Took care of Sattam's comments from our code review.
git-svn-id: https://hyracks.googlecode.com/svn/branches/hyracks_indexes@522 123451ca-8445-de46-9d55-352943316053
diff --git a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java
index 5dd5650..7aeaf77 100644
--- a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java
+++ b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/InsertPipelineExample.java
@@ -45,8 +45,8 @@
import edu.uci.ics.hyracks.examples.btree.helper.StorageManagerInterface;
import edu.uci.ics.hyracks.examples.btree.helper.TreeIndexRegistryProvider;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeOpHelperFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
@@ -143,8 +143,8 @@
// create factories and providers for secondary B-Tree
TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(primaryTypeTraits);
- ITreeIndexFrameFactory primaryInteriorFrameFactory = new NSMInteriorFrameFactory(primaryTupleWriterFactory);
- ITreeIndexFrameFactory primaryLeafFrameFactory = new NSMLeafFrameFactory(primaryTupleWriterFactory);
+ ITreeIndexFrameFactory primaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(primaryTupleWriterFactory);
+ ITreeIndexFrameFactory primaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(primaryTupleWriterFactory);
// the B-Tree expects its keyfields to be at the front of its input
// tuple
@@ -174,9 +174,9 @@
// create factories and providers for secondary B-Tree
TypeAwareTupleWriterFactory secondaryTupleWriterFactory = new TypeAwareTupleWriterFactory(secondaryTypeTraits);
- ITreeIndexFrameFactory secondaryInteriorFrameFactory = new NSMInteriorFrameFactory(
+ ITreeIndexFrameFactory secondaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
secondaryTupleWriterFactory);
- ITreeIndexFrameFactory secondaryLeafFrameFactory = new NSMLeafFrameFactory(secondaryTupleWriterFactory);
+ ITreeIndexFrameFactory secondaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(secondaryTupleWriterFactory);
// the B-Tree expects its keyfields to be at the front of its input
// tuple
diff --git a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
index 7e88bc1..1533714 100644
--- a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
+++ b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexBulkLoadExample.java
@@ -43,8 +43,8 @@
import edu.uci.ics.hyracks.examples.btree.helper.StorageManagerInterface;
import edu.uci.ics.hyracks.examples.btree.helper.TreeIndexRegistryProvider;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeOpHelperFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
@@ -146,8 +146,8 @@
// create factories and providers for B-Tree
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
- ITreeIndexFrameFactory interiorFrameFactory = new NSMInteriorFrameFactory(tupleWriterFactory);
- ITreeIndexFrameFactory leafFrameFactory = new NSMLeafFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider = TreeIndexRegistryProvider.INSTANCE;
IStorageManagerInterface storageManager = StorageManagerInterface.INSTANCE;
diff --git a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexEnlistFilesExample.java b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexEnlistFilesExample.java
index ce70cf7..c991a89 100644
--- a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexEnlistFilesExample.java
+++ b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexEnlistFilesExample.java
@@ -34,8 +34,8 @@
import edu.uci.ics.hyracks.examples.btree.helper.StorageManagerInterface;
import edu.uci.ics.hyracks.examples.btree.helper.TreeIndexRegistryProvider;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeOpHelperFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
@@ -101,8 +101,8 @@
// create factories and providers for B-Tree
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
- ITreeIndexFrameFactory interiorFrameFactory = new NSMInteriorFrameFactory(tupleWriterFactory);
- ITreeIndexFrameFactory leafFrameFactory = new NSMLeafFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider = TreeIndexRegistryProvider.INSTANCE;
IStorageManagerInterface storageManager = StorageManagerInterface.INSTANCE;
diff --git a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
index cc4e984..7e4ae9d 100644
--- a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
+++ b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/PrimaryIndexSearchExample.java
@@ -41,8 +41,8 @@
import edu.uci.ics.hyracks.examples.btree.helper.TreeIndexRegistryProvider;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeOpHelperFactory;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
@@ -103,8 +103,8 @@
// create factories and providers for B-Tree
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
- ITreeIndexFrameFactory interiorFrameFactory = new NSMInteriorFrameFactory(tupleWriterFactory);
- ITreeIndexFrameFactory leafFrameFactory = new NSMLeafFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider = TreeIndexRegistryProvider.INSTANCE;
IStorageManagerInterface storageManager = StorageManagerInterface.INSTANCE;
diff --git a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
index 6fbf53c..b061811 100644
--- a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
+++ b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexBulkLoadExample.java
@@ -37,8 +37,8 @@
import edu.uci.ics.hyracks.examples.btree.helper.StorageManagerInterface;
import edu.uci.ics.hyracks.examples.btree.helper.TreeIndexRegistryProvider;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeOpHelperFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
@@ -120,8 +120,8 @@
// create factories and providers for primary B-Tree
TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(primaryTypeTraits);
- ITreeIndexFrameFactory primaryInteriorFrameFactory = new NSMInteriorFrameFactory(primaryTupleWriterFactory);
- ITreeIndexFrameFactory primaryLeafFrameFactory = new NSMLeafFrameFactory(primaryTupleWriterFactory);
+ ITreeIndexFrameFactory primaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(primaryTupleWriterFactory);
+ ITreeIndexFrameFactory primaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(primaryTupleWriterFactory);
// use a disk-order scan to read primary index
IFileSplitProvider primarySplitProvider = JobHelper.createFileSplitProvider(splitNCs, options.primaryBTreeName);
@@ -150,9 +150,9 @@
// create factories and providers for secondary B-Tree
TypeAwareTupleWriterFactory secondaryTupleWriterFactory = new TypeAwareTupleWriterFactory(secondaryTypeTraits);
- ITreeIndexFrameFactory secondaryInteriorFrameFactory = new NSMInteriorFrameFactory(
+ ITreeIndexFrameFactory secondaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
secondaryTupleWriterFactory);
- ITreeIndexFrameFactory secondaryLeafFrameFactory = new NSMLeafFrameFactory(secondaryTupleWriterFactory);
+ ITreeIndexFrameFactory secondaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(secondaryTupleWriterFactory);
// the B-Tree expects its keyfields to be at the front of its input
// tuple
diff --git a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
index e39f359..5e5eaeb 100644
--- a/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
+++ b/hyracks-examples/btree-example/btreeclient/src/main/java/edu/uci/ics/hyracks/examples/btree/client/SecondaryIndexSearchExample.java
@@ -41,8 +41,8 @@
import edu.uci.ics.hyracks.examples.btree.helper.TreeIndexRegistryProvider;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeOpHelperFactory;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
@@ -111,9 +111,9 @@
// create factories and providers for secondary B-Tree
TypeAwareTupleWriterFactory secondaryTupleWriterFactory = new TypeAwareTupleWriterFactory(secondaryTypeTraits);
- ITreeIndexFrameFactory secondaryInteriorFrameFactory = new NSMInteriorFrameFactory(
+ ITreeIndexFrameFactory secondaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
secondaryTupleWriterFactory);
- ITreeIndexFrameFactory secondaryLeafFrameFactory = new NSMLeafFrameFactory(secondaryTupleWriterFactory);
+ ITreeIndexFrameFactory secondaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(secondaryTupleWriterFactory);
// schema of tuples coming out of primary index
RecordDescriptor primaryRecDesc = new RecordDescriptor(new ISerializerDeserializer[] {
@@ -129,8 +129,8 @@
// create factories and providers for secondary B-Tree
TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(primaryTypeTraits);
- ITreeIndexFrameFactory primaryInteriorFrameFactory = new NSMInteriorFrameFactory(primaryTupleWriterFactory);
- ITreeIndexFrameFactory primaryLeafFrameFactory = new NSMLeafFrameFactory(primaryTupleWriterFactory);
+ ITreeIndexFrameFactory primaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(primaryTupleWriterFactory);
+ ITreeIndexFrameFactory primaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(primaryTupleWriterFactory);
// comparators for btree, note that we only need a comparator for the
// non-unique key
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreeOperatorsTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreeOperatorsTest.java
index 803cc55..b4a1d60 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreeOperatorsTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/btree/BTreeOperatorsTest.java
@@ -49,8 +49,8 @@
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeOpHelperFactory;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
@@ -67,439 +67,571 @@
import edu.uci.ics.hyracks.tests.integration.AbstractIntegrationTest;
public class BTreeOperatorsTest extends AbstractIntegrationTest {
- static {
- TestStorageManagerComponentHolder.init(8192, 20, 20);
- }
+ static {
+ TestStorageManagerComponentHolder.init(8192, 20, 20);
+ }
- private IStorageManagerInterface storageManager = new TestStorageManagerInterface();
- private IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider = new TestTreeIndexRegistryProvider();
- private ITreeIndexOpHelperFactory opHelperFactory = new BTreeOpHelperFactory();
+ private IStorageManagerInterface storageManager = new TestStorageManagerInterface();
+ private IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider = new TestTreeIndexRegistryProvider();
+ private ITreeIndexOpHelperFactory opHelperFactory = new BTreeOpHelperFactory();
- private final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
- private final static String sep = System.getProperty("file.separator");
+ private final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat(
+ "ddMMyy-hhmmssSS");
+ private final static String sep = System.getProperty("file.separator");
- // field, type and key declarations for primary index
- private int primaryFieldCount = 6;
- private ITypeTrait[] primaryTypeTraits = new ITypeTrait[primaryFieldCount];
- private int primaryKeyFieldCount = 1;
- private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
- private TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(primaryTypeTraits);
- private ITreeIndexFrameFactory primaryInteriorFrameFactory = new NSMInteriorFrameFactory(primaryTupleWriterFactory);
- private ITreeIndexFrameFactory primaryLeafFrameFactory = new NSMLeafFrameFactory(primaryTupleWriterFactory);
+ // field, type and key declarations for primary index
+ private int primaryFieldCount = 6;
+ private ITypeTrait[] primaryTypeTraits = new ITypeTrait[primaryFieldCount];
+ private int primaryKeyFieldCount = 1;
+ private IBinaryComparatorFactory[] primaryComparatorFactories = new IBinaryComparatorFactory[primaryKeyFieldCount];
+ private TypeAwareTupleWriterFactory primaryTupleWriterFactory = new TypeAwareTupleWriterFactory(
+ primaryTypeTraits);
+ private ITreeIndexFrameFactory primaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
+ primaryTupleWriterFactory);
+ private ITreeIndexFrameFactory primaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(
+ primaryTupleWriterFactory);
- private static String primaryBtreeName = "primary" + simpleDateFormat.format(new Date());
- private static String primaryFileName = System.getProperty("java.io.tmpdir") + sep + primaryBtreeName;
+ private static String primaryBtreeName = "primary"
+ + simpleDateFormat.format(new Date());
+ private static String primaryFileName = System
+ .getProperty("java.io.tmpdir") + sep + primaryBtreeName;
- private IFileSplitProvider primaryBtreeSplitProvider = new ConstantFileSplitProvider(
- new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(primaryFileName))) });
+ private IFileSplitProvider primaryBtreeSplitProvider = new ConstantFileSplitProvider(
+ new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
+ primaryFileName))) });
- private RecordDescriptor primaryRecDesc = new RecordDescriptor(new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
+ private RecordDescriptor primaryRecDesc = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
- // field, type and key declarations for secondary indexes
- private int secondaryFieldCount = 2;
- private ITypeTrait[] secondaryTypeTraits = new ITypeTrait[secondaryFieldCount];
- private int secondaryKeyFieldCount = 2;
- private IBinaryComparatorFactory[] secondaryComparatorFactories = new IBinaryComparatorFactory[secondaryKeyFieldCount];
- private TypeAwareTupleWriterFactory secondaryTupleWriterFactory = new TypeAwareTupleWriterFactory(
- secondaryTypeTraits);
- private ITreeIndexFrameFactory secondaryInteriorFrameFactory = new NSMInteriorFrameFactory(
- secondaryTupleWriterFactory);
- private ITreeIndexFrameFactory secondaryLeafFrameFactory = new NSMLeafFrameFactory(secondaryTupleWriterFactory);
+ // field, type and key declarations for secondary indexes
+ private int secondaryFieldCount = 2;
+ private ITypeTrait[] secondaryTypeTraits = new ITypeTrait[secondaryFieldCount];
+ private int secondaryKeyFieldCount = 2;
+ private IBinaryComparatorFactory[] secondaryComparatorFactories = new IBinaryComparatorFactory[secondaryKeyFieldCount];
+ private TypeAwareTupleWriterFactory secondaryTupleWriterFactory = new TypeAwareTupleWriterFactory(
+ secondaryTypeTraits);
+ private ITreeIndexFrameFactory secondaryInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
+ secondaryTupleWriterFactory);
+ private ITreeIndexFrameFactory secondaryLeafFrameFactory = new BTreeNSMLeafFrameFactory(
+ secondaryTupleWriterFactory);
- private static String secondaryBtreeName = "secondary" + simpleDateFormat.format(new Date());
- private static String secondaryFileName = System.getProperty("java.io.tmpdir") + sep + secondaryBtreeName;
+ private static String secondaryBtreeName = "secondary"
+ + simpleDateFormat.format(new Date());
+ private static String secondaryFileName = System
+ .getProperty("java.io.tmpdir") + sep + secondaryBtreeName;
- private IFileSplitProvider secondaryBtreeSplitProvider = new ConstantFileSplitProvider(
- new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(secondaryFileName))) });
+ private IFileSplitProvider secondaryBtreeSplitProvider = new ConstantFileSplitProvider(
+ new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
+ secondaryFileName))) });
- private RecordDescriptor secondaryRecDesc = new RecordDescriptor(new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE });
+ private RecordDescriptor secondaryRecDesc = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
- @Before
- public void setup() {
- // field, type and key declarations for primary index
- primaryTypeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[2] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[3] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[4] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryTypeTraits[5] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- primaryComparatorFactories[0] = UTF8StringBinaryComparatorFactory.INSTANCE;
+ @Before
+ public void setup() {
+ // field, type and key declarations for primary index
+ primaryTypeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ primaryTypeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ primaryTypeTraits[2] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ primaryTypeTraits[3] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ primaryTypeTraits[4] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ primaryTypeTraits[5] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ primaryComparatorFactories[0] = UTF8StringBinaryComparatorFactory.INSTANCE;
- // field, type and key declarations for secondary indexes
- secondaryTypeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- secondaryTypeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- secondaryComparatorFactories[0] = UTF8StringBinaryComparatorFactory.INSTANCE;
- secondaryComparatorFactories[1] = UTF8StringBinaryComparatorFactory.INSTANCE;
- }
+ // field, type and key declarations for secondary indexes
+ secondaryTypeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ secondaryTypeTraits[1] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ secondaryComparatorFactories[0] = UTF8StringBinaryComparatorFactory.INSTANCE;
+ secondaryComparatorFactories[1] = UTF8StringBinaryComparatorFactory.INSTANCE;
+ }
- @Test
- public void loadPrimaryIndexTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ @Test
+ public void loadPrimaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- "data/tpch0.001/orders-part1.tbl"))) };
- IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(ordersSplits);
- RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID,
+ new FileReference(new File("data/tpch0.001/orders-part1.tbl"))) };
+ IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(
+ ordersSplits);
+ RecordDescriptor ordersDesc = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
- FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(spec, ordersSplitProvider,
- new DelimitedDataTupleParserFactory(new IValueParserFactory[] { UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE }, '|'), ordersDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, ordScanner, NC1_ID);
+ FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(
+ spec, ordersSplitProvider, new DelimitedDataTupleParserFactory(
+ new IValueParserFactory[] {
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE }, '|'),
+ ordersDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ ordScanner, NC1_ID);
- ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(spec, 1000, new int[] { 0 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE }, ordersDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter, NC1_ID);
+ ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(
+ spec,
+ 1000,
+ new int[] { 0 },
+ new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
+ ordersDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter,
+ NC1_ID);
- int[] fieldPermutation = { 0, 1, 2, 4, 5, 7 };
- TreeIndexBulkLoadOperatorDescriptor primaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
- storageManager, treeIndexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, fieldPermutation, 0.7f,
- opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeBulkLoad, NC1_ID);
+ int[] fieldPermutation = { 0, 1, 2, 4, 5, 7 };
+ TreeIndexBulkLoadOperatorDescriptor primaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(
+ spec, storageManager, treeIndexRegistryProvider,
+ primaryBtreeSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits,
+ primaryComparatorFactories, fieldPermutation, 0.7f,
+ opHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ primaryBtreeBulkLoad, NC1_ID);
- spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0, sorter, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0,
+ sorter, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), sorter, 0, primaryBtreeBulkLoad, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sorter, 0,
+ primaryBtreeBulkLoad, 0);
- spec.addRoot(primaryBtreeBulkLoad);
- runTest(spec);
- }
+ spec.addRoot(primaryBtreeBulkLoad);
+ runTest(spec);
+ }
- @Test
- public void showPrimaryIndexStats() throws Exception {
- JobSpecification spec = new JobSpecification();
+ @Test
+ public void showPrimaryIndexStats() throws Exception {
+ JobSpecification spec = new JobSpecification();
- TreeIndexStatsOperatorDescriptor primaryStatsOp = new TreeIndexStatsOperatorDescriptor(spec, storageManager,
- treeIndexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryStatsOp, NC1_ID);
+ TreeIndexStatsOperatorDescriptor primaryStatsOp = new TreeIndexStatsOperatorDescriptor(
+ spec, storageManager, treeIndexRegistryProvider,
+ primaryBtreeSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits,
+ primaryComparatorFactories, opHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ primaryStatsOp, NC1_ID);
- spec.addRoot(primaryStatsOp);
- runTest(spec);
- }
+ spec.addRoot(primaryStatsOp);
+ runTest(spec);
+ }
- @Test
- public void scanPrimaryIndexTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ @Test
+ public void scanPrimaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- // build dummy tuple containing nothing
- ArrayTupleBuilder tb = new ArrayTupleBuilder(primaryKeyFieldCount * 2);
- DataOutput dos = tb.getDataOutput();
+ // build dummy tuple containing nothing
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(primaryKeyFieldCount * 2);
+ DataOutput dos = tb.getDataOutput();
- tb.reset();
- UTF8StringSerializerDeserializer.INSTANCE.serialize("0", dos);
- tb.addFieldEndOffset();
+ tb.reset();
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("0", dos);
+ tb.addFieldEndOffset();
- ISerializerDeserializer[] keyRecDescSers = { UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE };
- RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
+ ISerializerDeserializer[] keyRecDescSers = {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+ RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
- ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(spec,
- keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(), tb.getSize());
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, keyProviderOp, NC1_ID);
+ ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(
+ spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(),
+ tb.getSize());
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ keyProviderOp, NC1_ID);
- int[] lowKeyFields = null; // - infinity
- int[] highKeyFields = null; // + infinity
+ int[] lowKeyFields = null; // - infinity
+ int[] highKeyFields = null; // + infinity
- BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
- storageManager, treeIndexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, true, lowKeyFields,
- highKeyFields, true, true, opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
+ BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(
+ spec, primaryRecDesc, storageManager,
+ treeIndexRegistryProvider, primaryBtreeSplitProvider,
+ primaryInteriorFrameFactory, primaryLeafFrameFactory,
+ primaryTypeTraits, primaryComparatorFactories, true,
+ lowKeyFields, highKeyFields, true, true, opHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ primaryBtreeSearchOp, NC1_ID);
- PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
+ PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC1_ID);
- spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryBtreeSearchOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), primaryBtreeSearchOp, 0, printer, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0,
+ primaryBtreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec),
+ primaryBtreeSearchOp, 0, printer, 0);
- spec.addRoot(printer);
- runTest(spec);
- }
+ spec.addRoot(printer);
+ runTest(spec);
+ }
- @Test
- public void searchPrimaryIndexTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ @Test
+ public void searchPrimaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- // build tuple containing low and high search key
- // high key and low key
- ArrayTupleBuilder tb = new ArrayTupleBuilder(primaryKeyFieldCount * 2);
- DataOutput dos = tb.getDataOutput();
+ // build tuple containing low and high search key
+ // high key and low key
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(primaryKeyFieldCount * 2);
+ DataOutput dos = tb.getDataOutput();
- tb.reset();
- // low key
- UTF8StringSerializerDeserializer.INSTANCE.serialize("100", dos);
- tb.addFieldEndOffset();
- // high key
- UTF8StringSerializerDeserializer.INSTANCE.serialize("200", dos);
- tb.addFieldEndOffset();
+ tb.reset();
+ // low key
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("100", dos);
+ tb.addFieldEndOffset();
+ // high key
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("200", dos);
+ tb.addFieldEndOffset();
- ISerializerDeserializer[] keyRecDescSers = { UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE };
- RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
+ ISerializerDeserializer[] keyRecDescSers = {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+ RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
- ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(spec,
- keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(), tb.getSize());
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, keyProviderOp, NC1_ID);
+ ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(
+ spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(),
+ tb.getSize());
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ keyProviderOp, NC1_ID);
- int[] lowKeyFields = { 0 };
- int[] highKeyFields = { 1 };
+ int[] lowKeyFields = { 0 };
+ int[] highKeyFields = { 1 };
- BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
- storageManager, treeIndexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, true, lowKeyFields,
- highKeyFields, true, true, opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
+ BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(
+ spec, primaryRecDesc, storageManager,
+ treeIndexRegistryProvider, primaryBtreeSplitProvider,
+ primaryInteriorFrameFactory, primaryLeafFrameFactory,
+ primaryTypeTraits, primaryComparatorFactories, true,
+ lowKeyFields, highKeyFields, true, true, opHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ primaryBtreeSearchOp, NC1_ID);
- PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
+ PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC1_ID);
- spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryBtreeSearchOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), primaryBtreeSearchOp, 0, printer, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0,
+ primaryBtreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec),
+ primaryBtreeSearchOp, 0, printer, 0);
- spec.addRoot(printer);
- runTest(spec);
- }
+ spec.addRoot(printer);
+ runTest(spec);
+ }
- @Test
- public void loadSecondaryIndexTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ @Test
+ public void loadSecondaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- // build dummy tuple containing nothing
- ArrayTupleBuilder tb = new ArrayTupleBuilder(primaryKeyFieldCount * 2);
- DataOutput dos = tb.getDataOutput();
+ // build dummy tuple containing nothing
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(primaryKeyFieldCount * 2);
+ DataOutput dos = tb.getDataOutput();
- tb.reset();
- UTF8StringSerializerDeserializer.INSTANCE.serialize("0", dos);
- tb.addFieldEndOffset();
+ tb.reset();
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("0", dos);
+ tb.addFieldEndOffset();
- ISerializerDeserializer[] keyRecDescSers = { UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE };
- RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
+ ISerializerDeserializer[] keyRecDescSers = {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+ RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
- ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(spec,
- keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(), tb.getSize());
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, keyProviderOp, NC1_ID);
+ ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(
+ spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(),
+ tb.getSize());
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ keyProviderOp, NC1_ID);
- int[] lowKeyFields = null; // - infinity
- int[] highKeyFields = null; // + infinity
+ int[] lowKeyFields = null; // - infinity
+ int[] highKeyFields = null; // + infinity
- // scan primary index
- BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
- storageManager, treeIndexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, true, lowKeyFields,
- highKeyFields, true, true, opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
+ // scan primary index
+ BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(
+ spec, primaryRecDesc, storageManager,
+ treeIndexRegistryProvider, primaryBtreeSplitProvider,
+ primaryInteriorFrameFactory, primaryLeafFrameFactory,
+ primaryTypeTraits, primaryComparatorFactories, true,
+ lowKeyFields, highKeyFields, true, true, opHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ primaryBtreeSearchOp, NC1_ID);
- // sort based on secondary keys
- ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(spec, 1000, new int[] { 3, 0 },
- new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE }, primaryRecDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter, NC1_ID);
+ // sort based on secondary keys
+ ExternalSortOperatorDescriptor sorter = new ExternalSortOperatorDescriptor(
+ spec,
+ 1000,
+ new int[] { 3, 0 },
+ new IBinaryComparatorFactory[] { UTF8StringBinaryComparatorFactory.INSTANCE },
+ primaryRecDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, sorter,
+ NC1_ID);
- // load secondary index
- int[] fieldPermutation = { 3, 0 };
- TreeIndexBulkLoadOperatorDescriptor secondaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
- storageManager, treeIndexRegistryProvider, secondaryBtreeSplitProvider, secondaryInteriorFrameFactory,
- secondaryLeafFrameFactory, secondaryTypeTraits, secondaryComparatorFactories, fieldPermutation, 0.7f,
- opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryBtreeBulkLoad, NC1_ID);
+ // load secondary index
+ int[] fieldPermutation = { 3, 0 };
+ TreeIndexBulkLoadOperatorDescriptor secondaryBtreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(
+ spec, storageManager, treeIndexRegistryProvider,
+ secondaryBtreeSplitProvider, secondaryInteriorFrameFactory,
+ secondaryLeafFrameFactory, secondaryTypeTraits,
+ secondaryComparatorFactories, fieldPermutation, 0.7f,
+ opHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ secondaryBtreeBulkLoad, NC1_ID);
- spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryBtreeSearchOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), primaryBtreeSearchOp, 0, sorter, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), sorter, 0, secondaryBtreeBulkLoad, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0,
+ primaryBtreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec),
+ primaryBtreeSearchOp, 0, sorter, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sorter, 0,
+ secondaryBtreeBulkLoad, 0);
- spec.addRoot(secondaryBtreeBulkLoad);
- runTest(spec);
- }
+ spec.addRoot(secondaryBtreeBulkLoad);
+ runTest(spec);
+ }
- @Test
- public void searchSecondaryIndexTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ @Test
+ public void searchSecondaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- // build tuple containing search keys (only use the first key as search
- // key)
- ArrayTupleBuilder tb = new ArrayTupleBuilder(secondaryKeyFieldCount);
- DataOutput dos = tb.getDataOutput();
+ // build tuple containing search keys (only use the first key as search
+ // key)
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(secondaryKeyFieldCount);
+ DataOutput dos = tb.getDataOutput();
- tb.reset();
- // low key
- UTF8StringSerializerDeserializer.INSTANCE.serialize("1998-07-21", dos);
- tb.addFieldEndOffset();
- // high key
- UTF8StringSerializerDeserializer.INSTANCE.serialize("2000-10-18", dos);
- tb.addFieldEndOffset();
+ tb.reset();
+ // low key
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("1998-07-21", dos);
+ tb.addFieldEndOffset();
+ // high key
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("2000-10-18", dos);
+ tb.addFieldEndOffset();
- ISerializerDeserializer[] keyRecDescSers = { UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE };
- RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
+ ISerializerDeserializer[] keyRecDescSers = {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+ RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
- ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(spec,
- keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(), tb.getSize());
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, keyProviderOp, NC1_ID);
+ ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(
+ spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(),
+ tb.getSize());
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ keyProviderOp, NC1_ID);
- int[] secondaryLowKeyFields = { 0 };
- int[] secondaryHighKeyFields = { 1 };
+ int[] secondaryLowKeyFields = { 0 };
+ int[] secondaryHighKeyFields = { 1 };
- // search secondary index
- BTreeSearchOperatorDescriptor secondaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec,
- secondaryRecDesc, storageManager, treeIndexRegistryProvider, secondaryBtreeSplitProvider,
- secondaryInteriorFrameFactory, secondaryLeafFrameFactory, secondaryTypeTraits,
- secondaryComparatorFactories, true, secondaryLowKeyFields, secondaryHighKeyFields, true, true,
- opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryBtreeSearchOp, NC1_ID);
+ // search secondary index
+ BTreeSearchOperatorDescriptor secondaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(
+ spec, secondaryRecDesc, storageManager,
+ treeIndexRegistryProvider, secondaryBtreeSplitProvider,
+ secondaryInteriorFrameFactory, secondaryLeafFrameFactory,
+ secondaryTypeTraits, secondaryComparatorFactories, true,
+ secondaryLowKeyFields, secondaryHighKeyFields, true, true,
+ opHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ secondaryBtreeSearchOp, NC1_ID);
- int[] primaryLowKeyFields = { 1 }; // second field from the tuples
- // coming from secondary index
- int[] primaryHighKeyFields = { 1 }; // second field from the tuples
- // coming from secondary index
+ int[] primaryLowKeyFields = { 1 }; // second field from the tuples
+ // coming from secondary index
+ int[] primaryHighKeyFields = { 1 }; // second field from the tuples
+ // coming from secondary index
- // search primary index
- BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
- storageManager, treeIndexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, true, primaryLowKeyFields,
- primaryHighKeyFields, true, true, opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
+ // search primary index
+ BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(
+ spec, primaryRecDesc, storageManager,
+ treeIndexRegistryProvider, primaryBtreeSplitProvider,
+ primaryInteriorFrameFactory, primaryLeafFrameFactory,
+ primaryTypeTraits, primaryComparatorFactories, true,
+ primaryLowKeyFields, primaryHighKeyFields, true, true,
+ opHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ primaryBtreeSearchOp, NC1_ID);
- PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
+ PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC1_ID);
- spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, secondaryBtreeSearchOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), secondaryBtreeSearchOp, 0, primaryBtreeSearchOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), primaryBtreeSearchOp, 0, printer, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0,
+ secondaryBtreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec),
+ secondaryBtreeSearchOp, 0, primaryBtreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec),
+ primaryBtreeSearchOp, 0, printer, 0);
- spec.addRoot(printer);
- runTest(spec);
- }
+ spec.addRoot(printer);
+ runTest(spec);
+ }
- @Test
- public void insertPipelineTest() throws Exception {
+ @Test
+ public void insertPipelineTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ JobSpecification spec = new JobSpecification();
- FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID, new FileReference(new File(
- "data/tpch0.001/orders-part2.tbl"))) };
- IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(ordersSplits);
- RecordDescriptor ordersDesc = new RecordDescriptor(new ISerializerDeserializer[] {
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE, UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE });
+ FileSplit[] ordersSplits = new FileSplit[] { new FileSplit(NC1_ID,
+ new FileReference(new File("data/tpch0.001/orders-part2.tbl"))) };
+ IFileSplitProvider ordersSplitProvider = new ConstantFileSplitProvider(
+ ordersSplits);
+ RecordDescriptor ordersDesc = new RecordDescriptor(
+ new ISerializerDeserializer[] {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE });
- FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(spec, ordersSplitProvider,
- new DelimitedDataTupleParserFactory(new IValueParserFactory[] { UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE,
- UTF8StringParserFactory.INSTANCE, UTF8StringParserFactory.INSTANCE }, '|'), ordersDesc);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, ordScanner, NC1_ID);
+ FileScanOperatorDescriptor ordScanner = new FileScanOperatorDescriptor(
+ spec, ordersSplitProvider, new DelimitedDataTupleParserFactory(
+ new IValueParserFactory[] {
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE,
+ UTF8StringParserFactory.INSTANCE }, '|'),
+ ordersDesc);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ ordScanner, NC1_ID);
- // insert into primary index
- int[] primaryFieldPermutation = { 0, 1, 2, 4, 5, 7 };
- TreeIndexInsertUpdateDeleteOperatorDescriptor primaryBtreeInsertOp = new TreeIndexInsertUpdateDeleteOperatorDescriptor(
- spec, ordersDesc, storageManager, treeIndexRegistryProvider, primaryBtreeSplitProvider,
- primaryInteriorFrameFactory, primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories,
- primaryFieldPermutation, IndexOp.INSERT, opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeInsertOp, NC1_ID);
+ // insert into primary index
+ int[] primaryFieldPermutation = { 0, 1, 2, 4, 5, 7 };
+ TreeIndexInsertUpdateDeleteOperatorDescriptor primaryBtreeInsertOp = new TreeIndexInsertUpdateDeleteOperatorDescriptor(
+ spec, ordersDesc, storageManager, treeIndexRegistryProvider,
+ primaryBtreeSplitProvider, primaryInteriorFrameFactory,
+ primaryLeafFrameFactory, primaryTypeTraits,
+ primaryComparatorFactories, primaryFieldPermutation,
+ IndexOp.INSERT, opHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ primaryBtreeInsertOp, NC1_ID);
- // first secondary index
- int[] fieldPermutationB = { 4, 0 };
- TreeIndexInsertUpdateDeleteOperatorDescriptor secondaryInsertOp = new TreeIndexInsertUpdateDeleteOperatorDescriptor(
- spec, ordersDesc, storageManager, treeIndexRegistryProvider, secondaryBtreeSplitProvider,
- secondaryInteriorFrameFactory, secondaryLeafFrameFactory, secondaryTypeTraits,
- secondaryComparatorFactories, fieldPermutationB, IndexOp.INSERT, opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryInsertOp, NC1_ID);
+ // first secondary index
+ int[] fieldPermutationB = { 4, 0 };
+ TreeIndexInsertUpdateDeleteOperatorDescriptor secondaryInsertOp = new TreeIndexInsertUpdateDeleteOperatorDescriptor(
+ spec, ordersDesc, storageManager, treeIndexRegistryProvider,
+ secondaryBtreeSplitProvider, secondaryInteriorFrameFactory,
+ secondaryLeafFrameFactory, secondaryTypeTraits,
+ secondaryComparatorFactories, fieldPermutationB,
+ IndexOp.INSERT, opHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ secondaryInsertOp, NC1_ID);
- NullSinkOperatorDescriptor nullSink = new NullSinkOperatorDescriptor(spec);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, nullSink, NC1_ID);
+ NullSinkOperatorDescriptor nullSink = new NullSinkOperatorDescriptor(
+ spec);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, nullSink,
+ NC1_ID);
- spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0, primaryBtreeInsertOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), ordScanner, 0,
+ primaryBtreeInsertOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), primaryBtreeInsertOp, 0, secondaryInsertOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec),
+ primaryBtreeInsertOp, 0, secondaryInsertOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), secondaryInsertOp, 0, nullSink, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), secondaryInsertOp,
+ 0, nullSink, 0);
- spec.addRoot(nullSink);
- runTest(spec);
- }
+ spec.addRoot(nullSink);
+ runTest(spec);
+ }
- @Test
- public void searchUpdatedSecondaryIndexTest() throws Exception {
- JobSpecification spec = new JobSpecification();
+ @Test
+ public void searchUpdatedSecondaryIndexTest() throws Exception {
+ JobSpecification spec = new JobSpecification();
- // build tuple containing search keys (only use the first key as search
- // key)
- ArrayTupleBuilder tb = new ArrayTupleBuilder(secondaryKeyFieldCount);
- DataOutput dos = tb.getDataOutput();
+ // build tuple containing search keys (only use the first key as search
+ // key)
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(secondaryKeyFieldCount);
+ DataOutput dos = tb.getDataOutput();
- tb.reset();
- // low key
- UTF8StringSerializerDeserializer.INSTANCE.serialize("1998-07-21", dos);
- tb.addFieldEndOffset();
- // high key
- UTF8StringSerializerDeserializer.INSTANCE.serialize("2000-10-18", dos);
- tb.addFieldEndOffset();
+ tb.reset();
+ // low key
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("1998-07-21", dos);
+ tb.addFieldEndOffset();
+ // high key
+ UTF8StringSerializerDeserializer.INSTANCE.serialize("2000-10-18", dos);
+ tb.addFieldEndOffset();
- ISerializerDeserializer[] keyRecDescSers = { UTF8StringSerializerDeserializer.INSTANCE,
- UTF8StringSerializerDeserializer.INSTANCE };
- RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
+ ISerializerDeserializer[] keyRecDescSers = {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ UTF8StringSerializerDeserializer.INSTANCE };
+ RecordDescriptor keyRecDesc = new RecordDescriptor(keyRecDescSers);
- ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(spec,
- keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(), tb.getSize());
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, keyProviderOp, NC1_ID);
+ ConstantTupleSourceOperatorDescriptor keyProviderOp = new ConstantTupleSourceOperatorDescriptor(
+ spec, keyRecDesc, tb.getFieldEndOffsets(), tb.getByteArray(),
+ tb.getSize());
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ keyProviderOp, NC1_ID);
- int[] secondaryLowKeyFields = { 0 };
- int[] secondaryHighKeyFields = { 1 };
+ int[] secondaryLowKeyFields = { 0 };
+ int[] secondaryHighKeyFields = { 1 };
- // search secondary index
- BTreeSearchOperatorDescriptor secondaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec,
- secondaryRecDesc, storageManager, treeIndexRegistryProvider, secondaryBtreeSplitProvider,
- secondaryInteriorFrameFactory, secondaryLeafFrameFactory, secondaryTypeTraits,
- secondaryComparatorFactories, true, secondaryLowKeyFields, secondaryHighKeyFields, true, true,
- opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, secondaryBtreeSearchOp, NC1_ID);
+ // search secondary index
+ BTreeSearchOperatorDescriptor secondaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(
+ spec, secondaryRecDesc, storageManager,
+ treeIndexRegistryProvider, secondaryBtreeSplitProvider,
+ secondaryInteriorFrameFactory, secondaryLeafFrameFactory,
+ secondaryTypeTraits, secondaryComparatorFactories, true,
+ secondaryLowKeyFields, secondaryHighKeyFields, true, true,
+ opHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ secondaryBtreeSearchOp, NC1_ID);
- // second field from the tuples coming from secondary index
- int[] primaryLowKeyFields = { 1 };
- // second field from the tuples coming from secondary index
- int[] primaryHighKeyFields = { 1 };
+ // second field from the tuples coming from secondary index
+ int[] primaryLowKeyFields = { 1 };
+ // second field from the tuples coming from secondary index
+ int[] primaryHighKeyFields = { 1 };
- // search primary index
- BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
- storageManager, treeIndexRegistryProvider, primaryBtreeSplitProvider, primaryInteriorFrameFactory,
- primaryLeafFrameFactory, primaryTypeTraits, primaryComparatorFactories, true, primaryLowKeyFields,
- primaryHighKeyFields, true, true, opHelperFactory);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, primaryBtreeSearchOp, NC1_ID);
+ // search primary index
+ BTreeSearchOperatorDescriptor primaryBtreeSearchOp = new BTreeSearchOperatorDescriptor(
+ spec, primaryRecDesc, storageManager,
+ treeIndexRegistryProvider, primaryBtreeSplitProvider,
+ primaryInteriorFrameFactory, primaryLeafFrameFactory,
+ primaryTypeTraits, primaryComparatorFactories, true,
+ primaryLowKeyFields, primaryHighKeyFields, true, true,
+ opHelperFactory);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec,
+ primaryBtreeSearchOp, NC1_ID);
- PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
- PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer, NC1_ID);
+ PrinterOperatorDescriptor printer = new PrinterOperatorDescriptor(spec);
+ PartitionConstraintHelper.addAbsoluteLocationConstraint(spec, printer,
+ NC1_ID);
- spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, secondaryBtreeSearchOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), secondaryBtreeSearchOp, 0, primaryBtreeSearchOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), primaryBtreeSearchOp, 0, printer, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0,
+ secondaryBtreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec),
+ secondaryBtreeSearchOp, 0, primaryBtreeSearchOp, 0);
+ spec.connect(new OneToOneConnectorDescriptor(spec),
+ primaryBtreeSearchOp, 0, printer, 0);
- spec.addRoot(printer);
- runTest(spec);
- }
+ spec.addRoot(printer);
+ runTest(spec);
+ }
- @AfterClass
- public static void cleanup() throws Exception {
- File primary = new File(primaryFileName);
- primary.deleteOnExit();
+ @AfterClass
+ public static void cleanup() throws Exception {
+ File primary = new File(primaryFileName);
+ primary.deleteOnExit();
- File secondary = new File(secondaryFileName);
- secondary.deleteOnExit();
- }
+ File secondary = new File(secondaryFileName);
+ secondary.deleteOnExit();
+ }
}
\ No newline at end of file
diff --git a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreeOperatorsTest.java b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreeOperatorsTest.java
index f754de7..5f6eb3a 100644
--- a/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreeOperatorsTest.java
+++ b/hyracks-examples/hyracks-integration-tests/src/test/java/edu/uci/ics/hyracks/tests/rtree/RTreeOperatorsTest.java
@@ -51,8 +51,8 @@
import edu.uci.ics.hyracks.dataflow.std.sort.ExternalSortOperatorDescriptor;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeOpHelperFactory;
import edu.uci.ics.hyracks.storage.am.btree.dataflow.BTreeSearchOperatorDescriptor;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
@@ -115,9 +115,9 @@
private IBinaryComparatorFactory[] primaryBTreeComparatorFactories = new IBinaryComparatorFactory[primaryBTreeKeyFieldCount];
private TypeAwareTupleWriterFactory primaryBTreeTupleWriterFactory = new TypeAwareTupleWriterFactory(
primaryBTreeTypeTraits);
- private ITreeIndexFrameFactory primaryBTreeInteriorFrameFactory = new NSMInteriorFrameFactory(
+ private ITreeIndexFrameFactory primaryBTreeInteriorFrameFactory = new BTreeNSMInteriorFrameFactory(
primaryBTreeTupleWriterFactory);
- private ITreeIndexFrameFactory primaryBTreeLeafFrameFactory = new NSMLeafFrameFactory(
+ private ITreeIndexFrameFactory primaryBTreeLeafFrameFactory = new BTreeNSMLeafFrameFactory(
primaryBTreeTupleWriterFactory);
private static String primaryBTreeName = "primaryBTree" + simpleDateFormat.format(new Date());
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeInteriorFrame.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeInteriorFrame.java
index 0f1c66a..25ab167 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeInteriorFrame.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeInteriorFrame.java
@@ -15,11 +15,15 @@
package edu.uci.ics.hyracks.storage.am.btree.api;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.btree.impls.RangePredicate;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
public interface IBTreeInteriorFrame extends ITreeIndexFrame {
+ public void insertSorted(ITupleReference tuple, MultiComparator cmp) throws HyracksDataException;
+
public int getChildPageId(RangePredicate pred, MultiComparator srcCmp);
public int getLeftmostChildPageId(MultiComparator cmp);
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeLeafFrame.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeLeafFrame.java
index 697edfa..53d892e 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeLeafFrame.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IBTreeLeafFrame.java
@@ -15,6 +15,7 @@
package edu.uci.ics.hyracks.storage.am.btree.api;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
@@ -23,6 +24,8 @@
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
public interface IBTreeLeafFrame extends ITreeIndexFrame {
+ public void insertSorted(ITupleReference tuple, MultiComparator cmp) throws HyracksDataException;
+
public void setNextLeaf(int nextPage);
public int getNextLeaf();
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IFrameCompressor.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IFrameCompressor.java
index 390524d..8e1d0a2 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IFrameCompressor.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IFrameCompressor.java
@@ -15,9 +15,9 @@
package edu.uci.ics.hyracks.storage.am.btree.api;
-import edu.uci.ics.hyracks.storage.am.btree.frames.FieldPrefixNSMLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeFieldPrefixNSMLeafFrame;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
public interface IFrameCompressor {
- public boolean compress(FieldPrefixNSMLeafFrame frame, MultiComparator cmp) throws Exception;
+ public boolean compress(BTreeFieldPrefixNSMLeafFrame frame, MultiComparator cmp) throws Exception;
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IPrefixSlotManager.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IPrefixSlotManager.java
index 5d78afc..d01db11 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IPrefixSlotManager.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/api/IPrefixSlotManager.java
@@ -16,7 +16,7 @@
package edu.uci.ics.hyracks.storage.am.btree.api;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.storage.am.btree.frames.FieldPrefixNSMLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeFieldPrefixNSMLeafFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.FindTupleMode;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.FindTupleNoExactMatchPolicy;
@@ -43,7 +43,7 @@
// all prefixes are recomputed during a reorg or compaction
public interface IPrefixSlotManager {
- public void setFrame(FieldPrefixNSMLeafFrame frame);
+ public void setFrame(BTreeFieldPrefixNSMLeafFrame frame);
public int decodeFirstSlotField(int slot);
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java
index a4e9d9b..1292ff2 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/compressors/FieldPrefixCompressor.java
@@ -24,7 +24,7 @@
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
import edu.uci.ics.hyracks.storage.am.btree.api.IFrameCompressor;
import edu.uci.ics.hyracks.storage.am.btree.api.IPrefixSlotManager;
-import edu.uci.ics.hyracks.storage.am.btree.frames.FieldPrefixNSMLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeFieldPrefixNSMLeafFrame;
import edu.uci.ics.hyracks.storage.am.btree.impls.FieldPrefixSlotManager;
import edu.uci.ics.hyracks.storage.am.btree.impls.FieldPrefixTupleReference;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
@@ -49,7 +49,7 @@
}
@Override
- public boolean compress(FieldPrefixNSMLeafFrame frame, MultiComparator cmp) throws Exception {
+ public boolean compress(BTreeFieldPrefixNSMLeafFrame frame, MultiComparator cmp) throws Exception {
int tupleCount = frame.getTupleCount();
if (tupleCount <= 0) {
frame.setPrefixTupleCount(0);
@@ -365,7 +365,7 @@
// the prefix length may be different for different keypartitions
// the occurrenceThreshold determines the minimum number of tuples that must
// share a common prefix in order for us to consider compressing them
- private ArrayList<KeyPartition> getKeyPartitions(FieldPrefixNSMLeafFrame frame, MultiComparator cmp,
+ private ArrayList<KeyPartition> getKeyPartitions(BTreeFieldPrefixNSMLeafFrame frame, MultiComparator cmp,
int occurrenceThreshold) {
IBinaryComparator[] cmps = cmp.getComparators();
int fieldCount = cmp.getKeyFieldCount();
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/FieldPrefixNSMLeafFrame.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
similarity index 99%
rename from hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/FieldPrefixNSMLeafFrame.java
rename to hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
index 2c1f003..bd4947c 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/FieldPrefixNSMLeafFrame.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrame.java
@@ -49,7 +49,7 @@
// WARNING: only works when tupleWriter is an instance of TypeAwareTupleWriter
-public class FieldPrefixNSMLeafFrame implements IBTreeLeafFrame {
+public class BTreeFieldPrefixNSMLeafFrame implements IBTreeLeafFrame {
protected static final int pageLsnOff = 0; // 0
protected static final int tupleCountOff = pageLsnOff + 4; // 4
@@ -74,7 +74,7 @@
private FieldPrefixTupleReference frameTuple;
private FieldPrefixPrefixTupleReference framePrefixTuple;
- public FieldPrefixNSMLeafFrame(ITreeIndexTupleWriter tupleWriter) {
+ public BTreeFieldPrefixNSMLeafFrame(ITreeIndexTupleWriter tupleWriter) {
this.tupleWriter = tupleWriter;
this.frameTuple = new FieldPrefixTupleReference(tupleWriter.createTupleReference());
ITypeTrait[] typeTraits = ((TypeAwareTupleWriter) tupleWriter).getTypeTraits();
@@ -458,7 +458,7 @@
public int split(ITreeIndexFrame rightFrame, ITupleReference tuple, MultiComparator cmp, ISplitKey splitKey)
throws Exception {
- FieldPrefixNSMLeafFrame rf = (FieldPrefixNSMLeafFrame) rightFrame;
+ BTreeFieldPrefixNSMLeafFrame rf = (BTreeFieldPrefixNSMLeafFrame) rightFrame;
frameTuple.setFieldCount(cmp.getFieldCount());
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/FieldPrefixNSMLeafFrameFactory.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java
similarity index 80%
rename from hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/FieldPrefixNSMLeafFrameFactory.java
rename to hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java
index 22da2f5..05b43d3 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/FieldPrefixNSMLeafFrameFactory.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeFieldPrefixNSMLeafFrameFactory.java
@@ -19,17 +19,17 @@
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
-public class FieldPrefixNSMLeafFrameFactory implements ITreeIndexFrameFactory {
+public class BTreeFieldPrefixNSMLeafFrameFactory implements ITreeIndexFrameFactory {
private static final long serialVersionUID = 1L;
private ITreeIndexTupleWriterFactory tupleWriterFactory;
- public FieldPrefixNSMLeafFrameFactory(ITreeIndexTupleWriterFactory tupleWriterFactory) {
+ public BTreeFieldPrefixNSMLeafFrameFactory(ITreeIndexTupleWriterFactory tupleWriterFactory) {
this.tupleWriterFactory = tupleWriterFactory;
}
@Override
public IBTreeLeafFrame createFrame() {
- return new FieldPrefixNSMLeafFrame(tupleWriterFactory.createTupleWriter());
+ return new BTreeFieldPrefixNSMLeafFrame(tupleWriterFactory.createTupleWriter());
}
}
\ No newline at end of file
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/NSMInteriorFrame.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
similarity index 99%
rename from hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/NSMInteriorFrame.java
rename to hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
index 12de948..6025003 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/NSMInteriorFrame.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrame.java
@@ -39,7 +39,7 @@
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.SlotOffTupleOff;
-public class NSMInteriorFrame extends TreeIndexNSMFrame implements IBTreeInteriorFrame {
+public class BTreeNSMInteriorFrame extends TreeIndexNSMFrame implements IBTreeInteriorFrame {
private static final int rightLeafOff = smFlagOff + 1;
@@ -48,7 +48,7 @@
// private SimpleTupleReference cmpFrameTuple = new SimpleTupleReference();
private ITreeIndexTupleReference cmpFrameTuple;
- public NSMInteriorFrame(ITreeIndexTupleWriter tupleWriter) {
+ public BTreeNSMInteriorFrame(ITreeIndexTupleWriter tupleWriter) {
super(tupleWriter, new OrderedSlotManager());
cmpFrameTuple = tupleWriter.createTupleReference();
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/NSMInteriorFrameFactory.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java
similarity index 82%
rename from hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/NSMInteriorFrameFactory.java
rename to hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java
index 0d93ed0..6b30ee0 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/NSMInteriorFrameFactory.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMInteriorFrameFactory.java
@@ -19,17 +19,17 @@
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
-public class NSMInteriorFrameFactory implements ITreeIndexFrameFactory {
+public class BTreeNSMInteriorFrameFactory implements ITreeIndexFrameFactory {
private static final long serialVersionUID = 1L;
private ITreeIndexTupleWriterFactory tupleWriterFactory;
- public NSMInteriorFrameFactory(ITreeIndexTupleWriterFactory tupleWriterFactory) {
+ public BTreeNSMInteriorFrameFactory(ITreeIndexTupleWriterFactory tupleWriterFactory) {
this.tupleWriterFactory = tupleWriterFactory;
}
@Override
public IBTreeInteriorFrame createFrame() {
- return new NSMInteriorFrame(tupleWriterFactory.createTupleWriter());
+ return new BTreeNSMInteriorFrame(tupleWriterFactory.createTupleWriter());
}
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/NSMLeafFrame.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
similarity index 97%
rename from hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/NSMLeafFrame.java
rename to hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
index 6cafa07..85fbec9 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/NSMLeafFrame.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMLeafFrame.java
@@ -30,11 +30,11 @@
import edu.uci.ics.hyracks.storage.am.common.ophelpers.FindTupleNoExactMatchPolicy;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-public class NSMLeafFrame extends TreeIndexNSMFrame implements IBTreeLeafFrame {
+public class BTreeNSMLeafFrame extends TreeIndexNSMFrame implements IBTreeLeafFrame {
protected static final int prevLeafOff = smFlagOff + 1;
protected static final int nextLeafOff = prevLeafOff + 4;
- public NSMLeafFrame(ITreeIndexTupleWriter tupleWriter) {
+ public BTreeNSMLeafFrame(ITreeIndexTupleWriter tupleWriter) {
super(tupleWriter, new OrderedSlotManager());
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/NSMLeafFrameFactory.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java
similarity index 82%
rename from hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/NSMLeafFrameFactory.java
rename to hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java
index d8f401f..d59b391 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/NSMLeafFrameFactory.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/frames/BTreeNSMLeafFrameFactory.java
@@ -19,17 +19,17 @@
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriterFactory;
-public class NSMLeafFrameFactory implements ITreeIndexFrameFactory {
+public class BTreeNSMLeafFrameFactory implements ITreeIndexFrameFactory {
private static final long serialVersionUID = 1L;
private ITreeIndexTupleWriterFactory tupleWriterFactory;
- public NSMLeafFrameFactory(ITreeIndexTupleWriterFactory tupleWriterFactory) {
+ public BTreeNSMLeafFrameFactory(ITreeIndexTupleWriterFactory tupleWriterFactory) {
this.tupleWriterFactory = tupleWriterFactory;
}
@Override
public IBTreeLeafFrame createFrame() {
- return new NSMLeafFrame(tupleWriterFactory.createTupleWriter());
+ return new BTreeNSMLeafFrame(tupleWriterFactory.createTupleWriter());
}
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
index 975d956..2831ff6 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/BTree.java
@@ -24,7 +24,7 @@
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMInteriorFrame;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrame;
import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexBulkLoadContext;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndex;
@@ -209,7 +209,7 @@
System.out.format(keyString);
if (!interiorFrame.isLeaf()) {
- ArrayList<Integer> children = ((NSMInteriorFrame) (interiorFrame)).getChildren(cmp);
+ ArrayList<Integer> children = ((BTreeNSMInteriorFrame) (interiorFrame)).getChildren(cmp);
for (int i = 0; i < children.size(); i++) {
printTree(children.get(i), node, i == children.size() - 1, leafFrame, interiorFrame, fields);
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixPrefixTupleReference.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixPrefixTupleReference.java
index 1caf23b..621281a 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixPrefixTupleReference.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixPrefixTupleReference.java
@@ -16,7 +16,7 @@
package edu.uci.ics.hyracks.storage.am.btree.impls;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
-import edu.uci.ics.hyracks.storage.am.btree.frames.FieldPrefixNSMLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeFieldPrefixNSMLeafFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleReference;
@@ -29,7 +29,7 @@
// assumes tuple index refers to prefix tuples
@Override
public void resetByTupleIndex(ITreeIndexFrame frame, int tupleIndex) {
- FieldPrefixNSMLeafFrame concreteFrame = (FieldPrefixNSMLeafFrame) frame;
+ BTreeFieldPrefixNSMLeafFrame concreteFrame = (BTreeFieldPrefixNSMLeafFrame) frame;
int prefixSlotOff = concreteFrame.slotManager.getPrefixSlotOff(tupleIndex);
int prefixSlot = concreteFrame.getBuffer().getInt(prefixSlotOff);
setFieldCount(concreteFrame.slotManager.decodeFirstSlotField(prefixSlot));
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixSlotManager.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixSlotManager.java
index 225dc83..6f98c8c 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixSlotManager.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixSlotManager.java
@@ -19,7 +19,7 @@
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.btree.api.IPrefixSlotManager;
-import edu.uci.ics.hyracks.storage.am.btree.frames.FieldPrefixNSMLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeFieldPrefixNSMLeafFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.FindTupleMode;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.FindTupleNoExactMatchPolicy;
@@ -33,7 +33,7 @@
public static final int GREATEST_SLOT = 0x00FFFFFF;
private ByteBuffer buf;
- private FieldPrefixNSMLeafFrame frame;
+ private BTreeFieldPrefixNSMLeafFrame frame;
public int decodeFirstSlotField(int slot) {
return (slot & 0xFF000000) >>> 24;
@@ -235,7 +235,7 @@
}
}
- public void setFrame(FieldPrefixNSMLeafFrame frame) {
+ public void setFrame(BTreeFieldPrefixNSMLeafFrame frame) {
this.frame = frame;
this.buf = frame.getBuffer();
}
diff --git a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixTupleReference.java b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixTupleReference.java
index 4628e60..9a655fc 100644
--- a/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixTupleReference.java
+++ b/hyracks-storage-am-btree/src/main/java/edu/uci/ics/hyracks/storage/am/btree/impls/FieldPrefixTupleReference.java
@@ -2,13 +2,13 @@
import java.nio.ByteBuffer;
-import edu.uci.ics.hyracks.storage.am.btree.frames.FieldPrefixNSMLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeFieldPrefixNSMLeafFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleReference;
public class FieldPrefixTupleReference implements ITreeIndexTupleReference {
- private FieldPrefixNSMLeafFrame frame;
+ private BTreeFieldPrefixNSMLeafFrame frame;
private int prefixTupleStartOff;
private int suffixTupleStartOff;
private int numPrefixFields;
@@ -21,7 +21,7 @@
@Override
public void resetByTupleIndex(ITreeIndexFrame frame, int tupleIndex) {
- this.frame = (FieldPrefixNSMLeafFrame) frame;
+ this.frame = (BTreeFieldPrefixNSMLeafFrame) frame;
int tupleSlotOff = this.frame.slotManager.getTupleSlotOff(tupleIndex);
int tupleSlot = this.frame.getBuffer().getInt(tupleSlotOff);
diff --git a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexFrame.java b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexFrame.java
index ee5a1e2..246efe4 100644
--- a/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexFrame.java
+++ b/hyracks-storage-am-common/src/main/java/edu/uci/ics/hyracks/storage/am/common/api/ITreeIndexFrame.java
@@ -88,8 +88,6 @@
public void setSmFlag(boolean smFlag);
- public void insertSorted(ITupleReference tuple, MultiComparator cmp) throws HyracksDataException;
-
public int getSlotSize();
// TODO: should be removed after new tuple format
diff --git a/hyracks-storage-am-invertedindex/pom.xml b/hyracks-storage-am-invertedindex/pom.xml
index 15d13dc..34f1135 100644
--- a/hyracks-storage-am-invertedindex/pom.xml
+++ b/hyracks-storage-am-invertedindex/pom.xml
@@ -59,13 +59,13 @@
<type>jar</type>
<scope>compile</scope>
</dependency>
- <dependency>
- <groupId>edu.uci.ics.fuzzyjoin</groupId>
- <artifactId>fuzzyjoin-core</artifactId>
- <version>0.0.3-SNAPSHOT</version>
- <type>jar</type>
- <scope>compile</scope>
- </dependency>
+ <dependency>
+ <groupId>edu.uci.ics.fuzzyjoin</groupId>
+ <artifactId>fuzzyjoin-core</artifactId>
+ <version>0.0.3</version>
+ <type>jar</type>
+ <scope>compile</scope>
+ </dependency>
<dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexOperatorDescriptorHelper.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexOperatorDescriptorHelper.java
new file mode 100644
index 0000000..b3afe4a
--- /dev/null
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexOperatorDescriptorHelper.java
@@ -0,0 +1,33 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.invertedindex.api;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOperatorDescriptorHelper;
+import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex;
+
+public interface IInvertedIndexOperatorDescriptorHelper extends ITreeIndexOperatorDescriptorHelper {
+ public IFileSplitProvider getInvIndexFileSplitProvider();
+
+ public IBinaryComparatorFactory[] getInvIndexComparatorFactories();
+
+ public ITypeTrait[] getInvIndexTypeTraits();
+
+ public IIndexRegistryProvider<InvertedIndex> getInvIndexRegistryProvider();
+}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexSearchModifier.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexSearchModifier.java
index b3819a2..8f7a3e3 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexSearchModifier.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexSearchModifier.java
@@ -1,8 +1,24 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.invertedindex.api;
import java.util.List;
public interface IInvertedIndexSearchModifier {
int getOccurrenceThreshold(List<IInvertedListCursor> invListCursors);
- int getPrefixLists(List<IInvertedListCursor> invListCursors);
+
+ int getPrefixLists(List<IInvertedListCursor> invListCursors);
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexSearcher.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexSearcher.java
index 9a591d0..36d64ea 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexSearcher.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedIndexSearcher.java
@@ -22,9 +22,14 @@
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
public interface IInvertedIndexSearcher {
- public void search(IInvertedIndexResultCursor resultCursor, ITupleReference queryTuple, int queryFieldIndex, IInvertedIndexSearchModifier searchModifier) throws Exception;
+ public void search(IInvertedIndexResultCursor resultCursor, ITupleReference queryTuple, int queryFieldIndex,
+ IInvertedIndexSearchModifier searchModifier) throws Exception;
+
public IFrameTupleAccessor createResultFrameTupleAccessor();
+
public ITupleReference createResultTupleReference();
+
public List<ByteBuffer> getResultBuffers();
- public int getNumValidResultBuffers();
+
+ public int getNumValidResultBuffers();
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedListBuilder.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedListBuilder.java
index 02b57d2..aaaef56 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedListBuilder.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedListBuilder.java
@@ -1,17 +1,32 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.invertedindex.api;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-public interface IInvertedListBuilder {
- public boolean startNewList(ITupleReference tuple, int numTokenFields);
-
- // returns true if successfully appended
- // returns false if not enough space in targetBuf
- public boolean appendElement(ITupleReference tuple, int numTokenFields, int numElementFields);
-
- public void setTargetBuffer(byte[] targetBuf, int startPos);
-
- public int getListSize();
-
- public int getPos();
+public interface IInvertedListBuilder {
+ public boolean startNewList(ITupleReference tuple, int numTokenFields);
+
+ // returns true if successfully appended
+ // returns false if not enough space in targetBuf
+ public boolean appendElement(ITupleReference tuple, int numTokenFields, int numElementFields);
+
+ public void setTargetBuffer(byte[] targetBuf, int startPos);
+
+ public int getListSize();
+
+ public int getPos();
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedListCursor.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedListCursor.java
index f03bdfc..9435f3c 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedListCursor.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/api/IInvertedListCursor.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.invertedindex.api;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
@@ -7,27 +22,35 @@
public interface IInvertedListCursor extends Comparable<IInvertedListCursor> {
void reset(int startPageId, int endPageId, int startOff, int numElements);
-
+
void pinPagesSync() throws HyracksDataException;
+
void pinPagesAsync() throws HyracksDataException;
+
void unpinPages() throws HyracksDataException;
-
+
boolean hasNext();
+
void next();
-
+
ITupleReference getTuple();
-
+
// getters
int getNumElements();
+
int getStartPageId();
+
int getEndPageId();
+
int getStartOff();
-
+
// jump to a specific element
void positionCursor(int elementIx);
- boolean containsKey(ITupleReference searchTuple, MultiComparator invListCmp);
-
+
+ boolean containsKey(ITupleReference searchTuple, MultiComparator invListCmp);
+
// for debugging
- String printInvList(ISerializerDeserializer[] serdes) throws HyracksDataException;
- String printCurrentElement(ISerializerDeserializer[] serdes) throws HyracksDataException;
+ String printInvList(ISerializerDeserializer[] serdes) throws HyracksDataException;
+
+ String printCurrentElement(ISerializerDeserializer[] serdes) throws HyracksDataException;
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/AbstractInvertedIndexOperatorDescriptor.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/AbstractInvertedIndexOperatorDescriptor.java
index 8cc0bd7..0d0c5bf 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/AbstractInvertedIndexOperatorDescriptor.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/AbstractInvertedIndexOperatorDescriptor.java
@@ -25,6 +25,7 @@
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOpHelperFactory;
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexOperatorDescriptorHelper;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
@@ -44,10 +45,10 @@
protected final ITypeTrait[] btreeTypeTraits;
protected final IBinaryComparatorFactory[] btreeComparatorFactories;
protected final ITreeIndexOpHelperFactory opHelperFactory;
-
+
// inverted index
protected final IFileSplitProvider invIndexFileSplitProvider;
- protected final IIndexRegistryProvider<InvertedIndex> invIndexRegistryProvider;
+ protected final IIndexRegistryProvider<InvertedIndex> invIndexRegistryProvider;
protected final ITypeTrait[] invIndexTypeTraits;
protected final IBinaryComparatorFactory[] invIndexComparatorFactories;
@@ -56,8 +57,7 @@
IFileSplitProvider btreeFileSplitProvider, IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider,
ITreeIndexFrameFactory interiorFrameFactory, ITreeIndexFrameFactory leafFrameFactory,
ITypeTrait[] btreeTypeTraits, IBinaryComparatorFactory[] btreeComparatorFactories, float btreeFillFactor,
- ITreeIndexOpHelperFactory opHelperFactory,
- IFileSplitProvider invIndexFileSplitProvider,
+ ITreeIndexOpHelperFactory opHelperFactory, IFileSplitProvider invIndexFileSplitProvider,
IIndexRegistryProvider<InvertedIndex> invIndexRegistryProvider, ITypeTrait[] invIndexTypeTraits,
IBinaryComparatorFactory[] invIndexComparatorFactories) {
super(spec, inputArity, outputArity);
@@ -143,7 +143,7 @@
public ITypeTrait[] getInvIndexTypeTraits() {
return invIndexTypeTraits;
}
-
+
@Override
public ITreeIndexOpHelperFactory getTreeIndexOpHelperFactory() {
return opHelperFactory;
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/IInvertedIndexOperatorDescriptorHelper.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/IInvertedIndexOperatorDescriptorHelper.java
deleted file mode 100644
index 762a86c..0000000
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/IInvertedIndexOperatorDescriptorHelper.java
+++ /dev/null
@@ -1,18 +0,0 @@
-package edu.uci.ics.hyracks.storage.am.invertedindex.dataflow;
-
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
-import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.ITreeIndexOperatorDescriptorHelper;
-import edu.uci.ics.hyracks.storage.am.common.dataflow.IIndexRegistryProvider;
-import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex;
-
-public interface IInvertedIndexOperatorDescriptorHelper extends ITreeIndexOperatorDescriptorHelper {
- public IFileSplitProvider getInvIndexFileSplitProvider();
-
- public IBinaryComparatorFactory[] getInvIndexComparatorFactories();
-
- public ITypeTrait[] getInvIndexTypeTraits();
-
- public IIndexRegistryProvider<InvertedIndex> getInvIndexRegistryProvider();
-}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorDescriptor.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorDescriptor.java
index 2aa0e0a..d003580 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorDescriptor.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorDescriptor.java
@@ -44,13 +44,13 @@
IIndexRegistryProvider<ITreeIndex> treeIndexRegistryProvider, ITreeIndexFrameFactory interiorFrameFactory,
ITreeIndexFrameFactory leafFrameFactory, ITypeTrait[] btreeTypeTraits,
IBinaryComparatorFactory[] btreeComparatorFactories, float btreeFillFactor,
- ITreeIndexOpHelperFactory opHelperFactory,
- IFileSplitProvider invIndexFileSplitProvider,
+ ITreeIndexOpHelperFactory opHelperFactory, IFileSplitProvider invIndexFileSplitProvider,
IIndexRegistryProvider<InvertedIndex> invIndexRegistryProvider, ITypeTrait[] invIndexTypeTraits,
IBinaryComparatorFactory[] invIndexComparatorFactories, IInvertedListBuilder invListBuilder) {
- super(spec, 1, 0, null, storageManager, btreeFileSplitProvider, treeIndexRegistryProvider, interiorFrameFactory,
- leafFrameFactory, btreeTypeTraits, btreeComparatorFactories, btreeFillFactor,
- opHelperFactory, invIndexFileSplitProvider, invIndexRegistryProvider, invIndexTypeTraits, invIndexComparatorFactories);
+ super(spec, 1, 0, null, storageManager, btreeFileSplitProvider, treeIndexRegistryProvider,
+ interiorFrameFactory, leafFrameFactory, btreeTypeTraits, btreeComparatorFactories, btreeFillFactor,
+ opHelperFactory, invIndexFileSplitProvider, invIndexRegistryProvider, invIndexTypeTraits,
+ invIndexComparatorFactories);
this.fieldPermutation = fieldPermutation;
this.btreeFillFactor = btreeFillFactor;
this.invListBuilder = invListBuilder;
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorNodePushable.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorNodePushable.java
index 060f166..4969124 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorNodePushable.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexBulkLoadOperatorNodePushable.java
@@ -31,22 +31,23 @@
public class InvertedIndexBulkLoadOperatorNodePushable extends AbstractUnaryInputSinkOperatorNodePushable {
private final TreeIndexOpHelper treeIndexOpHelper;
private float btreeFillFactor;
-
- private final InvertedIndexOpHelper invIndexOpHelper;
+
+ private final InvertedIndexOpHelper invIndexOpHelper;
protected final IInvertedListBuilder invListBuilder;
private InvertedIndex.BulkLoadContext bulkLoadCtx;
private final IHyracksStageletContext ctx;
-
+
private FrameTupleAccessor accessor;
private PermutingFrameTupleReference tuple = new PermutingFrameTupleReference();
-
+
private IRecordDescriptorProvider recordDescProvider;
-
+
public InvertedIndexBulkLoadOperatorNodePushable(AbstractInvertedIndexOperatorDescriptor opDesc,
IHyracksStageletContext ctx, int partition, int[] fieldPermutation, float btreeFillFactor,
IInvertedListBuilder invListBuilder, IRecordDescriptorProvider recordDescProvider) {
- treeIndexOpHelper = opDesc.getTreeIndexOpHelperFactory().createTreeIndexOpHelper(opDesc, ctx, partition, IndexHelperOpenMode.CREATE);
+ treeIndexOpHelper = opDesc.getTreeIndexOpHelperFactory().createTreeIndexOpHelper(opDesc, ctx, partition,
+ IndexHelperOpenMode.CREATE);
invIndexOpHelper = new InvertedIndexOpHelper(opDesc, ctx, partition, IndexHelperOpenMode.CREATE);
this.btreeFillFactor = btreeFillFactor;
this.recordDescProvider = recordDescProvider;
@@ -61,22 +62,23 @@
.getOperatorDescriptor();
RecordDescriptor recDesc = recordDescProvider.getInputRecordDescriptor(opDesc.getOperatorId(), 0);
accessor = new FrameTupleAccessor(treeIndexOpHelper.getHyracksStageletContext().getFrameSize(), recDesc);
-
+
// btree
try {
treeIndexOpHelper.init();
- treeIndexOpHelper.getTreeIndex().open(treeIndexOpHelper.getIndexFileId());
+ treeIndexOpHelper.getTreeIndex().open(treeIndexOpHelper.getIndexFileId());
} catch (Exception e) {
// cleanup in case of failure
treeIndexOpHelper.deinit();
throw new HyracksDataException(e);
}
-
+
// inverted index
try {
invIndexOpHelper.init();
- invIndexOpHelper.getInvIndex().open(invIndexOpHelper.getInvIndexFileId());
- bulkLoadCtx = invIndexOpHelper.getInvIndex().beginBulkLoad(invListBuilder, ctx.getFrameSize(), btreeFillFactor);
+ invIndexOpHelper.getInvIndex().open(invIndexOpHelper.getInvIndexFileId());
+ bulkLoadCtx = invIndexOpHelper.getInvIndex().beginBulkLoad(invListBuilder, ctx.getFrameSize(),
+ btreeFillFactor);
} catch (Exception e) {
// cleanup in case of failure
invIndexOpHelper.deinit();
@@ -95,7 +97,7 @@
}
@Override
- public void close() throws HyracksDataException {
+ public void close() throws HyracksDataException {
try {
invIndexOpHelper.getInvIndex().endBulkLoad(bulkLoadCtx);
} finally {
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexOpHelper.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexOpHelper.java
index 4beefcd..c16cfcd 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexOpHelper.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/dataflow/InvertedIndexOpHelper.java
@@ -25,30 +25,31 @@
import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexHelperOpenMode;
import edu.uci.ics.hyracks.storage.am.common.dataflow.IndexRegistry;
import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexOperatorDescriptorHelper;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex;
import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
public final class InvertedIndexOpHelper {
-
- private InvertedIndex invIndex;
- private int invIndexFileId = -1;
- private int partition;
+
+ private InvertedIndex invIndex;
+ private int invIndexFileId = -1;
+ private int partition;
private IInvertedIndexOperatorDescriptorHelper opDesc;
private IHyracksStageletContext ctx;
- private IndexHelperOpenMode mode;
+ private IndexHelperOpenMode mode;
- public InvertedIndexOpHelper(IInvertedIndexOperatorDescriptorHelper opDesc, final IHyracksStageletContext ctx, int partition,
- IndexHelperOpenMode mode) {
+ public InvertedIndexOpHelper(IInvertedIndexOperatorDescriptorHelper opDesc, final IHyracksStageletContext ctx,
+ int partition, IndexHelperOpenMode mode) {
this.opDesc = opDesc;
this.ctx = ctx;
this.mode = mode;
this.partition = partition;
}
- public void init() throws HyracksDataException {
+ public void init() throws HyracksDataException {
IBufferCache bufferCache = opDesc.getStorageManager().getBufferCache(ctx);
IFileMapProvider fileMapProvider = opDesc.getStorageManager().getFileMapProvider(ctx);
IFileSplitProvider fileSplitProvider = opDesc.getInvIndexFileSplitProvider();
@@ -56,82 +57,81 @@
FileReference f = fileSplitProvider.getFileSplits()[partition].getLocalFile();
boolean fileIsMapped = fileMapProvider.isMapped(f);
- switch (mode) {
-
- case OPEN: {
- if (!fileIsMapped) {
- throw new HyracksDataException(
- "Trying to open inverted index from unmapped file " + f.toString());
- }
- }
- break;
+ switch (mode) {
- case CREATE:
- case ENLIST: {
- if (!fileIsMapped) {
- bufferCache.createFile(f);
- }
- }
- break;
-
- }
-
+ case OPEN: {
+ if (!fileIsMapped) {
+ throw new HyracksDataException("Trying to open inverted index from unmapped file " + f.toString());
+ }
+ }
+ break;
+
+ case CREATE:
+ case ENLIST: {
+ if (!fileIsMapped) {
+ bufferCache.createFile(f);
+ }
+ }
+ break;
+
+ }
+
int fileId = fileMapProvider.lookupFileId(f);
try {
- bufferCache.openFile(fileId);
- } catch(HyracksDataException e) {
- // revert state of buffer cache since file failed to open
- if(!fileIsMapped) {
- bufferCache.deleteFile(fileId);
- }
- throw e;
+ bufferCache.openFile(fileId);
+ } catch (HyracksDataException e) {
+ // revert state of buffer cache since file failed to open
+ if (!fileIsMapped) {
+ bufferCache.deleteFile(fileId);
+ }
+ throw e;
}
-
- // only set btreeFileId member when openFile() succeeds,
+
+ // only set btreeFileId member when openFile() succeeds,
// otherwise deinit() will try to close the file that failed to open
- invIndexFileId = fileId;
+ invIndexFileId = fileId;
IndexRegistry<InvertedIndex> invIndexRegistry = opDesc.getInvIndexRegistryProvider().getRegistry(ctx);
invIndex = invIndexRegistry.get(invIndexFileId);
if (invIndex == null) {
- // create new inverted index and register it
+ // create new inverted index and register it
invIndexRegistry.lock();
- try {
- // check if inverted index has already been registered by another thread
- invIndex = invIndexRegistry.get(invIndexFileId);
- if (invIndex == null) {
- // this thread should create and register the inverted index
+ try {
+ // check if inverted index has already been registered by
+ // another thread
+ invIndex = invIndexRegistry.get(invIndexFileId);
+ if (invIndex == null) {
+ // this thread should create and register the inverted index
- IBinaryComparator[] comparators = new IBinaryComparator[opDesc
- .getInvIndexComparatorFactories().length];
- for (int i = 0; i < opDesc.getInvIndexComparatorFactories().length; i++) {
- comparators[i] = opDesc.getInvIndexComparatorFactories()[i]
- .createBinaryComparator();
- }
+ IBinaryComparator[] comparators = new IBinaryComparator[opDesc.getInvIndexComparatorFactories().length];
+ for (int i = 0; i < opDesc.getInvIndexComparatorFactories().length; i++) {
+ comparators[i] = opDesc.getInvIndexComparatorFactories()[i].createBinaryComparator();
+ }
- MultiComparator cmp = new MultiComparator(opDesc
- .getInvIndexTypeTraits(), comparators);
-
- // assumes btree has already been registered
- IFileSplitProvider btreeFileSplitProvider = opDesc.getTreeIndexFileSplitProvider();
- IndexRegistry<ITreeIndex> treeIndexRegistry = opDesc.getTreeIndexRegistryProvider().getRegistry(ctx);
- FileReference btreeFile = btreeFileSplitProvider.getFileSplits()[partition].getLocalFile();
- boolean btreeFileIsMapped = fileMapProvider.isMapped(btreeFile);
- if(!btreeFileIsMapped) {
- throw new HyracksDataException("Trying to create inverted index, but associated BTree file has not been mapped");
- }
- int btreeFileId = fileMapProvider.lookupFileId(f);
- BTree btree = (BTree)treeIndexRegistry.get(btreeFileId);
-
- invIndex = new InvertedIndex(bufferCache, btree, cmp);
- invIndex.open(invIndexFileId);
- invIndexRegistry.register(invIndexFileId, invIndex);
- }
- } finally {
- invIndexRegistry.unlock();
- }
- }
- }
+ MultiComparator cmp = new MultiComparator(opDesc.getInvIndexTypeTraits(), comparators);
+
+ // assumes btree has already been registered
+ IFileSplitProvider btreeFileSplitProvider = opDesc.getTreeIndexFileSplitProvider();
+ IndexRegistry<ITreeIndex> treeIndexRegistry = opDesc.getTreeIndexRegistryProvider()
+ .getRegistry(ctx);
+ FileReference btreeFile = btreeFileSplitProvider.getFileSplits()[partition].getLocalFile();
+ boolean btreeFileIsMapped = fileMapProvider.isMapped(btreeFile);
+ if (!btreeFileIsMapped) {
+ throw new HyracksDataException(
+ "Trying to create inverted index, but associated BTree file has not been mapped");
+ }
+ int btreeFileId = fileMapProvider.lookupFileId(f);
+ BTree btree = (BTree) treeIndexRegistry.get(btreeFileId);
+
+ invIndex = new InvertedIndex(bufferCache, btree, cmp);
+ invIndex.open(invIndexFileId);
+ invIndexRegistry.register(invIndexFileId, invIndex);
+ }
+ } finally {
+ invIndexRegistry.unlock();
+ }
+ }
+ }
public void deinit() throws HyracksDataException {
if (invIndexFileId != -1) {
@@ -140,19 +140,19 @@
}
}
- public InvertedIndex getInvIndex() {
- return invIndex;
- }
-
+ public InvertedIndex getInvIndex() {
+ return invIndex;
+ }
+
public IHyracksStageletContext getHyracksStageletContext() {
return ctx;
}
- public ITreeIndexOperatorDescriptorHelper getOperatorDescriptor() {
- return opDesc;
- }
-
- public int getInvIndexFileId() {
- return invIndexFileId;
- }
+ public ITreeIndexOperatorDescriptorHelper getOperatorDescriptor() {
+ return opDesc;
+ }
+
+ public int getInvIndexFileId() {
+ return invIndexFileId;
+ }
}
\ No newline at end of file
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListBuilder.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListBuilder.java
index d141db0..03fc9a1 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListBuilder.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListBuilder.java
@@ -1,61 +1,79 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.invertedindex.impls;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListBuilder;
-public class FixedSizeElementInvertedListBuilder implements IInvertedListBuilder {
- private final int listElementSize;
- private int listSize = 0;
-
- private byte[] targetBuf;
- private int pos;
-
- public FixedSizeElementInvertedListBuilder(ITypeTrait[] invListFields) {
- int tmp = 0;
- for(int i = 0; i < invListFields.length; i++) {
- tmp += invListFields[i].getStaticallyKnownDataLength();
- }
- listElementSize = tmp;
- }
-
- @Override
- public boolean startNewList(ITupleReference tuple, int tokenField) {
- if(pos + listElementSize > targetBuf.length) return false;
- else {
- listSize = 0;
- return true;
- }
- }
-
- @Override
- public boolean appendElement(ITupleReference tuple, int numTokenFields, int numElementFields) {
- if(pos + listElementSize > targetBuf.length) return false;
-
- for(int i = 0; i < numElementFields; i++) {
- int field = numTokenFields + i;
- System.arraycopy(tuple.getFieldData(field), tuple.getFieldStart(field), targetBuf, pos, tuple.getFieldLength(field));
- }
-
- listSize++;
- pos += listElementSize;
-
- return true;
- }
-
- @Override
- public void setTargetBuffer(byte[] targetBuf, int startPos) {
- this.targetBuf = targetBuf;
- this.pos = startPos;
- }
+public class FixedSizeElementInvertedListBuilder implements IInvertedListBuilder {
+ private final int listElementSize;
+ private int listSize = 0;
- @Override
- public int getListSize() {
- return listSize;
- }
+ private byte[] targetBuf;
+ private int pos;
- @Override
- public int getPos() {
- return pos;
- }
+ public FixedSizeElementInvertedListBuilder(ITypeTrait[] invListFields) {
+ int tmp = 0;
+ for (int i = 0; i < invListFields.length; i++) {
+ tmp += invListFields[i].getStaticallyKnownDataLength();
+ }
+ listElementSize = tmp;
+ }
+
+ @Override
+ public boolean startNewList(ITupleReference tuple, int tokenField) {
+ if (pos + listElementSize > targetBuf.length)
+ return false;
+ else {
+ listSize = 0;
+ return true;
+ }
+ }
+
+ @Override
+ public boolean appendElement(ITupleReference tuple, int numTokenFields, int numElementFields) {
+ if (pos + listElementSize > targetBuf.length)
+ return false;
+
+ for (int i = 0; i < numElementFields; i++) {
+ int field = numTokenFields + i;
+ System.arraycopy(tuple.getFieldData(field), tuple.getFieldStart(field), targetBuf, pos,
+ tuple.getFieldLength(field));
+ }
+
+ listSize++;
+ pos += listElementSize;
+
+ return true;
+ }
+
+ @Override
+ public void setTargetBuffer(byte[] targetBuf, int startPos) {
+ this.targetBuf = targetBuf;
+ this.pos = startPos;
+ }
+
+ @Override
+ public int getListSize() {
+ return listSize;
+ }
+
+ @Override
+ public int getPos() {
+ return pos;
+ }
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListCursor.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListCursor.java
index 9ed0ee1..f7ef56e 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListCursor.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeElementInvertedListCursor.java
@@ -21,99 +21,98 @@
private final int elementSize;
private int currentElementIx;
private int currentOff;
- private int currentPageIx;
-
+ private int currentPageIx;
+
private int startPageId;
private int endPageId;
private int startOff;
private int numElements;
-
+
private final FixedSizeTupleReference tuple;
private ICachedPage[] pages = new ICachedPage[10];
private int[] elementIndexes = new int[10];
-
+
public FixedSizeElementInvertedListCursor(IBufferCache bufferCache, int fileId, ITypeTrait[] invListFields) {
this.bufferCache = bufferCache;
this.fileId = fileId;
this.currentElementIx = 0;
- this.currentPageIx = 0;
-
+ this.currentPageIx = 0;
+
int tmp = 0;
- for(int i = 0; i < invListFields.length; i++) {
+ for (int i = 0; i < invListFields.length; i++) {
tmp += invListFields[i].getStaticallyKnownDataLength();
}
elementSize = tmp;
this.currentOff = -elementSize;
this.tuple = new FixedSizeTupleReference(invListFields);
}
-
+
@Override
public boolean hasNext() {
- if(currentElementIx < numElements) return true;
- else return false;
+ if (currentElementIx < numElements)
+ return true;
+ else
+ return false;
}
@Override
public void next() {
- //System.out.println("NEXT: " + currentOff + " " + elementSize + " " + bufferCache.getPageSize());
- if(currentOff + elementSize >= bufferCache.getPageSize()) {
+ if (currentOff + elementSize >= bufferCache.getPageSize()) {
currentPageIx++;
- currentOff = 0;
- }
- else {
+ currentOff = 0;
+ } else {
currentOff += elementSize;
}
-
+
currentElementIx++;
tuple.reset(pages[currentPageIx].getBuffer().array(), currentOff);
}
@Override
public void pinPagesAsync() {
-
-
+ // TODO: implement
}
@Override
- public void pinPagesSync() throws HyracksDataException {
+ public void pinPagesSync() throws HyracksDataException {
int pix = 0;
- for(int i = startPageId; i <= endPageId; i++) {
+ for (int i = startPageId; i <= endPageId; i++) {
pages[pix] = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, i), false);
pages[pix].acquireReadLatch();
pix++;
- }
+ }
}
@Override
- public void unpinPages() throws HyracksDataException {
+ public void unpinPages() throws HyracksDataException {
int numPages = endPageId - startPageId + 1;
- for(int i = 0; i < numPages; i++) {
+ for (int i = 0; i < numPages; i++) {
pages[i].releaseReadLatch();
bufferCache.unpin(pages[i]);
- }
+ }
}
-
+
@Override
public void positionCursor(int elementIx) {
int numPages = endPageId - startPageId + 1;
-
- currentPageIx = binarySearch(elementIndexes, 0, numPages, elementIx);
- if(currentPageIx < 0) {
- throw new IndexOutOfBoundsException("Requested index: " + elementIx + " from array with numElements: " + numElements);
+
+ currentPageIx = binarySearch(elementIndexes, 0, numPages, elementIx);
+ if (currentPageIx < 0) {
+ throw new IndexOutOfBoundsException("Requested index: " + elementIx + " from array with numElements: "
+ + numElements);
}
-
- if(currentPageIx == 0) {
+
+ if (currentPageIx == 0) {
currentOff = startOff + elementIx * elementSize;
- }
- else {
- int relativeElementIx = elementIx - elementIndexes[currentPageIx-1] - 1;
+ } else {
+ int relativeElementIx = elementIx - elementIndexes[currentPageIx - 1] - 1;
currentOff = relativeElementIx * elementSize;
}
-
+
currentElementIx = elementIx;
tuple.reset(pages[currentPageIx].getBuffer().array(), currentOff);
}
-
+
@Override
public boolean containsKey(ITupleReference searchTuple, MultiComparator invListCmp) {
int mid;
@@ -128,14 +127,14 @@
end = mid - 1;
} else if (cmp > 0) {
begin = mid + 1;
- } else {
+ } else {
return true;
}
}
-
+
return false;
- }
-
+ }
+
@Override
public void reset(int startPageId, int endPageId, int startOff, int numElements) {
this.startPageId = startPageId;
@@ -145,97 +144,100 @@
this.currentElementIx = 0;
this.currentPageIx = 0;
this.currentOff = startOff - elementSize;
-
+
int numPages = endPageId - startPageId + 1;
- if(numPages > pages.length) {
+ if (numPages > pages.length) {
pages = new ICachedPage[endPageId - startPageId + 1];
elementIndexes = new int[endPageId - startPageId + 1];
- }
-
+ }
+
// fill elementIndexes
// first page
int cumulElements = (bufferCache.getPageSize() - startOff) / elementSize;
- elementIndexes[0] = cumulElements-1;
-
- // middle, full pages
- for(int i = 1; i < numPages-1; i++) {
- elementIndexes[i] = elementIndexes[i-1] + (bufferCache.getPageSize() / elementSize);
+ elementIndexes[0] = cumulElements - 1;
+
+ // middle, full pages
+ for (int i = 1; i < numPages - 1; i++) {
+ elementIndexes[i] = elementIndexes[i - 1] + (bufferCache.getPageSize() / elementSize);
}
-
+
// last page
- elementIndexes[numPages-1] = numElements-1;
- }
-
+ elementIndexes[numPages - 1] = numElements - 1;
+ }
+
@Override
- public String printInvList(ISerializerDeserializer[] serdes) throws HyracksDataException {
+ public String printInvList(ISerializerDeserializer[] serdes) throws HyracksDataException {
int oldCurrentOff = currentOff;
int oldCurrentPageId = currentPageIx;
int oldCurrentElementIx = currentElementIx;
-
+
currentOff = startOff - elementSize;
currentPageIx = 0;
currentElementIx = 0;
-
+
StringBuilder strBuilder = new StringBuilder();
-
+
int count = 0;
- while(hasNext()) {
+ while (hasNext()) {
next();
- count++;
-
- // System.out.println(offset + " " + currentElementIx);
-
- for(int i = 0; i < tuple.getFieldCount(); i++) {
- ByteArrayInputStream inStream = new ByteArrayInputStream(tuple.getFieldData(i), tuple.getFieldStart(i), tuple.getFieldLength(i));
- DataInput dataIn = new DataInputStream(inStream);
+ count++;
+ for (int i = 0; i < tuple.getFieldCount(); i++) {
+ ByteArrayInputStream inStream = new ByteArrayInputStream(tuple.getFieldData(i), tuple.getFieldStart(i),
+ tuple.getFieldLength(i));
+ DataInput dataIn = new DataInputStream(inStream);
Object o = serdes[i].deserialize(dataIn);
strBuilder.append(o.toString());
- if(i+1 < tuple.getFieldCount()) strBuilder.append(",");
+ if (i + 1 < tuple.getFieldCount())
+ strBuilder.append(",");
}
strBuilder.append(" ");
}
- //System.out.println("PRINT COUNT: " + count);
-
+
// reset previous state
currentOff = oldCurrentOff;
currentPageIx = oldCurrentPageId;
currentElementIx = oldCurrentElementIx;
-
- return strBuilder.toString();
- }
-
- public String printCurrentElement(ISerializerDeserializer[] serdes) throws HyracksDataException {
- StringBuilder strBuilder = new StringBuilder();
- for(int i = 0; i < tuple.getFieldCount(); i++) {
- ByteArrayInputStream inStream = new ByteArrayInputStream(tuple.getFieldData(i), tuple.getFieldStart(i), tuple.getFieldLength(i));
- DataInput dataIn = new DataInputStream(inStream);
- Object o = serdes[i].deserialize(dataIn);
- strBuilder.append(o.toString());
- if(i+1 < tuple.getFieldCount()) strBuilder.append(",");
- }
+
return strBuilder.toString();
}
-
+
+ public String printCurrentElement(ISerializerDeserializer[] serdes) throws HyracksDataException {
+ StringBuilder strBuilder = new StringBuilder();
+ for (int i = 0; i < tuple.getFieldCount(); i++) {
+ ByteArrayInputStream inStream = new ByteArrayInputStream(tuple.getFieldData(i), tuple.getFieldStart(i),
+ tuple.getFieldLength(i));
+ DataInput dataIn = new DataInputStream(inStream);
+ Object o = serdes[i].deserialize(dataIn);
+ strBuilder.append(o.toString());
+ if (i + 1 < tuple.getFieldCount())
+ strBuilder.append(",");
+ }
+ return strBuilder.toString();
+ }
+
private int binarySearch(int[] arr, int arrStart, int arrLength, int key) {
int mid;
int begin = arrStart;
int end = arrStart + arrLength - 1;
while (begin <= end) {
- mid = (begin + end) / 2;
+ mid = (begin + end) / 2;
int cmp = (key - arr[mid]);
if (cmp < 0) {
end = mid - 1;
} else if (cmp > 0) {
begin = mid + 1;
- } else {
+ } else {
return mid;
}
}
-
- if(begin > arr.length - 1) return -1;
- if(key < arr[begin]) return begin;
- else return -1;
+
+ if (begin > arr.length - 1)
+ return -1;
+ if (key < arr[begin])
+ return begin;
+ else
+ return -1;
}
@Override
@@ -270,9 +272,9 @@
public ICachedPage getPage() {
return pages[currentPageIx];
}
-
+
@Override
- public ITupleReference getTuple() {
+ public ITupleReference getTuple() {
return tuple;
}
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeFrameTupleAccessor.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeFrameTupleAccessor.java
index 1c3c9e4..9858eb0 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeFrameTupleAccessor.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeFrameTupleAccessor.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.invertedindex.impls;
import java.nio.ByteBuffer;
@@ -10,27 +25,27 @@
private final int frameSize;
private ByteBuffer buffer;
-
+
private final ITypeTrait[] fields;
private final int[] fieldStartOffsets;
private final int tupleSize;
-
+
public FixedSizeFrameTupleAccessor(int frameSize, ITypeTrait[] fields) {
this.frameSize = frameSize;
this.fields = fields;
this.fieldStartOffsets = new int[fields.length];
this.fieldStartOffsets[0] = 0;
- for(int i = 1; i < fields.length; i++) {
- fieldStartOffsets[i] = fieldStartOffsets[i-1] + fields[i-1].getStaticallyKnownDataLength();
+ for (int i = 1; i < fields.length; i++) {
+ fieldStartOffsets[i] = fieldStartOffsets[i - 1] + fields[i - 1].getStaticallyKnownDataLength();
}
-
+
int tmp = 0;
- for(int i = 0; i < fields.length; i++) {
+ for (int i = 0; i < fields.length; i++) {
tmp += fields[i].getStaticallyKnownDataLength();
}
tupleSize = tmp;
}
-
+
@Override
public ByteBuffer getBuffer() {
return buffer;
@@ -68,7 +83,7 @@
@Override
public int getTupleEndOffset(int tupleIndex) {
- return getFieldEndOffset(tupleIndex, fields.length-1);
+ return getFieldEndOffset(tupleIndex, fields.length - 1);
}
@Override
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeFrameTupleAppender.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeFrameTupleAppender.java
index 7d95584..edc2304 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeFrameTupleAppender.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeFrameTupleAppender.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.invertedindex.impls;
import java.nio.ByteBuffer;
@@ -12,17 +27,17 @@
private final int tupleSize;
private ByteBuffer buffer;
private int tupleCount;
- private int tupleDataEndOffset;
+ private int tupleDataEndOffset;
public FixedSizeFrameTupleAppender(int frameSize, ITypeTrait[] fields) {
this.frameSize = frameSize;
int tmp = 0;
- for(int i = 0; i < fields.length; i++) {
+ for (int i = 0; i < fields.length; i++) {
tmp += fields[i].getStaticallyKnownDataLength();
}
tupleSize = tmp;
}
-
+
public void reset(ByteBuffer buffer, boolean clear) {
this.buffer = buffer;
if (clear) {
@@ -31,76 +46,77 @@
tupleDataEndOffset = 0;
}
}
-
+
public boolean append(byte[] bytes, int offset) {
if (tupleDataEndOffset + tupleSize + TUPLE_COUNT_SIZE <= frameSize) {
System.arraycopy(bytes, offset, buffer.array(), tupleDataEndOffset, tupleSize);
tupleDataEndOffset += tupleSize;
- tupleCount++;
+ tupleCount++;
return true;
}
return false;
- }
-
+ }
+
public boolean append(byte[] bytes, int offset, int length) {
if (tupleDataEndOffset + length + TUPLE_COUNT_SIZE <= frameSize) {
System.arraycopy(bytes, offset, buffer.array(), tupleDataEndOffset, length);
- tupleDataEndOffset += length;
+ tupleDataEndOffset += length;
return true;
}
return false;
}
-
+
public boolean append(int fieldValue) {
if (tupleDataEndOffset + 4 + TUPLE_COUNT_SIZE <= frameSize) {
- buffer.putInt(tupleDataEndOffset, fieldValue);
+ buffer.putInt(tupleDataEndOffset, fieldValue);
tupleDataEndOffset += 4;
- tupleCount++;
+ tupleCount++;
return true;
}
return false;
}
-
+
public boolean append(long fieldValue) {
if (tupleDataEndOffset + 8 + TUPLE_COUNT_SIZE <= frameSize) {
- buffer.putLong(tupleDataEndOffset, fieldValue);
+ buffer.putLong(tupleDataEndOffset, fieldValue);
tupleDataEndOffset += 8;
- tupleCount++;
+ tupleCount++;
return true;
}
return false;
}
-
+
public boolean append(char fieldValue) {
if (tupleDataEndOffset + 2 + TUPLE_COUNT_SIZE <= frameSize) {
- buffer.putLong(tupleDataEndOffset, fieldValue);
+ buffer.putLong(tupleDataEndOffset, fieldValue);
tupleDataEndOffset += 2;
- tupleCount++;
+ tupleCount++;
return true;
}
return false;
}
-
+
public boolean append(byte fieldValue) {
if (tupleDataEndOffset + 1 + TUPLE_COUNT_SIZE <= frameSize) {
- buffer.put(tupleDataEndOffset, fieldValue);
+ buffer.put(tupleDataEndOffset, fieldValue);
tupleDataEndOffset += 1;
- tupleCount++;
+ tupleCount++;
return true;
}
return false;
- }
-
+ }
+
// returns true if an entire tuple fits
// returns false otherwise
public boolean hasSpace() {
return tupleDataEndOffset + tupleSize + TUPLE_COUNT_SIZE <= frameSize;
}
-
+
public void incrementTupleCount(int count) {
- buffer.putInt(FrameHelper.getTupleCountOffset(frameSize), buffer.getInt(FrameHelper.getTupleCountOffset(frameSize)) + count);
+ buffer.putInt(FrameHelper.getTupleCountOffset(frameSize),
+ buffer.getInt(FrameHelper.getTupleCountOffset(frameSize)) + count);
}
-
+
public int getTupleCount() {
return tupleCount;
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeTupleReference.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeTupleReference.java
index 0ddde07..248b81e 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeTupleReference.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/FixedSizeTupleReference.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.invertedindex.impls;
import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
@@ -8,22 +23,22 @@
private final ITypeTrait[] typeTraits;
private final int[] fieldStartOffsets;
private byte[] data;
- private int startOff;
-
+ private int startOff;
+
public FixedSizeTupleReference(ITypeTrait[] typeTraits) {
this.typeTraits = typeTraits;
this.fieldStartOffsets = new int[typeTraits.length];
this.fieldStartOffsets[0] = 0;
- for(int i = 1; i < typeTraits.length; i++) {
- fieldStartOffsets[i] = fieldStartOffsets[i-1] + typeTraits[i-1].getStaticallyKnownDataLength();
+ for (int i = 1; i < typeTraits.length; i++) {
+ fieldStartOffsets[i] = fieldStartOffsets[i - 1] + typeTraits[i - 1].getStaticallyKnownDataLength();
}
}
-
+
public void reset(byte[] data, int startOff) {
this.data = data;
this.startOff = startOff;
}
-
+
@Override
public int getFieldCount() {
return typeTraits.length;
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java
index fc3cf15..9eab110 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndex.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.invertedindex.impls;
import java.nio.ByteBuffer;
@@ -24,6 +39,14 @@
import edu.uci.ics.hyracks.storage.common.buffercache.ICachedPage;
import edu.uci.ics.hyracks.storage.common.file.BufferedFileHandle;
+/**
+ * An inverted index consists of two files: 1. a file storing (paginated)
+ * inverted lists 2. a BTree-file mapping from tokens to inverted lists
+ *
+ * Implemented features: bulk loading and searching (based on T-Occurrence) Not
+ * implemented features: updates (insert/update/delete) Limitations: a query
+ * cannot exceed the size of a Hyracks frame
+ */
public class InvertedIndex {
private BTree btree;
@@ -32,8 +55,8 @@
private int fileId;
private final MultiComparator invListCmp;
private final int numTokenFields;
- private final int numInvListKeys;
-
+ private final int numInvListKeys;
+
public InvertedIndex(IBufferCache bufferCache, BTree btree, MultiComparator invListCmp) {
this.bufferCache = bufferCache;
this.btree = btree;
@@ -45,32 +68,26 @@
public void open(int fileId) {
this.fileId = fileId;
}
-
+
public void close() {
- this.fileId = -1;
+ this.fileId = -1;
}
-
- public BulkLoadContext beginBulkLoad(IInvertedListBuilder invListBuilder, int hyracksFrameSize, float btreeFillFactor) throws HyracksDataException {
+
+ public BulkLoadContext beginBulkLoad(IInvertedListBuilder invListBuilder, int hyracksFrameSize,
+ float btreeFillFactor) throws HyracksDataException {
BulkLoadContext ctx = new BulkLoadContext(invListBuilder, hyracksFrameSize, btreeFillFactor);
ctx.init(rootPageId, fileId);
return ctx;
}
-
- // ASSUMPTIONS:
- // the first btree.getMultiComparator().getKeyFieldCount() fields in tuple are btree keys (e.g., a string token)
- // the next invListCmp.getKeyFieldCount() fields in tuple are keys of the inverted list (e.g., primary key)
+
+ // ASSUMPTIONS:
+ // the first btree.getMultiComparator().getKeyFieldCount() fields in tuple
+ // are btree keys (e.g., a string token)
+ // the next invListCmp.getKeyFieldCount() fields in tuple are keys of the
+ // inverted list (e.g., primary key)
// key fields of inverted list are fixed size
- public void bulkLoadAddTuple(BulkLoadContext ctx, ITupleReference tuple)
- throws HyracksDataException {
-
- // debug
- //UTF8StringSerializerDeserializer serde = UTF8StringSerializerDeserializer.INSTANCE;
- //ByteArrayInputStream inStream = new ByteArrayInputStream(tuple.getFieldData(tokenField), tuple.getFieldStart(tokenField), tuple
- // .getFieldLength(tokenField));
- //DataInput dataIn = new DataInputStream(inStream);
- //Object o = serde.deserialize(dataIn);
- //System.out.println(o.toString());
-
+ public void bulkLoadAddTuple(BulkLoadContext ctx, ITupleReference tuple) throws HyracksDataException {
+
// first inverted list, copy token to baaos and start new list
if (ctx.currentInvListTokenBaaos.size() == 0) {
ctx.currentInvListStartPageId = ctx.currentPageId;
@@ -79,8 +96,8 @@
// remember current token
ctx.currentInvListTokenBaaos.reset();
for (int i = 0; i < numTokenFields; i++) {
- ctx.currentInvListTokenBaaos.write(tuple.getFieldData(i), tuple.getFieldStart(i), tuple
- .getFieldLength(i));
+ ctx.currentInvListTokenBaaos.write(tuple.getFieldData(i), tuple.getFieldStart(i),
+ tuple.getFieldLength(i));
}
if (!ctx.invListBuilder.startNewList(tuple, numTokenFields)) {
@@ -90,22 +107,22 @@
throw new IllegalStateException("Failed to create first inverted list.");
}
}
- }
-
+ }
+
// create new inverted list?
ctx.currentInvListToken.reset(ctx.currentInvListTokenBaaos.getByteArray(), 0);
if (ctx.tokenCmp.compare(tuple, ctx.currentInvListToken) != 0) {
-
+
// create entry in btree for last inverted list
createAndInsertBTreeTuple(ctx);
// remember new token
ctx.currentInvListTokenBaaos.reset();
for (int i = 0; i < numTokenFields; i++) {
- ctx.currentInvListTokenBaaos.write(tuple.getFieldData(i), tuple.getFieldStart(i), tuple
- .getFieldLength(i));
+ ctx.currentInvListTokenBaaos.write(tuple.getFieldData(i), tuple.getFieldStart(i),
+ tuple.getFieldLength(i));
}
-
+
// start new list
if (!ctx.invListBuilder.startNewList(tuple, numTokenFields)) {
ctx.pinNextPage();
@@ -118,7 +135,7 @@
ctx.currentInvListStartPageId = ctx.currentPageId;
ctx.currentInvListStartOffset = ctx.invListBuilder.getPos();
}
-
+
// append to current inverted list
if (!ctx.invListBuilder.appendElement(tuple, numTokenFields, numInvListKeys)) {
ctx.pinNextPage();
@@ -128,41 +145,45 @@
"Failed to append element to inverted list after switching to a new page.");
}
}
- }
-
- public boolean openCursor(ITreeIndexCursor btreeCursor, RangePredicate btreePred, BTreeOpContext btreeOpCtx, IInvertedListCursor invListCursor) throws Exception {
- btree.search(btreeCursor, btreePred, btreeOpCtx);
-
+ }
+
+ public boolean openCursor(ITreeIndexCursor btreeCursor, RangePredicate btreePred, BTreeOpContext btreeOpCtx,
+ IInvertedListCursor invListCursor) throws Exception {
+ btree.search(btreeCursor, btreePred, btreeOpCtx);
+
boolean ret = false;
- if(btreeCursor.hasNext()) {
-
+ if (btreeCursor.hasNext()) {
+
btreeCursor.next();
ITupleReference frameTuple = btreeCursor.getTuple();
-
- // TODO: hardcoded mapping of btree fields
- int startPageId = IntegerSerializerDeserializer.getInt(frameTuple.getFieldData(1), frameTuple.getFieldStart(1));
- int endPageId = IntegerSerializerDeserializer.getInt(frameTuple.getFieldData(2), frameTuple.getFieldStart(2));
- int startOff = IntegerSerializerDeserializer.getInt(frameTuple.getFieldData(3), frameTuple.getFieldStart(3));
- int numElements = IntegerSerializerDeserializer.getInt(frameTuple.getFieldData(4), frameTuple.getFieldStart(4));
-
+
+ // hardcoded mapping of btree fields
+ int startPageId = IntegerSerializerDeserializer.getInt(frameTuple.getFieldData(1),
+ frameTuple.getFieldStart(1));
+ int endPageId = IntegerSerializerDeserializer.getInt(frameTuple.getFieldData(2),
+ frameTuple.getFieldStart(2));
+ int startOff = IntegerSerializerDeserializer
+ .getInt(frameTuple.getFieldData(3), frameTuple.getFieldStart(3));
+ int numElements = IntegerSerializerDeserializer.getInt(frameTuple.getFieldData(4),
+ frameTuple.getFieldStart(4));
+
invListCursor.reset(startPageId, endPageId, startOff, numElements);
ret = true;
- }
- else {
+ } else {
invListCursor.reset(0, 0, 0, 0);
}
-
+
btreeCursor.close();
btreeCursor.reset();
-
+
return ret;
}
-
+
public void createAndInsertBTreeTuple(BulkLoadContext ctx) throws HyracksDataException {
// build tuple
ctx.btreeTupleBuilder.reset();
- ctx.btreeTupleBuilder.addField(ctx.currentInvListTokenBaaos.getByteArray(), 0, ctx.currentInvListTokenBaaos
- .size());
+ ctx.btreeTupleBuilder.addField(ctx.currentInvListTokenBaaos.getByteArray(), 0,
+ ctx.currentInvListTokenBaaos.size());
ctx.btreeTupleBuilder.addField(IntegerSerializerDeserializer.INSTANCE, ctx.currentInvListStartPageId);
ctx.btreeTupleBuilder.addField(IntegerSerializerDeserializer.INSTANCE, ctx.currentPageId);
ctx.btreeTupleBuilder.addField(IntegerSerializerDeserializer.INSTANCE, ctx.currentInvListStartOffset);
@@ -173,7 +194,7 @@
ctx.btreeTupleAppender.append(ctx.btreeTupleBuilder.getFieldEndOffsets(), ctx.btreeTupleBuilder.getByteArray(),
0, ctx.btreeTupleBuilder.getSize());
- // reset tuple reference
+ // reset tuple reference
ctx.btreeFrameTupleReference.reset(ctx.btreeFrameTupleAccessor, 0);
btree.bulkLoadAddTuple(ctx.btreeBulkLoadCtx, ctx.btreeFrameTupleReference);
@@ -189,19 +210,19 @@
public IBufferCache getBufferCache() {
return bufferCache;
}
-
+
public int getInvListsFileId() {
return fileId;
}
-
+
public MultiComparator getInvListElementCmp() {
return invListCmp;
}
-
+
public BTree getBTree() {
return btree;
}
-
+
public final class BulkLoadContext {
private final ByteBuffer btreeTupleBuffer;
private final ArrayTupleBuilder btreeTupleBuilder;
@@ -214,33 +235,35 @@
private int currentInvListStartPageId;
private int currentInvListStartOffset;
private final ByteArrayAccessibleOutputStream currentInvListTokenBaaos = new ByteArrayAccessibleOutputStream();
- private final FixedSizeTupleReference currentInvListToken = new FixedSizeTupleReference(invListCmp.getTypeTraits());
-
+ private final FixedSizeTupleReference currentInvListToken = new FixedSizeTupleReference(
+ invListCmp.getTypeTraits());
+
private int currentPageId;
private ICachedPage currentPage;
private final IInvertedListBuilder invListBuilder;
- private final MultiComparator tokenCmp;
-
+ private final MultiComparator tokenCmp;
+
public BulkLoadContext(IInvertedListBuilder invListBuilder, int hyracksFrameSize, float btreeFillFactor) {
this.invListBuilder = invListBuilder;
this.tokenCmp = btree.getMultiComparator();
this.btreeTupleBuffer = ByteBuffer.allocate(hyracksFrameSize);
this.btreeTupleBuilder = new ArrayTupleBuilder(tokenCmp.getFieldCount());
this.btreeTupleAppender = new FrameTupleAppender(hyracksFrameSize);
- // TODO: dummy record descriptor, serde never used anyway, only need
- // correct number of fields
- // tuple contains (token, start page, end page, start offset, num elements)
+ // TODO: serde never used, only need correct number of fields
+ // tuple contains (token, start page, end page, start offset, num
+ // elements)
RecordDescriptor recDesc = new RecordDescriptor(new ISerializerDeserializer[] {
IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE });
+ IntegerSerializerDeserializer.INSTANCE, IntegerSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE });
this.btreeFrameTupleAccessor = new FrameTupleAccessor(hyracksFrameSize, recDesc);
this.btreeFillFactor = btreeFillFactor;
}
public void init(int startPageId, int fileId) throws HyracksDataException {
- btreeBulkLoadCtx = btree.beginBulkLoad(BTree.DEFAULT_FILL_FACTOR, btree.getLeafFrameFactory().createFrame(),
- btree.getInteriorFrameFactory().createFrame(), btree.getFreePageManager().getMetaDataFrameFactory()
- .createFrame());
+ btreeBulkLoadCtx = btree.beginBulkLoad(BTree.DEFAULT_FILL_FACTOR,
+ btree.getLeafFrameFactory().createFrame(), btree.getInteriorFrameFactory().createFrame(), btree
+ .getFreePageManager().getMetaDataFrameFactory().createFrame());
currentPageId = startPageId;
currentPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), true);
currentPage.acquireWriteLatch();
@@ -262,5 +285,5 @@
currentPage = bufferCache.pin(BufferedFileHandle.getDiskPageId(fileId, currentPageId), true);
currentPage.acquireWriteLatch();
}
- };
+ };
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndexException.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndexException.java
index 5ba852a..494ac0c 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndexException.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/InvertedIndexException.java
@@ -1,8 +1,24 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.invertedindex.impls;
-public class InvertedIndexException extends Exception {
- private static final long serialVersionUID = 1L;
- public InvertedIndexException(String msg) {
- super(msg);
- }
+public class InvertedIndexException extends Exception {
+ private static final long serialVersionUID = 1L;
+
+ public InvertedIndexException(String msg) {
+ super(msg);
+ }
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/OccurrenceThresholdPanicException.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/OccurrenceThresholdPanicException.java
index 4711fb6..b0e737c 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/OccurrenceThresholdPanicException.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/OccurrenceThresholdPanicException.java
@@ -1,9 +1,24 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.invertedindex.impls;
public class OccurrenceThresholdPanicException extends InvertedIndexException {
- private static final long serialVersionUID = 1L;
-
- public OccurrenceThresholdPanicException(String msg) {
- super(msg);
- }
+ private static final long serialVersionUID = 1L;
+
+ public OccurrenceThresholdPanicException(String msg) {
+ super(msg);
+ }
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/SearchResultCursor.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/SearchResultCursor.java
index f1fe841..5a7230c 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/SearchResultCursor.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/SearchResultCursor.java
@@ -30,13 +30,13 @@
private FixedSizeTupleReference resultTuple;
private int numResultBuffers;
private int currentBufferIndex = 0;
- private int tupleIndex = 0;
-
+ private int tupleIndex = 0;
+
public SearchResultCursor(IFrameTupleAccessor fta, ITupleReference resultTuple) {
this.fta = fta;
- this.resultTuple = (FixedSizeTupleReference)resultTuple;
+ this.resultTuple = (FixedSizeTupleReference) resultTuple;
}
-
+
@Override
public boolean hasNext() {
if (currentBufferIndex < numResultBuffers && tupleIndex < fta.getTupleCount())
@@ -46,16 +46,16 @@
}
@Override
- public void next() {
- resultTuple.reset(fta.getBuffer().array(), fta.getTupleStartOffset(tupleIndex));
+ public void next() {
+ resultTuple.reset(fta.getBuffer().array(), fta.getTupleStartOffset(tupleIndex));
tupleIndex++;
- if(tupleIndex >= fta.getTupleCount()) {
- if(currentBufferIndex + 1 < numResultBuffers) {
+ if (tupleIndex >= fta.getTupleCount()) {
+ if (currentBufferIndex + 1 < numResultBuffers) {
currentBufferIndex++;
fta.reset(resultBuffers.get(currentBufferIndex));
tupleIndex = 0;
}
- }
+ }
}
@Override
@@ -64,12 +64,12 @@
}
@Override
- public void reset(IInvertedIndexSearcher invIndexSearcher) {
+ public void reset(IInvertedIndexSearcher invIndexSearcher) {
currentBufferIndex = 0;
tupleIndex = 0;
resultBuffers = invIndexSearcher.getResultBuffers();
numResultBuffers = invIndexSearcher.getNumValidResultBuffers();
- if(numResultBuffers > 0) {
+ if (numResultBuffers > 0) {
fta.reset(resultBuffers.get(0));
}
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcher.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcher.java
index 41d831f..5585856 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcher.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcher.java
@@ -15,6 +15,9 @@
package edu.uci.ics.hyracks.storage.am.invertedindex.impls;
+import java.io.ByteArrayInputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.IOException;
import java.nio.ByteBuffer;
@@ -52,243 +55,239 @@
public class TOccurrenceSearcher implements IInvertedIndexSearcher {
- protected final IHyracksStageletContext ctx;
- protected final FixedSizeFrameTupleAppender resultFrameTupleApp;
- protected final FixedSizeFrameTupleAccessor resultFrameTupleAcc;
- protected final FixedSizeTupleReference resultTuple;
- protected final int invListKeyLength;
- protected int currentNumResults;
+ protected final IHyracksStageletContext ctx;
+ protected final FixedSizeFrameTupleAppender resultFrameTupleApp;
+ protected final FixedSizeFrameTupleAccessor resultFrameTupleAcc;
+ protected final FixedSizeTupleReference resultTuple;
+ protected final int invListKeyLength;
+ protected int currentNumResults;
- protected List<ByteBuffer> newResultBuffers = new ArrayList<ByteBuffer>();
- protected List<ByteBuffer> prevResultBuffers = new ArrayList<ByteBuffer>();
- protected List<ByteBuffer> swap = null;
- protected int maxResultBufIdx = 0;
+ protected List<ByteBuffer> newResultBuffers = new ArrayList<ByteBuffer>();
+ protected List<ByteBuffer> prevResultBuffers = new ArrayList<ByteBuffer>();
+ protected List<ByteBuffer> swap = null;
+ protected int maxResultBufIdx = 0;
- protected final ITreeIndexFrame leafFrame;
- protected final ITreeIndexFrame interiorFrame;
- protected final ITreeIndexCursor btreeCursor;
- protected final FrameTupleReference searchKey = new FrameTupleReference();
- protected final RangePredicate btreePred = new RangePredicate(true, null, null, true, true, null, null);
- protected final BTreeOpContext btreeOpCtx;
+ protected final ITreeIndexFrame leafFrame;
+ protected final ITreeIndexFrame interiorFrame;
+ protected final ITreeIndexCursor btreeCursor;
+ protected final FrameTupleReference searchKey = new FrameTupleReference();
+ protected final RangePredicate btreePred = new RangePredicate(true, null, null, true, true, null, null);
+ protected final BTreeOpContext btreeOpCtx;
- protected RecordDescriptor queryTokenRecDesc = new RecordDescriptor(
- new ISerializerDeserializer[] { UTF8StringSerializerDeserializer.INSTANCE });
- protected ArrayTupleBuilder queryTokenBuilder = new ArrayTupleBuilder(queryTokenRecDesc.getFields().length);
- protected DataOutput queryTokenDos = queryTokenBuilder.getDataOutput();
- protected FrameTupleAppender queryTokenAppender;
- protected ByteBuffer queryTokenFrame;
+ protected RecordDescriptor queryTokenRecDesc = new RecordDescriptor(
+ new ISerializerDeserializer[] { UTF8StringSerializerDeserializer.INSTANCE });
+ protected ArrayTupleBuilder queryTokenBuilder = new ArrayTupleBuilder(queryTokenRecDesc.getFields().length);
+ protected DataOutput queryTokenDos = queryTokenBuilder.getDataOutput();
+ protected FrameTupleAppender queryTokenAppender;
+ protected ByteBuffer queryTokenFrame;
- protected final InvertedIndex invIndex;
- protected final IBinaryTokenizer queryTokenizer;
- protected final ITypeTrait[] invListFieldsWithCount;
- protected int occurrenceThreshold;
+ protected final InvertedIndex invIndex;
+ protected final IBinaryTokenizer queryTokenizer;
+ protected final ITypeTrait[] invListFieldsWithCount;
+ protected int occurrenceThreshold;
- protected final int cursorCacheSize = 10;
- protected List<IInvertedListCursor> invListCursorCache = new ArrayList<IInvertedListCursor>(cursorCacheSize);
- protected List<IInvertedListCursor> invListCursors = new ArrayList<IInvertedListCursor>(cursorCacheSize);
+ protected final int cursorCacheSize = 10;
+ protected List<IInvertedListCursor> invListCursorCache = new ArrayList<IInvertedListCursor>(cursorCacheSize);
+ protected List<IInvertedListCursor> invListCursors = new ArrayList<IInvertedListCursor>(cursorCacheSize);
- public TOccurrenceSearcher(IHyracksStageletContext ctx, InvertedIndex invIndex, IBinaryTokenizer queryTokenizer) {
- this.ctx = ctx;
- this.invIndex = invIndex;
- this.queryTokenizer = queryTokenizer;
+ public TOccurrenceSearcher(IHyracksStageletContext ctx, InvertedIndex invIndex, IBinaryTokenizer queryTokenizer) {
+ this.ctx = ctx;
+ this.invIndex = invIndex;
+ this.queryTokenizer = queryTokenizer;
- leafFrame = invIndex.getBTree().getLeafFrameFactory().createFrame();
- interiorFrame = invIndex.getBTree().getInteriorFrameFactory().createFrame();
+ leafFrame = invIndex.getBTree().getLeafFrameFactory().createFrame();
+ interiorFrame = invIndex.getBTree().getInteriorFrameFactory().createFrame();
- btreeCursor = new BTreeRangeSearchCursor((IBTreeLeafFrame)leafFrame);
- ITypeTrait[] invListFields = invIndex.getInvListElementCmp().getTypeTraits();
- invListFieldsWithCount = new TypeTrait[invListFields.length + 1];
- int tmp = 0;
- for(int i = 0; i < invListFields.length; i++) {
- invListFieldsWithCount[i] = invListFields[i];
- tmp += invListFields[i].getStaticallyKnownDataLength();
- }
- // using an integer for counting occurrences
- invListFieldsWithCount[invListFields.length] = new TypeTrait(4);
- invListKeyLength = tmp;
+ btreeCursor = new BTreeRangeSearchCursor((IBTreeLeafFrame) leafFrame);
+ ITypeTrait[] invListFields = invIndex.getInvListElementCmp().getTypeTraits();
+ invListFieldsWithCount = new TypeTrait[invListFields.length + 1];
+ int tmp = 0;
+ for (int i = 0; i < invListFields.length; i++) {
+ invListFieldsWithCount[i] = invListFields[i];
+ tmp += invListFields[i].getStaticallyKnownDataLength();
+ }
+ // using an integer for counting occurrences
+ invListFieldsWithCount[invListFields.length] = new TypeTrait(4);
+ invListKeyLength = tmp;
- btreeOpCtx = invIndex.getBTree().createOpContext(IndexOp.SEARCH, leafFrame,
- interiorFrame, null);
+ btreeOpCtx = invIndex.getBTree().createOpContext(IndexOp.SEARCH, leafFrame, interiorFrame, null);
- resultFrameTupleApp = new FixedSizeFrameTupleAppender(ctx.getFrameSize(), invListFieldsWithCount);
- resultFrameTupleAcc = new FixedSizeFrameTupleAccessor(ctx.getFrameSize(), invListFieldsWithCount);
- resultTuple = new FixedSizeTupleReference(invListFieldsWithCount);
- newResultBuffers.add(ctx.allocateFrame());
- prevResultBuffers.add(ctx.allocateFrame());
+ resultFrameTupleApp = new FixedSizeFrameTupleAppender(ctx.getFrameSize(), invListFieldsWithCount);
+ resultFrameTupleAcc = new FixedSizeFrameTupleAccessor(ctx.getFrameSize(), invListFieldsWithCount);
+ resultTuple = new FixedSizeTupleReference(invListFieldsWithCount);
+ newResultBuffers.add(ctx.allocateFrame());
+ prevResultBuffers.add(ctx.allocateFrame());
- MultiComparator searchCmp = invIndex.getBTree().getMultiComparator();
- btreePred.setLowKeyComparator(searchCmp);
- btreePred.setHighKeyComparator(searchCmp);
- btreePred.setLowKey(searchKey, true);
- btreePred.setHighKey(searchKey, true);
+ MultiComparator searchCmp = invIndex.getBTree().getMultiComparator();
+ btreePred.setLowKeyComparator(searchCmp);
+ btreePred.setHighKeyComparator(searchCmp);
+ btreePred.setLowKey(searchKey, true);
+ btreePred.setHighKey(searchKey, true);
- // pre-create cursor objects
- for (int i = 0; i < cursorCacheSize; i++) {
- invListCursorCache.add(new FixedSizeElementInvertedListCursor(invIndex.getBufferCache(), invIndex
- .getInvListsFileId(), invIndex.getInvListElementCmp().getTypeTraits()));
- }
+ // pre-create cursor objects
+ for (int i = 0; i < cursorCacheSize; i++) {
+ invListCursorCache.add(new FixedSizeElementInvertedListCursor(invIndex.getBufferCache(), invIndex
+ .getInvListsFileId(), invIndex.getInvListElementCmp().getTypeTraits()));
+ }
- queryTokenAppender = new FrameTupleAppender(ctx.getFrameSize());
- queryTokenFrame = ctx.allocateFrame();
+ queryTokenAppender = new FrameTupleAppender(ctx.getFrameSize());
+ queryTokenFrame = ctx.allocateFrame();
- currentNumResults = 0;
- }
+ currentNumResults = 0;
+ }
- public void reset() {
- for(ByteBuffer b : newResultBuffers) {
- resultFrameTupleApp.reset(b, true);
- }
- for(ByteBuffer b : prevResultBuffers) {
- resultFrameTupleApp.reset(b, true);
- }
- currentNumResults = 0;
- }
+ public void reset() {
+ for (ByteBuffer b : newResultBuffers) {
+ resultFrameTupleApp.reset(b, true);
+ }
+ for (ByteBuffer b : prevResultBuffers) {
+ resultFrameTupleApp.reset(b, true);
+ }
+ currentNumResults = 0;
+ }
- public void search(IInvertedIndexResultCursor resultCursor, ITupleReference queryTuple, int queryFieldIndex, IInvertedIndexSearchModifier searchModifier) throws Exception {
+ public void search(IInvertedIndexResultCursor resultCursor, ITupleReference queryTuple, int queryFieldIndex,
+ IInvertedIndexSearchModifier searchModifier) throws Exception {
- queryTokenAppender.reset(queryTokenFrame, true);
- queryTokenizer.reset(queryTuple.getFieldData(queryFieldIndex), queryTuple.getFieldStart(queryFieldIndex),
- queryTuple.getFieldLength(queryFieldIndex));
- while (queryTokenizer.hasNext()) {
- queryTokenizer.next();
+ queryTokenAppender.reset(queryTokenFrame, true);
+ queryTokenizer.reset(queryTuple.getFieldData(queryFieldIndex), queryTuple.getFieldStart(queryFieldIndex),
+ queryTuple.getFieldLength(queryFieldIndex));
- queryTokenBuilder.reset();
- try {
- IToken token = queryTokenizer.getToken();
- token.serializeToken(queryTokenDos);
- queryTokenBuilder.addFieldEndOffset();
- // WARNING: assuming one frame is big enough to hold all tokens
- queryTokenAppender.append(queryTokenBuilder.getFieldEndOffsets(), queryTokenBuilder.getByteArray(), 0,
- queryTokenBuilder.getSize());
- } catch (IOException e) {
- throw new HyracksDataException(e);
- }
- }
+ while (queryTokenizer.hasNext()) {
+ queryTokenizer.next();
+ queryTokenBuilder.reset();
+ try {
+ IToken token = queryTokenizer.getToken();
+ token.serializeToken(queryTokenDos);
+ queryTokenBuilder.addFieldEndOffset();
+ // WARNING: assuming one frame is big enough to hold all tokens
+ queryTokenAppender.append(queryTokenBuilder.getFieldEndOffsets(), queryTokenBuilder.getByteArray(), 0,
+ queryTokenBuilder.getSize());
+ } catch (IOException e) {
+ throw new HyracksDataException(e);
+ }
+ }
- FrameTupleAccessor queryTokenAccessor = new FrameTupleAccessor(ctx.getFrameSize(), queryTokenRecDesc);
- queryTokenAccessor.reset(queryTokenFrame);
- int numQueryTokens = queryTokenAccessor.getTupleCount();
+ FrameTupleAccessor queryTokenAccessor = new FrameTupleAccessor(ctx.getFrameSize(), queryTokenRecDesc);
+ queryTokenAccessor.reset(queryTokenFrame);
+ int numQueryTokens = queryTokenAccessor.getTupleCount();
- // expand cursor cache if necessary
- if (numQueryTokens > invListCursorCache.size()) {
- int diff = numQueryTokens - invListCursorCache.size();
- for (int i = 0; i < diff; i++) {
- invListCursorCache.add(new FixedSizeElementInvertedListCursor(invIndex.getBufferCache(), invIndex
- .getInvListsFileId(), invIndex.getInvListElementCmp().getTypeTraits()));
- }
- }
+ // expand cursor cache if necessary
+ if (numQueryTokens > invListCursorCache.size()) {
+ int diff = numQueryTokens - invListCursorCache.size();
+ for (int i = 0; i < diff; i++) {
+ invListCursorCache.add(new FixedSizeElementInvertedListCursor(invIndex.getBufferCache(), invIndex
+ .getInvListsFileId(), invIndex.getInvListElementCmp().getTypeTraits()));
+ }
+ }
- invListCursors.clear();
- for (int i = 0; i < numQueryTokens; i++) {
- searchKey.reset(queryTokenAccessor, i);
- invIndex.openCursor(btreeCursor, btreePred, btreeOpCtx, invListCursorCache.get(i));
- invListCursors.add(invListCursorCache.get(i));
- }
-
- occurrenceThreshold = searchModifier.getOccurrenceThreshold(invListCursors);
-
- // TODO: deal with panic cases properly
- if(occurrenceThreshold <= 0) {
- throw new OccurrenceThresholdPanicException("Merge Threshold is <= 0. Failing Search.");
- }
+ invListCursors.clear();
+ for (int i = 0; i < numQueryTokens; i++) {
+ searchKey.reset(queryTokenAccessor, i);
+ invIndex.openCursor(btreeCursor, btreePred, btreeOpCtx, invListCursorCache.get(i));
+ invListCursors.add(invListCursorCache.get(i));
+ }
- int numPrefixLists = searchModifier.getPrefixLists(invListCursors);
- maxResultBufIdx = mergePrefixLists(numPrefixLists, numQueryTokens);
- maxResultBufIdx = mergeSuffixLists(numPrefixLists, numQueryTokens, maxResultBufIdx);
+ occurrenceThreshold = searchModifier.getOccurrenceThreshold(invListCursors);
- resultCursor.reset(this);
+ // TODO: deal with panic cases properly
+ if (occurrenceThreshold <= 0) {
+ throw new OccurrenceThresholdPanicException("Merge Threshold is <= 0. Failing Search.");
+ }
- //printNewResults(maxResultBufIdx);
- }
+ int numPrefixLists = searchModifier.getPrefixLists(invListCursors);
+ maxResultBufIdx = mergePrefixLists(numPrefixLists, numQueryTokens);
+ maxResultBufIdx = mergeSuffixLists(numPrefixLists, numQueryTokens, maxResultBufIdx);
- protected int mergePrefixLists(int numPrefixTokens, int numQueryTokens) throws IOException {
- int maxPrevBufIdx = 0;
- for(int i = 0; i < numPrefixTokens; i++) {
- swap = prevResultBuffers;
- prevResultBuffers = newResultBuffers;
- newResultBuffers = swap;
- currentNumResults = 0;
+ resultCursor.reset(this);
+ }
- invListCursors.get(i).pinPagesSync();
- maxPrevBufIdx = mergePrefixList(invListCursors.get(i), prevResultBuffers, maxPrevBufIdx, newResultBuffers);
- invListCursors.get(i).unpinPages();
+ protected int mergePrefixLists(int numPrefixTokens, int numQueryTokens) throws IOException {
+ int maxPrevBufIdx = 0;
+ for (int i = 0; i < numPrefixTokens; i++) {
+ swap = prevResultBuffers;
+ prevResultBuffers = newResultBuffers;
+ newResultBuffers = swap;
+ currentNumResults = 0;
- //printNewResults(maxPrevBufIdx);
- }
+ invListCursors.get(i).pinPagesSync();
+ maxPrevBufIdx = mergePrefixList(invListCursors.get(i), prevResultBuffers, maxPrevBufIdx, newResultBuffers);
+ invListCursors.get(i).unpinPages();
+ }
+ return maxPrevBufIdx;
+ }
- return maxPrevBufIdx;
- }
+ protected int mergeSuffixLists(int numPrefixTokens, int numQueryTokens, int maxPrevBufIdx) throws IOException {
+ for (int i = numPrefixTokens; i < numQueryTokens; i++) {
+ swap = prevResultBuffers;
+ prevResultBuffers = newResultBuffers;
+ newResultBuffers = swap;
- protected int mergeSuffixLists(int numPrefixTokens, int numQueryTokens, int maxPrevBufIdx) throws IOException {
- for(int i = numPrefixTokens; i < numQueryTokens; i++) {
- swap = prevResultBuffers;
- prevResultBuffers = newResultBuffers;
- newResultBuffers = swap;
+ invListCursors.get(i).pinPagesSync();
+ int numInvListElements = invListCursors.get(i).getNumElements();
+ // should we binary search the next list or should we sort-merge it?
+ if (currentNumResults * Math.log(numInvListElements) < currentNumResults + numInvListElements) {
+ maxPrevBufIdx = mergeSuffixListProbe(invListCursors.get(i), prevResultBuffers, maxPrevBufIdx,
+ newResultBuffers, i, numQueryTokens);
+ } else {
+ maxPrevBufIdx = mergeSuffixListScan(invListCursors.get(i), prevResultBuffers, maxPrevBufIdx,
+ newResultBuffers, i, numQueryTokens);
+ }
+ invListCursors.get(i).unpinPages();
+ }
+ return maxPrevBufIdx;
+ }
- invListCursors.get(i).pinPagesSync();
- int numInvListElements = invListCursors.get(i).getNumElements();
- // should we binary search the next list or should we sort-merge it?
- if(currentNumResults * Math.log(numInvListElements) < currentNumResults + numInvListElements) {
- //System.out.println("PROBING LIST: " + i);
- maxPrevBufIdx = mergeSuffixListProbe(invListCursors.get(i), prevResultBuffers, maxPrevBufIdx, newResultBuffers, i, numQueryTokens);
- }
- else {
- //System.out.println("SCANNING LIST: " + i);
- maxPrevBufIdx = mergeSuffixListScan(invListCursors.get(i), prevResultBuffers, maxPrevBufIdx, newResultBuffers, i, numQueryTokens);
- }
- invListCursors.get(i).unpinPages();
- }
- return maxPrevBufIdx;
- }
+ protected int mergeSuffixListProbe(IInvertedListCursor invListCursor, List<ByteBuffer> prevResultBuffers,
+ int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens) throws IOException {
- protected int mergeSuffixListProbe(IInvertedListCursor invListCursor, List<ByteBuffer> prevResultBuffers, int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens) throws IOException {
+ int newBufIdx = 0;
+ ByteBuffer newCurrentBuffer = newResultBuffers.get(0);
- int newBufIdx = 0;
- ByteBuffer newCurrentBuffer = newResultBuffers.get(0);
+ int prevBufIdx = 0;
+ ByteBuffer prevCurrentBuffer = prevResultBuffers.get(0);
- int prevBufIdx = 0;
- ByteBuffer prevCurrentBuffer = prevResultBuffers.get(0);
+ int resultTidx = 0;
- int resultTidx = 0;
+ currentNumResults = 0;
- currentNumResults = 0;
+ MultiComparator invListCmp = invIndex.getInvListElementCmp();
- MultiComparator invListCmp = invIndex.getInvListElementCmp();
+ resultFrameTupleAcc.reset(prevCurrentBuffer);
+ resultFrameTupleApp.reset(newCurrentBuffer, true);
- resultFrameTupleAcc.reset(prevCurrentBuffer);
- resultFrameTupleApp.reset(newCurrentBuffer, true);
+ while (resultTidx < resultFrameTupleAcc.getTupleCount()) {
- while(resultTidx < resultFrameTupleAcc.getTupleCount()) {
+ resultTuple.reset(prevCurrentBuffer.array(), resultFrameTupleAcc.getTupleStartOffset(resultTidx));
+ int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0),
+ resultTuple.getFieldStart(resultTuple.getFieldCount() - 1));
- resultTuple.reset(prevCurrentBuffer.array(), resultFrameTupleAcc.getTupleStartOffset(resultTidx));
- int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0), resultTuple.getFieldStart(resultTuple.getFieldCount()-1));
+ if (invListCursor.containsKey(resultTuple, invListCmp)) {
+ count++;
+ newBufIdx = appendTupleToNewResults(resultTuple, count, newBufIdx);
+ } else {
+ if (count + numQueryTokens - invListIx > occurrenceThreshold) {
+ newBufIdx = appendTupleToNewResults(resultTuple, count, newBufIdx);
+ }
+ }
- if(invListCursor.containsKey(resultTuple, invListCmp)) {
- count++;
- newBufIdx = appendTupleToNewResults(resultTuple, count, newBufIdx);
- }
- else {
- if(count + numQueryTokens - invListIx > occurrenceThreshold) {
- newBufIdx = appendTupleToNewResults(resultTuple, count, newBufIdx);
- }
- }
+ resultTidx++;
+ if (resultTidx >= resultFrameTupleAcc.getTupleCount()) {
+ prevBufIdx++;
+ if (prevBufIdx <= maxPrevBufIdx) {
+ prevCurrentBuffer = prevResultBuffers.get(prevBufIdx);
+ resultFrameTupleAcc.reset(prevCurrentBuffer);
+ resultTidx = 0;
+ }
+ }
+ }
- resultTidx++;
- if (resultTidx >= resultFrameTupleAcc.getTupleCount()) {
- prevBufIdx++;
- if (prevBufIdx <= maxPrevBufIdx) {
- prevCurrentBuffer = prevResultBuffers.get(prevBufIdx);
- resultFrameTupleAcc.reset(prevCurrentBuffer);
- resultTidx = 0;
- }
- }
- }
+ return newBufIdx;
+ }
- return newBufIdx;
- }
-
- protected int mergeSuffixListScan(IInvertedListCursor invListCursor, List<ByteBuffer> prevResultBuffers, int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens) throws IOException {
- int newBufIdx = 0;
+ protected int mergeSuffixListScan(IInvertedListCursor invListCursor, List<ByteBuffer> prevResultBuffers,
+ int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens) throws IOException {
+ int newBufIdx = 0;
ByteBuffer newCurrentBuffer = newResultBuffers.get(0);
int prevBufIdx = 0;
@@ -296,7 +295,7 @@
boolean advanceCursor = true;
boolean advancePrevResult = false;
- int resultTidx = 0;
+ int resultTidx = 0;
MultiComparator invListCmp = invIndex.getInvListElementCmp();
@@ -304,31 +303,34 @@
resultFrameTupleApp.reset(newCurrentBuffer, true);
int invListTidx = 0;
- int invListNumTuples = invListCursor.getNumElements();
+ int invListNumTuples = invListCursor.getNumElements();
- if(invListCursor.hasNext()) invListCursor.next();
+ if (invListCursor.hasNext())
+ invListCursor.next();
- while(invListTidx < invListNumTuples && resultTidx < resultFrameTupleAcc.getTupleCount()) {
-
+ while (invListTidx < invListNumTuples && resultTidx < resultFrameTupleAcc.getTupleCount()) {
+
ITupleReference invListTuple = invListCursor.getTuple();
- resultTuple.reset(prevCurrentBuffer.array(), resultFrameTupleAcc.getTupleStartOffset(resultTidx));
+ resultTuple.reset(prevCurrentBuffer.array(), resultFrameTupleAcc.getTupleStartOffset(resultTidx));
int cmp = invListCmp.compare(invListTuple, resultTuple);
if (cmp == 0) {
- int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0), resultTuple.getFieldStart(resultTuple.getFieldCount()-1)) + 1;
- newBufIdx = appendTupleToNewResults(resultTuple, count, newBufIdx);
+ int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0),
+ resultTuple.getFieldStart(resultTuple.getFieldCount() - 1)) + 1;
+ newBufIdx = appendTupleToNewResults(resultTuple, count, newBufIdx);
advanceCursor = true;
advancePrevResult = true;
} else {
- if (cmp < 0) {
+ if (cmp < 0) {
advanceCursor = true;
advancePrevResult = false;
} else {
- int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0), resultTuple.getFieldStart(resultTuple.getFieldCount()-1));
- if(count + numQueryTokens - invListIx > occurrenceThreshold) {
- newBufIdx = appendTupleToNewResults(resultTuple, count, newBufIdx);
- }
+ int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0),
+ resultTuple.getFieldStart(resultTuple.getFieldCount() - 1));
+ if (count + numQueryTokens - invListIx > occurrenceThreshold) {
+ newBufIdx = appendTupleToNewResults(resultTuple, count, newBufIdx);
+ }
advanceCursor = false;
advancePrevResult = true;
}
@@ -344,23 +346,24 @@
resultTidx = 0;
}
}
- }
-
- if(advanceCursor) {
- invListTidx++;
- invListCursor.next();
- }
+ }
+
+ if (advanceCursor) {
+ invListTidx++;
+ invListCursor.next();
+ }
}
-
+
// append remaining elements from previous result set
- while(resultTidx < resultFrameTupleAcc.getTupleCount()) {
-
- resultTuple.reset(prevCurrentBuffer.array(), resultFrameTupleAcc.getTupleStartOffset(resultTidx));
-
- int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0), resultTuple.getFieldStart(resultTuple.getFieldCount()-1));
- if(count + numQueryTokens - invListIx > occurrenceThreshold) {
- newBufIdx = appendTupleToNewResults(resultTuple, count, newBufIdx);
- }
+ while (resultTidx < resultFrameTupleAcc.getTupleCount()) {
+
+ resultTuple.reset(prevCurrentBuffer.array(), resultFrameTupleAcc.getTupleStartOffset(resultTidx));
+
+ int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0),
+ resultTuple.getFieldStart(resultTuple.getFieldCount() - 1));
+ if (count + numQueryTokens - invListIx > occurrenceThreshold) {
+ newBufIdx = appendTupleToNewResults(resultTuple, count, newBufIdx);
+ }
resultTidx++;
if (resultTidx >= resultFrameTupleAcc.getTupleCount()) {
@@ -374,9 +377,10 @@
}
return newBufIdx;
- }
-
- protected int mergePrefixList(IInvertedListCursor invListCursor, List<ByteBuffer> prevResultBuffers, int maxPrevBufIdx, List<ByteBuffer> newResultBuffers) throws IOException {
+ }
+
+ protected int mergePrefixList(IInvertedListCursor invListCursor, List<ByteBuffer> prevResultBuffers,
+ int maxPrevBufIdx, List<ByteBuffer> newResultBuffers) throws IOException {
int newBufIdx = 0;
ByteBuffer newCurrentBuffer = newResultBuffers.get(0);
@@ -385,7 +389,7 @@
boolean advanceCursor = true;
boolean advancePrevResult = false;
- int resultTidx = 0;
+ int resultTidx = 0;
MultiComparator invListCmp = invIndex.getInvListElementCmp();
@@ -393,20 +397,21 @@
resultFrameTupleApp.reset(newCurrentBuffer, true);
int invListTidx = 0;
- int invListNumTuples = invListCursor.getNumElements();
+ int invListNumTuples = invListCursor.getNumElements();
- if(invListCursor.hasNext()) invListCursor.next();
+ if (invListCursor.hasNext())
+ invListCursor.next();
- while(invListTidx < invListNumTuples && resultTidx < resultFrameTupleAcc.getTupleCount()) {
-
+ while (invListTidx < invListNumTuples && resultTidx < resultFrameTupleAcc.getTupleCount()) {
+
ITupleReference invListTuple = invListCursor.getTuple();
-
- resultTuple.reset(prevCurrentBuffer.array(), resultFrameTupleAcc.getTupleStartOffset(resultTidx));
+ resultTuple.reset(prevCurrentBuffer.array(), resultFrameTupleAcc.getTupleStartOffset(resultTidx));
int cmp = invListCmp.compare(invListTuple, resultTuple);
if (cmp == 0) {
- int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0), resultTuple.getFieldStart(resultTuple.getFieldCount()-1)) + 1;
- newBufIdx = appendTupleToNewResults(resultTuple, count, newBufIdx);
+ int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0),
+ resultTuple.getFieldStart(resultTuple.getFieldCount() - 1)) + 1;
+ newBufIdx = appendTupleToNewResults(resultTuple, count, newBufIdx);
advanceCursor = true;
advancePrevResult = true;
} else {
@@ -416,7 +421,8 @@
advanceCursor = true;
advancePrevResult = false;
} else {
- int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0), resultTuple.getFieldStart(resultTuple.getFieldCount()-1));
+ int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0),
+ resultTuple.getFieldStart(resultTuple.getFieldCount() - 1));
newBufIdx = appendTupleToNewResults(resultTuple, count, newBufIdx);
advanceCursor = false;
advancePrevResult = true;
@@ -433,28 +439,29 @@
resultTidx = 0;
}
}
- }
-
- if(advanceCursor) {
- invListTidx++;
- invListCursor.next();
- }
+ }
+
+ if (advanceCursor) {
+ invListTidx++;
+ invListCursor.next();
+ }
}
- // append remaining new elements from inverted list
- while(invListTidx < invListNumTuples) {
+ // append remaining new elements from inverted list
+ while (invListTidx < invListNumTuples) {
ITupleReference invListTuple = invListCursor.getTuple();
newBufIdx = appendTupleToNewResults(invListTuple, 1, newBufIdx);
invListTidx++;
- invListCursor.next();
+ invListCursor.next();
}
// append remaining elements from previous result set
- while(resultTidx < resultFrameTupleAcc.getTupleCount()) {
-
- resultTuple.reset(prevCurrentBuffer.array(), resultFrameTupleAcc.getTupleStartOffset(resultTidx));
+ while (resultTidx < resultFrameTupleAcc.getTupleCount()) {
- int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0), resultTuple.getFieldStart(resultTuple.getFieldCount()-1));
+ resultTuple.reset(prevCurrentBuffer.array(), resultFrameTupleAcc.getTupleStartOffset(resultTidx));
+
+ int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0),
+ resultTuple.getFieldStart(resultTuple.getFieldCount() - 1));
newBufIdx = appendTupleToNewResults(resultTuple, count, newBufIdx);
resultTidx++;
@@ -468,236 +475,72 @@
}
}
- return newBufIdx;
+ return newBufIdx;
}
- protected int appendTupleToNewResults(ITupleReference tuple, int newCount, int newBufIdx) throws IOException {
- ByteBuffer newCurrentBuffer = newResultBuffers.get(newBufIdx);
+ protected int appendTupleToNewResults(ITupleReference tuple, int newCount, int newBufIdx) throws IOException {
+ ByteBuffer newCurrentBuffer = newResultBuffers.get(newBufIdx);
- if (!resultFrameTupleApp.hasSpace()) {
- newBufIdx++;
- if (newBufIdx >= newResultBuffers.size()) {
- newResultBuffers.add(ctx.allocateFrame());
- }
- newCurrentBuffer = newResultBuffers.get(newBufIdx);
- resultFrameTupleApp.reset(newCurrentBuffer, true);
- }
+ if (!resultFrameTupleApp.hasSpace()) {
+ newBufIdx++;
+ if (newBufIdx >= newResultBuffers.size()) {
+ newResultBuffers.add(ctx.allocateFrame());
+ }
+ newCurrentBuffer = newResultBuffers.get(newBufIdx);
+ resultFrameTupleApp.reset(newCurrentBuffer, true);
+ }
- // append key
- if (!resultFrameTupleApp.append(tuple.getFieldData(0), tuple.getFieldStart(0), invListKeyLength) ) {
- throw new IllegalStateException();
- }
+ // append key
+ if (!resultFrameTupleApp.append(tuple.getFieldData(0), tuple.getFieldStart(0), invListKeyLength)) {
+ throw new IllegalStateException();
+ }
- // append new count
- if (!resultFrameTupleApp.append(newCount) ) {
- throw new IllegalStateException();
- }
+ // append new count
+ if (!resultFrameTupleApp.append(newCount)) {
+ throw new IllegalStateException();
+ }
- resultFrameTupleApp.incrementTupleCount(1);
+ resultFrameTupleApp.incrementTupleCount(1);
- currentNumResults++;
+ currentNumResults++;
- return newBufIdx;
- }
+ return newBufIdx;
+ }
- public IFrameTupleAccessor createResultFrameTupleAccessor() {
- return new FixedSizeFrameTupleAccessor(ctx.getFrameSize(), invListFieldsWithCount);
- }
+ public IFrameTupleAccessor createResultFrameTupleAccessor() {
+ return new FixedSizeFrameTupleAccessor(ctx.getFrameSize(), invListFieldsWithCount);
+ }
- public ITupleReference createResultTupleReference() {
- return new FixedSizeTupleReference(invListFieldsWithCount);
- }
+ public ITupleReference createResultTupleReference() {
+ return new FixedSizeTupleReference(invListFieldsWithCount);
+ }
- @Override
- public List<ByteBuffer> getResultBuffers() {
- return newResultBuffers;
- }
+ @Override
+ public List<ByteBuffer> getResultBuffers() {
+ return newResultBuffers;
+ }
- @Override
- public int getNumValidResultBuffers() {
- return maxResultBufIdx + 1;
- }
+ @Override
+ public int getNumValidResultBuffers() {
+ return maxResultBufIdx + 1;
+ }
- public int getOccurrenceThreshold() {
- return occurrenceThreshold;
- }
+ public int getOccurrenceThreshold() {
+ return occurrenceThreshold;
+ }
- public void printNewResults(int maxResultBufIdx) {
- StringBuffer strBuffer = new StringBuffer();
- for(int i = 0; i <= maxResultBufIdx; i++) {
- ByteBuffer testBuf = newResultBuffers.get(i);
- resultFrameTupleAcc.reset(testBuf);
- for(int j = 0; j < resultFrameTupleAcc.getTupleCount(); j++) {
- strBuffer.append(IntegerSerializerDeserializer.getInt(resultFrameTupleAcc.getBuffer().array(), resultFrameTupleAcc.getFieldStartOffset(j, 0)) + ",");
- strBuffer.append(IntegerSerializerDeserializer.getInt(resultFrameTupleAcc.getBuffer().array(), resultFrameTupleAcc.getFieldStartOffset(j, 1)) + " ");
- }
- }
- System.out.println(strBuffer.toString());
- }
-
-
-
- // older slower code
- /*
- protected int mergeSuffixListScan(IInvertedListCursor invListCursor, List<ByteBuffer> prevResultBuffers, int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens) throws IOException {
-
- int newBufIdx = 0;
- ByteBuffer newCurrentBuffer = newResultBuffers.get(0);
-
- int prevBufIdx = 0;
- ByteBuffer prevCurrentBuffer = prevResultBuffers.get(0);
-
- boolean advanceCursor = true;
- boolean advancePrevResult = false;
- int resultTidx = 0;
-
- MultiComparator invListCmp = invIndex.getInvListElementCmp();
-
- resultFrameTupleAcc.reset(prevCurrentBuffer);
- resultFrameTupleApp.reset(newCurrentBuffer, true);
-
- ITupleReference invListTuple = null;
-
- int invListTidx = 0;
- int invListNumTuples = invListCursor.getNumElements();
-
- if(invListCursor.hasNext()) invListCursor.next();
-
- while(invListTidx < invListNumTuples || resultTidx < resultFrameTupleAcc.getTupleCount()) {
-
- invListTuple = null;
-
- int cmp = 0;
- if(invListTidx >= invListNumTuples) {
- resultTuple.reset(prevCurrentBuffer.array(), resultFrameTupleAcc.getTupleStartOffset(resultTidx));
- cmp = 1;
- } else if(resultTidx >= resultFrameTupleAcc.getTupleCount()) {
- invListTuple = invListCursor.getTuple();
- cmp = -1;
- } else {
- invListTuple = invListCursor.getTuple();
- resultTuple.reset(prevCurrentBuffer.array(), resultFrameTupleAcc.getTupleStartOffset(resultTidx));
- cmp = invListCmp.compare(invListTuple, resultTuple);
- }
-
- if (cmp == 0) {
- int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0), resultTuple.getFieldStart(resultTuple.getFieldCount()-1)) + 1;
- newBufIdx = appendTupleToNewResults(resultTuple, count, newBufIdx);
- advanceCursor = true;
- advancePrevResult = true;
- } else {
- if (cmp < 0) {
- advanceCursor = true;
- advancePrevResult = false;
- } else {
- int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0), resultTuple.getFieldStart(resultTuple.getFieldCount()-1));
- if(count + numQueryTokens - invListIx > occurrenceThreshold) {
- newBufIdx = appendTupleToNewResults(resultTuple, count, newBufIdx);
- }
- advanceCursor = false;
- advancePrevResult = true;
- }
- }
-
- if (advancePrevResult) {
- resultTidx++;
- if (resultTidx >= resultFrameTupleAcc.getTupleCount()) {
- prevBufIdx++;
- if (prevBufIdx <= maxPrevBufIdx) {
- prevCurrentBuffer = prevResultBuffers.get(prevBufIdx);
- resultFrameTupleAcc.reset(prevCurrentBuffer);
- resultTidx = 0;
- }
- }
- }
-
- if(advanceCursor) {
- invListTidx++;
- invListCursor.next();
- }
- }
-
- return newBufIdx;
- }
-
- protected int mergePrefixList(IInvertedListCursor invListCursor, List<ByteBuffer> prevResultBuffers, int maxPrevBufIdx, List<ByteBuffer> newResultBuffers) throws IOException {
- int newBufIdx = 0;
- ByteBuffer newCurrentBuffer = newResultBuffers.get(0);
-
- int prevBufIdx = 0;
- ByteBuffer prevCurrentBuffer = prevResultBuffers.get(0);
-
- boolean advanceCursor = true;
- boolean advancePrevResult = false;
- int resultTidx = 0;
-
- MultiComparator invListCmp = invIndex.getInvListElementCmp();
-
- resultFrameTupleAcc.reset(prevCurrentBuffer);
- resultFrameTupleApp.reset(newCurrentBuffer, true);
-
- ITupleReference invListTuple = null;
-
- int invListTidx = 0;
- int invListNumTuples = invListCursor.getNumElements();
-
- if(invListCursor.hasNext()) invListCursor.next();
-
- while(invListTidx < invListNumTuples || resultTidx < resultFrameTupleAcc.getTupleCount()) {
-
- invListTuple = null;
-
- int cmp = 0;
- if(invListTidx >= invListNumTuples) {
- resultTuple.reset(prevCurrentBuffer.array(), resultFrameTupleAcc.getTupleStartOffset(resultTidx));
- cmp = 1;
- } else if(resultTidx >= resultFrameTupleAcc.getTupleCount()) {
- invListTuple = invListCursor.getTuple();
- cmp = -1;
- } else {
- invListTuple = invListCursor.getTuple();
- resultTuple.reset(prevCurrentBuffer.array(), resultFrameTupleAcc.getTupleStartOffset(resultTidx));
- cmp = invListCmp.compare(invListTuple, resultTuple);
- }
-
- if (cmp == 0) {
- int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0), resultTuple.getFieldStart(resultTuple.getFieldCount()-1)) + 1;
- newBufIdx = appendTupleToNewResults(resultTuple, count, newBufIdx);
- advanceCursor = true;
- advancePrevResult = true;
- } else {
- if (cmp < 0) {
- int count = 1;
- newBufIdx = appendTupleToNewResults(invListTuple, count, newBufIdx);
- advanceCursor = true;
- advancePrevResult = false;
- } else {
- int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0), resultTuple.getFieldStart(resultTuple.getFieldCount()-1));
- newBufIdx = appendTupleToNewResults(resultTuple, count, newBufIdx);
- advanceCursor = false;
- advancePrevResult = true;
- }
- }
-
- if (advancePrevResult) {
- resultTidx++;
- if (resultTidx >= resultFrameTupleAcc.getTupleCount()) {
- prevBufIdx++;
- if (prevBufIdx <= maxPrevBufIdx) {
- prevCurrentBuffer = prevResultBuffers.get(prevBufIdx);
- resultFrameTupleAcc.reset(prevCurrentBuffer);
- resultTidx = 0;
- }
- }
- }
-
- if(advanceCursor) {
- invListTidx++;
- invListCursor.next();
- }
- }
-
- return newBufIdx;
- }
-*/
+ public void printNewResults(int maxResultBufIdx) {
+ StringBuffer strBuffer = new StringBuffer();
+ for (int i = 0; i <= maxResultBufIdx; i++) {
+ ByteBuffer testBuf = newResultBuffers.get(i);
+ resultFrameTupleAcc.reset(testBuf);
+ for (int j = 0; j < resultFrameTupleAcc.getTupleCount(); j++) {
+ strBuffer.append(IntegerSerializerDeserializer.getInt(resultFrameTupleAcc.getBuffer().array(),
+ resultFrameTupleAcc.getFieldStartOffset(j, 0)) + ",");
+ strBuffer.append(IntegerSerializerDeserializer.getInt(resultFrameTupleAcc.getBuffer().array(),
+ resultFrameTupleAcc.getFieldStartOffset(j, 1)) + " ");
+ }
+ }
+ System.out.println(strBuffer.toString());
+ }
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixProbeOnly.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixProbeOnly.java
index 4a4e91c..18b870b 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixProbeOnly.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixProbeOnly.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.invertedindex.impls;
import java.io.IOException;
@@ -16,50 +31,52 @@
IBinaryTokenizer queryTokenizer) {
super(ctx, invIndex, queryTokenizer);
}
-
+
protected int mergeSuffixLists(int numPrefixTokens, int numQueryTokens, int maxPrevBufIdx) throws IOException {
- for(int i = numPrefixTokens; i < numQueryTokens; i++) {
+ for (int i = numPrefixTokens; i < numQueryTokens; i++) {
swap = prevResultBuffers;
prevResultBuffers = newResultBuffers;
newResultBuffers = swap;
currentNumResults = 0;
-
+
invListCursors.get(i).pinPagesSync();
- maxPrevBufIdx = mergeSuffixListProbe(invListCursors.get(i), prevResultBuffers, maxPrevBufIdx, newResultBuffers, i, numQueryTokens);
- invListCursors.get(i).unpinPages();
- }
- return maxPrevBufIdx;
+ maxPrevBufIdx = mergeSuffixListProbe(invListCursors.get(i), prevResultBuffers, maxPrevBufIdx,
+ newResultBuffers, i, numQueryTokens);
+ invListCursors.get(i).unpinPages();
+ }
+ return maxPrevBufIdx;
}
-
- protected int mergeSuffixListProbe(IInvertedListCursor invListCursor, List<ByteBuffer> prevResultBuffers, int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens) throws IOException {
-
+
+ protected int mergeSuffixListProbe(IInvertedListCursor invListCursor, List<ByteBuffer> prevResultBuffers,
+ int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens) throws IOException {
+
int newBufIdx = 0;
ByteBuffer newCurrentBuffer = newResultBuffers.get(0);
int prevBufIdx = 0;
ByteBuffer prevCurrentBuffer = prevResultBuffers.get(0);
-
- int resultTidx = 0;
-
+
+ int resultTidx = 0;
+
MultiComparator invListCmp = invIndex.getInvListElementCmp();
-
+
resultFrameTupleAcc.reset(prevCurrentBuffer);
resultFrameTupleApp.reset(newCurrentBuffer, true);
-
- while(resultTidx < resultFrameTupleAcc.getTupleCount()) {
-
- resultTuple.reset(prevCurrentBuffer.array(), resultFrameTupleAcc.getTupleStartOffset(resultTidx));
- int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0), resultTuple.getFieldStart(resultTuple.getFieldCount()-1));
- if(invListCursor.containsKey(resultTuple, invListCmp)) {
+ while (resultTidx < resultFrameTupleAcc.getTupleCount()) {
+
+ resultTuple.reset(prevCurrentBuffer.array(), resultFrameTupleAcc.getTupleStartOffset(resultTidx));
+ int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0),
+ resultTuple.getFieldStart(resultTuple.getFieldCount() - 1));
+
+ if (invListCursor.containsKey(resultTuple, invListCmp)) {
count++;
newBufIdx = appendTupleToNewResults(resultTuple, count, newBufIdx);
- }
- else {
- if(count + numQueryTokens - invListIx > occurrenceThreshold) {
+ } else {
+ if (count + numQueryTokens - invListIx > occurrenceThreshold) {
newBufIdx = appendTupleToNewResults(resultTuple, count, newBufIdx);
}
- }
+ }
resultTidx++;
if (resultTidx >= resultFrameTupleAcc.getTupleCount()) {
@@ -71,7 +88,7 @@
}
}
}
-
+
return newBufIdx;
}
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixScanOnly.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixScanOnly.java
index 8db1b91..604c68d 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixScanOnly.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/impls/TOccurrenceSearcherSuffixScanOnly.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.invertedindex.impls;
import java.io.IOException;
@@ -17,59 +32,64 @@
IBinaryTokenizer queryTokenizer) {
super(ctx, invIndex, queryTokenizer);
}
-
+
protected int mergeSuffixLists(int numPrefixTokens, int numQueryTokens, int maxPrevBufIdx) throws IOException {
- for(int i = numPrefixTokens; i < numQueryTokens; i++) {
+ for (int i = numPrefixTokens; i < numQueryTokens; i++) {
swap = prevResultBuffers;
prevResultBuffers = newResultBuffers;
newResultBuffers = swap;
currentNumResults = 0;
-
+
invListCursors.get(i).pinPagesSync();
- maxPrevBufIdx = mergeSuffixListScan(invListCursors.get(i), prevResultBuffers, maxPrevBufIdx, newResultBuffers, i, numQueryTokens);
- invListCursors.get(i).unpinPages();
- }
- return maxPrevBufIdx;
+ maxPrevBufIdx = mergeSuffixListScan(invListCursors.get(i), prevResultBuffers, maxPrevBufIdx,
+ newResultBuffers, i, numQueryTokens);
+ invListCursors.get(i).unpinPages();
+ }
+ return maxPrevBufIdx;
}
-
- protected int mergeSuffixListScan(IInvertedListCursor invListCursor, List<ByteBuffer> prevResultBuffers, int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens) throws IOException {
-
+
+ protected int mergeSuffixListScan(IInvertedListCursor invListCursor, List<ByteBuffer> prevResultBuffers,
+ int maxPrevBufIdx, List<ByteBuffer> newResultBuffers, int invListIx, int numQueryTokens) throws IOException {
+
int newBufIdx = 0;
ByteBuffer newCurrentBuffer = newResultBuffers.get(0);
int prevBufIdx = 0;
ByteBuffer prevCurrentBuffer = prevResultBuffers.get(0);
-
+
boolean advanceCursor = true;
boolean advancePrevResult = false;
- int resultTidx = 0;
-
+ int resultTidx = 0;
+
MultiComparator invListCmp = invIndex.getInvListElementCmp();
-
+
resultFrameTupleAcc.reset(prevCurrentBuffer);
resultFrameTupleApp.reset(newCurrentBuffer, true);
-
- while(invListCursor.hasNext() && resultTidx < resultFrameTupleAcc.getTupleCount()) {
-
- if(advanceCursor) invListCursor.next();
-
+
+ while (invListCursor.hasNext() && resultTidx < resultFrameTupleAcc.getTupleCount()) {
+
+ if (advanceCursor)
+ invListCursor.next();
+
ITupleReference invListTuple = invListCursor.getTuple();
-
- resultTuple.reset(prevCurrentBuffer.array(), resultFrameTupleAcc.getTupleStartOffset(resultTidx));
-
+
+ resultTuple.reset(prevCurrentBuffer.array(), resultFrameTupleAcc.getTupleStartOffset(resultTidx));
+
int cmp = invListCmp.compare(invListTuple, resultTuple);
if (cmp == 0) {
- int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0), resultTuple.getFieldStart(resultTuple.getFieldCount()-1)) + 1;
- newBufIdx = appendTupleToNewResults(resultTuple, count, newBufIdx);
+ int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0),
+ resultTuple.getFieldStart(resultTuple.getFieldCount() - 1)) + 1;
+ newBufIdx = appendTupleToNewResults(resultTuple, count, newBufIdx);
advanceCursor = true;
advancePrevResult = true;
} else {
- if (cmp < 0) {
+ if (cmp < 0) {
advanceCursor = true;
advancePrevResult = false;
} else {
- int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0), resultTuple.getFieldStart(resultTuple.getFieldCount()-1));
- if(count + numQueryTokens - invListIx > occurrenceThreshold) {
+ int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0),
+ resultTuple.getFieldStart(resultTuple.getFieldCount() - 1));
+ if (count + numQueryTokens - invListIx > occurrenceThreshold) {
newBufIdx = appendTupleToNewResults(resultTuple, count, newBufIdx);
}
advanceCursor = false;
@@ -87,16 +107,16 @@
resultTidx = 0;
}
}
- }
+ }
}
-
+
// append remaining elements from previous result set
- //if(resultTidx < resultFrameTupleAcc.getTupleCount()) System.out.println("APPENDING FROM RESULTS");
- while(resultTidx < resultFrameTupleAcc.getTupleCount()) {
-
- int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0), resultTuple.getFieldStart(resultTuple.getFieldCount()-1));
+ while (resultTidx < resultFrameTupleAcc.getTupleCount()) {
+
+ int count = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0),
+ resultTuple.getFieldStart(resultTuple.getFieldCount() - 1));
newBufIdx = appendTupleToNewResults(resultTuple, count, newBufIdx);
-
+
resultTidx++;
if (resultTidx >= resultFrameTupleAcc.getTupleCount()) {
prevBufIdx++;
@@ -107,7 +127,7 @@
}
}
}
-
+
return newBufIdx;
}
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/ConjunctiveSearchModifier.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/ConjunctiveSearchModifier.java
index 1f1d32c..55945be 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/ConjunctiveSearchModifier.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/ConjunctiveSearchModifier.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.invertedindex.searchmodifiers;
import java.util.Collections;
@@ -12,7 +27,7 @@
public int getOccurrenceThreshold(List<IInvertedListCursor> invListCursors) {
return invListCursors.size();
}
-
+
@Override
public int getPrefixLists(List<IInvertedListCursor> invListCursors) {
Collections.sort(invListCursors);
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/EditDistanceSearchModifier.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/EditDistanceSearchModifier.java
index 19481af..ac109b6 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/EditDistanceSearchModifier.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/EditDistanceSearchModifier.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.invertedindex.searchmodifiers;
import java.util.Collections;
@@ -10,20 +25,20 @@
private int gramLength;
private int edThresh;
-
+
public EditDistanceSearchModifier(int gramLength, int edThresh) {
this.gramLength = gramLength;
this.edThresh = edThresh;
}
-
+
@Override
- public int getOccurrenceThreshold(List<IInvertedListCursor> invListCursors) {
+ public int getOccurrenceThreshold(List<IInvertedListCursor> invListCursors) {
return invListCursors.size() - edThresh * gramLength;
}
@Override
public int getPrefixLists(List<IInvertedListCursor> invListCursors) {
- Collections.sort(invListCursors);
+ Collections.sort(invListCursors);
return invListCursors.size() - getOccurrenceThreshold(invListCursors) + 1;
}
@@ -41,5 +56,5 @@
public void setEdThresh(int edThresh) {
this.edThresh = edThresh;
- }
+ }
}
diff --git a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/JaccardSearchModifier.java b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/JaccardSearchModifier.java
index 72298e3..f42841c 100644
--- a/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/JaccardSearchModifier.java
+++ b/hyracks-storage-am-invertedindex/src/main/java/edu/uci/ics/hyracks/storage/am/invertedindex/searchmodifiers/JaccardSearchModifier.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.invertedindex.searchmodifiers;
import java.util.Collections;
@@ -7,16 +22,16 @@
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifier;
public class JaccardSearchModifier implements IInvertedIndexSearchModifier {
-
+
private float jaccThresh;
-
+
public JaccardSearchModifier(float jaccThresh) {
this.jaccThresh = jaccThresh;
}
-
+
@Override
public int getOccurrenceThreshold(List<IInvertedListCursor> invListCursors) {
- return (int) Math.floor((float) invListCursors.size() * jaccThresh);
+ return (int) Math.floor((float) invListCursors.size() * jaccThresh);
}
@Override
diff --git a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java
index e485a70..3623f04 100644
--- a/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java
+++ b/hyracks-storage-am-rtree/src/main/java/edu/uci/ics/hyracks/storage/am/rtree/frames/RTreeNSMFrame.java
@@ -154,12 +154,7 @@
}
}
}
- }
-
- @Override
- public void insertSorted(ITupleReference tuple, MultiComparator cmp) throws HyracksDataException {
-
- }
+ }
@Override
public ITreeIndexTupleReference createTupleReference() {
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/AbstractBTreeTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/AbstractBTreeTest.java
index c2e2fd6..56ae6e9 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/AbstractBTreeTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/AbstractBTreeTest.java
@@ -3,11 +3,14 @@
import java.io.File;
import java.text.SimpleDateFormat;
import java.util.Date;
+import java.util.logging.Logger;
import org.junit.AfterClass;
public abstract class AbstractBTreeTest {
+ protected static final Logger LOGGER = Logger.getLogger(AbstractBTreeTest.class.getName());
+
protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
protected final static String tmpDir = System.getProperty("java.io.tmpdir");
protected final static String sep = System.getProperty("file.separator");
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeFieldPrefixNSMTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeFieldPrefixNSMTest.java
index cfbc622..2b2bf55 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeFieldPrefixNSMTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeFieldPrefixNSMTest.java
@@ -40,7 +40,7 @@
import edu.uci.ics.hyracks.dataflow.common.data.comparators.IntegerBinaryComparatorFactory;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.btree.api.IPrefixSlotManager;
-import edu.uci.ics.hyracks.storage.am.btree.frames.FieldPrefixNSMLeafFrame;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeFieldPrefixNSMLeafFrame;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeException;
import edu.uci.ics.hyracks.storage.am.btree.impls.FieldPrefixSlotManager;
import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexTupleWriter;
@@ -64,7 +64,7 @@
private ITupleReference createTuple(IHyracksStageletContext ctx, int f0,
int f1, int f2, boolean print) throws HyracksDataException {
if (print)
- System.out.println("CREATING: " + f0 + " " + f1 + " " + f2);
+ LOGGER.info("CREATING: " + f0 + " " + f1 + " " + f2);
ByteBuffer buf = ctx.allocateFrame();
FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
@@ -144,7 +144,7 @@
IPrefixSlotManager slotManager = new FieldPrefixSlotManager();
ITreeIndexTupleWriter tupleWriter = new TypeAwareTupleWriter(typeTraits);
- FieldPrefixNSMLeafFrame frame = new FieldPrefixNSMLeafFrame(
+ BTreeFieldPrefixNSMLeafFrame frame = new BTreeFieldPrefixNSMLeafFrame(
tupleWriter);
frame.setPage(page);
frame.initBuffer((byte) 0);
@@ -165,7 +165,7 @@
for (int i = 0; i < numRecords; i++) {
if ((i + 1) % 100 == 0)
- print("INSERTING " + (i + 1) + " / " + numRecords + "\n");
+ LOGGER.info("INSERTING " + (i + 1) + " / " + numRecords);
int a = rnd.nextInt() % smallMax;
int b = rnd.nextInt() % smallMax;
@@ -205,7 +205,7 @@
for (int i = 0; i < numRecords; i++) {
if ((i + 1) % 100 == 0)
- print("DELETING " + (i + 1) + " / " + numRecords + "\n");
+ LOGGER.info("DELETING " + (i + 1) + " / " + numRecords);
ITupleReference tuple = createTuple(ctx,
savedFields[i][0], savedFields[i][1],
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java
index 9fad24c..44cf18b 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeStatsTest.java
@@ -23,8 +23,8 @@
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeOpContext;
import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
@@ -49,7 +49,7 @@
// private static final int PAGE_SIZE = 256;
// private static final int NUM_PAGES = 10;
- //private static final int PAGE_SIZE = 32768;
+ // private static final int PAGE_SIZE = 32768;
private static final int PAGE_SIZE = 4096;
private static final int NUM_PAGES = 1000;
private static final int MAX_OPEN_FILES = 10;
@@ -85,9 +85,9 @@
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(
typeTraits);
- ITreeIndexFrameFactory leafFrameFactory = new NSMLeafFrameFactory(
+ ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(
tupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new NSMInteriorFrameFactory(
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(
tupleWriterFactory);
ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
@@ -108,7 +108,7 @@
long start = System.currentTimeMillis();
- print("INSERTING INTO TREE\n");
+ LOGGER.info("INSERTING INTO TREE");
ByteBuffer frame = ctx.allocateFrame();
FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
@@ -147,8 +147,8 @@
if (i % 10000 == 0) {
long end = System.currentTimeMillis();
- print("INSERTING " + i + " : " + f0 + " " + f1 + " "
- + (end - start) + "\n");
+ LOGGER.info("INSERTING " + i + " : " + f0 + " " + f1 + " "
+ + (end - start));
}
try {
@@ -161,8 +161,7 @@
TreeIndexStatsGatherer statsGatherer = new TreeIndexStatsGatherer(bufferCache, freePageManager, fileId, btree.getRootPageId());
TreeIndexStats stats = statsGatherer.gatherStats(leafFrame, interiorFrame, metaFrame);
- String s = stats.toString();
- System.out.println(s);
+ LOGGER.info(stats.toString());
TreeIndexBufferCacheWarmup bufferCacheWarmup = new TreeIndexBufferCacheWarmup(bufferCache, freePageManager, fileId);
bufferCacheWarmup.warmup(leafFrame, metaFrame, new int[] {1, 2}, new int[] {2, 5});
@@ -170,7 +169,5 @@
btree.close();
bufferCache.closeFile(fileId);
bufferCache.close();
-
- print("\n");
}
}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeTest.java
index 4c6bb8c..6fa5d43 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/BTreeTest.java
@@ -41,8 +41,8 @@
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeOpContext;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
@@ -83,7 +83,7 @@
@Test
public void test01() throws Exception {
- print("FIXED-LENGTH KEY TEST\n");
+ LOGGER.info("FIXED-LENGTH KEY TEST");
TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
@@ -107,8 +107,8 @@
MultiComparator cmp = new MultiComparator(typeTraits, cmps);
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
- ITreeIndexFrameFactory leafFrameFactory = new NSMLeafFrameFactory(tupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new NSMInteriorFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
@@ -126,7 +126,7 @@
long start = System.currentTimeMillis();
- print("INSERTING INTO TREE\n");
+ LOGGER.info("INSERTING INTO TREE");
ByteBuffer frame = ctx.allocateFrame();
FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
@@ -159,12 +159,9 @@
tuple.reset(accessor, 0);
- // System.out.println(tuple.getFieldCount() + " " +
- // tuple.getFieldLength(0) + " " + tuple.getFieldLength(1));
-
if (i % 1000 == 0) {
long end = System.currentTimeMillis();
- print("INSERTING " + i + " : " + f0 + " " + f1 + " " + (end - start) + "\n");
+ LOGGER.info("INSERTING " + i + " : " + f0 + " " + f1 + " " + (end - start));
}
try {
@@ -173,26 +170,20 @@
} catch (Exception e) {
e.printStackTrace();
}
-
- // btree.printTree(leafFrame, interiorFrame);
- // System.out.println();
}
// btree.printTree(leafFrame, interiorFrame);
- // System.out.println();
int maxPage = btree.getFreePageManager().getMaxPage(metaFrame);
- System.out.println("MAXPAGE: " + maxPage);
-
- String stats = btree.printStats();
- print(stats);
-
+ LOGGER.info("MAXPAGE: " + maxPage);
+ LOGGER.info(btree.printStats());
+
long end = System.currentTimeMillis();
long duration = end - start;
- print("DURATION: " + duration + "\n");
+ LOGGER.info("DURATION: " + duration);
// ordered scan
- print("ORDERED SCAN:\n");
+ LOGGER.info("ORDERED SCAN:");
ITreeIndexCursor scanCursor = new BTreeRangeSearchCursor(leafFrame);
RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
BTreeOpContext searchOpCtx = btree.createOpContext(IndexOp.SEARCH, leafFrame, interiorFrame, null);
@@ -202,7 +193,7 @@
scanCursor.next();
ITupleReference frameTuple = scanCursor.getTuple();
String rec = cmp.printTuple(frameTuple, recDescSers);
- print(rec + "\n");
+ LOGGER.info(rec);
}
} catch (Exception e) {
e.printStackTrace();
@@ -211,7 +202,7 @@
}
// disk-order scan
- print("DISK-ORDER SCAN:\n");
+ LOGGER.info("DISK-ORDER SCAN:");
TreeDiskOrderScanCursor diskOrderCursor = new TreeDiskOrderScanCursor(leafFrame);
BTreeOpContext diskOrderScanOpCtx = btree.createOpContext(IndexOp.DISKORDERSCAN, leafFrame, null, null);
btree.diskOrderScan(diskOrderCursor, leafFrame, metaFrame, diskOrderScanOpCtx);
@@ -220,7 +211,7 @@
diskOrderCursor.next();
ITupleReference frameTuple = diskOrderCursor.getTuple();
String rec = cmp.printTuple(frameTuple, recDescSers);
- print(rec + "\n");
+ LOGGER.info(rec);
}
} catch (Exception e) {
e.printStackTrace();
@@ -229,7 +220,7 @@
}
// range search in [-1000, 1000]
- print("RANGE SEARCH:\n");
+ LOGGER.info("RANGE SEARCH:");
ITreeIndexCursor rangeCursor = new BTreeRangeSearchCursor(leafFrame);
@@ -275,7 +266,7 @@
rangeCursor.next();
ITupleReference frameTuple = rangeCursor.getTuple();
String rec = cmp.printTuple(frameTuple, recDescSers);
- print(rec + "\n");
+ LOGGER.info(rec);
}
} catch (Exception e) {
e.printStackTrace();
@@ -286,8 +277,6 @@
btree.close();
bufferCache.closeFile(fileId);
bufferCache.close();
-
- print("\n");
}
// COMPOSITE KEY TEST (NON-UNIQUE B-TREE)
@@ -298,7 +287,7 @@
@Test
public void test02() throws Exception {
- print("COMPOSITE KEY TEST\n");
+ LOGGER.info("COMPOSITE KEY TEST");
TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
@@ -326,8 +315,8 @@
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
// SimpleTupleWriterFactory tupleWriterFactory = new
// SimpleTupleWriterFactory();
- ITreeIndexFrameFactory leafFrameFactory = new NSMLeafFrameFactory(tupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new NSMInteriorFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
@@ -345,7 +334,7 @@
long start = System.currentTimeMillis();
- print("INSERTING INTO TREE\n");
+ LOGGER.info("INSERTING INTO TREE");
ByteBuffer frame = ctx.allocateFrame();
FrameTupleAppender appender = new FrameTupleAppender(ctx.getFrameSize());
@@ -380,7 +369,7 @@
tuple.reset(accessor, 0);
if (i % 1000 == 0) {
- print("INSERTING " + i + " : " + f0 + " " + f1 + "\n");
+ LOGGER.info("INSERTING " + i + " : " + f0 + " " + f1);
}
try {
@@ -392,10 +381,10 @@
long end = System.currentTimeMillis();
long duration = end - start;
- print("DURATION: " + duration + "\n");
+ LOGGER.info("DURATION: " + duration);
// try a simple index scan
- print("ORDERED SCAN:\n");
+ LOGGER.info("ORDERED SCAN:");
ITreeIndexCursor scanCursor = new BTreeRangeSearchCursor(leafFrame);
RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
BTreeOpContext searchOpCtx = btree.createOpContext(IndexOp.SEARCH, leafFrame, interiorFrame, null);
@@ -406,7 +395,7 @@
scanCursor.next();
ITupleReference frameTuple = scanCursor.getTuple();
String rec = cmp.printTuple(frameTuple, recDescSers);
- print(rec + "\n");
+ LOGGER.info(rec);
}
} catch (Exception e) {
e.printStackTrace();
@@ -415,7 +404,7 @@
}
// range search in [(-3),(3)]
- print("RANGE SEARCH:\n");
+ LOGGER.info("RANGE SEARCH:");
ITreeIndexCursor rangeCursor = new BTreeRangeSearchCursor(leafFrame);
// build low and high keys
@@ -477,8 +466,6 @@
btree.close();
bufferCache.closeFile(fileId);
bufferCache.close();
-
- print("\n");
}
// VARIABLE-LENGTH TEST
@@ -489,7 +476,7 @@
@Test
public void test03() throws Exception {
- print("VARIABLE-LENGTH KEY TEST\n");
+ LOGGER.info("VARIABLE-LENGTH KEY TEST");
TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
@@ -515,8 +502,8 @@
SimpleTupleWriterFactory tupleWriterFactory = new SimpleTupleWriterFactory();
// TypeAwareTupleWriterFactory tupleWriterFactory = new
// TypeAwareTupleWriterFactory(typeTraits);
- ITreeIndexFrameFactory leafFrameFactory = new NSMLeafFrameFactory(tupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new NSMInteriorFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
@@ -563,23 +550,20 @@
tuple.reset(accessor, 0);
if (i % 1000 == 0) {
- // print("INSERTING " + i + ": " + cmp.printRecord(record, 0) +
- // "\n");
- print("INSERTING " + i + "\n");
+ LOGGER.info("INSERTING " + i);
}
try {
btree.insert(tuple, insertOpCtx);
} catch (Exception e) {
- // e.printStackTrace();
}
}
// btree.printTree();
- System.out.println("DONE INSERTING");
+ LOGGER.info("DONE INSERTING");
// ordered scan
- print("ORDERED SCAN:\n");
+ LOGGER.info("ORDERED SCAN:");
ITreeIndexCursor scanCursor = new BTreeRangeSearchCursor(leafFrame);
RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
BTreeOpContext searchOpCtx = btree.createOpContext(IndexOp.SEARCH, leafFrame, interiorFrame, null);
@@ -590,7 +574,7 @@
scanCursor.next();
ITupleReference frameTuple = scanCursor.getTuple();
String rec = cmp.printTuple(frameTuple, recDescSers);
- print(rec + "\n");
+ LOGGER.info(rec);
}
} catch (Exception e) {
e.printStackTrace();
@@ -599,7 +583,7 @@
}
// range search in ["cbf", cc7"]
- print("RANGE SEARCH:\n");
+ LOGGER.info("RANGE SEARCH:");
ITreeIndexCursor rangeCursor = new BTreeRangeSearchCursor(leafFrame);
@@ -645,7 +629,7 @@
rangeCursor.next();
ITupleReference frameTuple = rangeCursor.getTuple();
String rec = cmp.printTuple(frameTuple, recDescSers);
- print(rec + "\n");
+ LOGGER.info(rec);
}
} catch (Exception e) {
e.printStackTrace();
@@ -656,8 +640,6 @@
btree.close();
bufferCache.closeFile(fileId);
bufferCache.close();
-
- print("\n");
}
// DELETION TEST
@@ -669,7 +651,7 @@
@Test
public void test04() throws Exception {
- print("DELETION TEST\n");
+ LOGGER.info("DELETION TEST");
TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
@@ -695,8 +677,8 @@
// SimpleTupleWriterFactory tupleWriterFactory = new
// SimpleTupleWriterFactory();
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
- ITreeIndexFrameFactory leafFrameFactory = new NSMLeafFrameFactory(tupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new NSMInteriorFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
@@ -730,9 +712,9 @@
int runs = 3;
for (int run = 0; run < runs; run++) {
- print("DELETION TEST RUN: " + (run + 1) + "/" + runs + "\n");
+ LOGGER.info("DELETION TEST RUN: " + (run + 1) + "/" + runs);
- print("INSERTING INTO BTREE\n");
+ LOGGER.info("INSERTING INTO BTREE");
int maxLength = 10;
int ins = 10000;
String[] f0s = new String[ins];
@@ -758,9 +740,7 @@
tuple.reset(accessor, 0);
if (i % 1000 == 0) {
- print("INSERTING " + i + "\n");
- // print("INSERTING " + i + ": " + cmp.printRecord(record,
- // 0) + "\n");
+ LOGGER.info("INSERTING " + i);
}
try {
@@ -777,7 +757,7 @@
// btree.printTree();
// btree.printStats();
- print("DELETING FROM BTREE\n");
+ LOGGER.info("DELETING FROM BTREE");
int delDone = 0;
for (int i = 0; i < ins; i++) {
@@ -793,9 +773,7 @@
tuple.reset(accessor, 0);
if (i % 1000 == 0) {
- // print("DELETING " + i + ": " +
- // cmp.printRecord(records[i], 0) + "\n");
- print("DELETING " + i + "\n");
+ LOGGER.info("DELETING " + i);
}
try {
@@ -808,16 +786,15 @@
}
if (insDoneCmp[i] != delDone) {
- print("INCONSISTENT STATE, ERROR IN DELETION TEST\n");
- print("INSDONECMP: " + insDoneCmp[i] + " " + delDone + "\n");
+ LOGGER.info("INCONSISTENT STATE, ERROR IN DELETION TEST");
+ LOGGER.info("INSDONECMP: " + insDoneCmp[i] + " " + delDone);
break;
}
- // btree.printTree();
}
// btree.printTree(leafFrame, interiorFrame);
if (insDone != delDone) {
- print("ERROR! INSDONE: " + insDone + " DELDONE: " + delDone);
+ LOGGER.info("ERROR! INSDONE: " + insDone + " DELDONE: " + delDone);
break;
}
}
@@ -825,8 +802,6 @@
btree.close();
bufferCache.closeFile(fileId);
bufferCache.close();
-
- print("\n");
}
// BULK LOAD TEST
@@ -836,7 +811,7 @@
@Test
public void test05() throws Exception {
- print("BULK LOAD TEST\n");
+ LOGGER.info("BULK LOAD TEST");
TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
@@ -864,8 +839,8 @@
// SimpleTupleWriterFactory tupleWriterFactory = new
// SimpleTupleWriterFactory();
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
- ITreeIndexFrameFactory leafFrameFactory = new NSMLeafFrameFactory(tupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new NSMInteriorFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
ITreeIndexFrame leafFrame = leafFrameFactory.createFrame();
@@ -897,7 +872,7 @@
// generate sorted records
int ins = 100000;
- print("BULK LOADING " + ins + " RECORDS\n");
+ LOGGER.info("BULK LOADING " + ins + " RECORDS");
long start = System.currentTimeMillis();
for (int i = 0; i < ins; i++) {
@@ -923,10 +898,10 @@
long end = System.currentTimeMillis();
long duration = end - start;
- print("DURATION: " + duration + "\n");
+ LOGGER.info("DURATION: " + duration);
// range search
- print("RANGE SEARCH:\n");
+ LOGGER.info("RANGE SEARCH:");
ITreeIndexCursor rangeCursor = new BTreeRangeSearchCursor((IBTreeLeafFrame) leafFrame);
// build low and high keys
@@ -973,7 +948,7 @@
rangeCursor.next();
ITupleReference frameTuple = rangeCursor.getTuple();
String rec = cmp.printTuple(frameTuple, recDescSers);
- print(rec + "\n");
+ LOGGER.info(rec);
}
} catch (Exception e) {
e.printStackTrace();
@@ -984,8 +959,6 @@
btree.close();
bufferCache.closeFile(fileId);
bufferCache.close();
-
- print("\n");
}
// TIME-INTERVAL INTERSECTION DEMO FOR EVENT PEOPLE
@@ -994,7 +967,7 @@
@Test
public void test06() throws Exception {
- print("TIME-INTERVAL INTERSECTION DEMO\n");
+ LOGGER.info("TIME-INTERVAL INTERSECTION DEMO");
TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(ctx);
@@ -1021,8 +994,8 @@
// SimpleTupleWriterFactory tupleWriterFactory = new
// SimpleTupleWriterFactory();
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
- ITreeIndexFrameFactory leafFrameFactory = new NSMLeafFrameFactory(tupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new NSMInteriorFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(tupleWriterFactory);
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(tupleWriterFactory);
ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame) leafFrameFactory.createFrame();
@@ -1106,13 +1079,11 @@
tuple.reset(accessor, 0);
- // print("INSERTING " + i + " : " + f0 + " " + f1 + "\n");
- print("INSERTING " + i + "\n");
+ LOGGER.info("INSERTING " + i);
try {
btree.insert(tuple, insertOpCtx);
} catch (Exception e) {
- // e.printStackTrace();
}
}
// btree.printTree(leafFrame, interiorFrame);
@@ -1120,11 +1091,11 @@
long end = System.currentTimeMillis();
long duration = end - start;
- print("DURATION: " + duration + "\n");
+ LOGGER.info("DURATION: " + duration);
// try a simple index scan
- print("ORDERED SCAN:\n");
+ LOGGER.info("ORDERED SCAN:");
ITreeIndexCursor scanCursor = new BTreeRangeSearchCursor(leafFrame);
RangePredicate nullPred = new RangePredicate(true, null, null, true, true, null, null);
BTreeOpContext searchOpCtx = btree.createOpContext(IndexOp.SEARCH, leafFrame, interiorFrame, null);
@@ -1144,7 +1115,7 @@
}
// try a range search
- print("RANGE SEARCH:\n");
+ LOGGER.info("RANGE SEARCH:");
ITreeIndexCursor rangeCursor = new BTreeRangeSearchCursor(leafFrame);
// build low and high keys
@@ -1187,9 +1158,6 @@
searchCmps[1] = IntegerBinaryComparatorFactory.INSTANCE.createBinaryComparator();
MultiComparator searchCmp = new MultiComparator(typeTraits, searchCmps);
- // print("INDEX RANGE SEARCH ON: " + cmp.printKey(lowKey, 0) + " " +
- // cmp.printKey(highKey, 0) + "\n");
-
RangePredicate rangePred = new RangePredicate(true, lowKey, highKey, true, true, searchCmp, searchCmp);
btree.search(rangeCursor, rangePred, searchOpCtx);
@@ -1198,7 +1166,7 @@
rangeCursor.next();
ITupleReference frameTuple = rangeCursor.getTuple();
String rec = cmp.printTuple(frameTuple, recDescSers);
- print(rec + "\n");
+ LOGGER.info(rec);
}
} catch (Exception e) {
e.printStackTrace();
@@ -1209,8 +1177,6 @@
btree.close();
bufferCache.closeFile(fileId);
bufferCache.close();
-
- print("\n");
}
public static String randomString(int length, Random random) {
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/RangeSearchCursorTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/RangeSearchCursorTest.java
index bc6f991..1a3b219 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/RangeSearchCursorTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/RangeSearchCursorTest.java
@@ -47,9 +47,9 @@
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeInteriorFrame;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.btree.frames.FieldPrefixNSMLeafFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeFieldPrefixNSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeException;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeOpContext;
@@ -82,9 +82,9 @@
TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(
typeTraits);
- ITreeIndexFrameFactory leafFrameFactory = new NSMLeafFrameFactory(
+ ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(
tupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new NSMInteriorFrameFactory(
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(
tupleWriterFactory);
ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
@@ -116,7 +116,7 @@
@Test
public void uniqueIndexTest() throws Exception {
- System.out.println("TESTING RANGE SEARCH CURSOR ON UNIQUE INDEX");
+ LOGGER.info("TESTING RANGE SEARCH CURSOR ON UNIQUE INDEX");
TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
IBufferCache bufferCache = TestStorageManagerComponentHolder
@@ -191,8 +191,6 @@
int minSearchKey = -100;
int maxSearchKey = 100;
- // System.out.println("STARTING SEARCH TESTS");
-
// forward searches
performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
maxSearchKey, true, true, true, false);
@@ -221,7 +219,7 @@
@Test
public void nonUniqueIndexTest() throws Exception {
- System.out.println("TESTING RANGE SEARCH CURSOR ON NONUNIQUE INDEX");
+ LOGGER.info("TESTING RANGE SEARCH CURSOR ON NONUNIQUE INDEX");
TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
IBufferCache bufferCache = TestStorageManagerComponentHolder
@@ -295,8 +293,6 @@
int minSearchKey = -100;
int maxSearchKey = 100;
- // System.out.println("STARTING SEARCH TESTS");
-
// forward searches
performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
maxSearchKey, true, true, true, false);
@@ -325,10 +321,9 @@
@Test
public void nonUniqueFieldPrefixIndexTest() throws Exception {
- System.out
- .println("TESTING RANGE SEARCH CURSOR ON NONUNIQUE FIELD-PREFIX COMPRESSED INDEX");
+ LOGGER.info("TESTING RANGE SEARCH CURSOR ON NONUNIQUE FIELD-PREFIX COMPRESSED INDEX");
- ITreeIndexFrameFactory leafFrameFactory = new FieldPrefixNSMLeafFrameFactory(
+ ITreeIndexFrameFactory leafFrameFactory = new BTreeFieldPrefixNSMLeafFrameFactory(
tupleWriterFactory);
IBTreeLeafFrame leafFrame = (IBTreeLeafFrame)leafFrameFactory.createFrame();
@@ -404,8 +399,6 @@
int minSearchKey = -100;
int maxSearchKey = 100;
- // System.out.println("STARTING SEARCH TESTS");
-
// forward searches
performSearches(keys, btree, leafFrame, interiorFrame, minSearchKey,
maxSearchKey, true, true, true, false);
@@ -530,8 +523,6 @@
for (int i = minKey; i < maxKey; i++) {
for (int j = minKey; j < maxKey; j++) {
- // if(i != -100 || j != 1) continue;
-
results.clear();
expectedResults.clear();
@@ -583,29 +574,31 @@
else
u = ')';
- System.out.println("RANGE: " + l + " " + lowKey + " , "
+ LOGGER.info("RANGE: " + l + " " + lowKey + " , "
+ highKey + " " + u);
- for (Integer r : expectedResults)
- System.out.print(r + " ");
- System.out.println();
+ StringBuilder strBuilder = new StringBuilder();
+ for (Integer r : expectedResults) {
+ strBuilder.append(r + " ");
+ }
+ LOGGER.info(strBuilder.toString());
}
}
if (results.size() == expectedResults.size()) {
for (int k = 0; k < results.size(); k++) {
if (!results.get(k).equals(expectedResults.get(k))) {
- System.out.println("DIFFERENT RESULTS AT: i=" + i
+ LOGGER.info("DIFFERENT RESULTS AT: i=" + i
+ " j=" + j + " k=" + k);
- System.out.println(results.get(k) + " "
+ LOGGER.info(results.get(k) + " "
+ expectedResults.get(k));
return false;
}
}
} else {
- System.out.println("UNEQUAL NUMBER OF RESULTS AT: i=" + i
+ LOGGER.info("UNEQUAL NUMBER OF RESULTS AT: i=" + i
+ " j=" + j);
- System.out.println("RESULTS: " + results.size());
- System.out.println("EXPECTED RESULTS: "
+ LOGGER.info("RESULTS: " + results.size());
+ LOGGER.info("EXPECTED RESULTS: "
+ expectedResults.size());
return false;
}
diff --git a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StorageManagerTest.java b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StorageManagerTest.java
index 1f08145..1a510e6 100644
--- a/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StorageManagerTest.java
+++ b/hyracks-tests/hyracks-storage-am-btree-test/src/test/java/edu/uci/ics/hyracks/storage/am/btree/StorageManagerTest.java
@@ -90,7 +90,7 @@
private void pinRandomPage() {
int pageId = Math.abs(rnd.nextInt() % maxPages);
- System.out.println(workerId + " PINNING PAGE: " + pageId);
+ LOGGER.info(workerId + " PINNING PAGE: " + pageId);
try {
ICachedPage page = bufferCache.pin(BufferedFileHandle
@@ -105,14 +105,14 @@
break;
case FTA_READONLY: {
- System.out.println(workerId + " S LATCHING: " + pageId);
+ LOGGER.info(workerId + " S LATCHING: " + pageId);
page.acquireReadLatch();
latch = LatchType.LATCH_S;
}
break;
case FTA_WRITEONLY: {
- System.out.println(workerId + " X LATCHING: " + pageId);
+ LOGGER.info(workerId + " X LATCHING: " + pageId);
page.acquireWriteLatch();
latch = LatchType.LATCH_X;
}
@@ -120,11 +120,11 @@
case FTA_MIXED: {
if (rnd.nextInt() % 2 == 0) {
- System.out.println(workerId + " S LATCHING: " + pageId);
+ LOGGER.info(workerId + " S LATCHING: " + pageId);
page.acquireReadLatch();
latch = LatchType.LATCH_S;
} else {
- System.out.println(workerId + " X LATCHING: " + pageId);
+ LOGGER.info(workerId + " X LATCHING: " + pageId);
page.acquireWriteLatch();
latch = LatchType.LATCH_X;
}
@@ -148,16 +148,16 @@
if (plPage.latch != null) {
if (plPage.latch == LatchType.LATCH_S) {
- System.out.println(workerId + " S UNLATCHING: "
+ LOGGER.info(workerId + " S UNLATCHING: "
+ plPage.pageId);
plPage.page.releaseReadLatch();
} else {
- System.out.println(workerId + " X UNLATCHING: "
+ LOGGER.info(workerId + " X UNLATCHING: "
+ plPage.pageId);
plPage.page.releaseWriteLatch();
}
}
- System.out.println(workerId + " UNPINNING PAGE: "
+ LOGGER.info(workerId + " UNPINNING PAGE: "
+ plPage.pageId);
bufferCache.unpin(plPage.page);
@@ -168,7 +168,7 @@
}
private void openFile() {
- System.out.println(workerId + " OPENING FILE: " + fileId);
+ LOGGER.info(workerId + " OPENING FILE: " + fileId);
try {
bufferCache.openFile(fileId);
fileIsOpen = true;
@@ -178,7 +178,7 @@
}
private void closeFile() {
- System.out.println(workerId + " CLOSING FILE: " + fileId);
+ LOGGER.info(workerId + " CLOSING FILE: " + fileId);
try {
bufferCache.closeFile(fileId);
fileIsOpen = false;
@@ -195,7 +195,7 @@
while (loopCount < maxLoopCount) {
loopCount++;
- System.out.println(workerId + " LOOP: " + loopCount + "/"
+ LOGGER.info(workerId + " LOOP: " + loopCount + "/"
+ maxLoopCount);
if (fileIsOpen) {
diff --git a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/AbstractInvIndexSearchTest.java b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/AbstractInvIndexSearchTest.java
new file mode 100644
index 0000000..31f06a1
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/AbstractInvIndexSearchTest.java
@@ -0,0 +1,224 @@
+package edu.uci.ics.hyracks.storage.am.invertedindex;
+
+import java.io.DataOutput;
+import java.io.File;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Random;
+
+import org.junit.After;
+import org.junit.Before;
+
+import edu.uci.ics.fuzzyjoin.tokenizer.IBinaryTokenizer;
+import edu.uci.ics.fuzzyjoin.tokenizer.ITokenFactory;
+import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
+import edu.uci.ics.hyracks.api.context.IHyracksStageletContext;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
+import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
+import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.api.io.FileReference;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.comparators.IntegerBinaryComparatorFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.comparators.UTF8StringBinaryComparatorFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
+import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
+import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
+import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
+import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexResultCursor;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.TOccurrenceSearcher;
+import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
+import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
+import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
+import edu.uci.ics.hyracks.test.support.TestUtils;
+
+public abstract class AbstractInvIndexSearchTest extends AbstractInvIndexTest {
+ protected final int PAGE_SIZE = 32768;
+ protected final int NUM_PAGES = 100;
+ protected final int MAX_OPEN_FILES = 10;
+ protected final int HYRACKS_FRAME_SIZE = 32768;
+ protected IHyracksStageletContext stageletCtx = TestUtils
+ .create(HYRACKS_FRAME_SIZE);
+
+ protected IBufferCache bufferCache;
+ protected IFileMapProvider fmp;
+
+ // --- BTREE ---
+
+ // create file refs
+ protected FileReference btreeFile = new FileReference(new File(
+ btreeFileName));
+ protected int btreeFileId;
+
+ // declare btree fields
+ protected int fieldCount = 5;
+ protected ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
+
+ // declare btree keys
+ protected int btreeKeyFieldCount = 1;
+ protected IBinaryComparator[] btreeBinCmps = new IBinaryComparator[btreeKeyFieldCount];
+ protected MultiComparator btreeCmp = new MultiComparator(typeTraits,
+ btreeBinCmps);
+
+ // btree frame factories
+ protected TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(
+ typeTraits);
+ protected ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(
+ tupleWriterFactory);
+ protected ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(
+ tupleWriterFactory);
+ protected ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+
+ // btree frames
+ protected ITreeIndexFrame leafFrame = leafFrameFactory.createFrame();
+ protected ITreeIndexMetaDataFrame metaFrame = metaFrameFactory
+ .createFrame();
+
+ protected IFreePageManager freePageManager;
+
+ protected BTree btree;
+
+ // --- INVERTED INDEX ---
+
+ protected FileReference invListsFile = new FileReference(new File(
+ invListsFileName));
+ protected int invListsFileId;
+
+ protected int invListFields = 1;
+ protected ITypeTrait[] invListTypeTraits = new ITypeTrait[invListFields];
+
+ protected int invListKeys = 1;
+ protected IBinaryComparator[] invListBinCmps = new IBinaryComparator[invListKeys];
+ protected MultiComparator invListCmp = new MultiComparator(
+ invListTypeTraits, invListBinCmps);
+
+ protected InvertedIndex invIndex;
+
+ protected Random rnd = new Random();
+
+ protected ByteBuffer frame = stageletCtx.allocateFrame();
+ protected FrameTupleAppender appender = new FrameTupleAppender(
+ stageletCtx.getFrameSize());
+ protected ArrayTupleBuilder tb = new ArrayTupleBuilder(2);
+ protected DataOutput dos = tb.getDataOutput();
+
+ protected ISerializerDeserializer[] insertSerde = {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE };
+ protected RecordDescriptor insertRecDesc = new RecordDescriptor(insertSerde);
+ protected IFrameTupleAccessor accessor = new FrameTupleAccessor(
+ stageletCtx.getFrameSize(), insertRecDesc);
+
+ protected FrameTupleReference tuple = new FrameTupleReference();
+
+ protected ArrayList<ArrayList<Integer>> checkInvLists = new ArrayList<ArrayList<Integer>>();
+
+ protected int maxId = 1000000;
+ // protected int maxId = 1000;
+ protected int[] scanCountArray = new int[maxId];
+ protected ArrayList<Integer> expectedResults = new ArrayList<Integer>();
+
+ protected ISerializerDeserializer[] querySerde = { UTF8StringSerializerDeserializer.INSTANCE };
+ protected RecordDescriptor queryRecDesc = new RecordDescriptor(querySerde);
+
+ protected FrameTupleAppender queryAppender = new FrameTupleAppender(
+ stageletCtx.getFrameSize());
+ protected ArrayTupleBuilder queryTb = new ArrayTupleBuilder(
+ querySerde.length);
+ protected DataOutput queryDos = queryTb.getDataOutput();
+
+ protected IFrameTupleAccessor queryAccessor = new FrameTupleAccessor(
+ stageletCtx.getFrameSize(), queryRecDesc);
+ protected FrameTupleReference queryTuple = new FrameTupleReference();
+
+ protected ITokenFactory tokenFactory;
+ protected IBinaryTokenizer tokenizer;
+
+ protected TOccurrenceSearcher searcher;
+ protected IInvertedIndexResultCursor resultCursor;
+
+ /**
+ * Initialize members, generate data, and bulk load the inverted index.
+ *
+ */
+ @Before
+ public void start() throws Exception {
+ TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES,
+ MAX_OPEN_FILES);
+ bufferCache = TestStorageManagerComponentHolder
+ .getBufferCache(stageletCtx);
+ fmp = TestStorageManagerComponentHolder.getFileMapProvider(stageletCtx);
+
+ // --- BTREE ---
+
+ bufferCache.createFile(btreeFile);
+ btreeFileId = fmp.lookupFileId(btreeFile);
+ bufferCache.openFile(btreeFileId);
+
+ // token (key)
+ typeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ // startPageId
+ typeTraits[1] = new TypeTrait(4);
+ // endPageId
+ typeTraits[2] = new TypeTrait(4);
+ // startOff
+ typeTraits[3] = new TypeTrait(4);
+ // numElements
+ typeTraits[4] = new TypeTrait(4);
+
+ btreeBinCmps[0] = UTF8StringBinaryComparatorFactory.INSTANCE
+ .createBinaryComparator();
+
+ freePageManager = new LinkedListFreePageManager(bufferCache,
+ btreeFileId, 0, metaFrameFactory);
+
+ btree = new BTree(bufferCache, freePageManager, interiorFrameFactory,
+ leafFrameFactory, btreeCmp);
+ btree.create(btreeFileId, leafFrame, metaFrame);
+ btree.open(btreeFileId);
+
+ // --- INVERTED INDEX ---
+
+ bufferCache.createFile(invListsFile);
+ invListsFileId = fmp.lookupFileId(invListsFile);
+ bufferCache.openFile(invListsFileId);
+
+ invListTypeTraits[0] = new TypeTrait(4);
+ invListBinCmps[0] = IntegerBinaryComparatorFactory.INSTANCE
+ .createBinaryComparator();
+
+ invIndex = new InvertedIndex(bufferCache, btree, invListCmp);
+ invIndex.open(invListsFileId);
+
+ rnd.setSeed(50);
+
+ accessor.reset(frame);
+ queryAccessor.reset(frame);
+ }
+
+ @After
+ public void deinit() throws HyracksDataException {
+ AbstractInvIndexTest.tearDown();
+ btree.close();
+ invIndex.close();
+ bufferCache.closeFile(btreeFileId);
+ bufferCache.closeFile(invListsFileId);
+ bufferCache.close();
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/AbstractInvIndexTest.java b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/AbstractInvIndexTest.java
index 88fabba..cc8ab15 100644
--- a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/AbstractInvIndexTest.java
+++ b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/AbstractInvIndexTest.java
@@ -1,22 +1,43 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.invertedindex;
import java.io.File;
import java.text.SimpleDateFormat;
import java.util.Date;
+import java.util.logging.Logger;
public abstract class AbstractInvIndexTest {
- protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat("ddMMyy-hhmmssSS");
- protected final static String tmpDir = System.getProperty("java.io.tmpdir");
- protected final static String sep = System.getProperty("file.separator");
- protected final static String baseFileName = tmpDir + sep + simpleDateFormat.format(new Date());
- protected final static String btreeFileName = baseFileName + "btree";
- protected final static String invListsFileName = baseFileName + "invlists";
-
- public static void tearDown() {
- File btreeFile = new File(btreeFileName);
- btreeFile.deleteOnExit();
- File invListsFile = new File(invListsFileName);
- invListsFile.deleteOnExit();
- }
+ protected static final Logger LOGGER = Logger
+ .getLogger(AbstractInvIndexTest.class.getName());
+
+ protected final static SimpleDateFormat simpleDateFormat = new SimpleDateFormat(
+ "ddMMyy-hhmmssSS");
+ protected final static String tmpDir = System.getProperty("java.io.tmpdir");
+ protected final static String sep = System.getProperty("file.separator");
+ protected final static String baseFileName = tmpDir + sep
+ + simpleDateFormat.format(new Date());
+ protected final static String btreeFileName = baseFileName + "btree";
+ protected final static String invListsFileName = baseFileName + "invlists";
+
+ public static void tearDown() {
+ File btreeFile = new File(btreeFileName);
+ btreeFile.deleteOnExit();
+ File invListsFile = new File(invListsFileName);
+ invListsFile.deleteOnExit();
+ }
}
diff --git a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/BulkLoadTest.java b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/BulkLoadTest.java
index f55d086..fb0b7b9 100644
--- a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/BulkLoadTest.java
+++ b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/BulkLoadTest.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.invertedindex;
import java.io.DataOutput;
@@ -30,8 +45,8 @@
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
import edu.uci.ics.hyracks.storage.am.btree.api.IBTreeLeafFrame;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMLeafFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMInteriorFrameFactory;
+import edu.uci.ics.hyracks.storage.am.btree.frames.BTreeNSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeOpContext;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTreeRangeSearchCursor;
@@ -58,229 +73,273 @@
import edu.uci.ics.hyracks.test.support.TestUtils;
public class BulkLoadTest extends AbstractInvIndexTest {
-
- private static final int PAGE_SIZE = 32768;
- private static final int NUM_PAGES = 100;
- private static final int MAX_OPEN_FILES = 10;
- private static final int HYRACKS_FRAME_SIZE = 32768;
- private IHyracksStageletContext stageletCtx = TestUtils.create(HYRACKS_FRAME_SIZE);
- @Test
- public void singleFieldPayloadTest() throws Exception {
+ private static final int PAGE_SIZE = 32768;
+ private static final int NUM_PAGES = 100;
+ private static final int MAX_OPEN_FILES = 10;
+ private static final int HYRACKS_FRAME_SIZE = 32768;
+ private IHyracksStageletContext stageletCtx = TestUtils
+ .create(HYRACKS_FRAME_SIZE);
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- IBufferCache bufferCache = TestStorageManagerComponentHolder.getBufferCache(stageletCtx);
- IFileMapProvider fmp = TestStorageManagerComponentHolder.getFileMapProvider(stageletCtx);
-
- // create file refs
- FileReference btreeFile = new FileReference(new File(btreeFileName));
- bufferCache.createFile(btreeFile);
- int btreeFileId = fmp.lookupFileId(btreeFile);
- bufferCache.openFile(btreeFileId);
-
- FileReference invListsFile = new FileReference(new File(invListsFileName));
- bufferCache.createFile(invListsFile);
- int invListsFileId = fmp.lookupFileId(invListsFile);
- bufferCache.openFile(invListsFileId);
-
- // declare btree fields
- int fieldCount = 5;
- ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
- // token (key)
- typeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- // startPageId
- typeTraits[1] = new TypeTrait(4);
- // endPageId
- typeTraits[2] = new TypeTrait(4);
- // startOff
- typeTraits[3] = new TypeTrait(4);
- // numElements
- typeTraits[4] = new TypeTrait(4);
+ /**
+ * This test generates a list of <word-token, id> pairs which are pre-sorted
+ * on the token. Those pairs for the input to an inverted-index bulk load.
+ * The contents of the inverted lists are verified against the generated
+ * data.
+ *
+ */
+ @Test
+ public void singleFieldPayloadTest() throws Exception {
- // declare btree keys
- int keyFieldCount = 1;
- IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
- cmps[0] = UTF8StringBinaryComparatorFactory.INSTANCE.createBinaryComparator();
+ TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES,
+ MAX_OPEN_FILES);
+ IBufferCache bufferCache = TestStorageManagerComponentHolder
+ .getBufferCache(stageletCtx);
+ IFileMapProvider fmp = TestStorageManagerComponentHolder
+ .getFileMapProvider(stageletCtx);
- MultiComparator btreeCmp = new MultiComparator(typeTraits, cmps);
+ // create file refs
+ FileReference btreeFile = new FileReference(new File(btreeFileName));
+ bufferCache.createFile(btreeFile);
+ int btreeFileId = fmp.lookupFileId(btreeFile);
+ bufferCache.openFile(btreeFileId);
- TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
- ITreeIndexFrameFactory leafFrameFactory = new NSMLeafFrameFactory(tupleWriterFactory);
- ITreeIndexFrameFactory interiorFrameFactory = new NSMInteriorFrameFactory(tupleWriterFactory);
- ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
+ FileReference invListsFile = new FileReference(new File(
+ invListsFileName));
+ bufferCache.createFile(invListsFile);
+ int invListsFileId = fmp.lookupFileId(invListsFile);
+ bufferCache.openFile(invListsFileId);
- ITreeIndexFrame leafFrame = leafFrameFactory.createFrame();
- ITreeIndexFrame interiorFrame = interiorFrameFactory.createFrame();
- ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
+ // declare btree fields
+ int fieldCount = 5;
+ ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
+ // token (key)
+ typeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
+ // startPageId
+ typeTraits[1] = new TypeTrait(4);
+ // endPageId
+ typeTraits[2] = new TypeTrait(4);
+ // startOff
+ typeTraits[3] = new TypeTrait(4);
+ // numElements
+ typeTraits[4] = new TypeTrait(4);
- IFreePageManager freePageManager = new LinkedListFreePageManager(bufferCache, btreeFileId, 0, metaFrameFactory);
-
- BTree btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, btreeCmp);
- btree.create(btreeFileId, leafFrame, metaFrame);
- btree.open(btreeFileId);
-
- int invListFields = 1;
- ITypeTrait[] invListTypeTraits = new ITypeTrait[invListFields];
- invListTypeTraits[0] = new TypeTrait(4);
-
- int invListKeys = 1;
- IBinaryComparator[] invListBinCmps = new IBinaryComparator[invListKeys];
- invListBinCmps[0] = IntegerBinaryComparatorFactory.INSTANCE.createBinaryComparator();
-
- MultiComparator invListCmp = new MultiComparator(invListTypeTraits, invListBinCmps);
-
- InvertedIndex invIndex = new InvertedIndex(bufferCache, btree, invListCmp);
- invIndex.open(invListsFileId);
-
- Random rnd = new Random();
- rnd.setSeed(50);
+ // declare btree keys
+ int keyFieldCount = 1;
+ IBinaryComparator[] cmps = new IBinaryComparator[keyFieldCount];
+ cmps[0] = UTF8StringBinaryComparatorFactory.INSTANCE
+ .createBinaryComparator();
- ByteBuffer frame = stageletCtx.allocateFrame();
- FrameTupleAppender appender = new FrameTupleAppender(stageletCtx.getFrameSize());
- ArrayTupleBuilder tb = new ArrayTupleBuilder(2);
- DataOutput dos = tb.getDataOutput();
+ MultiComparator btreeCmp = new MultiComparator(typeTraits, cmps);
- ISerializerDeserializer[] insertSerde = { UTF8StringSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE };
- RecordDescriptor insertRecDesc = new RecordDescriptor(insertSerde);
- IFrameTupleAccessor accessor = new FrameTupleAccessor(stageletCtx.getFrameSize(), insertRecDesc);
- accessor.reset(frame);
- FrameTupleReference tuple = new FrameTupleReference();
+ TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(
+ typeTraits);
+ ITreeIndexFrameFactory leafFrameFactory = new BTreeNSMLeafFrameFactory(
+ tupleWriterFactory);
+ ITreeIndexFrameFactory interiorFrameFactory = new BTreeNSMInteriorFrameFactory(
+ tupleWriterFactory);
+ ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
- List<String> tokens = new ArrayList<String>();
- tokens.add("compilers");
- tokens.add("computer");
- tokens.add("databases");
- tokens.add("fast");
- tokens.add("hyracks");
- tokens.add("major");
- tokens.add("science");
- tokens.add("systems");
- tokens.add("university");
-
- ArrayList<ArrayList<Integer>> checkListElements = new ArrayList<ArrayList<Integer>>();
- for(int i = 0; i < tokens.size(); i++) {
- checkListElements.add(new ArrayList<Integer>());
- }
-
- int maxId = 1000000;
- int addProb = 0;
- int addProbStep = 10;
+ ITreeIndexFrame leafFrame = leafFrameFactory.createFrame();
+ ITreeIndexFrame interiorFrame = interiorFrameFactory.createFrame();
+ ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
- IInvertedListBuilder invListBuilder = new FixedSizeElementInvertedListBuilder(invListTypeTraits);
- InvertedIndex.BulkLoadContext ctx = invIndex.beginBulkLoad(invListBuilder, HYRACKS_FRAME_SIZE, BTree.DEFAULT_FILL_FACTOR);
-
- int totalElements = 0;
- for (int i = 0; i < tokens.size(); i++) {
+ IFreePageManager freePageManager = new LinkedListFreePageManager(
+ bufferCache, btreeFileId, 0, metaFrameFactory);
- addProb += addProbStep * (i+1);
- for (int j = 0; j < maxId; j++) {
- if ((Math.abs(rnd.nextInt()) % addProb) == 0) {
-
- totalElements++;
-
- tb.reset();
- UTF8StringSerializerDeserializer.INSTANCE.serialize(tokens.get(i), dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(j, dos);
- tb.addFieldEndOffset();
+ BTree btree = new BTree(bufferCache, freePageManager,
+ interiorFrameFactory, leafFrameFactory, btreeCmp);
+ btree.create(btreeFileId, leafFrame, metaFrame);
+ btree.open(btreeFileId);
- checkListElements.get(i).add(j);
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- try {
- invIndex.bulkLoadAddTuple(ctx, tuple);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- }
- }
- invIndex.endBulkLoad(ctx);
-
- // ------- START VERIFICATION -----------
-
- ITreeIndexCursor btreeCursor = new BTreeRangeSearchCursor((IBTreeLeafFrame)leafFrame);
- FrameTupleReference searchKey = new FrameTupleReference();
- RangePredicate btreePred = new RangePredicate(true, searchKey, searchKey, true, true, btreeCmp, btreeCmp);
-
- IInvertedListCursor invListCursor = new FixedSizeElementInvertedListCursor(bufferCache, invListsFileId, invListTypeTraits);
-
- ISerializerDeserializer[] tokenSerde = { UTF8StringSerializerDeserializer.INSTANCE};
- RecordDescriptor tokenRecDesc = new RecordDescriptor(tokenSerde);
- FrameTupleAppender tokenAppender = new FrameTupleAppender(stageletCtx.getFrameSize());
- ArrayTupleBuilder tokenTupleBuilder = new ArrayTupleBuilder(1);
- DataOutput tokenDos = tokenTupleBuilder.getDataOutput();
- IFrameTupleAccessor tokenAccessor = new FrameTupleAccessor(stageletCtx.getFrameSize(), tokenRecDesc);
- tokenAccessor.reset(frame);
-
- BTreeOpContext btreeOpCtx = invIndex.getBTree().createOpContext(IndexOp.SEARCH, leafFrame,
- interiorFrame, null);
-
- // verify created inverted lists one-by-one
- for(int i = 0; i < tokens.size(); i++) {
-
- tokenTupleBuilder.reset();
- UTF8StringSerializerDeserializer.INSTANCE.serialize(tokens.get(i), tokenDos);
- tokenTupleBuilder.addFieldEndOffset();
-
- tokenAppender.reset(frame, true);
- tokenAppender.append(tokenTupleBuilder.getFieldEndOffsets(), tokenTupleBuilder.getByteArray(), 0, tokenTupleBuilder.getSize());
-
- searchKey.reset(tokenAccessor, 0);
-
- invIndex.openCursor(btreeCursor, btreePred, btreeOpCtx, invListCursor);
-
- invListCursor.pinPagesSync();
- int checkIndex = 0;
- while(invListCursor.hasNext()) {
- invListCursor.next();
- ITupleReference invListTuple = invListCursor.getTuple();
- int invListElement = IntegerSerializerDeserializer.getInt(invListTuple.getFieldData(0), invListTuple.getFieldStart(0));
- int checkInvListElement = checkListElements.get(i).get(checkIndex).intValue();
- Assert.assertEquals(invListElement, checkInvListElement);
- checkIndex++;
- }
- invListCursor.unpinPages();
- Assert.assertEquals(checkIndex, checkListElements.get(i).size());
- }
-
- // check that non-existing tokens have an empty inverted list
- List<String> nonExistingTokens = new ArrayList<String>();
- nonExistingTokens.add("watermelon");
- nonExistingTokens.add("avocado");
- nonExistingTokens.add("lemon");
-
- for(int i = 0; i < nonExistingTokens.size(); i++) {
-
- tokenTupleBuilder.reset();
- UTF8StringSerializerDeserializer.INSTANCE.serialize(nonExistingTokens.get(i), tokenDos);
- tokenTupleBuilder.addFieldEndOffset();
-
- tokenAppender.reset(frame, true);
- tokenAppender.append(tokenTupleBuilder.getFieldEndOffsets(), tokenTupleBuilder.getByteArray(), 0, tokenTupleBuilder.getSize());
-
- searchKey.reset(tokenAccessor, 0);
-
- invIndex.openCursor(btreeCursor, btreePred, btreeOpCtx, invListCursor);
-
- invListCursor.pinPagesSync();
- Assert.assertEquals(invListCursor.hasNext(), false);
- invListCursor.unpinPages();
- }
-
- btree.close();
- bufferCache.closeFile(btreeFileId);
- bufferCache.closeFile(invListsFileId);
- bufferCache.close();
- }
-
- @AfterClass
- public static void deinit() {
- AbstractInvIndexTest.tearDown();
- }
+ int invListFields = 1;
+ ITypeTrait[] invListTypeTraits = new ITypeTrait[invListFields];
+ invListTypeTraits[0] = new TypeTrait(4);
+
+ int invListKeys = 1;
+ IBinaryComparator[] invListBinCmps = new IBinaryComparator[invListKeys];
+ invListBinCmps[0] = IntegerBinaryComparatorFactory.INSTANCE
+ .createBinaryComparator();
+
+ MultiComparator invListCmp = new MultiComparator(invListTypeTraits,
+ invListBinCmps);
+
+ InvertedIndex invIndex = new InvertedIndex(bufferCache, btree,
+ invListCmp);
+ invIndex.open(invListsFileId);
+
+ Random rnd = new Random();
+ rnd.setSeed(50);
+
+ ByteBuffer frame = stageletCtx.allocateFrame();
+ FrameTupleAppender appender = new FrameTupleAppender(
+ stageletCtx.getFrameSize());
+ ArrayTupleBuilder tb = new ArrayTupleBuilder(2);
+ DataOutput dos = tb.getDataOutput();
+
+ ISerializerDeserializer[] insertSerde = {
+ UTF8StringSerializerDeserializer.INSTANCE,
+ IntegerSerializerDeserializer.INSTANCE };
+ RecordDescriptor insertRecDesc = new RecordDescriptor(insertSerde);
+ IFrameTupleAccessor accessor = new FrameTupleAccessor(
+ stageletCtx.getFrameSize(), insertRecDesc);
+ accessor.reset(frame);
+ FrameTupleReference tuple = new FrameTupleReference();
+
+ List<String> tokens = new ArrayList<String>();
+ tokens.add("compilers");
+ tokens.add("computer");
+ tokens.add("databases");
+ tokens.add("fast");
+ tokens.add("hyracks");
+ tokens.add("major");
+ tokens.add("science");
+ tokens.add("systems");
+ tokens.add("university");
+
+ ArrayList<ArrayList<Integer>> checkListElements = new ArrayList<ArrayList<Integer>>();
+ for (int i = 0; i < tokens.size(); i++) {
+ checkListElements.add(new ArrayList<Integer>());
+ }
+
+ int maxId = 1000000;
+ int addProb = 0;
+ int addProbStep = 10;
+
+ IInvertedListBuilder invListBuilder = new FixedSizeElementInvertedListBuilder(
+ invListTypeTraits);
+ InvertedIndex.BulkLoadContext ctx = invIndex.beginBulkLoad(
+ invListBuilder, HYRACKS_FRAME_SIZE, BTree.DEFAULT_FILL_FACTOR);
+
+ int totalElements = 0;
+ for (int i = 0; i < tokens.size(); i++) {
+
+ addProb += addProbStep * (i + 1);
+ for (int j = 0; j < maxId; j++) {
+ if ((Math.abs(rnd.nextInt()) % addProb) == 0) {
+
+ totalElements++;
+
+ tb.reset();
+ UTF8StringSerializerDeserializer.INSTANCE.serialize(
+ tokens.get(i), dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(j, dos);
+ tb.addFieldEndOffset();
+
+ checkListElements.get(i).add(j);
+
+ appender.reset(frame, true);
+ appender.append(tb.getFieldEndOffsets(), tb.getByteArray(),
+ 0, tb.getSize());
+
+ tuple.reset(accessor, 0);
+
+ try {
+ invIndex.bulkLoadAddTuple(ctx, tuple);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+ invIndex.endBulkLoad(ctx);
+
+ // ------- START VERIFICATION -----------
+
+ ITreeIndexCursor btreeCursor = new BTreeRangeSearchCursor(
+ (IBTreeLeafFrame) leafFrame);
+ FrameTupleReference searchKey = new FrameTupleReference();
+ RangePredicate btreePred = new RangePredicate(true, searchKey,
+ searchKey, true, true, btreeCmp, btreeCmp);
+
+ IInvertedListCursor invListCursor = new FixedSizeElementInvertedListCursor(
+ bufferCache, invListsFileId, invListTypeTraits);
+
+ ISerializerDeserializer[] tokenSerde = { UTF8StringSerializerDeserializer.INSTANCE };
+ RecordDescriptor tokenRecDesc = new RecordDescriptor(tokenSerde);
+ FrameTupleAppender tokenAppender = new FrameTupleAppender(
+ stageletCtx.getFrameSize());
+ ArrayTupleBuilder tokenTupleBuilder = new ArrayTupleBuilder(1);
+ DataOutput tokenDos = tokenTupleBuilder.getDataOutput();
+ IFrameTupleAccessor tokenAccessor = new FrameTupleAccessor(
+ stageletCtx.getFrameSize(), tokenRecDesc);
+ tokenAccessor.reset(frame);
+
+ BTreeOpContext btreeOpCtx = invIndex.getBTree().createOpContext(
+ IndexOp.SEARCH, leafFrame, interiorFrame, null);
+
+ // verify created inverted lists one-by-one
+ for (int i = 0; i < tokens.size(); i++) {
+
+ tokenTupleBuilder.reset();
+ UTF8StringSerializerDeserializer.INSTANCE.serialize(tokens.get(i),
+ tokenDos);
+ tokenTupleBuilder.addFieldEndOffset();
+
+ tokenAppender.reset(frame, true);
+ tokenAppender.append(tokenTupleBuilder.getFieldEndOffsets(),
+ tokenTupleBuilder.getByteArray(), 0,
+ tokenTupleBuilder.getSize());
+
+ searchKey.reset(tokenAccessor, 0);
+
+ invIndex.openCursor(btreeCursor, btreePred, btreeOpCtx,
+ invListCursor);
+
+ invListCursor.pinPagesSync();
+ int checkIndex = 0;
+ while (invListCursor.hasNext()) {
+ invListCursor.next();
+ ITupleReference invListTuple = invListCursor.getTuple();
+ int invListElement = IntegerSerializerDeserializer.getInt(
+ invListTuple.getFieldData(0),
+ invListTuple.getFieldStart(0));
+ int checkInvListElement = checkListElements.get(i)
+ .get(checkIndex).intValue();
+ Assert.assertEquals(invListElement, checkInvListElement);
+ checkIndex++;
+ }
+ invListCursor.unpinPages();
+ Assert.assertEquals(checkIndex, checkListElements.get(i).size());
+ }
+
+ // check that non-existing tokens have an empty inverted list
+ List<String> nonExistingTokens = new ArrayList<String>();
+ nonExistingTokens.add("watermelon");
+ nonExistingTokens.add("avocado");
+ nonExistingTokens.add("lemon");
+
+ for (int i = 0; i < nonExistingTokens.size(); i++) {
+
+ tokenTupleBuilder.reset();
+ UTF8StringSerializerDeserializer.INSTANCE.serialize(
+ nonExistingTokens.get(i), tokenDos);
+ tokenTupleBuilder.addFieldEndOffset();
+
+ tokenAppender.reset(frame, true);
+ tokenAppender.append(tokenTupleBuilder.getFieldEndOffsets(),
+ tokenTupleBuilder.getByteArray(), 0,
+ tokenTupleBuilder.getSize());
+
+ searchKey.reset(tokenAccessor, 0);
+
+ invIndex.openCursor(btreeCursor, btreePred, btreeOpCtx,
+ invListCursor);
+
+ invListCursor.pinPagesSync();
+ Assert.assertEquals(invListCursor.hasNext(), false);
+ invListCursor.unpinPages();
+ }
+
+ btree.close();
+ bufferCache.closeFile(btreeFileId);
+ bufferCache.closeFile(invListsFileId);
+ bufferCache.close();
+ }
+
+ @AfterClass
+ public static void deinit() {
+ AbstractInvIndexTest.tearDown();
+ }
}
diff --git a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/FixedSizeFrameTupleTest.java b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/FixedSizeFrameTupleTest.java
index d48f181..0b49ec1 100644
--- a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/FixedSizeFrameTupleTest.java
+++ b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/FixedSizeFrameTupleTest.java
@@ -1,3 +1,18 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package edu.uci.ics.hyracks.storage.am.invertedindex;
import java.nio.ByteBuffer;
@@ -17,37 +32,47 @@
public class FixedSizeFrameTupleTest {
private static int FRAME_SIZE = 4096;
-
+
private Random rnd = new Random(50);
-
+
+ /**
+ * This test verifies the correct behavior of the FixedSizeFrameTuple class.
+ * Frames containing FixedSizeFrameTuple's require neither tuple slots nor
+ * field slots. The tests inserts generated data into a frame until the
+ * frame is full, and then verifies the frame's contents.
+ *
+ */
@Test
- public void singleFieldTest() throws Exception {
+ public void singleFieldTest() throws Exception {
ByteBuffer buffer = ByteBuffer.allocate(FRAME_SIZE);
-
+
ITypeTrait[] fields = new TypeTrait[1];
fields[0] = new TypeTrait(4);
-
- FixedSizeFrameTupleAppender ftapp = new FixedSizeFrameTupleAppender(FRAME_SIZE, fields);
- FixedSizeFrameTupleAccessor ftacc = new FixedSizeFrameTupleAccessor(FRAME_SIZE, fields);
-
- boolean frameHasSpace = true;
-
+
+ FixedSizeFrameTupleAppender ftapp = new FixedSizeFrameTupleAppender(
+ FRAME_SIZE, fields);
+ FixedSizeFrameTupleAccessor ftacc = new FixedSizeFrameTupleAccessor(
+ FRAME_SIZE, fields);
+
+ boolean frameHasSpace = true;
+
ArrayList<Integer> check = new ArrayList<Integer>();
-
+
ftapp.reset(buffer, true);
- while(frameHasSpace) {
+ while (frameHasSpace) {
int val = rnd.nextInt();
frameHasSpace = ftapp.append(val);
- if(frameHasSpace) {
+ if (frameHasSpace) {
check.add(val);
ftapp.incrementTupleCount(1);
}
}
-
- ftacc.reset(buffer);
- for(int i = 0; i < ftacc.getTupleCount(); i++) {
- int val = IntegerSerializerDeserializer.getInt(ftacc.getBuffer().array(), ftacc.getTupleStartOffset(i));
+
+ ftacc.reset(buffer);
+ for (int i = 0; i < ftacc.getTupleCount(); i++) {
+ int val = IntegerSerializerDeserializer.getInt(ftacc.getBuffer()
+ .array(), ftacc.getTupleStartOffset(i));
Assert.assertEquals(check.get(i).intValue(), val);
- }
+ }
}
}
diff --git a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchPerfTest.java b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchPerfTest.java
new file mode 100644
index 0000000..4d3942f
--- /dev/null
+++ b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchPerfTest.java
@@ -0,0 +1,296 @@
+/*
+ * Copyright 2009-2010 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.hyracks.storage.am.invertedindex;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Random;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import edu.uci.ics.fuzzyjoin.tokenizer.DelimitedUTF8StringBinaryTokenizer;
+import edu.uci.ics.fuzzyjoin.tokenizer.IBinaryTokenizer;
+import edu.uci.ics.fuzzyjoin.tokenizer.ITokenFactory;
+import edu.uci.ics.fuzzyjoin.tokenizer.UTF8WordTokenFactory;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
+import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifier;
+import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListBuilder;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.FixedSizeElementInvertedListBuilder;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.InvertedIndex;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.OccurrenceThresholdPanicException;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.SearchResultCursor;
+import edu.uci.ics.hyracks.storage.am.invertedindex.impls.TOccurrenceSearcher;
+import edu.uci.ics.hyracks.storage.am.invertedindex.searchmodifiers.ConjunctiveSearchModifier;
+import edu.uci.ics.hyracks.storage.am.invertedindex.searchmodifiers.JaccardSearchModifier;
+
+/**
+ * The purpose of this test is to evaluate the performance of searches against
+ * an inverted index. First, we generate random <token, id> pairs sorted on
+ * token, which are bulk loaded into an inverted index. Next, we build random
+ * queries from a list of predefined tokens in the index, and measure the
+ * performance of executing them with different search modifiers. We test the
+ * ConjunctiveSearchModifier and the JaccardSearchModifier.
+ *
+ */
+public class SearchPerfTest extends AbstractInvIndexSearchTest {
+
+ protected List<String> tokens = new ArrayList<String>();
+
+ @Before
+ public void start() throws Exception {
+ super.start();
+ tokenFactory = new UTF8WordTokenFactory();
+ tokenizer = new DelimitedUTF8StringBinaryTokenizer(true, false,
+ tokenFactory);
+ searcher = new TOccurrenceSearcher(stageletCtx, invIndex, tokenizer);
+ resultCursor = new SearchResultCursor(
+ searcher.createResultFrameTupleAccessor(),
+ searcher.createResultTupleReference());
+ loadData();
+ }
+
+ public void loadData() throws HyracksDataException {
+ tokens.add("compilers");
+ tokens.add("computer");
+ tokens.add("databases");
+ tokens.add("fast");
+ tokens.add("hyracks");
+ tokens.add("major");
+ tokens.add("science");
+ tokens.add("systems");
+ tokens.add("university");
+
+ for (int i = 0; i < tokens.size(); i++) {
+ checkInvLists.add(new ArrayList<Integer>());
+ }
+
+ // for generating length-skewed inverted lists
+ int addProb = 0;
+ int addProbStep = 10;
+
+ IInvertedListBuilder invListBuilder = new FixedSizeElementInvertedListBuilder(
+ invListTypeTraits);
+ InvertedIndex.BulkLoadContext ctx = invIndex.beginBulkLoad(
+ invListBuilder, HYRACKS_FRAME_SIZE, BTree.DEFAULT_FILL_FACTOR);
+
+ int totalElements = 0;
+ for (int i = 0; i < tokens.size(); i++) {
+
+ addProb += addProbStep * (i + 1);
+ for (int j = 0; j < maxId; j++) {
+ if ((Math.abs(rnd.nextInt()) % addProb) == 0) {
+
+ totalElements++;
+
+ tb.reset();
+ UTF8StringSerializerDeserializer.INSTANCE.serialize(
+ tokens.get(i), dos);
+ tb.addFieldEndOffset();
+ IntegerSerializerDeserializer.INSTANCE.serialize(j, dos);
+ tb.addFieldEndOffset();
+
+ checkInvLists.get(i).add(j);
+
+ appender.reset(frame, true);
+ appender.append(tb.getFieldEndOffsets(), tb.getByteArray(),
+ 0, tb.getSize());
+
+ tuple.reset(accessor, 0);
+
+ try {
+ invIndex.bulkLoadAddTuple(ctx, tuple);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+ invIndex.endBulkLoad(ctx);
+ }
+
+ /**
+ * Determine the expected results with the ScanCount algorithm. The
+ * ScanCount algorithm is very simple, so we can be confident the results
+ * are correct.
+ *
+ */
+ protected void fillExpectedResults(int[] queryTokenIndexes,
+ int numQueryTokens, int occurrenceThreshold) {
+ // reset scan count array
+ for (int i = 0; i < maxId; i++) {
+ scanCountArray[i] = 0;
+ }
+
+ // count occurrences
+ for (int i = 0; i < numQueryTokens; i++) {
+ ArrayList<Integer> list = checkInvLists.get(queryTokenIndexes[i]);
+ for (int j = 0; j < list.size(); j++) {
+ scanCountArray[list.get(j)]++;
+ }
+ }
+
+ // check threshold
+ expectedResults.clear();
+ for (int i = 0; i < maxId; i++) {
+ if (scanCountArray[i] >= occurrenceThreshold) {
+ expectedResults.add(i);
+ }
+ }
+ }
+
+ /**
+ * Generates a specified number of queries. Each query consists of a set of
+ * randomly chosen tokens that are picked from the pre-defined set of
+ * tokens. We run each query, measure it's time, and verify it's results
+ * against the results produced by ScanCount, implemented in
+ * fillExpectedResults().
+ *
+ */
+ private void runQueries(IInvertedIndexSearchModifier searchModifier,
+ int numQueries) throws Exception {
+
+ rnd.setSeed(50);
+
+ // generate random queries
+ int[] queryTokenIndexes = new int[tokens.size()];
+ for (int i = 0; i < numQueries; i++) {
+
+ int numQueryTokens = Math.abs(rnd.nextInt() % tokens.size()) + 1;
+ for (int j = 0; j < numQueryTokens; j++) {
+ queryTokenIndexes[j] = Math.abs(rnd.nextInt() % tokens.size());
+ }
+
+ StringBuilder strBuilder = new StringBuilder();
+ for (int j = 0; j < numQueryTokens; j++) {
+ strBuilder.append(tokens.get(queryTokenIndexes[j]));
+ if (j + 1 != numQueryTokens)
+ strBuilder.append(" ");
+ }
+
+ String queryString = strBuilder.toString();
+
+ queryTb.reset();
+ UTF8StringSerializerDeserializer.INSTANCE.serialize(queryString,
+ queryDos);
+ queryTb.addFieldEndOffset();
+
+ queryAppender.reset(frame, true);
+ queryAppender.append(queryTb.getFieldEndOffsets(),
+ queryTb.getByteArray(), 0, queryTb.getSize());
+ queryTuple.reset(queryAccessor, 0);
+
+ boolean panic = false;
+
+ int repeats = 1;
+ double totalTime = 0;
+ for (int j = 0; j < repeats; j++) {
+ long timeStart = System.currentTimeMillis();
+ try {
+ searcher.reset();
+ searcher.search(resultCursor, queryTuple, 0, searchModifier);
+ } catch (OccurrenceThresholdPanicException e) {
+ panic = true;
+ }
+ long timeEnd = System.currentTimeMillis();
+ totalTime += timeEnd - timeStart;
+ }
+ double avgTime = totalTime / (double) repeats;
+ LOGGER.info(i + ": " + "\"" + queryString + "\": " + avgTime + "ms");
+
+ if (!panic) {
+
+ fillExpectedResults(queryTokenIndexes, numQueryTokens,
+ searcher.getOccurrenceThreshold());
+
+ // verify results
+ int checkIndex = 0;
+ while (resultCursor.hasNext()) {
+ resultCursor.next();
+ ITupleReference resultTuple = resultCursor.getTuple();
+ int id = IntegerSerializerDeserializer.getInt(
+ resultTuple.getFieldData(0),
+ resultTuple.getFieldStart(0));
+ Assert.assertEquals(expectedResults.get(checkIndex)
+ .intValue(), id);
+ checkIndex++;
+ }
+
+ if (expectedResults.size() != checkIndex) {
+ LOGGER.info("CHECKING");
+ StringBuilder expectedStrBuilder = new StringBuilder();
+ for (Integer x : expectedResults) {
+ expectedStrBuilder.append(x + " ");
+ }
+ LOGGER.info(expectedStrBuilder.toString());
+ }
+
+ Assert.assertEquals(expectedResults.size(), checkIndex);
+ }
+ }
+ }
+
+ /**
+ * Runs 50 random conjunctive search queries to test the
+ * ConjunctiveSearchModifier.
+ *
+ */
+ @Test
+ public void conjunctiveKeywordQueryTest() throws Exception {
+ IInvertedIndexSearchModifier searchModifier = new ConjunctiveSearchModifier();
+ runQueries(searchModifier, 50);
+ }
+
+ /**
+ * Runs 50 random jaccard-based search queries with thresholds 1.0, 0.9,
+ * 0.8, 0.7, 0.6, 0.5. Tests the JaccardSearchModifier.
+ *
+ */
+ @Test
+ public void jaccardKeywordQueryTest() throws Exception {
+ JaccardSearchModifier searchModifier = new JaccardSearchModifier(1.0f);
+
+ LOGGER.info("JACCARD: " + 1.0f);
+ searchModifier.setJaccThresh(1.0f);
+ runQueries(searchModifier, 50);
+
+ LOGGER.info("JACCARD: " + 0.9f);
+ searchModifier.setJaccThresh(0.9f);
+ runQueries(searchModifier, 50);
+
+ LOGGER.info("JACCARD: " + 0.8f);
+ searchModifier.setJaccThresh(0.8f);
+ runQueries(searchModifier, 50);
+
+ LOGGER.info("JACCARD: " + 0.7f);
+ searchModifier.setJaccThresh(0.7f);
+ runQueries(searchModifier, 50);
+
+ LOGGER.info("JACCARD: " + 0.6f);
+ searchModifier.setJaccThresh(0.6f);
+ runQueries(searchModifier, 50);
+
+ LOGGER.info("JACCARD: " + 0.5f);
+ searchModifier.setJaccThresh(0.5f);
+ runQueries(searchModifier, 50);
+ }
+}
diff --git a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchTest.java b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchTest.java
index 81569fc..376735f 100644
--- a/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchTest.java
+++ b/hyracks-tests/hyracks-storage-am-invertedindex-test/src/test/java/edu/uci/ics/hyracks/storage/am/invertedindex/SearchTest.java
@@ -1,52 +1,22 @@
package edu.uci.ics.hyracks.storage.am.invertedindex;
-import java.io.DataOutput;
-import java.io.File;
-import java.nio.ByteBuffer;
+import java.io.DataOutputStream;
+import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
-import java.util.Random;
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
+import org.junit.Before;
import org.junit.Test;
-import edu.uci.ics.fuzzyjoin.tokenizer.DelimitedUTF8StringBinaryTokenizer;
-import edu.uci.ics.fuzzyjoin.tokenizer.IBinaryTokenizer;
-import edu.uci.ics.fuzzyjoin.tokenizer.ITokenFactory;
-import edu.uci.ics.fuzzyjoin.tokenizer.UTF8WordTokenFactory;
-import edu.uci.ics.hyracks.api.comm.IFrameTupleAccessor;
-import edu.uci.ics.hyracks.api.context.IHyracksStageletContext;
-import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
-import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
-import edu.uci.ics.hyracks.api.dataflow.value.ITypeTrait;
-import edu.uci.ics.hyracks.api.dataflow.value.RecordDescriptor;
-import edu.uci.ics.hyracks.api.dataflow.value.TypeTrait;
-import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.api.io.FileReference;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAccessor;
-import edu.uci.ics.hyracks.dataflow.common.comm.io.FrameTupleAppender;
-import edu.uci.ics.hyracks.dataflow.common.data.accessors.FrameTupleReference;
+import edu.uci.ics.fuzzyjoin.tokenizer.IToken;
+import edu.uci.ics.fuzzyjoin.tokenizer.NGramUTF8StringBinaryTokenizer;
+import edu.uci.ics.fuzzyjoin.tokenizer.UTF8NGramTokenFactory;
+import edu.uci.ics.hyracks.dataflow.common.comm.io.ByteArrayAccessibleOutputStream;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.ITupleReference;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.IntegerBinaryComparatorFactory;
-import edu.uci.ics.hyracks.dataflow.common.data.comparators.UTF8StringBinaryComparatorFactory;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.UTF8StringSerializerDeserializer;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMInteriorFrameFactory;
-import edu.uci.ics.hyracks.storage.am.btree.frames.NSMLeafFrameFactory;
import edu.uci.ics.hyracks.storage.am.btree.impls.BTree;
-import edu.uci.ics.hyracks.storage.am.common.api.IFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrame;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrame;
-import edu.uci.ics.hyracks.storage.am.common.api.ITreeIndexMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.frames.LIFOMetaDataFrameFactory;
-import edu.uci.ics.hyracks.storage.am.common.freepage.LinkedListFreePageManager;
-import edu.uci.ics.hyracks.storage.am.common.ophelpers.MultiComparator;
-import edu.uci.ics.hyracks.storage.am.common.tuples.TypeAwareTupleWriterFactory;
-import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexResultCursor;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedIndexSearchModifier;
import edu.uci.ics.hyracks.storage.am.invertedindex.api.IInvertedListBuilder;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.FixedSizeElementInvertedListBuilder;
@@ -55,357 +25,252 @@
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.SearchResultCursor;
import edu.uci.ics.hyracks.storage.am.invertedindex.impls.TOccurrenceSearcher;
import edu.uci.ics.hyracks.storage.am.invertedindex.searchmodifiers.ConjunctiveSearchModifier;
+import edu.uci.ics.hyracks.storage.am.invertedindex.searchmodifiers.EditDistanceSearchModifier;
import edu.uci.ics.hyracks.storage.am.invertedindex.searchmodifiers.JaccardSearchModifier;
-import edu.uci.ics.hyracks.storage.common.buffercache.IBufferCache;
-import edu.uci.ics.hyracks.storage.common.file.IFileMapProvider;
-import edu.uci.ics.hyracks.test.support.TestStorageManagerComponentHolder;
-import edu.uci.ics.hyracks.test.support.TestUtils;
-public class SearchTest extends AbstractInvIndexTest {
-
- // private static final int PAGE_SIZE = 65536;
- private static final int PAGE_SIZE = 32768;
- private static final int NUM_PAGES = 100;
- private static final int MAX_OPEN_FILES = 10;
- private static final int HYRACKS_FRAME_SIZE = 32768;
- private static IHyracksStageletContext stageletCtx = TestUtils.create(HYRACKS_FRAME_SIZE);
-
- private static IBufferCache bufferCache;
- private static IFileMapProvider fmp;
-
- // --- BTREE ---
-
- // create file refs
- private static FileReference btreeFile = new FileReference(new File(btreeFileName));
- private static int btreeFileId;
-
- // declare btree fields
- private static int fieldCount = 5;
- private static ITypeTrait[] typeTraits = new ITypeTrait[fieldCount];
-
- // declare btree keys
- private static int btreeKeyFieldCount = 1;
- private static IBinaryComparator[] btreeBinCmps = new IBinaryComparator[btreeKeyFieldCount];
- private static MultiComparator btreeCmp = new MultiComparator(typeTraits, btreeBinCmps);
-
- // btree frame factories
- private static TypeAwareTupleWriterFactory tupleWriterFactory = new TypeAwareTupleWriterFactory(typeTraits);
- private static ITreeIndexFrameFactory leafFrameFactory = new NSMLeafFrameFactory(tupleWriterFactory);
- private static ITreeIndexFrameFactory interiorFrameFactory = new NSMInteriorFrameFactory(tupleWriterFactory);
- private static ITreeIndexMetaDataFrameFactory metaFrameFactory = new LIFOMetaDataFrameFactory();
-
- // btree frames
- private static ITreeIndexFrame leafFrame = leafFrameFactory.createFrame();
- private static ITreeIndexMetaDataFrame metaFrame = metaFrameFactory.createFrame();
-
- private static IFreePageManager freePageManager;
-
- private static BTree btree;
-
-
- // --- INVERTED INDEX ---
-
- private static FileReference invListsFile = new FileReference(new File(invListsFileName));
- private static int invListsFileId;
-
- private static int invListFields = 1;
- private static ITypeTrait[] invListTypeTraits = new ITypeTrait[invListFields];
-
- private static int invListKeys = 1;
- private static IBinaryComparator[] invListBinCmps = new IBinaryComparator[invListKeys];
- private static MultiComparator invListCmp = new MultiComparator(invListTypeTraits, invListBinCmps);
-
- private static InvertedIndex invIndex;
-
- private static Random rnd = new Random();
-
- private static ByteBuffer frame = stageletCtx.allocateFrame();
- private static FrameTupleAppender appender = new FrameTupleAppender(stageletCtx.getFrameSize());
- private static ArrayTupleBuilder tb = new ArrayTupleBuilder(2);
- private static DataOutput dos = tb.getDataOutput();
+public class SearchTest extends AbstractInvIndexSearchTest {
- private static ISerializerDeserializer[] insertSerde = { UTF8StringSerializerDeserializer.INSTANCE,
- IntegerSerializerDeserializer.INSTANCE };
- private static RecordDescriptor insertRecDesc = new RecordDescriptor(insertSerde);
- private static IFrameTupleAccessor accessor = new FrameTupleAccessor(stageletCtx.getFrameSize(), insertRecDesc);
-
- private static FrameTupleReference tuple = new FrameTupleReference();
-
+ protected List<String> dataStrings = new ArrayList<String>();
+ protected List<String> firstNames = new ArrayList<String>();
+ protected List<String> lastNames = new ArrayList<String>();
- private static List<String> tokens = new ArrayList<String>();
- private static ArrayList<ArrayList<Integer>> checkInvLists = new ArrayList<ArrayList<Integer>>();
-
- private static int maxId = 1000000;
- //private static int maxId = 1000;
- private static int[] scanCountArray = new int[maxId];
- private static ArrayList<Integer> expectedResults = new ArrayList<Integer>();
-
- private static ISerializerDeserializer[] querySerde = { UTF8StringSerializerDeserializer.INSTANCE };
- private static RecordDescriptor queryRecDesc = new RecordDescriptor(querySerde);
+ @Before
+ public void start() throws Exception {
+ super.start();
+ tokenFactory = new UTF8NGramTokenFactory();
+ tokenizer = new NGramUTF8StringBinaryTokenizer(3, false, true, false,
+ tokenFactory);
+ searcher = new TOccurrenceSearcher(stageletCtx, invIndex, tokenizer);
+ resultCursor = new SearchResultCursor(
+ searcher.createResultFrameTupleAccessor(),
+ searcher.createResultTupleReference());
+ generateDataStrings();
+ loadData();
+ }
- private static FrameTupleAppender queryAppender = new FrameTupleAppender(stageletCtx.getFrameSize());
- private static ArrayTupleBuilder queryTb = new ArrayTupleBuilder(querySerde.length);
- private static DataOutput queryDos = queryTb.getDataOutput();
+ public void generateDataStrings() {
+ firstNames.add("Kathrin");
+ firstNames.add("Cathrin");
+ firstNames.add("Kathryn");
+ firstNames.add("Cathryn");
+ firstNames.add("Kathrine");
+ firstNames.add("Cathrine");
+ firstNames.add("Kathryne");
+ firstNames.add("Cathryne");
+ firstNames.add("Katherin");
+ firstNames.add("Catherin");
+ firstNames.add("Katheryn");
+ firstNames.add("Catheryn");
+ firstNames.add("Katherine");
+ firstNames.add("Catherine");
+ firstNames.add("Katheryne");
+ firstNames.add("Catheryne");
+ firstNames.add("John");
+ firstNames.add("Jack");
+ firstNames.add("Jonathan");
+ firstNames.add("Nathan");
- private static IFrameTupleAccessor queryAccessor = new FrameTupleAccessor(stageletCtx.getFrameSize(), queryRecDesc);
- private static FrameTupleReference queryTuple = new FrameTupleReference();
-
- private static ITokenFactory tokenFactory = new UTF8WordTokenFactory();
- private static IBinaryTokenizer queryTokenizer = new DelimitedUTF8StringBinaryTokenizer(true, false, tokenFactory);
-
- private static TOccurrenceSearcher searcher;
- private static IInvertedIndexResultCursor resultCursor;
-
- @BeforeClass
- public static void start() throws Exception {
- TestStorageManagerComponentHolder.init(PAGE_SIZE, NUM_PAGES, MAX_OPEN_FILES);
- bufferCache = TestStorageManagerComponentHolder.getBufferCache(stageletCtx);
- fmp = TestStorageManagerComponentHolder.getFileMapProvider(stageletCtx);
-
- // --- BTREE ---
-
- bufferCache.createFile(btreeFile);
- btreeFileId = fmp.lookupFileId(btreeFile);
- bufferCache.openFile(btreeFileId);
+ lastNames.add("Miller");
+ lastNames.add("Myller");
+ lastNames.add("Keller");
+ lastNames.add("Ketler");
+ lastNames.add("Muller");
+ lastNames.add("Fuller");
+ lastNames.add("Smith");
+ lastNames.add("Smyth");
+ lastNames.add("Smithe");
+ lastNames.add("Smythe");
- // token (key)
- typeTraits[0] = new TypeTrait(ITypeTrait.VARIABLE_LENGTH);
- // startPageId
- typeTraits[1] = new TypeTrait(4);
- // endPageId
- typeTraits[2] = new TypeTrait(4);
- // startOff
- typeTraits[3] = new TypeTrait(4);
- // numElements
- typeTraits[4] = new TypeTrait(4);
-
- btreeBinCmps[0] = UTF8StringBinaryComparatorFactory.INSTANCE.createBinaryComparator();
-
- freePageManager = new LinkedListFreePageManager(bufferCache, btreeFileId, 0, metaFrameFactory);
-
- btree = new BTree(bufferCache, freePageManager, interiorFrameFactory, leafFrameFactory, btreeCmp);
- btree.create(btreeFileId, leafFrame, metaFrame);
- btree.open(btreeFileId);
-
-
- // --- INVERTED INDEX ---
-
- bufferCache.createFile(invListsFile);
- invListsFileId = fmp.lookupFileId(invListsFile);
- bufferCache.openFile(invListsFileId);
-
- invListTypeTraits[0] = new TypeTrait(4);
- invListBinCmps[0] = IntegerBinaryComparatorFactory.INSTANCE.createBinaryComparator();
-
- invIndex = new InvertedIndex(bufferCache, btree, invListCmp);
- invIndex.open(invListsFileId);
-
- searcher = new TOccurrenceSearcher(stageletCtx, invIndex, queryTokenizer);
- resultCursor = new SearchResultCursor(searcher.createResultFrameTupleAccessor(), searcher.createResultTupleReference());
-
- rnd.setSeed(50);
-
- accessor.reset(frame);
- queryAccessor.reset(frame);
-
- loadData();
- }
-
- private static void loadData() throws HyracksDataException {
- tokens.add("compilers");
- tokens.add("computer");
- tokens.add("databases");
- tokens.add("fast");
- tokens.add("hyracks");
- tokens.add("major");
- tokens.add("science");
- tokens.add("systems");
- tokens.add("university");
-
- for(int i = 0; i < tokens.size(); i++) {
- checkInvLists.add(new ArrayList<Integer>());
- }
-
- // for generating length-skewed inverted lists
- int addProb = 0;
- int addProbStep = 10;
-
- IInvertedListBuilder invListBuilder = new FixedSizeElementInvertedListBuilder(invListTypeTraits);
- InvertedIndex.BulkLoadContext ctx = invIndex.beginBulkLoad(invListBuilder, HYRACKS_FRAME_SIZE, BTree.DEFAULT_FILL_FACTOR);
-
- int totalElements = 0;
- for (int i = 0; i < tokens.size(); i++) {
+ // Generate all 'firstName lastName' combinations as data strings
+ for (String f : firstNames) {
+ for (String l : lastNames) {
+ dataStrings.add(f + " " + l);
+ }
+ }
+ }
- addProb += addProbStep * (i+1);
- for (int j = 0; j < maxId; j++) {
- if ((Math.abs(rnd.nextInt()) % addProb) == 0) {
-
- totalElements++;
-
- tb.reset();
- UTF8StringSerializerDeserializer.INSTANCE.serialize(tokens.get(i), dos);
- tb.addFieldEndOffset();
- IntegerSerializerDeserializer.INSTANCE.serialize(j, dos);
- tb.addFieldEndOffset();
-
- checkInvLists.get(i).add(j);
-
- appender.reset(frame, true);
- appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0, tb.getSize());
-
- tuple.reset(accessor, 0);
-
- try {
- invIndex.bulkLoadAddTuple(ctx, tuple);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- }
- }
- invIndex.endBulkLoad(ctx);
- }
-
- private void fillExpectedResults(int[] queryTokenIndexes, int numQueryTokens, int occurrenceThreshold) {
- // reset scan count array
- for(int i = 0; i < maxId; i++) {
- scanCountArray[i] = 0;
- }
-
- // count occurrences
- for(int i = 0; i < numQueryTokens; i++) {
- ArrayList<Integer> list = checkInvLists.get(queryTokenIndexes[i]);
- for(int j = 0; j < list.size(); j++) {
- scanCountArray[list.get(j)]++;
- }
- }
-
- // check threshold
- expectedResults.clear();
- for(int i = 0; i < maxId; i++) {
- if(scanCountArray[i] >= occurrenceThreshold) {
- expectedResults.add(i);
- }
- }
- }
-
- private void runQueries(IInvertedIndexSearchModifier searchModifier) throws Exception {
-
- rnd.setSeed(50);
-
- // generate random queries
- int queries = 50;
- int[] queryTokenIndexes = new int[tokens.size()];
- for(int i = 0; i < queries; i++) {
-
- int numQueryTokens = Math.abs(rnd.nextInt() % tokens.size()) + 1;
- for(int j = 0; j < numQueryTokens; j++) {
- queryTokenIndexes[j] = Math.abs(rnd.nextInt() % tokens.size());
- }
-
- StringBuilder strBuilder = new StringBuilder();
- for(int j = 0; j < numQueryTokens; j++) {
- strBuilder.append(tokens.get(queryTokenIndexes[j]));
- if(j+1 != numQueryTokens) strBuilder.append(" ");
- }
-
- String queryString = strBuilder.toString();
-
- queryTb.reset();
- UTF8StringSerializerDeserializer.INSTANCE.serialize(queryString, queryDos);
- queryTb.addFieldEndOffset();
+ private class TokenIdPair implements Comparable<TokenIdPair> {
+ public ByteArrayAccessibleOutputStream baaos = new ByteArrayAccessibleOutputStream();
+ public DataOutputStream dos = new DataOutputStream(baaos);
+ public int id;
- queryAppender.reset(frame, true);
- queryAppender.append(queryTb.getFieldEndOffsets(), queryTb.getByteArray(), 0, queryTb.getSize());
- queryTuple.reset(queryAccessor, 0);
-
- boolean panic = false;
-
- int repeats = 1;
- double totalTime = 0;
- for(int j = 0; j < repeats; j++) {
- long timeStart = System.currentTimeMillis();
- try {
- searcher.reset();
- searcher.search(resultCursor, queryTuple, 0, searchModifier);
- } catch(OccurrenceThresholdPanicException e) {
- panic = true;
- }
- long timeEnd = System.currentTimeMillis();
- totalTime += timeEnd - timeStart;
- }
- double avgTime = totalTime / (double)repeats;
- System.out.println(i + ": " + "\"" + queryString + "\": " + avgTime + "ms");
-
- if(!panic) {
+ TokenIdPair(IToken token, int id) throws IOException {
+ token.serializeToken(dos);
+ this.id = id;
+ }
- fillExpectedResults(queryTokenIndexes, numQueryTokens, searcher.getOccurrenceThreshold());
-
- // verify results
- int checkIndex = 0;
- while(resultCursor.hasNext()) {
- resultCursor.next();
- ITupleReference resultTuple = resultCursor.getTuple();
- int id = IntegerSerializerDeserializer.getInt(resultTuple.getFieldData(0), resultTuple.getFieldStart(0));
- Assert.assertEquals(expectedResults.get(checkIndex).intValue(), id);
- checkIndex++;
- }
+ @Override
+ public int compareTo(TokenIdPair o) {
+ int cmp = btreeBinCmps[0].compare(baaos.getByteArray(), 0,
+ baaos.getByteArray().length, o.baaos.getByteArray(), 0,
+ o.baaos.getByteArray().length);
+ if (cmp == 0) {
+ return id - o.id;
+ } else {
+ return cmp;
+ }
+ }
+ }
- if(expectedResults.size() != checkIndex) {
- System.out.println("CHECKING");
- for(Integer x : expectedResults) {
- System.out.print(x + " ");
- }
- System.out.println();
- }
+ public void loadData() throws IOException {
+ List<TokenIdPair> pairs = new ArrayList<TokenIdPair>();
+ // generate pairs for subsequent sorting and bulk-loading
+ int id = 0;
+ for (String s : dataStrings) {
+ ByteArrayAccessibleOutputStream baaos = new ByteArrayAccessibleOutputStream();
+ DataOutputStream dos = new DataOutputStream(baaos);
+ UTF8StringSerializerDeserializer.INSTANCE.serialize(s, dos);
+ tokenizer.reset(baaos.getByteArray(), 0, baaos.size());
+ int tokenCount = 0;
+ while (tokenizer.hasNext()) {
+ tokenizer.next();
+ IToken token = tokenizer.getToken();
+ pairs.add(new TokenIdPair(token, id));
+ ++tokenCount;
+ }
+ ++id;
+ }
+ Collections.sort(pairs);
- Assert.assertEquals(expectedResults.size(), checkIndex);
- }
- }
- }
-
- @Test
- public void conjunctiveKeywordQueryTest() throws Exception {
- IInvertedIndexSearchModifier searchModifier = new ConjunctiveSearchModifier();
- runQueries(searchModifier);
- }
-
- @Test
- public void jaccardKeywordQueryTest() throws Exception {
- JaccardSearchModifier searchModifier = new JaccardSearchModifier(1.0f);
-
- System.out.println("JACCARD: " + 1.0f);
- searchModifier.setJaccThresh(1.0f);
- runQueries(searchModifier);
-
- System.out.println("JACCARD: " + 0.9f);
- searchModifier.setJaccThresh(0.9f);
- runQueries(searchModifier);
-
- System.out.println("JACCARD: " + 0.8f);
- searchModifier.setJaccThresh(0.8f);
- runQueries(searchModifier);
-
- System.out.println("JACCARD: " + 0.7f);
- searchModifier.setJaccThresh(0.7f);
- runQueries(searchModifier);
-
- System.out.println("JACCARD: " + 0.6f);
- searchModifier.setJaccThresh(0.6f);
- runQueries(searchModifier);
-
- System.out.println("JACCARD: " + 0.5f);
- searchModifier.setJaccThresh(0.5f);
- runQueries(searchModifier);
- }
-
- @AfterClass
- public static void deinit() throws HyracksDataException {
- AbstractInvIndexTest.tearDown();
- btree.close();
- invIndex.close();
- bufferCache.closeFile(btreeFileId);
- bufferCache.closeFile(invListsFileId);
- bufferCache.close();
- }
+ // bulk load index
+ IInvertedListBuilder invListBuilder = new FixedSizeElementInvertedListBuilder(
+ invListTypeTraits);
+ InvertedIndex.BulkLoadContext ctx = invIndex.beginBulkLoad(
+ invListBuilder, HYRACKS_FRAME_SIZE, BTree.DEFAULT_FILL_FACTOR);
+
+ for (TokenIdPair t : pairs) {
+ tb.reset();
+ tb.addField(t.baaos.getByteArray(), 0,
+ t.baaos.getByteArray().length);
+ IntegerSerializerDeserializer.INSTANCE.serialize(t.id, dos);
+ tb.addFieldEndOffset();
+
+ appender.reset(frame, true);
+ appender.append(tb.getFieldEndOffsets(), tb.getByteArray(), 0,
+ tb.getSize());
+
+ tuple.reset(accessor, 0);
+
+ try {
+ invIndex.bulkLoadAddTuple(ctx, tuple);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ invIndex.endBulkLoad(ctx);
+ }
+
+ /**
+ * Runs a specified number of randomly picked strings from dataStrings as
+ * queries. We run each query, measure it's time, and print it's results.
+ *
+ */
+ private void runQueries(IInvertedIndexSearchModifier searchModifier,
+ int numQueries) throws Exception {
+
+ rnd.setSeed(50);
+
+ for (int i = 0; i < numQueries; i++) {
+
+ int queryIndex = Math.abs(rnd.nextInt() % dataStrings.size());
+ String queryString = dataStrings.get(queryIndex);
+
+ queryTb.reset();
+ UTF8StringSerializerDeserializer.INSTANCE.serialize(queryString,
+ queryDos);
+ queryTb.addFieldEndOffset();
+
+ queryAppender.reset(frame, true);
+ queryAppender.append(queryTb.getFieldEndOffsets(),
+ queryTb.getByteArray(), 0, queryTb.getSize());
+ queryTuple.reset(queryAccessor, 0);
+
+ int repeats = 1;
+ double totalTime = 0;
+ for (int j = 0; j < repeats; j++) {
+ long timeStart = System.currentTimeMillis();
+ try {
+ searcher.reset();
+ searcher.search(resultCursor, queryTuple, 0, searchModifier);
+ } catch (OccurrenceThresholdPanicException e) {
+ // ignore panic queries
+ }
+ long timeEnd = System.currentTimeMillis();
+ totalTime += timeEnd - timeStart;
+ }
+ double avgTime = totalTime / (double) repeats;
+ StringBuilder strBuilder = new StringBuilder();
+ strBuilder.append(i + ": " + "\"" + queryString + "\": " + avgTime
+ + "ms" + "\n");
+ strBuilder.append("CANDIDATE RESULTS:\n");
+ while (resultCursor.hasNext()) {
+ resultCursor.next();
+ ITupleReference resultTuple = resultCursor.getTuple();
+ int id = IntegerSerializerDeserializer.getInt(
+ resultTuple.getFieldData(0),
+ resultTuple.getFieldStart(0));
+ strBuilder.append(id + " " + dataStrings.get(id));
+ strBuilder.append('\n');
+ }
+ // remove trailing newline
+ strBuilder.deleteCharAt(strBuilder.length() - 1);
+ LOGGER.info(strBuilder.toString());
+ }
+ }
+
+ /**
+ * Runs 5 random conjunctive search queries to test the
+ * ConjunctiveSearchModifier.
+ *
+ */
+ @Test
+ public void conjunctiveQueryTest() throws Exception {
+ IInvertedIndexSearchModifier searchModifier = new ConjunctiveSearchModifier();
+ runQueries(searchModifier, 5);
+ }
+
+ /**
+ * Runs 5 random jaccard-based search queries with thresholds 0.9, 0.8, 0.7.
+ * Tests the JaccardSearchModifier.
+ *
+ */
+ @Test
+ public void jaccardQueryTest() throws Exception {
+ JaccardSearchModifier searchModifier = new JaccardSearchModifier(1.0f);
+
+ LOGGER.info("JACCARD: " + 0.9f);
+ searchModifier.setJaccThresh(0.9f);
+ runQueries(searchModifier, 5);
+
+ LOGGER.info("JACCARD: " + 0.8f);
+ searchModifier.setJaccThresh(0.8f);
+ runQueries(searchModifier, 5);
+
+ LOGGER.info("JACCARD: " + 0.7f);
+ searchModifier.setJaccThresh(0.7f);
+ runQueries(searchModifier, 5);
+ }
+
+ /**
+ * Runs 5 random edit-distance based search queries with thresholds 1, 2, 3.
+ * Tests the EditDistanceSearchModifier.
+ *
+ */
+ @Test
+ public void editDistanceQueryTest() throws Exception {
+ EditDistanceSearchModifier searchModifier = new EditDistanceSearchModifier(
+ 3, 0);
+
+ LOGGER.info("EDIT DISTANCE: " + 1);
+ searchModifier.setEdThresh(1);
+ runQueries(searchModifier, 5);
+
+ LOGGER.info("EDIT DISTANCE: " + 2);
+ searchModifier.setEdThresh(2);
+ runQueries(searchModifier, 5);
+
+ LOGGER.info("EDIT DISTANCE: " + 3);
+ searchModifier.setEdThresh(3);
+ runQueries(searchModifier, 5);
+ }
}