[NO ISSUE] Misc Fixes for Point MBR RTree
- user-model changes: no
- storage format changes: no
- interface changes: no
Details:
1. Fix the MBR calculation of Point MBR RTrees when finalizing the
bulkload process.
2. Fix the nullFlagBytes of Point MBR Rtrees.
3. Add unit test cases to cover Point MBR RTrees.
Change-Id: Ice24112152a2d93c7d2316b7506e6d6e81f2df44
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2911
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleReferenceForPointMBR.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleReferenceForPointMBR.java
index 1432aba..e28d5a2 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleReferenceForPointMBR.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/tuples/LSMRTreeTupleReferenceForPointMBR.java
@@ -128,7 +128,9 @@
@Override
protected int getNullFlagsBytes() {
- return BitOperationUtils.getFlagBytes(inputTotalFieldCount + (antimatterAware ? 1 : 0));
+ // stored key field count + value field count
+ return BitOperationUtils.getFlagBytes(
+ storedKeyFieldCount + inputTotalFieldCount - inputKeyFieldCount + (antimatterAware ? 1 : 0));
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
index f12f423..635fe7a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-rtree/src/main/java/org/apache/hyracks/storage/am/rtree/impls/RTree.java
@@ -1006,6 +1006,7 @@
//set next guide MBR
//if propagateBulk didnt have to do anything this may be un-necessary
if (nodeFrontiers.size() > 1 && nodeFrontiers.indexOf(n) < nodeFrontiers.size() - 1) {
+ lowerFrame = nodeFrontiers.indexOf(n) != 0 ? prevInteriorFrame : leafFrame;
lowerFrame.setPage(n.page);
((RTreeNSMFrame) lowerFrame).adjustMBR();
interiorFrameTupleWriter.writeTupleFields(((RTreeNSMFrame) lowerFrame).getMBRTuples(), 0, mbr, 0);
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/AbstractRTreeBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/AbstractRTreeBulkLoadTest.java
index 67b24a2..2d6f97c 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/AbstractRTreeBulkLoadTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/AbstractRTreeBulkLoadTest.java
@@ -30,10 +30,16 @@
public abstract class AbstractRTreeBulkLoadTest extends AbstractRTreeTestDriver {
private final RTreeTestUtils rTreeTestUtils;
+ private final boolean isPoint;
public AbstractRTreeBulkLoadTest(boolean testRstarPolicy) {
+ this(testRstarPolicy, false);
+ }
+
+ public AbstractRTreeBulkLoadTest(boolean testRstarPolicy, boolean isPoint) {
super(testRstarPolicy);
this.rTreeTestUtils = new RTreeTestUtils();
+ this.isPoint = isPoint;
}
@Override
@@ -47,9 +53,9 @@
// We assume all fieldSerdes are of the same type. Check the first
// one to determine which field types to generate.
if (fieldSerdes[0] instanceof IntegerSerializerDeserializer) {
- rTreeTestUtils.bulkLoadIntTuples(ctx, numTuplesToInsert, getRandom());
+ rTreeTestUtils.bulkLoadIntTuples(ctx, numTuplesToInsert, getRandom(), isPoint);
} else if (fieldSerdes[0] instanceof DoubleSerializerDeserializer) {
- rTreeTestUtils.bulkLoadDoubleTuples(ctx, numTuplesToInsert, getRandom());
+ rTreeTestUtils.bulkLoadDoubleTuples(ctx, numTuplesToInsert, getRandom(), isPoint);
}
rTreeTestUtils.checkScan(ctx);
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/RTreeTestUtils.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/RTreeTestUtils.java
index 165bf43..92f5055 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/RTreeTestUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/storage/am/rtree/RTreeTestUtils.java
@@ -93,8 +93,13 @@
}
}
+ public void insertDoubleTuples(IIndexTestContext ctx, int numTuples, Random rnd) throws HyracksDataException {
+ insertDoubleTuples(ctx, numTuples, rnd, false);
+ }
+
@SuppressWarnings("unchecked")
- public void insertDoubleTuples(IIndexTestContext ctx, int numTuples, Random rnd) throws Exception {
+ public void insertDoubleTuples(IIndexTestContext ctx, int numTuples, Random rnd, boolean isPoint)
+ throws HyracksDataException {
int fieldCount = ctx.getFieldCount();
int numKeyFields = ctx.getKeyFieldCount();
double[] fieldValues = new double[ctx.getFieldCount()];
@@ -104,7 +109,7 @@
double maxValue = Math.ceil(Math.pow(numTuples, 1.0 / numKeyFields));
for (int i = 0; i < numTuples; i++) {
// Set keys.
- setDoubleKeyFields(fieldValues, numKeyFields, maxValue, rnd);
+ setDoubleKeyFields(fieldValues, numKeyFields, maxValue, rnd, isPoint);
// Set values.
setDoublePayloadFields(fieldValues, numKeyFields, fieldCount);
TupleUtils.createDoubleTuple(ctx.getTupleBuilder(), ctx.getTuple(), fieldValues);
@@ -126,15 +131,20 @@
}
}
- private void setDoubleKeyFields(double[] fieldValues, int numKeyFields, double maxValue, Random rnd) {
+ private void setDoubleKeyFields(double[] fieldValues, int numKeyFields, double maxValue, Random rnd,
+ boolean isPoint) {
int maxFieldPos = numKeyFields / 2;
for (int j = 0; j < maxFieldPos; j++) {
int k = maxFieldPos + j;
double firstValue = rnd.nextDouble() % maxValue;
double secondValue;
- do {
- secondValue = rnd.nextDouble() % maxValue;
- } while (secondValue < firstValue);
+ if (isPoint) {
+ secondValue = firstValue;
+ } else {
+ do {
+ secondValue = rnd.nextDouble() % maxValue;
+ } while (secondValue < firstValue);
+ }
fieldValues[j] = firstValue;
fieldValues[k] = secondValue;
}
@@ -155,8 +165,13 @@
return checkTuple;
}
- @SuppressWarnings("unchecked")
public void bulkLoadDoubleTuples(IIndexTestContext ctx, int numTuples, Random rnd) throws Exception {
+ bulkLoadDoubleTuples(ctx, numTuples, rnd, false);
+ }
+
+ @SuppressWarnings("unchecked")
+ public void bulkLoadDoubleTuples(IIndexTestContext ctx, int numTuples, Random rnd, boolean isPoint)
+ throws HyracksDataException {
int fieldCount = ctx.getFieldCount();
int numKeyFields = ctx.getKeyFieldCount();
double[] fieldValues = new double[ctx.getFieldCount()];
@@ -164,7 +179,7 @@
Collection<CheckTuple> tmpCheckTuples = createCheckTuplesCollection();
for (int i = 0; i < numTuples; i++) {
// Set keys.
- setDoubleKeyFields(fieldValues, numKeyFields, maxValue, rnd);
+ setDoubleKeyFields(fieldValues, numKeyFields, maxValue, rnd, isPoint);
// Set values.
setDoublePayloadFields(fieldValues, numKeyFields, fieldCount);
@@ -222,22 +237,56 @@
return checkTuple;
}
- @Override
- protected void setIntKeyFields(int[] fieldValues, int numKeyFields, int maxValue, Random rnd) {
+ @SuppressWarnings("unchecked")
+ public void bulkLoadIntTuples(IIndexTestContext ctx, int numTuples, Random rnd, boolean isPoint)
+ throws HyracksDataException {
+ int fieldCount = ctx.getFieldCount();
+ int numKeyFields = ctx.getKeyFieldCount();
+ int[] fieldValues = new int[ctx.getFieldCount()];
+ int maxValue = (int) Math.ceil(Math.pow(numTuples, 1.0 / numKeyFields));
+ Collection<CheckTuple> tmpCheckTuples = createCheckTuplesCollection();
+ for (int i = 0; i < numTuples; i++) {
+ // Set keys.
+ setIntKeyFields(fieldValues, numKeyFields, maxValue, rnd, isPoint);
+ // Set values.
+ setIntPayloadFields(fieldValues, numKeyFields, fieldCount);
+
+ // Set expected values. (We also use these as the pre-sorted stream
+ // for ordered indexes bulk loading).
+ ctx.insertCheckTuple(createIntCheckTuple(fieldValues, ctx.getKeyFieldCount()), tmpCheckTuples);
+ }
+ bulkLoadCheckTuples(ctx, tmpCheckTuples, false);
+
+ // Add tmpCheckTuples to ctx check tuples for comparing searches.
+ for (CheckTuple checkTuple : tmpCheckTuples) {
+ ctx.insertCheckTuple(checkTuple, ctx.getCheckTuples());
+ }
+ }
+
+ protected void setIntKeyFields(int[] fieldValues, int numKeyFields, int maxValue, Random rnd, boolean isPoint) {
int maxFieldPos = numKeyFields / 2;
for (int j = 0; j < maxFieldPos; j++) {
int k = maxFieldPos + j;
int firstValue = rnd.nextInt() % maxValue;
int secondValue;
- do {
- secondValue = rnd.nextInt() % maxValue;
- } while (secondValue < firstValue);
+ if (isPoint) {
+ secondValue = firstValue;
+ } else {
+ do {
+ secondValue = rnd.nextInt() % maxValue;
+ } while (secondValue < firstValue);
+ }
fieldValues[j] = firstValue;
fieldValues[k] = secondValue;
}
}
@Override
+ protected void setIntKeyFields(int[] fieldValues, int numKeyFields, int maxValue, Random rnd) {
+ setIntKeyFields(fieldValues, numKeyFields, maxValue, rnd, false);
+ }
+
+ @Override
protected void setIntPayloadFields(int[] fieldValues, int numKeyFields, int numFields) {
for (int j = numKeyFields; j < numFields; j++) {
fieldValues[j] = intPayloadValue++;
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreePointMBRBulkLoadTest.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreePointMBRBulkLoadTest.java
new file mode 100644
index 0000000..0df7caf
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/LSMRTreePointMBRBulkLoadTest.java
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.storage.am.lsm.rtree;
+
+import java.util.Random;
+
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
+import org.apache.hyracks.storage.am.config.AccessMethodTestsConfig;
+import org.apache.hyracks.storage.am.lsm.rtree.util.LSMRTreeTestContext;
+import org.apache.hyracks.storage.am.lsm.rtree.util.LSMRTreeTestHarness;
+import org.apache.hyracks.storage.am.rtree.AbstractRTreeBulkLoadTest;
+import org.apache.hyracks.storage.am.rtree.AbstractRTreeTestContext;
+import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
+import org.junit.After;
+import org.junit.Before;
+
+@SuppressWarnings("rawtypes")
+public class LSMRTreePointMBRBulkLoadTest extends AbstractRTreeBulkLoadTest {
+
+ public LSMRTreePointMBRBulkLoadTest() {
+ super(AccessMethodTestsConfig.LSM_RTREE_TEST_RSTAR_POLICY, true);
+ }
+
+ private final LSMRTreeTestHarness harness = new LSMRTreeTestHarness();
+
+ @Before
+ public void setUp() throws HyracksDataException {
+ harness.setUp();
+ }
+
+ @After
+ public void tearDown() throws HyracksDataException {
+ harness.tearDown();
+ }
+
+ @Override
+ protected AbstractRTreeTestContext createTestContext(ISerializerDeserializer[] fieldSerdes,
+ IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeys, RTreePolicyType rtreePolicyType)
+ throws Exception {
+ return LSMRTreeTestContext.create(harness.getIOManager(), harness.getVirtualBufferCaches(),
+ harness.getFileReference(), harness.getDiskBufferCache(), fieldSerdes, valueProviderFactories, numKeys,
+ rtreePolicyType, harness.getBoomFilterFalsePositiveRate(), harness.getMergePolicy(),
+ harness.getOperationTracker(), harness.getIOScheduler(), harness.getIOOperationCallbackFactory(),
+ harness.getMetadataPageManagerFactory(), true);
+ }
+
+ @Override
+ protected Random getRandom() {
+ return harness.getRandom();
+ }
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
index 2bd74af..acdb268 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-rtree-test/src/test/java/org/apache/hyracks/storage/am/lsm/rtree/util/LSMRTreeTestContext.java
@@ -79,6 +79,17 @@
double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
IMetadataPageManagerFactory metadataPageManagerFactory) throws Exception {
+ return create(ioManager, virtualBufferCaches, file, diskBufferCache, fieldSerdes, valueProviderFactories,
+ numKeyFields, rtreePolicyType, bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler,
+ ioOpCallbackFactory, metadataPageManagerFactory, false);
+ }
+
+ public static LSMRTreeTestContext create(IIOManager ioManager, List<IVirtualBufferCache> virtualBufferCaches,
+ FileReference file, IBufferCache diskBufferCache, ISerializerDeserializer[] fieldSerdes,
+ IPrimitiveValueProviderFactory[] valueProviderFactories, int numKeyFields, RTreePolicyType rtreePolicyType,
+ double bloomFilterFalsePositiveRate, ILSMMergePolicy mergePolicy, ILSMOperationTracker opTracker,
+ ILSMIOOperationScheduler ioScheduler, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
+ IMetadataPageManagerFactory metadataPageManagerFactory, boolean isPointMBR) throws Exception {
ITypeTraits[] typeTraits = SerdeUtils.serdesToTypeTraits(fieldSerdes);
IBinaryComparatorFactory[] rtreeCmpFactories =
SerdeUtils.serdesToComparatorFactories(fieldSerdes, numKeyFields);
@@ -95,7 +106,7 @@
typeTraits, rtreeCmpFactories, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
bloomFilterFalsePositiveRate, mergePolicy, opTracker, ioScheduler, ioOpCallbackFactory,
LSMRTreeUtils.proposeBestLinearizer(typeTraits, rtreeCmpFactories.length), null, btreeFields, null,
- null, null, true, false, metadataPageManagerFactory);
+ null, null, true, isPointMBR, metadataPageManagerFactory);
LSMRTreeTestContext testCtx = new LSMRTreeTestContext(fieldSerdes, lsmTree);
return testCtx;
}