Adjust creation of Rtree to reflect the new LSM Rtree Optimization
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java
index 72e818e..5a0db1d 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeOperationsHelper.java
@@ -67,6 +67,7 @@
protected IPrimitiveValueProviderFactory[] valueProviderFactories;
protected int numNestedSecondaryKeyFields;
protected ATypeTag keyType;
+ protected int[] primaryKeyFields;
protected SecondaryRTreeOperationsHelper(PhysicalOptimizationConfig physOptConf,
IAsterixPropertiesProvider propertiesProvider) {
@@ -83,7 +84,7 @@
secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, primaryComparatorFactories,
valueProviderFactories, RTreePolicyType.RTREE, AqlMetadataProvider.proposeLinearizer(keyType,
secondaryComparatorFactories.length), dataset.getDatasetId(), mergePolicyFactory,
- mergePolicyFactoryProperties);
+ mergePolicyFactoryProperties, primaryKeyFields);
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMRTreeResource);
@@ -96,8 +97,8 @@
dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMRTreeIOOperationCallbackFactory.INSTANCE, AqlMetadataProvider.proposeLinearizer(keyType,
secondaryComparatorFactories.length), storageProperties
- .getBloomFilterFalsePositiveRate()), localResourceFactoryProvider,
- NoOpOperationCallbackFactory.INSTANCE);
+ .getBloomFilterFalsePositiveRate(), primaryKeyFields),
+ localResourceFactoryProvider, NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, secondaryIndexCreateOp,
secondaryPartitionConstraint);
spec.addRoot(secondaryIndexCreateOp);
@@ -147,6 +148,10 @@
secondaryTypeTraits[numNestedSecondaryKeyFields + i] = primaryRecDesc.getTypeTraits()[i];
}
secondaryRecDesc = new RecordDescriptor(secondaryRecFields, secondaryTypeTraits);
+ primaryKeyFields = new int[numPrimaryKeys];
+ for (int i = 0; i < primaryKeyFields.length; i++) {
+ primaryKeyFields[i] = i + numNestedSecondaryKeyFields;
+ }
}
@Override
@@ -178,7 +183,8 @@
new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
- storageProperties.getBloomFilterFalsePositiveRate()), BTree.DEFAULT_FILL_FACTOR);
+ storageProperties.getBloomFilterFalsePositiveRate(), primaryKeyFields),
+ BTree.DEFAULT_FILL_FACTOR);
// Connect the operators.
spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, asterixAssignOp, 0);
@@ -207,7 +213,8 @@
new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER, LSMRTreeIOOperationCallbackFactory.INSTANCE,
AqlMetadataProvider.proposeLinearizer(keyType, secondaryComparatorFactories.length),
- storageProperties.getBloomFilterFalsePositiveRate()), NoOpOperationCallbackFactory.INSTANCE);
+ storageProperties.getBloomFilterFalsePositiveRate(), primaryKeyFields),
+ NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, compactOp,
secondaryPartitionConstraint);
spec.addRoot(compactOp);
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
index c45a121..78481fa 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
@@ -570,7 +570,8 @@
compactionInfo.second, isSecondary ? new SecondaryIndexOperationTrackerProvider(
dataset.getDatasetId()) : new PrimaryIndexOperationTrackerProvider(
dataset.getDatasetId()), rtcProvider, LSMBTreeIOOperationCallbackFactory.INSTANCE,
- storageProperties.getBloomFilterFalsePositiveRate(), !isSecondary), retainInput, searchCallbackFactory);
+ storageProperties.getBloomFilterFalsePositiveRate(), !isSecondary), retainInput,
+ searchCallbackFactory);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(btreeSearchOp, spPc.second);
@@ -628,6 +629,10 @@
IBinaryComparatorFactory[] primaryComparatorFactories = DatasetUtils.computeKeysBinaryComparatorFactories(
dataset, recType, context.getBinaryComparatorFactoryProvider());
+ int[] btreeFields = new int[primaryComparatorFactories.length];
+ for (int i = 0; i < btreeFields.length; i++) {
+ btreeFields[i] = i + numSecondaryKeys;
+ }
IAType nestedKeyType = NonTaggedFormatUtil.getNestedSpatialType(keyType.getTypeTag());
Pair<ILSMMergePolicyFactory, Map<String, String>> compactionInfo = DatasetUtils.getMergePolicyFactory(
@@ -641,8 +646,8 @@
compactionInfo.second, new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMRTreeIOOperationCallbackFactory.INSTANCE, proposeLinearizer(nestedKeyType.getTypeTag(),
- comparatorFactories.length), storageProperties.getBloomFilterFalsePositiveRate()),
- retainInput, searchCallbackFactory);
+ comparatorFactories.length), storageProperties.getBloomFilterFalsePositiveRate(),
+ btreeFields), retainInput, searchCallbackFactory);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(rtreeSearchOp, spPc.second);
} catch (MetadataException me) {
@@ -799,8 +804,9 @@
new LSMBTreeDataflowHelperFactory(new AsterixVirtualBufferCacheProvider(dataset.getDatasetId()),
compactionInfo.first, compactionInfo.second, new PrimaryIndexOperationTrackerProvider(
dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
- LSMBTreeIOOperationCallbackFactory.INSTANCE,
- storageProperties.getBloomFilterFalsePositiveRate(), true), NoOpOperationCallbackFactory.INSTANCE);
+ LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties
+ .getBloomFilterFalsePositiveRate(), true),
+ NoOpOperationCallbackFactory.INSTANCE);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(btreeBulkLoad,
splitsAndConstraint.second);
} catch (MetadataException me) {
@@ -1065,8 +1071,8 @@
compactionInfo.first, compactionInfo.second, new SecondaryIndexOperationTrackerProvider(
dataset.getDatasetId()), AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMBTreeIOOperationCallbackFactory.INSTANCE, storageProperties
- .getBloomFilterFalsePositiveRate(), false), filterFactory, modificationCallbackFactory,
- false, indexName);
+ .getBloomFilterFalsePositiveRate(), false), filterFactory,
+ modificationCallbackFactory, false, indexName);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(btreeBulkLoad,
splitsAndConstraint.second);
} catch (MetadataException e) {
@@ -1276,6 +1282,10 @@
IAsterixApplicationContextInfo appContext = (IAsterixApplicationContextInfo) context.getAppContext();
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = splitProviderAndPartitionConstraintsForInternalOrFeedDataset(
dataverseName, datasetName, indexName);
+ int[] btreeFields = new int[primaryComparatorFactories.length];
+ for (int k = 0; k < btreeFields.length; k++) {
+ btreeFields[k] = k + numSecondaryKeys;
+ }
// prepare callback
JobId jobId = ((JobEventListenerFactory) spec.getJobletEventListenerFactory()).getJobId();
@@ -1296,8 +1306,8 @@
compactionInfo.second, new SecondaryIndexOperationTrackerProvider(dataset.getDatasetId()),
AsterixRuntimeComponentsProvider.RUNTIME_PROVIDER,
LSMRTreeIOOperationCallbackFactory.INSTANCE, proposeLinearizer(nestedKeyType.getTypeTag(),
- comparatorFactories.length), storageProperties.getBloomFilterFalsePositiveRate()),
- filterFactory, modificationCallbackFactory, false, indexName);
+ comparatorFactories.length), storageProperties.getBloomFilterFalsePositiveRate(),
+ btreeFields), filterFactory, modificationCallbackFactory, false, indexName);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(rtreeUpdate, splitsAndConstraint.second);
} catch (MetadataException | IOException e) {
throw new AlgebricksException(e);
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
index ffdfcac..0776db8 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/resource/LSMRTreeLocalResourceMetadata.java
@@ -47,11 +47,12 @@
private final ILinearizeComparatorFactory linearizeCmpFactory;
private final ILSMMergePolicyFactory mergePolicyFactory;
private final Map<String, String> mergePolicyProperties;
+ private final int[] btreeFields;
public LSMRTreeLocalResourceMetadata(ITypeTraits[] typeTraits, IBinaryComparatorFactory[] rtreeCmpFactories,
IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
RTreePolicyType rtreePolicyType, ILinearizeComparatorFactory linearizeCmpFactory, int datasetID,
- ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties) {
+ ILSMMergePolicyFactory mergePolicyFactory, Map<String, String> mergePolicyProperties, int[] btreeFields) {
super(datasetID);
this.typeTraits = typeTraits;
this.rtreeCmpFactories = rtreeCmpFactories;
@@ -61,6 +62,7 @@
this.linearizeCmpFactory = linearizeCmpFactory;
this.mergePolicyFactory = mergePolicyFactory;
this.mergePolicyProperties = mergePolicyProperties;
+ this.btreeFields = btreeFields;
}
@Override
@@ -75,7 +77,7 @@
mergePolicyFactory.createMergePolicy(mergePolicyProperties), new BaseOperationTracker(
(DatasetLifecycleManager) runtimeContextProvider.getIndexLifecycleManager(), datasetID),
runtimeContextProvider.getLSMIOScheduler(), LSMRTreeIOOperationCallbackFactory.INSTANCE
- .createIOOperationCallback(), linearizeCmpFactory);
+ .createIOOperationCallback(), linearizeCmpFactory, btreeFields);
} catch (TreeIndexException e) {
throw new HyracksDataException(e);
}