[ASTERIXDB-2203][IDX] Eliminate sort operator for secondary primary index
- user model changes: no
- storage format changes: no
- interface changes: no
details:
This change is intended to remove the sort operator when creating
a secondary primary index. The secondary primary index DDL statement uses
the same code as the one for creating general secondary BTree indexes which
has the sort operator since it needs to sort by SK. The sort operator is
not needed for secondary primary index since there is no SK and PK is already
sorted.
Change-Id: I2287c355204355b7ce306388c85d58d69a1bb64a
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2240
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dmitry.lychagin@couchbase.com>
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
index 41def96..cf6338a 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
@@ -125,50 +125,57 @@
spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
return spec;
} else {
- // Create dummy key provider for feeding the primary index scan.
- IOperatorDescriptor keyProviderOp = DatasetUtil.createDummyKeyProviderOp(spec, dataset,
- metadataProvider);
+ // job spec:
+ // key provider -> primary idx -> (cast assign)? -> assign -> (select)? -> (sort)? -> bulk load -> sink
IndexUtil.bindJobEventListener(spec, metadataProvider);
- // Create primary index scan op.
- IOperatorDescriptor primaryScanOp = DatasetUtil.createPrimaryIndexScanOp(spec, metadataProvider, dataset);
+ // dummy key provider ----> primary index scan
+ IOperatorDescriptor sourceOp = DatasetUtil.createDummyKeyProviderOp(spec, dataset, metadataProvider);
+ IOperatorDescriptor targetOp = DatasetUtil.createPrimaryIndexScanOp(spec, metadataProvider, dataset);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
- // Assign op.
- IOperatorDescriptor sourceOp = primaryScanOp;
+ sourceOp = targetOp;
if (isOverridingKeyFieldTypes && !enforcedItemType.equals(itemType)) {
- sourceOp = createCastOp(spec, dataset.getDatasetType(), index.isEnforced());
- spec.connect(new OneToOneConnectorDescriptor(spec), primaryScanOp, 0, sourceOp, 0);
+ // primary index scan ----> cast assign
+ targetOp = createCastOp(spec, dataset.getDatasetType(), index.isEnforced());
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
+ sourceOp = targetOp;
}
- AlgebricksMetaOperatorDescriptor asterixAssignOp =
- createAssignOp(spec, index.getKeyFieldNames().size(), secondaryRecDesc);
+ // primary index OR cast assign ----> assign op
+ targetOp = createAssignOp(spec, index.getKeyFieldNames().size(), secondaryRecDesc);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
- // If any of the secondary fields are nullable, then add a select op that filters nulls.
- AlgebricksMetaOperatorDescriptor selectOp = null;
+ sourceOp = targetOp;
if (anySecondaryKeyIsNullable || isOverridingKeyFieldTypes) {
- selectOp = createFilterNullsSelectOp(spec, index.getKeyFieldNames().size(), secondaryRecDesc);
+ // if any of the secondary fields are nullable, then add a select op that filters nulls.
+ // assign op ----> select op
+ targetOp = createFilterNullsSelectOp(spec, index.getKeyFieldNames().size(), secondaryRecDesc);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
+ sourceOp = targetOp;
}
- // Sort by secondary keys.
- ExternalSortOperatorDescriptor sortOp = createSortOp(spec, secondaryComparatorFactories, secondaryRecDesc);
- // Create secondary BTree bulk load op.
- TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec, fieldPermutation,
- dataflowHelperFactory, GlobalConfig.DEFAULT_TREE_FILL_FACTOR);
+ // no need to sort if the index is secondary primary index
+ if (!index.getKeyFieldNames().isEmpty()) {
+ // sort by secondary keys.
+ // assign op OR select op ----> sort op
+ targetOp = createSortOp(spec, secondaryComparatorFactories, secondaryRecDesc);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
+ sourceOp = targetOp;
+ }
- AlgebricksMetaOperatorDescriptor metaOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
+ // assign op OR select op OR sort op ----> bulk load op
+ targetOp = createTreeIndexBulkLoadOp(spec, fieldPermutation, dataflowHelperFactory,
+ GlobalConfig.DEFAULT_TREE_FILL_FACTOR);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
+
+ // bulk load op ----> sink op
+ sourceOp = targetOp;
+ targetOp = new AlgebricksMetaOperatorDescriptor(spec, 1, 0,
new IPushRuntimeFactory[] { new SinkRuntimeFactory() },
new RecordDescriptor[] { secondaryRecDesc });
- // Connect the operators.
- spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, asterixAssignOp, 0);
- if (anySecondaryKeyIsNullable || isOverridingKeyFieldTypes) {
- spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, selectOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), selectOp, 0, sortOp, 0);
- } else {
- spec.connect(new OneToOneConnectorDescriptor(spec), asterixAssignOp, 0, sortOp, 0);
- }
- spec.connect(new OneToOneConnectorDescriptor(spec), sortOp, 0, secondaryBulkLoadOp, 0);
- spec.connect(new OneToOneConnectorDescriptor(spec), secondaryBulkLoadOp, 0, metaOp, 0);
- spec.addRoot(metaOp);
+ spec.connect(new OneToOneConnectorDescriptor(spec), sourceOp, 0, targetOp, 0);
+
+ spec.addRoot(targetOp);
spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy());
return spec;
}