[NO ISSUE][FUN] Use the correct tuple projector in query_partition()
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Update private storage-component() to include the size and component path.
- Update log for finding metadata key from an index.
Ext-ref: MB-66346
Change-Id: I17d777b2067ad51780b9ea740a8673acadc37906
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19651
Tested-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
Reviewed-by: Michael Blow <mblow@apache.org>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryPartitionDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryPartitionDatasource.java
index 4917b33..1101ace 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryPartitionDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryPartitionDatasource.java
@@ -28,6 +28,7 @@
import org.apache.asterix.metadata.declared.FunctionDataSource;
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.utils.IndexUtil;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
@@ -50,6 +51,7 @@
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.job.JobSpecification;
import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
+import org.apache.hyracks.storage.common.projection.ITupleProjectorFactory;
public class QueryPartitionDatasource extends FunctionDataSource {
@@ -111,8 +113,15 @@
ITupleFilterFactory tupleFilterFactory, long outputLimit, IOperatorSchema opSchema,
IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig,
IProjectionFiltrationInfo projectionInfo) throws AlgebricksException {
+ ARecordType metaItemType = null;
+ if (ds.hasMetaPart()) {
+ metaItemType = (ARecordType) schemaTypes[2];
+ }
+ ARecordType datasetType = (ARecordType) schemaTypes[1];
+ ITupleProjectorFactory tupleProjectorFactory = IndexUtil.createTupleProjectorFactory(context, typeEnv,
+ ds.getDatasetFormatInfo(), projectionInfo, datasetType, metaItemType, ds.getPrimaryKeys().size());
return metadataProvider.getBtreePartitionSearchRuntime(jobSpec, opSchema, typeEnv, context, ds,
- tupleFilterFactory, outputLimit, partitionNum);
+ tupleFilterFactory, tupleProjectorFactory, outputLimit, partitionNum);
}
@Override
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsReader.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsReader.java
index 81f680d..65d7015 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsReader.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsReader.java
@@ -33,15 +33,15 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMDiskComponent;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
import org.apache.hyracks.storage.am.lsm.common.impls.LSMComponentId;
+import org.apache.hyracks.util.JSONUtil;
public class StorageComponentsReader extends FunctionReader {
- private final List<String> components;
private final Iterator<String> it;
private final CharArrayRecord record;
public StorageComponentsReader(String nodeId, DatasetResource dsr) throws HyracksDataException {
- components = new ArrayList<>();
+ List<String> components = new ArrayList<>();
if (dsr != null && dsr.isOpen()) {
Map<Long, IndexInfo> indexes = dsr.getIndexes();
StringBuilder strBuilder = new StringBuilder();
@@ -56,7 +56,7 @@
strBuilder.append("\", \"path\":\"");
strBuilder.append(path);
strBuilder.append("\", \"components\":[");
- // syncronize over the opTracker
+ // synchronize over the opTracker
synchronized (index.getOperationTracker()) {
List<ILSMDiskComponent> diskComponents = index.getDiskComponents();
for (int i = diskComponents.size() - 1; i >= 0; i--) {
@@ -70,6 +70,10 @@
strBuilder.append(id.getMinId());
strBuilder.append(",\"max\":");
strBuilder.append(id.getMaxId());
+ strBuilder.append(",\"size\":");
+ strBuilder.append(c.getComponentSize());
+ strBuilder.append(",\"component\":");
+ JSONUtil.quoteAndEscape(strBuilder, String.valueOf(c));
strBuilder.append('}');
}
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 66c184b..f11a338 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -169,7 +169,6 @@
import org.apache.hyracks.storage.am.common.api.ITupleFilterFactory;
import org.apache.hyracks.storage.am.common.dataflow.IIndexDataflowHelperFactory;
import org.apache.hyracks.storage.am.common.dataflow.IndexDataflowHelperFactory;
-import org.apache.hyracks.storage.am.common.impls.DefaultTupleProjectorFactory;
import org.apache.hyracks.storage.am.common.ophelpers.IndexOperation;
import org.apache.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeBatchPointSearchOperatorDescriptor;
import org.apache.hyracks.storage.am.lsm.invertedindex.dataflow.BinaryTokenizerOperatorDescriptor;
@@ -598,11 +597,11 @@
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getBtreePartitionSearchRuntime(
JobSpecification jobSpec, IOperatorSchema opSchema, IVariableTypeEnvironment typeEnv, JobGenContext context,
- Dataset dataset, ITupleFilterFactory tupleFilterFactory, long outputLimit, int partitionNum)
- throws AlgebricksException {
+ Dataset dataset, ITupleFilterFactory tupleFilterFactory, ITupleProjectorFactory tupleProjectorFactory,
+ long outputLimit, int partitionNum) throws AlgebricksException {
return getBtreeSearchRuntime(jobSpec, opSchema, typeEnv, context, true, false, null, dataset,
dataset.getDatasetName(), null, null, true, true, false, null, null, null, tupleFilterFactory,
- outputLimit, false, false, DefaultTupleProjectorFactory.INSTANCE, false, partitionNum);
+ outputLimit, false, false, tupleProjectorFactory, false, partitionNum);
}
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getBtreeSearchRuntime(JobSpecification jobSpec,
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/util/ComponentUtils.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/util/ComponentUtils.java
index a8cbfdd..bfb0c3c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/util/ComponentUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/util/ComponentUtils.java
@@ -93,7 +93,7 @@
// was not found in all in-memory components, search in the disk components
fromDiskComponents(index, key, value);
if (value.getLength() == 0) {
- LOGGER.debug("{} was NOT found", key);
+ LOGGER.debug("{} was NOT found for index {}", key, index);
}
}
} else {