[NO ISSUE][COMP] Fixes and cleanup for external filters
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Enable filter pushdowns for external datasets
- Always register external dataset scans
- Make ExternalDataPrefix with KEY_PATH
Change-Id: I08b509fca0eb3ab9aad82e816dfe4849adf725eb
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17707
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hussainht@gmail.com>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushValueAccessAndFilterDownRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushValueAccessAndFilterDownRule.java
index 3b298d0..c74c293 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushValueAccessAndFilterDownRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushValueAccessAndFilterDownRule.java
@@ -113,11 +113,11 @@
pushdownProcessorsExecutor.add(new ColumnFilterPushdownProcessor(pushdownContext, context));
// Performs range-filter pushdowns
pushdownProcessorsExecutor.add(new ColumnRangeFilterPushdownProcessor(pushdownContext, context));
- // Performs prefix pushdowns
- pushdownProcessorsExecutor.add(new ExternalDatasetFilterPushdownProcessor(pushdownContext, context));
- // Inlines AND/OR expression (must be last to run)
- pushdownProcessorsExecutor.add(new InlineFilterExpressionsProcessor(pushdownContext, context));
}
+ // Performs prefix pushdowns
+ pushdownProcessorsExecutor.add(new ExternalDatasetFilterPushdownProcessor(pushdownContext, context));
+ // Inlines AND/OR expression (must be last to run)
+ pushdownProcessorsExecutor.add(new InlineFilterExpressionsProcessor(pushdownContext, context));
}
/**
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/PushdownProcessorsExecutor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/PushdownProcessorsExecutor.java
index 1530f55..ab6a26c 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/PushdownProcessorsExecutor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/PushdownProcessorsExecutor.java
@@ -67,7 +67,8 @@
if (dataset.getDatasetFormatInfo().getFormat() == DatasetConfig.DatasetFormat.COLUMN) {
info = createInternalColumnarDatasetInfo(scanDefineDescriptor, context);
} else if (dataset.getDatasetType() == DatasetConfig.DatasetType.EXTERNAL
- && DatasetUtil.isFieldAccessPushdownSupported(dataset)) {
+ && (DatasetUtil.isFieldAccessPushdownSupported(dataset)
+ || DatasetUtil.isFilterPushdownSupported(dataset))) {
info = createExternalDatasetProjectionInfo(scanDefineDescriptor, context);
}
setInfoToDataScan(scanOp, info);
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/PushdownOperatorVisitor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/PushdownOperatorVisitor.java
index 65b131a..dd9acd5 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/PushdownOperatorVisitor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/pushdown/visitor/PushdownOperatorVisitor.java
@@ -26,13 +26,11 @@
import org.apache.asterix.common.config.DatasetConfig;
import org.apache.asterix.common.config.DatasetConfig.DatasetFormat;
import org.apache.asterix.common.metadata.DataverseName;
-import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.metadata.declared.DataSource;
import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.declared.DatasetDataSource;
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
-import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.utils.ConstantExpressionUtil;
import org.apache.asterix.optimizer.rules.pushdown.PushdownContext;
@@ -202,11 +200,8 @@
Dataset dataset = getDataset(dataSource);
//Only external dataset can have pushed down expressions
- if (dataset.getDatasetType() == DatasetConfig.DatasetType.EXTERNAL
- && !ExternalDataUtils
- .supportsPushdown(((ExternalDatasetDetails) dataset.getDatasetDetails()).getProperties())
- || dataset.getDatasetType() == DatasetConfig.DatasetType.INTERNAL
- && dataset.getDatasetFormatInfo().getFormat() == DatasetFormat.ROW) {
+ if (dataset.getDatasetType() == DatasetConfig.DatasetType.INTERNAL
+ && dataset.getDatasetFormatInfo().getFormat() == DatasetFormat.ROW) {
return null;
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
index 899a179..ec56009 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
@@ -21,6 +21,7 @@
import static org.apache.asterix.external.util.ExternalDataConstants.COMPUTED_FIELD_PATTERN;
import static org.apache.asterix.external.util.ExternalDataConstants.DEFINITION_FIELD_NAME;
+import static org.apache.asterix.external.util.ExternalDataConstants.KEY_PATH;
import static org.apache.asterix.external.util.ExternalDataConstants.PREFIX_DEFAULT_DELIMITER;
import java.util.ArrayList;
@@ -68,7 +69,7 @@
}
public ExternalDataPrefix(Map<String, String> configuration) throws AlgebricksException {
- this(configuration.get(DEFINITION_FIELD_NAME));
+ this(getDefinitionOrPath(configuration));
}
public ExternalDataPrefix(String prefix) throws AlgebricksException {
@@ -245,4 +246,8 @@
return values;
}
+
+ private static String getDefinitionOrPath(Map<String, String> configuration) {
+ return configuration.getOrDefault(DEFINITION_FIELD_NAME, configuration.get(KEY_PATH));
+ }
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index 88eafcd..44964f3 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@ -699,8 +699,8 @@
if (dataset.getDatasetType() == DatasetType.INTERNAL) {
return dataset.getDatasetFormatInfo().getFormat() == DatasetConfig.DatasetFormat.COLUMN;
}
- // TODO add external dataset with dynamic prefixes
- return false;
+ // External dataset support filter pushdown
+ return true;
}
public static boolean isRangeFilterPushdownSupported(Dataset dataset) {