[ASTERIXDB-3262][COMP] Enable columnar filters by default
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
Make the use of columnar filters enabled by default.
- Fix ASTERIXDB-3265: The array filter reader has
its delimiters reversed
- Fix ASTERIXDB-3264: Columnar range-filter doesn't correctly
evaluate equality if a constant is the left operand
Change-Id: I9306db62ff0e991efed30346a8847b9edae9393e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17781
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Wail Alkowaileet <wael.y.k@gmail.com>
Reviewed-by: Hussain Towaileb <hussainht@gmail.com>
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/column-pushdown/meta.001.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/column-pushdown/meta.001.sqlpp
index e5e90c1..6ef6e72 100644
--- a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/column-pushdown/meta.001.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/column-pushdown/meta.001.sqlpp
@@ -40,6 +40,7 @@
CREATE DATASET `nation`(CH2Type) WITH META(CH2MetaType) PRIMARY KEY META().uid WITH {"storage-format": {"format": "column"}};
CREATE DATASET `region`(CH2Type) WITH META(CH2MetaType) PRIMARY KEY META().uid WITH {"storage-format": {"format": "column"}};
+SET `compiler.column.filter` "false";
SELECT s.s_i_id, SUM(s.s_order_cnt) as ordercount
FROM nation n, supplier su, stock s
WHERE s.s_w_id * s.s_i_id MOD 10000 = su.su_suppkey
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index a67d326..1c3b061 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -21,7 +21,7 @@
"compiler\.arrayindex" : true,
"compiler.batch.lookup" : true,
"compiler.cbo" : true,
- "compiler.column.filter" : false,
+ "compiler.column.filter" : true,
"compiler\.external\.field\.pushdown" : true,
"compiler.forcejoinorder" : false,
"compiler\.framesize" : 32768,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index 0e90266..93fea8f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -21,7 +21,7 @@
"compiler\.arrayindex" : true,
"compiler.batch.lookup" : true,
"compiler.cbo" : true,
- "compiler.column.filter" : false,
+ "compiler.column.filter" : true,
"compiler\.external\.field\.pushdown" : true,
"compiler.forcejoinorder" : false,
"compiler\.framesize" : 32768,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index 931fd06..e68b30a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -21,7 +21,7 @@
"compiler\.arrayindex" : true,
"compiler.batch.lookup" : true,
"compiler.cbo" : true,
- "compiler.column.filter" : false,
+ "compiler.column.filter" : true,
"compiler\.external\.field\.pushdown" : true,
"compiler.forcejoinorder" : false,
"compiler\.framesize" : 32768,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/array-access-pushdown/array-access-pushdown.010.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/array-access-pushdown/array-access-pushdown.010.plan
index 7783b10..3f46c9c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/array-access-pushdown/array-access-pushdown.010.plan
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/array-access-pushdown/array-access-pushdown.010.plan
@@ -33,7 +33,7 @@
-- STREAM_PROJECT |PARTITIONED|
exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$49, $$p] <- test.ColumnDataset project ({entities:{urls:[{display_url:any}]}}) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ data-scan []<-[$$49, $$p] <- test.ColumnDataset project ({entities:{urls:[{display_url:any}]}}) filter on: not(if-missing-or-null(eq(scan-collection($$p.getField("entities").getField("urls")).getField("display_url"), "string"), false)) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
-- DATASOURCE_SCAN |PARTITIONED|
exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/other-pushdowns/other-pushdowns.008.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/other-pushdowns/other-pushdowns.008.plan
index 26cb68c..d88c101 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/other-pushdowns/other-pushdowns.008.plan
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/other-pushdowns/other-pushdowns.008.plan
@@ -44,7 +44,7 @@
-- STREAM_SELECT |PARTITIONED|
exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$72, $$p1] <- test.ColumnDataset project ({entities:{urls:[{display_url:any}]}}) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ data-scan []<-[$$72, $$p1] <- test.ColumnDataset project ({entities:{urls:[{display_url:any}]}}) filter on: eq(lowercase(scan-collection($$p1.getField("entities").getField("urls")).getField("display_url")), "string") [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
-- DATASOURCE_SCAN |PARTITIONED|
exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/other-pushdowns/other-pushdowns.010.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/other-pushdowns/other-pushdowns.010.plan
index 41b9e05..5a7b0b3 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/other-pushdowns/other-pushdowns.010.plan
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/column/pushdown/other-pushdowns/other-pushdowns.010.plan
@@ -46,7 +46,7 @@
-- STREAM_SELECT |PARTITIONED|
exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$71, $$p1] <- test.ColumnDataset project ({entities:{urls:[{display_url:any,indices:any}]}}) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ data-scan []<-[$$71, $$p1] <- test.ColumnDataset project ({entities:{urls:[{display_url:any,indices:any}]}}) filter on: eq(lowercase(scan-collection($$p1.getField("entities").getField("urls")).getField("display_url")), "string") [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
-- DATASOURCE_SCAN |PARTITIONED|
exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/array-access-pushdown/array-access-pushdown.010.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/array-access-pushdown/array-access-pushdown.010.plan
index d0bf0f0..633350a 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/array-access-pushdown/array-access-pushdown.010.plan
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/array-access-pushdown/array-access-pushdown.010.plan
@@ -33,7 +33,7 @@
-- STREAM_PROJECT |PARTITIONED|
exchange [cardinality: 2.0, op-cost: 0.0, total-cost: 2.1]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$49, $$p] <- test.ColumnDataset project ({entities:{urls:[{display_url:any}]}}) [cardinality: 2.0, op-cost: 2.1, total-cost: 2.1]
+ data-scan []<-[$$49, $$p] <- test.ColumnDataset project ({entities:{urls:[{display_url:any}]}}) filter on: not(if-missing-or-null(eq(scan-collection($$p.getField("entities").getField("urls")).getField("display_url"), "string"), false)) [cardinality: 2.0, op-cost: 2.1, total-cost: 2.1]
-- DATASOURCE_SCAN |PARTITIONED|
exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/other-pushdowns/other-pushdowns.008.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/other-pushdowns/other-pushdowns.008.plan
index 224298e..6ededae 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/other-pushdowns/other-pushdowns.008.plan
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/other-pushdowns/other-pushdowns.008.plan
@@ -44,7 +44,7 @@
-- STREAM_SELECT |PARTITIONED|
exchange [cardinality: 2.0, op-cost: 0.0, total-cost: 2.1]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$72, $$p1] <- test.ColumnDataset project ({entities:{urls:[{display_url:any}]}}) [cardinality: 2.0, op-cost: 2.1, total-cost: 2.1]
+ data-scan []<-[$$72, $$p1] <- test.ColumnDataset project ({entities:{urls:[{display_url:any}]}}) filter on: eq(lowercase(scan-collection($$p1.getField("entities").getField("urls")).getField("display_url")), "string") [cardinality: 2.0, op-cost: 2.1, total-cost: 2.1]
-- DATASOURCE_SCAN |PARTITIONED|
exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/other-pushdowns/other-pushdowns.010.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/other-pushdowns/other-pushdowns.010.plan
index 0bf5e31..402e741 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/other-pushdowns/other-pushdowns.010.plan
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_cbo/column/pushdown/other-pushdowns/other-pushdowns.010.plan
@@ -46,7 +46,7 @@
-- STREAM_SELECT |PARTITIONED|
exchange [cardinality: 2.0, op-cost: 0.0, total-cost: 2.1]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
- data-scan []<-[$$71, $$p1] <- test.ColumnDataset project ({entities:{urls:[{display_url:any,indices:any}]}}) [cardinality: 2.0, op-cost: 2.1, total-cost: 2.1]
+ data-scan []<-[$$71, $$p1] <- test.ColumnDataset project ({entities:{urls:[{display_url:any,indices:any}]}}) filter on: eq(lowercase(scan-collection($$p1.getField("entities").getField("urls")).getField("display_url")), "string") [cardinality: 2.0, op-cost: 2.1, total-cost: 2.1]
-- DATASOURCE_SCAN |PARTITIONED|
exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
-- ONE_TO_ONE_EXCHANGE |PARTITIONED|
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/PathExtractorVisitor.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/PathExtractorVisitor.java
index 707f5b7..de4df8a 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/PathExtractorVisitor.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/metadata/schema/visitor/PathExtractorVisitor.java
@@ -133,9 +133,14 @@
} else {
// array
reader = readerFactory.createValueReader(primitiveNode.getTypeTag(), primitiveNode.getColumnIndex(), level,
- delimiters.toIntArray());
+ getReversedDelimiters());
}
readers.add(reader);
return reader;
}
+
+ private int[] getReversedDelimiters() {
+ Collections.reverse(delimiters);
+ return delimiters.toIntArray();
+ }
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/filter/ColumnRangeFilterBuilder.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/filter/ColumnRangeFilterBuilder.java
index be05381..f83b484 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/filter/ColumnRangeFilterBuilder.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/filter/ColumnRangeFilterBuilder.java
@@ -118,6 +118,11 @@
}
ComparisonKind comparisonKind = getComparisonKind(fid, constant.getType().getTypeTag());
+ if (comparisonKind == ComparisonKind.NEQ) {
+ // Ignore NEQ
+ return NoOpColumnFilterEvaluatorFactory.INSTANCE;
+ }
+
IColumnRangeFilterValueAccessorFactory constValue =
ConstantColumnRangeFilterValueAccessorFactory.createFactory(constant);
IColumnRangeFilterValueAccessorFactory min = new ColumnRangeFilterValueAccessorFactory(path, true);
@@ -153,15 +158,20 @@
}
private static ComparisonKind invert(ComparisonKind comparisonKind) {
- if (comparisonKind == ComparisonKind.LT) {
- return ComparisonKind.GE;
- } else if (comparisonKind == ComparisonKind.LE) {
- return ComparisonKind.GT;
- } else if (comparisonKind == ComparisonKind.GT) {
- return ComparisonKind.LE;
+ switch (comparisonKind) {
+ case EQ:
+ return ComparisonKind.EQ;
+ case LE:
+ return ComparisonKind.GT;
+ case GE:
+ return ComparisonKind.LT;
+ case LT:
+ return ComparisonKind.GE;
+ case GT:
+ return ComparisonKind.LE;
+ default:
+ throw new IllegalStateException("Unsupported comparison type: " + comparisonKind);
}
- //ComparisonKind.GE
- return ComparisonKind.LT;
}
private static IColumnRangeFilterEvaluatorFactory createEvaluator(ComparisonKind comparisonKind,
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
index 74f4447..98c4223 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java
@@ -45,5 +45,5 @@
public static final int EXTERNAL_SCAN_BUFFER_SIZE =
StorageUtil.getIntSizeInBytes(8, StorageUtil.StorageUnit.KILOBYTE);
public static final boolean BATCH_LOOKUP_DEFAULT = true;
- public static final boolean COLUMN_FILTER_DEFAULT = false;
+ public static final boolean COLUMN_FILTER_DEFAULT = true;
}