[ASTERIXDB-2771] Improving LSM filters on datasets with meta records
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
1. Fixed field permutation in INSERT and made it consistent with UPSERT.
The tuple order is [PKEY, RECORD, META, FILTER]
2. Fixed filters on meta on secondary indexes.
3. Enable extensions to use filters on meta.
4. Fixed getDeleteRuntime to make sure it works when datasets have both
filters and meta parts.
Change-Id: I3b7d4bc301e8f4b0bb1015e7754aa1e874f77709
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/7864
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
index c49670e..bd5d82a 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroduceLSMComponentFilterRule.java
@@ -123,7 +123,7 @@
IOptimizableFuncExpr optFuncExpr = analysisCtx.getMatchedFuncExpr(i);
boolean found = findMacthedExprFieldName(optFuncExpr, op, dataset, itemType, datasetIndexes, context,
filterSourceIndicator);
- // the field name source should be from the dataset record, i.e. source should be == 0
+ // the field name source should be consistent with the filter source indicator
if (found && optFuncExpr.getFieldName(0).equals(filterFieldName)
&& optFuncExpr.getFieldSource(0) == filterSourceIndicator) {
optFuncExprs.add(optFuncExpr);
@@ -572,8 +572,11 @@
IAType metaItemType = ((MetadataProvider) context.getMetadataProvider())
.findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
+ IAType recordItemType = ((MetadataProvider) context.getMetadataProvider())
+ .findType(dataset.getMetaItemTypeDataverseName(), dataset.getItemTypeName());
+ ARecordType recordType = (ARecordType) recordItemType;
ARecordType metaRecType = (ARecordType) metaItemType;
- int numSecondaryKeys = KeyFieldTypeUtil.getNumSecondaryKeys(index, filterSourceType, metaRecType);
+ int numSecondaryKeys = KeyFieldTypeUtil.getNumSecondaryKeys(index, recordType, metaRecType);
List<String> fieldName;
int keySource;
if (varIndex >= numSecondaryKeys) {
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index fd4fd90..3c56a98 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -425,7 +425,7 @@
return plan;
}
- private ILogicalOperator translateDelete(DatasetDataSource targetDatasource, Mutable<ILogicalExpression> varRef,
+ protected ILogicalOperator translateDelete(DatasetDataSource targetDatasource, Mutable<ILogicalExpression> varRef,
List<Mutable<ILogicalExpression>> varRefsForLoading, LogicalVariable seqVar, ILogicalOperator pkeyAssignOp,
ICompiledDmlStatement stmt) throws AlgebricksException {
SourceLocation sourceLoc = stmt.getSourceLocation();
@@ -454,7 +454,7 @@
return leafOperator;
}
- private ILogicalOperator translateUpsert(DatasetDataSource targetDatasource,
+ protected ILogicalOperator translateUpsert(DatasetDataSource targetDatasource,
Mutable<ILogicalExpression> payloadVarRef, List<Mutable<ILogicalExpression>> varRefsForLoading,
ILogicalOperator pkeyAssignOp, LogicalVariable unnestVar, ILogicalOperator topOp,
List<Mutable<ILogicalExpression>> pkeyExprs, LogicalVariable seqVar, ICompiledDmlStatement stmt,
@@ -580,7 +580,7 @@
return processReturningExpression(rootOperator, upsertOp, compiledUpsert, resultMetadata);
}
- private ILogicalOperator translateInsert(DatasetDataSource targetDatasource, Mutable<ILogicalExpression> varRef,
+ protected ILogicalOperator translateInsert(DatasetDataSource targetDatasource, Mutable<ILogicalExpression> varRef,
List<Mutable<ILogicalExpression>> varRefsForLoading, LogicalVariable seqVar, ILogicalOperator pkeyAssignOp,
ICompiledDmlStatement stmt, IResultMetadata resultMetadata) throws AlgebricksException {
SourceLocation sourceLoc = stmt.getSourceLocation();
@@ -616,7 +616,7 @@
return processReturningExpression(rootOperator, insertOp, compiledInsert, resultMetadata);
}
- private List<Mutable<ILogicalExpression>> generatedFilterExprs(ILogicalOperator pkeyAssignOp,
+ protected List<Mutable<ILogicalExpression>> generatedFilterExprs(ILogicalOperator pkeyAssignOp,
List<String> filterField, LogicalVariable seqVar, SourceLocation sourceLoc) {
List<LogicalVariable> filterVars = new ArrayList<>();
List<Mutable<ILogicalExpression>> filterAssignExprs = new ArrayList<>();
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter_on_meta_with_idx_0.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter_on_meta_with_idx_0.sqlpp
new file mode 100644
index 0000000..cfa2f00
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter_on_meta_with_idx_0.sqlpp
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+create type DocumentType as open{
+ location: point,
+ area_code: string,
+ text: string
+};
+
+create type KVMetaType as open{
+ `key`:string,
+ vbucket:int32,
+ seq:int64,
+ cas:int64,
+ expiration:int32,
+ flags:int32,
+ revSeq:int64,
+ lockTime:int32
+};
+
+CREATE TYPE OfficerLocation AS {
+ oid: int,
+ location: point,
+ vehicle: string
+};
+
+
+CREATE DATASET KVStore(DocumentType) with meta(KVMetaType) primary key meta().`key` with filter on meta().seq;
+CREATE DATASET OfficerLocations(OfficerLocation) PRIMARY KEY oid;
+
+CREATE INDEX s_location ON KVStore(location) type RTREE;
+CREATE INDEX o_location ON OfficerLocations(location) type RTREE;
+
+
+SELECT t FROM KVStore t, OfficerLocations o
+WHERE spatial_intersect(create_circle(t.location, 100.0), o.location) AND meta(t).seq > 100;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter_on_meta_with_idx_1.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter_on_meta_with_idx_1.sqlpp
new file mode 100644
index 0000000..4d74d24
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter_on_meta_with_idx_1.sqlpp
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+create type DocumentType as open{
+ location: point,
+ area_code: string,
+ text: string
+};
+
+create type KVMetaType as open{
+ `key`:string,
+ vbucket:int32,
+ seq:int64,
+ cas:int64,
+ expiration:int32,
+ flags:int32,
+ revSeq:int64,
+ lockTime:int32
+};
+
+CREATE TYPE OfficerLocation AS {
+ oid: int,
+ location: point,
+ vehicle: string
+};
+
+
+CREATE DATASET KVStore(DocumentType) with meta(KVMetaType) primary key meta().`key` with filter on meta().seq;
+CREATE DATASET OfficerLocations(OfficerLocation) PRIMARY KEY oid;
+
+
+SELECT t FROM KVStore t, OfficerLocations o
+WHERE spatial_intersect(create_circle(t.location, 100.0), o.location) AND meta(t).seq > 100;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter_on_meta_with_idx_2.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter_on_meta_with_idx_2.sqlpp
new file mode 100644
index 0000000..a63c3d31
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter_on_meta_with_idx_2.sqlpp
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+create type DocumentType as open{
+ location: point,
+ area_code: string,
+ attr1: int64,
+ text: string
+};
+
+create type KVMetaType as open{
+ `key`:string,
+ vbucket:int32,
+ seq:int64,
+ cas:int64,
+ expiration:int32,
+ flags:int32,
+ revSeq:int64,
+ lockTime:int32
+};
+
+CREATE TYPE OfficerLocation AS {
+ oid: int,
+ location: point,
+ attr2: int64
+};
+
+
+CREATE DATASET KVStore(DocumentType) with meta(KVMetaType) primary key meta().`key` with filter on meta().seq;
+CREATE DATASET OfficerLocations(OfficerLocation) PRIMARY KEY oid;
+
+CREATE INDEX s_location ON KVStore(attr1);
+CREATE INDEX o_location ON OfficerLocations(location) type RTREE;
+
+
+SELECT t FROM KVStore t, OfficerLocations o
+WHERE t.attr1 = o.attr2 AND meta(t).seq > 100;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter_on_meta_with_idx_3.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter_on_meta_with_idx_3.sqlpp
new file mode 100644
index 0000000..5ade370
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/filter_on_meta_with_idx_3.sqlpp
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+create type DocumentType as open{
+ location: point,
+ area_code: string,
+ attr1: int64,
+ text: string
+};
+
+create type KVMetaType as open{
+ `key`:string,
+ vbucket:int32,
+ seq:int64,
+ cas:int64,
+ expiration:int32,
+ flags:int32,
+ revSeq:int64,
+ lockTime:int32
+};
+
+CREATE TYPE OfficerLocation AS {
+ oid: int,
+ location: point,
+ attr2: int64
+};
+
+
+CREATE DATASET KVStore(DocumentType) with meta(KVMetaType) primary key meta().`key` with filter on meta().seq;
+CREATE DATASET OfficerLocations(OfficerLocation) PRIMARY KEY oid;
+
+
+SELECT t FROM KVStore t, OfficerLocations o
+WHERE t.attr1 = o.attr2 AND meta(t).seq > 100;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter_on_meta_with_idx_0.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter_on_meta_with_idx_0.plan
new file mode 100644
index 0000000..6ab1e92
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter_on_meta_with_idx_0.plan
@@ -0,0 +1,25 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- NESTED_LOOP |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter_on_meta_with_idx_1.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter_on_meta_with_idx_1.plan
new file mode 100644
index 0000000..6ab1e92
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter_on_meta_with_idx_1.plan
@@ -0,0 +1,25 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- NESTED_LOOP |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter_on_meta_with_idx_2.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter_on_meta_with_idx_2.plan
new file mode 100644
index 0000000..0e83dd0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter_on_meta_with_idx_2.plan
@@ -0,0 +1,25 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$33][$$34] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$33] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$34] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter_on_meta_with_idx_3.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter_on_meta_with_idx_3.plan
new file mode 100644
index 0000000..0e83dd0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/filter_on_meta_with_idx_3.plan
@@ -0,0 +1,25 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$33][$$34] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$33] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- STREAM_SELECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- BROADCAST_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$34] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create_dataset_with_filter_on_meta/create_dataset_with_filter_on_meta.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create_dataset_with_filter_on_meta/create_dataset_with_filter_on_meta.1.ddl.sqlpp
new file mode 100644
index 0000000..909d8d4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create_dataset_with_filter_on_meta/create_dataset_with_filter_on_meta.1.ddl.sqlpp
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"; you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+drop dataverse KeyVerse if exists;
+create dataverse KeyVerse;
+use KeyVerse;
+
+create type DocumentType as open{
+ location: point,
+ area_code: string,
+ text: string
+};
+
+create type KVMetaType as open{
+ `key`:string,
+ vbucket:int32,
+ seq:int64,
+ cas:int64,
+ expiration:int32,
+ flags:int32,
+ revSeq:int64,
+ lockTime:int32
+};
+
+create dataset KVStore(DocumentType) with meta(KVMetaType) primary key meta().`key` with filter on meta().seq;
+
+CREATE INDEX s_rating ON KVStore(area_code);
+CREATE INDEX s_location ON KVStore(location) type RTREE;
+//CREATE INDEX text_idx ON KVStore(text) type fulltext;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create_dataset_with_filter_on_meta/create_dataset_with_filter_on_meta.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create_dataset_with_filter_on_meta/create_dataset_with_filter_on_meta.2.query.sqlpp
new file mode 100644
index 0000000..03926a4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create_dataset_with_filter_on_meta/create_dataset_with_filter_on_meta.2.query.sqlpp
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"; you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+SELECT VALUE i FROM Metadata.`Index` i WHERE i.DataverseName = "KeyVerse";
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create_dataset_with_filter_on_meta/create_dataset_with_filter_on_meta.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create_dataset_with_filter_on_meta/create_dataset_with_filter_on_meta.1.adm
new file mode 100644
index 0000000..fcd415b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create_dataset_with_filter_on_meta/create_dataset_with_filter_on_meta.1.adm
@@ -0,0 +1,3 @@
+{ "DataverseName": "KeyVerse", "DatasetName": "KVStore", "IndexName": "KVStore", "IndexStructure": "BTREE", "SearchKey": [ [ "key" ] ], "IsPrimary": true, "Timestamp": "Mon Sep 07 18:14:11 PDT 2020", "PendingOp": 0, "SearchKeySourceIndicator": [ 1 ] }
+{ "DataverseName": "KeyVerse", "DatasetName": "KVStore", "IndexName": "s_location", "IndexStructure": "RTREE", "SearchKey": [ [ "location" ] ], "IsPrimary": false, "Timestamp": "Mon Sep 07 18:14:11 PDT 2020", "PendingOp": 0 }
+{ "DataverseName": "KeyVerse", "DatasetName": "KVStore", "IndexName": "s_rating", "IndexStructure": "BTREE", "SearchKey": [ [ "area_code" ] ], "IsPrimary": false, "Timestamp": "Mon Sep 07 18:14:11 PDT 2020", "PendingOp": 0 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 687b743..854028e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -4016,6 +4016,11 @@
<output-dir compare="Text">dataset-and-index-same-dataverse</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="ddl">
+ <compilation-unit name="create_dataset_with_filter_on_meta">
+ <output-dir compare="Text">create_dataset_with_filter_on_meta</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="dml">
<test-case FilePath="dml">
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 4e721cb..d84cade 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -728,9 +728,10 @@
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getDeleteRuntime(
IDataSource<DataSourceId> dataSource, IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv,
List<LogicalVariable> keys, LogicalVariable payload, List<LogicalVariable> additionalNonKeyFields,
- RecordDescriptor inputRecordDesc, JobGenContext context, JobSpecification spec) throws AlgebricksException {
+ List<LogicalVariable> additionalNonFilteringFields, RecordDescriptor inputRecordDesc, JobGenContext context,
+ JobSpecification spec) throws AlgebricksException {
return getInsertOrDeleteRuntime(IndexOperation.DELETE, dataSource, propagatedSchema, keys, payload,
- additionalNonKeyFields, inputRecordDesc, context, spec, false, null);
+ additionalNonKeyFields, inputRecordDesc, context, spec, false, additionalNonFilteringFields);
}
@Override
@@ -1085,12 +1086,7 @@
i++;
}
fieldPermutation[i++] = propagatedSchema.findVariable(payload);
- int[] filterFields = new int[numFilterFields];
- if (numFilterFields > 0) {
- int idx = propagatedSchema.findVariable(additionalNonKeyFields.get(0));
- fieldPermutation[i++] = idx;
- filterFields[0] = idx;
- }
+
if (additionalNonFilteringFields != null) {
for (LogicalVariable variable : additionalNonFilteringFields) {
int idx = propagatedSchema.findVariable(variable);
@@ -1098,6 +1094,13 @@
}
}
+ int[] filterFields = new int[numFilterFields];
+ if (numFilterFields > 0) {
+ int idx = propagatedSchema.findVariable(additionalNonKeyFields.get(0));
+ fieldPermutation[i++] = idx;
+ filterFields[0] = idx;
+ }
+
Index primaryIndex = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataset.getDataverseName(),
dataset.getDatasetName(), dataset.getDatasetName());
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint =
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
index 0cff284..bc5ac8b 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java
@@ -277,16 +277,14 @@
}
if (numFilterFields > 0) {
+ ARecordType filterItemType =
+ ((InternalDatasetDetails) dataset.getDatasetDetails()).getFilterSourceIndicator() == 0 ? itemType
+ : metaType;
secondaryFieldAccessEvalFactories[numSecondaryKeys] = metadataProvider.getDataFormat()
- .getFieldAccessEvaluatorFactory(metadataProvider.getFunctionManager(), itemType, filterFieldName,
- numPrimaryKeys, sourceLoc);
+ .getFieldAccessEvaluatorFactory(metadataProvider.getFunctionManager(), filterItemType,
+ filterFieldName, numPrimaryKeys, sourceLoc);
Pair<IAType, Boolean> keyTypePair;
- // since filter is not null, it's safe to cast to internal
- if (((InternalDatasetDetails) dataset.getDatasetDetails()).getFilterSourceIndicator() == 0) {
- keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType);
- } else {
- keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, metaType);
- }
+ keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, filterItemType);
IAType type = keyTypePair.first;
ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type);
secondaryRecFields[numPrimaryKeys + numSecondaryKeys] = serde;
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java
index 01e1af2..40f2610 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java
@@ -186,16 +186,14 @@
}
if (numFilterFields > 0) {
- secondaryFieldAccessEvalFactories[numSecondaryKeys] =
- metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory(
- metadataProvider.getFunctionManager(), itemType, filterFieldName, recordColumn, sourceLoc);
+ ARecordType filterItemType =
+ ((InternalDatasetDetails) dataset.getDatasetDetails()).getFilterSourceIndicator() == 0 ? itemType
+ : metaType;
+ secondaryFieldAccessEvalFactories[numSecondaryKeys] = metadataProvider.getDataFormat()
+ .getFieldAccessEvaluatorFactory(metadataProvider.getFunctionManager(), filterItemType,
+ filterFieldName, recordColumn, sourceLoc);
Pair<IAType, Boolean> keyTypePair;
- // since filter is not null, it's safe to cast to internal
- if (((InternalDatasetDetails) dataset.getDatasetDetails()).getFilterSourceIndicator() == 0) {
- keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType);
- } else {
- keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, metaType);
- }
+ keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, filterItemType);
IAType type = keyTypePair.first;
ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type);
secondaryRecFields[numPrimaryKeys + numSecondaryKeys] = serde;
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedInvertedIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedInvertedIndexOperationsHelper.java
index 734f913..cf84e56 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedInvertedIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedInvertedIndexOperationsHelper.java
@@ -25,6 +25,8 @@
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.metadata.entities.InternalDatasetDetails;
+import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.utils.NonTaggedFormatUtil;
import org.apache.asterix.runtime.utils.RuntimeUtils;
@@ -120,10 +122,13 @@
secondaryTypeTraits[0] = typeTraitProvider.getTypeTrait(secondaryKeyType);
}
if (numFilterFields > 0) {
- secondaryFieldAccessEvalFactories[numSecondaryKeys] =
- metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory(
- metadataProvider.getFunctionManager(), itemType, filterFieldName, recordColumn, sourceLoc);
- Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType);
+ ARecordType filterItemType =
+ ((InternalDatasetDetails) dataset.getDatasetDetails()).getFilterSourceIndicator() == 0 ? itemType
+ : metaType;
+ secondaryFieldAccessEvalFactories[numSecondaryKeys] = metadataProvider.getDataFormat()
+ .getFieldAccessEvaluatorFactory(metadataProvider.getFunctionManager(), filterItemType,
+ filterFieldName, recordColumn, sourceLoc);
+ Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, filterItemType);
IAType type = keyTypePair.first;
ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type);
secondaryRecFields[numPrimaryKeys + numSecondaryKeys] = serde;
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedRTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedRTreeOperationsHelper.java
index c870c60..d10c093 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedRTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedRTreeOperationsHelper.java
@@ -29,6 +29,8 @@
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.metadata.entities.InternalDatasetDetails;
+import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.utils.NonTaggedFormatUtil;
@@ -123,12 +125,15 @@
enforcedRecFields[numPrimaryKeys] = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(itemType);
enforcedRecDesc = new RecordDescriptor(enforcedRecFields, enforcedTypeTraits);
if (numFilterFields > 0) {
+ ARecordType filterItemType =
+ ((InternalDatasetDetails) dataset.getDatasetDetails()).getFilterSourceIndicator() == 0 ? itemType
+ : metaType;
rtreeFields = new int[numNestedSecondaryKeyFields + numPrimaryKeys];
for (int i = 0; i < rtreeFields.length; i++) {
rtreeFields[i] = i;
}
- Pair<IAType, Boolean> typePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType);
+ Pair<IAType, Boolean> typePair = Index.getNonNullableKeyFieldType(filterFieldName, filterItemType);
IAType type = typePair.first;
ISerializerDeserializer serde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(type);
secondaryRecFields[numPrimaryKeys + numNestedSecondaryKeyFields] = serde;
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
index 2ac2048..85205b3 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java
@@ -26,6 +26,8 @@
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.metadata.entities.InternalDatasetDetails;
+import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.utils.NonTaggedFormatUtil;
import org.apache.asterix.runtime.utils.RuntimeUtils;
@@ -122,10 +124,13 @@
secondaryTypeTraits[0] = typeTraitProvider.getTypeTrait(secondaryKeyType);
}
if (numFilterFields > 0) {
+ ARecordType filterItemType =
+ ((InternalDatasetDetails) dataset.getDatasetDetails()).getFilterSourceIndicator() == 0 ? itemType
+ : metaType;
secondaryFieldAccessEvalFactories[numSecondaryKeys] = metadataProvider.getDataFormat()
- .getFieldAccessEvaluatorFactory(metadataProvider.getFunctionManager(), itemType, filterFieldName,
- numPrimaryKeys, sourceLoc);
- Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType);
+ .getFieldAccessEvaluatorFactory(metadataProvider.getFunctionManager(), filterItemType,
+ filterFieldName, numPrimaryKeys, sourceLoc);
+ Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, filterItemType);
IAType type = keyTypePair.first;
ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type);
secondaryRecFields[numPrimaryKeys + numSecondaryKeys] = serde;
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
index fef5097..1d60772 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java
@@ -31,6 +31,8 @@
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.metadata.entities.InternalDatasetDetails;
+import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.utils.NonTaggedFormatUtil;
@@ -139,12 +141,15 @@
enforcedRecFields[numPrimaryKeys] = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(itemType);
enforcedRecDesc = new RecordDescriptor(enforcedRecFields, enforcedTypeTraits);
if (numFilterFields > 0) {
+ ARecordType filterItemType =
+ ((InternalDatasetDetails) dataset.getDatasetDetails()).getFilterSourceIndicator() == 0 ? itemType
+ : metaType;
rtreeFields = new int[numNestedSecondaryKeyFields + numPrimaryKeys];
for (int i = 0; i < rtreeFields.length; i++) {
rtreeFields[i] = i;
}
- Pair<IAType, Boolean> typePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType);
+ Pair<IAType, Boolean> typePair = Index.getNonNullableKeyFieldType(filterFieldName, filterItemType);
IAType type = typePair.first;
ISerializerDeserializer serde = SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(type);
secondaryRecFields[numPrimaryKeys + numNestedSecondaryKeyFields] = serde;
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
index 6d001dc..7aa3640 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
@@ -75,8 +75,9 @@
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> getDeleteRuntime(IDataSource<S> dataSource,
IOperatorSchema propagatedSchema, IVariableTypeEnvironment typeEnv, List<LogicalVariable> keys,
- LogicalVariable payLoadVar, List<LogicalVariable> additionalNonKeyFields, RecordDescriptor inputRecordDesc,
- JobGenContext context, JobSpecification jobSpec) throws AlgebricksException;
+ LogicalVariable payLoadVar, List<LogicalVariable> additionalNonKeyFields,
+ List<LogicalVariable> additionalNonFilteringFields, RecordDescriptor inputRecordDesc, JobGenContext context,
+ JobSpecification jobSpec) throws AlgebricksException;
/**
* Creates the insert runtime of IndexInsertDeletePOperator, which models
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/InsertDeleteUpsertPOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/InsertDeleteUpsertPOperator.java
index 927fb66..1d41291 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/InsertDeleteUpsertPOperator.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/InsertDeleteUpsertPOperator.java
@@ -114,7 +114,7 @@
break;
case DELETE:
runtimeAndConstraints = mp.getDeleteRuntime(dataSource, propagatedSchema, typeEnv, keys, payload,
- additionalFilteringKeys, inputDesc, context, spec);
+ additionalFilteringKeys, additionalNonFilteringFields, inputDesc, context, spec);
break;
case UPSERT:
runtimeAndConstraints = mp.getUpsertRuntime(dataSource, inputSchemas[0], typeEnv, keys, payload,