Add an integer ordered list to dataset "Dataset" and "Index" optinally.
For the case the primary or secondary index key contains fields from the meta
record associated with a dataset record, we need an integer ordered
list to indicate which record a key field comes from.
Change-Id: I979c642ebd60e53213369f1e2070146b4c26e805
Reviewed-on: https://asterix-gerrit.ics.uci.edu/683
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>
Reviewed-by: Michael Blow <michael.blow@couchbase.com>
Reviewed-by: Till Westmann <tillw@apache.org>
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
index ab28e28..5af6ccc 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java
@@ -176,7 +176,7 @@
DataSourceScanOperator scan = new DataSourceScanOperator(v,
createFeedDataSource(asid, targetDataset, sourceFeedName, subscriptionLocation,
metadataProvider, policy, outputType,
- null /* TODO(Adbullah): to figure out the meta type name*/, csLocations));
+ null /* TODO(Abdullah): to figure out the meta type name*/, csLocations));
List<Mutable<ILogicalOperator>> scanInpList = scan.getInputs();
scanInpList.addAll(unnest.getInputs());
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
index 5ede3bb..e85fecf 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/util/EquivalenceClassUtils.java
@@ -96,16 +96,21 @@
metaFieldNameToIndexMap.put(metaFieldNames[metaFieldIndex], metaFieldIndex);
}
}
+ List<Integer> keySourceIndicators = datasetDetails.getKeySourceIndicator();
LogicalVariable recordVar = hasMeta ? indexSearchVars.get(indexSearchVars.size() - 2)
: indexSearchVars.get(indexSearchVars.size() - 1);
LogicalVariable metaRecordVar = hasMeta ? indexSearchVars.get(indexSearchVars.size() - 1) : null;
for (int pkIndex = 0; pkIndex < primaryKey.size(); ++pkIndex) {
LogicalVariable referredRecordVar = recordVar;
String pkFieldName = primaryKey.get(pkIndex).get(0);
- Integer fieldIndexInRecord = fieldNameToIndexMap.get(pkFieldName);
- if (fieldIndexInRecord == null && hasMeta) {
+ int source = keySourceIndicators == null ? 0 : keySourceIndicators.get(pkIndex);
+ Integer fieldIndexInRecord;
+ if (source == 0) {
+ // The field is from the main record.
+ fieldIndexInRecord = fieldNameToIndexMap.get(pkFieldName);
+ } else {
+ // The field is from the auxiliary meta record.
referredRecordVar = metaRecordVar;
- pkFieldName = primaryKey.get(pkIndex).get(1);
fieldIndexInRecord = metaFieldNameToIndexMap.get(pkFieldName);
}
LogicalVariable var = indexSearchVars.get(pkIndex);
diff --git a/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java b/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
index f4691f1..95a8802 100644
--- a/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
+++ b/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
@@ -35,8 +35,6 @@
*/
public class ValidateUtil {
- private static final String META = "meta()";
-
/**
* Validates the field that will be used as filter for the components of an LSM index.
*
@@ -89,7 +87,8 @@
* (if the validation failed), IOException
*/
public static List<IAType> validatePartitioningExpressions(ARecordType recType, ARecordType metaRecType,
- List<List<String>> partitioningExprs, boolean autogenerated) throws AsterixException, IOException {
+ List<List<String>> partitioningExprs, List<Integer> keySourceIndicators, boolean autogenerated)
+ throws AsterixException, IOException {
List<IAType> partitioningExprTypes = new ArrayList<IAType>(partitioningExprs.size());
if (autogenerated) {
if (partitioningExprs.size() > 1) {
@@ -107,12 +106,8 @@
} else {
for (int i = 0; i < partitioningExprs.size(); i++) {
List<String> fieldName = partitioningExprs.get(i);
- List<String> metaFieldName = null;
- boolean useMeta = fieldName.get(0).equals(META);
- if (useMeta) {
- metaFieldName = fieldName.subList(1, fieldName.size());
- }
- IAType fieldType = useMeta ? metaRecType.getSubFieldType(metaFieldName)
+ boolean useMeta = keySourceIndicators.get(i) > 0;
+ IAType fieldType = useMeta ? metaRecType.getSubFieldType(fieldName)
: recType.getSubFieldType(fieldName);
switch (fieldType.getTypeTag()) {
case INT8:
diff --git a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
index 87859dc..75f325b 100644
--- a/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
+++ b/asterix-app/src/main/java/org/apache/asterix/aql/translator/QueryTranslator.java
@@ -595,10 +595,12 @@
List<List<String>> partitioningExprs = ((InternalDetailsDecl) dd.getDatasetDetailsDecl())
.getPartitioningExprs();
+ List<Integer> keySourceIndicators = ((InternalDetailsDecl) dd.getDatasetDetailsDecl())
+ .getKeySourceIndicators();
boolean autogenerated = ((InternalDetailsDecl) dd.getDatasetDetailsDecl()).isAutogenerated();
ARecordType aRecordType = (ARecordType) itemType;
List<IAType> partitioningTypes = ValidateUtil.validatePartitioningExpressions(aRecordType,
- metaRecType, partitioningExprs, autogenerated);
+ metaRecType, partitioningExprs, keySourceIndicators, autogenerated);
List<String> filterField = ((InternalDetailsDecl) dd.getDatasetDetailsDecl()).getFilterField();
if (filterField != null) {
@@ -615,7 +617,7 @@
}
datasetDetails = new InternalDatasetDetails(InternalDatasetDetails.FileStructure.BTREE,
InternalDatasetDetails.PartitioningStrategy.HASH, partitioningExprs, partitioningExprs,
- partitioningTypes, autogenerated, filterField, temp);
+ keySourceIndicators, partitioningTypes, autogenerated, filterField, temp);
break;
}
case EXTERNAL: {
@@ -957,7 +959,7 @@
//Add an entry for the files index
filesIndex = new Index(dataverseName, datasetName,
ExternalIndexingOperations.getFilesIndexName(datasetName), IndexType.BTREE,
- ExternalIndexingOperations.FILE_INDEX_FIELD_NAMES,
+ ExternalIndexingOperations.FILE_INDEX_FIELD_NAMES, null,
ExternalIndexingOperations.FILE_INDEX_FIELD_TYPES, false, false,
IMetadataEntity.PENDING_ADD_OP);
MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(), filesIndex);
@@ -993,8 +995,8 @@
//#. add a new index with PendingAddOp
Index index = new Index(dataverseName, datasetName, indexName, stmtCreateIndex.getIndexType(), indexFields,
- indexFieldTypes, stmtCreateIndex.getGramLength(), stmtCreateIndex.isEnforced(), false,
- IMetadataEntity.PENDING_ADD_OP);
+ stmtCreateIndex.getFieldSourceIndicators(), indexFieldTypes, stmtCreateIndex.getGramLength(),
+ stmtCreateIndex.isEnforced(), false, IMetadataEntity.PENDING_ADD_OP);
MetadataManager.INSTANCE.addIndex(metadataProvider.getMetadataTxnContext(), index);
ARecordType enforcedType = null;
@@ -1580,8 +1582,8 @@
MetadataManager.INSTANCE.dropIndex(mdTxnCtx, dataverseName, datasetName, indexName);
MetadataManager.INSTANCE.addIndex(mdTxnCtx,
new Index(dataverseName, datasetName, indexName, index.getIndexType(), index.getKeyFieldNames(),
- index.getKeyFieldTypes(), index.isEnforcingKeyFileds(), index.isPrimaryIndex(),
- IMetadataEntity.PENDING_DROP_OP));
+ index.getKeyFieldSourceIndicators(), index.getKeyFieldTypes(),
+ index.isEnforcingKeyFileds(), index.isPrimaryIndex(), IMetadataEntity.PENDING_DROP_OP));
//#. commit the existing transaction before calling runJob.
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
@@ -1633,8 +1635,9 @@
MetadataManager.INSTANCE.addIndex(mdTxnCtx,
new Index(dataverseName, datasetName, externalIndex.getIndexName(),
externalIndex.getIndexType(), externalIndex.getKeyFieldNames(),
- index.getKeyFieldTypes(), index.isEnforcingKeyFileds(),
- externalIndex.isPrimaryIndex(), IMetadataEntity.PENDING_DROP_OP));
+ externalIndex.getKeyFieldSourceIndicators(), index.getKeyFieldTypes(),
+ index.isEnforcingKeyFileds(), externalIndex.isPrimaryIndex(),
+ IMetadataEntity.PENDING_DROP_OP));
}
}
}
@@ -1643,8 +1646,8 @@
MetadataManager.INSTANCE.dropIndex(mdTxnCtx, dataverseName, datasetName, indexName);
MetadataManager.INSTANCE.addIndex(mdTxnCtx,
new Index(dataverseName, datasetName, indexName, index.getIndexType(), index.getKeyFieldNames(),
- index.getKeyFieldTypes(), index.isEnforcingKeyFileds(), index.isPrimaryIndex(),
- IMetadataEntity.PENDING_DROP_OP));
+ index.getKeyFieldSourceIndicators(), index.getKeyFieldTypes(),
+ index.isEnforcingKeyFileds(), index.isPrimaryIndex(), IMetadataEntity.PENDING_DROP_OP));
//#. commit the existing transaction before calling runJob.
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
@@ -2947,8 +2950,8 @@
DropStatement dropStmt = new DropStatement(new Identifier(dataverseNameTo), pregelixStmt.getDatasetNameTo(),
true);
this.handleDatasetDropStatement(metadataProvider, dropStmt, hcc);
- IDatasetDetailsDecl idd = new InternalDetailsDecl(toIndex.getKeyFieldNames(), false, null,
- toDataset.getDatasetDetails().isTemp());
+ IDatasetDetailsDecl idd = new InternalDetailsDecl(toIndex.getKeyFieldNames(),
+ toIndex.getKeyFieldSourceIndicators(), false, null, toDataset.getDatasetDetails().isTemp());
DatasetDecl createToDataset = new DatasetDecl(new Identifier(dataverseNameTo),
pregelixStmt.getDatasetNameTo(), new Identifier(toDataset.getItemTypeDataverseName()),
new Identifier(toDataset.getItemTypeName()),
diff --git a/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-6/dataset_with_meta-6.1.ddl.aql b/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-6/dataset_with_meta-6.1.ddl.aql
new file mode 100644
index 0000000..510137f
--- /dev/null
+++ b/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-6/dataset_with_meta-6.1.ddl.aql
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type EmptyType as open {
+ id: string
+}
+
+create type LineType as open {
+ id:int32,
+ text: string
+}
+
+create dataset Book(LineType) with meta(EmptyType)
+primary key id;
+
+create index MetaIndex on Book(meta().id) type btree;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-1/query_dataset_with_meta-5-1.2.update.aql b/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-6/dataset_with_meta-6.2.update.aql
similarity index 100%
copy from asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-1/query_dataset_with_meta-5-1.2.update.aql
copy to asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-6/dataset_with_meta-6.2.update.aql
diff --git a/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-6/dataset_with_meta-6.3.query.aql b/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-6/dataset_with_meta-6.3.query.aql
new file mode 100644
index 0000000..615cb49
--- /dev/null
+++ b/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-6/dataset_with_meta-6.3.query.aql
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+for $x in dataset('Metadata.Index')
+where $x.DataverseName='test'
+return $x
diff --git a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-1/query_dataset_with_meta-5-1.4.ddl.aql b/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-6/dataset_with_meta-6.4.ddl.aql
similarity index 100%
copy from asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-1/query_dataset_with_meta-5-1.4.ddl.aql
copy to asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-6/dataset_with_meta-6.4.ddl.aql
diff --git a/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-7/dataset_with_meta-6.1.ddl.aql b/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-7/dataset_with_meta-6.1.ddl.aql
new file mode 100644
index 0000000..41b50d0
--- /dev/null
+++ b/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-7/dataset_with_meta-6.1.ddl.aql
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type EmptyType as open {
+ id: string
+}
+
+create type LineType as open {
+ id:int32,
+ text: string
+}
+
+create dataset Book(LineType) with meta(EmptyType)
+primary key id;
+
+create index MetaIndex on Book(meta().id, id) type btree;
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-1/query_dataset_with_meta-5-1.2.update.aql b/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-7/dataset_with_meta-6.2.update.aql
similarity index 100%
copy from asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-1/query_dataset_with_meta-5-1.2.update.aql
copy to asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-7/dataset_with_meta-6.2.update.aql
diff --git a/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-7/dataset_with_meta-6.3.query.aql b/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-7/dataset_with_meta-6.3.query.aql
new file mode 100644
index 0000000..615cb49
--- /dev/null
+++ b/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-7/dataset_with_meta-6.3.query.aql
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+for $x in dataset('Metadata.Index')
+where $x.DataverseName='test'
+return $x
diff --git a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-1/query_dataset_with_meta-5-1.4.ddl.aql b/asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-7/dataset_with_meta-6.4.ddl.aql
similarity index 100%
copy from asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-1/query_dataset_with_meta-5-1.4.ddl.aql
copy to asterix-app/src/test/resources/metadata/queries/basic/dataset_with_meta-7/dataset_with_meta-6.4.ddl.aql
diff --git a/asterix-app/src/test/resources/metadata/queries/basic/meta01/meta01.1.ddl.aql b/asterix-app/src/test/resources/metadata/queries/basic/meta01/meta01.1.ddl.aql
index 5df40a8..ccb5b3e 100644
--- a/asterix-app/src/test/resources/metadata/queries/basic/meta01/meta01.1.ddl.aql
+++ b/asterix-app/src/test/resources/metadata/queries/basic/meta01/meta01.1.ddl.aql
@@ -22,6 +22,7 @@
* Date : 15 Sep 2012
*/
+drop dataverse test if exists;
drop dataverse testdv if exists;
create dataverse testdv;
diff --git a/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-4/dataset_with_meta-4.3.adm b/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-4/dataset_with_meta-4.3.adm
index ec8c3bf..581328f 100644
--- a/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-4/dataset_with_meta-4.3.adm
+++ b/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-4/dataset_with_meta-4.3.adm
@@ -1 +1 @@
-{ "DataverseName": "test", "DatasetName": "Book", "DatatypeDataverseName": "test", "DatatypeName": "LineType", "DatasetType": "INTERNAL", "GroupName": "DEFAULT_NG_ALL_NODES", "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "meta()", "id" ] ], "PrimaryKey": [ [ "meta()", "id" ] ], "Autogenerated": false }, "ExternalDetails": null, "Hints": {{ }}, "Timestamp": "Wed Feb 24 22:29:39 PST 2016", "DatasetId": 101i32, "PendingOp": 0i32, "MetatypeDataverseName": "test", "MetatypeName": "LineType" }
+{ "DataverseName": "test", "DatasetName": "Book", "DatatypeDataverseName": "test", "DatatypeName": "LineType", "DatasetType": "INTERNAL", "GroupName": "DEFAULT_NG_ALL_NODES", "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "id" ] ], "PrimaryKey": [ [ "id" ] ], "Autogenerated": false, "KeySourceIndicator": [ 1i8 ] }, "ExternalDetails": null, "Hints": {{ }}, "Timestamp": "Mon Feb 29 15:58:41 PST 2016", "DatasetId": 104i32, "PendingOp": 0i32, "MetatypeDataverseName": "test", "MetatypeName": "LineType" }
diff --git a/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-5/dataset_with_meta-5.3.adm b/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-5/dataset_with_meta-5.3.adm
index 722f3a5..485ca4e 100644
--- a/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-5/dataset_with_meta-5.3.adm
+++ b/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-5/dataset_with_meta-5.3.adm
@@ -1 +1 @@
-{ "DataverseName": "test", "DatasetName": "Book", "DatatypeDataverseName": "test", "DatatypeName": "LineType", "DatasetType": "INTERNAL", "GroupName": "DEFAULT_NG_ALL_NODES", "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "meta()", "key" ] ], "PrimaryKey": [ [ "meta()", "key" ] ], "Autogenerated": false }, "ExternalDetails": null, "Hints": {{ }}, "Timestamp": "Wed Feb 24 22:55:29 PST 2016", "DatasetId": 105i32, "PendingOp": 0i32, "MetatypeDataverseName": "test", "MetatypeName": "EmptyType" }
+{ "DataverseName": "test", "DatasetName": "Book", "DatatypeDataverseName": "test", "DatatypeName": "LineType", "DatasetType": "INTERNAL", "GroupName": "DEFAULT_NG_ALL_NODES", "CompactionPolicy": "prefix", "CompactionPolicyProperties": [ { "Name": "max-mergable-component-size", "Value": "1073741824" }, { "Name": "max-tolerance-component-count", "Value": "5" } ], "InternalDetails": { "FileStructure": "BTREE", "PartitioningStrategy": "HASH", "PartitioningKey": [ [ "key" ] ], "PrimaryKey": [ [ "key" ] ], "Autogenerated": false, "KeySourceIndicator": [ 1i8 ] }, "ExternalDetails": null, "Hints": {{ }}, "Timestamp": "Mon Feb 29 15:58:41 PST 2016", "DatasetId": 105i32, "PendingOp": 0i32, "MetatypeDataverseName": "test", "MetatypeName": "EmptyType" }
diff --git a/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-6/dataset_with_meta-6.3.adm b/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-6/dataset_with_meta-6.3.adm
new file mode 100644
index 0000000..1c33fc9
--- /dev/null
+++ b/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-6/dataset_with_meta-6.3.adm
@@ -0,0 +1,2 @@
+{ "DataverseName": "test", "DatasetName": "Book", "IndexName": "Book", "IndexStructure": "BTREE", "SearchKey": [ [ "id" ] ], "IsPrimary": true, "Timestamp": "Mon Feb 29 18:41:47 PST 2016", "PendingOp": 0i32 }
+{ "DataverseName": "test", "DatasetName": "Book", "IndexName": "MetaIndex", "IndexStructure": "BTREE", "SearchKey": [ [ "id" ] ], "IsPrimary": false, "Timestamp": "Mon Feb 29 18:42:53 PST 2016", "PendingOp": 0i32, "SearchKeySourceIndicator": [ 1i8 ] }
diff --git a/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-7/dataset_with_meta-7.3.adm b/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-7/dataset_with_meta-7.3.adm
new file mode 100644
index 0000000..52dc45a
--- /dev/null
+++ b/asterix-app/src/test/resources/metadata/results/basic/dataset_with_meta-7/dataset_with_meta-7.3.adm
@@ -0,0 +1,2 @@
+{ "DataverseName": "test", "DatasetName": "Book", "IndexName": "Book", "IndexStructure": "BTREE", "SearchKey": [ [ "id" ] ], "IsPrimary": true, "Timestamp": "Mon Feb 29 18:45:56 PST 2016", "PendingOp": 0i32 }
+{ "DataverseName": "test", "DatasetName": "Book", "IndexName": "MetaIndex", "IndexStructure": "BTREE", "SearchKey": [ [ "id" ], [ "id" ] ], "IsPrimary": false, "Timestamp": "Mon Feb 29 18:45:57 PST 2016", "PendingOp": 0i32, "SearchKeySourceIndicator": [ 1i8, 0i8 ] }
diff --git a/asterix-app/src/test/resources/metadata/testsuite.xml b/asterix-app/src/test/resources/metadata/testsuite.xml
index 550d93d..595eb84 100644
--- a/asterix-app/src/test/resources/metadata/testsuite.xml
+++ b/asterix-app/src/test/resources/metadata/testsuite.xml
@@ -44,6 +44,16 @@
</compilation-unit>
</test-case>
<test-case FilePath="basic">
+ <compilation-unit name="dataset_with_meta-6">
+ <output-dir compare="Text">dataset_with_meta-6</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="basic">
+ <compilation-unit name="dataset_with_meta-7">
+ <output-dir compare="Text">dataset_with_meta-7</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="basic">
<compilation-unit name="meta01">
<output-dir compare="Text">meta01</output-dir>
</compilation-unit>
diff --git a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-1/query_dataset_with_meta-5-1.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-1/query_dataset_with_meta-1.1.ddl.aql
similarity index 100%
rename from asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-1/query_dataset_with_meta-5-1.1.ddl.aql
rename to asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-1/query_dataset_with_meta-1.1.ddl.aql
diff --git a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-1/query_dataset_with_meta-5-1.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-1/query_dataset_with_meta-1.2.update.aql
similarity index 100%
rename from asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-1/query_dataset_with_meta-5-1.2.update.aql
rename to asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-1/query_dataset_with_meta-1.2.update.aql
diff --git a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-1/query_dataset_with_meta-5-1.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-1/query_dataset_with_meta-1.3.query.aql
similarity index 100%
rename from asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-1/query_dataset_with_meta-5-1.3.query.aql
rename to asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-1/query_dataset_with_meta-1.3.query.aql
diff --git a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-1/query_dataset_with_meta-5-1.4.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-1/query_dataset_with_meta-1.4.ddl.aql
similarity index 100%
rename from asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-1/query_dataset_with_meta-5-1.4.ddl.aql
rename to asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-1/query_dataset_with_meta-1.4.ddl.aql
diff --git a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-2/query_dataset_with_meta-5-2.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-2/query_dataset_with_meta-2.1.ddl.aql
similarity index 100%
rename from asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-2/query_dataset_with_meta-5-2.1.ddl.aql
rename to asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-2/query_dataset_with_meta-2.1.ddl.aql
diff --git a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-2/query_dataset_with_meta-5-2.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-2/query_dataset_with_meta-2.2.update.aql
similarity index 100%
rename from asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-2/query_dataset_with_meta-5-2.2.update.aql
rename to asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-2/query_dataset_with_meta-2.2.update.aql
diff --git a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-2/query_dataset_with_meta-5-2.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-2/query_dataset_with_meta-2.3.query.aql
similarity index 100%
rename from asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-2/query_dataset_with_meta-5-2.3.query.aql
rename to asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-2/query_dataset_with_meta-2.3.query.aql
diff --git a/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-2/query_dataset_with_meta-5-2.4.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-2/query_dataset_with_meta-2.4.ddl.aql
similarity index 100%
rename from asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-2/query_dataset_with_meta-5-2.4.ddl.aql
rename to asterix-app/src/test/resources/runtimets/queries/meta/query_dataset_with_meta-2/query_dataset_with_meta-2.4.ddl.aql
diff --git a/asterix-lang-aql/src/main/javacc/AQL.html b/asterix-lang-aql/src/main/javacc/AQL.html
index 3da5e77..d3670d6 100644
--- a/asterix-lang-aql/src/main/javacc/AQL.html
+++ b/asterix-lang-aql/src/main/javacc/AQL.html
@@ -342,7 +342,7 @@
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod21">DatasetSpecification</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>( "external" <DATASET> <A HREF="#prod30">QualifiedName</A> <LEFTPAREN> <A HREF="#prod27">TypeName</A> <RIGHTPAREN> <A HREF="#prod28">IfNotExists</A> "using" <A HREF="#prod31">AdapterName</A> <A HREF="#prod32">Configuration</A> ( "on" <A HREF="#prod18">Identifier</A> )? ( "hints" <A HREF="#prod33">Properties</A> )? ( "using" "compaction" "policy" <A HREF="#prod34">CompactionPolicy</A> ( <A HREF="#prod32">Configuration</A> )? )? | ( "internal" | "temporary" )? <DATASET> <A HREF="#prod30">QualifiedName</A> <LEFTPAREN> <A HREF="#prod27">TypeName</A> ( <COMMA> <A HREF="#prod27">TypeName</A> )? <RIGHTPAREN> <A HREF="#prod28">IfNotExists</A> <A HREF="#prod35">PrimaryKey</A> ( "autogenerated" )? ( "on" <A HREF="#prod18">Identifier</A> )? ( "hints" <A HREF="#prod33">Properties</A> )? ( "using" "compaction" "policy" <A HREF="#prod34">CompactionPolicy</A> ( <A HREF="#prod32">Configuration</A> )? )? ( "with filter on" <A HREF="#prod36">NestedField</A> )? )</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( "external" <DATASET> <A HREF="#prod30">QualifiedName</A> <LEFTPAREN> <A HREF="#prod27">TypeName</A> <RIGHTPAREN> <A HREF="#prod28">IfNotExists</A> "using" <A HREF="#prod31">AdapterName</A> <A HREF="#prod32">Configuration</A> ( "on" <A HREF="#prod18">Identifier</A> )? ( "hints" <A HREF="#prod33">Properties</A> )? ( "using" "compaction" "policy" <A HREF="#prod34">CompactionPolicy</A> ( <A HREF="#prod32">Configuration</A> )? )? | ( "internal" | "temporary" )? <DATASET> <A HREF="#prod30">QualifiedName</A> <LEFTPAREN> <A HREF="#prod27">TypeName</A> <RIGHTPAREN> ( <WITH> <A HREF="#prod18">Identifier</A> <LEFTPAREN> <A HREF="#prod27">TypeName</A> <RIGHTPAREN> )? <A HREF="#prod28">IfNotExists</A> <A HREF="#prod35">PrimaryKey</A> ( "autogenerated" )? ( "on" <A HREF="#prod18">Identifier</A> )? ( "hints" <A HREF="#prod33">Properties</A> )? ( "using" "compaction" "policy" <A HREF="#prod34">CompactionPolicy</A> ( <A HREF="#prod32">Configuration</A> )? )? ( "with filter on" <A HREF="#prod36">NestedField</A> )? )</TD>
</TR>
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod16">RefreshExternalDatasetStatement</A></TD>
@@ -562,7 +562,7 @@
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod36">NestedField</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>( "@" )? <A HREF="#prod18">Identifier</A> ( <DOT> <A HREF="#prod18">Identifier</A> )*</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod18">Identifier</A> ( <LEFTPAREN> <RIGHTPAREN> )? ( <DOT> <A HREF="#prod18">Identifier</A> )*</TD>
</TR>
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod40">StringLiteral</A></TD>
diff --git a/asterix-lang-aql/src/main/javacc/AQL.jj b/asterix-lang-aql/src/main/javacc/AQL.jj
index ea95871..8205021 100644
--- a/asterix-lang-aql/src/main/javacc/AQL.jj
+++ b/asterix-lang-aql/src/main/javacc/AQL.jj
@@ -408,14 +408,14 @@
Map<String,String> properties = null;
Map<String,String> compactionPolicyProperties = null;
FunctionSignature appliedFunction = null;
- List<List<String>> primaryKeyFields = null;
+ Pair<List<Integer>, List<List<String>>> primaryKeyFields = null;
String nodeGroupName = null;
Map<String,String> hints = new HashMap<String,String>();
DatasetDecl dsetDecl = null;
boolean autogenerated = false;
String compactionPolicy = null;
boolean temp = false;
- List<String> filterField = null;
+ Pair<Integer, List<String>> filterField = null;
Pair<Identifier,Identifier> metaTypeComponents = new Pair<Identifier, Identifier>(null, null);
}
{
@@ -471,9 +471,13 @@
( "using" "compaction" "policy" compactionPolicy = CompactionPolicy() (compactionPolicyProperties = Configuration())? )?
( "with filter on" filterField = NestedField() )?
{
- InternalDetailsDecl idd = new InternalDetailsDecl(primaryKeyFields,
+ if(filterField!=null && filterField.first!=0){
+ throw new ParseException("A filter field can only be a field in the main record of the dataset.");
+ }
+ InternalDetailsDecl idd = new InternalDetailsDecl(primaryKeyFields.second,
+ primaryKeyFields.first,
autogenerated,
- filterField,
+ filterField == null? null : filterField.second,
temp);
dsetDecl = new DatasetDecl(nameComponents.first,
nameComponents.second,
@@ -537,7 +541,7 @@
String indexName = null;
boolean ifNotExists = false;
Pair<Identifier,Identifier> nameComponents = null;
- Pair<List<String>, TypeExpression> fieldPair = null;
+ Pair<Integer, Pair<List<String>, TypeExpression>> fieldPair = null;
IndexParams indexType = null;
boolean enforced = false;
}
@@ -547,11 +551,13 @@
"on" nameComponents = QualifiedName()
<LEFTPAREN> ( fieldPair = OpenField()
{
- cis.addFieldExprPair(fieldPair);
+ cis.addFieldExprPair(fieldPair.second);
+ cis.addFieldIndexIndicator(fieldPair.first);
}
) (<COMMA> fieldPair = OpenField()
{
- cis.addFieldExprPair(fieldPair);
+ cis.addFieldExprPair(fieldPair.second);
+ cis.addFieldIndexIndicator(fieldPair.first);
}
)* <RIGHTPAREN> ( "type" indexType = IndexType() )? ( "enforced" { enforced = true; } )?
{
@@ -817,23 +823,26 @@
}
}
-List<List<String>> PrimaryKey() throws ParseException:
+Pair<List<Integer>, List<List<String>>> PrimaryKey() throws ParseException:
{
- List<String> tmp = null;
- List<List<String>> primaryKeyFields = new ArrayList<List<String>>();
+ Pair<Integer, List<String>> tmp = null;
+ List<Integer> keyFieldSourceIndicators = new ArrayList<Integer>();
+ List<List<String>> primaryKeyFields = new ArrayList<List<String>>();
}
{
"primary" "key" tmp = NestedField()
{
- primaryKeyFields.add(tmp);
+ keyFieldSourceIndicators.add(tmp.first);
+ primaryKeyFields.add(tmp.second);
}
( <COMMA> tmp = NestedField()
{
- primaryKeyFields.add(tmp);
+ keyFieldSourceIndicators.add(tmp.first);
+ primaryKeyFields.add(tmp.second);
}
)*
{
- return primaryKeyFields;
+ return new Pair<List<Integer>, List<List<String>>> (keyFieldSourceIndicators, primaryKeyFields);
}
}
@@ -1381,23 +1390,25 @@
)
}
-Pair<List<String>, TypeExpression> OpenField() throws ParseException:
+Pair<Integer, Pair<List<String>, TypeExpression>> OpenField() throws ParseException:
{
TypeExpression fieldType = null;
- List<String> fieldList = null;
+ Pair<Integer, List<String>> fieldList = null;
}
{
fieldList = NestedField()
( <COLON> fieldType = IndexedTypeExpr() )?
{
- return new Pair<List<String>, TypeExpression>(fieldList, fieldType);
+ return new Pair<Integer, Pair<List<String>, TypeExpression>>
+ (fieldList.first, new Pair<List<String>, TypeExpression>(fieldList.second, fieldType));
}
}
-List<String> NestedField() throws ParseException:
+Pair<Integer, List<String>> NestedField() throws ParseException:
{
List<String> exprList = new ArrayList<String>();
String lit = null;
+ int source = 0;
}
{
lit = Identifier()
@@ -1405,14 +1416,14 @@
boolean meetParens = false;
}
(
+ LOOKAHEAD(1)
<LEFTPAREN><RIGHTPAREN>
{
- if(lit.equals("meta")){
- exprList.add(lit+"()");
- }else{
+ if(!lit.equals("meta")){
throw new ParseException("The string before () has to be \"meta\".");
}
meetParens = true;
+ source = 1;
}
)?
{
@@ -1427,7 +1438,7 @@
}
)*
{
- return exprList;
+ return new Pair<Integer, List<String>>(source, exprList);
}
}
diff --git a/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java b/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java
index 6e8f41e..d3d681a 100644
--- a/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java
+++ b/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateIndexStatement.java
@@ -34,7 +34,8 @@
private Identifier indexName;
private Identifier dataverseName;
private Identifier datasetName;
- private List<Pair<List<String>, TypeExpression>> fieldExprs = new ArrayList<Pair<List<String>, TypeExpression>>();
+ private List<Pair<List<String>, TypeExpression>> fieldExprs = new ArrayList<>();
+ private List<Integer> fieldIndexIndicators = new ArrayList<>();
private IndexType indexType = IndexType.BTREE;
private boolean enforced;
private boolean ifNotExists;
@@ -85,6 +86,14 @@
this.fieldExprs.add(fp);
}
+ public List<Integer> getFieldSourceIndicators() {
+ return fieldIndexIndicators;
+ }
+
+ public void addFieldIndexIndicator(Integer index) {
+ fieldIndexIndicators.add(index);
+ }
+
public IndexType getIndexType() {
return indexType;
}
diff --git a/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FeedDetailsDecl.java b/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FeedDetailsDecl.java
deleted file mode 100644
index 1a42aa3..0000000
--- a/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/FeedDetailsDecl.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.lang.common.statement;
-
-import java.util.List;
-import java.util.Map;
-
-import org.apache.asterix.common.functions.FunctionSignature;
-
-public class FeedDetailsDecl extends InternalDetailsDecl {
- private final Map<String, String> configuration;
- private final String adapterFactoryClassname;
- private final FunctionSignature functionSignature;
-
- public FeedDetailsDecl(String adapterFactoryClassname, Map<String, String> configuration,
- FunctionSignature signature, List<List<String>> partitioningExpr, List<String> filterField) {
- super(partitioningExpr, false, filterField, false);
- this.adapterFactoryClassname = adapterFactoryClassname;
- this.configuration = configuration;
- this.functionSignature = signature;
- }
-
- public Map<String, String> getConfiguration() {
- return configuration;
- }
-
- public String getAdapterFactoryClassname() {
- return adapterFactoryClassname;
- }
-
- public FunctionSignature getSignature() {
- return functionSignature;
- }
-
- public FunctionSignature getFunctionSignature() {
- return functionSignature;
- }
-}
diff --git a/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/InternalDetailsDecl.java b/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/InternalDetailsDecl.java
index 431f9fb..41d97a8 100644
--- a/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/InternalDetailsDecl.java
+++ b/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/InternalDetailsDecl.java
@@ -22,13 +22,15 @@
public class InternalDetailsDecl implements IDatasetDetailsDecl {
private final List<List<String>> partitioningExprs;
+ private final List<Integer> keySourceIndicators;
private final boolean autogenerated;
private final boolean temp;
private final List<String> filterField;
- public InternalDetailsDecl(List<List<String>> partitioningExpr, boolean autogenerated, List<String> filterField,
- boolean temp) {
+ public InternalDetailsDecl(List<List<String>> partitioningExpr, List<Integer> keySourceIndicators,
+ boolean autogenerated, List<String> filterField, boolean temp) {
this.partitioningExprs = partitioningExpr;
+ this.keySourceIndicators = keySourceIndicators;
this.autogenerated = autogenerated;
this.filterField = filterField;
this.temp = temp;
@@ -38,6 +40,10 @@
return partitioningExprs;
}
+ public List<Integer> getKeySourceIndicators() {
+ return keySourceIndicators;
+ }
+
public boolean isAutogenerated() {
return autogenerated;
}
diff --git a/asterix-lang-sqlpp/src/main/javacc/SQLPP.html b/asterix-lang-sqlpp/src/main/javacc/SQLPP.html
index 42aaa06..ebe7926 100644
--- a/asterix-lang-sqlpp/src/main/javacc/SQLPP.html
+++ b/asterix-lang-sqlpp/src/main/javacc/SQLPP.html
@@ -403,7 +403,7 @@
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod21">DatasetSpecification</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>( <EXTERNAL> <DATASET> <A HREF="#prod30">QualifiedName</A> <LEFTPAREN> <A HREF="#prod27">TypeName</A> <RIGHTPAREN> <A HREF="#prod28">IfNotExists</A> <USING> <A HREF="#prod31">AdapterName</A> <A HREF="#prod32">Configuration</A> ( <ON> <A HREF="#prod18">Identifier</A> )? ( <HINTS> <A HREF="#prod33">Properties</A> )? ( <USING> <COMPACTION> <POLICY> <A HREF="#prod34">CompactionPolicy</A> ( <A HREF="#prod32">Configuration</A> )? )? | ( <INTERNAL> | <TEMPORARY> )? <DATASET> <A HREF="#prod30">QualifiedName</A> <LEFTPAREN> <A HREF="#prod27">TypeName</A> ( <COMMA> <A HREF="#prod27">TypeName</A> )? <RIGHTPAREN> <A HREF="#prod28">IfNotExists</A> <A HREF="#prod35">PrimaryKey</A> ( <AUTOGENERATED> )? ( <ON> <A HREF="#prod18">Identifier</A> )? ( <HINTS> <A HREF="#prod33">Properties</A> )? ( <USING> <COMPACTION> <POLICY> <A HREF="#prod34">CompactionPolicy</A> ( <A HREF="#prod32">Configuration</A> )? )? ( <WITH> <FILTER> <ON> <A HREF="#prod36">NestedField</A> )? )</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <EXTERNAL> <DATASET> <A HREF="#prod30">QualifiedName</A> <LEFTPAREN> <A HREF="#prod27">TypeName</A> <RIGHTPAREN> <A HREF="#prod28">IfNotExists</A> <USING> <A HREF="#prod31">AdapterName</A> <A HREF="#prod32">Configuration</A> ( <ON> <A HREF="#prod18">Identifier</A> )? ( <HINTS> <A HREF="#prod33">Properties</A> )? ( <USING> <COMPACTION> <POLICY> <A HREF="#prod34">CompactionPolicy</A> ( <A HREF="#prod32">Configuration</A> )? )? | ( <INTERNAL> | <TEMPORARY> )? <DATASET> <A HREF="#prod30">QualifiedName</A> <LEFTPAREN> <A HREF="#prod27">TypeName</A> <RIGHTPAREN> ( <WITH> <A HREF="#prod18">Identifier</A> <LEFTPAREN> <A HREF="#prod27">TypeName</A> <RIGHTPAREN> )? <A HREF="#prod28">IfNotExists</A> <A HREF="#prod35">PrimaryKey</A> ( <AUTOGENERATED> )? ( <ON> <A HREF="#prod18">Identifier</A> )? ( <HINTS> <A HREF="#prod33">Properties</A> )? ( <USING> <COMPACTION> <POLICY> <A HREF="#prod34">CompactionPolicy</A> ( <A HREF="#prod32">Configuration</A> )? )? ( <WITH> <FILTER> <ON> <A HREF="#prod36">NestedField</A> )? )</TD>
</TR>
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod16">RefreshExternalDatasetStatement</A></TD>
@@ -623,7 +623,7 @@
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod36">NestedField</A></TD>
<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
-<TD ALIGN=LEFT VALIGN=BASELINE>( <ATT> )? <A HREF="#prod18">Identifier</A> ( <DOT> <A HREF="#prod18">Identifier</A> )*</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod18">Identifier</A> ( <LEFTPAREN> <RIGHTPAREN> )? ( <DOT> <A HREF="#prod18">Identifier</A> )*</TD>
</TR>
<TR>
<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod40">QuotedString</A></TD>
diff --git a/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index 6259faa..ec915e9 100644
--- a/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -421,14 +421,14 @@
Map<String,String> properties = null;
Map<String,String> compactionPolicyProperties = null;
FunctionSignature appliedFunction = null;
- List<List<String>> primaryKeyFields = null;
+ Pair<List<Integer>, List<List<String>>> primaryKeyFields = null;
String nodeGroupName = null;
Map<String,String> hints = new HashMap<String,String>();
DatasetDecl dsetDecl = null;
boolean autogenerated = false;
String compactionPolicy = null;
boolean temp = false;
- List<String> filterField = null;
+ Pair<Integer, List<String>> filterField = null;
Pair<Identifier,Identifier> metaTypeComponents = new Pair<Identifier, Identifier>(null, null);
}
{
@@ -481,9 +481,13 @@
( <USING> <COMPACTION> <POLICY> compactionPolicy = CompactionPolicy() (LOOKAHEAD(1) compactionPolicyProperties = Configuration())? )?
( LOOKAHEAD(2) <WITH> <FILTER> <ON> filterField = NestedField() )?
{
- InternalDetailsDecl idd = new InternalDetailsDecl(primaryKeyFields,
+ if(filterField!=null && filterField.first!=0){
+ throw new ParseException("A filter field can only be a field in the main record of the dataset.");
+ }
+ InternalDetailsDecl idd = new InternalDetailsDecl(primaryKeyFields.second,
+ primaryKeyFields.first,
autogenerated,
- filterField,
+ filterField == null? null : filterField.second,
temp);
dsetDecl = new DatasetDecl(nameComponents.first,
nameComponents.second,
@@ -547,7 +551,7 @@
String indexName = null;
boolean ifNotExists = false;
Pair<Identifier,Identifier> nameComponents = null;
- Pair<List<String>, TypeExpression> fieldPair = null;
+ Pair<Integer, Pair<List<String>, TypeExpression>> fieldPair = null;
IndexParams indexType = null;
boolean enforced = false;
}
@@ -557,11 +561,13 @@
<ON> nameComponents = QualifiedName()
<LEFTPAREN> ( fieldPair = OpenField()
{
- cis.addFieldExprPair(fieldPair);
+ cis.addFieldExprPair(fieldPair.second);
+ cis.addFieldIndexIndicator(fieldPair.first);
}
) (<COMMA> fieldPair = OpenField()
{
- cis.addFieldExprPair(fieldPair);
+ cis.addFieldExprPair(fieldPair.second);
+ cis.addFieldIndexIndicator(fieldPair.first);
}
)* <RIGHTPAREN> ( <TYPE> indexType = IndexType() )? ( <ENFORCED> { enforced = true; } )?
{
@@ -825,23 +831,26 @@
}
}
-List<List<String>> PrimaryKey() throws ParseException:
+Pair<List<Integer>, List<List<String>>> PrimaryKey() throws ParseException:
{
- List<String> tmp = null;
+ Pair<Integer, List<String>> tmp = null;
+ List<Integer> keyFieldSourceIndicators = new ArrayList<Integer>();
List<List<String>> primaryKeyFields = new ArrayList<List<String>>();
}
{
<PRIMARY> <KEY> tmp = NestedField()
{
- primaryKeyFields.add(tmp);
+ keyFieldSourceIndicators.add(tmp.first);
+ primaryKeyFields.add(tmp.second);
}
( <COMMA> tmp = NestedField()
{
- primaryKeyFields.add(tmp);
+ keyFieldSourceIndicators.add(tmp.first);
+ primaryKeyFields.add(tmp.second);
}
)*
{
- return primaryKeyFields;
+ return new Pair<List<Integer>, List<List<String>>> (keyFieldSourceIndicators, primaryKeyFields);
}
}
@@ -1380,23 +1389,25 @@
)
}
-Pair<List<String>, TypeExpression> OpenField() throws ParseException:
+Pair<Integer, Pair<List<String>, TypeExpression>> OpenField() throws ParseException:
{
TypeExpression fieldType = null;
- List<String> fieldList = null;
+ Pair<Integer, List<String>> fieldList = null;
}
{
fieldList = NestedField()
( <COLON> fieldType = IndexedTypeExpr() )?
{
- return new Pair<List<String>, TypeExpression>(fieldList, fieldType);
+ return new Pair<Integer, Pair<List<String>, TypeExpression>>
+ (fieldList.first, new Pair<List<String>, TypeExpression>(fieldList.second, fieldType));
}
}
-List<String> NestedField() throws ParseException:
+Pair<Integer, List<String>> NestedField() throws ParseException:
{
List<String> exprList = new ArrayList<String>();
String lit = null;
+ int source = 0;
}
{
lit = Identifier()
@@ -1404,14 +1415,14 @@
boolean meetParens = false;
}
(
+ LOOKAHEAD(1)
<LEFTPAREN><RIGHTPAREN>
{
- if(lit.toLowerCase().equals("meta")){
- exprList.add(lit.toLowerCase() + "()");
- }else{
+ if(!lit.toLowerCase().equals("meta")){
throw new ParseException("The string before () has to be \"meta\".");
}
meetParens = true;
+ source = 1;
}
)?
{
@@ -1426,7 +1437,7 @@
}
)*
{
- return exprList;
+ return new Pair<Integer, List<String>>(source, exprList);
}
}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataCache.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataCache.java
index b1eaf0f..78a7d6f 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataCache.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataCache.java
@@ -156,8 +156,8 @@
if (dataset.getDatasetType() == DatasetType.INTERNAL) {
InternalDatasetDetails id = (InternalDatasetDetails) dataset.getDatasetDetails();
Index index = new Index(dataset.getDataverseName(), dataset.getDatasetName(),
- dataset.getDatasetName(), IndexType.BTREE, id.getPartitioningKey(), id.getPrimaryKeyType(),
- false, true, dataset.getPendingOp());
+ dataset.getDatasetName(), IndexType.BTREE, id.getPartitioningKey(),
+ id.getKeySourceIndicator(), id.getPrimaryKeyType(), false, true, dataset.getPendingOp());
addIndexIfNotExistsInternal(index);
}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
index 088a85b..542c12d 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
@@ -174,7 +174,7 @@
} catch (TreeIndexDuplicateKeyException e) {
throw new MetadataException(
"A dataverse with this name " + dataverse.getDataverseName() + " already exists.", e);
- } catch (ACIDException|IndexException|IOException e) {
+ } catch (ACIDException | IndexException | IOException e) {
throw new MetadataException(e);
}
}
@@ -191,15 +191,15 @@
// Add the primary index for the dataset.
InternalDatasetDetails id = (InternalDatasetDetails) dataset.getDatasetDetails();
Index primaryIndex = new Index(dataset.getDataverseName(), dataset.getDatasetName(),
- dataset.getDatasetName(), IndexType.BTREE, id.getPrimaryKey(), id.getPrimaryKeyType(), false,
- true, dataset.getPendingOp());
+ dataset.getDatasetName(), IndexType.BTREE, id.getPrimaryKey(), id.getKeySourceIndicator(),
+ id.getPrimaryKeyType(), false, true, dataset.getPendingOp());
addIndex(jobId, primaryIndex);
}
} catch (TreeIndexDuplicateKeyException e) {
throw new MetadataException("A dataset with this name " + dataset.getDatasetName()
+ " already exists in dataverse '" + dataset.getDataverseName() + "'.", e);
- } catch (ACIDException|IndexException|IOException e) {
+ } catch (ACIDException | IndexException | IOException e) {
throw new MetadataException(e);
}
}
@@ -212,7 +212,7 @@
insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.INDEX_DATASET, tuple);
} catch (TreeIndexDuplicateKeyException e) {
throw new MetadataException("An index with name '" + index.getIndexName() + "' already exists.", e);
- } catch (ACIDException|IndexException|IOException e) {
+ } catch (ACIDException | IndexException | IOException e) {
throw new MetadataException(e);
}
}
@@ -225,7 +225,7 @@
insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.NODE_DATASET, tuple);
} catch (TreeIndexDuplicateKeyException e) {
throw new MetadataException("A node with name '" + node.getNodeName() + "' already exists.", e);
- } catch (ACIDException|IndexException|IOException e) {
+ } catch (ACIDException | IndexException | IOException e) {
throw new MetadataException(e);
}
}
@@ -239,7 +239,7 @@
} catch (TreeIndexDuplicateKeyException e) {
throw new MetadataException("A nodegroup with name '" + nodeGroup.getNodeGroupName() + "' already exists.",
e);
- } catch (ACIDException|IndexException|IOException e) {
+ } catch (ACIDException | IndexException | IOException e) {
throw new MetadataException(e);
}
}
@@ -252,7 +252,7 @@
insertTupleIntoIndex(jobId, MetadataPrimaryIndexes.DATATYPE_DATASET, tuple);
} catch (TreeIndexDuplicateKeyException e) {
throw new MetadataException("A datatype with name '" + datatype.getDatatypeName() + "' already exists.", e);
- } catch (ACIDException|IndexException|IOException e) {
+ } catch (ACIDException | IndexException | IOException e) {
throw new MetadataException(e);
}
}
@@ -268,7 +268,7 @@
} catch (TreeIndexDuplicateKeyException e) {
throw new MetadataException("A function with this name " + function.getName() + " and arity "
+ function.getArity() + " already exists in dataverse '" + function.getDataverseName() + "'.", e);
- } catch (ACIDException|IndexException|IOException e) {
+ } catch (ACIDException | IndexException | IOException e) {
throw new MetadataException(e);
}
}
@@ -1082,6 +1082,7 @@
// TODO: Can use Hyrack's TupleUtils for this, once we switch to a newer
// Hyracks version.
public ITupleReference createTuple(String... fields) throws HyracksDataException {
+ @SuppressWarnings("unchecked")
ISerializerDeserializer<AString> stringSerde = AqlSerializerDeserializerProvider.INSTANCE
.getSerializerDeserializer(BuiltinType.ASTRING);
AMutableString aString = new AMutableString("");
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java
index 84a75ef..a8c3e44 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataTransactionContext.java
@@ -128,7 +128,7 @@
}
public void dropIndex(String dataverseName, String datasetName, String indexName) {
- Index index = new Index(dataverseName, datasetName, indexName, null, null, null, false, false,
+ Index index = new Index(dataverseName, datasetName, indexName, null, null, null, null, false, false,
IMetadataEntity.PENDING_NO_OP);
droppedCache.addIndexIfNotExists(index);
logAndApply(new MetadataLogicalOperation(index, false));
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
index 3e709bf..ec0d21f 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -226,7 +226,7 @@
public static void insertInitialDatasets(MetadataTransactionContext mdTxnCtx) throws Exception {
for (int i = 0; i < primaryIndexes.length; i++) {
IDatasetDetails id = new InternalDatasetDetails(FileStructure.BTREE, PartitioningStrategy.HASH,
- primaryIndexes[i].getPartitioningExpr(), primaryIndexes[i].getPartitioningExpr(),
+ primaryIndexes[i].getPartitioningExpr(), primaryIndexes[i].getPartitioningExpr(), null,
primaryIndexes[i].getPartitioningExprType(), false, null, false);
MetadataManager.INSTANCE.addDataset(mdTxnCtx, new Dataset(primaryIndexes[i].getDataverseName(),
primaryIndexes[i].getIndexedDatasetName(), primaryIndexes[i].getDataverseName(),
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java
index d401864..db3ccc1 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Index.java
@@ -46,6 +46,7 @@
private final String indexName;
private final IndexType indexType;
private final List<List<String>> keyFieldNames;
+ private final List<Integer> keyFieldSourceIndicators;
private final List<IAType> keyFieldTypes;
private final boolean enforceKeyFields;
private final boolean isPrimaryIndex;
@@ -55,13 +56,14 @@
private int pendingOp;
public Index(String dataverseName, String datasetName, String indexName, IndexType indexType,
- List<List<String>> keyFieldNames, List<IAType> keyFieldTypes, int gramLength, boolean enforceKeyFields,
- boolean isPrimaryIndex, int pendingOp) {
+ List<List<String>> keyFieldNames, List<Integer> keyFieldSourceIndicators, List<IAType> keyFieldTypes,
+ int gramLength, boolean enforceKeyFields, boolean isPrimaryIndex, int pendingOp) {
this.dataverseName = dataverseName;
this.datasetName = datasetName;
this.indexName = indexName;
this.indexType = indexType;
this.keyFieldNames = keyFieldNames;
+ this.keyFieldSourceIndicators = keyFieldSourceIndicators;
this.keyFieldTypes = keyFieldTypes;
this.gramLength = gramLength;
this.enforceKeyFields = enforceKeyFields;
@@ -70,18 +72,10 @@
}
public Index(String dataverseName, String datasetName, String indexName, IndexType indexType,
- List<List<String>> keyFieldNames, List<IAType> keyFieldTypes, boolean enforceKeyFields,
- boolean isPrimaryIndex, int pendingOp) {
- this.dataverseName = dataverseName;
- this.datasetName = datasetName;
- this.indexName = indexName;
- this.indexType = indexType;
- this.keyFieldNames = keyFieldNames;
- this.keyFieldTypes = keyFieldTypes;
- this.gramLength = -1;
- this.enforceKeyFields = enforceKeyFields;
- this.isPrimaryIndex = isPrimaryIndex;
- this.pendingOp = pendingOp;
+ List<List<String>> keyFieldNames, List<Integer> keyFieldSourceIndicators, List<IAType> keyFieldTypes,
+ boolean enforceKeyFields, boolean isPrimaryIndex, int pendingOp) {
+ this(dataverseName, datasetName, indexName, indexType, keyFieldNames, keyFieldSourceIndicators, keyFieldTypes,
+ -1, enforceKeyFields, isPrimaryIndex, pendingOp);
}
public String getDataverseName() {
@@ -100,6 +94,10 @@
return keyFieldNames;
}
+ public List<Integer> getKeyFieldSourceIndicators() {
+ return keyFieldSourceIndicators;
+ }
+
public List<IAType> getKeyFieldTypes() {
return keyFieldTypes;
}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java
index a772c39..6971037 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java
@@ -30,6 +30,8 @@
import org.apache.asterix.metadata.IDatasetDetails;
import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes;
import org.apache.asterix.om.base.ABoolean;
+import org.apache.asterix.om.base.AInt8;
+import org.apache.asterix.om.base.AMutableInt8;
import org.apache.asterix.om.base.AMutableString;
import org.apache.asterix.om.base.AString;
import org.apache.asterix.om.types.AOrderedListType;
@@ -52,24 +54,28 @@
HASH
};
- protected final FileStructure fileStructure;
- protected final PartitioningStrategy partitioningStrategy;
- protected final List<List<String>> partitioningKeys;
- protected final List<List<String>> primaryKeys;
- protected final List<IAType> primaryKeyTypes;
- protected final boolean autogenerated;
- protected final boolean temp;
- protected long lastAccessTime;
- protected final List<String> filterField;
+ private final FileStructure fileStructure;
+ private final PartitioningStrategy partitioningStrategy;
+ private final List<List<String>> partitioningKeys;
+ private final List<List<String>> primaryKeys;
+ private final List<IAType> primaryKeyTypes;
+ private final boolean autogenerated;
+ private final boolean temp;
+ private long lastAccessTime;
+ private final List<String> filterField;
+ private final List<Integer> keySourceIndicators;
+
public static final String FILTER_FIELD_NAME = "FilterField";
+ public static final String KEY_FILD_SOURCE_INDICATOR_FIELD_NAME = "KeySourceIndicator";
public InternalDatasetDetails(FileStructure fileStructure, PartitioningStrategy partitioningStrategy,
- List<List<String>> partitioningKey, List<List<String>> primaryKey, List<IAType> primaryKeyType,
- boolean autogenerated, List<String> filterField, boolean temp) {
+ List<List<String>> partitioningKey, List<List<String>> primaryKey, List<Integer> keyFieldIndicators,
+ List<IAType> primaryKeyType, boolean autogenerated, List<String> filterField, boolean temp) {
this.fileStructure = fileStructure;
this.partitioningStrategy = partitioningStrategy;
this.partitioningKeys = partitioningKey;
this.primaryKeys = primaryKey;
+ this.keySourceIndicators = keyFieldIndicators;
this.primaryKeyTypes = primaryKeyType;
this.autogenerated = autogenerated;
this.filterField = filterField;
@@ -89,6 +95,10 @@
return primaryKeys;
}
+ public List<Integer> getKeySourceIndicator() {
+ return keySourceIndicators;
+ }
+
public List<IAType> getPrimaryKeyType() {
return primaryKeyTypes;
}
@@ -130,14 +140,19 @@
ArrayBackedValueStorage itemValue = new ArrayBackedValueStorage();
OrderedListBuilder primaryKeyListBuilder = new OrderedListBuilder();
AOrderedListType stringList = new AOrderedListType(BuiltinType.ASTRING, null);
+ AOrderedListType int8List = new AOrderedListType(BuiltinType.AINT8, null);
internalRecordBuilder.reset(MetadataRecordTypes.INTERNAL_DETAILS_RECORDTYPE);
AMutableString aString = new AMutableString("");
+ AMutableInt8 aInt8 = new AMutableInt8((byte) 0);
@SuppressWarnings("unchecked")
ISerializerDeserializer<ABoolean> booleanSerde = AqlSerializerDeserializerProvider.INSTANCE
.getSerializerDeserializer(BuiltinType.ABOOLEAN);
@SuppressWarnings("unchecked")
ISerializerDeserializer<AString> stringSerde = AqlSerializerDeserializerProvider.INSTANCE
.getSerializerDeserializer(BuiltinType.ASTRING);
+ @SuppressWarnings("unchecked")
+ ISerializerDeserializer<AInt8> int8Serde = AqlSerializerDeserializerProvider.INSTANCE
+ .getSerializerDeserializer(BuiltinType.AINT8);
// write field 0
fieldValue.reset();
@@ -199,6 +214,7 @@
internalRecordBuilder.addField(MetadataRecordTypes.INTERNAL_DETAILS_ARECORD_AUTOGENERATED_FIELD_INDEX,
fieldValue);
+ // write filter fields if any
List<String> filterField = getFilterField();
if (filterField != null) {
listBuilder.reset(stringList);
@@ -217,6 +233,34 @@
internalRecordBuilder.addField(nameValue, fieldValue);
}
+ // write key source indicators if any
+ List<Integer> keySourceIndicator = getKeySourceIndicator();
+ boolean needSerialization = false;
+ if (keySourceIndicator != null) {
+ for (int source : keySourceIndicator) {
+ if (source != 0) {
+ needSerialization = true;
+ break;
+ }
+ }
+ }
+ if (needSerialization) {
+ listBuilder.reset(int8List);
+ ArrayBackedValueStorage nameValue = new ArrayBackedValueStorage();
+ nameValue.reset();
+ aString.setValue(KEY_FILD_SOURCE_INDICATOR_FIELD_NAME);
+ stringSerde.serialize(aString, nameValue.getDataOutput());
+ for (int source : keySourceIndicator) {
+ itemValue.reset();
+ aInt8.setValue((byte) source);
+ int8Serde.serialize(aInt8, itemValue.getDataOutput());
+ listBuilder.addItem(itemValue);
+ }
+ fieldValue.reset();
+ listBuilder.write(fieldValue.getDataOutput(), true);
+ internalRecordBuilder.addField(nameValue, fieldValue);
+ }
+
internalRecordBuilder.write(out, true);
}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
index 12543c3..0825aa4 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
@@ -52,6 +52,7 @@
import org.apache.asterix.om.base.ABoolean;
import org.apache.asterix.om.base.ADateTime;
import org.apache.asterix.om.base.AInt32;
+import org.apache.asterix.om.base.AInt8;
import org.apache.asterix.om.base.AMutableInt32;
import org.apache.asterix.om.base.AMutableString;
import org.apache.asterix.om.base.AOrderedList;
@@ -183,10 +184,27 @@
}
}
+ // Read a field-source-indicator field.
+ List<Integer> keyFieldSourceIndicator = new ArrayList<>();
+ int keyFieldSourceIndicatorIndex = datasetDetailsRecord.getType()
+ .getFieldIndex(InternalDatasetDetails.KEY_FILD_SOURCE_INDICATOR_FIELD_NAME);
+ if (keyFieldSourceIndicatorIndex >= 0) {
+ cursor = ((AOrderedList) datasetDetailsRecord.getValueByPos(keyFieldSourceIndicatorIndex))
+ .getCursor();
+ while (cursor.next()) {
+ keyFieldSourceIndicator.add((int) ((AInt8) cursor.get()).getByteValue());
+ }
+ } else {
+ for (int index = 0; index < partitioningKey.size(); ++index) {
+ keyFieldSourceIndicator.add(0);
+ }
+ }
+
// Temporary dataset only lives in the compiler therefore the temp field is false.
// DatasetTupleTranslator always read from the metadata node, so the temp flag should be always false.
datasetDetails = new InternalDatasetDetails(fileStructure, partitioningStrategy, partitioningKey,
- partitioningKey, partitioningKeyType, autogenerated, filterField, false);
+ partitioningKey, keyFieldSourceIndicator, partitioningKeyType, autogenerated, filterField,
+ false);
break;
}
diff --git a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
index 37f0e48..f310a03 100644
--- a/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
+++ b/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
@@ -41,6 +41,8 @@
import org.apache.asterix.om.base.ABoolean;
import org.apache.asterix.om.base.ACollectionCursor;
import org.apache.asterix.om.base.AInt32;
+import org.apache.asterix.om.base.AInt8;
+import org.apache.asterix.om.base.AMutableInt8;
import org.apache.asterix.om.base.AOrderedList;
import org.apache.asterix.om.base.ARecord;
import org.apache.asterix.om.base.AString;
@@ -70,18 +72,24 @@
public static final String GRAM_LENGTH_FIELD_NAME = "GramLength";
public static final String INDEX_SEARCHKEY_TYPE_FIELD_NAME = "SearchKeyType";
public static final String INDEX_ISENFORCED_FIELD_NAME = "IsEnforced";
+ public static final String INDEX_SEARCHKEY_SOURCE_INDICATOR_FIELD_NAME = "SearchKeySourceIndicator";
private OrderedListBuilder listBuilder = new OrderedListBuilder();
private OrderedListBuilder primaryKeyListBuilder = new OrderedListBuilder();
private AOrderedListType stringList = new AOrderedListType(BuiltinType.ASTRING, null);
+ private AOrderedListType int8List = new AOrderedListType(BuiltinType.AINT8, null);
private ArrayBackedValueStorage nameValue = new ArrayBackedValueStorage();
private ArrayBackedValueStorage itemValue = new ArrayBackedValueStorage();
private List<List<String>> searchKey;
private List<IAType> searchKeyType;
+ private AMutableInt8 aInt8 = new AMutableInt8((byte) 0);
@SuppressWarnings("unchecked")
- protected ISerializerDeserializer<AInt32> intSerde = AqlSerializerDeserializerProvider.INSTANCE
+ private ISerializerDeserializer<AInt32> intSerde = AqlSerializerDeserializerProvider.INSTANCE
.getSerializerDeserializer(BuiltinType.AINT32);
@SuppressWarnings("unchecked")
+ private ISerializerDeserializer<AInt8> int8Serde = AqlSerializerDeserializerProvider.INSTANCE
+ .getSerializerDeserializer(BuiltinType.AINT8);
+ @SuppressWarnings("unchecked")
private ISerializerDeserializer<ARecord> recordSerde = AqlSerializerDeserializerProvider.INSTANCE
.getSerializerDeserializer(MetadataRecordTypes.INDEX_RECORDTYPE);
private final MetadataNode metadataNode;
@@ -161,8 +169,22 @@
if (gramLenPos >= 0) {
gramLength = ((AInt32) rec.getValueByPos(gramLenPos)).getIntegerValue();
}
- return new Index(dvName, dsName, indexName, indexStructure, searchKey, searchKeyType, gramLength,
- isEnforcingKeys, isPrimaryIndex, pendingOp);
+
+ // Read a field-source-indicator field.
+ List<Integer> keyFieldSourceIndicator = new ArrayList<>();
+ int keyFieldSourceIndicatorIndex = rec.getType().getFieldIndex(INDEX_SEARCHKEY_SOURCE_INDICATOR_FIELD_NAME);
+ if (keyFieldSourceIndicatorIndex >= 0) {
+ IACursor cursor = ((AOrderedList) rec.getValueByPos(keyFieldSourceIndicatorIndex)).getCursor();
+ while (cursor.next()) {
+ keyFieldSourceIndicator.add((int) ((AInt8) cursor.get()).getByteValue());
+ }
+ } else {
+ for (int index = 0; index < searchKey.size(); ++index) {
+ keyFieldSourceIndicator.add(0);
+ }
+ }
+ return new Index(dvName, dsName, indexName, indexStructure, searchKey, keyFieldSourceIndicator, searchKeyType,
+ gramLength, isEnforcingKeys, isPrimaryIndex, pendingOp);
}
@Override
@@ -288,6 +310,33 @@
recordBuilder.addField(nameValue, fieldValue);
}
+ List<Integer> keySourceIndicator = instance.getKeyFieldSourceIndicators();
+ boolean needSerialization = false;
+ if (keySourceIndicator != null) {
+ for (int source : keySourceIndicator) {
+ if (source != 0) {
+ needSerialization = true;
+ break;
+ }
+ }
+ }
+ if (needSerialization) {
+ listBuilder.reset(int8List);
+ ArrayBackedValueStorage nameValue = new ArrayBackedValueStorage();
+ nameValue.reset();
+ aString.setValue(INDEX_SEARCHKEY_SOURCE_INDICATOR_FIELD_NAME);
+ stringSerde.serialize(aString, nameValue.getDataOutput());
+ for (int source : keySourceIndicator) {
+ itemValue.reset();
+ aInt8.setValue((byte) source);
+ int8Serde.serialize(aInt8, itemValue.getDataOutput());
+ listBuilder.addItem(itemValue);
+ }
+ fieldValue.reset();
+ listBuilder.write(fieldValue.getDataOutput(), true);
+ recordBuilder.addField(nameValue, fieldValue);
+ }
+
// write record
recordBuilder.write(tupleBuilder.getDataOutput(), true);
tupleBuilder.addFieldEndOffset();
diff --git a/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslatorTest.java b/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslatorTest.java
index 9328d30..182f296 100644
--- a/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslatorTest.java
+++ b/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslatorTest.java
@@ -40,24 +40,37 @@
@Test
public void test() throws MetadataException, IOException {
- Map<String, String> compactionPolicyProperties = new HashMap<>();
- compactionPolicyProperties.put("max-mergable-component-size", "1073741824");
- compactionPolicyProperties.put("max-tolerance-component-count", "3");
+ Integer[] indicators = { 0, 1, null };
+ for (Integer indicator : indicators) {
+ Map<String, String> compactionPolicyProperties = new HashMap<>();
+ compactionPolicyProperties.put("max-mergable-component-size", "1073741824");
+ compactionPolicyProperties.put("max-tolerance-component-count", "3");
- InternalDatasetDetails details = new InternalDatasetDetails(FileStructure.BTREE, PartitioningStrategy.HASH,
- Collections.singletonList(Collections.singletonList("row_id")),
- Collections.singletonList(Collections.singletonList("row_id")),
- Collections.singletonList(BuiltinType.AINT64), false, Collections.emptyList(), false);
+ InternalDatasetDetails details = new InternalDatasetDetails(FileStructure.BTREE, PartitioningStrategy.HASH,
+ Collections.singletonList(Collections.singletonList("row_id")),
+ Collections.singletonList(Collections.singletonList("row_id")),
+ indicator == null ? null : Collections.singletonList(indicator),
+ Collections.singletonList(BuiltinType.AINT64), false, Collections.emptyList(), false);
- Dataset dataset = new Dataset("test", "log", "foo", "LogType", "CB", "MetaType", "DEFAULT_NG_ALL_NODES",
- "prefix", compactionPolicyProperties, details, Collections.emptyMap(), DatasetType.INTERNAL, 115, 0);
+ Dataset dataset = new Dataset("test", "log", "foo", "LogType", "CB", "MetaType", "DEFAULT_NG_ALL_NODES",
+ "prefix", compactionPolicyProperties, details, Collections.emptyMap(), DatasetType.INTERNAL, 115,
+ 0);
- MetadataRecordTypes.init();
- MetadataPrimaryIndexes.init();
- DatasetTupleTranslator dtTranslator = new DatasetTupleTranslator(true);
- ITupleReference tuple = dtTranslator.getTupleFromMetadataEntity(dataset);
- Dataset deserializedDataset = dtTranslator.getMetadataEntityFromTuple(tuple);
- Assert.assertEquals(dataset.getMetaItemTypeDataverseName(), deserializedDataset.getMetaItemTypeDataverseName());
- Assert.assertEquals(dataset.getMetaItemTypeName(), deserializedDataset.getMetaItemTypeName());
+ MetadataRecordTypes.init();
+ MetadataPrimaryIndexes.init();
+ DatasetTupleTranslator dtTranslator = new DatasetTupleTranslator(true);
+ ITupleReference tuple = dtTranslator.getTupleFromMetadataEntity(dataset);
+ Dataset deserializedDataset = dtTranslator.getMetadataEntityFromTuple(tuple);
+ Assert.assertEquals(dataset.getMetaItemTypeDataverseName(),
+ deserializedDataset.getMetaItemTypeDataverseName());
+ Assert.assertEquals(dataset.getMetaItemTypeName(), deserializedDataset.getMetaItemTypeName());
+ if (indicator == null) {
+ Assert.assertEquals(Collections.singletonList(new Integer(0)),
+ ((InternalDatasetDetails) deserializedDataset.getDatasetDetails()).getKeySourceIndicator());
+ } else {
+ Assert.assertEquals(((InternalDatasetDetails) dataset.getDatasetDetails()).getKeySourceIndicator(),
+ ((InternalDatasetDetails) deserializedDataset.getDatasetDetails()).getKeySourceIndicator());
+ }
+ }
}
}
diff --git a/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java b/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java
new file mode 100644
index 0000000..de548e2
--- /dev/null
+++ b/asterix-metadata/src/test/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslatorTest.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.metadata.entitytupletranslators;
+
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.asterix.common.config.DatasetConfig.DatasetType;
+import org.apache.asterix.common.config.DatasetConfig.IndexType;
+import org.apache.asterix.metadata.MetadataException;
+import org.apache.asterix.metadata.MetadataNode;
+import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes;
+import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes;
+import org.apache.asterix.metadata.entities.Dataset;
+import org.apache.asterix.metadata.entities.Datatype;
+import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.metadata.entities.InternalDatasetDetails;
+import org.apache.asterix.metadata.entities.InternalDatasetDetails.FileStructure;
+import org.apache.asterix.metadata.entities.InternalDatasetDetails.PartitioningStrategy;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class IndexTupleTranslatorTest {
+
+ @Test
+ public void test() throws MetadataException, IOException {
+ Integer[] indicators = { 0, 1, null };
+ for (Integer indicator : indicators) {
+ Map<String, String> compactionPolicyProperties = new HashMap<>();
+ compactionPolicyProperties.put("max-mergable-component-size", "1073741824");
+ compactionPolicyProperties.put("max-tolerance-component-count", "3");
+
+ InternalDatasetDetails details = new InternalDatasetDetails(FileStructure.BTREE, PartitioningStrategy.HASH,
+ Collections.singletonList(Collections.singletonList("row_id")),
+ Collections.singletonList(Collections.singletonList("row_id")),
+ indicator == null ? null : Collections.singletonList(indicator),
+ Collections.singletonList(BuiltinType.AINT64), false, Collections.emptyList(), false);
+
+ Dataset dataset = new Dataset("test", "d1", "foo", "LogType", "CB", "MetaType", "DEFAULT_NG_ALL_NODES",
+ "prefix", compactionPolicyProperties, details, Collections.emptyMap(), DatasetType.INTERNAL, 115,
+ 0);
+
+ Index index = new Index("test", "d1", "i1", IndexType.BTREE,
+ Collections.singletonList(Collections.singletonList("row_id")),
+ indicator == null ? null : Collections.singletonList(indicator),
+ Collections.singletonList(BuiltinType.AINT64), -1, false, false, 0);
+
+ MetadataRecordTypes.init();
+ MetadataPrimaryIndexes.init();
+
+ MetadataNode mockMetadataNode = mock(MetadataNode.class);
+ when(mockMetadataNode.getDatatype(any(), anyString(), anyString())).thenReturn(new Datatype("test", "d1",
+ new ARecordType("", new String[] { "row_id" }, new IAType[] { BuiltinType.AINT64 }, true), true));
+ when(mockMetadataNode.getDataset(any(), anyString(), anyString())).thenReturn(dataset);
+
+ IndexTupleTranslator idxTranslator = new IndexTupleTranslator(null, mockMetadataNode, true);
+ ITupleReference tuple = idxTranslator.getTupleFromMetadataEntity(index);
+ Index deserializedIndex = idxTranslator.getMetadataEntityFromTuple(tuple);
+ if (indicator == null) {
+ Assert.assertEquals(Collections.singletonList(new Integer(0)),
+ deserializedIndex.getKeyFieldSourceIndicators());
+ } else {
+ Assert.assertEquals(index.getKeyFieldSourceIndicators(),
+ deserializedIndex.getKeyFieldSourceIndicators());
+ }
+ }
+ }
+}