[NO ISSUE][OTH] Create dataset without type specification
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
With this change, users can create datasets without providing type
specification. For example the following statement creates a user
datase with id as primary key of type string:
CREATE DATASET user PRIMARY KEY(id: string)
Change-Id: I333d2b0dbbccef51dd3a11c611af10ac6a665ee3
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17485
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Peeyush Gupta <peeyush.gupta@couchbase.com>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
index b8eba74..d56963e 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/operators/physical/BTreeSearchPOperator.java
@@ -146,6 +146,8 @@
IProjectionFiltrationInfo<?> metaProjectionInfo = unnestMapOp.getMetaProjectionInfo();
ARecordType datasetType = (ARecordType) metadataProvider.findType(dataset);
ARecordType metaItemType = (ARecordType) metadataProvider.findMetaType(dataset);
+ datasetType = (ARecordType) metadataProvider.findTypeForDatasetWithoutType(datasetType,
+ metaItemType, dataset);
tupleProjectorFactory = IndexUtil.createTupleProjectorFactory(formatInfo, projectionInfo,
metaProjectionInfo, datasetType, metaItemType, dataset.getPrimaryKeys().size());
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
index 09ad4d1..4a0c4ae 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/IntroduceSecondaryIndexInsertDeleteRule.java
@@ -190,6 +190,8 @@
if (dataset.hasMetaPart()) {
metaType = (ARecordType) mp.findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
}
+ recType = (ARecordType) mp.findTypeForDatasetWithoutType(recType, metaType, dataset);
+
List<Index> indexes = mp.getDatasetIndexes(dataset.getDataverseName(), dataset.getDatasetName());
Stream<Index> indexStream = indexes.stream();
indexStream = indexStream.filter(index -> index.getIndexType() != IndexType.SAMPLE);
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroducePrimaryIndexForAggregationRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroducePrimaryIndexForAggregationRule.java
index cdeee0f..fde1ad7 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroducePrimaryIndexForAggregationRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/IntroducePrimaryIndexForAggregationRule.java
@@ -232,6 +232,9 @@
ARecordType recordType = (ARecordType) ((MetadataProvider) context.getMetadataProvider()).findType(dataset);
ARecordType metaRecordType =
(ARecordType) ((MetadataProvider) context.getMetadataProvider()).findMetaType(dataset);
+ recordType = (ARecordType) ((MetadataProvider) context.getMetadataProvider())
+ .findTypeForDatasetWithoutType(recordType, metaRecordType, dataset);
+
// create the operator that will replace the dataset scan/search
AbstractUnnestMapOperator primaryIndexUnnestOperator =
(AbstractUnnestMapOperator) AccessMethodUtils.createSecondaryIndexUnnestMap(dataset, recordType,
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
index bbfb365..29ee113 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/OptimizableOperatorSubTree.java
@@ -335,6 +335,8 @@
ARecordType metaItemType = (ARecordType) metadataProvider.findType(ds.getMetaItemTypeDataverseName(),
ds.getMetaItemTypeName());
+ rType = (ARecordType) metadataProvider.findTypeForDatasetWithoutType(rType, metaItemType, ds);
+
// First index is always the primary datasource in this subtree.
if (i == 0) {
setDataset(ds);
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index daa1d2f..bd06729 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -207,6 +207,8 @@
IAType itemType = metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
IAType metaItemType =
metadataProvider.findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
+ itemType = metadataProvider.findTypeForDatasetWithoutType(itemType, metaItemType, dataset);
+
DatasetDataSource targetDatasource =
validateDatasetInfo(metadataProvider, stmt.getDataverseName(), stmt.getDatasetName(), sourceLoc);
List<List<String>> partitionKeys = targetDatasource.getDataset().getPrimaryKeys();
@@ -694,6 +696,8 @@
IAType itemType = metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
IAType metaItemType =
metadataProvider.findType(dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
+ itemType = (ARecordType) metadataProvider.findTypeForDatasetWithoutType(itemType, metaItemType, dataset);
+
INodeDomain domain = metadataProvider.findNodeDomain(dataset.getNodeGroupName());
return new DatasetDataSource(sourceId, dataset, itemType, metaItemType, DataSource.Type.INTERNAL_DATASET,
dataset.getDatasetDetails(), domain);
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
index e51a539..04db82e 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/util/ValidateUtil.java
@@ -26,7 +26,10 @@
import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.lang.common.expression.TypeExpression;
+import org.apache.asterix.lang.common.expression.TypeReferenceExpression;
import org.apache.asterix.lang.common.statement.CreateViewStatement;
+import org.apache.asterix.metadata.entities.BuiltinTypeMap;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.utils.KeyFieldTypeUtil;
import org.apache.asterix.om.typecomputer.impl.TypeComputeUtils;
@@ -127,14 +130,22 @@
List<List<String>> partitioningExprs, List<Integer> keySourceIndicators, boolean autogenerated,
SourceLocation sourceLoc) throws AlgebricksException {
return validatePartitioningExpressionsImpl(recType, metaRecType, partitioningExprs, keySourceIndicators,
- autogenerated, true, sourceLoc);
+ autogenerated, true, sourceLoc, null);
+ }
+
+ public static List<IAType> validatePartitioningExpressions(ARecordType recType, ARecordType metaRecType,
+ List<List<String>> partitioningExprs, List<Integer> keySourceIndicators, boolean autogenerated,
+ SourceLocation sourceLoc, List<TypeExpression> partitioningExprTypes) throws AlgebricksException {
+ return validatePartitioningExpressionsImpl(recType, metaRecType, partitioningExprs, keySourceIndicators,
+ autogenerated, true, sourceLoc, partitioningExprTypes);
}
private static List<IAType> validatePartitioningExpressionsImpl(ARecordType recType, ARecordType metaRecType,
List<List<String>> partitioningExprs, List<Integer> keySourceIndicators, boolean autogenerated,
- boolean forPrimaryKey, SourceLocation sourceLoc) throws AlgebricksException {
+ boolean forPrimaryKey, SourceLocation sourceLoc, List<TypeExpression> partitioningExprTypes)
+ throws AlgebricksException {
String keyKindDisplayName = forPrimaryKey ? PRIMARY : "";
- List<IAType> partitioningExprTypes = new ArrayList<>(partitioningExprs.size());
+ List<IAType> computedPartitioningExprTypes = new ArrayList<>(partitioningExprs.size());
if (autogenerated) {
if (partitioningExprs.size() > 1) {
throw new CompilationException(ErrorCode.COMPILATION_CANNOT_AUTOGENERATE_COMPOSITE_KEY, sourceLoc,
@@ -143,37 +154,53 @@
List<String> fieldName = partitioningExprs.get(0);
IAType fieldType = recType.getSubFieldType(fieldName);
if (fieldType == null) {
- String unTypeField = fieldName.get(0) == null ? "" : fieldName.get(0);
- throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND, sourceLoc,
- LogRedactionUtil.userData(unTypeField));
+ if (partitioningExprTypes != null && partitioningExprTypes.size() > 0) {
+ String typeName =
+ ((TypeReferenceExpression) partitioningExprTypes.get(0)).getIdent().second.getValue();
+ fieldType = BuiltinTypeMap.getBuiltinType(typeName);
+ } else {
+ String unTypeField = fieldName.get(0) == null ? "" : fieldName.get(0);
+ throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND, sourceLoc,
+ LogRedactionUtil.userData(unTypeField));
+ }
}
- partitioningExprTypes.add(fieldType);
+ computedPartitioningExprTypes.add(fieldType);
ATypeTag pkTypeTag = fieldType.getTypeTag();
if (pkTypeTag != ATypeTag.UUID) {
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_AUTOGENERATED_TYPE, sourceLoc,
keyKindDisplayName, pkTypeTag.name(), ATypeTag.UUID.name());
}
} else {
- partitioningExprTypes =
- KeyFieldTypeUtil.getKeyTypes(recType, metaRecType, partitioningExprs, keySourceIndicators);
+ if (partitioningExprTypes == null) {
+ computedPartitioningExprTypes =
+ KeyFieldTypeUtil.getKeyTypes(recType, metaRecType, partitioningExprs, keySourceIndicators);
+ }
for (int i = 0; i < partitioningExprs.size(); i++) {
List<String> partitioningExpr = partitioningExprs.get(i);
- IAType fieldType = partitioningExprTypes.get(i);
- if (fieldType == null) {
- throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND, sourceLoc,
- LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(partitioningExpr)));
- }
- if (forPrimaryKey) {
- boolean nullable = KeyFieldTypeUtil.chooseSource(keySourceIndicators, i, recType, metaRecType)
- .isSubFieldNullable(partitioningExpr);
- if (nullable) {
- // key field is nullable
- throw new CompilationException(ErrorCode.COMPILATION_KEY_CANNOT_BE_NULLABLE, sourceLoc,
- keyKindDisplayName,
+ IAType fieldType;
+ if (partitioningExprTypes != null) {
+ String typeName =
+ ((TypeReferenceExpression) partitioningExprTypes.get(i)).getIdent().second.getValue();
+ fieldType = BuiltinTypeMap.getBuiltinType(typeName);
+ computedPartitioningExprTypes.add(fieldType);
+ } else {
+ fieldType = computedPartitioningExprTypes.get(i);
+ if (fieldType == null) {
+ throw new CompilationException(ErrorCode.COMPILATION_FIELD_NOT_FOUND, sourceLoc,
LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(partitioningExpr)));
}
- } else {
- fieldType = TypeComputeUtils.getActualType(fieldType);
+ if (forPrimaryKey) {
+ boolean nullable = KeyFieldTypeUtil.chooseSource(keySourceIndicators, i, recType, metaRecType)
+ .isSubFieldNullable(partitioningExpr);
+ if (nullable) {
+ // key field is nullable
+ throw new CompilationException(ErrorCode.COMPILATION_KEY_CANNOT_BE_NULLABLE, sourceLoc,
+ keyKindDisplayName,
+ LogRedactionUtil.userData(RecordUtil.toFullyQualifiedName(partitioningExpr)));
+ }
+ } else {
+ fieldType = TypeComputeUtils.getActualType(fieldType);
+ }
}
switch (fieldType.getTypeTag()) {
case TINYINT:
@@ -201,7 +228,7 @@
}
}
}
- return partitioningExprTypes;
+ return computedPartitioningExprTypes;
}
/**
@@ -337,7 +364,7 @@
validatePartitioningExpressionsImpl(itemType, null,
keyFields.stream().map(Collections::singletonList).collect(Collectors.toList()),
- Collections.nCopies(keyFields.size(), Index.RECORD_INDICATOR), false, !isForeignKey, sourceLoc);
+ Collections.nCopies(keyFields.size(), Index.RECORD_INDICATOR), false, !isForeignKey, sourceLoc, null);
return keyFields;
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexRewriter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexRewriter.java
index e0ff9c2..3d96595 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexRewriter.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexRewriter.java
@@ -143,6 +143,8 @@
Index index = validateIndex(f, metadataProvider, loc, dataverseName, datasetName, indexName);
ARecordType dsType = (ARecordType) metadataProvider.findType(dataset);
ARecordType metaType = DatasetUtil.getMetaType(metadataProvider, dataset);
+ dsType = (ARecordType) metadataProvider.findTypeForDatasetWithoutType(dsType, metaType, dataset);
+
List<IAType> dsKeyTypes = KeyFieldTypeUtil.getPartitoningKeyTypes(dataset, dsType, metaType);
List<Pair<IAType, Boolean>> secKeyTypes = KeyFieldTypeUtil.getBTreeIndexKeyTypes(index, dsType, metaType);
int numPrimaryKeys = dsKeyTypes.size();
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 3fa7423..f48e2b8 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -805,12 +805,15 @@
List<List<String>> partitioningExprs =
((InternalDetailsDecl) dd.getDatasetDetailsDecl()).getPartitioningExprs();
+ List<TypeExpression> partitioningExprTypes =
+ ((InternalDetailsDecl) dd.getDatasetDetailsDecl()).getPartitioningExprTypes();
List<Integer> keySourceIndicators =
((InternalDetailsDecl) dd.getDatasetDetailsDecl()).getKeySourceIndicators();
boolean autogenerated = ((InternalDetailsDecl) dd.getDatasetDetailsDecl()).isAutogenerated();
ARecordType aRecordType = (ARecordType) itemType;
- List<IAType> partitioningTypes = ValidateUtil.validatePartitioningExpressions(aRecordType,
- metaRecType, partitioningExprs, keySourceIndicators, autogenerated, sourceLoc);
+ List<IAType> partitioningTypes =
+ ValidateUtil.validatePartitioningExpressions(aRecordType, metaRecType, partitioningExprs,
+ keySourceIndicators, autogenerated, sourceLoc, partitioningExprTypes);
List<String> filterField = ((InternalDetailsDecl) dd.getDatasetDetailsDecl()).getFilterField();
Integer filterSourceIndicator =
@@ -827,9 +830,12 @@
compactionPolicy = StorageConstants.DEFAULT_FILTERED_DATASET_COMPACTION_POLICY_NAME;
compactionPolicyProperties = StorageConstants.DEFAULT_COMPACTION_POLICY_PROPERTIES;
}
+ boolean isDatasetWithoutTypeSpec = aRecordType.getFieldNames().length == 0 && metaRecType == null;
+
datasetDetails = new InternalDatasetDetails(InternalDatasetDetails.FileStructure.BTREE,
InternalDatasetDetails.PartitioningStrategy.HASH, partitioningExprs, partitioningExprs,
- keySourceIndicators, partitioningTypes, autogenerated, filterSourceIndicator, filterField);
+ keySourceIndicators, partitioningTypes, autogenerated, filterSourceIndicator, filterField,
+ isDatasetWithoutTypeSpec);
break;
case EXTERNAL:
ExternalDetailsDecl externalDetails = (ExternalDetailsDecl) dd.getDatasetDetailsDecl();
@@ -1128,6 +1134,9 @@
metaRecordType = (ARecordType) metaDt.getDatatype();
}
*/
+ if (!ds.hasMetaPart()) {
+ aRecordType = (ARecordType) metadataProvider.findTypeForDatasetWithoutType(aRecordType, null, ds);
+ }
List<List<IAType>> indexFieldTypes = new ArrayList<>(indexedElementsCount);
boolean hadUnnest = false;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java
index da72d2c..619c2cc 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RebalanceUtil.java
@@ -334,6 +334,7 @@
ARecordType itemType =
(ARecordType) metadataProvider.findType(source.getItemTypeDataverseName(), source.getItemTypeName());
ARecordType metaType = DatasetUtil.getMetaType(metadataProvider, source);
+ itemType = (ARecordType) metadataProvider.findTypeForDatasetWithoutType(itemType, metaType, source);
int numberOfPrimaryKeys = source.getPrimaryKeys().size();
// This could be expensive if record structure is "complex"
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-1/create-dataset.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-1/create-dataset.01.ddl.sqlpp
new file mode 100644
index 0000000..ed74a0f
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-1/create-dataset.01.ddl.sqlpp
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create dataset orders primary key (my_id: string);
+
+create dataset orders if not exists primary key (my_id: string);
+
+create dataset users if not exists primary key (my_id: int32, address.city:string, name.first: string, name.last:string);
+
+CREATE PRIMARY INDEX users_primary_index ON users;
+
+CREATE INDEX users_first_name ON users(name.first) TYPE BTREE;
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-1/create-dataset.02.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-1/create-dataset.02.update.sqlpp
new file mode 100644
index 0000000..a634582
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-1/create-dataset.02.update.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+INSERT INTO test.orders([
+{"my_id": "a", "f": null },
+{"my_id": "b"},
+{"my_id": "c", "f": {"inner_f": "foo", "inner_f2": {"f3": "bar"} } }
+]);
+
+INSERT INTO test.users([
+{"my_id": 1, "address":{"city": "C1"}, "name":{"first": "F1", "last": "L1"}},
+{"my_id": 2, "address":{"city": "C2"}, "name":{"first": "F2", "last": "L1"}},
+{"my_id": 2, "address":{"city": "C2"}, "name":{"first": "F1", "last": "L2"}}
+]);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-1/create-dataset.03.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-1/create-dataset.03.query.sqlpp
new file mode 100644
index 0000000..ef8a873
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-1/create-dataset.03.query.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+select value DatasetName
+from Metadata.`Dataset`
+where decode_dataverse_name(DataverseName) = ["test"]
+order by DatasetName
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-1/create-dataset.04.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-1/create-dataset.04.query.sqlpp
new file mode 100644
index 0000000..5d286e9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-1/create-dataset.04.query.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+select DatasetName, InternalDetails.PrimaryKeyTypes
+from Metadata.`Dataset`
+where decode_dataverse_name(DataverseName) = ["test"]
+order by DatasetName
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-1/create-dataset.05.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-1/create-dataset.05.query.sqlpp
new file mode 100644
index 0000000..ed8f335
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-1/create-dataset.05.query.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+select address, name
+from test.users d
+where d.my_id=2
+order by name.first
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-1/create-dataset.06.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-1/create-dataset.06.query.sqlpp
new file mode 100644
index 0000000..cfc0689
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-1/create-dataset.06.query.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+select address, name
+from test.users d
+where name.first="F1"
+order by name.last
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-1/create-dataset.07.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-1/create-dataset.07.ddl.sqlpp
new file mode 100644
index 0000000..3b957ba
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-1/create-dataset.07.ddl.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+drop dataset orders;
+drop dataset users;
+
+drop dataverse test;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-2/create-dataset.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-2/create-dataset.01.ddl.sqlpp
new file mode 100644
index 0000000..c83eb10
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-2/create-dataset.01.ddl.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+create dataset orders primary key (my_id: string);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-2/create-dataset.02.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-2/create-dataset.02.update.sqlpp
new file mode 100644
index 0000000..832aa49
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-2/create-dataset.02.update.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+INSERT INTO test.orders([
+{"f": "a" }
+]);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-2/create-dataset.03.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-2/create-dataset.03.ddl.sqlpp
new file mode 100644
index 0000000..f0f5dc9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-2/create-dataset.03.ddl.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataset test.orders;
+
+drop dataverse test;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-1/create-dataset.03.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-1/create-dataset.03.adm
new file mode 100644
index 0000000..cb54e90
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-1/create-dataset.03.adm
@@ -0,0 +1,2 @@
+"orders"
+"users"
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-1/create-dataset.04.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-1/create-dataset.04.adm
new file mode 100644
index 0000000..47a5320
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-1/create-dataset.04.adm
@@ -0,0 +1,2 @@
+{ "DatasetName": "orders", "PrimaryKeyTypes": [ "string" ] }
+{ "DatasetName": "users", "PrimaryKeyTypes": [ "int32", "string", "string", "string" ] }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-1/create-dataset.05.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-1/create-dataset.05.adm
new file mode 100644
index 0000000..c7c9e15
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-1/create-dataset.05.adm
@@ -0,0 +1,2 @@
+{ "address": { "city": "C2" }, "name": { "first": "F1", "last": "L2" } }
+{ "address": { "city": "C2" }, "name": { "first": "F2", "last": "L1" } }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-1/create-dataset.06.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-1/create-dataset.06.adm
new file mode 100644
index 0000000..20645b1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-1/create-dataset.06.adm
@@ -0,0 +1,2 @@
+{ "address": { "city": "C1" }, "name": { "first": "F1", "last": "L1" } }
+{ "address": { "city": "C2" }, "name": { "first": "F1", "last": "L2" } }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index b9c9d69..05f0219 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -4141,6 +4141,18 @@
</test-group>
<test-group name="ddl">
<test-case FilePath="ddl">
+ <compilation-unit name="create-dataset-1">
+ <output-dir compare="Clean-JSON">create-dataset-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl">
+ <compilation-unit name="create-dataset-2">
+ <output-dir compare="Clean-JSON">create-dataset-2</output-dir>
+ <source-location>false</source-location>
+ <expected-error>type mismatch: missing a required closed field my_id: string</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl">
<compilation-unit name="analyze-dataset-1">
<output-dir compare="Text">analyze-dataset-1</output-dir>
</compilation-unit>
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnSecondaryIndexSchemaUtil.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnSecondaryIndexSchemaUtil.java
index 87c111f..414cd1f 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnSecondaryIndexSchemaUtil.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/ColumnSecondaryIndexSchemaUtil.java
@@ -49,6 +49,18 @@
return new ARecordType("root", result.getFieldNames(), result.getFieldTypes(), true);
}
+ public static ARecordType getRecordTypeWithFieldTypes(List<List<String>> paths, List<IAType> types)
+ throws AlgebricksException {
+ ARecordType result = DataProjectionFiltrationInfo.EMPTY_TYPE;
+ for (int i = 0; i < paths.size(); i++) {
+ List<String> path = paths.get(i);
+ ARecordType type = getRecordType(path, "root", 0, types.get(i));
+ result = (ARecordType) RecordMergeTypeComputer.merge(result, type);
+ }
+
+ return new ARecordType("root", result.getFieldNames(), result.getFieldTypes(), true);
+ }
+
/**
* Get the expected type for an array index
*
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/InternalDetailsDecl.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/InternalDetailsDecl.java
index 3ddd261..9dd29a5 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/InternalDetailsDecl.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/InternalDetailsDecl.java
@@ -22,6 +22,7 @@
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.lang.common.expression.TypeExpression;
public class InternalDetailsDecl implements IDatasetDetailsDecl {
private final List<List<String>> partitioningExprs;
@@ -29,10 +30,11 @@
private final Integer filterSourceIndicator;
private final boolean autogenerated;
private final List<String> filterField;
+ private final List<TypeExpression> partitioningExprTypes;
public InternalDetailsDecl(List<List<String>> partitioningExpr, List<Integer> keySourceIndicators,
- boolean autogenerated, Integer filterSourceIndicator, List<String> filterField)
- throws CompilationException {
+ boolean autogenerated, Integer filterSourceIndicator, List<String> filterField,
+ List<TypeExpression> partitioningExprTypes) throws CompilationException {
this.partitioningExprs = partitioningExpr;
this.keySourceIndicators = keySourceIndicators;
this.autogenerated = autogenerated;
@@ -42,12 +44,23 @@
}
this.filterField = filterField;
this.filterSourceIndicator = filterSourceIndicator;
+ this.partitioningExprTypes = partitioningExprTypes;
+ }
+
+ public InternalDetailsDecl(List<List<String>> partitioningExpr, List<Integer> keySourceIndicators,
+ boolean autogenerated, Integer filterSourceIndicator, List<String> filterField)
+ throws CompilationException {
+ this(partitioningExpr, keySourceIndicators, autogenerated, filterSourceIndicator, filterField, null);
}
public List<List<String>> getPartitioningExprs() {
return partitioningExprs;
}
+ public List<TypeExpression> getPartitioningExprTypes() {
+ return partitioningExprTypes;
+ }
+
public List<Integer> getKeySourceIndicators() {
return keySourceIndicators;
}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index 5b0760b..464efb4 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -207,6 +207,7 @@
import org.apache.asterix.lang.sqlpp.util.ExpressionToVariableUtil;
import org.apache.asterix.lang.sqlpp.util.FunctionMapUtil;
import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil;
+import org.apache.asterix.metadata.bootstrap.MetadataBuiltinEntities;
import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.asterix.om.exceptions.TypeMismatchException;
import org.apache.asterix.om.functions.BuiltinFunctions;
@@ -1115,6 +1116,7 @@
TypeExpression typeExpr = null;
TypeExpression metaTypeExpr = null;
Pair<List<Integer>, List<List<String>>> primaryKeyFields = null;
+ Triple<List<Integer>, List<List<String>>, List<TypeExpression>> primaryKeyFieldsWithTypes = null;
Map<String,String> hints = new HashMap<String,String>();
DatasetDecl stmt = null;
boolean autogenerated = false;
@@ -1123,7 +1125,7 @@
}
{
nameComponents = QualifiedName()
- typeExpr = DatasetTypeSpecification()
+ (typeExpr = DatasetTypeSpecification())?
(
{ String name; }
<WITH>
@@ -1137,23 +1139,36 @@
metaTypeExpr = DatasetTypeSpecification()
)?
ifNotExists = IfNotExists()
- primaryKeyFields = PrimaryKey()
+ (LOOKAHEAD(3) primaryKeyFieldsWithTypes = PrimaryKeyWithType()
+ | primaryKeyFields = PrimaryKey())
(<AUTOGENERATED> { autogenerated = true; } )?
( <HINTS> hints = Properties() )?
( LOOKAHEAD(2) <WITH> <FILTER> <ON> filterField = NestedField() )?
( <WITH> withRecord = RecordConstructor() )?
{
try {
- InternalDetailsDecl idd = new InternalDetailsDecl(primaryKeyFields.second, primaryKeyFields.first, autogenerated,
- filterField == null? null : filterField.first, filterField == null? null : filterField.second);
- DatasetDeclParametersUtil.adjustInlineTypeDecl(typeExpr, primaryKeyFields.second, primaryKeyFields.first, false);
- if (metaTypeExpr != null) {
- DatasetDeclParametersUtil.adjustInlineTypeDecl(metaTypeExpr, primaryKeyFields.second, primaryKeyFields.first,
- true);
+ if (typeExpr == null) {
+ InternalDetailsDecl idd = new InternalDetailsDecl(primaryKeyFieldsWithTypes.second,
+ primaryKeyFieldsWithTypes.first, autogenerated, filterField == null? null : filterField.first,
+ filterField == null? null : filterField.second, primaryKeyFieldsWithTypes.third);
+ final TypeReferenceExpression anyObjectReference = new TypeReferenceExpression(
+ new Pair(MetadataBuiltinEntities.ANY_OBJECT_DATATYPE.getDataverseName(),
+ new Identifier(MetadataBuiltinEntities.ANY_OBJECT_DATATYPE.getDatatypeName())));
+ stmt = new DatasetDecl(nameComponents.first, nameComponents.second, anyObjectReference, null, hints,
+ DatasetType.INTERNAL, idd, withRecord, ifNotExists);
+ return addSourceLocation(stmt, startStmtToken);
+ } else {
+ InternalDetailsDecl idd = new InternalDetailsDecl(primaryKeyFields.second, primaryKeyFields.first, autogenerated,
+ filterField == null? null : filterField.first, filterField == null? null : filterField.second);
+ DatasetDeclParametersUtil.adjustInlineTypeDecl(typeExpr, primaryKeyFields.second, primaryKeyFields.first, false);
+ if (metaTypeExpr != null) {
+ DatasetDeclParametersUtil.adjustInlineTypeDecl(metaTypeExpr, primaryKeyFields.second, primaryKeyFields.first,
+ true);
+ }
+ stmt = new DatasetDecl(nameComponents.first, nameComponents.second, typeExpr, metaTypeExpr, hints,
+ DatasetType.INTERNAL, idd, withRecord, ifNotExists);
+ return addSourceLocation(stmt, startStmtToken);
}
- stmt = new DatasetDecl(nameComponents.first, nameComponents.second, typeExpr, metaTypeExpr, hints,
- DatasetType.INTERNAL, idd, withRecord, ifNotExists);
- return addSourceLocation(stmt, startStmtToken);
} catch (CompilationException e) {
throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage());
}
@@ -2142,6 +2157,45 @@
}
}
+Triple<List<Integer>, List<List<String>>, List<TypeExpression>> PrimaryKeyWithType() throws ParseException:
+{
+ Triple<List<Integer>, List<List<String>>, List<TypeExpression>> primaryKeyFieldsWithTypes = null;
+}
+{
+ <PRIMARY> <KEY> <LEFTPAREN> primaryKeyFieldsWithTypes = PrimaryKeyFieldsWithType() <RIGHTPAREN>
+ {
+ return primaryKeyFieldsWithTypes;
+ }
+}
+
+Triple<List<Integer>, List<List<String>>, List<TypeExpression>> PrimaryKeyFieldsWithType() throws ParseException:
+{
+ Pair<Integer, List<String>> tmp = null;
+ List<Integer> keyFieldSourceIndicators = new ArrayList<Integer>();
+ List<List<String>> primaryKeyFields = new ArrayList<List<String>>();
+ List<TypeExpression> primaryKeyFieldTypes = new ArrayList<TypeExpression>();
+ TypeExpression type = null;
+}
+{
+ tmp = NestedField() <COLON> type = TypeReference()
+ {
+ keyFieldSourceIndicators.add(tmp.first);
+ primaryKeyFields.add(tmp.second);
+ primaryKeyFieldTypes.add(type);
+ }
+ ( <COMMA> tmp = NestedField() <COLON> type = TypeReference()
+ {
+ keyFieldSourceIndicators.add(tmp.first);
+ primaryKeyFields.add(tmp.second);
+ primaryKeyFieldTypes.add(type);
+ }
+ )*
+ {
+ return new Triple<List<Integer>, List<List<String>>, List<TypeExpression>> (keyFieldSourceIndicators,
+ primaryKeyFields, primaryKeyFieldTypes);
+ }
+}
+
Pair<List<Integer>, List<List<String>>> PrimaryKey() throws ParseException:
{
Pair<List<Integer>, List<List<String>>> primaryKeyFields = null;
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java
index ee629b1..4b9483c 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataManagerUtil.java
@@ -23,6 +23,7 @@
import java.util.ArrayList;
import java.util.List;
+import org.apache.asterix.column.util.ColumnSecondaryIndexSchemaUtil;
import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.common.config.DatasetConfig;
import org.apache.asterix.common.exceptions.AsterixException;
@@ -39,6 +40,7 @@
import org.apache.asterix.metadata.entities.FullTextConfigMetadataEntity;
import org.apache.asterix.metadata.entities.FullTextFilterMetadataEntity;
import org.apache.asterix.metadata.entities.Index;
+import org.apache.asterix.metadata.entities.InternalDatasetDetails;
import org.apache.asterix.metadata.entities.NodeGroup;
import org.apache.asterix.metadata.entities.Synonym;
import org.apache.asterix.metadata.utils.MetadataConstants;
@@ -62,6 +64,33 @@
return type != null ? type.getDatatype() : null;
}
+ /**
+ * Checks if a dataset is created without type specification and has no meta part. For such datasets,
+ * creates and returns a record type based on the primary key and primary key types information included in the
+ * internal details.
+ *
+ * @param itemType record type of the dataset
+ * @param metaItemType record type of the meta part of the dataset
+ * @param dataset the actual dataset
+ * @return type computed from primary keys if dataset without type spec, otherwise the original itemType itself
+ * @throws AlgebricksException
+ */
+ public static IAType findTypeForDatasetWithoutType(IAType itemType, IAType metaItemType, Dataset dataset)
+ throws AlgebricksException {
+ ARecordType recordType = (ARecordType) itemType;
+ if (recordType.getFieldNames().length == 0 && metaItemType == null
+ && dataset.getDatasetType() == DatasetConfig.DatasetType.INTERNAL) {
+ InternalDatasetDetails dsDetails = (InternalDatasetDetails) dataset.getDatasetDetails();
+ return findType(dsDetails.getPrimaryKey(), dsDetails.getPrimaryKeyType());
+ }
+ return itemType;
+ }
+
+ private static IAType findType(List<List<String>> primaryKeys, List<IAType> primaryKeyTypes)
+ throws AlgebricksException {
+ return ColumnSecondaryIndexSchemaUtil.getRecordTypeWithFieldTypes(primaryKeys, primaryKeyTypes);
+ }
+
public static Datatype findTypeEntity(MetadataTransactionContext mdTxnCtx, DataverseName dataverseName,
String typeName) throws AlgebricksException {
if (dataverseName == null || typeName == null) {
@@ -210,6 +239,8 @@
IAType itemType = findType(mdTxnCtx, dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
IAType metaItemType = findType(mdTxnCtx, dataset.getMetaItemTypeDataverseName(), dataset.getMetaItemTypeName());
+ itemType = findTypeForDatasetWithoutType(itemType, metaItemType, dataset);
+
INodeDomain domain = findNodeDomain(clusterStateManager, mdTxnCtx, dataset.getNodeGroupName());
return new DatasetDataSource(id, dataset, itemType, metaItemType, datasourceType, dataset.getDatasetDetails(),
domain);
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 4862389..00fe7ae 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -365,6 +365,11 @@
return MetadataManagerUtil.findTypeEntity(mdTxnCtx, dataverseName, typeName);
}
+ public IAType findTypeForDatasetWithoutType(IAType recordType, IAType metaRecordType, Dataset dataset)
+ throws AlgebricksException {
+ return MetadataManagerUtil.findTypeForDatasetWithoutType(recordType, metaRecordType, dataset);
+ }
+
public IAType findType(DataverseName dataverseName, String typeName) throws AlgebricksException {
return MetadataManagerUtil.findType(mdTxnCtx, dataverseName, typeName);
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
index 4b225ec..b97880e 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java
@@ -710,6 +710,7 @@
}
ARecordType itemType = (ARecordType) metadataProvider.findType(this);
ARecordType metaType = (ARecordType) metadataProvider.findMetaType(this);
+ itemType = (ARecordType) metadataProvider.findTypeForDatasetWithoutType(itemType, metaType, this);
// Set the serde/traits for primary keys
for (int i = 0; i < numPrimaryKeys; i++) {
@@ -773,6 +774,8 @@
throws AlgebricksException {
ARecordType recordType = (ARecordType) metadataProvider.findType(this);
ARecordType metaType = (ARecordType) metadataProvider.findMetaType(this);
+ recordType = (ARecordType) metadataProvider.findTypeForDatasetWithoutType(recordType, metaType, this);
+
List<List<String>> partitioningKeys = getPrimaryKeys();
int numPrimaryKeys = partitioningKeys.size();
IBinaryHashFunctionFactory[] hashFuncFactories = new IBinaryHashFunctionFactory[numPrimaryKeys];
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java
index c82b86a..fd19937 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/InternalDatasetDetails.java
@@ -65,15 +65,17 @@
private final Integer filterSourceIndicator;
private final List<String> filterField;
private final List<Integer> keySourceIndicators;
+ private final boolean isDatasetWithoutTypeSpecification;
public static final String FILTER_FIELD_NAME = "FilterField";
public static final String FILTER_SOURCE_INDICATOR_FIELD_NAME = "FilterSourceIndicator";
public static final String KEY_FILD_SOURCE_INDICATOR_FIELD_NAME = "KeySourceIndicator";
+ public static final String PRIMARY_KEY_TYPES_FIELD_NAME = "PrimaryKeyTypes";
public InternalDatasetDetails(FileStructure fileStructure, PartitioningStrategy partitioningStrategy,
List<List<String>> partitioningKey, List<List<String>> primaryKey, List<Integer> keyFieldIndicators,
- List<IAType> primaryKeyType, boolean autogenerated, Integer filterSourceIndicator,
- List<String> filterField) {
+ List<IAType> primaryKeyType, boolean autogenerated, Integer filterSourceIndicator, List<String> filterField,
+ boolean isDatasetWithoutTypeSpecification) {
this.fileStructure = fileStructure;
this.partitioningStrategy = partitioningStrategy;
this.partitioningKeys = partitioningKey;
@@ -96,6 +98,15 @@
this.filterSourceIndicator = null;
this.filterField = null;
}
+ this.isDatasetWithoutTypeSpecification = isDatasetWithoutTypeSpecification;
+ }
+
+ public InternalDatasetDetails(FileStructure fileStructure, PartitioningStrategy partitioningStrategy,
+ List<List<String>> partitioningKey, List<List<String>> primaryKey, List<Integer> keyFieldIndicators,
+ List<IAType> primaryKeyType, boolean autogenerated, Integer filterSourceIndicator,
+ List<String> filterField) {
+ this(fileStructure, partitioningStrategy, partitioningKey, primaryKey, keyFieldIndicators, primaryKeyType,
+ autogenerated, filterSourceIndicator, filterField, false);
}
public List<List<String>> getPartitioningKey() {
@@ -223,6 +234,25 @@
internalRecordBuilder.addField(MetadataRecordTypes.INTERNAL_DETAILS_ARECORD_AUTOGENERATED_FIELD_INDEX,
fieldValue);
+ // Serialize Primary Key types if available and if dataset without type specification
+ if (isDatasetWithoutTypeSpecification && primaryKeyTypes != null && !primaryKeyTypes.isEmpty()) {
+ ArrayBackedValueStorage nameValue = new ArrayBackedValueStorage();
+ nameValue.reset();
+ aString.setValue(PRIMARY_KEY_TYPES_FIELD_NAME);
+ stringSerde.serialize(aString, nameValue.getDataOutput());
+
+ listBuilder.reset(AOrderedListType.FULL_OPEN_ORDEREDLIST_TYPE);
+ for (IAType keyType : primaryKeyTypes) {
+ itemValue.reset();
+ aString.setValue(keyType.getTypeName());
+ stringSerde.serialize(aString, itemValue.getDataOutput());
+ listBuilder.addItem(itemValue);
+ }
+ fieldValue.reset();
+ listBuilder.write(fieldValue.getDataOutput(), true);
+ internalRecordBuilder.addField(nameValue, fieldValue);
+ }
+
// write filter fields if any
Integer filterSourceIndicator = getFilterSourceIndicator();
List<String> filterField = getFilterField();
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
index 790faa5..e659b8f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
@@ -44,6 +44,7 @@
import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes;
import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes;
import org.apache.asterix.metadata.dataset.DatasetFormatInfo;
+import org.apache.asterix.metadata.entities.BuiltinTypeMap;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.ExternalDatasetDetails;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
@@ -161,6 +162,20 @@
partitioningKeyType.add(BuiltinType.ASTRING);
}
+ // Check if there is a primary key types field
+ List<IAType> primaryKeyTypes = null;
+ int primaryKeyTypesPos = datasetDetailsRecord.getType()
+ .getFieldIndex(InternalDatasetDetails.PRIMARY_KEY_TYPES_FIELD_NAME);
+ if (primaryKeyTypesPos >= 0) {
+ cursor = ((AOrderedList) datasetDetailsRecord.getValueByPos(primaryKeyTypesPos)).getCursor();
+ primaryKeyTypes = new ArrayList<>();
+ while (cursor.next()) {
+ String primaryKeyTypeName = ((AString) cursor.get()).getStringValue();
+ IAType primaryKeyType = BuiltinTypeMap.getBuiltinType(primaryKeyTypeName);
+ primaryKeyTypes.add(primaryKeyType);
+ }
+ }
+
boolean autogenerated = ((ABoolean) datasetDetailsRecord
.getValueByPos(MetadataRecordTypes.INTERNAL_DETAILS_ARECORD_AUTOGENERATED_FIELD_INDEX))
.getBoolean();
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
index b00a706..43d0ed6 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
@@ -42,6 +42,7 @@
import org.apache.asterix.metadata.MetadataNode;
import org.apache.asterix.metadata.bootstrap.MetadataPrimaryIndexes;
import org.apache.asterix.metadata.bootstrap.MetadataRecordTypes;
+import org.apache.asterix.metadata.declared.MetadataManagerUtil;
import org.apache.asterix.metadata.entities.BuiltinTypeMap;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
@@ -333,6 +334,7 @@
metaDt = (ARecordType) metadataNode.getDatatype(txnId, metatypeDataverseName, metatypeName)
.getDatatype();
}
+ recordDt = (ARecordType) MetadataManagerUtil.findTypeForDatasetWithoutType(recordDt, metaDt, dataset);
searchKeyType = new ArrayList<>(searchElementCount);
for (int i = 0; i < searchElementCount; i++) {
Pair<List<List<String>>, List<List<String>>> searchElement = searchElements.get(i);
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index 2c1bdea..304d902 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@ -330,6 +330,8 @@
if (dataset.hasMetaPart()) {
metaItemType = (ARecordType) metadataProvider.findMetaType(dataset);
}
+ itemType = (ARecordType) metadataProvider.findTypeForDatasetWithoutType(itemType, metaItemType, dataset);
+
JobSpecification spec = RuntimeUtils.createJobSpecification(metadataProvider.getApplicationContext());
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint =
metadataProvider.getSplitProviderAndConstraints(dataset);
@@ -435,6 +437,8 @@
int numFilterFields = DatasetUtil.getFilterField(dataset) == null ? 0 : 1;
ARecordType itemType = (ARecordType) metadataProvider.findType(dataset);
ARecordType metaItemType = (ARecordType) metadataProvider.findMetaType(dataset);
+ itemType = (ARecordType) metadataProvider.findTypeForDatasetWithoutType(itemType, metaItemType, dataset);
+
Index primaryIndex = metadataProvider.getIndex(dataset.getDataverseName(), dataset.getDatasetName(),
dataset.getDatasetName());
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint =
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SampleOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SampleOperationsHelper.java
index 056a8c2..7b4587b 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SampleOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SampleOperationsHelper.java
@@ -124,6 +124,8 @@
itemType =
(ARecordType) metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
metaType = DatasetUtil.getMetaType(metadataProvider, dataset);
+ itemType = (ARecordType) metadataProvider.findTypeForDatasetWithoutType(itemType, metaType, dataset);
+
recordDesc = dataset.getPrimaryRecordDescriptor(metadataProvider);
comparatorFactories = dataset.getPrimaryComparatorFactories(metadataProvider, itemType, metaType);
groupbyNumFrames = getGroupByNumFrames(metadataProvider, sourceLoc);
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index ea7da19..76c2a73 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@ -128,9 +128,11 @@
this.dataset = dataset;
this.index = index;
this.metadataProvider = metadataProvider;
- this.itemType =
+ ARecordType recordType =
(ARecordType) metadataProvider.findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName());
this.metaType = DatasetUtil.getMetaType(metadataProvider, dataset);
+ this.itemType = (ARecordType) metadataProvider.findTypeForDatasetWithoutType(recordType, metaType, dataset);
+
Pair<ARecordType, ARecordType> enforcedTypes = getEnforcedType(index, itemType, metaType);
this.enforcedItemType = enforcedTypes.first;
this.enforcedMetaType = enforcedTypes.second;