Merge commit '3e2623cf64827e891ecd8c6dea13d03f5599c6a5' into HEAD
Change-Id: I89091d8bf8575c73d26f497d5fbffe780adbedf9
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index eefb7f3..cf14555 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -99,6 +99,8 @@
import org.apache.asterix.lang.common.base.IStatementRewriter;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.lang.common.expression.IndexedTypeExpression;
+import org.apache.asterix.lang.common.expression.TypeExpression;
+import org.apache.asterix.lang.common.expression.TypeReferenceExpression;
import org.apache.asterix.lang.common.statement.CompactStatement;
import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
import org.apache.asterix.lang.common.statement.CreateAdapterStatement;
@@ -585,15 +587,43 @@
DatasetDecl dd = (DatasetDecl) stmt;
DataverseName dataverseName = getActiveDataverseName(dd.getDataverse());
String datasetName = dd.getName().getValue();
- DataverseName itemTypeDataverseName = getActiveDataverseName(dd.getItemTypeDataverse());
- String itemTypeName = dd.getItemTypeName().getValue();
+ TypeExpression itemTypeExpr = dd.getItemType();
+ DataverseName itemTypeDataverseName = null;
+ String itemTypeName = null;
+ switch (itemTypeExpr.getTypeKind()) {
+ case TYPEREFERENCE:
+ TypeReferenceExpression itemTypeRefExpr = (TypeReferenceExpression) itemTypeExpr;
+ Pair<DataverseName, Identifier> itemTypeIdent = itemTypeRefExpr.getIdent();
+ itemTypeDataverseName = itemTypeIdent.first != null ? itemTypeIdent.first : dataverseName;
+ itemTypeName = itemTypeRefExpr.getIdent().second.getValue();
+ break;
+ case RECORD:
+ break;
+ default:
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, stmt.getSourceLocation(),
+ String.valueOf(itemTypeExpr.getTypeKind()));
+ }
+
+ TypeExpression metaItemTypeExpr = dd.getMetaItemType();
DataverseName metaItemTypeDataverseName = null;
String metaItemTypeName = null;
- Identifier metaItemTypeId = dd.getMetaItemTypeName();
- if (metaItemTypeId != null) {
- metaItemTypeName = metaItemTypeId.getValue();
- metaItemTypeDataverseName = getActiveDataverseName(dd.getMetaItemTypeDataverse());
+ if (metaItemTypeExpr != null) {
+ switch (metaItemTypeExpr.getTypeKind()) {
+ case TYPEREFERENCE:
+ TypeReferenceExpression metaItemTypeRefExpr = (TypeReferenceExpression) metaItemTypeExpr;
+ Pair<DataverseName, Identifier> metaItemTypeIdent = metaItemTypeRefExpr.getIdent();
+ metaItemTypeDataverseName =
+ metaItemTypeIdent.first != null ? metaItemTypeIdent.first : dataverseName;
+ metaItemTypeName = metaItemTypeRefExpr.getIdent().second.getValue();
+ break;
+ case RECORD:
+ break;
+ default:
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, stmt.getSourceLocation(),
+ String.valueOf(metaItemTypeExpr.getTypeKind()));
+ }
}
+
Identifier ngNameId = dd.getNodegroupName();
String nodegroupName = ngNameId == null ? null : ngNameId.getValue();
String compactionPolicy = dd.getCompactionPolicy();
@@ -604,14 +634,16 @@
compactionPolicy, defaultCompactionPolicy, dd.getDatasetDetailsDecl());
try {
doCreateDatasetStatement(metadataProvider, dd, dataverseName, datasetName, itemTypeDataverseName,
- itemTypeName, metaItemTypeDataverseName, metaItemTypeName, hcc, requestParameters);
+ itemTypeExpr, itemTypeName, metaItemTypeExpr, metaItemTypeDataverseName, metaItemTypeName, hcc,
+ requestParameters);
} finally {
metadataProvider.getLocks().unlock();
}
}
protected void doCreateDatasetStatement(MetadataProvider metadataProvider, DatasetDecl dd,
- DataverseName dataverseName, String datasetName, DataverseName itemTypeDataverseName, String itemTypeName,
+ DataverseName dataverseName, String datasetName, DataverseName itemTypeDataverseName,
+ TypeExpression itemTypeExpr, String itemTypeName, TypeExpression metaItemTypeExpr,
DataverseName metaItemTypeDataverseName, String metaItemTypeName, IHyracksClientConnection hcc,
IRequestParameters requestParameters) throws Exception {
MutableObject<ProgressState> progress = new MutableObject<>(ProgressState.NO_PROGRESS);
@@ -637,10 +669,23 @@
throw new CompilationException(ErrorCode.DATASET_EXISTS, sourceLoc, datasetName, dataverseName);
}
}
- Datatype dt = MetadataManager.INSTANCE.getDatatype(metadataProvider.getMetadataTxnContext(),
- itemTypeDataverseName, itemTypeName);
- if (dt == null) {
- throw new CompilationException(ErrorCode.UNKNOWN_TYPE, sourceLoc, itemTypeName);
+ IAType itemType;
+ switch (itemTypeExpr.getTypeKind()) {
+ case TYPEREFERENCE:
+ itemType = metadataProvider.findType(itemTypeDataverseName, itemTypeName);
+ break;
+ case RECORD:
+ itemTypeDataverseName = dataverseName;
+ itemTypeName = DatasetUtil.createInlineTypeName(datasetName, false);
+ lockUtil.createTypeBegin(lockManager, metadataProvider.getLocks(), itemTypeDataverseName,
+ itemTypeName);
+ itemType = translateType(itemTypeDataverseName, itemTypeName, itemTypeExpr, mdTxnCtx);
+ MetadataManager.INSTANCE.addDatatype(mdTxnCtx,
+ new Datatype(itemTypeDataverseName, itemTypeName, itemType, true));
+ break;
+ default:
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc,
+ String.valueOf(itemTypeExpr.getTypeKind()));
}
String ngName = ngNameId != null ? ngNameId.getValue()
: configureNodegroupForDataset(appCtx, dd.getHints(), dataverseName, datasetName, metadataProvider,
@@ -654,19 +699,35 @@
}
switch (dd.getDatasetType()) {
case INTERNAL:
- IAType itemType = dt.getDatatype();
if (itemType.getTypeTag() != ATypeTag.OBJECT) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"Dataset type has to be a record type.");
}
IAType metaItemType = null;
- if (metaItemTypeDataverseName != null && metaItemTypeName != null) {
- metaItemType = metadataProvider.findType(metaItemTypeDataverseName, metaItemTypeName);
- }
- if (metaItemType != null && metaItemType.getTypeTag() != ATypeTag.OBJECT) {
- throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
- "Dataset meta type has to be a record type.");
+ if (metaItemTypeExpr != null) {
+ switch (metaItemTypeExpr.getTypeKind()) {
+ case TYPEREFERENCE:
+ metaItemType = metadataProvider.findType(metaItemTypeDataverseName, metaItemTypeName);
+ if (metaItemType.getTypeTag() != ATypeTag.OBJECT) {
+ throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
+ "Dataset meta type has to be a record type.");
+ }
+ break;
+ case RECORD:
+ metaItemTypeDataverseName = dataverseName;
+ metaItemTypeName = DatasetUtil.createInlineTypeName(datasetName, true);
+ lockUtil.createTypeBegin(lockManager, metadataProvider.getLocks(),
+ metaItemTypeDataverseName, metaItemTypeName);
+ metaItemType = translateType(metaItemTypeDataverseName, metaItemTypeName,
+ metaItemTypeExpr, mdTxnCtx);
+ MetadataManager.INSTANCE.addDatatype(mdTxnCtx,
+ new Datatype(metaItemTypeDataverseName, metaItemTypeName, metaItemType, true));
+ break;
+ default:
+ throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, sourceLoc,
+ String.valueOf(metaItemTypeExpr.getTypeKind()));
+ }
}
ARecordType metaRecType = (ARecordType) metaItemType;
@@ -1274,10 +1335,7 @@
throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc,
"Cannot redefine builtin type " + typeName + ".");
} else {
- Map<TypeSignature, IAType> typeMap = TypeTranslator.computeTypes(dataverseName,
- stmtCreateType.getIdent().getValue(), stmtCreateType.getTypeDef(), dataverseName, mdTxnCtx);
- TypeSignature typeSignature = new TypeSignature(dataverseName, typeName);
- IAType type = typeMap.get(typeSignature);
+ IAType type = translateType(dataverseName, typeName, stmtCreateType.getTypeDef(), mdTxnCtx);
MetadataManager.INSTANCE.addDatatype(mdTxnCtx, new Datatype(dataverseName, typeName, type, false));
}
}
@@ -1290,6 +1348,14 @@
}
}
+ private IAType translateType(DataverseName dataverseName, String typeName, TypeExpression typeDef,
+ MetadataTransactionContext mdTxnCtx) throws AlgebricksException {
+ Map<TypeSignature, IAType> typeMap =
+ TypeTranslator.computeTypes(dataverseName, typeName, typeDef, dataverseName, mdTxnCtx);
+ TypeSignature typeSignature = new TypeSignature(dataverseName, typeName);
+ return typeMap.get(typeSignature);
+ }
+
protected void handleDataverseDropStatement(MetadataProvider metadataProvider, Statement stmt,
IHyracksClientConnection hcc, IRequestParameters requestParameters) throws Exception {
DataverseDropStatement stmtDropDataverse = (DataverseDropStatement) stmt;
@@ -1515,8 +1581,39 @@
}
}
validateDatasetState(metadataProvider, ds, sourceLoc);
+
+ // prepare to drop item and meta types if they were created as inline types
+ DataverseName itemTypeDataverseName = ds.getItemTypeDataverseName();
+ String itemTypeName = ds.getItemTypeName();
+ boolean isInlineItemType = DatasetUtil.isInlineTypeName(ds, itemTypeDataverseName, itemTypeName);
+ if (isInlineItemType) {
+ lockUtil.dropTypeBegin(lockManager, metadataProvider.getLocks(), itemTypeDataverseName, itemTypeName);
+ }
+ DataverseName metaTypeDataverseName = ds.getMetaItemTypeDataverseName();
+ String metaTypeName = ds.getMetaItemTypeName();
+ boolean isInlineMetaType =
+ metaTypeName != null && DatasetUtil.isInlineTypeName(ds, metaTypeDataverseName, metaTypeName);
+ if (isInlineMetaType) {
+ lockUtil.dropTypeBegin(lockManager, metadataProvider.getLocks(), metaTypeDataverseName, metaTypeName);
+ }
+ Datatype inlineItemType = isInlineItemType
+ ? MetadataManager.INSTANCE.getDatatype(mdTxnCtx.getValue(), itemTypeDataverseName, itemTypeName)
+ : null;
+ Datatype inlineMetaType = isInlineMetaType
+ ? MetadataManager.INSTANCE.getDatatype(mdTxnCtx.getValue(), metaTypeDataverseName, metaTypeName)
+ : null;
+
ds.drop(metadataProvider, mdTxnCtx, jobsToExecute, bActiveTxn, progress, hcc, dropCorrespondingNodeGroup,
sourceLoc);
+
+ // drop inline item and meta types
+ if (isInlineItemType && inlineItemType.getIsAnonymous()) {
+ MetadataManager.INSTANCE.dropDatatype(mdTxnCtx.getValue(), itemTypeDataverseName, itemTypeName);
+ }
+ if (isInlineMetaType && inlineMetaType.getIsAnonymous()) {
+ MetadataManager.INSTANCE.dropDatatype(mdTxnCtx.getValue(), metaTypeDataverseName, metaTypeName);
+ }
+
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx.getValue());
} catch (Exception e) {
if (bActiveTxn.booleanValue()) {
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.1.ddl.sqlpp
new file mode 100644
index 0000000..6230d4b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.1.ddl.sqlpp
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+/* Metadata test function */
+
+CREATE FUNCTION listMetadata() {
+ SELECT "Dataset" AS en, d.DatasetName, d.DatatypeDataverseName, d.DatatypeName,
+ d.MetatypeDataverseName, d.MetatypeName
+ FROM Metadata.`Dataset` d
+ WHERE d.DataverseName = "test"
+ UNION ALL
+ SELECT "Datatype" AS en, dt.DatatypeName, dt.Derived
+ FROM Metadata.`Datatype` dt
+ WHERE dt.DataverseName = "test"
+ ORDER BY en, DatasetName, DatatypeName
+};
+
+/* Internal datasets */
+
+CREATE DATASET A_Customers_Default_Closed(
+ c_custkey integer not null,
+ c_name string not null,
+ c_phone string,
+ c_comment string
+) PRIMARY KEY c_custkey ;
+
+CREATE DATASET A_Customers_Closed(
+ c_custkey integer not null,
+ c_name string not null,
+ c_phone string,
+ c_comment string
+) CLOSED TYPE PRIMARY KEY c_custkey;
+
+CREATE DATASET A_Customers_Open(
+ c_custkey integer not null,
+ c_name string not null,
+ c_phone string,
+ c_comment string
+) OPEN TYPE PRIMARY KEY c_custkey;
+
+/* External datasets */
+
+CREATE EXTERNAL DATASET B_Orders_Default_Closed(
+ o_orderkey integer not null,
+ o_custkey integer not null,
+ o_orderstatus string not null,
+ o_totalprice double not null,
+ o_orderdate string not null,
+ o_orderpriority string not null,
+ o_clerk string not null,
+ o_shippriority integer not null,
+ o_comment string
+)
+USING `localfs`
+((`path`=`asterix_nc1://data/tpch0.001/orders.tbl`),
+(`input-format`=`text-input-format`),(`format`=`delimited-text`),(`delimiter`=`|`));
+
+CREATE EXTERNAL DATASET B_Orders_Closed(
+ o_orderkey integer not null,
+ o_custkey integer not null,
+ o_orderstatus string not null,
+ o_totalprice double not null,
+ o_orderdate string not null,
+ o_orderpriority string not null,
+ o_clerk string not null,
+ o_shippriority integer not null,
+ o_comment string
+) CLOSED TYPE
+USING `localfs`
+((`path`=`asterix_nc1://data/tpch0.001/orders.tbl`),
+(`input-format`=`text-input-format`),(`format`=`delimited-text`),(`delimiter`=`|`));
+
+CREATE EXTERNAL DATASET B_Orders_Open(
+ o_orderkey integer not null,
+ o_custkey integer not null,
+ o_orderstatus string not null,
+ o_totalprice double not null,
+ o_orderdate string not null,
+ o_orderpriority string not null,
+ o_clerk string not null,
+ o_shippriority integer not null,
+ o_comment string
+) OPEN TYPE
+USING `localfs`
+((`path`=`asterix_nc1://data/tpch0.001/orders.tbl`),
+(`input-format`=`text-input-format`),(`format`=`delimited-text`),(`delimiter`=`|`));
+
+/* Internal datasets with inline META type */
+
+CREATE DATASET C_Customers_Meta_Default_Closed(
+ c_custkey integer not null,
+ c_name string not null,
+ c_phone string,
+ c_comment string
+)
+WITH META(c_x integer not null, c_y integer)
+PRIMARY KEY c_custkey ;
+
+CREATE DATASET C_Customers_Meta_Closed(
+ c_custkey integer not null,
+ c_name string not null,
+ c_phone string,
+ c_comment string
+) CLOSED TYPE
+WITH META(
+ c_x integer not null,
+ c_y integer
+) CLOSED TYPE
+PRIMARY KEY c_custkey;
+
+CREATE DATASET C_Customers_Meta_Open(
+ c_custkey integer not null,
+ c_name string not null,
+ c_phone string,
+ c_comment string
+) OPEN TYPE
+WITH META(
+ c_x integer not null,
+ c_y integer
+) OPEN TYPE
+PRIMARY KEY c_custkey;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.2.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.2.query.sqlpp
new file mode 100644
index 0000000..52a6324
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.2.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+listMetadata();
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.3.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.3.ddl.sqlpp
new file mode 100644
index 0000000..8a08888
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.3.ddl.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+DROP DATASET A_Customers_Default_Closed;
+DROP DATASET A_Customers_Closed;
+DROP DATASET A_Customers_Open;
+DROP DATASET B_Orders_Default_Closed;
+DROP DATASET B_Orders_Closed;
+DROP DATASET B_Orders_Open;
+DROP DATASET C_Customers_Meta_Default_Closed;
+DROP DATASET C_Customers_Meta_Closed;
+DROP DATASET C_Customers_Meta_Open;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.4.query.sqlpp
new file mode 100644
index 0000000..ad16400
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.4.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Test that inline types are deleted when dataset is dropped
+ */
+
+USE test;
+
+array_count(listMetadata());
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.2.adm
new file mode 100644
index 0000000..8a539c8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.2.adm
@@ -0,0 +1,21 @@
+{ "en": "Dataset", "DatatypeName": "$d$t$i$A_Customers_Closed", "DatasetName": "A_Customers_Closed", "DatatypeDataverseName": "test" }
+{ "en": "Dataset", "DatatypeName": "$d$t$i$A_Customers_Default_Closed", "DatasetName": "A_Customers_Default_Closed", "DatatypeDataverseName": "test" }
+{ "en": "Dataset", "DatatypeName": "$d$t$i$A_Customers_Open", "DatasetName": "A_Customers_Open", "DatatypeDataverseName": "test" }
+{ "en": "Dataset", "DatatypeName": "$d$t$i$B_Orders_Closed", "DatasetName": "B_Orders_Closed", "DatatypeDataverseName": "test" }
+{ "en": "Dataset", "DatatypeName": "$d$t$i$B_Orders_Default_Closed", "DatasetName": "B_Orders_Default_Closed", "DatatypeDataverseName": "test" }
+{ "en": "Dataset", "DatatypeName": "$d$t$i$B_Orders_Open", "DatasetName": "B_Orders_Open", "DatatypeDataverseName": "test" }
+{ "en": "Dataset", "DatatypeName": "$d$t$i$C_Customers_Meta_Closed", "DatasetName": "C_Customers_Meta_Closed", "DatatypeDataverseName": "test", "MetatypeDataverseName": "test", "MetatypeName": "$d$t$m$C_Customers_Meta_Closed" }
+{ "en": "Dataset", "DatatypeName": "$d$t$i$C_Customers_Meta_Default_Closed", "DatasetName": "C_Customers_Meta_Default_Closed", "DatatypeDataverseName": "test", "MetatypeDataverseName": "test", "MetatypeName": "$d$t$m$C_Customers_Meta_Default_Closed" }
+{ "en": "Dataset", "DatatypeName": "$d$t$i$C_Customers_Meta_Open", "DatasetName": "C_Customers_Meta_Open", "DatatypeDataverseName": "test", "MetatypeDataverseName": "test", "MetatypeName": "$d$t$m$C_Customers_Meta_Open" }
+{ "en": "Datatype", "DatatypeName": "$d$t$i$A_Customers_Closed", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": false, "Fields": [ { "FieldName": "c_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_name", "FieldType": "string", "IsNullable": false }, { "FieldName": "c_phone", "FieldType": "string", "IsNullable": true }, { "FieldName": "c_comment", "FieldType": "string", "IsNullable": true } ] } } }
+{ "en": "Datatype", "DatatypeName": "$d$t$i$A_Customers_Default_Closed", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": false, "Fields": [ { "FieldName": "c_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_name", "FieldType": "string", "IsNullable": false }, { "FieldName": "c_phone", "FieldType": "string", "IsNullable": true }, { "FieldName": "c_comment", "FieldType": "string", "IsNullable": true } ] } } }
+{ "en": "Datatype", "DatatypeName": "$d$t$i$A_Customers_Open", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "c_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_name", "FieldType": "string", "IsNullable": false }, { "FieldName": "c_phone", "FieldType": "string", "IsNullable": true }, { "FieldName": "c_comment", "FieldType": "string", "IsNullable": true } ] } } }
+{ "en": "Datatype", "DatatypeName": "$d$t$i$B_Orders_Closed", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": false, "Fields": [ { "FieldName": "o_orderkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "o_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "o_orderstatus", "FieldType": "string", "IsNullable": false }, { "FieldName": "o_totalprice", "FieldType": "double", "IsNullable": false }, { "FieldName": "o_orderdate", "FieldType": "string", "IsNullable": false }, { "FieldName": "o_orderpriority", "FieldType": "string", "IsNullable": false }, { "FieldName": "o_clerk", "FieldType": "string", "IsNullable": false }, { "FieldName": "o_shippriority", "FieldType": "int32", "IsNullable": false }, { "FieldName": "o_comment", "FieldType": "string", "IsNullable": true } ] } } }
+{ "en": "Datatype", "DatatypeName": "$d$t$i$B_Orders_Default_Closed", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": false, "Fields": [ { "FieldName": "o_orderkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "o_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "o_orderstatus", "FieldType": "string", "IsNullable": false }, { "FieldName": "o_totalprice", "FieldType": "double", "IsNullable": false }, { "FieldName": "o_orderdate", "FieldType": "string", "IsNullable": false }, { "FieldName": "o_orderpriority", "FieldType": "string", "IsNullable": false }, { "FieldName": "o_clerk", "FieldType": "string", "IsNullable": false }, { "FieldName": "o_shippriority", "FieldType": "int32", "IsNullable": false }, { "FieldName": "o_comment", "FieldType": "string", "IsNullable": true } ] } } }
+{ "en": "Datatype", "DatatypeName": "$d$t$i$B_Orders_Open", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "o_orderkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "o_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "o_orderstatus", "FieldType": "string", "IsNullable": false }, { "FieldName": "o_totalprice", "FieldType": "double", "IsNullable": false }, { "FieldName": "o_orderdate", "FieldType": "string", "IsNullable": false }, { "FieldName": "o_orderpriority", "FieldType": "string", "IsNullable": false }, { "FieldName": "o_clerk", "FieldType": "string", "IsNullable": false }, { "FieldName": "o_shippriority", "FieldType": "int32", "IsNullable": false }, { "FieldName": "o_comment", "FieldType": "string", "IsNullable": true } ] } } }
+{ "en": "Datatype", "DatatypeName": "$d$t$i$C_Customers_Meta_Closed", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": false, "Fields": [ { "FieldName": "c_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_name", "FieldType": "string", "IsNullable": false }, { "FieldName": "c_phone", "FieldType": "string", "IsNullable": true }, { "FieldName": "c_comment", "FieldType": "string", "IsNullable": true } ] } } }
+{ "en": "Datatype", "DatatypeName": "$d$t$i$C_Customers_Meta_Default_Closed", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": false, "Fields": [ { "FieldName": "c_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_name", "FieldType": "string", "IsNullable": false }, { "FieldName": "c_phone", "FieldType": "string", "IsNullable": true }, { "FieldName": "c_comment", "FieldType": "string", "IsNullable": true } ] } } }
+{ "en": "Datatype", "DatatypeName": "$d$t$i$C_Customers_Meta_Open", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "c_custkey", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_name", "FieldType": "string", "IsNullable": false }, { "FieldName": "c_phone", "FieldType": "string", "IsNullable": true }, { "FieldName": "c_comment", "FieldType": "string", "IsNullable": true } ] } } }
+{ "en": "Datatype", "DatatypeName": "$d$t$m$C_Customers_Meta_Closed", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": false, "Fields": [ { "FieldName": "c_x", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_y", "FieldType": "int32", "IsNullable": true } ] } } }
+{ "en": "Datatype", "DatatypeName": "$d$t$m$C_Customers_Meta_Default_Closed", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": false, "Fields": [ { "FieldName": "c_x", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_y", "FieldType": "int32", "IsNullable": true } ] } } }
+{ "en": "Datatype", "DatatypeName": "$d$t$m$C_Customers_Meta_Open", "Derived": { "Tag": "RECORD", "IsAnonymous": true, "Record": { "IsOpen": true, "Fields": [ { "FieldName": "c_x", "FieldType": "int32", "IsNullable": false }, { "FieldName": "c_y", "FieldType": "int32", "IsNullable": true } ] } } }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.4.adm
new file mode 100644
index 0000000..c227083
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/ddl/create-dataset-inline-type-1/create-dataset-inline-type-1.4.adm
@@ -0,0 +1 @@
+0
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feeds_01/feeds_01.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feeds_01/feeds_01.1.adm
index 1dc31dc..3ab469e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feeds_01/feeds_01.1.adm
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/feeds/feeds_01/feeds_01.1.adm
@@ -1 +1 @@
-{ "DataverseName": "feeds", "FeedName": "TweetFeed", "AdapterConfiguration": {{ { "Name": "path", "Value": "asterix_nc1://data/twitter/obamatweets.adm" }, { "Name": "feed", "Value": "TweetFeed" }, { "Name": "adapter-name", "Value": "localfs" }, { "Name": "is-feed", "Value": "true" }, { "Name": "parser", "Value": "adm" }, { "Name": "reader", "Value": "localfs" }, { "Name": "format", "Value": "adm" }, { "Name": "tuple-interval", "Value": "10" }, { "Name": "linkName", "Value": "localfs" }, { "Name": "type-name", "Value": "TweetType" }, { "Name": "dataverse", "Value": "feeds" } }}, "Timestamp": "Wed Feb 26 20:33:46 AST 2020" }
\ No newline at end of file
+{ "DataverseName": "feeds", "FeedName": "TweetFeed", "AdapterConfiguration": {{ { "Name": "path", "Value": "asterix_nc1://data/twitter/obamatweets.adm" }, { "Name": "feed", "Value": "TweetFeed" }, { "Name": "adapter-name", "Value": "localfs" }, { "Name": "is-feed", "Value": "true" }, { "Name": "parser", "Value": "adm" }, { "Name": "reader", "Value": "localfs" }, { "Name": "format", "Value": "adm" }, { "Name": "tuple-interval", "Value": "10" }, { "Name": "type-name", "Value": "TweetType" }, { "Name": "dataverse", "Value": "feeds" } }}, "Timestamp": "Tue Mar 31 10:30:06 PDT 2020" }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/cross-dataverse/cross-dv01/cross-dv01.1.ast b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/cross-dataverse/cross-dv01/cross-dv01.1.ast
index c3a4877..ffe5bc3 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/cross-dataverse/cross-dv01/cross-dv01.1.ast
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results_parser_sqlpp/cross-dataverse/cross-dv01/cross-dv01.1.ast
@@ -17,7 +17,7 @@
dept : string
}
]
-DatasetDecl ugdstd(stdType) partitioned by [[id]]
-DatasetDecl gdstd(stdType) partitioned by [[id]]
+DatasetDecl ugdstd(student.stdType) partitioned by [[id]]
+DatasetDecl gdstd(student.stdType) partitioned by [[id]]
DatasetDecl prof(tchrType) partitioned by [[id]]
-DatasetDecl pstdoc(tchrType) partitioned by [[id]]
+DatasetDecl pstdoc(tchrType) partitioned by [[id]]
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index b473e60..e4b909e 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -3927,6 +3927,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="ddl">
+ <compilation-unit name="create-dataset-inline-type-1">
+ <output-dir compare="Text">create-dataset-inline-type-1</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="ddl">
<compilation-unit name="drop-primary-index">
<output-dir compare="Text">drop-primary-index</output-dir>
<expected-error>Cannot drop index "ds". Drop dataset "ds" to remove this index</expected-error>
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
index d081e56..078316c 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/adapter/factory/GenericAdapterFactory.java
@@ -25,6 +25,7 @@
import org.apache.asterix.common.api.IApplicationContext;
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
+import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.external.IDataSourceAdapter;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.external.api.IDataFlowController;
@@ -122,7 +123,7 @@
private void restoreExternalObjects(IServiceContext serviceContext, ILibraryManager libraryManager)
throws HyracksDataException, AlgebricksException {
if (dataSourceFactory == null) {
- dataSourceFactory = DatasourceFactoryProvider.getExternalDataSourceFactory(libraryManager, configuration);
+ dataSourceFactory = createExternalDataSourceFactory(configuration, libraryManager);
// create and configure parser factory
if (dataSourceFactory.isIndexible() && (files != null)) {
((IIndexibleExternalDataSource) dataSourceFactory).setSnapshot(files, indexingOp);
@@ -131,7 +132,7 @@
}
if (dataParserFactory == null) {
// create and configure parser factory
- dataParserFactory = ParserFactoryProvider.getDataParserFactory(libraryManager, configuration);
+ dataParserFactory = createDataParserFactory(configuration, libraryManager);
dataParserFactory.setRecordType(recordType);
dataParserFactory.setMetaType(metaType);
dataParserFactory.configure(configuration);
@@ -144,14 +145,13 @@
this.configuration = configuration;
IApplicationContext appCtx = (IApplicationContext) serviceContext.getApplicationContext();
ExternalDataUtils.validateDataSourceParameters(configuration);
- dataSourceFactory =
- DatasourceFactoryProvider.getExternalDataSourceFactory(appCtx.getLibraryManager(), configuration);
+ dataSourceFactory = createExternalDataSourceFactory(configuration, appCtx.getLibraryManager());
if (dataSourceFactory.isIndexible() && (files != null)) {
((IIndexibleExternalDataSource) dataSourceFactory).setSnapshot(files, indexingOp);
}
dataSourceFactory.configure(serviceContext, configuration);
ExternalDataUtils.validateDataParserParameters(configuration);
- dataParserFactory = ParserFactoryProvider.getDataParserFactory(appCtx.getLibraryManager(), configuration);
+ dataParserFactory = createDataParserFactory(configuration, appCtx.getLibraryManager());
dataParserFactory.setRecordType(recordType);
dataParserFactory.setMetaType(metaType);
dataParserFactory.configure(configuration);
@@ -222,4 +222,14 @@
dataParserFactory.configure(Collections.emptyMap());
configuration = Collections.emptyMap();
}
+
+ protected IExternalDataSourceFactory createExternalDataSourceFactory(Map<String, String> configuration,
+ ILibraryManager libraryManager) throws HyracksDataException, AsterixException {
+ return DatasourceFactoryProvider.getExternalDataSourceFactory(libraryManager, configuration);
+ }
+
+ protected IDataParserFactory createDataParserFactory(Map<String, String> configuration,
+ ILibraryManager libraryManager) throws AsterixException {
+ return ParserFactoryProvider.getDataParserFactory(libraryManager, configuration);
+ }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataCompatibilityUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataCompatibilityUtils.java
index 77cbb96..e222e99 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataCompatibilityUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataCompatibilityUtils.java
@@ -60,11 +60,8 @@
}
public static void prepare(String adapterName, Map<String, String> configuration) {
- // Adapter name in some cases can carry the link name for external datasets, always add it to configuration
- configuration.put(ExternalDataConstants.KEY_LINK_NAME, adapterName);
-
- if (!configuration.containsKey(ExternalDataConstants.KEY_READER)) { // SThree
- configuration.put(ExternalDataConstants.KEY_READER, adapterName); // myAwsLink
+ if (!configuration.containsKey(ExternalDataConstants.KEY_READER)) {
+ configuration.put(ExternalDataConstants.KEY_READER, adapterName);
}
if (!configuration.containsKey(ExternalDataConstants.KEY_PARSER)) {
if (configuration.containsKey(ExternalDataConstants.KEY_FORMAT)) {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index e44144a..26f5402 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -231,16 +231,6 @@
public static final String ERROR_PARSE_RECORD = "Parser failed to parse record";
- // TODO(Hussain): Move link related items to a different place
- /**
- * Common external link fields
- */
- public static final String KEY_DATAVERSE_NAME = "dataverseName";
- public static final String KEY_LINK_NAME = "linkName";
- public static final String KEY_LINK_TYPE = "linkType";
- public static final String[] KEY_EXTERNAL_DATASET_REQUIRED_CONNECTION_PARAMETERS =
- new String[] { KEY_DATAVERSE_NAME, KEY_LINK_NAME, KEY_LINK_TYPE };
-
public static class AwsS3Constants {
public static final String REGION_FIELD_NAME = "region";
public static final String ACCESS_KEY_FIELD_NAME = "accessKey";
diff --git a/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj b/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj
index 88017db..ca515e5 100644
--- a/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj
+++ b/asterixdb/asterix-lang-aql/src/main/javacc/AQL.jj
@@ -499,6 +499,7 @@
Pair<DataverseName,Identifier> nameComponents = null;
boolean ifNotExists = false;
Pair<DataverseName,Identifier> typeComponents = null;
+ Pair<DataverseName,Identifier> metaTypeComponents = null;
String adapterName = null;
Map<String,String> properties = null;
FunctionSignature appliedFunction = null;
@@ -508,7 +509,6 @@
DatasetDecl dsetDecl = null;
boolean autogenerated = false;
Pair<Integer, List<String>> filterField = null;
- Pair<DataverseName,Identifier> metaTypeComponents = new Pair<DataverseName, Identifier>(null, null);
RecordConstructor withRecord = null;
}
{
@@ -527,10 +527,8 @@
try{
dsetDecl = new DatasetDecl(nameComponents.first,
nameComponents.second,
- typeComponents.first,
- typeComponents.second,
- metaTypeComponents.first,
- metaTypeComponents.second,
+ new TypeReferenceExpression(typeComponents),
+ null,
nodeGroupName != null? new Identifier(nodeGroupName): null,
hints,
DatasetType.EXTERNAL,
@@ -574,10 +572,8 @@
try{
dsetDecl = new DatasetDecl(nameComponents.first,
nameComponents.second,
- typeComponents.first,
- typeComponents.second,
- metaTypeComponents.first,
- metaTypeComponents.second,
+ new TypeReferenceExpression(typeComponents),
+ metaTypeComponents != null ? new TypeReferenceExpression(metaTypeComponents) : null,
nodeGroupName != null ? new Identifier(nodeGroupName) : null,
hints,
DatasetType.INTERNAL,
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java
index 811a494..c0ad1b8 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/DatasetDecl.java
@@ -26,6 +26,7 @@
import org.apache.asterix.lang.common.base.AbstractStatement;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.lang.common.expression.RecordConstructor;
+import org.apache.asterix.lang.common.expression.TypeExpression;
import org.apache.asterix.lang.common.struct.Identifier;
import org.apache.asterix.lang.common.util.ConfigurationUtil;
import org.apache.asterix.lang.common.util.DatasetDeclParametersUtil;
@@ -37,10 +38,8 @@
public class DatasetDecl extends AbstractStatement {
protected final Identifier name;
protected final DataverseName dataverse;
- protected final DataverseName itemTypeDataverse;
- protected final Identifier itemTypeName;
- protected final DataverseName metaItemTypeDataverse;
- protected final Identifier metaItemTypeName;
+ protected final TypeExpression itemType;
+ protected final TypeExpression metaItemType;
protected final Identifier nodegroupName;
protected final DatasetType datasetType;
protected final IDatasetDetailsDecl datasetDetailsDecl;
@@ -48,16 +47,13 @@
private AdmObjectNode withObjectNode;
protected final boolean ifNotExists;
- public DatasetDecl(DataverseName dataverse, Identifier name, DataverseName itemTypeDataverse,
- Identifier itemTypeName, DataverseName metaItemTypeDataverse, Identifier metaItemTypeName,
+ public DatasetDecl(DataverseName dataverse, Identifier name, TypeExpression itemType, TypeExpression metaItemType,
Identifier nodeGroupName, Map<String, String> hints, DatasetType datasetType, IDatasetDetailsDecl idd,
RecordConstructor withRecord, boolean ifNotExists) throws CompilationException {
this.dataverse = dataverse;
this.name = name;
- this.itemTypeName = itemTypeName;
- this.itemTypeDataverse = itemTypeDataverse == null ? dataverse : itemTypeDataverse;
- this.metaItemTypeName = metaItemTypeName;
- this.metaItemTypeDataverse = metaItemTypeDataverse == null ? dataverse : metaItemTypeDataverse;
+ this.itemType = itemType;
+ this.metaItemType = metaItemType;
this.nodegroupName = nodeGroupName;
this.hints = hints;
this.withObjectNode = DatasetDeclParametersUtil.validateAndGetWithObjectNode(withRecord, datasetType);
@@ -82,20 +78,12 @@
return dataverse;
}
- public Identifier getItemTypeName() {
- return itemTypeName;
+ public TypeExpression getItemType() {
+ return itemType;
}
- public DataverseName getItemTypeDataverse() {
- return itemTypeDataverse;
- }
-
- public Identifier getMetaItemTypeName() {
- return metaItemTypeName;
- }
-
- public DataverseName getMetaItemTypeDataverse() {
- return metaItemTypeDataverse;
+ public TypeExpression getMetaItemType() {
+ return metaItemType;
}
public Identifier getNodegroupName() {
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
index 4903e4b..7d78746 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
@@ -432,17 +432,18 @@
public Void visit(DatasetDecl dd, Integer step) throws CompilationException {
if (dd.getDatasetType() == DatasetType.INTERNAL) {
out.print(skip(step) + "create " + datasetSymbol + generateFullName(dd.getDataverse(), dd.getName())
- + generateIfNotExists(dd.getIfNotExists()) + "("
- + generateFullName(dd.getItemTypeDataverse(), dd.getItemTypeName()) + ")" + " primary key ");
+ + generateIfNotExists(dd.getIfNotExists()) + "(");
+ dd.getItemType().accept(this, step + 2);
+ out.print(skip(step) + ") primary key ");
printDelimitedKeys(((InternalDetailsDecl) dd.getDatasetDetailsDecl()).getPartitioningExprs(), ",");
if (((InternalDetailsDecl) dd.getDatasetDetailsDecl()).isAutogenerated()) {
out.print(" autogenerated ");
}
} else if (dd.getDatasetType() == DatasetType.EXTERNAL) {
- out.print(
- skip(step) + "create external " + datasetSymbol + generateFullName(dd.getDataverse(), dd.getName())
- + "(" + generateFullName(dd.getItemTypeDataverse(), dd.getItemTypeName()) + ")"
- + generateIfNotExists(dd.getIfNotExists()));
+ out.print(skip(step) + "create external " + datasetSymbol
+ + generateFullName(dd.getDataverse(), dd.getName()) + "(");
+ dd.getItemType().accept(this, step + 2);
+ out.print(skip(step) + ")" + generateIfNotExists(dd.getIfNotExists()));
ExternalDetailsDecl externalDetails = (ExternalDetailsDecl) dd.getDatasetDetailsDecl();
out.print(" using " + revertStringToQuoted(externalDetails.getAdapter()));
printConfiguration(externalDetails.getProperties());
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/QueryPrintVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/QueryPrintVisitor.java
index a734918..1dfec2a 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/QueryPrintVisitor.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/QueryPrintVisitor.java
@@ -367,15 +367,18 @@
@Override
public Void visit(DatasetDecl dd, Integer step) throws CompilationException {
if (dd.getDatasetType() == DatasetType.INTERNAL) {
- String line = skip(step) + "DatasetDecl " + dd.getName() + "(" + dd.getItemTypeName() + ")"
- + " partitioned by " + ((InternalDetailsDecl) dd.getDatasetDetailsDecl()).getPartitioningExprs();
+ out.print(skip(step) + "DatasetDecl " + dd.getName() + "(");
+ dd.getItemType().accept(this, step + 2);
+ out.print(skip(step) + ") partitioned by "
+ + ((InternalDetailsDecl) dd.getDatasetDetailsDecl()).getPartitioningExprs());
if (((InternalDetailsDecl) dd.getDatasetDetailsDecl()).isAutogenerated()) {
- line += " [autogenerated]";
+ out.print(" [autogenerated]");
}
- out.println(line);
+ out.println();
} else if (dd.getDatasetType() == DatasetType.EXTERNAL) {
- out.println(skip(step) + "DatasetDecl " + dd.getName() + "(" + dd.getItemTypeName() + ")"
- + "is an external dataset");
+ out.print(skip(step) + "DatasetDecl " + dd.getName() + "(");
+ dd.getItemType().accept(this, step + 2);
+ out.println(skip(step) + ")is an external dataset");
}
return null;
}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index 53faff9..17c94bd 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -385,15 +385,15 @@
hintCollector.clear();
try {
return parseFunction.parse();
+ } catch (SqlppParseException e) {
+ throw new CompilationException(ErrorCode.PARSE_ERROR, e.getSourceLocation(), LogRedactionUtil.userData(getMessage(e)));
+ } catch (ParseException e) {
+ throw new CompilationException(ErrorCode.PARSE_ERROR, LogRedactionUtil.userData(getMessage(e)));
} catch (Error e) {
// this is here as the JavaCharStream that's below the lexer sometimes throws Errors that are not handled
// by the ANTLR-generated lexer or parser (e.g it does this for invalid backslash u + 4 hex digits escapes)
final String msg = e.getClass().getSimpleName() + (e.getMessage() != null ? ": " + e.getMessage() : "");
throw new CompilationException(ErrorCode.PARSE_ERROR, LogRedactionUtil.userData(msg));
- } catch (SqlppParseException e) {
- throw new CompilationException(ErrorCode.PARSE_ERROR, e.getSourceLocation(), LogRedactionUtil.userData(getMessage(e)));
- } catch (ParseException e) {
- throw new CompilationException(ErrorCode.PARSE_ERROR, LogRedactionUtil.userData(getMessage(e)));
} finally {
reportUnclaimedHints();
}
@@ -478,13 +478,21 @@
}
private SqlppParseException createUnexpectedTokenError() {
- return createUnexpectedTokenError(null);
+ return createUnexpectedTokenError(token, null);
+ }
+
+ private SqlppParseException createUnexpectedTokenError(Token t) {
+ return createUnexpectedTokenError(t, null);
}
private SqlppParseException createUnexpectedTokenError(String expected) {
- String message = "Unexpected token: " + LogRedactionUtil.userData(token.image) +
+ return createUnexpectedTokenError(token, expected);
+ }
+
+ private SqlppParseException createUnexpectedTokenError(Token t, String expected) {
+ String message = "Unexpected token: " + LogRedactionUtil.userData(t.image) +
(expected == null ? "" : ". Expected: " + LogRedactionUtil.userData(expected));
- return new SqlppParseException(getSourceLocation(token), message);
+ return new SqlppParseException(getSourceLocation(t), message);
}
private boolean laToken(int idx, int kind) {
@@ -751,22 +759,19 @@
{
Pair<DataverseName,Identifier> nameComponents = null;
boolean ifNotExists = false;
- Pair<DataverseName,Identifier> typeComponents = null;
- String adapterName = null;
- Map<String,String> properties = null;
- FunctionSignature appliedFunction = null;
+ TypeExpression typeExpr = null;
+ TypeExpression metaTypeExpr = null;
Pair<List<Integer>, List<List<String>>> primaryKeyFields = null;
String nodeGroupName = null;
Map<String,String> hints = new HashMap<String,String>();
DatasetDecl stmt = null;
boolean autogenerated = false;
Pair<Integer, List<String>> filterField = null;
- Pair<DataverseName,Identifier> metaTypeComponents = new Pair<DataverseName, Identifier>(null, null);
RecordConstructor withRecord = null;
}
{
nameComponents = QualifiedName()
- <LEFTPAREN> typeComponents = TypeName() <RIGHTPAREN>
+ typeExpr = DatasetTypeSpecification()
(
{ String name; }
<WITH>
@@ -777,7 +782,7 @@
"We can only support one additional associated field called \"meta\".");
}
}
- <LEFTPAREN> metaTypeComponents = TypeName() <RIGHTPAREN>
+ metaTypeExpr = DatasetTypeSpecification()
)?
ifNotExists = IfNotExists()
primaryKeyFields = PrimaryKey()
@@ -794,8 +799,7 @@
InternalDetailsDecl idd = new InternalDetailsDecl(primaryKeyFields.second, primaryKeyFields.first, autogenerated,
filterField == null? null : filterField.second);
try {
- stmt = new DatasetDecl(nameComponents.first, nameComponents.second, typeComponents.first, typeComponents.second,
- metaTypeComponents.first, metaTypeComponents.second,
+ stmt = new DatasetDecl(nameComponents.first, nameComponents.second, typeExpr, metaTypeExpr,
nodeGroupName != null ? new Identifier(nodeGroupName) : null, hints, DatasetType.INTERNAL, idd, withRecord,
ifNotExists);
return addSourceLocation(stmt, startStmtToken);
@@ -808,19 +812,18 @@
DatasetDecl ExternalDatasetSpecification(Token startStmtToken) throws ParseException:
{
Pair<DataverseName,Identifier> nameComponents = null;
+ TypeExpression typeExpr = null;
boolean ifNotExists = false;
- Pair<DataverseName,Identifier> typeComponents = null;
String adapterName = null;
Map<String,String> properties = null;
String nodeGroupName = null;
Map<String,String> hints = new HashMap<String,String>();
DatasetDecl stmt = null;
- Pair<DataverseName,Identifier> metaTypeComponents = new Pair<DataverseName, Identifier>(null, null);
RecordConstructor withRecord = null;
}
{
nameComponents = QualifiedName()
- <LEFTPAREN> typeComponents = TypeName() <RIGHTPAREN>
+ typeExpr = DatasetTypeSpecification()
ifNotExists = IfNotExists()
<USING> adapterName = AdapterName() properties = Configuration()
( <ON> nodeGroupName = Identifier() )?
@@ -831,8 +834,7 @@
edd.setAdapter(adapterName);
edd.setProperties(properties);
try {
- stmt = new DatasetDecl(nameComponents.first, nameComponents.second, typeComponents.first, typeComponents.second,
- metaTypeComponents.first, metaTypeComponents.second,
+ stmt = new DatasetDecl(nameComponents.first, nameComponents.second, typeExpr, null,
nodeGroupName != null? new Identifier(nodeGroupName): null, hints, DatasetType.EXTERNAL, edd, withRecord,
ifNotExists);
return addSourceLocation(stmt, startStmtToken);
@@ -842,6 +844,76 @@
}
}
+TypeExpression DatasetTypeSpecification() throws ParseException:
+{
+ TypeExpression typeExpr = null;
+}
+{
+ (
+ LOOKAHEAD(3) typeExpr = DatasetRecordTypeSpecification(true)
+ | typeExpr = DatasetReferenceTypeSpecification()
+ )
+ {
+ return typeExpr;
+ }
+}
+
+TypeExpression DatasetReferenceTypeSpecification() throws ParseException:
+{
+ TypeExpression typeExpr = null;
+}
+{
+ <LEFTPAREN> typeExpr = TypeReference() <RIGHTPAREN>
+ {
+ return typeExpr;
+ }
+}
+
+TypeExpression DatasetRecordTypeSpecification(boolean allowRecordKindModifier) throws ParseException:
+{
+ RecordTypeDefinition recordTypeDef = null;
+ RecordTypeDefinition.RecordKind recordKind = null;
+ Token recordKindToken = null;
+}
+{
+ <LEFTPAREN> recordTypeDef = DatasetRecordTypeDef() <RIGHTPAREN>
+ ( recordKind = RecordTypeKind() { recordKindToken = token; } <TYPE> )?
+ {
+ if (recordKind == null) {
+ recordKind = RecordTypeDefinition.RecordKind.CLOSED;
+ } else if (!allowRecordKindModifier) {
+ throw createUnexpectedTokenError(recordKindToken);
+ }
+ recordTypeDef.setRecordKind(recordKind);
+ return recordTypeDef;
+ }
+}
+
+RecordTypeDefinition DatasetRecordTypeDef() throws ParseException:
+{
+ RecordTypeDefinition recType = new RecordTypeDefinition();
+}
+{
+ DatasetRecordField(recType) ( <COMMA> DatasetRecordField(recType) )*
+ {
+ return recType;
+ }
+}
+
+void DatasetRecordField(RecordTypeDefinition recType) throws ParseException:
+{
+ String fieldName;
+ TypeExpression type = null;
+ boolean isUnknownable = true;
+}
+{
+ fieldName = Identifier()
+ type = TypeReference() ( <NOT> <NULL> { isUnknownable = false; } )?
+ {
+ recType.addField(fieldName, type, isUnknownable);
+ }
+}
+
RefreshExternalDatasetStatement RefreshExternalDatasetStatement() throws ParseException:
{
Token startToken = null;
@@ -2077,15 +2149,28 @@
}
}
+RecordTypeDefinition.RecordKind RecordTypeKind() throws ParseException:
+{
+ RecordTypeDefinition.RecordKind recordKind = null;
+}
+{
+ (
+ <CLOSED> { recordKind = RecordTypeDefinition.RecordKind.CLOSED; }
+ | <OPEN> { recordKind = RecordTypeDefinition.RecordKind.OPEN; }
+ )
+ {
+ return recordKind;
+ }
+}
+
RecordTypeDefinition RecordTypeDef() throws ParseException:
{
Token startToken = null;
RecordTypeDefinition recType = new RecordTypeDefinition();
- RecordTypeDefinition.RecordKind recordKind = null;
+ RecordTypeDefinition.RecordKind recordKind = RecordTypeDefinition.RecordKind.OPEN;
}
{
- ( <CLOSED> { recordKind = RecordTypeDefinition.RecordKind.CLOSED; }
- | <OPEN> { recordKind = RecordTypeDefinition.RecordKind.OPEN; } )?
+ ( recordKind = RecordTypeKind() )?
<LEFTBRACE>
{
startToken = token;
@@ -2112,9 +2197,6 @@
)?
<RIGHTBRACE>
{
- if (recordKind == null) {
- recordKind = RecordTypeDefinition.RecordKind.OPEN;
- }
recType.setRecordKind(recordKind);
return addSourceLocation(recType, startToken);
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index 84166ce..78c026d 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@ -100,6 +100,8 @@
*/
public static final byte OP_UPSERT = 0x03;
+ private static final String DATASET_INLINE_TYPE_PREFIX = "$d$t$";
+
private DatasetUtil() {
}
@@ -208,9 +210,13 @@
* field is actually a key by making sure the field is coming from the right record (data record or meta record),
* e.g. if the field name happens to be equal to the key name but the field is coming from the data record while
* the key is coming from the meta record.
- * @param keySourceIndicator indicates where the key is coming from, 1 from meta record, 0 from data record
- * @param keyIndex the key index we're checking the field against
- * @param fieldFromMeta whether the field is coming from the meta record or the data record
+ *
+ * @param keySourceIndicator
+ * indicates where the key is coming from, 1 from meta record, 0 from data record
+ * @param keyIndex
+ * the key index we're checking the field against
+ * @param fieldFromMeta
+ * whether the field is coming from the meta record or the data record
* @return true if the key source matches the field source. Otherwise, false.
*/
private static boolean keySourceMatches(List<Integer> keySourceIndicator, int keyIndex, boolean fieldFromMeta) {
@@ -590,4 +596,12 @@
return nodeGroup;
}
+ public static String createInlineTypeName(String datasetName, boolean forMetaItemType) {
+ char typeChar = forMetaItemType ? 'm' : 'i';
+ return DATASET_INLINE_TYPE_PREFIX + typeChar + '$' + datasetName;
+ }
+
+ public static boolean isInlineTypeName(Dataset dataset, DataverseName typeDataverseName, String typeName) {
+ return dataset.getDataverseName().equals(typeDataverseName) && typeName.startsWith(DATASET_INLINE_TYPE_PREFIX);
+ }
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockUtil.java
index 88d292b..1b88a06 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/MetadataLockUtil.java
@@ -55,14 +55,16 @@
DataverseName metaItemTypeDataverseName, String metaItemTypeName, String nodeGroupName,
String compactionPolicyName, boolean isDefaultCompactionPolicy) throws AlgebricksException {
lockMgr.acquireDataverseReadLock(locks, dataverseName);
- if (!dataverseName.equals(itemTypeDataverseName)) {
+ if (itemTypeDataverseName != null && !dataverseName.equals(itemTypeDataverseName)) {
lockMgr.acquireDataverseReadLock(locks, itemTypeDataverseName);
}
if (metaItemTypeDataverseName != null && !metaItemTypeDataverseName.equals(dataverseName)
&& !metaItemTypeDataverseName.equals(itemTypeDataverseName)) {
lockMgr.acquireDataverseReadLock(locks, metaItemTypeDataverseName);
}
- lockMgr.acquireDataTypeReadLock(locks, itemTypeDataverseName, itemTypeName);
+ if (itemTypeName != null) {
+ lockMgr.acquireDataTypeReadLock(locks, itemTypeDataverseName, itemTypeName);
+ }
if (metaItemTypeDataverseName != null && !metaItemTypeDataverseName.equals(itemTypeDataverseName)
&& !metaItemTypeName.equals(itemTypeName)) {
lockMgr.acquireDataTypeReadLock(locks, metaItemTypeDataverseName, metaItemTypeName);