[ASTERIXDB-3287][SQL++] Support parsing COPY TO statement
- user model changes: yes
- storage format changes: no
- interface changes: no
Details:
- Support parsing COPY TO
- Change the syntax for COPY FROM to be more aligned
with the COPY TO
Change-Id: I5e3f42c9f3a39b1308c54d3bc365256510746a0c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17877
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
Tested-by: Wail Alkowaileet <wael.y.k@gmail.com>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
index 012df01..3a1d9ac 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
@@ -351,7 +351,7 @@
@Override
public Statement.Kind getKind() {
- return Statement.Kind.COPY;
+ return Statement.Kind.COPY_FROM;
}
@Override
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index d156b25..e482144 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -224,7 +224,7 @@
lds = new LoadableDataSource(dataset, itemType, metaItemType,
((CompiledLoadFromFileStatement) stmt).getAdapter(),
((CompiledLoadFromFileStatement) stmt).getProperties());
- } else if (stmt.getKind() == Statement.Kind.COPY) {
+ } else if (stmt.getKind() == Statement.Kind.COPY_FROM) {
CompiledCopyFromFileStatement copyStmt = (CompiledCopyFromFileStatement) stmt;
lds = new LoadableDataSource(dataset, copyStmt.getItemType().getDatatype(), metaItemType,
copyStmt.getAdapter(), copyStmt.getProperties());
@@ -316,7 +316,7 @@
leafOperator.getInputs().add(new MutableObject<>(insertOp));
leafOperator.setSourceLocation(sourceLoc);
return new ALogicalPlanImpl(new MutableObject<>(leafOperator));
- } else if (stmt.getKind() == Statement.Kind.COPY) {
+ } else if (stmt.getKind() == Statement.Kind.COPY_FROM) {
InsertDeleteUpsertOperator upsertOp = new InsertDeleteUpsertOperator(targetDatasource, payloadRef,
varRefsForLoading, InsertDeleteUpsertOperator.Kind.UPSERT, false);
upsertOp.setAdditionalFilteringExpressions(additionalFilteringExpressions);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index 9d94327..24db0c0 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -210,7 +210,7 @@
// establish facts
final boolean isQuery = query != null;
final boolean isLoad = statement != null && statement.getKind() == Statement.Kind.LOAD;
- final boolean isCopy = statement != null && statement.getKind() == Statement.Kind.COPY;
+ final boolean isCopy = statement != null && statement.getKind() == Statement.Kind.COPY_FROM;
final SourceLocation sourceLoc =
query != null ? query.getSourceLocation() : statement != null ? statement.getSourceLocation() : null;
final boolean isExplainOnly = isQuery && query.isExplain();
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index 75ff63b..6dd6cc9 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -120,7 +120,7 @@
import org.apache.asterix.lang.common.statement.AnalyzeStatement;
import org.apache.asterix.lang.common.statement.CompactStatement;
import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
-import org.apache.asterix.lang.common.statement.CopyStatement;
+import org.apache.asterix.lang.common.statement.CopyFromStatement;
import org.apache.asterix.lang.common.statement.CreateAdapterStatement;
import org.apache.asterix.lang.common.statement.CreateDatabaseStatement;
import org.apache.asterix.lang.common.statement.CreateDataverseStatement;
@@ -459,11 +459,11 @@
}
handleLoadStatement(metadataProvider, stmt, hcc);
break;
- case COPY:
+ case COPY_FROM:
if (stats.getProfileType() == Stats.ProfileType.FULL) {
this.jobFlags.add(JobFlag.PROFILE_RUNTIME);
}
- handleCopyStatement(metadataProvider, stmt, hcc);
+ handleCopyFromStatement(metadataProvider, stmt, hcc);
break;
case INSERT:
case UPSERT:
@@ -3842,15 +3842,20 @@
}
}
- protected Map<String, String> createExternalDataPropertiesForCopyStmt(String databaseName,
- DataverseName dataverseName, CopyStatement copyStatement, Datatype itemType,
+ protected Map<String, String> createExternalDataPropertiesForCopyFromStmt(String databaseName,
+ DataverseName dataverseName, CopyFromStatement copyFromStatement, Datatype itemType,
MetadataTransactionContext mdTxnCtx) throws AlgebricksException {
- return copyStatement.getExternalDetails().getProperties();
+ ExternalDetailsDecl edd = copyFromStatement.getExternalDetails();
+ Map<String, String> properties = copyFromStatement.getExternalDetails().getProperties();
+ String path = copyFromStatement.getPath();
+ String pathKey = ExternalDataUtils.getPathKey(edd.getAdapter());
+ properties.put(pathKey, path);
+ return properties;
}
- protected void handleCopyStatement(MetadataProvider metadataProvider, Statement stmt, IHyracksClientConnection hcc)
- throws Exception {
- CopyStatement copyStmt = (CopyStatement) stmt;
+ protected void handleCopyFromStatement(MetadataProvider metadataProvider, Statement stmt,
+ IHyracksClientConnection hcc) throws Exception {
+ CopyFromStatement copyStmt = (CopyFromStatement) stmt;
String datasetName = copyStmt.getDatasetName();
metadataProvider.validateDatabaseObjectName(copyStmt.getNamespace(), datasetName, copyStmt.getSourceLocation());
Namespace stmtActiveNamespace = getActiveNamespace(copyStmt.getNamespace());
@@ -3887,8 +3892,8 @@
new Datatype(itemTypeDatabaseName, itemTypeDataverseName, itemTypeName, itemTypeEntity, true);
}
ExternalDetailsDecl externalDetails = copyStmt.getExternalDetails();
- Map<String, String> properties =
- createExternalDataPropertiesForCopyStmt(databaseName, dataverseName, copyStmt, itemType, mdTxnCtx);
+ Map<String, String> properties = createExternalDataPropertiesForCopyFromStmt(databaseName, dataverseName,
+ copyStmt, itemType, mdTxnCtx);
ExternalDataUtils.normalize(properties);
ExternalDataUtils.validate(properties);
validateExternalDatasetProperties(externalDetails, properties, copyStmt.getSourceLocation(), mdTxnCtx,
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy/copy-1/copy-1.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy/copy-1/copy-1.2.update.sqlpp
index 002d43d..86b749f 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy/copy-1/copy-1.2.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy/copy-1/copy-1.2.update.sqlpp
@@ -33,9 +33,12 @@
}
]);
-copy Customers
-using localfs
-(("path"="asterix_nc1://data/nontagged/customerData.json"),("format"="adm"));
+COPY Customers
+FROM localfs
+PATH ("asterix_nc1://data/nontagged/customerData.json")
+WITH {
+ "format": "adm"
+};
upsert into Customers([
{
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy/copy-2/copy-2.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy/copy-2/copy-2.2.update.sqlpp
index baae560..60ad511 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy/copy-2/copy-2.2.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy/copy-2/copy-2.2.update.sqlpp
@@ -28,13 +28,15 @@
}
]);
-copy test1 USING S3 (
-("region"="us-west-2"),
-("serviceEndpoint"="http://127.0.0.1:8001"),
-("container"="playground"),
-("definition"="data_dir"),
-("format"="json")
-);
+COPY test1
+FROM S3
+PATH ("data_dir")
+WITH {
+ "format": "json",
+ "container": "playground",
+ "serviceEndpoint": "http://127.0.0.1:8001",
+ "region": "us-west-2"
+};
upsert into test1([
{
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-3/create-dataset-3.02.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-3/create-dataset-3.02.update.sqlpp
index ab30804..30e4526 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-3/create-dataset-3.02.update.sqlpp
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/ddl/create-dataset-3/create-dataset-3.02.update.sqlpp
@@ -18,6 +18,9 @@
*/
use test;
-copy orders
-using localfs
-(("path"="asterix_nc1://data/nontagged/orderData.json"),("format"="adm"));
+COPY orders
+FROM localfs
+PATH("asterix_nc1://data/nontagged/orderData.json")
+WITH {
+ "format": "adm"
+};
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 62dc466..439e852 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -1057,4 +1057,17 @@
+ ExternalDataConstants.KEY_EMBED_FILTER_VALUES + "' is enabled");
}
}
+
+ public static String getPathKey(String adapter) {
+ String normalizedAdapter = adapter.toUpperCase();
+ switch (normalizedAdapter) {
+ case ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3:
+ case ExternalDataConstants.KEY_ADAPTER_NAME_AZURE_BLOB:
+ case ExternalDataConstants.KEY_ADAPTER_NAME_AZURE_DATA_LAKE:
+ case ExternalDataConstants.KEY_ADAPTER_NAME_GCS:
+ return ExternalDataConstants.DEFINITION_FIELD_NAME;
+ default:
+ return ExternalDataConstants.KEY_PATH;
+ }
+ }
}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java
index 404360c..42c7eb6 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/Statement.java
@@ -27,21 +27,29 @@
Kind getKind();
/**
- * get a byte representing the statement category.
- * Each category describes the type of modifications this statement does.
+ * get a byte representing the statement category.
+ * Each category describes the type of modifications this statement does.
*
* @return kind byte
*/
byte getCategory();
class Category {
- /** no modifications */
+ /**
+ * no modifications
+ */
public static final byte QUERY = 0x01;
- /** modify data */
+ /**
+ * modify data
+ */
public static final byte UPDATE = 0x02;
- /** modify metadata */
+ /**
+ * modify metadata
+ */
public static final byte DDL = 0x04;
- /** modify anything */
+ /**
+ * modify anything
+ */
public static final byte PROCEDURE = 0x08;
private Category() {
@@ -113,6 +121,7 @@
COMPACT,
SUBSCRIBE_FEED,
EXTENSION,
- COPY
+ COPY_FROM,
+ COPY_TO,
}
}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyFromStatement.java
similarity index 76%
rename from asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyStatement.java
rename to asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyFromStatement.java
index a07371d..51c9803 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyFromStatement.java
@@ -28,18 +28,30 @@
import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
import org.apache.asterix.object.base.AdmObjectNode;
-public class CopyStatement extends AbstractStatement {
+public class CopyFromStatement extends AbstractStatement {
private final Namespace namespace;
private final String datasetName;
+ private final String path;
private final TypeExpression typeExpr;
private final ExternalDetailsDecl externalDetails;
private final AdmObjectNode withObjectNode;
- public CopyStatement(Namespace namespace, String datasetName, TypeExpression typeExpr,
+ public CopyFromStatement(Namespace namespace, String datasetName, TypeExpression typeExpr,
+ ExternalDetailsDecl externalDetails, RecordConstructor withRecord) throws CompilationException {
+ this(namespace, datasetName, null, typeExpr, externalDetails, withRecord);
+ }
+
+ public CopyFromStatement(Namespace namespace, String datasetName, String path, TypeExpression typeExpr,
+ ExternalDetailsDecl externalDetails) throws CompilationException {
+ this(namespace, datasetName, path, typeExpr, externalDetails, null);
+ }
+
+ private CopyFromStatement(Namespace namespace, String datasetName, String path, TypeExpression typeExpr,
ExternalDetailsDecl externalDetails, RecordConstructor withRecord) throws CompilationException {
this.namespace = namespace;
this.datasetName = datasetName;
+ this.path = path;
this.typeExpr = typeExpr;
this.externalDetails = externalDetails;
this.withObjectNode = withRecord == null ? new AdmObjectNode() : ExpressionUtils.toNode(withRecord);
@@ -55,7 +67,7 @@
@Override
public Kind getKind() {
- return Kind.COPY;
+ return Kind.COPY_FROM;
}
public String getDatasetName() {
@@ -75,6 +87,10 @@
return withObjectNode;
}
+ public String getPath() {
+ return path;
+ }
+
@Override
public byte getCategory() {
return Category.UPDATE;
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToStatement.java
new file mode 100644
index 0000000..52e3570
--- /dev/null
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToStatement.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.common.statement;
+
+import static org.apache.asterix.lang.common.base.Statement.Category.QUERY;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.metadata.Namespace;
+import org.apache.asterix.lang.common.base.AbstractStatement;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.IReturningStatement;
+import org.apache.asterix.lang.common.clause.OrderbyClause;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
+
+public class CopyToStatement extends AbstractStatement implements IReturningStatement {
+ private final Namespace namespace;
+ private final String datasetName;
+ private final VariableExpr sourceVariable;
+ private final ExternalDetailsDecl externalDetailsDecl;
+ private final Map<Integer, VariableExpr> partitionsVariables;
+ private final List<OrderbyClause.OrderModifier> orderbyModifiers;
+ private final List<OrderbyClause.NullOrderModifier> orderbyNullModifierList;
+
+ private Query query;
+ private Expression pathExpression;
+
+ private List<Expression> partitionExpressions;
+ private List<Expression> orderbyList;
+ private int varCounter;
+
+ public CopyToStatement(Namespace namespace, String datasetName, Query query, VariableExpr sourceVariable,
+ ExternalDetailsDecl externalDetailsDecl, Expression pathExpression, List<Expression> partitionExpressions,
+ Map<Integer, VariableExpr> partitionsVariables, List<Expression> orderbyList,
+ List<OrderbyClause.OrderModifier> orderbyModifiers,
+ List<OrderbyClause.NullOrderModifier> orderbyNullModifierList, int varCounter) {
+ this.namespace = namespace;
+ this.datasetName = datasetName;
+ this.query = query;
+ this.sourceVariable = sourceVariable;
+ this.externalDetailsDecl = externalDetailsDecl;
+ this.pathExpression = pathExpression;
+ this.partitionExpressions = partitionExpressions;
+ this.partitionsVariables = partitionsVariables;
+ this.orderbyList = orderbyList;
+ this.orderbyModifiers = orderbyModifiers;
+ this.orderbyNullModifierList = orderbyNullModifierList;
+ this.varCounter = varCounter;
+ }
+
+ @Override
+ public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws CompilationException {
+ return visitor.visit(this, arg);
+ }
+
+ @Override
+ public Kind getKind() {
+ return Kind.COPY_TO;
+ }
+
+ @Override
+ public byte getCategory() {
+ return QUERY;
+ }
+
+ public Namespace getNamespace() {
+ return namespace;
+ }
+
+ public String getDatasetName() {
+ return datasetName;
+ }
+
+ public void setQuery(Query query) {
+ this.query = query;
+ }
+
+ public Query getQuery() {
+ return query;
+ }
+
+ public VariableExpr getSourceVariable() {
+ return sourceVariable;
+ }
+
+ public ExternalDetailsDecl getExternalDetailsDecl() {
+ return externalDetailsDecl;
+ }
+
+ public Expression getPathExpression() {
+ return pathExpression;
+ }
+
+ public void setPathExpression(Expression pathExpression) {
+ this.pathExpression = pathExpression;
+ }
+
+ public List<Expression> getPartitionExpressions() {
+ return partitionExpressions;
+ }
+
+ public void setPartitionExpressions(List<Expression> partitionExpressions) {
+ this.partitionExpressions = partitionExpressions;
+ }
+
+ public Map<Integer, VariableExpr> getPartitionsVariables() {
+ return partitionsVariables;
+ }
+
+ public List<Expression> getOrderbyList() {
+ return orderbyList;
+ }
+
+ public void setOrderbyList(List<Expression> orderbyList) {
+ this.orderbyList = orderbyList;
+ }
+
+ public List<OrderbyClause.OrderModifier> getOrderbyModifiers() {
+ return orderbyModifiers;
+ }
+
+ public List<OrderbyClause.NullOrderModifier> getOrderbyNullModifierList() {
+ return orderbyNullModifierList;
+ }
+
+ public boolean hasOverClause() {
+ return !partitionExpressions.isEmpty();
+ }
+
+ public boolean hasOrderClause() {
+ return !orderbyList.isEmpty();
+ }
+
+ @Override
+ public int getVarCounter() {
+ return varCounter;
+ }
+
+ @Override
+ public void setVarCounter(int varCounter) {
+ this.varCounter = varCounter;
+ }
+
+ @Override
+ public boolean isTopLevel() {
+ return true;
+ }
+
+ @Override
+ public List<Expression> getDirectlyEnclosedExpressions() {
+ List<Expression> topLevelExpressions = new ArrayList<>();
+ topLevelExpressions.add(query.getBody());
+ topLevelExpressions.add(pathExpression);
+ topLevelExpressions.addAll(partitionExpressions);
+ topLevelExpressions.addAll(orderbyList);
+ return topLevelExpressions;
+ }
+
+ @Override
+ public Expression getBody() {
+ return query.getBody();
+ }
+
+ @Override
+ public void setBody(Expression expr) {
+ query.setBody(expr);
+ }
+}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ConfigurationUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ConfigurationUtil.java
index 4bb799b..422ffe3 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ConfigurationUtil.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ConfigurationUtil.java
@@ -24,6 +24,7 @@
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.lang.common.expression.RecordConstructor;
import org.apache.asterix.object.base.AdmObjectNode;
import org.apache.asterix.object.base.AdmStringNode;
import org.apache.asterix.object.base.IAdmNode;
@@ -33,12 +34,15 @@
private ConfigurationUtil() {
}
+ public static Map<String, String> toProperties(RecordConstructor recordConstructor) throws CompilationException {
+ return toProperties(ExpressionUtils.toNode(recordConstructor));
+ }
+
/**
* Convert the parameters object to a Map<String,String>
* This method should go away once we store the with object as it is in storage
*
- * @param parameters
- * the parameters passed for the merge policy in the with clause
+ * @param parameters the parameters passed for the merge policy in the with clause
* @return the parameters as a map
*/
public static Map<String, String> toProperties(AdmObjectNode parameters) throws CompilationException {
@@ -53,10 +57,8 @@
/**
* Get string value of {@link IAdmNode}
*
- * @param value
- * IAdmNode value should be of type integer or string
- * @return
- * string value of <code>value</code>
+ * @param value IAdmNode value should be of type integer or string
+ * @return string value of <code>value</code>
* @throws CompilationException
*/
public static String getStringValue(IAdmNode value) throws CompilationException {
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ExpressionUtils.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ExpressionUtils.java
index 925fada..3e91e22 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ExpressionUtils.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/ExpressionUtils.java
@@ -18,8 +18,6 @@
*/
package org.apache.asterix.lang.common.util;
-import static org.apache.asterix.lang.common.base.Literal.Type.DOUBLE;
-
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java
index 077748d..d91873c 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/AbstractInlineUdfsVisitor.java
@@ -52,6 +52,7 @@
import org.apache.asterix.lang.common.expression.VariableExpr;
import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
import org.apache.asterix.lang.common.rewrites.VariableSubstitutionEnvironment;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.statement.InsertStatement;
import org.apache.asterix.lang.common.statement.Query;
@@ -86,10 +87,8 @@
}
/**
- * @param letClauses
- * , a list of let-binding clauses
- * @param returnExpr
- * , a return expression
+ * @param letClauses , a list of let-binding clauses
+ * @param returnExpr , a return expression
* @return a query expression which is upto a specific langauge, e.g., FLWOGR in AQL and expression query in SQL++.
*/
protected abstract Expression generateQueryExpression(List<LetClause> letClauses, Expression returnExpr)
@@ -283,6 +282,29 @@
return changed || rewrittenBodyExpression.first;
}
+ @Override
+ public Boolean visit(CopyToStatement stmtCopy, Void arg) throws CompilationException {
+ boolean changed = false;
+
+ Pair<Boolean, Expression> queryBody = inlineUdfsAndViewsInExpr(stmtCopy.getBody());
+ changed |= queryBody.first;
+ stmtCopy.setBody(queryBody.second);
+
+ Pair<Boolean, Expression> path = inlineUdfsAndViewsInExpr(stmtCopy.getPathExpression());
+ changed |= path.first;
+ stmtCopy.setPathExpression(path.second);
+
+ Pair<Boolean, List<Expression>> part = inlineUdfsInExprList(stmtCopy.getPartitionExpressions());
+ changed |= part.first;
+ stmtCopy.setPartitionExpressions(part.second);
+
+ Pair<Boolean, List<Expression>> order = inlineUdfsInExprList(stmtCopy.getOrderbyList());
+ changed |= order.first;
+ stmtCopy.setOrderbyList(order.second);
+
+ return changed;
+ }
+
protected Pair<Boolean, Expression> inlineUdfsAndViewsInExpr(Expression expr) throws CompilationException {
if (expr.getKind() != Kind.CALL_EXPRESSION) {
boolean r = expr.accept(this, null);
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
index 8edff28..7ac9967 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
@@ -70,7 +70,8 @@
import org.apache.asterix.lang.common.statement.AnalyzeStatement;
import org.apache.asterix.lang.common.statement.CompactStatement;
import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
-import org.apache.asterix.lang.common.statement.CopyStatement;
+import org.apache.asterix.lang.common.statement.CopyFromStatement;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.CreateAdapterStatement;
import org.apache.asterix.lang.common.statement.CreateDatabaseStatement;
import org.apache.asterix.lang.common.statement.CreateDataverseStatement;
@@ -549,16 +550,66 @@
}
@Override
- public Void visit(CopyStatement stmtCopy, Integer step) throws CompilationException {
+ public Void visit(CopyFromStatement stmtCopy, Integer step) throws CompilationException {
out.print(skip(step) + "copy " + datasetSymbol
- + generateFullName(stmtCopy.getDataverseName(), stmtCopy.getDatasetName()) + " using "
- + revertStringToQuoted(stmtCopy.getExternalDetails().getAdapter()) + " ");
+ + generateFullName(stmtCopy.getDataverseName(), stmtCopy.getDatasetName()) + " from "
+ + revertStringToQuoted(stmtCopy.getExternalDetails().getAdapter()) + " with ");
printConfiguration(stmtCopy.getExternalDetails().getProperties());
out.println();
return null;
}
@Override
+ public Void visit(CopyToStatement cto, Integer step) throws CompilationException {
+ out.println(skip(step) + "copy ");
+ if (cto.getQuery() != null) {
+ out.print("(");
+ cto.getQuery().accept(this, step + 2);
+ out.print(")");
+ } else {
+ out.print(datasetSymbol + generateFullName(cto.getNamespace().getDataverseName(), cto.getDatasetName()));
+ }
+ out.print(" as ");
+ cto.getSourceVariable().accept(this, step);
+ out.println();
+
+ out.print("path (");
+ cto.getPathExpression().accept(this, step + 1);
+ out.print(")");
+
+ out.println();
+
+ if (cto.hasOverClause()) {
+ out.print("over (");
+ List<Expression> partitionExprs = cto.getPartitionExpressions();
+ Map<Integer, VariableExpr> partitionVars = cto.getPartitionsVariables();
+ out.print(skip(step + 1) + "partition ");
+ for (int i = 0; i < partitionExprs.size(); i++) {
+ if (i > 0) {
+ out.print(COMMA + " ");
+ }
+ partitionExprs.get(i).accept(this, step + 2);
+ VariableExpr partVar = partitionVars.get(i);
+ if (partVar != null) {
+ out.print(" as ");
+ partVar.accept(this, step);
+ }
+ }
+ out.println();
+ if (cto.hasOrderClause()) {
+ out.print(skip(step + 1) + "order ");
+ printDelimitedObyExpressions(cto.getOrderbyList(), cto.getOrderbyModifiers(),
+ cto.getOrderbyNullModifierList(), step + 1);
+ out.println();
+ }
+ }
+
+ out.println("with ");
+ printConfiguration(cto.getExternalDetailsDecl().getProperties());
+ return null;
+ }
+
+ @Override
public Void visit(DropDatasetStatement del, Integer step) throws CompilationException {
out.println(
skip(step) + "drop " + datasetSymbol + generateFullName(del.getDataverseName(), del.getDatasetName())
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java
index 6dcfb83..4ea7789 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/GatherFunctionCallsVisitor.java
@@ -48,6 +48,7 @@
import org.apache.asterix.lang.common.expression.TypeReferenceExpression;
import org.apache.asterix.lang.common.expression.UnaryExpr;
import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.InsertStatement;
import org.apache.asterix.lang.common.statement.Query;
import org.apache.asterix.lang.common.struct.Identifier;
@@ -247,4 +248,19 @@
}
return null;
}
+
+ @Override
+ public Void visit(CopyToStatement stmtCopy, Void arg) throws CompilationException {
+ stmtCopy.getQuery().accept(this, arg);
+ stmtCopy.getPathExpression().accept(this, arg);
+ acceptList(stmtCopy.getPartitionExpressions(), arg);
+ acceptList(stmtCopy.getOrderbyList(), arg);
+ return null;
+ }
+
+ private void acceptList(List<Expression> expressions, Void arg) throws CompilationException {
+ for (Expression expression : expressions) {
+ expression.accept(this, arg);
+ }
+ }
}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java
index 6c0eea9..0118f4c 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/AbstractQueryExpressionVisitor.java
@@ -29,7 +29,8 @@
import org.apache.asterix.lang.common.statement.AnalyzeStatement;
import org.apache.asterix.lang.common.statement.CompactStatement;
import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
-import org.apache.asterix.lang.common.statement.CopyStatement;
+import org.apache.asterix.lang.common.statement.CopyFromStatement;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.CreateAdapterStatement;
import org.apache.asterix.lang.common.statement.CreateDatabaseStatement;
import org.apache.asterix.lang.common.statement.CreateDataverseStatement;
@@ -120,7 +121,12 @@
}
@Override
- public R visit(CopyStatement stmtCopy, T arg) throws CompilationException {
+ public R visit(CopyFromStatement stmtCopy, T arg) throws CompilationException {
+ return null;
+ }
+
+ @Override
+ public R visit(CopyToStatement stmtCopy, T arg) throws CompilationException {
return null;
}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java
index 4f70cb0..1ed9b3c 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/base/ILangVisitor.java
@@ -47,7 +47,8 @@
import org.apache.asterix.lang.common.statement.AnalyzeStatement;
import org.apache.asterix.lang.common.statement.CompactStatement;
import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
-import org.apache.asterix.lang.common.statement.CopyStatement;
+import org.apache.asterix.lang.common.statement.CopyFromStatement;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.CreateAdapterStatement;
import org.apache.asterix.lang.common.statement.CreateDatabaseStatement;
import org.apache.asterix.lang.common.statement.CreateDataverseStatement;
@@ -104,7 +105,9 @@
R visit(LoadStatement stmtLoad, T arg) throws CompilationException;
- R visit(CopyStatement stmtCopy, T arg) throws CompilationException;
+ R visit(CopyFromStatement stmtCopy, T arg) throws CompilationException;
+
+ R visit(CopyToStatement cto, T arg) throws CompilationException;
R visit(DropDatasetStatement del, T arg) throws CompilationException;
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppStatementRewriter.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppStatementRewriter.java
index f9b0e7a..30b20bb 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppStatementRewriter.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppStatementRewriter.java
@@ -21,6 +21,7 @@
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.lang.common.base.IStatementRewriter;
import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppCopyToRewriteVisitor;
import org.apache.asterix.lang.sqlpp.visitor.SqlppDeleteRewriteVisitor;
import org.apache.asterix.lang.sqlpp.visitor.SqlppSynonymRewriteVisitor;
import org.apache.asterix.metadata.declared.MetadataProvider;
@@ -34,6 +35,7 @@
case INSERT:
case UPSERT:
case DELETE:
+ case COPY_TO:
return true;
default:
return false;
@@ -45,6 +47,7 @@
if (stmt != null) {
stmt.accept(SqlppSynonymRewriteVisitor.INSTANCE, metadataProvider);
stmt.accept(SqlppDeleteRewriteVisitor.INSTANCE, metadataProvider);
+ stmt.accept(SqlppCopyToRewriteVisitor.INSTANCE, metadataProvider);
}
}
}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppCopyToRewriteVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppCopyToRewriteVisitor.java
new file mode 100644
index 0000000..68a2023
--- /dev/null
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/visitor/SqlppCopyToRewriteVisitor.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.sqlpp.rewrites.visitor;
+
+import java.util.Collections;
+
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.lang.common.expression.CallExpr;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
+import org.apache.asterix.lang.common.statement.Query;
+import org.apache.asterix.lang.common.util.FunctionUtil;
+import org.apache.asterix.lang.sqlpp.clause.FromClause;
+import org.apache.asterix.lang.sqlpp.clause.FromTerm;
+import org.apache.asterix.lang.sqlpp.clause.SelectBlock;
+import org.apache.asterix.lang.sqlpp.clause.SelectClause;
+import org.apache.asterix.lang.sqlpp.clause.SelectElement;
+import org.apache.asterix.lang.sqlpp.clause.SelectSetOperation;
+import org.apache.asterix.lang.sqlpp.expression.SelectExpression;
+import org.apache.asterix.lang.sqlpp.struct.SetOperationInput;
+import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppAstVisitor;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+
+public class SqlppCopyToRewriteVisitor extends AbstractSqlppAstVisitor<Void, MetadataProvider> {
+ public static final SqlppCopyToRewriteVisitor INSTANCE = new SqlppCopyToRewriteVisitor();
+
+ @Override
+ public Void visit(CopyToStatement stmtCopy, MetadataProvider metadataProvider) throws CompilationException {
+ if (stmtCopy.getQuery() == null) {
+ setQuery(stmtCopy);
+ }
+ return null;
+ }
+
+ private void setQuery(CopyToStatement stmtCopy) {
+ DataverseName dataverseName = stmtCopy.getNamespace().getDataverseName();
+ String datasetName = stmtCopy.getDatasetName();
+ CallExpr callExpression =
+ FunctionUtil.makeDatasetCallExpr(stmtCopy.getNamespace().getDatabaseName(), dataverseName, datasetName);
+ callExpression.setSourceLocation(stmtCopy.getSourceLocation());
+
+ // From clause.
+ VariableExpr var = stmtCopy.getSourceVariable();
+ FromTerm fromTerm = new FromTerm(callExpression, var, null, null);
+ fromTerm.setSourceLocation(var.getSourceLocation());
+ FromClause fromClause = new FromClause(Collections.singletonList(fromTerm));
+ fromClause.setSourceLocation(var.getSourceLocation());
+
+ // Select clause.
+ VariableExpr returnExpr = new VariableExpr(var.getVar());
+ returnExpr.setIsNewVar(false);
+ returnExpr.setSourceLocation(var.getSourceLocation());
+ SelectElement selectElement = new SelectElement(returnExpr);
+ selectElement.setSourceLocation(stmtCopy.getSourceLocation());
+ SelectClause selectClause = new SelectClause(selectElement, null, false);
+ selectClause.setSourceLocation(stmtCopy.getSourceLocation());
+
+ // Construct the select expression.
+ SelectBlock selectBlock = new SelectBlock(selectClause, fromClause, null, null, null);
+ selectBlock.setSourceLocation(var.getSourceLocation());
+ SelectSetOperation selectSetOperation = new SelectSetOperation(new SetOperationInput(selectBlock, null), null);
+ selectSetOperation.setSourceLocation(var.getSourceLocation());
+ SelectExpression selectExpression = new SelectExpression(null, selectSetOperation, null, null, false);
+ selectExpression.setSourceLocation(var.getSourceLocation());
+ Query query = new Query(false, false, selectExpression, 0);
+ query.setBody(selectExpression);
+ query.setSourceLocation(stmtCopy.getSourceLocation());
+
+ // return the copy statement.
+ stmtCopy.setQuery(query);
+ }
+}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckDatasetOnlyResolutionVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckDatasetOnlyResolutionVisitor.java
index f700d21..ad4385c 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckDatasetOnlyResolutionVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckDatasetOnlyResolutionVisitor.java
@@ -39,6 +39,7 @@
import org.apache.asterix.lang.common.expression.RecordConstructor;
import org.apache.asterix.lang.common.expression.UnaryExpr;
import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.statement.InsertStatement;
import org.apache.asterix.lang.common.statement.Query;
@@ -106,6 +107,11 @@
}
@Override
+ public Boolean visit(CopyToStatement stmtCopy, VariableExpr arg) throws CompilationException {
+ return contains(stmtCopy.getQuery(), arg);
+ }
+
+ @Override
public Boolean visit(Query q, VariableExpr arg) throws CompilationException {
return contains(q, arg);
}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckSubqueryVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckSubqueryVisitor.java
index 399e463..387b7f7 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckSubqueryVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/CheckSubqueryVisitor.java
@@ -44,6 +44,7 @@
import org.apache.asterix.lang.common.expression.RecordConstructor;
import org.apache.asterix.lang.common.expression.UnaryExpr;
import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.statement.Query;
import org.apache.asterix.lang.common.struct.Identifier;
@@ -313,6 +314,13 @@
}
@Override
+ public Boolean visit(CopyToStatement stmtCopy, ILangExpression arg) throws CompilationException {
+ return stmtCopy.getQuery().accept(this, arg) || stmtCopy.getPathExpression().accept(this, arg)
+ || visitExprList(stmtCopy.getPartitionExpressions(), arg)
+ || visitExprList(stmtCopy.getOrderbyList(), arg);
+ }
+
+ @Override
public Boolean visit(CallExpr callExpr, ILangExpression arg) throws CompilationException {
return visitExprList(callExpr.getExprList(), arg)
|| (callExpr.hasAggregateFilterExpr() && visit(callExpr.getAggregateFilterExpr(), arg));
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java
index 8ab87e7..c1c124a 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/FreeVariableVisitor.java
@@ -47,6 +47,7 @@
import org.apache.asterix.lang.common.expression.RecordConstructor;
import org.apache.asterix.lang.common.expression.UnaryExpr;
import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.statement.Query;
import org.apache.asterix.lang.common.struct.Identifier;
@@ -512,6 +513,15 @@
return null;
}
+ @Override
+ public Void visit(CopyToStatement stmtCopy, Collection<VariableExpr> freeVars) throws CompilationException {
+ stmtCopy.getBody().accept(this, freeVars);
+ stmtCopy.getPathExpression().accept(this, freeVars);
+ visit(stmtCopy.getPartitionExpressions(), freeVars);
+ visit(stmtCopy.getOrderbyList(), freeVars);
+ return null;
+ }
+
private void visitLetWhereClauses(List<? extends AbstractClause> clauseList, Collection<VariableExpr> freeVars)
throws CompilationException {
if (clauseList == null || clauseList.isEmpty()) {
@@ -541,16 +551,12 @@
/**
* Removes all binding variables defined in the select block for a free variable collection.
- * @param selectFreeVars,
- * free variables.
- * @param fromBindingVars,
- * binding variables defined in the from clause of a select block.
- * @param letsBindingVars,
- * binding variables defined in the let clauses of the select block.
- * @param gbyBindingVars
- * binding variables defined in the groupby clauses of the select block
- * @param gbyLetsBindingVars
- * binding variables defined in the let clauses after groupby of the select block.
+ *
+ * @param selectFreeVars, free variables.
+ * @param fromBindingVars, binding variables defined in the from clause of a select block.
+ * @param letsBindingVars, binding variables defined in the let clauses of the select block.
+ * @param gbyBindingVars binding variables defined in the groupby clauses of the select block
+ * @param gbyLetsBindingVars binding variables defined in the let clauses after groupby of the select block.
*/
private void removeAllBindingVarsInSelectBlock(Collection<VariableExpr> selectFreeVars,
Collection<VariableExpr> fromBindingVars, Collection<VariableExpr> letsBindingVars,
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppContainsExpressionVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppContainsExpressionVisitor.java
index 8e8e959..368cf1d 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppContainsExpressionVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppContainsExpressionVisitor.java
@@ -43,6 +43,7 @@
import org.apache.asterix.lang.common.expression.RecordConstructor;
import org.apache.asterix.lang.common.expression.UnaryExpr;
import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.struct.Identifier;
import org.apache.asterix.lang.common.struct.QuantifiedPair;
import org.apache.asterix.lang.sqlpp.clause.AbstractBinaryCorrelateClause;
@@ -307,6 +308,13 @@
|| (callExpr.hasAggregateFilterExpr() && visit(callExpr.getAggregateFilterExpr(), arg));
}
+ @Override
+ public Boolean visit(CopyToStatement stmtCopy, T arg) throws CompilationException {
+ return stmtCopy.getQuery().accept(this, arg) || stmtCopy.getPathExpression().accept(this, arg)
+ || visitExprList(stmtCopy.getPartitionExpressions(), arg)
+ || visitExprList(stmtCopy.getOrderbyList(), arg);
+ }
+
private boolean visit(ILangExpression expr, T arg) throws CompilationException {
return expr != null && expr.accept(this, arg);
}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java
index dd48680..f2519f1 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppExpressionScopingVisitor.java
@@ -42,6 +42,7 @@
import org.apache.asterix.lang.common.expression.VariableExpr;
import org.apache.asterix.lang.common.parser.ScopeChecker;
import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.statement.InsertStatement;
import org.apache.asterix.lang.common.statement.Query;
@@ -75,10 +76,8 @@
}
/**
- * @param context,
- * manages ids of variables and guarantees uniqueness of variables.
- * @param externalVars
- * pre-defined (external) variables that must be added to the initial scope
+ * @param context, manages ids of variables and guarantees uniqueness of variables.
+ * @param externalVars pre-defined (external) variables that must be added to the initial scope
*/
public AbstractSqlppExpressionScopingVisitor(LangRewritingContext context, Collection<VarIdentifier> externalVars) {
this.context = context;
@@ -415,6 +414,33 @@
}
@Override
+ public Expression visit(CopyToStatement stmtCopy, ILangExpression arg) throws CompilationException {
+ stmtCopy.setBody(stmtCopy.getBody().accept(this, stmtCopy));
+
+ // Scope that only contains the source variable
+ Scope currentNewScope = scopeChecker.createNewScope();
+ VariableExpr sourceVarExpr = stmtCopy.getSourceVariable();
+ addNewVarSymbolToScope(currentNewScope, sourceVarExpr.getVar(), stmtCopy.getSourceLocation());
+
+ // Visit partition exprs
+ stmtCopy.setPartitionExpressions(visit(stmtCopy.getPartitionExpressions(), stmtCopy));
+
+ // Add partition variables to the current scope
+ Collection<VariableExpr> partitionVars = stmtCopy.getPartitionsVariables().values();
+ for (VariableExpr partVarExpr : partitionVars) {
+ addNewVarSymbolToScope(currentNewScope, partVarExpr.getVar(), partVarExpr.getSourceLocation());
+ }
+
+ // Visit order by
+ stmtCopy.setOrderbyList(visit(stmtCopy.getOrderbyList(), stmtCopy));
+
+ // Visit path expr
+ stmtCopy.setPathExpression(stmtCopy.getPathExpression().accept(this, stmtCopy));
+
+ return null;
+ }
+
+ @Override
public Expression visit(IVisitorExtension ve, ILangExpression arg) throws CompilationException {
return ve.variableScopeDispatch(this, arg, scopeChecker);
}
@@ -455,7 +481,7 @@
* A single name identifier is first attempted to be resolved as a variable reference. If that fails
* because there's no variable with such name then (second stage) it's resolved as a field access on a context
* variable (if there's only one context variable defined in the local scope).
- *
+ * <p>
* See {@link org.apache.asterix.lang.sqlpp.rewrites.visitor.VariableCheckAndRewriteVisitor}
*/
CONTEXT_VARIABLE
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppSimpleExpressionVisitor.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppSimpleExpressionVisitor.java
index 6bbb740..16fe67a 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppSimpleExpressionVisitor.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/visitor/base/AbstractSqlppSimpleExpressionVisitor.java
@@ -45,6 +45,7 @@
import org.apache.asterix.lang.common.expression.RecordConstructor;
import org.apache.asterix.lang.common.expression.UnaryExpr;
import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.statement.InsertStatement;
import org.apache.asterix.lang.common.statement.Query;
@@ -358,6 +359,15 @@
}
@Override
+ public Expression visit(CopyToStatement stmtCopy, ILangExpression arg) throws CompilationException {
+ stmtCopy.setBody(stmtCopy.getBody().accept(this, arg));
+ stmtCopy.setPathExpression(stmtCopy.getPathExpression().accept(this, arg));
+ stmtCopy.setPartitionExpressions(visit(stmtCopy.getPartitionExpressions(), arg));
+ stmtCopy.setOrderbyList(visit(stmtCopy.getOrderbyList(), arg));
+ return null;
+ }
+
+ @Override
public Expression visit(FieldAccessor fa, ILangExpression arg) throws CompilationException {
fa.setExpr(visit(fa.getExpr(), arg));
return fa;
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index 3675876..f2dc18e 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -166,7 +166,8 @@
import org.apache.asterix.lang.common.statement.InsertStatement;
import org.apache.asterix.lang.common.statement.InternalDetailsDecl;
import org.apache.asterix.lang.common.statement.LoadStatement;
-import org.apache.asterix.lang.common.statement.CopyStatement;
+import org.apache.asterix.lang.common.statement.CopyFromStatement;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.NodeGroupDropStatement;
import org.apache.asterix.lang.common.statement.NodegroupDecl;
import org.apache.asterix.lang.common.statement.Query;
@@ -182,6 +183,7 @@
import org.apache.asterix.lang.common.struct.OperatorType;
import org.apache.asterix.lang.common.struct.QuantifiedPair;
import org.apache.asterix.lang.common.struct.VarIdentifier;
+import org.apache.asterix.lang.common.util.ConfigurationUtil;
import org.apache.asterix.lang.common.util.DatasetDeclParametersUtil;
import org.apache.asterix.lang.common.util.ExpressionUtils;
import org.apache.asterix.lang.common.util.RangeMapBuilder;
@@ -928,6 +930,14 @@
throw new SqlppParseException(getSourceLocation(startToken), "Invalid primary key specification.");
}
}
+
+ private Map<String, String> getConfiguration(RecordConstructor withRecord) throws SqlppParseException {
+ try {
+ return ConfigurationUtil.toProperties(withRecord);
+ } catch(AlgebricksException e) {
+ throw new SqlppParseException(withRecord.getSourceLocation(), e.getMessage());
+ }
+ }
}
PARSER_END(SQLPPParser)
@@ -2848,31 +2858,47 @@
}
}
-CopyStatement CopyStatement() throws ParseException:
+Statement CopyStatement() throws ParseException:
{
- Token startToken = null;
- Namespace namespace = null;
- Identifier datasetName = null;
- TypeExpression typeExpr = null;
- boolean alreadySorted = false;
- String adapterName;
- Map<String,String> properties;
- Pair<Namespace,Identifier> nameComponents = null;
+ Token startToken = null;
+ Pair<Namespace,Identifier> nameComponents = null;
+ Query query = null;
+ Statement stmt = null;
+ TypeExpression typeExpr = null;
+ VariableExpr alias = null;
}
{
- <COPY> (<INTO>)? { startToken = token; } nameComponents = QualifiedName()
- {
- namespace = nameComponents.first;
- datasetName = nameComponents.second;
- }
- (<AS> typeExpr = DatasetTypeSpecification(RecordTypeDefinition.RecordKind.OPEN))?
- <USING> adapterName = AdapterName() properties = Configuration()
+ <COPY>
+ ( LOOKAHEAD(1) <INTO> { startToken = token; } nameComponents = QualifiedName() stmt = CopyFromStatement(startToken, nameComponents, typeExpr)
+ | LOOKAHEAD(1) <LEFTPAREN> { startToken = token; } query = Query() <RIGHTPAREN> (<AS>)? alias = Variable() stmt = CopyToStatement(startToken, nameComponents, query, alias)
+ | { startToken = token; } nameComponents = QualifiedName()
+ (<AS>)? (typeExpr = DatasetTypeSpecification(RecordTypeDefinition.RecordKind.OPEN) | alias = Variable())?
+ (stmt = CopyFromStatement(startToken, nameComponents, typeExpr) | stmt = CopyToStatement(startToken, nameComponents, query, alias))
+ )
+ {
+ return stmt;
+ }
+}
+
+CopyFromStatement CopyFromStatement(Token startToken, Pair<Namespace, Identifier> nameComponents, TypeExpression typeExpr) throws ParseException:
+{
+ Namespace namespace = nameComponents.first;
+ Identifier datasetName = nameComponents.second;
+ String adapterName;
+ RecordConstructor withRecord;
+ String sourcePath;
+}
+{
+ <FROM> adapterName = AdapterName()
+ <PATH> <LEFTPAREN> sourcePath = ConcatenatedStrings() <RIGHTPAREN>
+ <WITH> withRecord = RecordConstructor()
{
ExternalDetailsDecl edd = new ExternalDetailsDecl();
edd.setAdapter(adapterName);
- edd.setProperties(properties);
+ edd.setProperties(getConfiguration(withRecord));
+
try {
- CopyStatement stmt = new CopyStatement(namespace, datasetName.getValue(), typeExpr, edd, null);
+ CopyFromStatement stmt = new CopyFromStatement(namespace, datasetName.getValue(), sourcePath, typeExpr, edd);
return addSourceLocation(stmt, startToken);
} catch (CompilationException e){
throw new SqlppParseException(getSourceLocation(startToken), e.getMessage());
@@ -2880,6 +2906,76 @@
}
}
+CopyToStatement CopyToStatement(Token startToken, Pair<Namespace, Identifier> nameComponents, Query query, VariableExpr alias) throws ParseException:
+{
+ VariableExpr usedAlias = alias;
+ String adapterName;
+ RecordConstructor withRecord;
+ Namespace namespace = nameComponents == null ? null : nameComponents.first;
+ String datasetName = nameComponents == null ? null : nameComponents.second.getValue();
+ Expression pathExpr = null;
+
+ List<Expression> partitionExprs = new ArrayList<Expression>();
+ Map<Integer, VariableExpr> partitionVarExprs = new HashMap<Integer, VariableExpr>();
+ List<Expression> orderbyList = new ArrayList<Expression>();
+ List<OrderbyClause.OrderModifier> orderbyModifierList = new ArrayList<OrderbyClause.OrderModifier>();
+ List<OrderbyClause.NullOrderModifier> orderbyNullModifierList = new ArrayList<OrderbyClause.NullOrderModifier>();
+}
+{
+ <TO> adapterName = AdapterName()
+ <PATH> <LEFTPAREN> pathExpr = Expression() <RIGHTPAREN>
+ (CopyToOverClause(partitionExprs, partitionVarExprs, orderbyList, orderbyModifierList, orderbyNullModifierList))?
+ <WITH> withRecord = RecordConstructor()
+ {
+ ExternalDetailsDecl edd = new ExternalDetailsDecl();
+ edd.setAdapter(adapterName);
+ edd.setProperties(getConfiguration(withRecord));
+
+ if(namespace == null) {
+ namespace = defaultNamespace;
+ }
+
+ if(usedAlias == null) {
+ usedAlias = new VariableExpr(SqlppVariableUtil.toInternalVariableIdentifier(datasetName));
+ }
+
+ CopyToStatement stmt = new CopyToStatement(namespace, datasetName, query, usedAlias, edd, pathExpr,
+ partitionExprs, partitionVarExprs, orderbyList, orderbyModifierList, orderbyNullModifierList, getVarCounter());
+ return addSourceLocation(stmt, startToken);
+ }
+}
+
+ void CopyToOverClause(List<Expression> partitionExprs,
+ Map<Integer, VariableExpr> partitionVarExprs,
+ List<Expression> orderbyList,
+ List<OrderbyClause.OrderModifier> orderbyModifierList,
+ List<OrderbyClause.NullOrderModifier> orderbyNullModifierList) throws ParseException:
+{
+ Expression partitionExpr = null;
+ VariableExpr partitionVar = null;
+ OrderbyClause orderByClause = null;
+}
+{
+ <OVER> <LEFTPAREN> <IDENTIFIER> {expectToken(PARTITION);} <BY>
+ partitionExpr = Expression() { partitionExprs.add(partitionExpr);}
+ ((<AS>)? partitionVar = Variable() { partitionVarExprs.put(0, partitionVar); })?
+ (
+ <COMMA> partitionExpr = Expression() { partitionExprs.add(partitionExpr); }
+ ((<AS>)? partitionVar = Variable() { partitionVarExprs.put(partitionExprs.size() - 1, partitionVar); })?
+ )*
+ (
+ orderByClause = OrderbyClause()
+ {
+ orderbyList.addAll(orderByClause.getOrderbyList());
+ orderbyModifierList.addAll(orderByClause.getModifierList());
+ orderbyNullModifierList.addAll(orderByClause.getNullModifierList());
+ }
+ )?
+ <RIGHTPAREN>
+}
+
+
+
LoadStatement LoadStatement() throws ParseException:
{
Token startToken = null;
@@ -3120,6 +3216,19 @@
}
}
+String ConcatenatedStrings() throws ParseException:
+{
+ String stringVal;
+ StringBuilder stringBuilder = new StringBuilder();
+}
+{
+ stringVal = StringLiteral() { stringBuilder.append(stringVal); }
+ (<COMMA> stringVal = StringLiteral() { stringBuilder.append(',').append(stringVal); })*
+ {
+ return stringBuilder.toString();
+ }
+}
+
Map<String,String> Properties() throws ParseException:
{
Map<String,String> properties = new HashMap<String,String>();