[ASTERIXDB-3287][COMP] Translate and optimize COPY TO
- user model changes: yes
- storage format changes: no
- interface changes: yes
Details:
The last patch to support COPY TO includes
translating and compiling the COPY TO statement.
Change-Id: I81ce4b60a73735096ac7b0efbe70995bff7b9a10
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17895
Reviewed-by: Wail Alkowaileet <wael.y.k@gmail.com>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExpressionToPlanTranslator.java
index 0116576..21ea72d 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExpressionToPlanTranslator.java
@@ -20,6 +20,7 @@
import org.apache.asterix.lang.common.statement.Query;
import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
+import org.apache.asterix.translator.CompiledStatements.ICompiledStatement;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan;
import org.apache.hyracks.api.result.IResultMetadata;
@@ -32,32 +33,25 @@
/**
* Translate a query.
*
- * @param query,
- * the AST of a query.
- * @param outputDatasetName,
- * the output dataset name (only for insert/delete).
- * @param stmt,
- * the compiled dml statement (only for insert/delete).
- * @param resultMetadata,
- * some result metadata that can be retrieved with the result
+ * @param query, the AST of a query.
+ * @param outputDatasetName, the output dataset name (only for insert/delete).
+ * @param stmt, the compiled statement (only for insert, delete, and copy to).
+ * @param resultMetadata, some result metadata that can be retrieved with the result
* @return a logical query plan for the query.
- * @throws AlgebricksException
*/
- public ILogicalPlan translate(Query query, String outputDatasetName, ICompiledDmlStatement stmt,
+ ILogicalPlan translate(Query query, String outputDatasetName, ICompiledStatement stmt,
IResultMetadata resultMetadata) throws AlgebricksException;
/**
* Translates a load statement.
*
- * @param stmt,
- * the compiled load statement.
+ * @param stmt, the compiled load statement.
* @return a logical query plan for the Copy/Load statement.
- * @throws AlgebricksException
*/
- public ILogicalPlan translateCopyOrLoad(ICompiledDmlStatement stmt) throws AlgebricksException;
+ ILogicalPlan translateCopyOrLoad(ICompiledDmlStatement stmt) throws AlgebricksException;
/**
* @return the current minimum available variable id.
*/
- public int getVarCounter();
+ int getVarCounter();
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
index a98aacd..0a0aa5c 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java
@@ -71,6 +71,7 @@
import org.apache.asterix.optimizer.rules.LoadRecordFieldsRule;
import org.apache.asterix.optimizer.rules.MetaFunctionToMetaVariableRule;
import org.apache.asterix.optimizer.rules.NestGroupByRule;
+import org.apache.asterix.optimizer.rules.NormalizeWritingPathRule;
import org.apache.asterix.optimizer.rules.PullSelectOutOfSpatialJoin;
import org.apache.asterix.optimizer.rules.PushAggFuncIntoStandaloneAggregateRule;
import org.apache.asterix.optimizer.rules.PushAggregateIntoNestedSubplanRule;
@@ -194,6 +195,7 @@
public static List<IAlgebraicRewriteRule> buildNormalizationRuleCollection(ICcApplicationContext appCtx) {
List<IAlgebraicRewriteRule> normalization = new LinkedList<>();
normalization.add(new CheckInsertUpsertReturningRule());
+ normalization.add(new NormalizeWritingPathRule(appCtx));
normalization.add(new IntroduceUnnestForCollectionToSequenceRule());
normalization.add(new EliminateSubplanRule());
// The following rule must run before PushAggregateIntoNestedSubplanRule
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/NormalizeWritingPathRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/NormalizeWritingPathRule.java
new file mode 100644
index 0000000..4532493
--- /dev/null
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/NormalizeWritingPathRule.java
@@ -0,0 +1,195 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.optimizer.rules;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.common.dataflow.ICcApplicationContext;
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.utils.ConstantExpressionUtil;
+import org.apache.asterix.optimizer.rules.visitor.ConstantFoldingVisitor;
+import org.apache.commons.lang3.mutable.Mutable;
+import org.apache.commons.lang3.mutable.MutableObject;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression;
+import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator;
+import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalExpressionTag;
+import org.apache.hyracks.algebricks.core.algebra.base.LogicalOperatorTag;
+import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment;
+import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression;
+import org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
+import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import org.apache.hyracks.api.exceptions.SourceLocation;
+
+public class NormalizeWritingPathRule implements IAlgebraicRewriteRule {
+ private final ConstantFoldingVisitor cfv;
+ private boolean checked = false;
+
+ public NormalizeWritingPathRule(ICcApplicationContext appCtx) {
+ cfv = new ConstantFoldingVisitor(appCtx);
+ }
+
+ @Override
+ public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context)
+ throws AlgebricksException {
+ if (checked) {
+ return false;
+ }
+
+ checked = true;
+ ILogicalOperator distributeResultOp = opRef.getValue();
+ ILogicalOperator op = distributeResultOp.getInputs().get(0).getValue();
+ if (distributeResultOp.getOperatorTag() != LogicalOperatorTag.DISTRIBUTE_RESULT
+ || op.getOperatorTag() != LogicalOperatorTag.WRITE) {
+ return false;
+ }
+
+ cfv.reset(context);
+ WriteOperator writeOp = (WriteOperator) op;
+ Mutable<ILogicalExpression> pathExprRef = writeOp.getPathExpression();
+ IVariableTypeEnvironment typeEnv = context.getOutputTypeEnvironment(distributeResultOp);
+ MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider();
+ IFunctionInfo info = metadataProvider.lookupFunction(BuiltinFunctions.TO_STRING);
+ boolean changed = normalize(pathExprRef, info, typeEnv);
+
+ context.computeAndSetTypeEnvironmentForOperator(distributeResultOp);
+ return changed;
+ }
+
+ private boolean normalize(Mutable<ILogicalExpression> exprRef, IFunctionInfo toStringInfo,
+ IVariableTypeEnvironment typeEnv) throws AlgebricksException {
+ ILogicalExpression expression = exprRef.getValue();
+ if (expression.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return wrapInToStringIfNeeded(exprRef, toStringInfo, typeEnv);
+ }
+
+ AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expression;
+ if (!BuiltinFunctions.STRING_CONCAT.equals(funcExpr.getFunctionIdentifier())) {
+ return wrapInToStringIfNeeded(exprRef, toStringInfo, typeEnv);
+ }
+
+ Mutable<ILogicalExpression> concatArgRef = funcExpr.getArguments().get(0);
+ ILogicalExpression concatArg = concatArgRef.getValue();
+ if (concatArg.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) {
+ return wrapInToStringIfNeeded(concatArgRef, toStringInfo, typeEnv);
+ }
+
+ AbstractFunctionCallExpression listConst = (AbstractFunctionCallExpression) concatArg;
+ List<Mutable<ILogicalExpression>> args = listConst.getArguments();
+ List<Mutable<ILogicalExpression>> newArgs = new ArrayList<>();
+ StringBuilder builder = new StringBuilder();
+ int foldedConst = 0;
+ boolean changed = false;
+ SourceLocation constSrcLocation = null;
+ for (int i = 0; i < args.size(); i++) {
+ Mutable<ILogicalExpression> argRef = args.get(i);
+ changed |= wrapInToStringIfNeeded(argRef, toStringInfo, typeEnv);
+ ILogicalExpression argExpr = argRef.getValue();
+ ATypeTag typeTag = evaluateAndAppendString(argExpr, builder, typeEnv);
+
+ if (typeTag == ATypeTag.MISSING || typeTag == ATypeTag.NULL) {
+ // Fail early as no data will be written
+ throw new CompilationException(ErrorCode.NON_STRING_WRITE_PATH, argExpr.getSourceLocation(), typeTag);
+ } else if (typeTag == ATypeTag.STRING) {
+ if (foldedConst++ == 0) {
+ constSrcLocation = argExpr.getSourceLocation();
+ }
+ } else {
+ changed |= addFoldedArgument(foldedConst, builder, args, i, newArgs, constSrcLocation);
+ newArgs.add(argRef);
+ }
+ }
+
+ changed |= addFoldedArgument(foldedConst, builder, args, args.size(), newArgs, constSrcLocation);
+ args.clear();
+ args.addAll(newArgs);
+
+ return changed;
+ }
+
+ private boolean addFoldedArgument(int foldedConst, StringBuilder builder, List<Mutable<ILogicalExpression>> args,
+ int i, List<Mutable<ILogicalExpression>> newArgs, SourceLocation constSrcLocation) {
+ boolean changed = false;
+ if (foldedConst == 1) {
+ newArgs.add(args.get(i - 1));
+ builder.setLength(0);
+ } else if (foldedConst > 1) {
+ ILogicalExpression contExpr = ConstantExpressionUtil.create(builder.toString(), constSrcLocation);
+ newArgs.add(new MutableObject<>(contExpr));
+ builder.setLength(0);
+ changed = true;
+ }
+ return changed;
+ }
+
+ private ATypeTag evaluateAndAppendString(ILogicalExpression argExpr, StringBuilder builder,
+ IVariableTypeEnvironment typeEnv) throws AlgebricksException {
+ Pair<Boolean, ILogicalExpression> pair = argExpr.accept(cfv, null);
+ ILogicalExpression foldedExpr = pair.getSecond();
+
+ if (foldedExpr == null || foldedExpr.getExpressionTag() != LogicalExpressionTag.CONSTANT) {
+ return ATypeTag.ANY;
+ }
+
+ ATypeTag typeTag = ((IAType) typeEnv.getType(foldedExpr)).getTypeTag();
+ if (typeTag == ATypeTag.STRING) {
+ builder.append(ConstantExpressionUtil.getStringConstant(foldedExpr));
+ }
+
+ return typeTag;
+ }
+
+ private boolean wrapInToStringIfNeeded(Mutable<ILogicalExpression> exprRef, IFunctionInfo fInfo,
+ IVariableTypeEnvironment typeEnv) throws AlgebricksException {
+ ILogicalExpression expr = exprRef.getValue();
+ IAType type = (IAType) typeEnv.getType(expr);
+ if (isString(type) || isNullOrMissing(type)) {
+ return false;
+ }
+
+ ScalarFunctionCallExpression toString = new ScalarFunctionCallExpression(fInfo, new MutableObject<>(expr));
+ exprRef.setValue(toString);
+ return true;
+ }
+
+ private boolean isNullOrMissing(IAType type) {
+ ATypeTag typeTag = type.getTypeTag();
+ return typeTag == ATypeTag.NULL || typeTag == ATypeTag.MISSING;
+ }
+
+ private boolean isString(IAType type) {
+ IAType actualType = type;
+ if (actualType.getTypeTag() == ATypeTag.UNION) {
+ actualType = ((AUnionType) actualType).getActualType();
+ }
+ return actualType.getTypeTag() == ATypeTag.STRING;
+ }
+
+}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
index 3a1d9ac..54d4cfc 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
@@ -18,13 +18,17 @@
*/
package org.apache.asterix.translator;
+import java.util.List;
import java.util.Map;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.external.feed.management.FeedConnectionRequest;
import org.apache.asterix.lang.common.base.Expression;
import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.lang.common.clause.OrderbyClause;
import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
+import org.apache.asterix.lang.common.statement.ExternalDetailsDecl;
import org.apache.asterix.lang.common.statement.Query;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Datatype;
@@ -586,4 +590,85 @@
return index;
}
}
+
+ public static class CompiledCopyToStatement extends AbstractCompiledStatement {
+ private final Query query;
+ private final VariableExpr sourceVariable;
+ private final String adapter;
+ private final Map<String, String> properties;
+ private final List<Expression> pathExpressions;
+ private final List<Expression> partitionExpressions;
+ private final Map<Integer, VariableExpr> partitionsVariables;
+ private final List<Expression> orderbyList;
+ private final List<OrderbyClause.OrderModifier> orderbyModifiers;
+ private final List<OrderbyClause.NullOrderModifier> orderbyNullModifierList;
+
+ public CompiledCopyToStatement(CopyToStatement copyToStatement) {
+ this.query = copyToStatement.getQuery();
+ this.sourceVariable = copyToStatement.getSourceVariable();
+ ExternalDetailsDecl eddDecl = copyToStatement.getExternalDetailsDecl();
+ this.adapter = eddDecl.getAdapter();
+ this.properties = eddDecl.getProperties();
+ this.pathExpressions = copyToStatement.getPathExpressions();
+ this.partitionExpressions = copyToStatement.getPartitionExpressions();
+ this.partitionsVariables = copyToStatement.getPartitionsVariables();
+ this.orderbyList = copyToStatement.getOrderbyList();
+ this.orderbyModifiers = copyToStatement.getOrderbyModifiers();
+ this.orderbyNullModifierList = copyToStatement.getOrderbyNullModifierList();
+ }
+
+ @Override
+ public Statement.Kind getKind() {
+ return Statement.Kind.COPY_TO;
+ }
+
+ public Query getQuery() {
+ return query;
+ }
+
+ public VariableExpr getSourceVariable() {
+ return sourceVariable;
+ }
+
+ public String getAdapter() {
+ return adapter;
+ }
+
+ public Map<String, String> getProperties() {
+ return properties;
+ }
+
+ public List<Expression> getPathExpressions() {
+ return pathExpressions;
+ }
+
+ public boolean isPartitioned() {
+ return !partitionExpressions.isEmpty();
+ }
+
+ public boolean isOrdered() {
+ return !orderbyList.isEmpty();
+ }
+
+ public List<Expression> getPartitionExpressions() {
+ return partitionExpressions;
+ }
+
+ public VariableExpr getPartitionsVariables(int index) {
+ return partitionsVariables.get(index);
+ }
+
+ public List<Expression> getOrderbyExpressions() {
+ return orderbyList;
+ }
+
+ public List<OrderbyClause.OrderModifier> getOrderbyModifiers() {
+ return orderbyModifiers;
+ }
+
+ public List<OrderbyClause.NullOrderModifier> getOrderbyNullModifiers() {
+ return orderbyNullModifierList;
+ }
+ }
+
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index e482144..4829d0b 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -83,10 +83,12 @@
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.asterix.metadata.declared.ResultSetDataSink;
import org.apache.asterix.metadata.declared.ResultSetSinkId;
+import org.apache.asterix.metadata.declared.WriteDataSink;
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Function;
import org.apache.asterix.metadata.entities.InternalDatasetDetails;
import org.apache.asterix.metadata.functions.ExternalFunctionCompilerUtil;
+import org.apache.asterix.metadata.provider.ExternalWriterProvider;
import org.apache.asterix.metadata.utils.DatasetUtil;
import org.apache.asterix.om.base.ABoolean;
import org.apache.asterix.om.base.AInt32;
@@ -99,6 +101,7 @@
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.utils.ConstantExpressionUtil;
import org.apache.asterix.translator.CompiledStatements.CompiledCopyFromFileStatement;
import org.apache.asterix.translator.CompiledStatements.CompiledInsertStatement;
import org.apache.asterix.translator.CompiledStatements.CompiledLoadFromFileStatement;
@@ -148,6 +151,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SubplanOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnionAllOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator;
+import org.apache.hyracks.algebricks.core.algebra.operators.logical.WriteOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.LogicalOperatorDeepCopyWithNewVariablesVisitor;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
import org.apache.hyracks.algebricks.core.algebra.plan.ALogicalPlanImpl;
@@ -195,6 +199,7 @@
return context.getVarCounter();
}
+ @Override
public ILogicalPlan translateCopyOrLoad(ICompiledDmlStatement stmt) throws AlgebricksException {
SourceLocation sourceLoc = stmt.getSourceLocation();
Dataset dataset =
@@ -345,9 +350,111 @@
}
@Override
- public ILogicalPlan translate(Query expr, String outputDatasetName, ICompiledDmlStatement stmt,
+ public ILogicalPlan translate(Query expr, String outputDatasetName, CompiledStatements.ICompiledStatement stmt,
IResultMetadata resultMetadata) throws AlgebricksException {
- return translate(expr, outputDatasetName, stmt, null, resultMetadata);
+ if (stmt != null && stmt.getKind() == Statement.Kind.COPY_TO) {
+ return translateCopyTo(expr, stmt, resultMetadata);
+ }
+ return translate(expr, outputDatasetName, (ICompiledDmlStatement) stmt, null, resultMetadata);
+ }
+
+ private ILogicalPlan translateCopyTo(Query expr, CompiledStatements.ICompiledStatement stmt,
+ IResultMetadata resultMetadata) throws AlgebricksException {
+ CompiledStatements.CompiledCopyToStatement copyTo = (CompiledStatements.CompiledCopyToStatement) stmt;
+ MutableObject<ILogicalOperator> base = new MutableObject<>(new EmptyTupleSourceOperator());
+ Pair<ILogicalOperator, LogicalVariable> p = expr.accept(this, base);
+ ArrayList<Mutable<ILogicalOperator>> globalPlanRoots = new ArrayList<>();
+ ILogicalOperator topOp = p.first;
+ List<LogicalVariable> liveVars = new ArrayList<>();
+ VariableUtilities.getLiveVariables(topOp, liveVars);
+ LogicalVariable resVar = liveVars.get(0);
+ Mutable<ILogicalOperator> topOpRef = new MutableObject<>(topOp);
+
+ // First, set source variable so the following operations has the source variable visible
+ context.setVar(copyTo.getSourceVariable(), resVar);
+ VariableReferenceExpression sourceExpr = new VariableReferenceExpression(resVar);
+ Mutable<ILogicalExpression> sourceExprRef = new MutableObject<>(sourceExpr);
+
+ // Set partition expression(s) if any. Prepare partition variables to be visible to path expression
+ List<Mutable<ILogicalExpression>> partitionExpressionRefs = new ArrayList<>();
+ if (copyTo.isPartitioned()) {
+ List<Expression> astPartitionExpressions = copyTo.getPartitionExpressions();
+ for (int i = 0; i < astPartitionExpressions.size(); i++) {
+ Expression expression = astPartitionExpressions.get(i);
+ Pair<ILogicalExpression, Mutable<ILogicalOperator>> partExprPair =
+ langExprToAlgExpression(expression, topOpRef);
+ LogicalVariable partVar = getVariable(copyTo.getPartitionsVariables(i));
+ Pair<Mutable<ILogicalExpression>, Mutable<ILogicalOperator>> wrappedPair =
+ wrapInAssign(partVar, partExprPair.first, topOpRef);
+ partitionExpressionRefs.add(wrappedPair.first);
+ topOpRef = wrappedPair.second;
+ }
+ }
+
+ // Set order expression(s) if any. We do order first to prevent partition variables to be used
+ List<Pair<OrderOperator.IOrder, Mutable<ILogicalExpression>>> orderExprListOut = new ArrayList<>();
+ List<Expression> orderExprList = copyTo.getOrderbyExpressions();
+ List<OrderbyClause.OrderModifier> orderModifierList = copyTo.getOrderbyModifiers();
+ List<OrderbyClause.NullOrderModifier> nullOrderModifierList = copyTo.getOrderbyNullModifiers();
+ for (int i = 0; i < orderExprList.size(); i++) {
+ Expression orderExpr = orderExprList.get(i);
+ OrderbyClause.OrderModifier orderModifier = orderModifierList.get(i);
+ OrderbyClause.NullOrderModifier nullOrderModifier = nullOrderModifierList.get(i);
+ Pair<ILogicalExpression, Mutable<ILogicalOperator>> orderExprResult =
+ langExprToAlgExpression(orderExpr, topOpRef);
+ Pair<Mutable<ILogicalExpression>, Mutable<ILogicalOperator>> wrappedPair =
+ wrapInAssign(context.newVar(), orderExprResult.first, orderExprResult.second);
+ addOrderByExpression(orderExprListOut, wrappedPair.first.getValue(), orderModifier, nullOrderModifier);
+ topOpRef = wrappedPair.second;
+ }
+
+ // Set Path
+ // astPathExpressions has at least one expression see CopyToStatement constructor
+ List<Expression> astPathExpressions = copyTo.getPathExpressions();
+ ILogicalExpression fullPathExpr = null;
+ WriteDataSink writeDataSink;
+ String separator = String.valueOf(ExternalWriterProvider.getSeparator(copyTo.getAdapter()));
+ List<Mutable<ILogicalExpression>> pathExprs = new ArrayList<>(astPathExpressions.size());
+ Pair<ILogicalExpression, Mutable<ILogicalOperator>> pathExprPair;
+ for (int i = 0; i < astPathExpressions.size(); i++) {
+ Expression pathExpr = astPathExpressions.get(i);
+ if (i > 0) {
+ // Add separator
+ ILogicalExpression algExpr = ConstantExpressionUtil.create(separator, pathExpr.getSourceLocation());
+ pathExprs.add(new MutableObject<>(algExpr));
+ }
+ pathExprPair = langExprToAlgExpression(pathExpr, topOpRef);
+ pathExprs.add(new MutableObject<>(pathExprPair.first));
+ topOpRef = pathExprPair.second;
+ SourceLocation srcLoc = astPathExpressions.get(0).getSourceLocation();
+
+ // concat arg
+ IFunctionInfo arrayConInfo = metadataProvider.lookupFunction(BuiltinFunctions.ORDERED_LIST_CONSTRUCTOR);
+ ScalarFunctionCallExpression arrayCon = new ScalarFunctionCallExpression(arrayConInfo, pathExprs);
+ arrayCon.setSourceLocation(srcLoc);
+ Mutable<ILogicalExpression> arrayContRef = new MutableObject<>(arrayCon);
+
+ // concat
+ IFunctionInfo concatInfo = metadataProvider.lookupFunction(BuiltinFunctions.STRING_CONCAT);
+ ScalarFunctionCallExpression concat = new ScalarFunctionCallExpression(concatInfo, arrayContRef);
+ concat.setSourceLocation(srcLoc);
+
+ fullPathExpr = concat;
+ }
+
+ writeDataSink = new WriteDataSink(copyTo.getAdapter(), copyTo.getProperties());
+ // writeOperator
+ WriteOperator writeOperator = new WriteOperator(sourceExprRef, new MutableObject<>(fullPathExpr),
+ partitionExpressionRefs, orderExprListOut, writeDataSink);
+ writeOperator.getInputs().add(topOpRef);
+
+ ResultSetSinkId rssId = new ResultSetSinkId(metadataProvider.getResultSetId());
+ ResultSetDataSink sink = new ResultSetDataSink(rssId, null);
+ DistributeResultOperator newTop = new DistributeResultOperator(new ArrayList<>(), sink, resultMetadata);
+ newTop.getInputs().add(new MutableObject<>(writeOperator));
+
+ globalPlanRoots.add(new MutableObject<>(newTop));
+ return new ALogicalPlanImpl(globalPlanRoots);
}
public ILogicalPlan translate(Query expr, String outputDatasetName, ICompiledDmlStatement stmt,
@@ -2234,4 +2341,29 @@
constExpr.setSourceLocation(sourceLoc);
return constExpr;
}
+
+ private LogicalVariable getVariable(VariableExpr variableExpr) {
+ LogicalVariable variable;
+ if (variableExpr != null) {
+ variable = context.newVar(variableExpr.getVar().getValue());
+ context.setVar(variableExpr, variable);
+ } else {
+ variable = context.newVar();
+ }
+
+ return variable;
+ }
+
+ private Pair<Mutable<ILogicalExpression>, Mutable<ILogicalOperator>> wrapInAssign(LogicalVariable variable,
+ ILogicalExpression expression, Mutable<ILogicalOperator> topOpRef) {
+ AssignOperator assignOperator = new AssignOperator(variable, new MutableObject<>(expression));
+ assignOperator.getInputs().add(topOpRef);
+
+ Pair<Mutable<ILogicalExpression>, Mutable<ILogicalOperator>> pair = new Pair<>(null, null);
+ VariableReferenceExpression partitionExpr = new VariableReferenceExpression(variable);
+
+ pair.first = new MutableObject<>(partitionExpr);
+ pair.second = new MutableObject<>(assignOperator);
+ return pair;
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
index 24db0c0..716d36e 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java
@@ -70,6 +70,7 @@
import org.apache.asterix.optimizer.base.AsterixOptimizationContext;
import org.apache.asterix.runtime.job.listener.JobEventListenerFactory;
import org.apache.asterix.translator.CompiledStatements.ICompiledDmlStatement;
+import org.apache.asterix.translator.CompiledStatements.ICompiledStatement;
import org.apache.asterix.translator.ExecutionPlans;
import org.apache.asterix.translator.IRequestParameters;
import org.apache.asterix.translator.ResultMetadata;
@@ -202,10 +203,9 @@
}
public JobSpecification compileQuery(IClusterInfoCollector clusterInfoCollector, MetadataProvider metadataProvider,
- Query query, int varCounter, String outputDatasetName, SessionOutput output,
- ICompiledDmlStatement statement, Map<VarIdentifier, IAObject> externalVars, IResponsePrinter printer,
- IWarningCollector warningCollector, IRequestParameters requestParameters)
- throws AlgebricksException, ACIDException {
+ Query query, int varCounter, String outputDatasetName, SessionOutput output, ICompiledStatement statement,
+ Map<VarIdentifier, IAObject> externalVars, IResponsePrinter printer, IWarningCollector warningCollector,
+ IRequestParameters requestParameters) throws AlgebricksException, ACIDException {
// establish facts
final boolean isQuery = query != null;
@@ -226,7 +226,7 @@
ILangExpressionToPlanTranslator t =
translatorFactory.createExpressionToPlanTranslator(metadataProvider, varCounter, externalVars);
ResultMetadata resultMetadata = new ResultMetadata(output.config().fmt());
- ILogicalPlan plan = isLoad || isCopy ? t.translateCopyOrLoad(statement)
+ ILogicalPlan plan = isLoad || isCopy ? t.translateCopyOrLoad((ICompiledDmlStatement) statement)
: t.translate(query, outputDatasetName, statement, resultMetadata);
ICcApplicationContext ccAppContext = metadataProvider.getApplicationContext();
@@ -401,8 +401,9 @@
: PlanPrettyPrinter.createStringPlanPrettyPrinter();
}
- private byte getStatementCategory(Query query, ICompiledDmlStatement statement) {
- return statement != null ? statement.getCategory()
+ private byte getStatementCategory(Query query, ICompiledStatement statement) {
+ return statement != null && statement.getKind() != Statement.Kind.COPY_TO
+ ? ((ICompiledDmlStatement) statement).getCategory()
: query != null ? Statement.Category.QUERY : Statement.Category.DDL;
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index ed6d529..f0cc75b 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -105,6 +105,7 @@
import org.apache.asterix.external.operators.FeedIntakeOperatorNodePushable;
import org.apache.asterix.external.util.ExternalDataConstants;
import org.apache.asterix.external.util.ExternalDataUtils;
+import org.apache.asterix.external.util.WriterValidationUtil;
import org.apache.asterix.lang.common.base.IQueryRewriter;
import org.apache.asterix.lang.common.base.IReturningStatement;
import org.apache.asterix.lang.common.base.IRewriterFactory;
@@ -121,6 +122,7 @@
import org.apache.asterix.lang.common.statement.CompactStatement;
import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
import org.apache.asterix.lang.common.statement.CopyFromStatement;
+import org.apache.asterix.lang.common.statement.CopyToStatement;
import org.apache.asterix.lang.common.statement.CreateAdapterStatement;
import org.apache.asterix.lang.common.statement.CreateDatabaseStatement;
import org.apache.asterix.lang.common.statement.CreateDataverseStatement;
@@ -220,6 +222,7 @@
import org.apache.asterix.transaction.management.service.transaction.GlobalTxInfo;
import org.apache.asterix.translator.AbstractLangTranslator;
import org.apache.asterix.translator.ClientRequest;
+import org.apache.asterix.translator.CompiledStatements;
import org.apache.asterix.translator.CompiledStatements.CompiledCopyFromFileStatement;
import org.apache.asterix.translator.CompiledStatements.CompiledDeleteStatement;
import org.apache.asterix.translator.CompiledStatements.CompiledInsertStatement;
@@ -465,6 +468,16 @@
}
handleCopyFromStatement(metadataProvider, stmt, hcc);
break;
+ case COPY_TO:
+ metadataProvider.setResultSetId(new ResultSetId(resultSetIdCounter.getAndInc()));
+ // The result should to be read just once
+ metadataProvider.setMaxResultReads(1);
+ if (stats.getProfileType() == Stats.ProfileType.FULL) {
+ this.jobFlags.add(JobFlag.PROFILE_RUNTIME);
+ }
+ handleCopyToStatement(metadataProvider, stmt, hcc, resultSet, resultDelivery, outMetadata,
+ requestParameters, stmtParams, stats);
+ break;
case INSERT:
case UPSERT:
if (((InsertStatement) stmt).getReturnExpression() != null) {
@@ -3945,6 +3958,76 @@
}
}
+ protected void handleCopyToStatement(MetadataProvider metadataProvider, Statement stmt,
+ IHyracksClientConnection hcc, IResultSet resultSet, ResultDelivery resultDelivery,
+ ResultMetadata outMetadata, IRequestParameters requestParameters, Map<String, IAObject> stmtParams,
+ Stats stats) throws Exception {
+ CopyToStatement copyTo = (CopyToStatement) stmt;
+ final IRequestTracker requestTracker = appCtx.getRequestTracker();
+ final ClientRequest clientRequest =
+ (ClientRequest) requestTracker.get(requestParameters.getRequestReference().getUuid());
+ final IMetadataLocker locker = new IMetadataLocker() {
+ @Override
+ public void lock() throws RuntimeDataException, InterruptedException {
+ try {
+ compilationLock.readLock().lockInterruptibly();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ ensureNotCancelled(clientRequest);
+ throw e;
+ }
+ }
+
+ @Override
+ public void unlock() {
+ metadataProvider.getLocks().unlock();
+ compilationLock.readLock().unlock();
+ }
+ };
+ final IStatementCompiler compiler = () -> {
+ long compileStart = System.nanoTime();
+ MetadataTransactionContext mdTxnCtx = MetadataManager.INSTANCE.beginTransaction();
+ boolean bActiveTxn = true;
+ metadataProvider.setMetadataTxnContext(mdTxnCtx);
+ try {
+ ExternalDetailsDecl edd = copyTo.getExternalDetailsDecl();
+ edd.setProperties(createAndValidateAdapterConfigurationForCopyToStmt(edd,
+ ExternalDataConstants.WRITER_SUPPORTED_ADAPTERS, copyTo.getSourceLocation(), mdTxnCtx));
+
+ Map<VarIdentifier, IAObject> externalVars = createExternalVariables(copyTo, stmtParams);
+ // Query Rewriting (happens under the same ongoing metadata transaction)
+ LangRewritingContext langRewritingContext = createLangRewritingContext(metadataProvider,
+ declaredFunctions, null, warningCollector, copyTo.getVarCounter());
+ Pair<IReturningStatement, Integer> rewrittenResult = apiFramework.reWriteQuery(langRewritingContext,
+ copyTo, sessionOutput, true, true, externalVars.keySet());
+
+ CompiledStatements.CompiledCopyToStatement compiledCopyToStatement =
+ new CompiledStatements.CompiledCopyToStatement(copyTo);
+
+ // Query Compilation (happens under the same ongoing metadata transaction)
+ final JobSpecification jobSpec = apiFramework.compileQuery(hcc, metadataProvider, copyTo.getQuery(),
+ rewrittenResult.second, null, sessionOutput, compiledCopyToStatement, externalVars,
+ responsePrinter, warningCollector, requestParameters);
+ // update stats with count of compile-time warnings. needs to be adapted for multi-statement.
+ stats.updateTotalWarningsCount(warningCollector.getTotalWarningsCount());
+ afterCompile();
+ MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
+ stats.setCompileTime(System.nanoTime() - compileStart);
+ bActiveTxn = false;
+ return isCompileOnly() ? null : jobSpec;
+ } catch (Exception e) {
+ LOGGER.log(Level.INFO, e.getMessage(), e);
+ if (bActiveTxn) {
+ abort(e, e, mdTxnCtx);
+ }
+ throw e;
+ }
+ };
+
+ deliverResult(hcc, resultSet, compiler, metadataProvider, locker, resultDelivery, outMetadata, stats,
+ requestParameters, true, null);
+ }
+
public JobSpecification handleInsertUpsertStatement(MetadataProvider metadataProvider, Statement stmt,
IHyracksClientConnection hcc, IResultSet resultSet, ResultDelivery resultDelivery,
ResultMetadata outMetadata, Stats stats, IRequestParameters requestParameters,
@@ -5501,6 +5584,15 @@
validateAdapterSpecificProperties(details, srcLoc, appCtx);
}
+ protected Map<String, String> createAndValidateAdapterConfigurationForCopyToStmt(
+ ExternalDetailsDecl externalDetailsDecl, Set<String> supportedAdapters, SourceLocation sourceLocation,
+ MetadataTransactionContext mdTxnCtx) throws AlgebricksException {
+ String adapterName = externalDetailsDecl.getAdapter();
+ Map<String, String> properties = externalDetailsDecl.getProperties();
+ WriterValidationUtil.validateWriterConfiguration(adapterName, supportedAdapters, properties, sourceLocation);
+ return properties;
+ }
+
/**
* Ensures that the external source container is present
*
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/early-missing/early-missing.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/early-missing/early-missing.01.ddl.sqlpp
new file mode 100644
index 0000000..b68c38b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/early-missing/early-missing.01.ddl.sqlpp
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+CREATE TYPE OpenType AS {
+};
+
+CREATE EXTERNAL DATASET Customer(OpenType) USING S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-2"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
+ ("container"="playground"),
+ ("definition"="external-filter/car/{company:string}/customer/{customer_id:int}"),
+ ("embed-filter-values" = "false"),
+ ("format"="json")
+);
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/early-missing/early-missing.02.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/early-missing/early-missing.02.update.sqlpp
new file mode 100644
index 0000000..434801b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/early-missing/early-missing.02.update.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+COPY Customer c
+TO S3
+PATH (MISSING)
+WITH {
+ "accessKeyId":"dummyAccessKey",
+ "secretAccessKey":"dummySecretKey",
+ "region":"us-west-2",
+ "serviceEndpoint":"http://127.0.0.1:8001",
+ "container":"playground",
+ "format":"json"
+}
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/early-missing/early-missing.03.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/early-missing/early-missing.03.update.sqlpp
new file mode 100644
index 0000000..d02486b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/early-missing/early-missing.03.update.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+COPY Customer c
+TO S3
+PATH (1 + "hello world")
+WITH {
+ "accessKeyId":"dummyAccessKey",
+ "secretAccessKey":"dummySecretKey",
+ "region":"us-west-2",
+ "serviceEndpoint":"http://127.0.0.1:8001",
+ "container":"playground",
+ "format":"json"
+}
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/long-path/long-path.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/long-path/long-path.01.ddl.sqlpp
new file mode 100644
index 0000000..b68c38b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/long-path/long-path.01.ddl.sqlpp
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+CREATE TYPE OpenType AS {
+};
+
+CREATE EXTERNAL DATASET Customer(OpenType) USING S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-2"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
+ ("container"="playground"),
+ ("definition"="external-filter/car/{company:string}/customer/{customer_id:int}"),
+ ("embed-filter-values" = "false"),
+ ("format"="json")
+);
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/long-path/long-path.02.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/long-path/long-path.02.update.sqlpp
new file mode 100644
index 0000000..ef35a36
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/long-path/long-path.02.update.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+COPY Customer c
+TO S3
+PATH ("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
+WITH {
+ "accessKeyId":"dummyAccessKey",
+ "secretAccessKey":"dummySecretKey",
+ "region":"us-west-2",
+ "serviceEndpoint":"http://127.0.0.1:8001",
+ "container":"playground",
+ "format":"json"
+}
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/long-path/long-path.03.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/long-path/long-path.03.query.sqlpp
new file mode 100644
index 0000000..e5cc591
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/long-path/long-path.03.query.sqlpp
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+// param max-warnings:json=1
+USE test;
+
+COPY (
+ SELECT VALUE c
+ FROM Customer c
+ -- Minimize the number of warnings
+ WHERE c.company = "ford"
+) toWrite
+TO S3
+PATH (company, "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")
+OVER (
+ PARTITION BY toWrite.company company
+)
+WITH {
+ "accessKeyId":"dummyAccessKey",
+ "secretAccessKey":"dummySecretKey",
+ "region":"us-west-2",
+ "serviceEndpoint":"http://127.0.0.1:8001",
+ "container":"playground",
+ "format":"json"
+}
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/non-empty-folder/non-empty-folder.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/non-empty-folder/non-empty-folder.01.ddl.sqlpp
new file mode 100644
index 0000000..b68c38b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/non-empty-folder/non-empty-folder.01.ddl.sqlpp
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+CREATE TYPE OpenType AS {
+};
+
+CREATE EXTERNAL DATASET Customer(OpenType) USING S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-2"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
+ ("container"="playground"),
+ ("definition"="external-filter/car/{company:string}/customer/{customer_id:int}"),
+ ("embed-filter-values" = "false"),
+ ("format"="json")
+);
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/non-empty-folder/non-empty-folder.02.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/non-empty-folder/non-empty-folder.02.update.sqlpp
new file mode 100644
index 0000000..a0e04db
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/non-empty-folder/non-empty-folder.02.update.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+COPY Customer c
+TO S3
+PATH ("copy-to-result", "duplicate-write")
+WITH {
+ "accessKeyId":"dummyAccessKey",
+ "secretAccessKey":"dummySecretKey",
+ "region":"us-west-2",
+ "serviceEndpoint":"http://127.0.0.1:8001",
+ "container":"playground",
+ "format":"json"
+}
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/non-empty-folder/non-empty-folder.03.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/non-empty-folder/non-empty-folder.03.update.sqlpp
new file mode 100644
index 0000000..a0e04db
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/non-empty-folder/non-empty-folder.03.update.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+COPY Customer c
+TO S3
+PATH ("copy-to-result", "duplicate-write")
+WITH {
+ "accessKeyId":"dummyAccessKey",
+ "secretAccessKey":"dummySecretKey",
+ "region":"us-west-2",
+ "serviceEndpoint":"http://127.0.0.1:8001",
+ "container":"playground",
+ "format":"json"
+}
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.01.ddl.sqlpp
new file mode 100644
index 0000000..b68c38b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.01.ddl.sqlpp
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+CREATE TYPE OpenType AS {
+};
+
+CREATE EXTERNAL DATASET Customer(OpenType) USING S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-2"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
+ ("container"="playground"),
+ ("definition"="external-filter/car/{company:string}/customer/{customer_id:int}"),
+ ("embed-filter-values" = "false"),
+ ("format"="json")
+);
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.02.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.02.query.sqlpp
new file mode 100644
index 0000000..158ea04
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.02.query.sqlpp
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+// param max-warnings:json=1
+USE test;
+
+COPY Customer c
+TO S3
+PATH ("copy-to-result", myMissingValue)
+OVER (
+ PARTITION BY c.doesNotExist myMissingValue
+)
+WITH {
+ "accessKeyId":"dummyAccessKey",
+ "secretAccessKey":"dummySecretKey",
+ "region":"us-west-2",
+ "serviceEndpoint":"http://127.0.0.1:8001",
+ "container":"playground",
+ "format":"json",
+ "compression":"gzip"
+}
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.03.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.03.query.sqlpp
new file mode 100644
index 0000000..0dfb3d3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.03.query.sqlpp
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+// param max-warnings:json=1
+USE test;
+
+COPY (
+ SELECT DISTINCT (CASE WHEN c.company = "ford"
+ THEN MISSING
+ ELSE c.company
+ END) company,
+ c.year
+ FROM Customer c
+) toWrite
+TO S3
+PATH ("copy-to-result/someMissing", company)
+OVER (
+ PARTITION BY toWrite.company company
+)
+WITH {
+ "accessKeyId":"dummyAccessKey",
+ "secretAccessKey":"dummySecretKey",
+ "region":"us-west-2",
+ "serviceEndpoint":"http://127.0.0.1:8001",
+ "container":"playground",
+ "format":"json"
+}
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.04.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.04.ddl.sqlpp
new file mode 100644
index 0000000..5277555
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.04.ddl.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+CREATE EXTERNAL DATASET CustomerCopy(OpenType) USING S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-2"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
+ ("container"="playground"),
+ ("definition"="copy-to-result/someMissing/{company:string}"),
+ ("embed-filter-values" = "false"),
+ ("format"="json")
+);
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.05.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.05.query.sqlpp
new file mode 100644
index 0000000..e4f8eac
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/runtime-missing/runtime-missing.05.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+USE test;
+
+SELECT VALUE c
+FROM CustomerCopy c
+ORDER BY c.company,
+ c.year
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/supported-adapter-format-compression/supported-adapters.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/supported-adapter-format-compression/supported-adapters.01.ddl.sqlpp
new file mode 100644
index 0000000..b68c38b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/supported-adapter-format-compression/supported-adapters.01.ddl.sqlpp
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+CREATE TYPE OpenType AS {
+};
+
+CREATE EXTERNAL DATASET Customer(OpenType) USING S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-2"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
+ ("container"="playground"),
+ ("definition"="external-filter/car/{company:string}/customer/{customer_id:int}"),
+ ("embed-filter-values" = "false"),
+ ("format"="json")
+);
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/supported-adapter-format-compression/supported-adapters.02.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/supported-adapter-format-compression/supported-adapters.02.update.sqlpp
new file mode 100644
index 0000000..b016330
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/supported-adapter-format-compression/supported-adapters.02.update.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+COPY Customer c
+TO AZUREBLOB
+PATH ("copy-to-result")
+WITH {
+ "accessKeyId":"dummyAccessKey",
+ "secretAccessKey":"dummySecretKey",
+ "region":"us-west-2",
+ "serviceEndpoint":"http://127.0.0.1:8001",
+ "container":"playground",
+ "format":"json"
+}
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/supported-adapter-format-compression/supported-adapters.03.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/supported-adapter-format-compression/supported-adapters.03.update.sqlpp
new file mode 100644
index 0000000..2e227d6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/supported-adapter-format-compression/supported-adapters.03.update.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+COPY Customer c
+TO S3
+PATH ("copy-to-result")
+WITH {
+ "accessKeyId":"dummyAccessKey",
+ "secretAccessKey":"dummySecretKey",
+ "region":"us-west-2",
+ "serviceEndpoint":"http://127.0.0.1:8001",
+ "container":"playground",
+ "format":"csv"
+}
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/supported-adapter-format-compression/supported-adapters.04.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/supported-adapter-format-compression/supported-adapters.04.update.sqlpp
new file mode 100644
index 0000000..38934b9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/negative/supported-adapter-format-compression/supported-adapters.04.update.sqlpp
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+COPY Customer c
+TO S3
+PATH ("copy-to-result")
+WITH {
+ "accessKeyId":"dummyAccessKey",
+ "secretAccessKey":"dummySecretKey",
+ "region":"us-west-2",
+ "serviceEndpoint":"http://127.0.0.1:8001",
+ "container":"playground",
+ "format":"json",
+ "compression": "rar"
+}
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/partition/partition.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/partition/partition.01.ddl.sqlpp
new file mode 100644
index 0000000..b68c38b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/partition/partition.01.ddl.sqlpp
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+CREATE TYPE OpenType AS {
+};
+
+CREATE EXTERNAL DATASET Customer(OpenType) USING S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-2"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
+ ("container"="playground"),
+ ("definition"="external-filter/car/{company:string}/customer/{customer_id:int}"),
+ ("embed-filter-values" = "false"),
+ ("format"="json")
+);
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/partition/partition.02.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/partition/partition.02.update.sqlpp
new file mode 100644
index 0000000..f1a22d0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/partition/partition.02.update.sqlpp
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+COPY Customer c
+TO S3
+PATH ("copy-to-result", "car", company, "customer", customer_id)
+OVER (
+ PARTITION BY c.company company,
+ c.customer_id customer_id
+)
+WITH {
+ "accessKeyId":"dummyAccessKey",
+ "secretAccessKey":"dummySecretKey",
+ "region":"us-west-2",
+ "serviceEndpoint":"http://127.0.0.1:8001",
+ "container":"playground",
+ "format":"json",
+ "compression":"gzip"
+}
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/partition/partition.03.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/partition/partition.03.ddl.sqlpp
new file mode 100644
index 0000000..14d1d92
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/partition/partition.03.ddl.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+CREATE EXTERNAL DATASET CustomerCopy(OpenType) USING S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-2"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
+ ("container"="playground"),
+ ("definition"="copy-to-result/car/{company:string}/customer/{customer_id:int}"),
+ ("embed-filter-values" = "false"),
+ ("format"="json")
+);
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/partition/partition.04.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/partition/partition.04.query.sqlpp
new file mode 100644
index 0000000..eace40c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/partition/partition.04.query.sqlpp
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SELECT VALUE c
+FROM Customer c
+ORDER BY c.customer_id,
+ c.company,
+ c.year,
+ c.month,
+ c.day;
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/partition/partition.05.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/partition/partition.05.query.sqlpp
new file mode 100644
index 0000000..7c3dab0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/partition/partition.05.query.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SELECT VALUE c
+FROM CustomerCopy c
+ORDER BY c.customer_id,
+ c.company,
+ c.year,
+ c.month,
+ c.day;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/partition/partition.06.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/partition/partition.06.query.sqlpp
new file mode 100644
index 0000000..c642b24
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/partition/partition.06.query.sqlpp
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SELECT VALUE c
+FROM CustomerCopy c
+WHERE c.company = "ford"
+ORDER BY c.customer_id,
+ c.company,
+ c.year,
+ c.month,
+ c.day;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.01.ddl.sqlpp
new file mode 100644
index 0000000..b68c38b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.01.ddl.sqlpp
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+CREATE TYPE OpenType AS {
+};
+
+CREATE EXTERNAL DATASET Customer(OpenType) USING S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-2"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
+ ("container"="playground"),
+ ("definition"="external-filter/car/{company:string}/customer/{customer_id:int}"),
+ ("embed-filter-values" = "false"),
+ ("format"="json")
+);
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.02.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.02.update.sqlpp
new file mode 100644
index 0000000..6df40c3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.02.update.sqlpp
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+COPY (
+ SELECT DISTINCT UPPERCASE(c.company) company, c.year
+ FROM Customer c
+) toWriter
+TO S3
+PATH ("copy-to-result", "company-year", company, year)
+OVER (
+ PARTITION BY toWriter.company company,
+ toWriter.year year
+)
+WITH {
+ "accessKeyId":"dummyAccessKey",
+ "secretAccessKey":"dummySecretKey",
+ "region":"us-west-2",
+ "serviceEndpoint":"http://127.0.0.1:8001",
+ "container":"playground",
+ "format":"json",
+ "compression":"gzip"
+}
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.03.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.03.ddl.sqlpp
new file mode 100644
index 0000000..6c19d01
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.03.ddl.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+CREATE EXTERNAL DATASET CustomerCopy(OpenType) USING S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-2"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
+ ("container"="playground"),
+ ("definition"="copy-to-result/company-year/{company:string}/{year:int}"),
+ ("embed-filter-values" = "false"),
+ ("format"="json")
+);
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.04.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.04.query.sqlpp
new file mode 100644
index 0000000..525de23
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.04.query.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SELECT VALUE c
+FROM CustomerCopy c
+ORDER BY c.company,
+ c.year
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.05.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.05.query.sqlpp
new file mode 100644
index 0000000..0b989d4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/query/query.05.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SELECT VALUE c
+FROM CustomerCopy c
+WHERE c.company = "FORD"
+ORDER BY c.company,
+ c.year
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/simple-write/simple-write.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/simple-write/simple-write.01.ddl.sqlpp
new file mode 100644
index 0000000..b68c38b
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/simple-write/simple-write.01.ddl.sqlpp
@@ -0,0 +1,39 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+
+USE test;
+
+CREATE TYPE OpenType AS {
+};
+
+CREATE EXTERNAL DATASET Customer(OpenType) USING S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-2"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
+ ("container"="playground"),
+ ("definition"="external-filter/car/{company:string}/customer/{customer_id:int}"),
+ ("embed-filter-values" = "false"),
+ ("format"="json")
+);
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/simple-write/simple-write.02.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/simple-write/simple-write.02.update.sqlpp
new file mode 100644
index 0000000..94aa7a0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/simple-write/simple-write.02.update.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+COPY Customer c
+TO S3
+PATH ("copy-to-result", "simple-write")
+WITH {
+ "accessKeyId":"dummyAccessKey",
+ "secretAccessKey":"dummySecretKey",
+ "region":"us-west-2",
+ "serviceEndpoint":"http://127.0.0.1:8001",
+ "container":"playground",
+ "format":"json"
+}
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/simple-write/simple-write.03.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/simple-write/simple-write.03.ddl.sqlpp
new file mode 100644
index 0000000..5f03abd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/simple-write/simple-write.03.ddl.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+CREATE EXTERNAL DATASET CustomerCopy(OpenType) USING S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-2"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
+ ("container"="playground"),
+ ("definition"="copy-to-result/simple-write"),
+ ("embed-filter-values" = "false"),
+ ("format"="json")
+);
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/simple-write/simple-write.04.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/simple-write/simple-write.04.query.sqlpp
new file mode 100644
index 0000000..eace40c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/simple-write/simple-write.04.query.sqlpp
@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SELECT VALUE c
+FROM Customer c
+ORDER BY c.customer_id,
+ c.company,
+ c.year,
+ c.month,
+ c.day;
+
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/simple-write/simple-write.05.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/simple-write/simple-write.05.query.sqlpp
new file mode 100644
index 0000000..7c3dab0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/copy-to/simple-write/simple-write.05.query.sqlpp
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+SELECT VALUE c
+FROM CustomerCopy c
+ORDER BY c.customer_id,
+ c.company,
+ c.year,
+ c.month,
+ c.day;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/negative/long-path/long-path.03.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/negative/long-path/long-path.03.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/negative/long-path/long-path.03.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/negative/runtime-missing/runtime-missing.02.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/negative/runtime-missing/runtime-missing.02.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/negative/runtime-missing/runtime-missing.02.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/negative/runtime-missing/runtime-missing.03.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/negative/runtime-missing/runtime-missing.03.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/negative/runtime-missing/runtime-missing.03.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/negative/runtime-missing/runtime-missing.05.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/negative/runtime-missing/runtime-missing.05.adm
new file mode 100644
index 0000000..53b6004
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/negative/runtime-missing/runtime-missing.05.adm
@@ -0,0 +1,6 @@
+{ "company": "lexus", "year": 2001 }
+{ "company": "lexus", "year": 2002 }
+{ "company": "lexus", "year": 2003 }
+{ "company": "toyota", "year": 2001 }
+{ "company": "toyota", "year": 2002 }
+{ "company": "toyota", "year": 2003 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/partition/partition.04.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/partition/partition.04.adm
new file mode 100644
index 0000000..4e13836
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/partition/partition.04.adm
@@ -0,0 +1,81 @@
+{ "customer_id": 1, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/partition/partition.05.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/partition/partition.05.adm
new file mode 100644
index 0000000..4e13836
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/partition/partition.05.adm
@@ -0,0 +1,81 @@
+{ "customer_id": 1, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/partition/partition.06.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/partition/partition.06.adm
new file mode 100644
index 0000000..330cafe
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/partition/partition.06.adm
@@ -0,0 +1,27 @@
+{ "customer_id": 1, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "ford", "year": 2003, "month": 3, "day": 3 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/query/query.04.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/query/query.04.adm
new file mode 100644
index 0000000..dd696c1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/query/query.04.adm
@@ -0,0 +1,9 @@
+{ "company": "FORD", "year": 2001 }
+{ "company": "FORD", "year": 2002 }
+{ "company": "FORD", "year": 2003 }
+{ "company": "LEXUS", "year": 2001 }
+{ "company": "LEXUS", "year": 2002 }
+{ "company": "LEXUS", "year": 2003 }
+{ "company": "TOYOTA", "year": 2001 }
+{ "company": "TOYOTA", "year": 2002 }
+{ "company": "TOYOTA", "year": 2003 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/query/query.05.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/query/query.05.adm
new file mode 100644
index 0000000..17e601c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/query/query.05.adm
@@ -0,0 +1,3 @@
+{ "company": "FORD", "year": 2001 }
+{ "company": "FORD", "year": 2002 }
+{ "company": "FORD", "year": 2003 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/simple-write/simple-write.04.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/simple-write/simple-write.04.adm
new file mode 100644
index 0000000..4e13836
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/simple-write/simple-write.04.adm
@@ -0,0 +1,81 @@
+{ "customer_id": 1, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/simple-write/simple-write.05.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/simple-write/simple-write.05.adm
new file mode 100644
index 0000000..4e13836
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/simple-write/simple-write.05.adm
@@ -0,0 +1,81 @@
+{ "customer_id": 1, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "lexus", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "lexus", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "lexus", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "toyota", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "toyota", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "toyota", "year": 2003, "month": 3, "day": 3 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/simple-write/simple-write.06.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/simple-write/simple-write.06.adm
new file mode 100644
index 0000000..330cafe
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/copy-to/simple-write/simple-write.06.adm
@@ -0,0 +1,27 @@
+{ "customer_id": 1, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 1, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 1, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 1, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 2, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 2, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 2, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "ford", "year": 2001, "month": 1, "day": 1 }
+{ "customer_id": 3, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "ford", "year": 2002, "month": 2, "day": 2 }
+{ "customer_id": 3, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "ford", "year": 2003, "month": 3, "day": 3 }
+{ "customer_id": 3, "company": "ford", "year": 2003, "month": 3, "day": 3 }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 143881f6..5b53b41 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -18,6 +18,58 @@
! under the License.
!-->
<test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" QueryOffsetPath="queries_sqlpp" QueryFileExtension=".sqlpp">
+ <test-group name="copy-to">
+ <test-case FilePath="copy-to">
+ <compilation-unit name="partition">
+ <output-dir compare="Text">partition</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to">
+ <compilation-unit name="simple-write">
+ <output-dir compare="Text">simple-write</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to">
+ <compilation-unit name="query">
+ <output-dir compare="Text">query</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to/negative">
+ <compilation-unit name="early-missing">
+ <output-dir compare="Text">early-missing</output-dir>
+ <expected-error>ASX0064: Path expression produced a value of type 'missing'. Path must be of type string</expected-error>
+ <expected-error>ASX0064: Path expression produced a value of type 'null'. Path must be of type string</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to/negative" check-warnings="true">
+ <compilation-unit name="long-path">
+ <output-dir compare="Text">long-path</output-dir>
+ <expected-error>ASX0065: Length of the file path 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa' exceeds the maximum length of '1024 bytes' allowed in S3</expected-error>
+ <expected-warn>ASX0065: Length of the file path 'ford/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/' exceeds the maximum length of '1024 bytes' allowed in S3 (in line 29, at column 7)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to/negative">
+ <compilation-unit name="non-empty-folder">
+ <output-dir compare="Text">non-empty-folder</output-dir>
+ <expected-error>ASX0062: Cannot write to a non-empty directory 'copy-to-result/duplicate-write'</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to/negative" check-warnings="true">
+ <compilation-unit name="runtime-missing">
+ <output-dir compare="Text">runtime-missing</output-dir>
+ <expected-warn>ASX0064: Path expression produced a value of type 'missing'. Path must be of type string (in line 24, at column 7)</expected-warn>
+ <expected-warn>ASX0064: Path expression produced a value of type 'missing'. Path must be of type string (in line 31, at column 7)</expected-warn>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="copy-to/negative">
+ <compilation-unit name="supported-adapter-format-compression">
+ <output-dir compare="Text">supported-adapter-format-compression</output-dir>
+ <expected-error>ASX1188: Unsupported writing adapter 'AZUREBLOB'. Supported adapters: [localfs, s3]</expected-error>
+ <expected-error>ASX1189: Unsupported writing format 'csv'. Supported formats: [json]</expected-error>
+ <expected-error>ASX1096: Unknown compression scheme rar. Supported schemes are [gzip]</expected-error>
+ </compilation-unit>
+ </test-case>
+ </test-group>
<test-group name="aws-s3-external-dataset">
<test-case FilePath="external-dataset">
<compilation-unit name="common/json/json">
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriter.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriter.java
index 64eee70..fb8150b 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriter.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/AbstractCloudExternalFileWriter.java
@@ -64,6 +64,10 @@
@Override
public void validate(String directory) throws HyracksDataException {
+ if (checkAndWarnExceedingMaxLength(directory)) {
+ return;
+ }
+
if (partitionedPath && !cloudClient.isEmptyPrefix(bucket, directory)) {
throw new RuntimeDataException(ErrorCode.DIRECTORY_IS_NOT_EMPTY, pathSourceLocation, directory);
}
@@ -72,11 +76,7 @@
@Override
public final boolean newFile(String directory, String fileName) throws HyracksDataException {
String fullPath = directory + fileName;
- if (isExceedingMaxLength(fullPath)) {
- if (warningCollector.shouldWarn()) {
- warningCollector.warn(Warning.of(pathSourceLocation, ErrorCode.WRITE_PATH_LENGTH_EXCEEDS_MAX_LENGTH,
- fullPath, getPathMaxLengthInBytes(), getAdapterName()));
- }
+ if (checkAndWarnExceedingMaxLength(fullPath)) {
return false;
}
@@ -96,7 +96,9 @@
@Override
public final void abort() throws HyracksDataException {
- bufferedWriter.abort();
+ if (bufferedWriter != null) {
+ bufferedWriter.abort();
+ }
printer.close();
}
@@ -109,7 +111,16 @@
abstract int getPathMaxLengthInBytes();
- private boolean isExceedingMaxLength(String path) {
- return Utf8.encodedLength(path) >= getPathMaxLengthInBytes();
+ private boolean checkAndWarnExceedingMaxLength(String fullPath) {
+ boolean exceeding = isExceedingMaxLength(fullPath, getPathMaxLengthInBytes());
+ if (exceeding && warningCollector.shouldWarn()) {
+ warningCollector.warn(Warning.of(pathSourceLocation, ErrorCode.WRITE_PATH_LENGTH_EXCEEDS_MAX_LENGTH,
+ fullPath, getPathMaxLengthInBytes(), getAdapterName()));
+ }
+ return exceeding;
+ }
+
+ static boolean isExceedingMaxLength(String path, int maxLength) {
+ return Utf8.encodedLength(path) >= maxLength;
}
}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriter.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriter.java
index 5fd4ea6..648dda5 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriter.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriter.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.api.exceptions.SourceLocation;
final class S3ExternalFileWriter extends AbstractCloudExternalFileWriter {
+ static int MAX_LENGTH_IN_BYTES = 1024;
S3ExternalFileWriter(IExternalFilePrinter printer, ICloudClient cloudClient, String bucket, boolean partitionedPath,
IWarningCollector warningCollector, SourceLocation pathSourceLocation) {
@@ -38,6 +39,6 @@
@Override
int getPathMaxLengthInBytes() {
- return 1024;
+ return MAX_LENGTH_IN_BYTES;
}
}
diff --git a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
index d7a51cd..508225d 100644
--- a/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
+++ b/asterixdb/asterix-cloud/src/main/java/org/apache/asterix/cloud/writer/S3ExternalFileWriterFactory.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.cloud.writer;
+import static org.apache.asterix.cloud.writer.AbstractCloudExternalFileWriter.isExceedingMaxLength;
+
import java.io.IOException;
import java.util.Collections;
import java.util.Map;
@@ -134,9 +136,17 @@
}
private void doValidate(ICloudClient testClient, String bucket) throws IOException, AlgebricksException {
- if (staticPath != null && !testClient.isEmptyPrefix(bucket, staticPath)) {
- // Ensure that the static path is empty
- throw new CompilationException(ErrorCode.DIRECTORY_IS_NOT_EMPTY, pathSourceLocation, staticPath);
+ if (staticPath != null) {
+ if (isExceedingMaxLength(staticPath, S3ExternalFileWriter.MAX_LENGTH_IN_BYTES)) {
+ throw new CompilationException(ErrorCode.WRITE_PATH_LENGTH_EXCEEDS_MAX_LENGTH, pathSourceLocation,
+ staticPath, S3ExternalFileWriter.MAX_LENGTH_IN_BYTES,
+ ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3);
+ }
+
+ if (!testClient.isEmptyPrefix(bucket, staticPath)) {
+ // Ensure that the static path is empty
+ throw new CompilationException(ErrorCode.DIRECTORY_IS_NOT_EMPTY, pathSourceLocation, staticPath);
+ }
}
Random random = new Random();
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java
index 4c848b1..00fe855 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/WriterValidationUtil.java
@@ -18,8 +18,10 @@
*/
package org.apache.asterix.external.util;
+import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
@@ -85,7 +87,8 @@
String normalizedValue = value.toLowerCase();
if (!supportedSet.contains(normalizedValue)) {
- throw CompilationException.create(errorCode, sourceLocation, value, supportedSet.toString());
+ List<String> sorted = supportedSet.stream().sorted().collect(Collectors.toList());
+ throw CompilationException.create(errorCode, sourceLocation, value, sorted.toString());
}
}
diff --git a/asterixdb/asterix-metadata/pom.xml b/asterixdb/asterix-metadata/pom.xml
index e0b5387..d870fb5 100644
--- a/asterixdb/asterix-metadata/pom.xml
+++ b/asterixdb/asterix-metadata/pom.xml
@@ -185,5 +185,10 @@
<artifactId>asterix-column</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.asterix</groupId>
+ <artifactId>asterix-cloud</artifactId>
+ <version>${project.version}</version>
+ </dependency>
</dependencies>
</project>
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 28bce7e..cfa8054 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -85,6 +85,7 @@
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.entities.Synonym;
import org.apache.asterix.metadata.feeds.FeedMetadataUtil;
+import org.apache.asterix.metadata.provider.ExternalWriterProvider;
import org.apache.asterix.metadata.utils.DataPartitioningProvider;
import org.apache.asterix.metadata.utils.DatasetUtil;
import org.apache.asterix.metadata.utils.FullTextUtil;
@@ -95,6 +96,7 @@
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.utils.ConstantExpressionUtil;
import org.apache.asterix.om.utils.NonTaggedFormatUtil;
import org.apache.asterix.runtime.base.AsterixTupleFilterFactory;
import org.apache.asterix.runtime.formats.FormatUtils;
@@ -104,10 +106,12 @@
import org.apache.asterix.runtime.operators.LSMSecondaryInsertDeleteWithNestedPlanOperatorDescriptor;
import org.apache.asterix.runtime.operators.LSMSecondaryUpsertOperatorDescriptor;
import org.apache.asterix.runtime.operators.LSMSecondaryUpsertWithNestedPlanOperatorDescriptor;
+import org.apache.asterix.runtime.writer.ExternalWriterFactory;
+import org.apache.asterix.runtime.writer.IExternalFilePrinterFactory;
+import org.apache.asterix.runtime.writer.IExternalFileWriterFactory;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.common.utils.Quadruple;
import org.apache.hyracks.algebricks.common.utils.Triple;
@@ -135,6 +139,7 @@
import org.apache.hyracks.algebricks.runtime.base.AlgebricksPipeline;
import org.apache.hyracks.algebricks.runtime.base.IPushRuntimeFactory;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.algebricks.runtime.operators.writer.SinkExternalWriterRuntimeFactory;
import org.apache.hyracks.api.dataflow.IOperatorDescriptor;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
@@ -746,9 +751,18 @@
IScalarEvaluatorFactory dynamicPathEvalFactory, ILogicalExpression staticPathExpr,
SourceLocation pathSourceLocation, IWriteDataSink sink, RecordDescriptor inputDesc, Object sourceType)
throws AlgebricksException {
-
- // TODO implement
- throw new NotImplementedException();
+ String staticPath = staticPathExpr != null ? ConstantExpressionUtil.getStringConstant(staticPathExpr) : null;
+ IExternalFileWriterFactory fileWriterFactory =
+ ExternalWriterProvider.createWriterFactory(appCtx, sink, staticPath, pathSourceLocation);
+ fileWriterFactory.validate();
+ String fileExtension = ExternalWriterProvider.getFileExtension(sink);
+ int maxResult = ExternalWriterProvider.getMaxResult(sink);
+ IExternalFilePrinterFactory printerFactory = ExternalWriterProvider.createPrinter(sink, sourceType);
+ ExternalWriterFactory writerFactory = new ExternalWriterFactory(fileWriterFactory, printerFactory,
+ fileExtension, maxResult, dynamicPathEvalFactory, staticPath, pathSourceLocation);
+ SinkExternalWriterRuntimeFactory runtime = new SinkExternalWriterRuntimeFactory(sourceColumn, partitionColumns,
+ partitionComparatorFactories, inputDesc, writerFactory);
+ return new Pair<>(runtime, null);
}
@Override
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java
new file mode 100644
index 0000000..9142556
--- /dev/null
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/provider/ExternalWriterProvider.java
@@ -0,0 +1,142 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.metadata.provider;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.asterix.cloud.writer.S3ExternalFileWriterFactory;
+import org.apache.asterix.common.dataflow.ICcApplicationContext;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.external.writer.LocalFSExternalFileWriterFactory;
+import org.apache.asterix.external.writer.compressor.GzipExternalFileCompressStreamFactory;
+import org.apache.asterix.external.writer.compressor.IExternalFileCompressStreamFactory;
+import org.apache.asterix.external.writer.compressor.NoOpExternalFileCompressStreamFactory;
+import org.apache.asterix.external.writer.printer.TextualExternalFilePrinterFactory;
+import org.apache.asterix.formats.nontagged.CleanJSONPrinterFactoryProvider;
+import org.apache.asterix.runtime.writer.ExternalFileWriterConfiguration;
+import org.apache.asterix.runtime.writer.IExternalFileFilterWriterFactoryProvider;
+import org.apache.asterix.runtime.writer.IExternalFilePrinterFactory;
+import org.apache.asterix.runtime.writer.IExternalFileWriterFactory;
+import org.apache.hyracks.algebricks.core.algebra.metadata.IWriteDataSink;
+import org.apache.hyracks.algebricks.data.IPrinterFactory;
+import org.apache.hyracks.api.exceptions.SourceLocation;
+import org.apache.hyracks.control.cc.ClusterControllerService;
+
+public class ExternalWriterProvider {
+ private static final Map<String, IExternalFileFilterWriterFactoryProvider> CREATOR_MAP;
+ private static final Map<String, IExternalFileCompressStreamFactory> STREAM_COMPRESSORS;
+
+ private ExternalWriterProvider() {
+ }
+
+ static {
+ CREATOR_MAP = new HashMap<>();
+ addCreator(ExternalDataConstants.KEY_ADAPTER_NAME_LOCALFS, LocalFSExternalFileWriterFactory.PROVIDER);
+ addCreator(ExternalDataConstants.KEY_ADAPTER_NAME_AWS_S3, S3ExternalFileWriterFactory.PROVIDER);
+
+ STREAM_COMPRESSORS = new HashMap<>();
+ STREAM_COMPRESSORS.put(ExternalDataConstants.KEY_COMPRESSION_GZIP,
+ GzipExternalFileCompressStreamFactory.INSTANCE);
+ }
+
+ public static IExternalFileWriterFactory createWriterFactory(ICcApplicationContext appCtx, IWriteDataSink sink,
+ String staticPath, SourceLocation pathExpressionLocation) {
+ String adapterName = sink.getAdapterName().toLowerCase();
+ IExternalFileFilterWriterFactoryProvider creator = CREATOR_MAP.get(adapterName);
+
+ if (creator == null) {
+ throw new UnsupportedOperationException("Unsupported adapter " + adapterName);
+ }
+
+ return creator.create(createConfiguration(appCtx, sink, staticPath, pathExpressionLocation));
+ }
+
+ public static String getFileExtension(IWriteDataSink sink) {
+ Map<String, String> configuration = sink.getConfiguration();
+ String format = getFormat(configuration);
+ String compression = getCompression(configuration);
+ return format + (compression.isEmpty() ? "" : "." + compression);
+ }
+
+ public static int getMaxResult(IWriteDataSink sink) {
+ String maxResultString = sink.getConfiguration().get(ExternalDataConstants.KEY_WRITER_MAX_RESULT);
+ if (maxResultString == null) {
+ return ExternalDataConstants.WRITER_MAX_RESULT_DEFAULT;
+ }
+ return Integer.parseInt(maxResultString);
+ }
+
+ private static ExternalFileWriterConfiguration createConfiguration(ICcApplicationContext appCtx,
+ IWriteDataSink sink, String staticPath, SourceLocation pathExpressionLocation) {
+ Map<String, String> params = sink.getConfiguration();
+ boolean singleNodeCluster = isSingleNodeCluster(appCtx);
+
+ return new ExternalFileWriterConfiguration(params, pathExpressionLocation, staticPath, singleNodeCluster);
+ }
+
+ private static boolean isSingleNodeCluster(ICcApplicationContext appCtx) {
+ ClusterControllerService ccs = (ClusterControllerService) appCtx.getServiceContext().getControllerService();
+ return ccs.getNodeManager().getIpAddressNodeNameMap().size() == 1;
+ }
+
+ private static void addCreator(String adapterName, IExternalFileFilterWriterFactoryProvider creator) {
+ IExternalFileFilterWriterFactoryProvider registeredCreator = CREATOR_MAP.get(adapterName.toLowerCase());
+ if (registeredCreator != null) {
+ throw new IllegalStateException(
+ "Adapter " + adapterName + " is registered to " + registeredCreator.getClass().getName());
+ }
+ CREATOR_MAP.put(adapterName.toLowerCase(), creator);
+ }
+
+ public static IExternalFilePrinterFactory createPrinter(IWriteDataSink sink, Object sourceType) {
+ Map<String, String> configuration = sink.getConfiguration();
+ String format = configuration.get(ExternalDataConstants.KEY_FORMAT);
+
+ // Only JSON is supported for now
+ if (!ExternalDataConstants.FORMAT_JSON_LOWER_CASE.equalsIgnoreCase(format)) {
+ throw new UnsupportedOperationException("Unsupported format " + format);
+ }
+
+ String compression = getCompression(configuration);
+ IExternalFileCompressStreamFactory compressStreamFactory =
+ STREAM_COMPRESSORS.getOrDefault(compression, NoOpExternalFileCompressStreamFactory.INSTANCE);
+
+ IPrinterFactory printerFactory = CleanJSONPrinterFactoryProvider.INSTANCE.getPrinterFactory(sourceType);
+ return new TextualExternalFilePrinterFactory(printerFactory, compressStreamFactory);
+ }
+
+ private static String getFormat(Map<String, String> configuration) {
+ return configuration.get(ExternalDataConstants.KEY_FORMAT);
+ }
+
+ private static String getCompression(Map<String, String> configuration) {
+ return configuration.getOrDefault(ExternalDataConstants.KEY_WRITER_COMPRESSION, "");
+ }
+
+ public static char getSeparator(String adapterName) {
+ IExternalFileFilterWriterFactoryProvider creator = CREATOR_MAP.get(adapterName.toLowerCase());
+
+ if (creator == null) {
+ throw new UnsupportedOperationException("Unsupported adapter " + adapterName);
+ }
+
+ return creator.getSeparator();
+ }
+}