[ASTERIXDB-3347][COMP] Refactor COPY TO to support different types of write destinations
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
This change refactors the COPY TO code
to allow for easier extendability for supporting
other types of destinations to write to.
Change-Id: Id90a30c1e9f41ca82ef28f6edd0569a69002572c
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18146
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hussainht@gmail.com>
Reviewed-by: Wail Alkowaileet <wael.y.k@gmail.com>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
index 0ff0b72..8025202 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/CompiledStatements.java
@@ -602,6 +602,8 @@
private final List<Expression> orderbyList;
private final List<OrderbyClause.OrderModifier> orderByModifiers;
private final List<OrderbyClause.NullOrderModifier> orderByNullModifierList;
+ private final List<Expression> keyExpressions;
+ private final boolean autogenerated;
public CompiledCopyToStatement(CopyToStatement copyToStatement) {
this.query = copyToStatement.getQuery();
@@ -615,6 +617,8 @@
this.orderbyList = copyToStatement.getOrderByList();
this.orderByModifiers = copyToStatement.getOrderByModifiers();
this.orderByNullModifierList = copyToStatement.getOrderByNullModifierList();
+ this.keyExpressions = copyToStatement.getKeyExpressions();
+ this.autogenerated = copyToStatement.isAutogenerated();
}
@Override
@@ -669,6 +673,14 @@
public List<OrderbyClause.NullOrderModifier> getOrderByNullModifiers() {
return orderByNullModifierList;
}
+
+ public List<Expression> getKeyExpressions() {
+ return keyExpressions;
+ }
+
+ public boolean isAutogenerated() {
+ return autogenerated;
+ }
}
}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToStatement.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToStatement.java
index dbe7b35..2520755 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToStatement.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CopyToStatement.java
@@ -21,6 +21,7 @@
import static org.apache.asterix.lang.common.base.Statement.Category.QUERY;
import java.util.ArrayList;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -42,7 +43,8 @@
private final Map<Integer, VariableExpr> partitionsVariables;
private final List<OrderbyClause.OrderModifier> orderByModifiers;
private final List<OrderbyClause.NullOrderModifier> orderByNullModifierList;
-
+ private final List<Expression> keyExpressions;
+ private final boolean autogenerated;
private Namespace namespace;
private Query query;
private List<Expression> pathExpressions;
@@ -52,10 +54,29 @@
private int varCounter;
public CopyToStatement(Namespace namespace, String datasetName, Query query, VariableExpr sourceVariable,
+ ExternalDetailsDecl externalDetailsDecl, int varCounter, List<Expression> keyExpressions,
+ boolean autogenerated) {
+ this(namespace, datasetName, query, sourceVariable, externalDetailsDecl, new ArrayList<>(), new ArrayList<>(),
+ new HashMap<>(), new ArrayList<>(), new ArrayList<>(), new ArrayList<>(), varCounter, keyExpressions,
+ autogenerated);
+ }
+
+ public CopyToStatement(Namespace namespace, String datasetName, Query query, VariableExpr sourceVariable,
ExternalDetailsDecl externalDetailsDecl, List<Expression> pathExpressions,
List<Expression> partitionExpressions, Map<Integer, VariableExpr> partitionsVariables,
List<Expression> orderbyList, List<OrderbyClause.OrderModifier> orderByModifiers,
List<OrderbyClause.NullOrderModifier> orderByNullModifierList, int varCounter) {
+ this(namespace, datasetName, query, sourceVariable, externalDetailsDecl, pathExpressions, partitionExpressions,
+ partitionsVariables, orderbyList, orderByModifiers, orderByNullModifierList, varCounter,
+ new ArrayList<>(), false);
+ }
+
+ private CopyToStatement(Namespace namespace, String datasetName, Query query, VariableExpr sourceVariable,
+ ExternalDetailsDecl externalDetailsDecl, List<Expression> pathExpressions,
+ List<Expression> partitionExpressions, Map<Integer, VariableExpr> partitionsVariables,
+ List<Expression> orderbyList, List<OrderbyClause.OrderModifier> orderByModifiers,
+ List<OrderbyClause.NullOrderModifier> orderByNullModifierList, int varCounter,
+ List<Expression> keyExpressions, boolean autogenerated) {
this.namespace = namespace;
this.datasetName = datasetName;
this.query = query;
@@ -68,6 +89,8 @@
this.orderByModifiers = orderByModifiers;
this.orderByNullModifierList = orderByNullModifierList;
this.varCounter = varCounter;
+ this.keyExpressions = keyExpressions;
+ this.autogenerated = autogenerated;
if (pathExpressions.isEmpty()) {
// Ensure path expressions to have at least an empty string
@@ -203,4 +226,20 @@
public void setBody(Expression expr) {
query.setBody(expr);
}
+
+ public List<Expression> getKeyExpressions() {
+ return keyExpressions;
+ }
+
+ public boolean isAutogenerated() {
+ return autogenerated;
+ }
+
+ public boolean isSinkFileStore() {
+ return keyExpressions.isEmpty() && !autogenerated;
+ }
+
+ public boolean isSinkDatabaseWithKey() {
+ return !keyExpressions.isEmpty() || autogenerated;
+ }
}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
index 3091b30..52e2678 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/visitor/FormatPrintVisitor.java
@@ -573,6 +573,20 @@
cto.getSourceVariable().accept(this, step);
out.println();
+ if (cto.isSinkFileStore()) {
+ formatPrintCopyToFileStore(cto, step);
+ } else if (cto.isSinkDatabaseWithKey()) {
+ formatPrintCopyToDatabaseWithKey(cto, step);
+ } else {
+ throw new IllegalStateException("NYI: This should never happen");
+ }
+
+ out.println("with ");
+ printConfiguration(cto.getExternalDetailsDecl().getProperties());
+ return null;
+ }
+
+ private void formatPrintCopyToFileStore(CopyToStatement cto, Integer step) throws CompilationException {
out.print("path (");
printDelimitedExpressions(cto.getPathExpressions(), COMMA, step + 1);
out.print(")");
@@ -606,10 +620,16 @@
}
out.println(')');
}
+ }
- out.println("with ");
- printConfiguration(cto.getExternalDetailsDecl().getProperties());
- return null;
+ private void formatPrintCopyToDatabaseWithKey(CopyToStatement cto, Integer step) throws CompilationException {
+ out.print("key ");
+ if (!cto.getKeyExpressions().isEmpty()) {
+ printDelimitedExpressions(cto.getKeyExpressions(), COMMA, step + 1);
+ } else {
+ out.print("autogenerated");
+ }
+ out.println();
}
@Override
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index a81807c..a1b7daf 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -2869,12 +2869,12 @@
}
{
<COPY>
- ( LOOKAHEAD(1) <INTO> { startToken = token; }
+ ( <INTO> { startToken = token; }
nameComponents = QualifiedName()
((<AS>)? (typeExpr = DatasetTypeSpecification(RecordTypeDefinition.RecordKind.OPEN)))?
stmt = CopyFromStatement(startToken, nameComponents, typeExpr)
- | LOOKAHEAD(1) <LEFTPAREN> { startToken = token; } query = Query() <RIGHTPAREN> (<AS>)? alias = Variable() stmt = CopyToStatement(startToken, nameComponents, query, alias)
- | { startToken = token; } nameComponents = QualifiedName()
+ | <LEFTPAREN> { startToken = token; } query = Query() <RIGHTPAREN> (<AS>)? alias = Variable() stmt = CopyToStatement(startToken, nameComponents, query, alias)
+ | { startToken = token; } nameComponents = QualifiedName()
(<AS>)? (typeExpr = DatasetTypeSpecification(RecordTypeDefinition.RecordKind.OPEN) | alias = Variable())?
(stmt = CopyFromStatement(startToken, nameComponents, typeExpr) | stmt = CopyToStatement(startToken, nameComponents, query, alias))
)