SQL++ parser:
1. refactored asterix-aql to become asterix-lang-common and asterix-lang-aql, where the former is the common part for different languages;
2. added asterix-lang-sqlpp on top of asterix-lang-common;
3. ported parser tests, optimizer tests and runtime tests in asterix-app to their sql++ version, and added parser tests for all the queries.
Change-Id: Ie5af4e3b692ca017ec047a1ba3b404a51beb3a2e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/466
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <tillw@apache.org>
diff --git a/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/clause/DistinctClause.java b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/clause/DistinctClause.java
new file mode 100644
index 0000000..789c5bd
--- /dev/null
+++ b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/clause/DistinctClause.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.aql.clause;
+
+import java.util.List;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.lang.aql.visitor.base.IAQLVisitor;
+import org.apache.asterix.lang.common.base.Clause;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
+
+public class DistinctClause implements Clause {
+
+ private List<Expression> distinctByExprs;
+
+ public DistinctClause(List<Expression> distinctByExpr) {
+ this.distinctByExprs = distinctByExpr;
+ }
+
+ public List<Expression> getDistinctByExpr() {
+ return distinctByExprs;
+ }
+
+ @Override
+ public ClauseType getClauseType() {
+ return ClauseType.DISTINCT_BY_CLAUSE;
+ }
+
+ @Override
+ public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws AsterixException {
+ return ((IAQLVisitor<R, T>) visitor).visit(this, arg);
+ }
+
+}
diff --git a/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/clause/ForClause.java b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/clause/ForClause.java
new file mode 100644
index 0000000..39e7bbb
--- /dev/null
+++ b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/clause/ForClause.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.aql.clause;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.lang.aql.visitor.base.IAQLVisitor;
+import org.apache.asterix.lang.common.base.Clause;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
+
+public class ForClause implements Clause {
+ private VariableExpr varExpr = null;
+ private VariableExpr posExpr = null;
+ private Expression inExpr = null;
+
+ public ForClause() {
+ super();
+ }
+
+ public ForClause(VariableExpr varExpr, Expression inExpr) {
+ super();
+ this.varExpr = varExpr;
+ this.inExpr = inExpr;
+ }
+
+ public ForClause(VariableExpr varExpr, Expression inExpr, VariableExpr posExpr) {
+ super();
+ this.varExpr = varExpr;
+ this.inExpr = inExpr;
+ this.posExpr = posExpr;
+ }
+
+ public VariableExpr getVarExpr() {
+ return varExpr;
+ }
+
+ public void setVarExpr(VariableExpr varExpr) {
+ this.varExpr = varExpr;
+ }
+
+ public Expression getInExpr() {
+ return inExpr;
+ }
+
+ public void setInExpr(Expression inExpr) {
+ this.inExpr = inExpr;
+ }
+
+ @Override
+ public ClauseType getClauseType() {
+ return ClauseType.FOR_CLAUSE;
+ }
+
+ @Override
+ public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws AsterixException {
+ return ((IAQLVisitor<R, T>) visitor).visit(this, arg);
+ }
+
+ public void setPosExpr(VariableExpr posExpr) {
+ this.posExpr = posExpr;
+ }
+
+ public VariableExpr getPosVarExpr() {
+ return posExpr;
+ }
+
+ public boolean hasPosVar() {
+ return posExpr != null;
+ }
+}
diff --git a/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/clause/JoinClause.java b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/clause/JoinClause.java
new file mode 100644
index 0000000..558bd52
--- /dev/null
+++ b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/clause/JoinClause.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.aql.clause;
+
+import java.util.List;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.lang.aql.visitor.base.IAQLPlusVisitor;
+import org.apache.asterix.lang.common.base.Clause;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
+
+public class JoinClause implements Clause {
+
+ public static enum JoinKind {
+ INNER,
+ LEFT_OUTER
+ }
+
+ private Expression whereExpr;
+ private List<Clause> leftClauses, rightClauses;
+ private final JoinKind kind;
+
+ public JoinClause() {
+ kind = JoinKind.INNER;
+ }
+
+ public JoinClause(JoinKind kind) {
+ this.kind = kind;
+ }
+
+ @Override
+ public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws AsterixException {
+ return ((IAQLPlusVisitor<R, T>) visitor).visitJoinClause(this, arg);
+ }
+
+ @Override
+ public ClauseType getClauseType() {
+ return null;
+ }
+
+ public List<Clause> getLeftClauses() {
+ return leftClauses;
+ }
+
+ public List<Clause> getRightClauses() {
+ return rightClauses;
+ }
+
+ public Expression getWhereExpr() {
+ return whereExpr;
+ }
+
+ public void setLeftClauses(List<Clause> leftClauses) {
+ this.leftClauses = leftClauses;
+ }
+
+ public void setRightClauses(List<Clause> righClauses) {
+ this.rightClauses = righClauses;
+ }
+
+ public void setWhereExpr(Expression whereExpr) {
+ this.whereExpr = whereExpr;
+ }
+
+ public JoinKind getKind() {
+ return kind;
+ }
+}
diff --git a/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/clause/MetaVariableClause.java b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/clause/MetaVariableClause.java
new file mode 100644
index 0000000..a33699e
--- /dev/null
+++ b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/clause/MetaVariableClause.java
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.aql.clause;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.lang.aql.visitor.base.IAQLPlusVisitor;
+import org.apache.asterix.lang.common.base.Clause;
+import org.apache.asterix.lang.common.struct.VarIdentifier;
+import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
+
+public class MetaVariableClause implements Clause {
+ private VarIdentifier var;
+
+ @Override
+ public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws AsterixException {
+ return ((IAQLPlusVisitor<R, T>) visitor).visitMetaVariableClause(this, arg);
+ }
+
+ @Override
+ public ClauseType getClauseType() {
+ return null;
+ }
+
+ public VarIdentifier getVar() {
+ return var;
+ }
+
+ public void setVar(VarIdentifier var) {
+ this.var = var;
+ }
+}
diff --git a/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/expression/FLWOGRExpression.java b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/expression/FLWOGRExpression.java
new file mode 100644
index 0000000..0761396
--- /dev/null
+++ b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/expression/FLWOGRExpression.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.aql.expression;
+
+import java.util.List;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.lang.aql.visitor.base.IAQLVisitor;
+import org.apache.asterix.lang.common.base.Clause;
+import org.apache.asterix.lang.common.base.Clause.ClauseType;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
+
+public class FLWOGRExpression implements Expression {
+ private List<Clause> clauseList;
+ private Expression returnExpr;
+
+ public FLWOGRExpression() {
+ super();
+ }
+
+ public FLWOGRExpression(List<Clause> clauseList, Expression returnExpr) {
+ super();
+ this.clauseList = clauseList;
+ this.returnExpr = returnExpr;
+ }
+
+ public List<Clause> getClauseList() {
+ return clauseList;
+ }
+
+ public void setClauseList(List<Clause> clauseList) {
+ this.clauseList = clauseList;
+ }
+
+ public Expression getReturnExpr() {
+ return returnExpr;
+ }
+
+ public void setReturnExpr(Expression returnExpr) {
+ this.returnExpr = returnExpr;
+ }
+
+ @Override
+ public Kind getKind() {
+ return Kind.FLWOGR_EXPRESSION;
+ }
+
+ public boolean noForClause() {
+ for (Clause c : clauseList) {
+ if (c.getClauseType() == ClauseType.FOR_CLAUSE) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ @Override
+ public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws AsterixException {
+ return ((IAQLVisitor<R, T>) visitor).visit(this, arg);
+ }
+}
diff --git a/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/expression/MetaVariableExpr.java b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/expression/MetaVariableExpr.java
new file mode 100644
index 0000000..99c7902
--- /dev/null
+++ b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/expression/MetaVariableExpr.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.aql.expression;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.lang.aql.visitor.base.IAQLPlusVisitor;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
+
+public class MetaVariableExpr extends VariableExpr {
+
+ @Override
+ public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws AsterixException {
+ return ((IAQLPlusVisitor<R, T>) visitor).visitMetaVariableExpr(this, arg);
+ }
+
+ @Override
+ public boolean getIsNewVar() {
+ return false;
+ }
+
+ @Override
+ public Kind getKind() {
+ return Kind.METAVARIABLE_EXPRESSION;
+ }
+}
diff --git a/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/expression/UnionExpr.java b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/expression/UnionExpr.java
new file mode 100644
index 0000000..2c8756b
--- /dev/null
+++ b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/expression/UnionExpr.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.aql.expression;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.lang.aql.visitor.base.IAQLVisitor;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
+
+public class UnionExpr implements Expression {
+
+ private List<Expression> exprs;
+
+ public UnionExpr() {
+ exprs = new ArrayList<Expression>();
+ }
+
+ public UnionExpr(List<Expression> exprs) {
+ this.exprs = exprs;
+ }
+
+ @Override
+ public Kind getKind() {
+ return Kind.UNION_EXPRESSION;
+ }
+
+ public List<Expression> getExprs() {
+ return exprs;
+ }
+
+ public void setExprs(List<Expression> exprs) {
+ this.exprs = exprs;
+ }
+
+ public void addExpr(Expression exp) {
+ exprs.add(exp);
+ }
+
+ @Override
+ public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws AsterixException {
+ return ((IAQLVisitor<R, T>) visitor).visit(this, arg);
+ }
+}
diff --git a/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlRewriter.java b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlRewriter.java
new file mode 100644
index 0000000..dc99549
--- /dev/null
+++ b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlRewriter.java
@@ -0,0 +1,234 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.aql.rewrites;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.lang.aql.clause.DistinctClause;
+import org.apache.asterix.lang.aql.clause.ForClause;
+import org.apache.asterix.lang.aql.expression.FLWOGRExpression;
+import org.apache.asterix.lang.aql.expression.UnionExpr;
+import org.apache.asterix.lang.aql.util.FunctionUtils;
+import org.apache.asterix.lang.aql.visitor.AQLInlineUdfsVisitor;
+import org.apache.asterix.lang.aql.visitor.base.IAQLVisitor;
+import org.apache.asterix.lang.common.base.Clause;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.Expression.Kind;
+import org.apache.asterix.lang.common.clause.GroupbyClause;
+import org.apache.asterix.lang.common.clause.LetClause;
+import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
+import org.apache.asterix.lang.common.statement.FunctionDecl;
+import org.apache.asterix.lang.common.statement.Query;
+import org.apache.asterix.lang.common.struct.VarIdentifier;
+import org.apache.asterix.lang.common.visitor.GatherFunctionCallsVisitor;
+import org.apache.asterix.metadata.MetadataManager;
+import org.apache.asterix.metadata.MetadataTransactionContext;
+import org.apache.asterix.metadata.declared.AqlMetadataProvider;
+import org.apache.asterix.metadata.entities.Function;
+import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
+
+public final class AqlRewriter {
+
+ private final Query topExpr;
+ private final List<FunctionDecl> declaredFunctions;
+ private final LangRewritingContext context;
+ private final MetadataTransactionContext mdTxnCtx;
+ private final AqlMetadataProvider metadataProvider;
+
+ public AqlRewriter(List<FunctionDecl> declaredFunctions, Query topExpr, AqlMetadataProvider metadataProvider) {
+ this.topExpr = topExpr;
+ context = new LangRewritingContext(topExpr.getVarCounter());
+ this.declaredFunctions = declaredFunctions;
+ this.mdTxnCtx = metadataProvider.getMetadataTxnContext();
+ this.metadataProvider = metadataProvider;
+ }
+
+ public Query getExpr() {
+ return topExpr;
+ }
+
+ public int getVarCounter() {
+ return context.getVarCounter();
+ }
+
+ public void rewrite() throws AsterixException {
+ wrapInLets();
+ inlineDeclaredUdfs();
+ }
+
+ private void wrapInLets() {
+ // If the top expression of the main statement is not a FLWOR, it wraps
+ // it into a let clause.
+ if (topExpr == null) {
+ return;
+ }
+ Expression body = topExpr.getBody();
+ if (body.getKind() != Kind.FLWOGR_EXPRESSION) {
+ VarIdentifier var = context.newVariable();
+ VariableExpr v = new VariableExpr(var);
+ LetClause c1 = new LetClause(v, body);
+ ArrayList<Clause> clauseList = new ArrayList<Clause>(1);
+ clauseList.add(c1);
+ FLWOGRExpression newBody = new FLWOGRExpression(clauseList, new VariableExpr(var));
+ topExpr.setBody(newBody);
+ }
+ }
+
+ private void inlineDeclaredUdfs() throws AsterixException {
+ if (topExpr == null) {
+ return;
+ }
+ List<FunctionSignature> funIds = new ArrayList<FunctionSignature>();
+ for (FunctionDecl fdecl : declaredFunctions) {
+ funIds.add(fdecl.getSignature());
+ }
+
+ List<FunctionDecl> otherFDecls = new ArrayList<FunctionDecl>();
+ buildOtherUdfs(topExpr.getBody(), otherFDecls, funIds);
+ declaredFunctions.addAll(otherFDecls);
+ if (!declaredFunctions.isEmpty()) {
+ AQLInlineUdfsVisitor visitor = new AQLInlineUdfsVisitor(context);
+ while (topExpr.accept(visitor, declaredFunctions)) {
+ // loop until no more changes
+ }
+ }
+ declaredFunctions.removeAll(otherFDecls);
+ }
+
+ private void buildOtherUdfs(Expression expression, List<FunctionDecl> functionDecls,
+ List<FunctionSignature> declaredFunctions) throws AsterixException {
+ if (expression == null) {
+ return;
+ }
+ String value = metadataProvider.getConfig().get(FunctionUtils.IMPORT_PRIVATE_FUNCTIONS);
+ boolean includePrivateFunctions = (value != null) ? Boolean.valueOf(value.toLowerCase()) : false;
+ Set<FunctionSignature> functionCalls = getFunctionCalls(expression);
+ for (FunctionSignature signature : functionCalls) {
+
+ if (declaredFunctions != null && declaredFunctions.contains(signature)) {
+ continue;
+ }
+
+ Function function = lookupUserDefinedFunctionDecl(signature);
+ if (function == null) {
+ if (AsterixBuiltinFunctions.isBuiltinCompilerFunction(signature, includePrivateFunctions)) {
+ continue;
+ }
+ StringBuilder messageBuilder = new StringBuilder();
+ if (functionDecls.size() > 0) {
+ messageBuilder.append(" function " + functionDecls.get(functionDecls.size() - 1).getSignature()
+ + " depends upon function " + signature + " which is undefined");
+ } else {
+ messageBuilder.append(" function " + signature + " is undefined ");
+ }
+ throw new AsterixException(messageBuilder.toString());
+ }
+
+ if (function.getLanguage().equalsIgnoreCase(Function.LANGUAGE_AQL)) {
+ FunctionDecl functionDecl = FunctionUtils.getFunctionDecl(function);
+ if (functionDecl != null) {
+ if (functionDecls.contains(functionDecl)) {
+ throw new AsterixException("ERROR:Recursive invocation "
+ + functionDecls.get(functionDecls.size() - 1).getSignature() + " <==> "
+ + functionDecl.getSignature());
+ }
+ functionDecls.add(functionDecl);
+ buildOtherUdfs(functionDecl.getFuncBody(), functionDecls, declaredFunctions);
+ }
+ }
+ }
+
+ }
+
+ private Function lookupUserDefinedFunctionDecl(FunctionSignature signature) throws AsterixException {
+ if (signature.getNamespace() == null) {
+ return null;
+ }
+ return MetadataManager.INSTANCE.getFunction(mdTxnCtx, signature);
+ }
+
+ private Set<FunctionSignature> getFunctionCalls(Expression expression) throws AsterixException {
+ GatherFunctionCalls gfc = new GatherFunctionCalls();
+ expression.accept(gfc, null);
+ return gfc.getCalls();
+ }
+
+ private static class GatherFunctionCalls extends GatherFunctionCallsVisitor implements IAQLVisitor<Void, Void> {
+
+ public GatherFunctionCalls() {
+ }
+
+ @Override
+ public Void visit(DistinctClause dc, Void arg) throws AsterixException {
+ for (Expression e : dc.getDistinctByExpr()) {
+ e.accept(this, arg);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visit(FLWOGRExpression flwor, Void arg) throws AsterixException {
+ for (Clause c : flwor.getClauseList()) {
+ c.accept(this, arg);
+ }
+ flwor.getReturnExpr().accept(this, arg);
+ return null;
+ }
+
+ @Override
+ public Void visit(ForClause fc, Void arg) throws AsterixException {
+ fc.getInExpr().accept(this, arg);
+ if (fc.getPosVarExpr() != null) {
+ fc.getPosVarExpr().accept(this, arg);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visit(GroupbyClause gc, Void arg) throws AsterixException {
+ for (GbyVariableExpressionPair p : gc.getGbyPairList()) {
+ p.getExpr().accept(this, arg);
+ }
+ for (GbyVariableExpressionPair p : gc.getDecorPairList()) {
+ p.getExpr().accept(this, arg);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visit(LetClause lc, Void arg) throws AsterixException {
+ lc.getBindingExpr().accept(this, arg);
+ return null;
+ }
+
+ @Override
+ public Void visit(UnionExpr u, Void arg) throws AsterixException {
+ for (Expression e : u.getExprs()) {
+ e.accept(this, arg);
+ }
+ return null;
+ }
+ }
+}
diff --git a/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/statement/SubscribeFeedStatement.java b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/statement/SubscribeFeedStatement.java
new file mode 100644
index 0000000..a86c409
--- /dev/null
+++ b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/statement/SubscribeFeedStatement.java
@@ -0,0 +1,209 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.aql.statement;
+
+import java.io.StringReader;
+import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.feeds.FeedActivity;
+import org.apache.asterix.common.feeds.FeedConnectionRequest;
+import org.apache.asterix.common.feeds.FeedId;
+import org.apache.asterix.common.feeds.FeedPolicyAccessor;
+import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.lang.aql.parser.AQLParser;
+import org.apache.asterix.lang.aql.parser.ParseException;
+import org.apache.asterix.lang.aql.util.FunctionUtils;
+import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.lang.common.statement.InsertStatement;
+import org.apache.asterix.lang.common.statement.Query;
+import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
+import org.apache.asterix.metadata.MetadataException;
+import org.apache.asterix.metadata.MetadataManager;
+import org.apache.asterix.metadata.MetadataTransactionContext;
+import org.apache.asterix.metadata.entities.DatasourceAdapter.AdapterType;
+import org.apache.asterix.metadata.entities.Feed;
+import org.apache.asterix.metadata.entities.Function;
+import org.apache.asterix.metadata.entities.PrimaryFeed;
+import org.apache.asterix.metadata.entities.SecondaryFeed;
+import org.apache.asterix.metadata.feeds.FeedUtil;
+import org.apache.asterix.metadata.feeds.IFeedAdapterFactory;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import org.apache.hyracks.algebricks.common.utils.Triple;
+
+/**
+ * Represents the AQL statement for subscribing to a feed.
+ * This AQL statement is private and may not be used by the end-user.
+ */
+public class SubscribeFeedStatement implements Statement {
+
+ private static final Logger LOGGER = Logger.getLogger(SubscribeFeedStatement.class.getName());
+ private final FeedConnectionRequest connectionRequest;
+ private Query query;
+ private int varCounter;
+ private final String[] locations;
+
+ public static final String WAIT_FOR_COMPLETION = "wait-for-completion-feed";
+
+ public SubscribeFeedStatement(String[] locations, FeedConnectionRequest subscriptionRequest) {
+ this.connectionRequest = subscriptionRequest;
+ this.varCounter = 0;
+ this.locations = locations;
+ }
+
+ public void initialize(MetadataTransactionContext mdTxnCtx) throws MetadataException {
+ this.query = new Query();
+ FeedId sourceFeedId = connectionRequest.getFeedJointKey().getFeedId();
+ Feed subscriberFeed = MetadataManager.INSTANCE.getFeed(mdTxnCtx,
+ connectionRequest.getReceivingFeedId().getDataverse(),
+ connectionRequest.getReceivingFeedId().getFeedName());
+ if (subscriberFeed == null) {
+ throw new IllegalStateException(" Subscriber feed " + subscriberFeed + " not found.");
+ }
+
+ String feedOutputType = getOutputType(mdTxnCtx);
+ FunctionSignature appliedFunction = subscriberFeed.getAppliedFunction();
+ Function function = null;
+ if (appliedFunction != null) {
+ function = MetadataManager.INSTANCE.getFunction(mdTxnCtx, appliedFunction);
+ if (function == null) {
+ throw new MetadataException(" Unknown function " + function);
+ } else if (function.getParams().size() > 1) {
+ throw new MetadataException(
+ " Incompatible function: " + appliedFunction + " Number if arguments must be 1");
+ }
+ }
+
+ StringBuilder builder = new StringBuilder();
+ builder.append("use dataverse " + sourceFeedId.getDataverse() + ";\n");
+ builder.append("set" + " " + FunctionUtils.IMPORT_PRIVATE_FUNCTIONS + " " + "'" + Boolean.TRUE + "'" + ";\n");
+ builder.append("set" + " " + FeedActivity.FeedActivityDetails.FEED_POLICY_NAME + " " + "'"
+ + connectionRequest.getPolicy() + "'" + ";\n");
+
+ builder.append("insert into dataset " + connectionRequest.getTargetDataset() + " ");
+ builder.append(" (" + " for $x in feed-collect ('" + sourceFeedId.getDataverse() + "'" + "," + "'"
+ + sourceFeedId.getFeedName() + "'" + "," + "'" + connectionRequest.getReceivingFeedId().getFeedName()
+ + "'" + "," + "'" + connectionRequest.getSubscriptionLocation().name() + "'" + "," + "'"
+ + connectionRequest.getTargetDataset() + "'" + "," + "'" + feedOutputType + "'" + ")");
+
+ List<String> functionsToApply = connectionRequest.getFunctionsToApply();
+ if (functionsToApply != null && functionsToApply.isEmpty()) {
+ builder.append(" return $x");
+ } else {
+ String rValueName = "x";
+ String lValueName = "y";
+ int variableIndex = 0;
+ for (String functionName : functionsToApply) {
+ function = MetadataManager.INSTANCE.getFunction(mdTxnCtx, appliedFunction);
+ variableIndex++;
+ switch (function.getLanguage().toUpperCase()) {
+ case Function.LANGUAGE_AQL:
+ builder.append(
+ " let " + "$" + lValueName + variableIndex + ":=(" + function.getFunctionBody() + ")");
+ builder.append("\n");
+ break;
+ case Function.LANGUAGE_JAVA:
+ builder.append(" let " + "$" + lValueName + variableIndex + ":=" + functionName + "(" + "$"
+ + rValueName + ")");
+ rValueName = lValueName + variableIndex;
+ break;
+ }
+ builder.append("\n");
+ }
+ builder.append("return $" + lValueName + variableIndex);
+ }
+ builder.append(")");
+ builder.append(";");
+ if (LOGGER.isLoggable(Level.INFO)) {
+ LOGGER.info("Connect feed statement translated to\n" + builder.toString());
+ }
+ AQLParser parser = new AQLParser(new StringReader(builder.toString()));
+
+ List<Statement> statements;
+ try {
+ statements = parser.Statement();
+ query = ((InsertStatement) statements.get(3)).getQuery();
+ } catch (ParseException pe) {
+ throw new MetadataException(pe);
+ }
+
+ }
+
+ public Query getQuery() {
+ return query;
+ }
+
+ public int getVarCounter() {
+ return varCounter;
+ }
+
+ @Override
+ public Kind getKind() {
+ return Kind.SUBSCRIBE_FEED;
+ }
+
+ public String getPolicy() {
+ return connectionRequest.getPolicy();
+ }
+
+ public FeedConnectionRequest getSubscriptionRequest() {
+ return connectionRequest;
+ }
+
+ @Override
+ public <R, T> R accept(ILangVisitor<R, T> visitor, T arg) throws AsterixException {
+ return null;
+ }
+
+ public String getDataverseName() {
+ return connectionRequest.getReceivingFeedId().getDataverse();
+ }
+
+ private String getOutputType(MetadataTransactionContext mdTxnCtx) throws MetadataException {
+ String outputType = null;
+ FeedId feedId = connectionRequest.getReceivingFeedId();
+ Feed feed = MetadataManager.INSTANCE.getFeed(mdTxnCtx, feedId.getDataverse(), feedId.getFeedName());
+ FeedPolicyAccessor policyAccessor = new FeedPolicyAccessor(connectionRequest.getPolicyParameters());
+ try {
+ switch (feed.getFeedType()) {
+ case PRIMARY:
+ Triple<IFeedAdapterFactory, ARecordType, AdapterType> factoryOutput = null;
+
+ factoryOutput = FeedUtil.getPrimaryFeedFactoryAndOutput((PrimaryFeed) feed, policyAccessor,
+ mdTxnCtx);
+ outputType = factoryOutput.second.getTypeName();
+ break;
+ case SECONDARY:
+ outputType = FeedUtil.getSecondaryFeedOutput((SecondaryFeed) feed, policyAccessor, mdTxnCtx);
+ break;
+ }
+ return outputType;
+
+ } catch (AlgebricksException ae) {
+ throw new MetadataException(ae);
+ }
+ }
+
+ public String[] getLocations() {
+ return locations;
+ }
+}
diff --git a/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/util/AQLFormatPrintUtil.java b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/util/AQLFormatPrintUtil.java
new file mode 100644
index 0000000..c97e642
--- /dev/null
+++ b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/util/AQLFormatPrintUtil.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.aql.util;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintWriter;
+import java.util.List;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.lang.aql.visitor.AQLFormatPrintVisitor;
+import org.apache.asterix.lang.aql.visitor.AQLToSQLPPPrintVisitor;
+import org.apache.asterix.lang.common.base.ILangExpression;
+import org.apache.asterix.lang.common.base.Statement;
+
+public class AQLFormatPrintUtil {
+
+ public static void print(ILangExpression expr, PrintWriter output) throws AsterixException {
+ AQLFormatPrintVisitor visitor = new AQLFormatPrintVisitor(output);
+ expr.accept(visitor, 0);
+ }
+
+ public static void print(List<Statement> exprs, PrintWriter output) throws AsterixException {
+ AQLFormatPrintVisitor visitor = new AQLFormatPrintVisitor(output);
+ for (Statement expr : exprs) {
+ expr.accept(visitor, 0);
+ }
+ }
+
+ public static String toString(List<Statement> exprs) throws AsterixException {
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintWriter output = new PrintWriter(bos);
+ AQLFormatPrintVisitor visitor = new AQLFormatPrintVisitor(output);
+ for (Statement expr : exprs) {
+ expr.accept(visitor, 0);
+ }
+ output.close();
+ return new String(bos.toByteArray());
+ }
+
+ public static String toSQLPPString(List<Statement> exprs) throws AsterixException {
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ PrintWriter output = new PrintWriter(bos);
+ AQLToSQLPPPrintVisitor visitor = new AQLToSQLPPPrintVisitor(output);
+ for (Statement expr : exprs) {
+ expr.accept(visitor, 0);
+ }
+ output.close();
+ return new String(bos.toByteArray());
+ }
+}
diff --git a/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/util/AQLVariableSubstitutionUtil.java b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/util/AQLVariableSubstitutionUtil.java
new file mode 100644
index 0000000..06e6fa7
--- /dev/null
+++ b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/util/AQLVariableSubstitutionUtil.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.aql.util;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.lang.aql.visitor.AQLCloneAndSubstituteVariablesVisitor;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.ILangExpression;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
+import org.apache.asterix.lang.common.rewrites.VariableSubstitutionEnvironment;
+
+public class AQLVariableSubstitutionUtil {
+
+ public static List<ILangExpression> substituteVariable(List<ILangExpression> expressions,
+ Map<VariableExpr, Expression> varExprMap) throws AsterixException {
+ AQLCloneAndSubstituteVariablesVisitor visitor = new AQLCloneAndSubstituteVariablesVisitor(
+ new LangRewritingContext(0));
+ VariableSubstitutionEnvironment env = new VariableSubstitutionEnvironment(varExprMap);
+ List<ILangExpression> newExprs = new ArrayList<ILangExpression>();
+ for (ILangExpression expression : expressions) {
+ newExprs.add(expression.accept(visitor, env).first);
+ }
+ return newExprs;
+ }
+
+ public static ILangExpression substituteVariable(ILangExpression expression,
+ Map<VariableExpr, Expression> varExprMap) throws AsterixException {
+ AQLCloneAndSubstituteVariablesVisitor visitor = new AQLCloneAndSubstituteVariablesVisitor(
+ new LangRewritingContext(0));
+ VariableSubstitutionEnvironment env = new VariableSubstitutionEnvironment(varExprMap);
+ return expression.accept(visitor, env).first;
+ }
+
+}
diff --git a/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/util/FunctionUtils.java b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/util/FunctionUtils.java
new file mode 100644
index 0000000..2f326ae
--- /dev/null
+++ b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/util/FunctionUtils.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.lang.aql.util;
+
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.lang.aql.parser.AQLParser;
+import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.lang.common.statement.FunctionDecl;
+import org.apache.asterix.lang.common.struct.VarIdentifier;
+import org.apache.asterix.metadata.entities.Function;
+import org.apache.asterix.om.functions.AsterixBuiltinFunctions;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
+
+public class FunctionUtils {
+
+ public static final String IMPORT_PRIVATE_FUNCTIONS = "import-private-functions";
+
+ public static FunctionDecl getFunctionDecl(Function function) throws AsterixException {
+ String functionBody = function.getFunctionBody();
+ List<String> params = function.getParams();
+ List<VarIdentifier> varIdentifiers = new ArrayList<VarIdentifier>();
+
+ StringBuilder builder = new StringBuilder();
+ builder.append(" use dataverse " + function.getDataverseName() + ";");
+ builder.append(" declare function " + function.getName().split("@")[0]);
+ builder.append("(");
+ for (String param : params) {
+ VarIdentifier varId = new VarIdentifier(param);
+ varIdentifiers.add(varId);
+ builder.append(param);
+ builder.append(",");
+ }
+ if (params.size() > 0) {
+ builder.delete(builder.length() - 1, builder.length());
+ }
+ builder.append(")");
+ builder.append("{");
+ builder.append("\n");
+ builder.append(functionBody);
+ builder.append("\n");
+ builder.append("}");
+
+ AQLParser parser = new AQLParser(new StringReader(new String(builder)));
+
+ List<Statement> statements = parser.parse();
+ FunctionDecl decl = (FunctionDecl) statements.get(1);
+ return decl;
+ }
+
+ public static IFunctionInfo getFunctionInfo(FunctionIdentifier fi) {
+ return AsterixBuiltinFunctions.getAsterixFunctionInfo(fi);
+ }
+
+ public static IFunctionInfo getFunctionInfo(FunctionSignature fs) {
+ return getFunctionInfo(new FunctionIdentifier(fs.getNamespace(), fs.getName(), fs.getArity()));
+ }
+
+}
diff --git a/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/util/RangeMapBuilder.java b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/util/RangeMapBuilder.java
new file mode 100644
index 0000000..a15fb45
--- /dev/null
+++ b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/util/RangeMapBuilder.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.aql.util;
+
+import java.io.DataOutput;
+import java.util.List;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.formats.nontagged.AqlBinaryComparatorFactoryProvider;
+import org.apache.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
+import org.apache.asterix.lang.aql.parser.AQLParser;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.Expression.Kind;
+import org.apache.asterix.lang.common.base.Literal;
+import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.lang.common.expression.ListConstructor;
+import org.apache.asterix.lang.common.expression.LiteralExpr;
+import org.apache.asterix.lang.common.literal.DoubleLiteral;
+import org.apache.asterix.lang.common.literal.FloatLiteral;
+import org.apache.asterix.lang.common.literal.IntegerLiteral;
+import org.apache.asterix.lang.common.literal.LongIntegerLiteral;
+import org.apache.asterix.lang.common.literal.StringLiteral;
+import org.apache.asterix.lang.common.statement.Query;
+import org.apache.asterix.om.base.AMutableDouble;
+import org.apache.asterix.om.base.AMutableFloat;
+import org.apache.asterix.om.base.AMutableInt32;
+import org.apache.asterix.om.base.AMutableInt64;
+import org.apache.asterix.om.base.AMutableString;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.hyracks.algebricks.common.exceptions.NotImplementedException;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
+import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.partition.range.IRangeMap;
+import org.apache.hyracks.dataflow.common.data.partition.range.RangeMap;
+
+public abstract class RangeMapBuilder {
+
+ public static IRangeMap parseHint(Object hint) throws AsterixException {
+ ArrayBackedValueStorage abvs = new ArrayBackedValueStorage();
+ DataOutput out = abvs.getDataOutput();;
+ abvs.reset();
+
+ AQLParser parser = new AQLParser((String) hint);
+ List<Statement> hintStatements = parser.parse();
+ if (hintStatements.size() != 1) {
+ throw new AsterixException("Only one range statement is allowed for the range hint.");
+ }
+
+ // Translate the query into a Range Map
+ if (hintStatements.get(0).getKind() != Statement.Kind.QUERY) {
+ throw new AsterixException("Not a proper query for the range hint.");
+ }
+ Query q = (Query) hintStatements.get(0);
+
+ if (q.getBody().getKind() != Kind.LIST_CONSTRUCTOR_EXPRESSION) {
+ throw new AsterixException("The range hint must be a list.");
+ }
+ List<Expression> el = ((ListConstructor) q.getBody()).getExprList();
+ int offsets[] = new int[el.size()];
+
+ // Loop over list of literals
+ for (int i = 0; i < el.size(); ++i) {
+ Expression item = el.get(i);
+ if (item.getKind() == Kind.LITERAL_EXPRESSION) {
+ parseLiteralToBytes(item, out);
+ offsets[i] = abvs.getLength();
+ }
+ // TODO Add support for composite fields.
+ }
+
+ return new RangeMap(1, abvs.getByteArray(), offsets);
+ }
+
+ @SuppressWarnings("unchecked")
+ private static void parseLiteralToBytes(Expression item, DataOutput out) throws AsterixException {
+ AMutableDouble aDouble = new AMutableDouble(0);
+ AMutableFloat aFloat = new AMutableFloat(0);
+ AMutableInt64 aInt64 = new AMutableInt64(0);
+ AMutableInt32 aInt32 = new AMutableInt32(0);
+ AMutableString aString = new AMutableString("");
+ @SuppressWarnings("rawtypes")
+ ISerializerDeserializer serde;
+
+ Literal l = ((LiteralExpr) item).getValue();
+ try {
+ switch (l.getLiteralType()) {
+ case DOUBLE:
+ DoubleLiteral dl = (DoubleLiteral) l;
+ serde = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ADOUBLE);
+ aDouble.setValue(dl.getValue());
+ serde.serialize(aDouble, out);
+ break;
+ case FLOAT:
+ FloatLiteral fl = (FloatLiteral) l;
+ serde = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AFLOAT);
+ aFloat.setValue(fl.getValue());
+ serde.serialize(aFloat, out);
+ break;
+ case INTEGER:
+ IntegerLiteral il = (IntegerLiteral) l;
+ serde = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT32);
+ aInt32.setValue(il.getValue());
+ serde.serialize(aInt32, out);
+ break;
+ case LONG:
+ LongIntegerLiteral lil = (LongIntegerLiteral) l;
+ serde = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT64);
+ aInt64.setValue(lil.getValue());
+ serde.serialize(aInt64, out);
+ break;
+ case STRING:
+ StringLiteral sl = (StringLiteral) l;
+ serde = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ASTRING);
+ aString.setValue(sl.getValue());
+ serde.serialize(aString, out);
+ break;
+ default:
+ throw new NotImplementedException("The range map builder has not been implemented for "
+ + item.getKind() + " type of expressions.");
+ }
+ } catch (HyracksDataException e) {
+ throw new AsterixException(e.getMessage());
+ }
+ }
+
+ public static void verifyRangeOrder(IRangeMap rangeMap, boolean ascending) throws AsterixException {
+ // TODO Add support for composite fields.
+ int fieldIndex = 0;
+ int fieldType = rangeMap.getTag(0, 0);
+ AqlBinaryComparatorFactoryProvider comparatorFactory = AqlBinaryComparatorFactoryProvider.INSTANCE;
+ IBinaryComparatorFactory bcf = comparatorFactory
+ .getBinaryComparatorFactory(ATypeTag.VALUE_TYPE_MAPPING[fieldType], ascending);
+ IBinaryComparator comparator = bcf.createBinaryComparator();
+ int c = 0;
+ for (int split = 1; split < rangeMap.getSplitCount(); ++split) {
+ if (fieldType != rangeMap.getTag(fieldIndex, split)) {
+ throw new AsterixException("Range field contains more than a single type of items (" + fieldType
+ + " and " + rangeMap.getTag(fieldIndex, split) + ").");
+ }
+ int previousSplit = split - 1;
+ try {
+ c = comparator.compare(rangeMap.getByteArray(fieldIndex, previousSplit),
+ rangeMap.getStartOffset(fieldIndex, previousSplit),
+ rangeMap.getLength(fieldIndex, previousSplit), rangeMap.getByteArray(fieldIndex, split),
+ rangeMap.getStartOffset(fieldIndex, split), rangeMap.getLength(fieldIndex, split));
+ } catch (HyracksDataException e) {
+ throw new AsterixException(e);
+ }
+ if (c >= 0) {
+ throw new AsterixException("Range fields are not in sorted order.");
+ }
+ }
+ }
+}
diff --git a/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/visitor/AQLCloneAndSubstituteVariablesVisitor.java b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/visitor/AQLCloneAndSubstituteVariablesVisitor.java
new file mode 100644
index 0000000..511354e
--- /dev/null
+++ b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/visitor/AQLCloneAndSubstituteVariablesVisitor.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.aql.visitor;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.lang.aql.clause.DistinctClause;
+import org.apache.asterix.lang.aql.clause.ForClause;
+import org.apache.asterix.lang.aql.expression.FLWOGRExpression;
+import org.apache.asterix.lang.aql.expression.UnionExpr;
+import org.apache.asterix.lang.aql.visitor.base.IAQLVisitor;
+import org.apache.asterix.lang.common.base.Clause;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.ILangExpression;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
+import org.apache.asterix.lang.common.rewrites.VariableSubstitutionEnvironment;
+import org.apache.asterix.lang.common.utils.VariableCloneAndSubstitutionUtil;
+import org.apache.asterix.lang.common.visitor.CloneAndSubstituteVariablesVisitor;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+
+public class AQLCloneAndSubstituteVariablesVisitor extends CloneAndSubstituteVariablesVisitor implements
+ IAQLVisitor<Pair<ILangExpression, VariableSubstitutionEnvironment>, VariableSubstitutionEnvironment> {
+
+ private LangRewritingContext context;
+
+ public AQLCloneAndSubstituteVariablesVisitor(LangRewritingContext context) {
+ super(context);
+ this.context = context;
+ }
+
+ @Override
+ public Pair<ILangExpression, VariableSubstitutionEnvironment> visit(ForClause fc,
+ VariableSubstitutionEnvironment env) throws AsterixException {
+ Pair<ILangExpression, VariableSubstitutionEnvironment> p1 = fc.getInExpr().accept(this, env);
+ VariableExpr varExpr = fc.getVarExpr();
+ VariableExpr newVe = generateNewVariable(context, varExpr);
+ VariableSubstitutionEnvironment resultEnv = new VariableSubstitutionEnvironment(env);
+ resultEnv.removeSubstitution(varExpr);
+
+ VariableExpr posVarExpr = null;
+ if (fc.hasPosVar()) {
+ posVarExpr = fc.getPosVarExpr();
+ resultEnv.removeSubstitution(posVarExpr);
+ }
+ ForClause newFor = new ForClause(newVe, (Expression) p1.first, posVarExpr);
+ return new Pair<ILangExpression, VariableSubstitutionEnvironment>(newFor, resultEnv);
+ }
+
+ @Override
+ public Pair<ILangExpression, VariableSubstitutionEnvironment> visit(FLWOGRExpression flwor,
+ VariableSubstitutionEnvironment env) throws AsterixException {
+ List<Clause> newClauses = new ArrayList<Clause>(flwor.getClauseList().size());
+ VariableSubstitutionEnvironment currentEnv = env;
+ for (Clause c : flwor.getClauseList()) {
+ Pair<ILangExpression, VariableSubstitutionEnvironment> p1 = c.accept(this, currentEnv);
+ currentEnv = p1.second;
+ newClauses.add((Clause) p1.first);
+ }
+ Pair<ILangExpression, VariableSubstitutionEnvironment> p2 = flwor.getReturnExpr().accept(this, currentEnv);
+ Expression newReturnExpr = (Expression) p2.first;
+ FLWOGRExpression newFlwor = new FLWOGRExpression(newClauses, newReturnExpr);
+ return new Pair<ILangExpression, VariableSubstitutionEnvironment>(newFlwor, p2.second);
+ }
+
+ @Override
+ public Pair<ILangExpression, VariableSubstitutionEnvironment> visit(UnionExpr u,
+ VariableSubstitutionEnvironment env) throws AsterixException {
+ List<Expression> exprList = VariableCloneAndSubstitutionUtil.visitAndCloneExprList(u.getExprs(), env, this);
+ UnionExpr newU = new UnionExpr(exprList);
+ return new Pair<ILangExpression, VariableSubstitutionEnvironment>(newU, env);
+ }
+
+ @Override
+ public Pair<ILangExpression, VariableSubstitutionEnvironment> visit(DistinctClause dc,
+ VariableSubstitutionEnvironment env) throws AsterixException {
+ List<Expression> exprList = VariableCloneAndSubstitutionUtil.visitAndCloneExprList(dc.getDistinctByExpr(), env,
+ this);
+ DistinctClause dc2 = new DistinctClause(exprList);
+ return new Pair<ILangExpression, VariableSubstitutionEnvironment>(dc2, env);
+ }
+}
\ No newline at end of file
diff --git a/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/visitor/AQLFormatPrintVisitor.java b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/visitor/AQLFormatPrintVisitor.java
new file mode 100644
index 0000000..ae186ab
--- /dev/null
+++ b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/visitor/AQLFormatPrintVisitor.java
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.aql.visitor;
+
+import java.io.PrintWriter;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.lang.aql.clause.DistinctClause;
+import org.apache.asterix.lang.aql.clause.ForClause;
+import org.apache.asterix.lang.aql.expression.FLWOGRExpression;
+import org.apache.asterix.lang.aql.expression.UnionExpr;
+import org.apache.asterix.lang.aql.visitor.base.IAQLVisitor;
+import org.apache.asterix.lang.common.base.Clause;
+import org.apache.asterix.lang.common.visitor.FormatPrintVisitor;
+
+public class AQLFormatPrintVisitor extends FormatPrintVisitor implements IAQLVisitor<Void, Integer> {
+
+ private final PrintWriter out;
+
+ public AQLFormatPrintVisitor() {
+ super();
+ out = new PrintWriter(System.out);
+ }
+
+ public AQLFormatPrintVisitor(PrintWriter out) {
+ super(out);
+ this.out = out;
+ }
+
+ @Override
+ public Void visit(FLWOGRExpression flwor, Integer step) throws AsterixException {
+ for (Clause cl : flwor.getClauseList()) {
+ cl.accept(this, step);
+ }
+ out.print(skip(step) + "return ");
+ flwor.getReturnExpr().accept(this, step + 2);
+ return null;
+ }
+
+ @Override
+ public Void visit(ForClause fc, Integer step) throws AsterixException {
+ out.print("for ");
+ fc.getVarExpr().accept(this, step + 2);
+ if (fc.hasPosVar()) {
+ out.print(" at ");
+ fc.getPosVarExpr().accept(this, step + 2);
+ }
+ out.print(" in ");
+ fc.getInExpr().accept(this, step + 2);
+ out.println();
+ return null;
+ }
+
+ @Override
+ public Void visit(UnionExpr u, Integer step) throws AsterixException {
+ printDelimitedExpressions(u.getExprs(), "\n" + skip(step) + "union\n", step);
+ return null;
+ }
+
+ @Override
+ public Void visit(DistinctClause dc, Integer step) throws AsterixException {
+ out.print(skip(step) + "distinct by ");
+ printDelimitedExpressions(dc.getDistinctByExpr(), COMMA, step + 2);
+ out.println();
+ return null;
+ }
+
+}
diff --git a/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/visitor/AQLInlineUdfsVisitor.java b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/visitor/AQLInlineUdfsVisitor.java
new file mode 100644
index 0000000..b53d888
--- /dev/null
+++ b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/visitor/AQLInlineUdfsVisitor.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.aql.visitor;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.lang.aql.clause.DistinctClause;
+import org.apache.asterix.lang.aql.clause.ForClause;
+import org.apache.asterix.lang.aql.expression.FLWOGRExpression;
+import org.apache.asterix.lang.aql.expression.UnionExpr;
+import org.apache.asterix.lang.aql.visitor.base.IAQLVisitor;
+import org.apache.asterix.lang.common.base.Clause;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.clause.LetClause;
+import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
+import org.apache.asterix.lang.common.statement.FunctionDecl;
+import org.apache.asterix.lang.common.visitor.AbstractInlineUdfsVisitor;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+
+public class AQLInlineUdfsVisitor extends AbstractInlineUdfsVisitor
+ implements IAQLVisitor<Boolean, List<FunctionDecl>> {
+
+ public AQLInlineUdfsVisitor(LangRewritingContext context) {
+ super(context, new AQLCloneAndSubstituteVariablesVisitor(context));
+ }
+
+ @Override
+ public Boolean visit(FLWOGRExpression flwor, List<FunctionDecl> arg) throws AsterixException {
+ boolean changed = false;
+ for (Clause c : flwor.getClauseList()) {
+ if (c.accept(this, arg)) {
+ changed = true;
+ }
+ }
+ Pair<Boolean, Expression> p = inlineUdfsInExpr(flwor.getReturnExpr(), arg);
+ flwor.setReturnExpr(p.second);
+ return changed || p.first;
+ }
+
+ @Override
+ public Boolean visit(ForClause fc, List<FunctionDecl> arg) throws AsterixException {
+ Pair<Boolean, Expression> p = inlineUdfsInExpr(fc.getInExpr(), arg);
+ fc.setInExpr(p.second);
+ return p.first;
+ }
+
+ @Override
+ public Boolean visit(UnionExpr u, List<FunctionDecl> fds) throws AsterixException {
+ Pair<Boolean, ArrayList<Expression>> p = inlineUdfsInExprList(u.getExprs(), fds);
+ u.setExprs(p.second);
+ return p.first;
+ }
+
+ @Override
+ public Boolean visit(DistinctClause dc, List<FunctionDecl> arg) throws AsterixException {
+ boolean changed = false;
+ for (Expression expr : dc.getDistinctByExpr()) {
+ changed = expr.accept(this, arg);
+ }
+ return changed;
+ }
+
+ @Override
+ protected Expression generateQueryExpression(List<LetClause> letClauses, Expression returnExpr) {
+ List<Clause> letList = new ArrayList<Clause>();
+ letList.addAll(letClauses);
+ return new FLWOGRExpression(letList, returnExpr);
+ }
+}
diff --git a/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/visitor/AQLPrintVisitor.java b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/visitor/AQLPrintVisitor.java
new file mode 100644
index 0000000..2798ef8
--- /dev/null
+++ b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/visitor/AQLPrintVisitor.java
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.aql.visitor;
+
+import java.io.PrintWriter;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.lang.aql.clause.DistinctClause;
+import org.apache.asterix.lang.aql.clause.ForClause;
+import org.apache.asterix.lang.aql.expression.FLWOGRExpression;
+import org.apache.asterix.lang.aql.expression.UnionExpr;
+import org.apache.asterix.lang.aql.visitor.base.IAQLVisitor;
+import org.apache.asterix.lang.common.base.Clause;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.visitor.QueryPrintVisitor;
+
+public class AQLPrintVisitor extends QueryPrintVisitor implements IAQLVisitor<Void, Integer> {
+
+ private final PrintWriter out;
+
+ public AQLPrintVisitor() {
+ super();
+ out = new PrintWriter(System.out);
+ }
+
+ public AQLPrintVisitor(PrintWriter out) {
+ super(out);
+ this.out = out;
+ }
+
+ @Override
+ public Void visit(FLWOGRExpression flwor, Integer step) throws AsterixException {
+ out.println(skip(step) + "FLWOGR [");
+ for (Clause cl : flwor.getClauseList()) {
+ cl.accept(this, step + 1);
+ }
+ out.println(skip(step + 1) + "Return");
+ flwor.getReturnExpr().accept(this, step + 2);
+ out.println(skip(step) + "]");
+ return null;
+ }
+
+ @Override
+ public Void visit(ForClause fc, Integer step) throws AsterixException {
+ out.print(skip(step) + "For ");
+ fc.getVarExpr().accept(this, 0);
+ out.println(skip(step + 1) + "In ");
+ fc.getInExpr().accept(this, step + 1);
+ return null;
+ }
+
+ @Override
+ public Void visit(UnionExpr u, Integer step) throws AsterixException {
+ out.println(skip(step) + "Union [");
+ for (Expression expr : u.getExprs()) {
+ expr.accept(this, step + 1);
+ }
+ out.println(skip(step) + "]");
+ return null;
+ }
+
+ @Override
+ public Void visit(DistinctClause dc, Integer step) throws AsterixException {
+ out.print(skip(step) + "Distinct ");
+ for (Expression expr : dc.getDistinctByExpr()) {
+ expr.accept(this, step + 1);
+ }
+ return null;
+ }
+
+}
diff --git a/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/visitor/AQLToSQLPPPrintVisitor.java b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/visitor/AQLToSQLPPPrintVisitor.java
new file mode 100644
index 0000000..ba837b5
--- /dev/null
+++ b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/visitor/AQLToSQLPPPrintVisitor.java
@@ -0,0 +1,632 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.aql.visitor;
+
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.lang.aql.clause.DistinctClause;
+import org.apache.asterix.lang.aql.clause.ForClause;
+import org.apache.asterix.lang.aql.expression.FLWOGRExpression;
+import org.apache.asterix.lang.aql.expression.UnionExpr;
+import org.apache.asterix.lang.aql.util.AQLVariableSubstitutionUtil;
+import org.apache.asterix.lang.aql.visitor.base.IAQLVisitor;
+import org.apache.asterix.lang.common.base.Clause;
+import org.apache.asterix.lang.common.base.Clause.ClauseType;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.Expression.Kind;
+import org.apache.asterix.lang.common.clause.GroupbyClause;
+import org.apache.asterix.lang.common.clause.LetClause;
+import org.apache.asterix.lang.common.clause.WhereClause;
+import org.apache.asterix.lang.common.expression.CallExpr;
+import org.apache.asterix.lang.common.expression.FieldAccessor;
+import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair;
+import org.apache.asterix.lang.common.expression.LiteralExpr;
+import org.apache.asterix.lang.common.expression.OperatorExpr;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.statement.DataverseDecl;
+import org.apache.asterix.lang.common.statement.DeleteStatement;
+import org.apache.asterix.lang.common.statement.InsertStatement;
+import org.apache.asterix.lang.common.statement.Query;
+import org.apache.asterix.lang.common.struct.Identifier;
+import org.apache.asterix.lang.common.struct.VarIdentifier;
+import org.apache.asterix.lang.common.visitor.FormatPrintVisitor;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+
+public class AQLToSQLPPPrintVisitor extends FormatPrintVisitor implements IAQLVisitor<Void, Integer> {
+
+ private final PrintWriter out;
+ private final Set<String> reservedKeywords = new HashSet<String>();
+ private int generatedId = 0;
+
+ public AQLToSQLPPPrintVisitor() {
+ this(new PrintWriter(System.out));
+ }
+
+ public AQLToSQLPPPrintVisitor(PrintWriter out) {
+ super(out);
+ this.out = out;
+ initialize();
+ }
+
+ private void initialize() {
+ dataverseSymbol = " database ";
+ datasetSymbol = " table ";
+ assignSymbol = "=";
+ reservedKeywords.addAll(Arrays.asList(new String[] { "order", "value", "nest", "keyword", "all" }));
+ }
+
+ @Override
+ public Void visit(FLWOGRExpression flwor, Integer step) throws AsterixException {
+ if (step != 0) {
+ out.println("(");
+ }
+ List<Clause> clauseList = new ArrayList<Clause>();
+ clauseList.addAll(flwor.getClauseList());
+
+ // Processes data-independent let clauses.
+ if (hasFor(clauseList)) {
+ processLeadingLetClauses(step, clauseList);
+ }
+
+ // Distill unnecessary order-bys before a group-by.
+ distillRedundantOrderby(clauseList);
+
+ // Correlated "for" clauses after group-by.
+ Pair<GroupbyClause, List<Clause>> extraction = extractUnnestAfterGroupby(clauseList);
+ GroupbyClause cuttingGbyClause = extraction.first;
+ List<Clause> unnestClauseList = extraction.second;
+ Expression returnExpr = flwor.getReturnExpr();
+ if (unnestClauseList.size() == 0) {
+ if (hasFor(clauseList)) {
+ out.print(skip(step) + "select element ");
+ returnExpr.accept(this, step + 2);
+ out.println();
+ } else {
+ // The FLOWGR only contains let-return, then inline let binding expressions into the return expression.
+ Map<VariableExpr, Expression> varExprMap = extractLetBindingVariables(clauseList, cuttingGbyClause);
+ returnExpr = (Expression) AQLVariableSubstitutionUtil.substituteVariable(returnExpr, varExprMap);
+ returnExpr.accept(this, step);
+ return null;
+ }
+ }
+
+ String generated = generateVariableSymbol();
+ if (unnestClauseList.size() > 0) {
+ Map<VariableExpr, Expression> varExprMap = extractDefinedCollectionVariables(clauseList, cuttingGbyClause,
+ generated);
+
+ returnExpr = (Expression) AQLVariableSubstitutionUtil.substituteVariable(returnExpr, varExprMap);
+ List<Clause> newUnnestClauses = new ArrayList<Clause>();
+ for (Clause nestedCl : unnestClauseList) {
+ newUnnestClauses.add((Clause) AQLVariableSubstitutionUtil.substituteVariable(nestedCl, varExprMap));
+ }
+ unnestClauseList = newUnnestClauses;
+
+ out.print(skip(step) + "select element " + (hasDistinct(unnestClauseList) ? "distinct " : ""));
+ returnExpr.accept(this, step + 2);
+ out.println();
+ out.println(skip(step) + "from");
+ out.print(skip(step + 2) + "( select element " + (hasDistinct(clauseList) ? "distinct " : "") + "{");
+ int index = 0;
+ int size = varExprMap.size();
+ for (VariableExpr var : varExprMap.keySet()) {
+ out.print("\'" + var.getVar().getValue().substring(1) + "\':" + var.getVar().getValue().substring(1));
+ if (++index < size) {
+ out.print(COMMA);
+ }
+ }
+ out.println("}");
+ }
+
+ reorder(clauseList);
+ reorder(unnestClauseList);
+
+ mergeConsecutiveWhereClauses(clauseList);
+ mergeConsecutiveWhereClauses(unnestClauseList);
+
+ boolean firstFor = true;
+ boolean firstLet = true;
+ int forStep = unnestClauseList.size() == 0 ? step : step + 3;
+ int size = clauseList.size();
+ // Processes all other clauses, with special printing for consecutive
+ // "for"s.
+ for (int i = 0; i < size; ++i) {
+ Clause cl = clauseList.get(i);
+ if (cl.getClauseType() == ClauseType.FOR_CLAUSE) {
+ boolean hasConsequentFor = false;
+ if (i < size - 1) {
+ Clause nextCl = clauseList.get(i + 1);
+ hasConsequentFor = nextCl.getClauseType() == ClauseType.FOR_CLAUSE;
+ }
+ visitForClause((ForClause) cl, forStep, firstFor, hasConsequentFor);
+ firstFor = false;
+ } else if (cl.getClauseType() == ClauseType.LET_CLAUSE) {
+ boolean hasConsequentLet = false;
+ if (i < size - 1) {
+ Clause nextCl = clauseList.get(i + 1);
+ hasConsequentLet = nextCl.getClauseType() == ClauseType.LET_CLAUSE;
+ }
+ visitLetClause((LetClause) cl, forStep, firstLet, hasConsequentLet);
+ firstLet = false;
+ } else {
+ cl.accept(this, forStep);
+ }
+
+ if (cl.getClauseType() == ClauseType.FROM_CLAUSE || cl.getClauseType() == ClauseType.GROUP_BY_CLAUSE) {
+ firstLet = true;
+ }
+ }
+
+ if (unnestClauseList.size() > 0) {
+ out.println(skip(forStep - 1) + ") as " + generated.substring(1) + ",");
+ for (Clause nestedCl : unnestClauseList) {
+ if (nestedCl.getClauseType() == ClauseType.FOR_CLAUSE) {
+ visitForClause((ForClause) nestedCl, step - 1, firstFor, false);
+ } else {
+ nestedCl.accept(this, step);
+ }
+ }
+ }
+
+ if (step > 0) {
+ out.print(skip(step - 2) + ")");
+ }
+ return null;
+ }
+
+ @Override
+ public Void visit(ForClause fc, Integer step) throws AsterixException {
+ // The processing of a "for" clause depends on its neighbor clauses,
+ // hence the logic goes to visit(FLWOGRExpression).
+ return null;
+ }
+
+ private void visitForClause(ForClause fc, Integer step, boolean startFor, boolean hasConsequentFor)
+ throws AsterixException {
+ if (startFor) {
+ out.print(skip(step) + "from ");
+ } else {
+ out.print(skip(step + 3));
+ }
+ fc.getInExpr().accept(this, step + 2);
+ out.print(" as ");
+ fc.getVarExpr().accept(this, step + 2);
+ if (fc.hasPosVar()) {
+ out.print(" at ");
+ fc.getPosVarExpr().accept(this, step + 2);
+ }
+ if (hasConsequentFor) {
+ out.print(COMMA);
+ }
+ out.println();
+ }
+
+ private void visitLetClause(LetClause lc, Integer step, boolean startLet, boolean hasConsequentLet)
+ throws AsterixException {
+ if (startLet) {
+ out.print(skip(step) + "with ");
+ } else {
+ out.print(skip(step + 3));
+ }
+ lc.getVarExpr().accept(this, step + 3);
+ out.print(" as ");
+ lc.getBindingExpr().accept(this, step + 3);
+ if (hasConsequentLet) {
+ out.print(COMMA);
+ }
+ out.println();
+ }
+
+ @Override
+ public Void visit(Query q, Integer step) throws AsterixException {
+ Expression expr = q.getBody();
+ if (expr != null) {
+ if (expr.getKind() != Kind.FLWOGR_EXPRESSION) {
+ out.print("select element ");
+ expr.accept(this, step + 2);
+ } else {
+ expr.accept(this, step);
+ }
+ }
+ if (q.isTopLevel()) {
+ out.println(SEMICOLON);
+ }
+ return null;
+ }
+
+ @Override
+ public Void visit(DataverseDecl dv, Integer step) throws AsterixException {
+ out.println(skip(step) + "use " + normalize(dv.getDataverseName().getValue()) + ";\n\n");
+ return null;
+ }
+
+ @Override
+ public Void visit(UnionExpr u, Integer step) throws AsterixException {
+ printDelimitedExpressions(u.getExprs(), "\n" + skip(step) + "union\n" + skip(step), step);
+ return null;
+ }
+
+ @Override
+ public Void visit(DistinctClause dc, Integer step) throws AsterixException {
+ return null;
+ }
+
+ @Override
+ public Void visit(VariableExpr v, Integer step) {
+ String varStr = v.getVar().getValue().substring(1);
+ if (reservedKeywords.contains(varStr)) {
+ varStr = varStr + "s";
+ }
+ out.print(varStr);
+ return null;
+ }
+
+ @Override
+ public Void visit(LetClause lc, Integer step) throws AsterixException {
+ out.print(skip(step) + "with ");
+ lc.getVarExpr().accept(this, step + 2);
+ out.print(" as ");
+ Expression bindingExpr = lc.getBindingExpr();
+ bindingExpr.accept(this, step + 2);
+ out.println();
+ return null;
+ }
+
+ @Override
+ public Void visit(CallExpr callExpr, Integer step) throws AsterixException {
+ FunctionSignature signature = callExpr.getFunctionSignature();
+ if (signature.getNamespace() != null && signature.getNamespace().equals("Metadata")
+ && signature.getName().equals("dataset") && signature.getArity() == 1) {
+ LiteralExpr expr = (LiteralExpr) callExpr.getExprList().get(0);
+ out.print(normalize(expr.getValue().getStringValue()));
+ } else {
+ printHints(callExpr.getHints(), step);
+ out.print(generateFullName(callExpr.getFunctionSignature().getNamespace(),
+ callExpr.getFunctionSignature().getName()) + "(");
+ printDelimitedExpressions(callExpr.getExprList(), COMMA, step);
+ out.print(")");
+ }
+ return null;
+ }
+
+ @Override
+ public Void visit(GroupbyClause gc, Integer step) throws AsterixException {
+ if (gc.hasHashGroupByHint()) {
+ out.println(skip(step) + "/* +hash */");
+ }
+ out.print(skip(step) + "group by ");
+ printDelimitedGbyExpressions(gc.getGbyPairList(), step + 2);
+ out.println();
+ return null;
+ }
+
+ @Override
+ public Void visit(InsertStatement insert, Integer step) throws AsterixException {
+ out.print(skip(step) + "insert into " + generateFullName(insert.getDataverseName(), insert.getDatasetName())
+ + "\n");
+ insert.getQuery().accept(this, step);
+ out.println(SEMICOLON);
+ return null;
+ }
+
+ @Override
+ public Void visit(DeleteStatement del, Integer step) throws AsterixException {
+ out.print(skip(step) + "delete ");
+ del.getVariableExpr().accept(this, step + 2);
+ out.println(skip(step) + " from " + generateFullName(del.getDataverseName(), del.getDatasetName()));
+ if (del.getCondition() != null) {
+ out.print(skip(step) + " where ");
+ del.getCondition().accept(this, step + 2);
+ }
+ out.println(SEMICOLON);
+ return null;
+ }
+
+ @Override
+ protected String normalize(String str) {
+ if (needQuotes(str) || containsReservedKeyWord(str.toLowerCase())) {
+ return revertStringToQuoted(str);
+ }
+ return str;
+ }
+
+ protected boolean containsReservedKeyWord(String str) {
+ if (reservedKeywords.contains(str)) {
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ protected void printDelimitedGbyExpressions(List<GbyVariableExpressionPair> gbyList, int step)
+ throws AsterixException {
+ int gbySize = gbyList.size();
+ int gbyIndex = 0;
+ for (GbyVariableExpressionPair pair : gbyList) {
+ pair.getExpr().accept(this, step);
+ if (pair.getVar() != null) {
+ out.print(" as ");
+ pair.getVar().accept(this, step);
+ }
+ if (++gbyIndex < gbySize) {
+ out.print(COMMA);
+ }
+ }
+ }
+
+ @Override
+ protected void printDelimitedIdentifiers(List<Identifier> ids, String delimiter) {
+ int index = 0;
+ int size = ids.size();
+ for (Identifier id : ids) {
+ String idStr = id.getValue();
+ if (idStr.startsWith("$")) {
+ id = new Identifier(idStr.substring(1));
+ }
+ out.print(id);
+ if (++index < size) {
+ out.print(delimiter);
+ }
+ }
+ }
+
+ // Collects produced variables from group-by.
+ private List<VariableExpr> collectProducedVariablesFromGroupby(GroupbyClause gbyClause) {
+ List<VariableExpr> producedVars = new ArrayList<VariableExpr>();
+ for (GbyVariableExpressionPair keyPair : gbyClause.getGbyPairList()) {
+ producedVars.add(keyPair.getVar());
+ }
+ for (GbyVariableExpressionPair keyPair : gbyClause.getDecorPairList()) {
+ producedVars.add(keyPair.getVar());
+ }
+ producedVars.addAll(gbyClause.getWithVarList());
+ return producedVars;
+ }
+
+ // Randomly generates a new variable symbol.
+ private String generateVariableSymbol() {
+ return "$gen" + generatedId++;
+ }
+
+ // Removes all redundant order by clauses.
+ private void distillRedundantOrderby(List<Clause> clauseList) {
+ List<Clause> redundantOrderbys = new ArrayList<Clause>();
+ boolean gbyAfterOrderby = false;
+ for (Clause cl : clauseList) {
+ if (cl.getClauseType() == ClauseType.ORDER_BY_CLAUSE) {
+ redundantOrderbys.add(cl);
+ }
+ if (cl.getClauseType() == ClauseType.GROUP_BY_CLAUSE) {
+ gbyAfterOrderby = true;
+ break;
+ }
+ }
+ if (gbyAfterOrderby) {
+ clauseList.removeAll(redundantOrderbys);
+ }
+
+ redundantOrderbys.clear();
+ for (Clause cl : clauseList) {
+ if (cl.getClauseType() == ClauseType.ORDER_BY_CLAUSE) {
+ redundantOrderbys.add(cl);
+ }
+ }
+ if (redundantOrderbys.size() > 0) {
+ redundantOrderbys.remove(redundantOrderbys.size() - 1);
+ }
+ clauseList.removeAll(redundantOrderbys);
+ }
+
+ // Processes leading "let"s in a FLWOGR.
+ private void processLeadingLetClauses(Integer step, List<Clause> clauseList) throws AsterixException {
+ List<Clause> processedLetList = new ArrayList<Clause>();
+ boolean firstLet = true;
+ int size = clauseList.size();
+ for (int i = 0; i < size; ++i) {
+ Clause cl = clauseList.get(i);
+ if (cl.getClauseType() != ClauseType.LET_CLAUSE) {
+ break;
+ }
+ boolean hasConsequentLet = false;
+ if (i < size - 1) {
+ Clause nextCl = clauseList.get(i + 1);
+ hasConsequentLet = nextCl.getClauseType() == ClauseType.LET_CLAUSE;
+ }
+ visitLetClause((LetClause) cl, step, firstLet, hasConsequentLet);
+ firstLet = false;
+ processedLetList.add(cl);
+ }
+ clauseList.removeAll(processedLetList);
+ }
+
+ // Extracts all clauses that led by a "for" clause after the first group-by
+ // clause in the input clause list.
+ // Those extracted clauses will be removed from the input clause list.
+ /**
+ * @param clauseList
+ * , a list of clauses
+ * @return the cutting group-by clause and the list of extracted clauses.
+ * @throws AsterixException
+ */
+ private Pair<GroupbyClause, List<Clause>> extractUnnestAfterGroupby(List<Clause> clauseList)
+ throws AsterixException {
+ List<Clause> nestedClauses = new ArrayList<Clause>();
+ GroupbyClause cuttingGbyClause = null;
+ boolean meetGroupBy = false;
+ boolean nestedClauseStarted = false;
+ for (Clause cl : clauseList) {
+ if (cl.getClauseType() == ClauseType.GROUP_BY_CLAUSE) {
+ meetGroupBy = true;
+ cuttingGbyClause = (GroupbyClause) cl;
+ continue;
+ }
+ if (meetGroupBy && cl.getClauseType() == ClauseType.FOR_CLAUSE) {
+ nestedClauseStarted = true;
+ }
+ if (nestedClauseStarted) {
+ nestedClauses.add(cl);
+ }
+ }
+ clauseList.removeAll(nestedClauses);
+ return new Pair<GroupbyClause, List<Clause>>(cuttingGbyClause, nestedClauses);
+ }
+
+ // Extracts the variables to be substituted with a path access.
+ private Map<VariableExpr, Expression> extractDefinedCollectionVariables(List<Clause> clauses,
+ GroupbyClause cuttingGbyClause, String generatedAlias) {
+ Map<VariableExpr, Expression> varExprMap = new HashMap<VariableExpr, Expression>();
+ List<VariableExpr> varToSubstitute = collectProducedVariablesFromGroupby(cuttingGbyClause);
+ int gbyIndex = clauses.indexOf(cuttingGbyClause);
+ for (int i = gbyIndex + 1; i < clauses.size(); i++) {
+ Clause cl = clauses.get(i);
+ if (cl.getClauseType() == ClauseType.LET_CLAUSE) {
+ varToSubstitute.add(((LetClause) cl).getVarExpr());
+ }
+ }
+ for (VariableExpr var : varToSubstitute) {
+ varExprMap.put(var, new FieldAccessor(new VariableExpr(new VarIdentifier(generatedAlias)),
+ new VarIdentifier(var.getVar().getValue().substring(1))));
+ }
+ return varExprMap;
+ }
+
+ // Extracts the variables to be substituted.
+ private Map<VariableExpr, Expression> extractLetBindingVariables(List<Clause> clauses,
+ GroupbyClause cuttingGbyClause) throws AsterixException {
+ Map<VariableExpr, Expression> varExprMap = new HashMap<VariableExpr, Expression>();
+ int gbyIndex = clauses.indexOf(cuttingGbyClause);
+ for (int i = gbyIndex + 1; i < clauses.size(); i++) {
+ Clause cl = clauses.get(i);
+ if (cl.getClauseType() == ClauseType.LET_CLAUSE) {
+ LetClause letClause = (LetClause) cl;
+ // inline let variables one by one iteratively.
+ letClause.setBindingExpr((Expression) AQLVariableSubstitutionUtil
+ .substituteVariable(letClause.getBindingExpr(), varExprMap));
+ varExprMap.put(letClause.getVarExpr(), letClause.getBindingExpr());
+ }
+ }
+ return varExprMap;
+ }
+
+ // Re-order clauses.
+ private List<Clause> reorder(List<Clause> clauses) {
+ Comparator<Clause> comparator = new ClauseComparator();
+ List<Clause> results = new ArrayList<Clause>();
+ int size = clauses.size();
+ int start = 0;
+ for (int index = 0; index < size; ++index) {
+ Clause clause = clauses.get(index);
+ if (clause.getClauseType() == ClauseType.GROUP_BY_CLAUSE) {
+ List<Clause> subList = clauses.subList(start, index);
+ Collections.sort(subList, comparator);
+ results.addAll(subList);
+ results.add(clause);
+ start = index + 1;
+ }
+ }
+ if (start < clauses.size()) {
+ List<Clause> subList = clauses.subList(start, size);
+ Collections.sort(subList, comparator);
+ results.addAll(subList);
+ }
+ return results;
+ }
+
+ // Merge consecutive "where" clauses.
+ private void mergeConsecutiveWhereClauses(List<Clause> clauses) {
+ List<Clause> results = new ArrayList<Clause>();
+ int size = clauses.size();
+ for (int index = 0; index < size;) {
+ Clause clause = clauses.get(index);
+ if (clause.getClauseType() != ClauseType.WHERE_CLAUSE) {
+ results.add(clause);
+ ++index;
+ } else {
+ List<Expression> expressions = new ArrayList<Expression>();
+ Clause firstWhereClause = clause;
+ do {
+ WhereClause whereClause = (WhereClause) clause;
+ expressions.add(whereClause.getWhereExpr());
+ if (++index >= size) {
+ break;
+ }
+ clause = clauses.get(index);
+ } while (clause.getClauseType() == ClauseType.WHERE_CLAUSE);
+ if (expressions.size() > 1) {
+ OperatorExpr newWhereExpr = new OperatorExpr();
+ newWhereExpr.setExprList(expressions);
+ newWhereExpr.setCurrentop(true);
+ for (int operatorIndex = 0; operatorIndex < expressions.size(); ++operatorIndex) {
+ newWhereExpr.addOperator("and");
+ }
+ results.add(new WhereClause(newWhereExpr));
+ } else {
+ results.add(firstWhereClause);
+ }
+ }
+ }
+ clauses.clear();
+ clauses.addAll(results);
+ }
+
+ // Where there is a distinct clause.
+ protected boolean hasDistinct(List<Clause> clauses) {
+ for (Clause clause : clauses) {
+ if (clause.getClauseType() == ClauseType.DISTINCT_BY_CLAUSE) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ // Whether the list of clauses contains a for clause.
+ private boolean hasFor(List<Clause> clauses) {
+ for (Clause cl : clauses) {
+ if (cl.getClauseType() == ClauseType.FOR_CLAUSE) {
+ return true;
+ }
+ }
+ return false;
+ }
+}
+
+/**
+ * This comparator is used to safely mutate the order of clauses in a FLWOGR
+ * expression. Note: clauses before and after a group-by cannot be re-aligned.
+ */
+class ClauseComparator implements Comparator<Clause> {
+
+ @Override
+ public int compare(Clause left, Clause right) {
+ int ordinalLeft = left.getClauseType().ordinal();
+ int ordinalRight = right.getClauseType().ordinal();
+ return ordinalLeft > ordinalRight ? 1 : (ordinalLeft == ordinalRight ? 0 : -1);
+ }
+}
diff --git a/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/visitor/base/AbstractAqlQueryExpressionVisitor.java b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/visitor/base/AbstractAqlQueryExpressionVisitor.java
new file mode 100644
index 0000000..977c3b1
--- /dev/null
+++ b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/visitor/base/AbstractAqlQueryExpressionVisitor.java
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.aql.visitor.base;
+
+import org.apache.asterix.lang.common.visitor.base.AbstractQueryExpressionVisitor;
+
+public abstract class AbstractAqlQueryExpressionVisitor<R, T> extends AbstractQueryExpressionVisitor<R, T>
+ implements IAQLVisitor<R, T> {
+
+}
diff --git a/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/visitor/base/IAQLPlusVisitor.java b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/visitor/base/IAQLPlusVisitor.java
new file mode 100644
index 0000000..f2037e7
--- /dev/null
+++ b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/visitor/base/IAQLPlusVisitor.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.aql.visitor.base;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.lang.aql.clause.JoinClause;
+import org.apache.asterix.lang.aql.clause.MetaVariableClause;
+import org.apache.asterix.lang.aql.expression.MetaVariableExpr;
+
+public interface IAQLPlusVisitor<R, T> extends IAQLVisitor<R, T> {
+
+ R visitJoinClause(JoinClause c, T arg) throws AsterixException;
+
+ R visitMetaVariableClause(MetaVariableClause c, T arg) throws AsterixException;
+
+ R visitMetaVariableExpr(MetaVariableExpr v, T arg) throws AsterixException;
+}
diff --git a/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/visitor/base/IAQLVisitor.java b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/visitor/base/IAQLVisitor.java
new file mode 100644
index 0000000..d990005
--- /dev/null
+++ b/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/visitor/base/IAQLVisitor.java
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.lang.aql.visitor.base;
+
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.lang.aql.clause.DistinctClause;
+import org.apache.asterix.lang.aql.clause.ForClause;
+import org.apache.asterix.lang.aql.expression.FLWOGRExpression;
+import org.apache.asterix.lang.aql.expression.UnionExpr;
+import org.apache.asterix.lang.common.visitor.base.ILangVisitor;
+
+public interface IAQLVisitor<R, T> extends ILangVisitor<R, T> {
+
+ R visit(FLWOGRExpression flwogreExpr, T arg) throws AsterixException;
+
+ R visit(UnionExpr u, T arg) throws AsterixException;
+
+ R visit(ForClause forClause, T arg) throws AsterixException;
+
+ R visit(DistinctClause distinctClause, T arg) throws AsterixException;
+
+}
diff --git a/asterix-lang-aql/src/main/javacc/AQL.html b/asterix-lang-aql/src/main/javacc/AQL.html
new file mode 100644
index 0000000..b50d554
--- /dev/null
+++ b/asterix-lang-aql/src/main/javacc/AQL.html
@@ -0,0 +1,774 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 3.2//EN">
+<HTML>
+<HEAD>
+<TITLE>BNF for AQL.jj</TITLE>
+</HEAD>
+<BODY>
+<H1 ALIGN=CENTER>BNF for AQL.jj</H1>
+<H2 ALIGN=CENTER>TOKENS</H2>
+<TABLE>
+ <!-- Token -->
+ <TR>
+ <TD>
+ <PRE>
+<DEFAULT,IN_DBL_BRACE> TOKEN : {
+<ASC: "asc">
+| <AT: "at">
+| <BY: "by">
+| <DATASET: "dataset">
+| <DECOR: "decor">
+| <DESC: "desc">
+| <DISTINCT: "distinct">
+| <ELSE: "else">
+| <EVERY: "every">
+| <FOR: "for">
+| <FROM: "from">
+| <GROUP: "group">
+| <IF: "if">
+| <IN: "in">
+| <LET: "let">
+| <LIMIT: "limit">
+| <OFFSET: "offset">
+| <ORDER: "order">
+| <RETURN: "return">
+| <SATISFIES: "satisfies">
+| <SELECT: "select">
+| <SOME: "some">
+| <THEN: "then">
+| <UNION: "union">
+| <WHERE: "where">
+| <WITH: "with">
+| <KEEPING: "keeping">
+}
+
+ </PRE>
+ </TD>
+ </TR>
+ <!-- Token -->
+ <TR>
+ <TD>
+ <PRE>
+<DEFAULT,IN_DBL_BRACE> TOKEN : {
+<CARET: "^">
+| <DIV: "/">
+| <IDIV: "idiv">
+| <MINUS: "-">
+| <MOD: "%">
+| <MUL: "*">
+| <PLUS: "+">
+| <LEFTPAREN: "(">
+| <RIGHTPAREN: ")">
+| <LEFTBRACKET: "[">
+| <RIGHTBRACKET: "]">
+| <COLON: ":">
+| <COMMA: ",">
+| <DOT: ".">
+| <QUES: "?">
+| <LT: "<">
+| <GT: ">">
+| <LE: "<=">
+| <GE: ">=">
+| <EQ: "=">
+| <NE: "!=">
+| <SIMILAR: "~=">
+| <ASSIGN: ":=">
+| <AND: "and">
+| <OR: "or">
+}
+
+ </PRE>
+ </TD>
+ </TR>
+ <!-- Token -->
+ <TR>
+ <TD>
+ <PRE>
+<DEFAULT,IN_DBL_BRACE> TOKEN : {
+<LEFTBRACE: "{"> : DEFAULT
+}
+
+ </PRE>
+ </TD>
+ </TR>
+ <!-- Token -->
+ <TR>
+ <TD>
+ <PRE>
+<DEFAULT> TOKEN : {
+<RIGHTBRACE: "}"> : {
+}
+
+ </PRE>
+ </TD>
+ </TR>
+ <!-- Token -->
+ <TR>
+ <TD>
+ <PRE>
+<DEFAULT,IN_DBL_BRACE> TOKEN : {
+<LEFTDBLBRACE: "{{"> : IN_DBL_BRACE
+}
+
+ </PRE>
+ </TD>
+ </TR>
+ <!-- Token -->
+ <TR>
+ <TD>
+ <PRE>
+<IN_DBL_BRACE> TOKEN : {
+<RIGHTDBLBRACE: "}}"> : {
+}
+
+ </PRE>
+ </TD>
+ </TR>
+ <!-- Token -->
+ <TR>
+ <TD>
+ <PRE>
+<DEFAULT,IN_DBL_BRACE> TOKEN : {
+<INTEGER_LITERAL: (<DIGIT>)+>
+}
+
+ </PRE>
+ </TD>
+ </TR>
+ <!-- Token -->
+ <TR>
+ <TD>
+ <PRE>
+<DEFAULT,IN_DBL_BRACE> TOKEN : {
+<NULL: "null">
+| <TRUE: "true">
+| <FALSE: "false">
+}
+
+ </PRE>
+ </TD>
+ </TR>
+ <!-- Token -->
+ <TR>
+ <TD>
+ <PRE>
+<DEFAULT,IN_DBL_BRACE> TOKEN : {
+<#DIGIT: ["0"-"9"]>
+}
+
+ </PRE>
+ </TD>
+ </TR>
+ <!-- Token -->
+ <TR>
+ <TD>
+ <PRE>
+<DEFAULT,IN_DBL_BRACE> TOKEN : {
+<DOUBLE_LITERAL: <DIGITS> | <DIGITS> ("." <DIGITS>)? | "." <DIGITS>>
+| <FLOAT_LITERAL: <DIGITS> ("f" | "F") | <DIGITS> ("." <DIGITS> ("f" | "F"))? | "." <DIGITS> ("f" | "F")>
+| <DIGITS: (<DIGIT>)+>
+}
+
+ </PRE>
+ </TD>
+ </TR>
+ <!-- Token -->
+ <TR>
+ <TD>
+ <PRE>
+<DEFAULT,IN_DBL_BRACE> TOKEN : {
+<#LETTER: ["A"-"Z","a"-"z"]>
+| <SPECIALCHARS: ["$","_","-"]>
+}
+
+ </PRE>
+ </TD>
+ </TR>
+ <!-- Token -->
+ <TR>
+ <TD>
+ <PRE>
+<DEFAULT,IN_DBL_BRACE> TOKEN : {
+<STRING_LITERAL: "\"" (<EscapeQuot> | <EscapeBslash> | <EscapeSlash> | <EscapeBspace> | <EscapeFormf> | <EscapeNl> | <EscapeCr> | <EscapeTab> | ~["\"","\\"])* "\"" | "\'" (<EscapeApos> | <EscapeBslash> | <EscapeSlash> | <EscapeBspace> | <EscapeFormf> | <EscapeNl> | <EscapeCr> | <EscapeTab> | ~["\'","\\"])* "\'">
+| <#EscapeQuot: "\\\"">
+| <#EscapeApos: "\\\'">
+| <#EscapeBslash: "\\\\">
+| <#EscapeSlash: "\\/">
+| <#EscapeBspace: "\\b">
+| <#EscapeFormf: "\\f">
+| <#EscapeNl: "\\n">
+| <#EscapeCr: "\\r">
+| <#EscapeTab: "\\t">
+}
+
+ </PRE>
+ </TD>
+ </TR>
+ <!-- Token -->
+ <TR>
+ <TD>
+ <PRE>
+<DEFAULT,IN_DBL_BRACE> TOKEN : {
+<IDENTIFIER: <LETTER> (<LETTER> | <DIGIT> | <SPECIALCHARS>)*>
+}
+
+ </PRE>
+ </TD>
+ </TR>
+ <!-- Token -->
+ <TR>
+ <TD>
+ <PRE>
+<DEFAULT,IN_DBL_BRACE> TOKEN : {
+<VARIABLE: "$" <LETTER> (<LETTER> | <DIGIT> | "_")*>
+}
+
+ </PRE>
+ </TD>
+ </TR>
+ <!-- Token -->
+ <TR>
+ <TD>
+ <PRE>
+<DEFAULT,IN_DBL_BRACE> SKIP : {
+" "
+| "\t"
+| "\r"
+| "\n"
+}
+
+ </PRE>
+ </TD>
+ </TR>
+ <!-- Token -->
+ <TR>
+ <TD>
+ <PRE>
+<DEFAULT,IN_DBL_BRACE> SKIP : {
+<"//" (~["\n"])* "\n">
+}
+
+ </PRE>
+ </TD>
+ </TR>
+ <!-- Token -->
+ <TR>
+ <TD>
+ <PRE>
+<DEFAULT,IN_DBL_BRACE> SKIP : {
+<"//" (~["\n","\r"])* ("\n" | "\r" | "\r\n")?>
+}
+
+ </PRE>
+ </TD>
+ </TR>
+ <!-- Token -->
+ <TR>
+ <TD>
+ <PRE>
+<DEFAULT,IN_DBL_BRACE> SKIP : {
+"/*" : INSIDE_COMMENT
+}
+
+ </PRE>
+ </TD>
+ </TR>
+ <!-- Token -->
+ <TR>
+ <TD>
+ <PRE>
+<INSIDE_COMMENT> SPECIAL : {
+<"+" (" ")* (~["*"])*>
+}
+
+ </PRE>
+ </TD>
+ </TR>
+ <!-- Token -->
+ <TR>
+ <TD>
+ <PRE>
+<INSIDE_COMMENT> SKIP : {
+"/*" : {
+}
+
+ </PRE>
+ </TD>
+ </TR>
+ <!-- Token -->
+ <TR>
+ <TD>
+ <PRE>
+<INSIDE_COMMENT> SKIP : {
+"*/" : {
+| <~[]>
+}
+
+ </PRE>
+ </TD>
+ </TR>
+</TABLE>
+<H2 ALIGN=CENTER>NON-TERMINALS</H2>
+<TABLE>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod1">Statement</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod2">SingleStatement</A> ( ";" )? )* <EOF></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod2">SingleStatement</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod3">DataverseDeclaration</A> | <A HREF="#prod4">FunctionDeclaration</A> | <A HREF="#prod5">CreateStatement</A> | <A HREF="#prod6">LoadStatement</A> | <A HREF="#prod7">DropStatement</A> | <A HREF="#prod8">WriteStatement</A> | <A HREF="#prod9">SetStatement</A> | <A HREF="#prod10">InsertStatement</A> | <A HREF="#prod11">DeleteStatement</A> | <A HREF="#prod12">UpdateStatement</A> | <A HREF="#prod13">FeedStatement</A> | <A HREF="#prod14">CompactStatement</A> | <A HREF="#prod15">Query</A> | <A HREF="#prod16">RefreshExternalDatasetStatement</A> | <A HREF="#prod17">RunStatement</A> )</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod3">DataverseDeclaration</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"use" "dataverse" <A HREF="#prod18">Identifier</A></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod5">CreateStatement</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"create" ( <A HREF="#prod19">TypeSpecification</A> | <A HREF="#prod20">NodegroupSpecification</A> | <A HREF="#prod21">DatasetSpecification</A> | <A HREF="#prod22">IndexSpecification</A> | <A HREF="#prod23">DataverseSpecification</A> | <A HREF="#prod24">FunctionSpecification</A> | <A HREF="#prod25">FeedSpecification</A> | <A HREF="#prod26">FeedPolicySpecification</A> )</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod19">TypeSpecification</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"type" <A HREF="#prod27">TypeName</A> <A HREF="#prod28">IfNotExists</A> "as" <A HREF="#prod29">TypeExpr</A></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod20">NodegroupSpecification</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"nodegroup" <A HREF="#prod18">Identifier</A> <A HREF="#prod28">IfNotExists</A> "on" <A HREF="#prod18">Identifier</A> ( <COMMA> <A HREF="#prod18">Identifier</A> )*</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod21">DatasetSpecification</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( "external" <DATASET> <A HREF="#prod30">QualifiedName</A> <LEFTPAREN> <A HREF="#prod18">Identifier</A> <RIGHTPAREN> <A HREF="#prod28">IfNotExists</A> "using" <A HREF="#prod31">AdapterName</A> <A HREF="#prod32">Configuration</A> ( "on" <A HREF="#prod18">Identifier</A> )? ( "hints" <A HREF="#prod33">Properties</A> )? ( "using" "compaction" "policy" <A HREF="#prod34">CompactionPolicy</A> ( <A HREF="#prod32">Configuration</A> )? )? | ( "internal" | "temporary" )? <DATASET> <A HREF="#prod30">QualifiedName</A> <LEFTPAREN> <A HREF="#prod18">Identifier</A> <RIGHTPAREN> <A HREF="#prod28">IfNotExists</A> <A HREF="#prod35">PrimaryKey</A> ( "autogenerated" )? ( "on" <A HREF="#prod18">Identifier</A> )? ( "hints" <A HREF="#prod33">Properties</A> )? ( "using" "compaction" "policy" <A HREF="#prod34">CompactionPolicy</A> ( <A HREF="#prod32">Configuration</A> )? )? ( "with filter on" <A HREF="#prod36">NestedField</A> )? )</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod16">RefreshExternalDatasetStatement</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"refresh external" <DATASET> <A HREF="#prod30">QualifiedName</A></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod17">RunStatement</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"run" <A HREF="#prod18">Identifier</A> <LEFTPAREN> ( <A HREF="#prod18">Identifier</A> ( <COMMA> )? )* <RIGHTPAREN> <FROM> <DATASET> <A HREF="#prod30">QualifiedName</A> "to" <DATASET> <A HREF="#prod30">QualifiedName</A></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod22">IndexSpecification</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"index" <A HREF="#prod18">Identifier</A> <A HREF="#prod28">IfNotExists</A> "on" <A HREF="#prod30">QualifiedName</A> <LEFTPAREN> ( <A HREF="#prod37">OpenField</A> ) ( <COMMA> <A HREF="#prod37">OpenField</A> )* <RIGHTPAREN> ( "type" <A HREF="#prod38">IndexType</A> )? ( "enforced" )?</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod34">CompactionPolicy</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod18">Identifier</A></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod39">FilterField</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod18">Identifier</A></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod38">IndexType</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( "btree" | "rtree" | "keyword" | "ngram" <LEFTPAREN> <INTEGER_LITERAL> <RIGHTPAREN> )</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod23">DataverseSpecification</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"dataverse" <A HREF="#prod18">Identifier</A> <A HREF="#prod28">IfNotExists</A> ( "with format" <A HREF="#prod40">StringLiteral</A> )?</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod24">FunctionSpecification</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"function" <A HREF="#prod41">FunctionName</A> <A HREF="#prod28">IfNotExists</A> <A HREF="#prod42">ParameterList</A> <LEFTBRACE> <A HREF="#prod43">Expression</A> <RIGHTBRACE></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod25">FeedSpecification</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( "secondary" "feed" <A HREF="#prod30">QualifiedName</A> <A HREF="#prod28">IfNotExists</A> <FROM> "feed" <A HREF="#prod30">QualifiedName</A> ( <A HREF="#prod44">ApplyFunction</A> )? | ( "primary" )? "feed" <A HREF="#prod30">QualifiedName</A> <A HREF="#prod28">IfNotExists</A> "using" <A HREF="#prod31">AdapterName</A> <A HREF="#prod32">Configuration</A> ( <A HREF="#prod44">ApplyFunction</A> )? )</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod26">FeedPolicySpecification</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( "ingestion" "policy" <A HREF="#prod18">Identifier</A> <A HREF="#prod28">IfNotExists</A> <FROM> ( "policy" <A HREF="#prod18">Identifier</A> <A HREF="#prod32">Configuration</A> ( "definition" <A HREF="#prod40">StringLiteral</A> )? | "path" <A HREF="#prod18">Identifier</A> ( "definition" <A HREF="#prod40">StringLiteral</A> )? ) )</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod42">ParameterList</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><LEFTPAREN> ( <VARIABLE> ( <COMMA> <VARIABLE> )* )? <RIGHTPAREN></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod28">IfNotExists</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( "if not exists" )?</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod44">ApplyFunction</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"apply" "function" <A HREF="#prod41">FunctionName</A></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod45">GetPolicy</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"using" "policy" <A HREF="#prod18">Identifier</A></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod46">FunctionSignature</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod41">FunctionName</A> "@" <INTEGER_LITERAL></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod35">PrimaryKey</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"primary" "key" <A HREF="#prod36">NestedField</A> ( <COMMA> <A HREF="#prod36">NestedField</A> )*</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod7">DropStatement</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"drop" ( <DATASET> <A HREF="#prod30">QualifiedName</A> <A HREF="#prod47">IfExists</A> | "index" <A HREF="#prod48">DoubleQualifiedName</A> <A HREF="#prod47">IfExists</A> | "nodegroup" <A HREF="#prod18">Identifier</A> <A HREF="#prod47">IfExists</A> | "type" <A HREF="#prod27">TypeName</A> <A HREF="#prod47">IfExists</A> | "dataverse" <A HREF="#prod18">Identifier</A> <A HREF="#prod47">IfExists</A> | "function" <A HREF="#prod46">FunctionSignature</A> <A HREF="#prod47">IfExists</A> | "feed" <A HREF="#prod30">QualifiedName</A> <A HREF="#prod47">IfExists</A> )</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod47">IfExists</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <IF> "exists" )?</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod10">InsertStatement</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"insert" "into" <DATASET> <A HREF="#prod30">QualifiedName</A> <A HREF="#prod15">Query</A></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod11">DeleteStatement</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"delete" <A HREF="#prod49">Variable</A> <FROM> <DATASET> <A HREF="#prod30">QualifiedName</A> ( <WHERE> <A HREF="#prod43">Expression</A> )?</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod12">UpdateStatement</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"update" <A HREF="#prod49">Variable</A> <IN> <A HREF="#prod43">Expression</A> <WHERE> <A HREF="#prod43">Expression</A> <LEFTPAREN> ( <A HREF="#prod50">UpdateClause</A> ( <COMMA> <A HREF="#prod50">UpdateClause</A> )* ) <RIGHTPAREN></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod50">UpdateClause</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( "set" <A HREF="#prod43">Expression</A> <ASSIGN> <A HREF="#prod43">Expression</A> | <A HREF="#prod10">InsertStatement</A> | <A HREF="#prod11">DeleteStatement</A> | <A HREF="#prod12">UpdateStatement</A> | <IF> <LEFTPAREN> <A HREF="#prod43">Expression</A> <RIGHTPAREN> <THEN> <A HREF="#prod50">UpdateClause</A> ( <ELSE> <A HREF="#prod50">UpdateClause</A> )? )</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod9">SetStatement</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"set" <A HREF="#prod18">Identifier</A> <A HREF="#prod40">StringLiteral</A></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod8">WriteStatement</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"write" "output" "to" <A HREF="#prod18">Identifier</A> <COLON> <A HREF="#prod40">StringLiteral</A> ( "using" <A HREF="#prod40">StringLiteral</A> )?</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod6">LoadStatement</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"load" <DATASET> <A HREF="#prod30">QualifiedName</A> "using" <A HREF="#prod31">AdapterName</A> <A HREF="#prod32">Configuration</A> ( "pre-sorted" )?</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod31">AdapterName</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod18">Identifier</A></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod14">CompactStatement</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"compact" <DATASET> <A HREF="#prod30">QualifiedName</A></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod13">FeedStatement</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( "connect" "feed" <A HREF="#prod30">QualifiedName</A> "to" <DATASET> <A HREF="#prod30">QualifiedName</A> ( <A HREF="#prod45">GetPolicy</A> )? | "disconnect" "feed" <A HREF="#prod30">QualifiedName</A> <FROM> <DATASET> <A HREF="#prod30">QualifiedName</A> )</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod32">Configuration</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><LEFTPAREN> ( <A HREF="#prod51">KeyValuePair</A> ( <COMMA> <A HREF="#prod51">KeyValuePair</A> )* )? <RIGHTPAREN></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod51">KeyValuePair</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><LEFTPAREN> <A HREF="#prod40">StringLiteral</A> <EQ> <A HREF="#prod40">StringLiteral</A> <RIGHTPAREN></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod33">Properties</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <LEFTPAREN> <A HREF="#prod52">Property</A> ( <COMMA> <A HREF="#prod52">Property</A> )* <RIGHTPAREN> )?</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod52">Property</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod18">Identifier</A> <EQ> ( <A HREF="#prod40">StringLiteral</A> | <INTEGER_LITERAL> )</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod53">IndexedTypeExpr</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod54">TypeReference</A> | <A HREF="#prod55">OrderedListTypeDef</A> | <A HREF="#prod56">UnorderedListTypeDef</A> )</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod29">TypeExpr</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod57">RecordTypeDef</A> | <A HREF="#prod54">TypeReference</A> | <A HREF="#prod55">OrderedListTypeDef</A> | <A HREF="#prod56">UnorderedListTypeDef</A> )</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod57">RecordTypeDef</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( "closed" | "open" )? <LEFTBRACE> ( <A HREF="#prod58">RecordField</A> ( <COMMA> <A HREF="#prod58">RecordField</A> )* )? <RIGHTBRACE></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod58">RecordField</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod18">Identifier</A> <COLON> <A HREF="#prod29">TypeExpr</A> ( <QUES> )?</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod54">TypeReference</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod18">Identifier</A></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod55">OrderedListTypeDef</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><LEFTBRACKET> ( <A HREF="#prod29">TypeExpr</A> ) <RIGHTBRACKET></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod56">UnorderedListTypeDef</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><LEFTDBLBRACE> ( <A HREF="#prod29">TypeExpr</A> ) <RIGHTDBLBRACE></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod41">FunctionName</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod18">Identifier</A> ( <DOT> <A HREF="#prod18">Identifier</A> ( "#" <A HREF="#prod18">Identifier</A> )? | "#" <A HREF="#prod18">Identifier</A> )?</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod27">TypeName</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod30">QualifiedName</A></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod18">Identifier</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <IDENTIFIER> | <A HREF="#prod40">StringLiteral</A> )</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod37">OpenField</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod36">NestedField</A> ( <COLON> <A HREF="#prod53">IndexedTypeExpr</A> )?</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod36">NestedField</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod18">Identifier</A> ( <DOT> <A HREF="#prod18">Identifier</A> )*</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod40">StringLiteral</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><STRING_LITERAL></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod30">QualifiedName</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod18">Identifier</A> ( <DOT> <A HREF="#prod18">Identifier</A> )?</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod48">DoubleQualifiedName</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod18">Identifier</A> <DOT> <A HREF="#prod18">Identifier</A> ( <DOT> <A HREF="#prod18">Identifier</A> )?</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod4">FunctionDeclaration</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>"declare" "function" <A HREF="#prod18">Identifier</A> <A HREF="#prod42">ParameterList</A> <LEFTBRACE> <A HREF="#prod43">Expression</A> <RIGHTBRACE></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod15">Query</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod43">Expression</A></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod43">Expression</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod59">OperatorExpr</A> | <A HREF="#prod60">IfThenElse</A> | <A HREF="#prod61">FLWOGR</A> | <A HREF="#prod62">QuantifiedExpression</A> )</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod59">OperatorExpr</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod63">AndExpr</A> ( <OR> <A HREF="#prod63">AndExpr</A> )*</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod63">AndExpr</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod64">RelExpr</A> ( <AND> <A HREF="#prod64">RelExpr</A> )*</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod64">RelExpr</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod65">AddExpr</A> ( ( <LT> | <GT> | <LE> | <GE> | <EQ> | <NE> | <SIMILAR> ) <A HREF="#prod65">AddExpr</A> )?</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod65">AddExpr</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod66">MultExpr</A> ( ( <PLUS> | <MINUS> ) <A HREF="#prod66">MultExpr</A> )*</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod66">MultExpr</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod67">UnionExpr</A> ( ( <MUL> | <DIV> | <MOD> | <CARET> | <IDIV> ) <A HREF="#prod67">UnionExpr</A> )*</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod67">UnionExpr</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod68">UnaryExpr</A> ( <UNION> ( <A HREF="#prod68">UnaryExpr</A> ) )*</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod68">UnaryExpr</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( ( <PLUS> | <MINUS> ) )? <A HREF="#prod69">ValueExpr</A></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod69">ValueExpr</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod70">PrimaryExpr</A> ( <A HREF="#prod71">Field</A> | <A HREF="#prod72">Index</A> )*</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod71">Field</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><DOT> <A HREF="#prod18">Identifier</A></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod72">Index</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><LEFTBRACKET> ( <A HREF="#prod43">Expression</A> | <QUES> ) <RIGHTBRACKET></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod70">PrimaryExpr</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod73">FunctionCallExpr</A> | <A HREF="#prod74">Literal</A> | <A HREF="#prod75">DatasetAccessExpression</A> | <A HREF="#prod76">VariableRef</A> | <A HREF="#prod77">ListConstructor</A> | <A HREF="#prod78">RecordConstructor</A> | <A HREF="#prod79">ParenthesizedExpression</A> )</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod74">Literal</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod40">StringLiteral</A> | <INTEGER_LITERAL> | <FLOAT_LITERAL> | <DOUBLE_LITERAL> | <NULL> | <TRUE> | <FALSE> )</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod76">VariableRef</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><VARIABLE></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod49">Variable</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><VARIABLE></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod77">ListConstructor</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod80">OrderedListConstructor</A> | <A HREF="#prod81">UnorderedListConstructor</A> )</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod80">OrderedListConstructor</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><LEFTBRACKET> <A HREF="#prod82">ExpressionList</A> <RIGHTBRACKET></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod81">UnorderedListConstructor</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><LEFTDBLBRACE> <A HREF="#prod82">ExpressionList</A> <RIGHTDBLBRACE></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod82">ExpressionList</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod43">Expression</A> ( <COMMA> <A HREF="#prod82">ExpressionList</A> )? )? ( <A HREF="#prod83">Comma</A> )?</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod83">Comma</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><COMMA></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod78">RecordConstructor</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><LEFTBRACE> ( <A HREF="#prod84">FieldBinding</A> ( <COMMA> <A HREF="#prod84">FieldBinding</A> )* )? <RIGHTBRACE></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod84">FieldBinding</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod43">Expression</A> <COLON> <A HREF="#prod43">Expression</A></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod73">FunctionCallExpr</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><A HREF="#prod41">FunctionName</A> <LEFTPAREN> ( <A HREF="#prod43">Expression</A> ( <COMMA> <A HREF="#prod43">Expression</A> )* )? <RIGHTPAREN></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod75">DatasetAccessExpression</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><DATASET> ( ( <A HREF="#prod18">Identifier</A> ( <DOT> <A HREF="#prod18">Identifier</A> )? ) | ( <LEFTPAREN> <A HREF="#prod43">Expression</A> <RIGHTPAREN> ) )</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod79">ParenthesizedExpression</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><LEFTPAREN> <A HREF="#prod43">Expression</A> <RIGHTPAREN></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod60">IfThenElse</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><IF> <LEFTPAREN> <A HREF="#prod43">Expression</A> <RIGHTPAREN> <THEN> <A HREF="#prod43">Expression</A> <ELSE> <A HREF="#prod43">Expression</A></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod61">FLWOGR</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod85">ForClause</A> | <A HREF="#prod86">LetClause</A> ) ( <A HREF="#prod87">Clause</A> )* ( <RETURN> | <SELECT> ) <A HREF="#prod43">Expression</A></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod87">Clause</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <A HREF="#prod85">ForClause</A> | <A HREF="#prod86">LetClause</A> | <A HREF="#prod88">WhereClause</A> | <A HREF="#prod89">OrderbyClause</A> | <A HREF="#prod90">GroupClause</A> | <A HREF="#prod91">LimitClause</A> | <A HREF="#prod92">DistinctClause</A> )</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod85">ForClause</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <FOR> | <FROM> ) <A HREF="#prod49">Variable</A> ( <AT> <A HREF="#prod49">Variable</A> )? <IN> ( <A HREF="#prod43">Expression</A> )</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod86">LetClause</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <LET> | <WITH> ) <A HREF="#prod49">Variable</A> <ASSIGN> <A HREF="#prod43">Expression</A></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod88">WhereClause</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><WHERE> <A HREF="#prod43">Expression</A></TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod89">OrderbyClause</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( <ORDER> <BY> <A HREF="#prod43">Expression</A> ( ( <ASC> ) | ( <DESC> ) )? ( <COMMA> <A HREF="#prod43">Expression</A> ( ( <ASC> ) | ( <DESC> ) )? )* )</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod90">GroupClause</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><GROUP> <BY> ( <A HREF="#prod49">Variable</A> <ASSIGN> )? <A HREF="#prod43">Expression</A> ( <COMMA> ( <A HREF="#prod49">Variable</A> <ASSIGN> )? <A HREF="#prod43">Expression</A> )* ( <DECOR> <A HREF="#prod49">Variable</A> <ASSIGN> <A HREF="#prod43">Expression</A> ( <COMMA> <DECOR> <A HREF="#prod49">Variable</A> <ASSIGN> <A HREF="#prod43">Expression</A> )* )? ( <WITH> | <KEEPING> ) <A HREF="#prod76">VariableRef</A> ( <COMMA> <A HREF="#prod76">VariableRef</A> )*</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod91">LimitClause</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><LIMIT> <A HREF="#prod43">Expression</A> ( <OFFSET> <A HREF="#prod43">Expression</A> )?</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod92">DistinctClause</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE><DISTINCT> <BY> <A HREF="#prod43">Expression</A> ( <COMMA> <A HREF="#prod43">Expression</A> )*</TD>
+</TR>
+<TR>
+<TD ALIGN=RIGHT VALIGN=BASELINE><A NAME="prod62">QuantifiedExpression</A></TD>
+<TD ALIGN=CENTER VALIGN=BASELINE>::=</TD>
+<TD ALIGN=LEFT VALIGN=BASELINE>( ( <SOME> ) | ( <EVERY> ) ) <A HREF="#prod49">Variable</A> <IN> <A HREF="#prod43">Expression</A> ( <COMMA> <A HREF="#prod49">Variable</A> <IN> <A HREF="#prod43">Expression</A> )* <SATISFIES> <A HREF="#prod43">Expression</A></TD>
+</TR>
+</TABLE>
+</BODY>
+</HTML>
diff --git a/asterix-lang-aql/src/main/javacc/AQL.jj b/asterix-lang-aql/src/main/javacc/AQL.jj
new file mode 100644
index 0000000..bb574d0
--- /dev/null
+++ b/asterix-lang-aql/src/main/javacc/AQL.jj
@@ -0,0 +1,2684 @@
+options {
+
+
+ STATIC = false;
+
+}
+
+
+PARSER_BEGIN(AQLParser)
+
+package org.apache.asterix.lang.aql.parser;
+
+// For AQLParserTokenManager
+import org.apache.xerces.util.IntStack;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.common.annotations.AutoDataGen;
+import org.apache.asterix.common.annotations.DateBetweenYearsDataGen;
+import org.apache.asterix.common.annotations.DatetimeAddRandHoursDataGen;
+import org.apache.asterix.common.annotations.DatetimeBetweenYearsDataGen;
+import org.apache.asterix.common.annotations.FieldIntervalDataGen;
+import org.apache.asterix.common.annotations.FieldValFileDataGen;
+import org.apache.asterix.common.annotations.FieldValFileSameIndexDataGen;
+import org.apache.asterix.common.annotations.IRecordFieldDataGen;
+import org.apache.asterix.common.annotations.InsertRandIntDataGen;
+import org.apache.asterix.common.annotations.ListDataGen;
+import org.apache.asterix.common.annotations.ListValFileDataGen;
+import org.apache.asterix.common.annotations.SkipSecondaryIndexSearchExpressionAnnotation;
+import org.apache.asterix.common.annotations.TypeDataGen;
+import org.apache.asterix.common.annotations.UndeclaredFieldsDataGen;
+import org.apache.asterix.common.config.DatasetConfig.DatasetType;
+import org.apache.asterix.common.config.DatasetConfig.IndexType;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.lang.aql.clause.DistinctClause;
+import org.apache.asterix.lang.aql.clause.ForClause;
+import org.apache.asterix.lang.aql.expression.FLWOGRExpression;
+import org.apache.asterix.lang.aql.expression.UnionExpr;
+import org.apache.asterix.lang.aql.util.RangeMapBuilder;
+import org.apache.asterix.lang.common.base.Clause;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.IParser;
+import org.apache.asterix.lang.common.base.Literal;
+import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.lang.common.clause.GroupbyClause;
+import org.apache.asterix.lang.common.clause.LetClause;
+import org.apache.asterix.lang.common.clause.LimitClause;
+import org.apache.asterix.lang.common.clause.OrderbyClause;
+import org.apache.asterix.lang.common.clause.UpdateClause;
+import org.apache.asterix.lang.common.clause.WhereClause;
+import org.apache.asterix.lang.common.context.RootScopeFactory;
+import org.apache.asterix.lang.common.context.Scope;
+import org.apache.asterix.lang.common.expression.AbstractAccessor;
+import org.apache.asterix.lang.common.expression.CallExpr;
+import org.apache.asterix.lang.common.expression.FieldAccessor;
+import org.apache.asterix.lang.common.expression.FieldBinding;
+import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair;
+import org.apache.asterix.lang.common.expression.IfExpr;
+import org.apache.asterix.lang.common.expression.IndexAccessor;
+import org.apache.asterix.lang.common.expression.ListConstructor;
+import org.apache.asterix.lang.common.expression.LiteralExpr;
+import org.apache.asterix.lang.common.expression.OperatorExpr;
+import org.apache.asterix.lang.common.expression.OrderedListTypeDefinition;
+import org.apache.asterix.lang.common.expression.QuantifiedExpression;
+import org.apache.asterix.lang.common.expression.RecordConstructor;
+import org.apache.asterix.lang.common.expression.RecordTypeDefinition;
+import org.apache.asterix.lang.common.expression.TypeExpression;
+import org.apache.asterix.lang.common.expression.TypeReferenceExpression;
+import org.apache.asterix.lang.common.expression.UnaryExpr;
+import org.apache.asterix.lang.common.expression.UnorderedListTypeDefinition;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.expression.UnaryExpr.Sign;
+import org.apache.asterix.lang.common.literal.DoubleLiteral;
+import org.apache.asterix.lang.common.literal.FalseLiteral;
+import org.apache.asterix.lang.common.literal.FloatLiteral;
+import org.apache.asterix.lang.common.literal.LongIntegerLiteral;
+import org.apache.asterix.lang.common.literal.NullLiteral;
+import org.apache.asterix.lang.common.literal.StringLiteral;
+import org.apache.asterix.lang.common.literal.TrueLiteral;
+import org.apache.asterix.lang.common.parser.ScopeChecker;
+import org.apache.asterix.lang.common.statement.CompactStatement;
+import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
+import org.apache.asterix.lang.common.statement.CreateDataverseStatement;
+import org.apache.asterix.lang.common.statement.CreateFeedPolicyStatement;
+import org.apache.asterix.lang.common.statement.CreateFeedStatement;
+import org.apache.asterix.lang.common.statement.CreateFunctionStatement;
+import org.apache.asterix.lang.common.statement.CreateIndexStatement;
+import org.apache.asterix.lang.common.statement.CreatePrimaryFeedStatement;
+import org.apache.asterix.lang.common.statement.CreateSecondaryFeedStatement;
+import org.apache.asterix.lang.common.statement.DatasetDecl;
+import org.apache.asterix.lang.common.statement.DataverseDecl;
+import org.apache.asterix.lang.common.statement.DataverseDropStatement;
+import org.apache.asterix.lang.common.statement.DeleteStatement;
+import org.apache.asterix.lang.common.statement.DisconnectFeedStatement;
+import org.apache.asterix.lang.common.statement.DropStatement;
+import org.apache.asterix.lang.common.statement.ExternalDetailsDecl;
+import org.apache.asterix.lang.common.statement.FeedDropStatement;
+import org.apache.asterix.lang.common.statement.FunctionDecl;
+import org.apache.asterix.lang.common.statement.FunctionDropStatement;
+import org.apache.asterix.lang.common.statement.IndexDropStatement;
+import org.apache.asterix.lang.common.statement.InsertStatement;
+import org.apache.asterix.lang.common.statement.InternalDetailsDecl;
+import org.apache.asterix.lang.common.statement.LoadStatement;
+import org.apache.asterix.lang.common.statement.NodeGroupDropStatement;
+import org.apache.asterix.lang.common.statement.NodegroupDecl;
+import org.apache.asterix.lang.common.statement.Query;
+import org.apache.asterix.lang.common.statement.RefreshExternalDatasetStatement;
+import org.apache.asterix.lang.common.statement.RunStatement;
+import org.apache.asterix.lang.common.statement.SetStatement;
+import org.apache.asterix.lang.common.statement.TypeDecl;
+import org.apache.asterix.lang.common.statement.TypeDropStatement;
+import org.apache.asterix.lang.common.statement.UpdateStatement;
+import org.apache.asterix.lang.common.statement.WriteStatement;
+import org.apache.asterix.lang.common.struct.Identifier;
+import org.apache.asterix.lang.common.struct.QuantifiedPair;
+import org.apache.asterix.lang.common.struct.VarIdentifier;
+import org.apache.asterix.metadata.bootstrap.MetadataConstants;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.common.utils.Triple;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionAnnotation;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IndexedNLJoinExpressionAnnotation;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+
+
+public class AQLParser extends ScopeChecker implements IParser {
+
+ // optimizer hints
+ private static final String AUTO_HINT = "auto";
+ private static final String BROADCAST_JOIN_HINT = "bcast";
+ private static final String COMPOSE_VAL_FILES_HINT = "compose-val-files";
+ private static final String DATE_BETWEEN_YEARS_HINT = "date-between-years";
+ private static final String DATETIME_ADD_RAND_HOURS_HINT = "datetime-add-rand-hours";
+ private static final String DATETIME_BETWEEN_YEARS_HINT = "datetime-between-years";
+ private static final String HASH_GROUP_BY_HINT = "hash";
+ private static final String INDEXED_NESTED_LOOP_JOIN_HINT = "indexnl";
+ private static final String INMEMORY_HINT = "inmem";
+ private static final String INSERT_RAND_INT_HINT = "insert-rand-int";
+ private static final String INTERVAL_HINT = "interval";
+ private static final String LIST_HINT = "list";
+ private static final String LIST_VAL_FILE_HINT = "list-val-file";
+ private static final String RANGE_HINT = "range";
+ private static final String SKIP_SECONDARY_INDEX_SEARCH_HINT = "skip-index";
+ private static final String VAL_FILE_HINT = "val-files";
+ private static final String VAL_FILE_SAME_INDEX_HINT = "val-file-same-idx";
+
+ private static final String GEN_FIELDS_HINT = "gen-fields";
+
+ // data generator hints
+ private static final String DGEN_HINT = "dgen";
+
+ private static class IndexParams {
+ public IndexType type;
+ public int gramLength;
+
+ public IndexParams(IndexType type, int gramLength) {
+ this.type = type;
+ this.gramLength = gramLength;
+ }
+ };
+
+ private static class FunctionName {
+ public String dataverse = null;
+ public String library = null;
+ public String function = null;
+ public String hint = null;
+ }
+
+ private static String getHint(Token t) {
+ if (t.specialToken == null) {
+ return null;
+ }
+ String s = t.specialToken.image;
+ int n = s.length();
+ if (n < 2) {
+ return null;
+ }
+ return s.substring(1).trim();
+ }
+
+ private static IRecordFieldDataGen parseFieldDataGen(String hint) throws ParseException {
+ IRecordFieldDataGen rfdg = null;
+ String splits[] = hint.split(" +");
+ if (splits[0].equals(VAL_FILE_HINT)) {
+ File[] valFiles = new File[splits.length - 1];
+ for (int k=1; k<splits.length; k++) {
+ valFiles[k-1] = new File(splits[k]);
+ }
+ rfdg = new FieldValFileDataGen(valFiles);
+ } else if (splits[0].equals(VAL_FILE_SAME_INDEX_HINT)) {
+ rfdg = new FieldValFileSameIndexDataGen(new File(splits[1]), splits[2]);
+ } else if (splits[0].equals(LIST_VAL_FILE_HINT)) {
+ rfdg = new ListValFileDataGen(new File(splits[1]), Integer.parseInt(splits[2]), Integer.parseInt(splits[3]));
+ } else if (splits[0].equals(LIST_HINT)) {
+ rfdg = new ListDataGen(Integer.parseInt(splits[1]), Integer.parseInt(splits[2]));
+ } else if (splits[0].equals(INTERVAL_HINT)) {
+ FieldIntervalDataGen.ValueType vt;
+ if (splits[1].equals("int")) {
+ vt = FieldIntervalDataGen.ValueType.INT;
+ } else if (splits[1].equals("long")) {
+ vt = FieldIntervalDataGen.ValueType.LONG;
+ } else if (splits[1].equals("float")) {
+ vt = FieldIntervalDataGen.ValueType.FLOAT;
+ } else if (splits[1].equals("double")) {
+ vt = FieldIntervalDataGen.ValueType.DOUBLE;
+ } else {
+ throw new ParseException("Unknown type for interval data gen: " + splits[1]);
+ }
+ rfdg = new FieldIntervalDataGen(vt, splits[2], splits[3]);
+ } else if (splits[0].equals(INSERT_RAND_INT_HINT)) {
+ rfdg = new InsertRandIntDataGen(splits[1], splits[2]);
+ } else if (splits[0].equals(DATE_BETWEEN_YEARS_HINT)) {
+ rfdg = new DateBetweenYearsDataGen(Integer.parseInt(splits[1]), Integer.parseInt(splits[2]));
+ } else if (splits[0].equals(DATETIME_BETWEEN_YEARS_HINT)) {
+ rfdg = new DatetimeBetweenYearsDataGen(Integer.parseInt(splits[1]), Integer.parseInt(splits[2]));
+ } else if (splits[0].equals(DATETIME_ADD_RAND_HOURS_HINT)) {
+ rfdg = new DatetimeAddRandHoursDataGen(Integer.parseInt(splits[1]), Integer.parseInt(splits[2]), splits[3]);
+ } else if (splits[0].equals(AUTO_HINT)) {
+ rfdg = new AutoDataGen(splits[1]);
+ }
+ return rfdg;
+ }
+
+ public AQLParser(String s){
+ this(new StringReader(s));
+ super.setInput(s);
+ }
+
+ public static void main(String args[]) throws ParseException, TokenMgrError, IOException, FileNotFoundException, AsterixException {
+ File file = new File(args[0]);
+ Reader fis = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
+ AQLParser parser = new AQLParser(fis);
+ List<Statement> st = parser.parse();
+ //st.accept(new AQLPrintVisitor(), 0);
+ }
+
+ public List<Statement> parse() throws AsterixException {
+ try {
+ return Statement();
+ } catch (Error e) {
+ // this is here as the JavaCharStream that's below the lexer somtimes throws Errors that are not handled
+ // by the ANTLR-generated lexer or parser (e.g it does this for invalid backslash u + 4 hex digits escapes)
+ throw new AsterixException(new ParseException(e.getMessage()));
+ } catch (ParseException e){
+ throw new AsterixException(e.getMessage());
+ }
+ }
+}
+
+PARSER_END(AQLParser)
+
+
+List<Statement> Statement() throws ParseException:
+{
+ scopeStack.push(RootScopeFactory.createRootScope(this));
+ List<Statement> decls = new ArrayList<Statement>();
+ Statement stmt = null;
+}
+{
+ ( stmt = SingleStatement() (";") ?
+ {
+ decls.add(stmt);
+ }
+ )*
+ <EOF>
+ {
+ return decls;
+ }
+}
+
+Statement SingleStatement() throws ParseException:
+{
+ Statement stmt = null;
+}
+{
+ (
+ stmt = DataverseDeclaration()
+ | stmt = FunctionDeclaration()
+ | stmt = CreateStatement()
+ | stmt = LoadStatement()
+ | stmt = DropStatement()
+ | stmt = WriteStatement()
+ | stmt = SetStatement()
+ | stmt = InsertStatement()
+ | stmt = DeleteStatement()
+ | stmt = UpdateStatement()
+ | stmt = FeedStatement()
+ | stmt = CompactStatement()
+ | stmt = Query()
+ | stmt = RefreshExternalDatasetStatement()
+ | stmt = RunStatement()
+ )
+ {
+ return stmt;
+ }
+}
+
+DataverseDecl DataverseDeclaration() throws ParseException:
+{
+ String dvName = null;
+}
+{
+ "use" "dataverse" dvName = Identifier()
+ {
+ defaultDataverse = dvName;
+ return new DataverseDecl(new Identifier(dvName));
+ }
+}
+
+Statement CreateStatement() throws ParseException:
+{
+ String hint = null;
+ boolean dgen = false;
+ Statement stmt = null;
+}
+{
+ "create"
+ (
+ {
+ hint = getHint(token);
+ if (hint != null && hint.startsWith(DGEN_HINT)) {
+ dgen = true;
+ }
+ }
+ stmt = TypeSpecification(hint, dgen)
+ | stmt = NodegroupSpecification()
+ | stmt = DatasetSpecification()
+ | stmt = IndexSpecification()
+ | stmt = DataverseSpecification()
+ | stmt = FunctionSpecification()
+ | stmt = FeedSpecification()
+ | stmt = FeedPolicySpecification()
+ )
+ {
+ return stmt;
+ }
+}
+
+TypeDecl TypeSpecification(String hint, boolean dgen) throws ParseException:
+{
+ Pair<Identifier,Identifier> nameComponents = null;
+ boolean ifNotExists = false;
+ TypeExpression typeExpr = null;
+}
+{
+ "type" nameComponents = TypeName() ifNotExists = IfNotExists()
+ "as" typeExpr = TypeExpr()
+ {
+ long numValues = -1;
+ String filename = null;
+ if (dgen) {
+ String splits[] = hint.split(" +");
+ if (splits.length != 3) {
+ throw new ParseException("Expecting /*+ dgen <filename> <numberOfItems> */");
+ }
+ filename = splits[1];
+ numValues = Long.parseLong(splits[2]);
+ }
+ TypeDataGen tddg = new TypeDataGen(dgen, filename, numValues);
+ return new TypeDecl(nameComponents.first, nameComponents.second, typeExpr, tddg, ifNotExists);
+ }
+}
+
+
+NodegroupDecl NodegroupSpecification() throws ParseException:
+{
+ String name = null;
+ String tmp = null;
+ boolean ifNotExists = false;
+ List<Identifier>ncNames = null;
+}
+{
+ "nodegroup" name = Identifier()
+ ifNotExists = IfNotExists() "on" tmp = Identifier()
+ {
+ ncNames = new ArrayList<Identifier>();
+ ncNames.add(new Identifier(tmp));
+ }
+ ( <COMMA> tmp = Identifier()
+ {
+ ncNames.add(new Identifier(tmp));
+ }
+ )*
+ {
+ return new NodegroupDecl(new Identifier(name), ncNames, ifNotExists);
+ }
+}
+
+DatasetDecl DatasetSpecification() throws ParseException:
+{
+ Pair<Identifier,Identifier> nameComponents = null;
+ boolean ifNotExists = false;
+ String typeName = null;
+ String adapterName = null;
+ Map<String,String> properties = null;
+ Map<String,String> compactionPolicyProperties = null;
+ FunctionSignature appliedFunction = null;
+ List<List<String>> primaryKeyFields = null;
+ String nodeGroupName = null;
+ Map<String,String> hints = new HashMap<String,String>();
+ DatasetDecl dsetDecl = null;
+ boolean autogenerated = false;
+ String compactionPolicy = null;
+ boolean temp = false;
+ List<String> filterField = null;
+}
+{
+ (
+ "external" <DATASET> nameComponents = QualifiedName()
+ <LEFTPAREN> typeName = Identifier() <RIGHTPAREN>
+ ifNotExists = IfNotExists()
+ "using" adapterName = AdapterName() properties = Configuration()
+ ("on" nodeGroupName = Identifier() )?
+ ( "hints" hints = Properties() )?
+ ( "using" "compaction" "policy" compactionPolicy = CompactionPolicy() (compactionPolicyProperties = Configuration())? )?
+ {
+ ExternalDetailsDecl edd = new ExternalDetailsDecl();
+ edd.setAdapter(adapterName);
+ edd.setProperties(properties);
+ dsetDecl = new DatasetDecl(nameComponents.first,
+ nameComponents.second,
+ new Identifier(typeName),
+ nodeGroupName != null? new Identifier(nodeGroupName): null,
+ compactionPolicy,
+ compactionPolicyProperties,
+ hints,
+ DatasetType.EXTERNAL,
+ edd,
+ ifNotExists);
+ }
+
+ | ("internal" | "temporary" {
+ temp = token.image.toLowerCase().equals("temporary");
+ }
+ )?
+ <DATASET> nameComponents = QualifiedName()
+ <LEFTPAREN> typeName = Identifier() <RIGHTPAREN>
+ ifNotExists = IfNotExists()
+ primaryKeyFields = PrimaryKey()
+ ("autogenerated" { autogenerated = true; } )?
+ ("on" nodeGroupName = Identifier() )?
+ ( "hints" hints = Properties() )?
+ ( "using" "compaction" "policy" compactionPolicy = CompactionPolicy() (compactionPolicyProperties = Configuration())? )?
+ ( "with filter on" filterField = NestedField() )?
+ {
+ InternalDetailsDecl idd = new InternalDetailsDecl(primaryKeyFields,
+ autogenerated,
+ filterField,
+ temp);
+ dsetDecl = new DatasetDecl(nameComponents.first,
+ nameComponents.second,
+ new Identifier(typeName),
+ nodeGroupName != null ? new Identifier(nodeGroupName) : null,
+ compactionPolicy,
+ compactionPolicyProperties,
+ hints,
+ DatasetType.INTERNAL,
+ idd,
+ ifNotExists);
+ }
+ )
+ {
+ return dsetDecl;
+ }
+}
+
+RefreshExternalDatasetStatement RefreshExternalDatasetStatement() throws ParseException:
+{
+ RefreshExternalDatasetStatement redss = new RefreshExternalDatasetStatement();
+ Pair<Identifier,Identifier> nameComponents = null;
+ String datasetName = null;
+}
+{
+ "refresh external" <DATASET> nameComponents = QualifiedName()
+ {
+ redss.setDataverseName(nameComponents.first);
+ redss.setDatasetName(nameComponents.second);
+ return redss;
+ }
+}
+
+RunStatement RunStatement() throws ParseException:
+{
+ String system = null;
+ String tmp;
+ ArrayList<String> parameters = new ArrayList<String>();
+ Pair<Identifier,Identifier> nameComponentsFrom = null;
+ Pair<Identifier,Identifier> nameComponentsTo = null;
+}
+{
+ "run" system = Identifier()<LEFTPAREN> ( tmp = Identifier() [<COMMA>]
+ {
+ parameters.add(tmp);
+ }
+ )*<RIGHTPAREN>
+ <FROM> <DATASET> nameComponentsFrom = QualifiedName()
+ "to" <DATASET> nameComponentsTo = QualifiedName()
+ {
+ return new RunStatement(system, parameters, nameComponentsFrom.first, nameComponentsFrom.second, nameComponentsTo.first, nameComponentsTo.second);
+ }
+}
+
+CreateIndexStatement IndexSpecification() throws ParseException:
+{
+ CreateIndexStatement cis = new CreateIndexStatement();
+ String indexName = null;
+ boolean ifNotExists = false;
+ Pair<Identifier,Identifier> nameComponents = null;
+ Pair<List<String>, TypeExpression> fieldPair = null;
+ IndexParams indexType = null;
+ boolean enforced = false;
+}
+{
+ "index" indexName = Identifier()
+ ifNotExists = IfNotExists()
+ "on" nameComponents = QualifiedName()
+ <LEFTPAREN> ( fieldPair = OpenField()
+ {
+ cis.addFieldExprPair(fieldPair);
+ }
+ ) (<COMMA> fieldPair = OpenField()
+ {
+ cis.addFieldExprPair(fieldPair);
+ }
+ )* <RIGHTPAREN> ( "type" indexType = IndexType() )? ( "enforced" { enforced = true; } )?
+ {
+ cis.setIndexName(new Identifier(indexName));
+ cis.setIfNotExists(ifNotExists);
+ cis.setDataverseName(nameComponents.first);
+ cis.setDatasetName(nameComponents.second);
+ if (indexType != null) {
+ cis.setIndexType(indexType.type);
+ cis.setGramLength(indexType.gramLength);
+ }
+ cis.setEnforced(enforced);
+ return cis;
+ }
+}
+
+String CompactionPolicy() throws ParseException :
+{
+ String compactionPolicy = null;
+}
+{
+ compactionPolicy = Identifier()
+ {
+ return compactionPolicy;
+ }
+}
+
+String FilterField() throws ParseException :
+{
+ String filterField = null;
+}
+{
+ filterField = Identifier()
+ {
+ return filterField;
+ }
+}
+
+IndexParams IndexType() throws ParseException:
+{
+ IndexType type = null;
+ int gramLength = 0;
+}
+{
+ ("btree"
+ {
+ type = IndexType.BTREE;
+ }
+ | "rtree"
+ {
+ type = IndexType.RTREE;
+ }
+ | "keyword"
+ {
+ type = IndexType.LENGTH_PARTITIONED_WORD_INVIX;
+ }
+ | "ngram" <LEFTPAREN> <INTEGER_LITERAL>
+ {
+ type = IndexType.LENGTH_PARTITIONED_NGRAM_INVIX;
+ gramLength = Integer.valueOf(token.image);
+ }
+ <RIGHTPAREN>)
+ {
+ return new IndexParams(type, gramLength);
+ }
+}
+
+CreateDataverseStatement DataverseSpecification() throws ParseException :
+{
+ String dvName = null;
+ boolean ifNotExists = false;
+ String format = null;
+}
+{
+ "dataverse" dvName = Identifier()
+ ifNotExists = IfNotExists()
+ ( "with format" format = StringLiteral() )?
+ {
+ return new CreateDataverseStatement(new Identifier(dvName), format, ifNotExists);
+ }
+}
+
+CreateFunctionStatement FunctionSpecification() throws ParseException:
+{
+ FunctionSignature signature;
+ boolean ifNotExists = false;
+ List<VarIdentifier> paramList = new ArrayList<VarIdentifier>();
+ String functionBody;
+ VarIdentifier var = null;
+ Expression functionBodyExpr;
+ Token beginPos;
+ Token endPos;
+ FunctionName fctName = null;
+
+ createNewScope();
+}
+{
+ "function" fctName = FunctionName()
+ ifNotExists = IfNotExists()
+ paramList = ParameterList()
+ <LEFTBRACE>
+ {
+ beginPos = token;
+ }
+ functionBodyExpr = Expression() <RIGHTBRACE>
+ {
+ endPos = token;
+ functionBody = extractFragment(beginPos.beginLine, beginPos.beginColumn, endPos.beginLine, endPos.beginColumn);
+ // TODO use fctName.library
+ signature = new FunctionSignature(fctName.dataverse, fctName.function, paramList.size());
+ getCurrentScope().addFunctionDescriptor(signature, false);
+ removeCurrentScope();
+ return new CreateFunctionStatement(signature, paramList, functionBody, ifNotExists);
+ }
+}
+
+CreateFeedStatement FeedSpecification() throws ParseException:
+{
+ Pair<Identifier,Identifier> nameComponents = null;
+ boolean ifNotExists = false;
+ String adapterName = null;
+ Map<String,String> properties = null;
+ FunctionSignature appliedFunction = null;
+ CreateFeedStatement cfs = null;
+ Pair<Identifier,Identifier> sourceNameComponents = null;
+
+}
+{
+ (
+ "secondary" "feed" nameComponents = QualifiedName() ifNotExists = IfNotExists()
+ <FROM> "feed" sourceNameComponents = QualifiedName() (appliedFunction = ApplyFunction())?
+ {
+ cfs = new CreateSecondaryFeedStatement(nameComponents,
+ sourceNameComponents, appliedFunction, ifNotExists);
+ }
+ |
+ ("primary")? "feed" nameComponents = QualifiedName() ifNotExists = IfNotExists()
+ "using" adapterName = AdapterName() properties = Configuration() (appliedFunction = ApplyFunction())?
+ {
+ cfs = new CreatePrimaryFeedStatement(nameComponents,
+ adapterName, properties, appliedFunction, ifNotExists);
+ }
+ )
+ {
+ return cfs;
+ }
+}
+
+CreateFeedPolicyStatement FeedPolicySpecification() throws ParseException:
+{
+ String policyName = null;
+ String basePolicyName = null;
+ String sourcePolicyFile = null;
+ String definition = null;
+ boolean ifNotExists = false;
+ Map<String,String> properties = null;
+ CreateFeedPolicyStatement cfps = null;
+}
+{
+ (
+ "ingestion" "policy" policyName = Identifier() ifNotExists = IfNotExists()
+ <FROM>
+ ("policy" basePolicyName = Identifier() properties = Configuration() ("definition" definition = StringLiteral())?
+ {
+ cfps = new CreateFeedPolicyStatement(policyName,
+ basePolicyName, properties, definition, ifNotExists);
+ }
+ | "path" sourcePolicyFile = Identifier() ("definition" definition = StringLiteral())?
+ {
+ cfps = new CreateFeedPolicyStatement(policyName, sourcePolicyFile, definition, ifNotExists);
+ }
+ )
+
+ )
+ {
+ return cfps;
+ }
+}
+
+
+
+List<VarIdentifier> ParameterList() throws ParseException:
+{
+ List<VarIdentifier> paramList = new ArrayList<VarIdentifier>();
+ VarIdentifier var = null;
+}
+{
+ <LEFTPAREN> (<VARIABLE>
+ {
+ var = new VarIdentifier();
+ var.setValue(token.image);
+ paramList.add(var);
+ getCurrentScope().addNewVarSymbolToScope(var);
+ }
+ (<COMMA> <VARIABLE>
+ {
+ var = new VarIdentifier();
+ var.setValue(token.image);
+ paramList.add(var);
+ getCurrentScope().addNewVarSymbolToScope(var);
+ }
+ )*)? <RIGHTPAREN>
+ {
+ return paramList;
+ }
+}
+
+boolean IfNotExists() throws ParseException:
+{
+}
+{
+ ( "if not exists"
+ {
+ return true;
+ }
+ )?
+ {
+ return false;
+ }
+}
+
+FunctionSignature ApplyFunction() throws ParseException:
+{
+ FunctionName functioName = null;
+ FunctionSignature funcSig = null;
+}
+{
+ "apply" "function" functioName = FunctionName()
+ {
+ String fqFunctionName = functioName.library == null ? functioName.function : functioName.library + "#" + functioName.function;
+ return new FunctionSignature(functioName.dataverse, fqFunctionName, 1);
+ }
+}
+
+String GetPolicy() throws ParseException:
+{
+ String policy = null;
+}
+{
+ "using" "policy" policy = Identifier()
+ {
+ return policy;
+ }
+
+}
+
+FunctionSignature FunctionSignature() throws ParseException:
+{
+ FunctionName fctName = null;
+ int arity = 0;
+}
+{
+ fctName = FunctionName() "@" <INTEGER_LITERAL>
+ {
+ arity = new Integer(token.image);
+ if (arity < 0 && arity != FunctionIdentifier.VARARGS) {
+ throw new ParseException(" invalid arity:" + arity);
+ }
+
+ // TODO use fctName.library
+ String fqFunctionName = fctName.library == null ? fctName.function : fctName.library + "#" + fctName.function;
+ return new FunctionSignature(fctName.dataverse, fqFunctionName, arity);
+ }
+}
+
+List<List<String>> PrimaryKey() throws ParseException:
+{
+ List<String> tmp = null;
+ List<List<String>> primaryKeyFields = new ArrayList<List<String>>();
+}
+{
+ "primary" "key" tmp = NestedField()
+ {
+ primaryKeyFields.add(tmp);
+ }
+ ( <COMMA> tmp = NestedField()
+ {
+ primaryKeyFields.add(tmp);
+ }
+ )*
+ {
+ return primaryKeyFields;
+ }
+}
+
+Statement DropStatement() throws ParseException:
+{
+ String id = null;
+ Pair<Identifier,Identifier> pairId = null;
+ Triple<Identifier,Identifier,Identifier> tripleId = null;
+ FunctionSignature funcSig = null;
+ boolean ifExists = false;
+ Statement stmt = null;
+}
+{
+ "drop"
+ (
+ <DATASET> pairId = QualifiedName() ifExists = IfExists()
+ {
+ stmt = new DropStatement(pairId.first, pairId.second, ifExists);
+ }
+ | "index" tripleId = DoubleQualifiedName() ifExists = IfExists()
+ {
+ stmt = new IndexDropStatement(tripleId.first, tripleId.second, tripleId.third, ifExists);
+ }
+ | "nodegroup" id = Identifier() ifExists = IfExists()
+ {
+ stmt = new NodeGroupDropStatement(new Identifier(id), ifExists);
+ }
+ | "type" pairId = TypeName() ifExists = IfExists()
+ {
+ stmt = new TypeDropStatement(pairId.first, pairId.second, ifExists);
+ }
+ | "dataverse" id = Identifier() ifExists = IfExists()
+ {
+ stmt = new DataverseDropStatement(new Identifier(id), ifExists);
+ }
+ | "function" funcSig = FunctionSignature() ifExists = IfExists()
+ {
+ stmt = new FunctionDropStatement(funcSig, ifExists);
+ }
+ | "feed" pairId = QualifiedName() ifExists = IfExists()
+ {
+ stmt = new FeedDropStatement(pairId.first, pairId.second, ifExists);
+ }
+ )
+ {
+ return stmt;
+ }
+}
+
+boolean IfExists() throws ParseException :
+{
+}
+{
+ ( <IF> "exists"
+ {
+ return true;
+ }
+ )?
+ {
+ return false;
+ }
+}
+
+InsertStatement InsertStatement() throws ParseException:
+{
+ Pair<Identifier,Identifier> nameComponents = null;
+ Query query;
+}
+{
+ "insert" "into" <DATASET> nameComponents = QualifiedName() query = Query()
+ {
+ query.setTopLevel(false);
+ return new InsertStatement(nameComponents.first, nameComponents.second, query, getVarCounter());
+ }
+}
+
+DeleteStatement DeleteStatement() throws ParseException:
+{
+ VariableExpr var = null;
+ Expression condition = null;
+ Pair<Identifier, Identifier> nameComponents;
+ // This is related to the new metadata lock management
+ setDataverses(new ArrayList<String>());
+ setDatasets(new ArrayList<String>());
+
+}
+{
+ "delete" var = Variable()
+ {
+ getCurrentScope().addNewVarSymbolToScope(var.getVar());
+ }
+ <FROM> <DATASET> nameComponents = QualifiedName()
+ (<WHERE> condition = Expression())?
+ {
+ // First we get the dataverses and datasets that we want to lock
+ List<String> dataverses = getDataverses();
+ List<String> datasets = getDatasets();
+ // we remove the pointer to the dataverses and datasets
+ setDataverses(null);
+ setDatasets(null);
+ return new DeleteStatement(var, nameComponents.first, nameComponents.second,
+ condition, getVarCounter(), dataverses, datasets);
+ }
+}
+
+UpdateStatement UpdateStatement() throws ParseException:
+{
+ VariableExpr vars;
+ Expression target;
+ Expression condition;
+ UpdateClause uc;
+ List<UpdateClause> ucs = new ArrayList<UpdateClause>();
+}
+{
+ "update" vars = Variable() <IN> target = Expression()
+ <WHERE> condition = Expression()
+ <LEFTPAREN> (uc = UpdateClause()
+ {
+ ucs.add(uc);
+ }
+ (<COMMA> uc = UpdateClause()
+ {
+ ucs.add(uc);
+ }
+ )*) <RIGHTPAREN>
+ {
+ return new UpdateStatement(vars, target, condition, ucs);
+ }
+}
+
+UpdateClause UpdateClause() throws ParseException:
+{
+ Expression target = null;
+ Expression value = null ;
+ InsertStatement is = null;
+ DeleteStatement ds = null;
+ UpdateStatement us = null;
+ Expression condition = null;
+ UpdateClause ifbranch = null;
+ UpdateClause elsebranch = null;
+}
+{
+ ("set" target = Expression() <ASSIGN> value = Expression()
+ | is = InsertStatement()
+ | ds = DeleteStatement()
+ | us = UpdateStatement()
+ | <IF> <LEFTPAREN> condition = Expression() <RIGHTPAREN>
+ <THEN> ifbranch = UpdateClause()
+ [LOOKAHEAD(1) <ELSE> elsebranch = UpdateClause()]
+ {
+ return new UpdateClause(target, value, is, ds, us, condition, ifbranch, elsebranch);
+ }
+ )
+}
+
+Statement SetStatement() throws ParseException:
+{
+ String pn = null;
+ String pv = null;
+}
+{
+ "set" pn = Identifier() pv = StringLiteral()
+ {
+ return new SetStatement(pn, pv);
+ }
+}
+
+Statement WriteStatement() throws ParseException:
+{
+ String nodeName = null;
+ String fileName = null;
+ Query query;
+ String writerClass = null;
+ Pair<Identifier,Identifier> nameComponents = null;
+}
+{
+ "write" "output" "to" nodeName = Identifier() <COLON> fileName = StringLiteral()
+ ( "using" writerClass = StringLiteral() )?
+ {
+ return new WriteStatement(new Identifier(nodeName), fileName, writerClass);
+ }
+}
+
+LoadStatement LoadStatement() throws ParseException:
+{
+ Identifier dataverseName = null;
+ Identifier datasetName = null;
+ boolean alreadySorted = false;
+ String adapterName;
+ Map<String,String> properties;
+ Pair<Identifier,Identifier> nameComponents = null;
+}
+{
+ "load" <DATASET> nameComponents = QualifiedName()
+ {
+ dataverseName = nameComponents.first;
+ datasetName = nameComponents.second;
+ }
+ "using" adapterName = AdapterName() properties = Configuration()
+ ("pre-sorted"
+ {
+ alreadySorted = true;
+ }
+ )?
+ {
+ return new LoadStatement(dataverseName, datasetName, adapterName, properties, alreadySorted);
+ }
+}
+
+
+String AdapterName() throws ParseException :
+{
+ String adapterName = null;
+}
+{
+ adapterName = Identifier()
+ {
+ return adapterName;
+ }
+}
+
+Statement CompactStatement() throws ParseException:
+{
+ Pair<Identifier,Identifier> nameComponents = null;
+ Statement stmt = null;
+}
+{
+ "compact" <DATASET> nameComponents = QualifiedName()
+ {
+ stmt = new CompactStatement(nameComponents.first, nameComponents.second);
+ }
+ {
+ return stmt;
+ }
+}
+
+Statement FeedStatement() throws ParseException:
+{
+ Pair<Identifier,Identifier> feedNameComponents = null;
+ Pair<Identifier,Identifier> datasetNameComponents = null;
+
+ Map<String,String> configuration = null;
+ Statement stmt = null;
+ String policy = null;
+}
+{
+ (
+ "connect" "feed" feedNameComponents = QualifiedName() "to" <DATASET> datasetNameComponents = QualifiedName() (policy = GetPolicy())?
+ {
+ stmt = new ConnectFeedStatement(feedNameComponents, datasetNameComponents, policy, getVarCounter());
+ }
+ | "disconnect" "feed" feedNameComponents = QualifiedName() <FROM> <DATASET> datasetNameComponents = QualifiedName()
+ {
+ stmt = new DisconnectFeedStatement(feedNameComponents, datasetNameComponents);
+ }
+ )
+ {
+ return stmt;
+ }
+}
+
+Map<String,String> Configuration() throws ParseException :
+{
+ Map<String,String> configuration = new LinkedHashMap<String,String>();
+ Pair<String, String> keyValuePair = null;
+}
+{
+ <LEFTPAREN> ( keyValuePair = KeyValuePair()
+ {
+ configuration.put(keyValuePair.first, keyValuePair.second);
+ }
+ ( <COMMA> keyValuePair = KeyValuePair()
+ {
+ configuration.put(keyValuePair.first, keyValuePair.second);
+ }
+ )* )? <RIGHTPAREN>
+ {
+ return configuration;
+ }
+}
+
+Pair<String, String> KeyValuePair() throws ParseException:
+{
+ String key;
+ String value;
+}
+{
+ <LEFTPAREN> key = StringLiteral() <EQ> value = StringLiteral() <RIGHTPAREN>
+ {
+ return new Pair<String, String>(key, value);
+ }
+}
+
+Map<String,String> Properties() throws ParseException:
+{
+ Map<String,String> properties = new HashMap<String,String>();
+ Pair<String, String> property;
+}
+{
+ ( <LEFTPAREN> property = Property()
+ {
+ properties.put(property.first, property.second);
+ }
+ ( <COMMA> property = Property()
+ {
+ properties.put(property.first, property.second);
+ }
+ )* <RIGHTPAREN> )?
+ {
+ return properties;
+ }
+}
+
+Pair<String, String> Property() throws ParseException:
+{
+ String key;
+ String value;
+}
+{
+ key = Identifier() <EQ> ( value = StringLiteral() | <INTEGER_LITERAL>
+ {
+ try {
+ value = "" + Long.valueOf(token.image);
+ } catch (NumberFormatException nfe) {
+ throw new ParseException("inapproriate value: " + token.image);
+ }
+ }
+ )
+ {
+ return new Pair<String, String>(key.toUpperCase(), value);
+ }
+}
+
+TypeExpression IndexedTypeExpr() throws ParseException:
+{
+ TypeExpression typeExpr = null;
+}
+{
+ (
+ typeExpr = TypeReference()
+ | typeExpr = OrderedListTypeDef()
+ | typeExpr = UnorderedListTypeDef()
+ )
+ {
+ return typeExpr;
+ }
+}
+
+TypeExpression TypeExpr() throws ParseException:
+{
+ TypeExpression typeExpr = null;
+}
+{
+ (
+ typeExpr = RecordTypeDef()
+ | typeExpr = TypeReference()
+ | typeExpr = OrderedListTypeDef()
+ | typeExpr = UnorderedListTypeDef()
+ )
+ {
+ return typeExpr;
+ }
+}
+
+RecordTypeDefinition RecordTypeDef() throws ParseException:
+{
+ RecordTypeDefinition recType = new RecordTypeDefinition();
+ RecordTypeDefinition.RecordKind recordKind = null;
+}
+{
+ ( "closed" { recordKind = RecordTypeDefinition.RecordKind.CLOSED; }
+ | "open" { recordKind = RecordTypeDefinition.RecordKind.OPEN; } )?
+ <LEFTBRACE>
+ {
+ String hint = getHint(token);
+ if (hint != null) {
+ String splits[] = hint.split(" +");
+ if (splits[0].equals(GEN_FIELDS_HINT)) {
+ if (splits.length != 5) {
+ throw new ParseException("Expecting: /*+ gen-fields <type> <min> <max> <prefix>*/");
+ }
+ if (!splits[1].equals("int")) {
+ throw new ParseException("The only supported type for gen-fields is int.");
+ }
+ UndeclaredFieldsDataGen ufdg = new UndeclaredFieldsDataGen(UndeclaredFieldsDataGen.Type.INT,
+ Integer.parseInt(splits[2]), Integer.parseInt(splits[3]), splits[4]);
+ recType.setUndeclaredFieldsDataGen(ufdg);
+ }
+ }
+
+ }
+ (
+ RecordField(recType)
+ ( <COMMA> RecordField(recType) )*
+ )?
+ <RIGHTBRACE>
+ {
+ if (recordKind == null) {
+ recordKind = RecordTypeDefinition.RecordKind.OPEN;
+ }
+ recType.setRecordKind(recordKind);
+ return recType;
+ }
+}
+
+void RecordField(RecordTypeDefinition recType) throws ParseException:
+{
+ String fieldName;
+ TypeExpression type = null;
+ boolean nullable = false;
+}
+{
+ fieldName = Identifier()
+ {
+ String hint = getHint(token);
+ IRecordFieldDataGen rfdg = hint != null ? parseFieldDataGen(hint) : null;
+ }
+ <COLON> type = TypeExpr() (<QUES> { nullable = true; } )?
+ {
+ recType.addField(fieldName, type, nullable, rfdg);
+ }
+}
+
+TypeReferenceExpression TypeReference() throws ParseException:
+{
+ String id = null;
+}
+{
+ id = Identifier()
+ {
+ if (id.equalsIgnoreCase("int")) {
+ id = "int64";
+ }
+
+ return new TypeReferenceExpression(new Identifier(id));
+ }
+}
+
+OrderedListTypeDefinition OrderedListTypeDef() throws ParseException:
+{
+ TypeExpression type = null;
+}
+{
+ <LEFTBRACKET>
+ ( type = TypeExpr() )
+ <RIGHTBRACKET>
+ {
+ return new OrderedListTypeDefinition(type);
+ }
+}
+
+
+UnorderedListTypeDefinition UnorderedListTypeDef() throws ParseException:
+{
+ TypeExpression type = null;
+}
+{
+ <LEFTDBLBRACE>
+ ( type = TypeExpr() )
+ <RIGHTDBLBRACE>
+ {
+ return new UnorderedListTypeDefinition(type);
+ }
+}
+
+FunctionName FunctionName() throws ParseException:
+{
+ String first = null;
+ String second = null;
+ String third = null;
+ boolean secondAfterDot = false;
+}
+{
+ first = Identifier()
+ {
+ FunctionName result = new FunctionName();
+ result.hint = getHint(token);
+ }
+ ( <DOT> second = Identifier()
+ {
+ secondAfterDot = true;
+ }
+ ("#" third = Identifier())? | "#" second = Identifier() )?
+ {
+ if (second == null) {
+ result.dataverse = defaultDataverse;
+ result.library = null;
+ result.function = first;
+ } else if (third == null) {
+ if (secondAfterDot) {
+ result.dataverse = first;
+ result.library = null;
+ result.function = second;
+ } else {
+ result.dataverse = defaultDataverse;
+ result.library = first;
+ result.function = second;
+ }
+ } else {
+ result.dataverse = first;
+ result.library = second;
+ result.function = third;
+ }
+
+ if (result.function.equalsIgnoreCase("int")) {
+ result.function = "int64";
+ }
+ return result;
+ }
+}
+
+
+Pair<Identifier,Identifier> TypeName() throws ParseException:
+{
+ Pair<Identifier,Identifier> name = null;
+}
+{
+ name = QualifiedName()
+ {
+ if (name.first == null) {
+ name.first = new Identifier(defaultDataverse);
+ }
+ return name;
+ }
+}
+
+String Identifier() throws ParseException:
+{
+ String lit = null;
+}
+{
+ (<IDENTIFIER>
+ {
+ return token.image;
+ }
+ | lit = StringLiteral()
+ {
+ return lit;
+ }
+ )
+}
+
+Pair<List<String>, TypeExpression> OpenField() throws ParseException:
+{
+ TypeExpression fieldType = null;
+ List<String> fieldList = null;
+}
+{
+ fieldList = NestedField()
+ ( <COLON> fieldType = IndexedTypeExpr() )?
+ {
+ return new Pair<List<String>, TypeExpression>(fieldList, fieldType);
+ }
+}
+
+List<String> NestedField() throws ParseException:
+{
+ List<String> exprList = new ArrayList<String>();
+ String lit = null;
+}
+{
+ lit = Identifier()
+ {
+ exprList.add(lit);
+ }
+ (<DOT>
+ lit = Identifier()
+ {
+ exprList.add(lit);
+ }
+ )*
+ {
+ return exprList;
+ }
+}
+
+
+
+String StringLiteral() throws ParseException:
+{
+}
+{
+ <STRING_LITERAL>
+ {
+ return removeQuotesAndEscapes(token.image);
+ }
+}
+
+Pair<Identifier,Identifier> QualifiedName() throws ParseException:
+{
+ String first = null;
+ String second = null;
+}
+{
+ first = Identifier() (<DOT> second = Identifier())?
+ {
+ Identifier id1 = null;
+ Identifier id2 = null;
+ if (second == null) {
+ id2 = new Identifier(first);
+ } else
+ {
+ id1 = new Identifier(first);
+ id2 = new Identifier(second);
+ }
+ return new Pair<Identifier,Identifier>(id1, id2);
+ }
+}
+
+Triple<Identifier,Identifier,Identifier> DoubleQualifiedName() throws ParseException:
+{
+ String first = null;
+ String second = null;
+ String third = null;
+}
+{
+ first = Identifier() <DOT> second = Identifier() (<DOT> third = Identifier())?
+ {
+ Identifier id1 = null;
+ Identifier id2 = null;
+ Identifier id3 = null;
+ if (third == null) {
+ id2 = new Identifier(first);
+ id3 = new Identifier(second);
+ } else {
+ id1 = new Identifier(first);
+ id2 = new Identifier(second);
+ id3 = new Identifier(third);
+ }
+ return new Triple<Identifier,Identifier,Identifier>(id1, id2, id3);
+ }
+}
+
+FunctionDecl FunctionDeclaration() throws ParseException:
+{
+ FunctionDecl funcDecl;
+ FunctionSignature signature;
+ String functionName;
+ List<VarIdentifier> paramList = new ArrayList<VarIdentifier>();
+ Expression funcBody;
+ createNewScope();
+}
+{
+ "declare" "function" functionName = Identifier()
+ paramList = ParameterList()
+ <LEFTBRACE> funcBody = Expression() <RIGHTBRACE>
+ {
+ signature = new FunctionSignature(defaultDataverse, functionName, paramList.size());
+ getCurrentScope().addFunctionDescriptor(signature, false);
+ funcDecl = new FunctionDecl(signature, paramList, funcBody);
+ removeCurrentScope();
+ return funcDecl;
+ }
+}
+
+
+Query Query() throws ParseException:
+{
+ Query query = new Query();
+ // we set the pointers to the dataverses and datasets lists to fill them with entities to be locked
+ setDataverses(query.getDataverses());
+ setDatasets(query.getDatasets());
+ Expression expr;
+}
+{
+ expr = Expression()
+ {
+ query.setBody(expr);
+ query.setVarCounter(getVarCounter());
+ // we remove the pointers to the locked entities before we return the query object
+ setDataverses(null);
+ setDatasets(null);
+ return query;
+ }
+
+}
+
+
+
+Expression Expression():
+{
+ Expression expr = null;
+ Expression exprP = null;
+}
+{
+(
+
+//OperatorExpr | IfThenElse | FLWOGRExpression | QuantifiedExpression
+ expr = OperatorExpr()
+ | expr = IfThenElse()
+ | expr = FLWOGR()
+ | expr = QuantifiedExpression()
+
+
+)
+ {
+ return (exprP==null) ? expr : exprP;
+ }
+}
+
+
+
+Expression OperatorExpr()throws ParseException:
+{
+ OperatorExpr op = null;
+ Expression operand = null;
+}
+{
+ operand = AndExpr()
+ (
+
+ <OR>
+ {
+ if (op == null) {
+ op = new OperatorExpr();
+ op.addOperand(operand);
+ op.setCurrentop(true);
+ }
+ op.addOperator(token.image);
+ }
+
+ operand = AndExpr()
+ {
+ op.addOperand(operand);
+ }
+
+ )*
+
+ {
+ return op==null? operand: op;
+ }
+}
+
+Expression AndExpr()throws ParseException:
+{
+ OperatorExpr op = null;
+ Expression operand = null;
+}
+{
+ operand = RelExpr()
+ (
+
+ <AND>
+ {
+ if (op == null) {
+ op = new OperatorExpr();
+ op.addOperand(operand);
+ op.setCurrentop(true);
+ }
+ op.addOperator(token.image);
+ }
+
+ operand = RelExpr()
+ {
+ op.addOperand(operand);
+ }
+
+ )*
+
+ {
+ return op==null? operand: op;
+ }
+}
+
+
+
+Expression RelExpr()throws ParseException:
+{
+ OperatorExpr op = null;
+ Expression operand = null;
+ boolean broadcast = false;
+ IExpressionAnnotation annotation = null;
+}
+{
+ operand = AddExpr()
+ {
+ if (operand instanceof VariableExpr) {
+ String hint = getHint(token);
+ if (hint != null && hint.equals(BROADCAST_JOIN_HINT)) {
+ broadcast = true;
+ }
+ }
+ }
+
+ (
+ LOOKAHEAD(2)( <LT> | <GT> | <LE> | <GE> | <EQ> | <NE> |<SIMILAR>)
+ {
+ String mhint = getHint(token);
+ if (mhint != null) {
+ if (mhint.equals(INDEXED_NESTED_LOOP_JOIN_HINT)) {
+ annotation = IndexedNLJoinExpressionAnnotation.INSTANCE;
+ } else if (mhint.equals(SKIP_SECONDARY_INDEX_SEARCH_HINT)) {
+ annotation = SkipSecondaryIndexSearchExpressionAnnotation.INSTANCE;
+ }
+ }
+ if (op == null) {
+ op = new OperatorExpr();
+ op.addOperand(operand, broadcast);
+ op.setCurrentop(true);
+ broadcast = false;
+ }
+ op.addOperator(token.image);
+ }
+
+ operand = AddExpr()
+ {
+ broadcast = false;
+ if (operand instanceof VariableExpr) {
+ String hint = getHint(token);
+ if (hint != null && hint.equals(BROADCAST_JOIN_HINT)) {
+ broadcast = true;
+ }
+ }
+ op.addOperand(operand, broadcast);
+ }
+ )?
+
+ {
+ if (annotation != null) {
+ op.addHint(annotation);
+ }
+ return op==null? operand: op;
+ }
+}
+
+Expression AddExpr()throws ParseException:
+{
+ OperatorExpr op = null;
+ Expression operand = null;
+}
+{
+ operand = MultExpr()
+
+ ( (<PLUS> | <MINUS>)
+ {
+ if (op == null) {
+ op = new OperatorExpr();
+ op.addOperand(operand);
+ op.setCurrentop(true);
+ }
+ ((OperatorExpr)op).addOperator(token.image);
+ }
+
+ operand = MultExpr()
+ {
+ op.addOperand(operand);
+ }
+ )*
+
+ {
+ return op==null? operand: op;
+ }
+}
+
+Expression MultExpr()throws ParseException:
+{
+ OperatorExpr op = null;
+ Expression operand = null;
+}
+{
+ operand = UnionExpr()
+
+ (( <MUL> | <DIV> | <MOD> | <CARET> | <IDIV>)
+ {
+ if (op == null) {
+ op = new OperatorExpr();
+ op.addOperand(operand);
+ op.setCurrentop(true);
+ }
+ op.addOperator(token.image);
+ }
+ operand = UnionExpr()
+ {
+ op.addOperand(operand);
+ }
+ )*
+
+ {
+ return op==null?operand:op;
+ }
+}
+
+Expression UnionExpr() throws ParseException:
+{
+ UnionExpr union = null;
+ Expression operand1 = null;
+ Expression operand2 = null;
+}
+{
+ operand1 = UnaryExpr()
+ (<UNION>
+ (operand2 = UnaryExpr()) {
+ if (union == null) {
+ union = new UnionExpr();
+ union.addExpr(operand1);
+ }
+ union.addExpr(operand2);
+ } )*
+ {
+ return (union == null)? operand1: union;
+ }
+}
+
+Expression UnaryExpr() throws ParseException:
+{
+ Expression uexpr = null;
+ Expression expr = null;
+}
+{
+ ( (<PLUS> | <MINUS>)
+ {
+ uexpr = new UnaryExpr();
+ if("+".equals(token.image))
+ ((UnaryExpr)uexpr).setSign(Sign.POSITIVE);
+ else if("-".equals(token.image))
+ ((UnaryExpr)uexpr).setSign(Sign.NEGATIVE);
+ else
+ throw new ParseException();
+ }
+ )?
+
+ expr = ValueExpr()
+ {
+ if(uexpr!=null){
+ ((UnaryExpr)uexpr).setExpr(expr);
+ return uexpr;
+ }
+ else{
+ return expr;
+ }
+ }
+}
+
+Expression ValueExpr()throws ParseException:
+{
+ Expression expr = null;
+ Identifier ident = null;
+ AbstractAccessor fa = null;
+ Expression indexExpr = null;
+}
+{
+ expr = PrimaryExpr() ( ident = Field()
+ {
+ fa = (fa == null ? new FieldAccessor(expr, ident)
+ : new FieldAccessor(fa, ident));
+ }
+ | indexExpr = Index()
+ {
+ fa = (fa == null ? new IndexAccessor(expr, indexExpr)
+ : new IndexAccessor(fa, indexExpr));
+ }
+ )*
+ {
+ return fa == null ? expr : fa;
+ }
+}
+
+Identifier Field() throws ParseException:
+{
+ String ident = null;
+}
+{
+ <DOT> ident = Identifier()
+ {
+ return new Identifier(ident);
+ }
+}
+
+Expression Index() throws ParseException:
+{
+ Expression expr = null;
+}
+{
+ <LEFTBRACKET> ( expr = Expression()
+ {
+ if(expr.getKind() == Expression.Kind.LITERAL_EXPRESSION)
+ {
+ Literal lit = ((LiteralExpr)expr).getValue();
+ if(lit.getLiteralType() != Literal.Type.INTEGER &&
+ lit.getLiteralType() != Literal.Type.LONG) {
+ throw new ParseException("Index should be an INTEGER");
+ }
+ }
+ }
+
+ | <QUES> // ANY
+
+ )
+
+ <RIGHTBRACKET>
+ {
+ return expr;
+ }
+}
+
+
+Expression PrimaryExpr()throws ParseException:
+{
+ Expression expr = null;
+}
+{
+ ( LOOKAHEAD(2)
+ expr = FunctionCallExpr()
+ | expr = Literal()
+ | expr = DatasetAccessExpression()
+ | expr = VariableRef()
+ {
+ if(((VariableExpr)expr).getIsNewVar() == true)
+ throw new ParseException("can't find variable " + ((VariableExpr)expr).getVar());
+ }
+ | expr = ListConstructor()
+ | expr = RecordConstructor()
+ | expr = ParenthesizedExpression()
+ )
+ {
+ return expr;
+ }
+}
+
+Expression Literal() throws ParseException:
+{
+ LiteralExpr lit = new LiteralExpr();
+ String str = null;
+}
+{
+ ( str = StringLiteral()
+ {
+ lit.setValue(new StringLiteral(str));
+ }
+ | <INTEGER_LITERAL>
+ {
+ lit.setValue(new LongIntegerLiteral(new Long(token.image)));
+ }
+ | <FLOAT_LITERAL>
+ {
+ lit.setValue(new FloatLiteral(new Float(token.image)));
+ }
+ | <DOUBLE_LITERAL>
+ {
+ lit.setValue(new DoubleLiteral(new Double(token.image)));
+ }
+ | <NULL>
+ {
+ lit.setValue(NullLiteral.INSTANCE);
+ }
+ | <TRUE>
+ {
+ lit.setValue(TrueLiteral.INSTANCE);
+ }
+ | <FALSE>
+ {
+ lit.setValue(FalseLiteral.INSTANCE);
+ }
+ )
+ {
+ return lit;
+ }
+}
+
+
+VariableExpr VariableRef() throws ParseException:
+{
+ VariableExpr varExp = new VariableExpr();
+ VarIdentifier var = new VarIdentifier();
+}
+{
+ <VARIABLE>
+ {
+ String varName = token.image;
+ Identifier ident = lookupSymbol(varName);
+ if (isInForbiddenScopes(varName)) {
+ throw new ParseException("Inside limit clauses, it is disallowed to reference a variable having the same name as any variable bound in the same scope as the limit clause.");
+ }
+ if(ident != null) { // exist such ident
+ varExp.setIsNewVar(false);
+ varExp.setVar((VarIdentifier)ident);
+ } else {
+ varExp.setVar(var);
+ }
+ var.setValue(varName);
+ return varExp;
+ }
+}
+
+
+VariableExpr Variable() throws ParseException:
+{
+ VariableExpr varExp = new VariableExpr();
+ VarIdentifier var = new VarIdentifier();
+}
+{
+ <VARIABLE>
+ {
+ Identifier ident = lookupSymbol(token.image);
+ if(ident != null) { // exist such ident
+ varExp.setIsNewVar(false);
+ }
+ varExp.setVar(var);
+ var.setValue(token.image);
+ return varExp;
+ }
+}
+
+Expression ListConstructor() throws ParseException:
+{
+ Expression expr = null;
+}
+{
+ (
+ expr = OrderedListConstructor() | expr = UnorderedListConstructor()
+ )
+
+ {
+ return expr;
+ }
+}
+
+
+ListConstructor OrderedListConstructor() throws ParseException:
+{
+ ListConstructor expr = new ListConstructor();
+ List<Expression> exprList = null;
+ expr.setType(ListConstructor.Type.ORDERED_LIST_CONSTRUCTOR);
+}
+{
+ <LEFTBRACKET> exprList = ExpressionList() <RIGHTBRACKET>
+ {
+ expr.setExprList(exprList);
+ return expr;
+ }
+}
+
+ListConstructor UnorderedListConstructor() throws ParseException:
+{
+ ListConstructor expr = new ListConstructor();
+ List<Expression> exprList = null;
+ expr.setType(ListConstructor.Type.UNORDERED_LIST_CONSTRUCTOR);
+}
+{
+ <LEFTDBLBRACE> exprList = ExpressionList() <RIGHTDBLBRACE>
+ {
+ expr.setExprList(exprList);
+ return expr;
+ }
+}
+
+List<Expression> ExpressionList() throws ParseException:
+{
+ Expression expr = null;
+ List<Expression> list = null;
+ List<Expression> exprList = new ArrayList<Expression>();
+}
+{
+ (
+ expr = Expression() { exprList.add(expr); }
+ (LOOKAHEAD(1) <COMMA> list = ExpressionList() { exprList.addAll(list); })?
+ )?
+ (LOOKAHEAD(1) Comma())?
+ {
+ return exprList;
+ }
+}
+
+void Comma():
+{}
+{
+ <COMMA>
+}
+
+RecordConstructor RecordConstructor() throws ParseException:
+{
+ RecordConstructor expr = new RecordConstructor();
+ FieldBinding tmp = null;
+ List<FieldBinding> fbList = new ArrayList<FieldBinding>();
+}
+{
+ <LEFTBRACE> (tmp = FieldBinding()
+ {
+ fbList.add(tmp);
+ }
+ (<COMMA> tmp = FieldBinding() { fbList.add(tmp); })*)? <RIGHTBRACE>
+ {
+ expr.setFbList(fbList);
+ return expr;
+ }
+}
+
+FieldBinding FieldBinding() throws ParseException:
+{
+ FieldBinding fb = new FieldBinding();
+ Expression left, right;
+}
+{
+ left = Expression() <COLON> right = Expression()
+ {
+ fb.setLeftExpr(left);
+ fb.setRightExpr(right);
+ return fb;
+ }
+}
+
+
+Expression FunctionCallExpr() throws ParseException:
+{
+ CallExpr callExpr;
+ List<Expression> argList = new ArrayList<Expression>();
+ Expression tmp;
+ int arity = 0;
+ FunctionName funcName = null;
+ String hint = null;
+}
+{
+ funcName = FunctionName()
+ {
+ hint = funcName.hint;
+ }
+ <LEFTPAREN> (tmp = Expression()
+ {
+ argList.add(tmp);
+ arity ++;
+ }
+ (<COMMA> tmp = Expression()
+ {
+ argList.add(tmp);
+ arity++;
+ }
+ )*)? <RIGHTPAREN>
+ {
+ // TODO use funcName.library
+ String fqFunctionName = funcName.library == null ? funcName.function : funcName.library + "#" + funcName.function;
+ FunctionSignature signature
+ = lookupFunctionSignature(funcName.dataverse, fqFunctionName, arity);
+ if (signature == null) {
+ signature = new FunctionSignature(funcName.dataverse, fqFunctionName, arity);
+ }
+ callExpr = new CallExpr(signature,argList);
+ if (hint != null) {
+ if (hint.startsWith(INDEXED_NESTED_LOOP_JOIN_HINT)) {
+ callExpr.addHint(IndexedNLJoinExpressionAnnotation.INSTANCE);
+ } else if (hint.startsWith(SKIP_SECONDARY_INDEX_SEARCH_HINT)) {
+ callExpr.addHint(SkipSecondaryIndexSearchExpressionAnnotation.INSTANCE);
+ }
+ }
+ return callExpr;
+ }
+}
+
+
+Expression DatasetAccessExpression() throws ParseException:
+{
+ String funcName;
+ String arg1 = null;
+ String arg2 = null;
+ Expression nameArg;
+}
+{
+ <DATASET>
+ {
+ funcName = token.image;
+ }
+ ( ( arg1 = Identifier() ( <DOT> arg2 = Identifier() )? )
+ {
+ String name = arg2 == null ? arg1 : arg1 + "." + arg2;
+ LiteralExpr ds = new LiteralExpr();
+ ds.setValue( new StringLiteral(name) );
+ nameArg = ds;
+ if(arg2 != null){
+ addDataverse(arg1.toString());
+ addDataset(name);
+ } else {
+ addDataset(defaultDataverse + "." + name);
+ }
+ }
+ | ( <LEFTPAREN> nameArg = Expression() <RIGHTPAREN> ) )
+ {
+ String dataverse = MetadataConstants.METADATA_DATAVERSE_NAME;
+ FunctionSignature signature = lookupFunctionSignature(dataverse, funcName, 1);
+ if (signature == null) {
+ signature = new FunctionSignature(dataverse, funcName, 1);
+ }
+ List<Expression> argList = new ArrayList<Expression>();
+ argList.add(nameArg);
+ return new CallExpr(signature, argList);
+ }
+}
+
+Expression ParenthesizedExpression() throws ParseException:
+{
+ Expression expr;
+}
+{
+ <LEFTPAREN> expr = Expression() <RIGHTPAREN>
+ {
+ return expr;
+ }
+}
+
+Expression IfThenElse() throws ParseException:
+{
+ Expression condExpr;
+ Expression thenExpr;
+ Expression elseExpr;
+ IfExpr ifExpr = new IfExpr();
+}
+{
+ <IF> <LEFTPAREN> condExpr = Expression() <RIGHTPAREN> <THEN> thenExpr = Expression() <ELSE> elseExpr = Expression()
+
+ {
+ ifExpr.setCondExpr(condExpr);
+ ifExpr.setThenExpr(thenExpr);
+ ifExpr.setElseExpr(elseExpr);
+ return ifExpr;
+ }
+}
+
+Expression FLWOGR() throws ParseException:
+{
+ FLWOGRExpression flworg = new FLWOGRExpression();
+ List<Clause> clauseList = new ArrayList<Clause>();
+ Expression returnExpr;
+ Clause tmp;
+ createNewScope();
+}
+{
+ (tmp = ForClause() {clauseList.add(tmp);} | tmp = LetClause() {clauseList.add(tmp);})
+ (tmp = Clause() {clauseList.add(tmp);})* (<RETURN>|<SELECT>) returnExpr = Expression()
+
+ {
+ flworg.setClauseList(clauseList);
+ flworg.setReturnExpr(returnExpr);
+ removeCurrentScope();
+ return flworg;
+ }
+}
+
+Clause Clause()throws ParseException :
+{
+ Clause clause;
+}
+{
+ (
+ clause = ForClause()
+ | clause = LetClause()
+ | clause = WhereClause()
+ | clause = OrderbyClause()
+ | clause = GroupClause()
+ | clause = LimitClause()
+ | clause = DistinctClause()
+ )
+ {
+ return clause;
+ }
+}
+
+Clause ForClause()throws ParseException :
+{
+ ForClause fc = new ForClause();
+ VariableExpr varExp;
+ VariableExpr varPos = null;
+ Expression inExp;
+ extendCurrentScope();
+}
+{
+ (<FOR>|<FROM>) varExp = Variable() (<AT> varPos = Variable())? <IN> ( inExp = Expression() )
+ {
+ fc.setVarExpr(varExp);
+ getCurrentScope().addNewVarSymbolToScope(varExp.getVar());
+ fc.setInExpr(inExp);
+ if (varPos != null) {
+ fc.setPosExpr(varPos);
+ getCurrentScope().addNewVarSymbolToScope(varPos.getVar());
+ }
+ return fc;
+ }
+}
+
+Clause LetClause() throws ParseException:
+{
+ LetClause lc = new LetClause();
+ VariableExpr varExp;
+ Expression beExp;
+ extendCurrentScope();
+}
+{
+ (<LET>|<WITH>) varExp = Variable() <ASSIGN> beExp = Expression()
+ {
+ getCurrentScope().addNewVarSymbolToScope(varExp.getVar());
+ lc.setVarExpr(varExp);
+ lc.setBindingExpr(beExp);
+ return lc;
+ }
+}
+
+Clause WhereClause()throws ParseException :
+{
+ WhereClause wc = new WhereClause();
+ Expression whereExpr;
+}
+{
+ <WHERE> whereExpr = Expression()
+ {
+ wc.setWhereExpr(whereExpr);
+ return wc;
+ }
+}
+
+Clause OrderbyClause()throws ParseException :
+{
+ OrderbyClause oc = new OrderbyClause();
+ Expression orderbyExpr;
+ List<Expression> orderbyList = new ArrayList<Expression>();
+ List<OrderbyClause.OrderModifier> modifierList = new ArrayList<OrderbyClause.OrderModifier >();
+ int numOfOrderby = 0;
+}
+{
+ (
+ <ORDER>
+ {
+ String hint = getHint(token);
+ if (hint != null) {
+ if (hint.startsWith(INMEMORY_HINT)) {
+ String splits[] = hint.split(" +");
+ int numFrames = Integer.parseInt(splits[1]);
+ int numTuples = Integer.parseInt(splits[2]);
+ oc.setNumFrames(numFrames);
+ oc.setNumTuples(numTuples);
+ }
+ if (hint.startsWith(RANGE_HINT)) {
+ try{
+ oc.setRangeMap(RangeMapBuilder.parseHint(hint.substring(RANGE_HINT.length())));
+ } catch (AsterixException e) {
+ throw new ParseException(e.getMessage());
+ }
+ }
+ }
+ }
+ <BY> orderbyExpr = Expression()
+ {
+ orderbyList.add(orderbyExpr);
+ OrderbyClause.OrderModifier modif = OrderbyClause.OrderModifier.ASC;
+ }
+ ( (<ASC> { modif = OrderbyClause.OrderModifier.ASC; })
+ | (<DESC> { modif = OrderbyClause.OrderModifier.DESC; }))?
+ {
+ modifierList.add(modif);
+ }
+
+ (<COMMA> orderbyExpr = Expression()
+ {
+ orderbyList.add(orderbyExpr);
+ modif = OrderbyClause.OrderModifier.ASC;
+ }
+ ( (<ASC> { modif = OrderbyClause.OrderModifier.ASC; })
+ | (<DESC> { modif = OrderbyClause.OrderModifier.DESC; }))?
+ {
+ modifierList.add(modif);
+ }
+ )*
+)
+ {
+ oc.setModifierList(modifierList);
+ oc.setOrderbyList(orderbyList);
+ return oc;
+ }
+}
+Clause GroupClause()throws ParseException :
+{
+ GroupbyClause gbc = new GroupbyClause();
+ // GbyVariableExpressionPair pair = new GbyVariableExpressionPair();
+ List<GbyVariableExpressionPair> vePairList = new ArrayList<GbyVariableExpressionPair>();
+ List<GbyVariableExpressionPair> decorPairList = new ArrayList<GbyVariableExpressionPair>();
+ List<VariableExpr> withVarList= new ArrayList<VariableExpr>();
+ VariableExpr var = null;
+ VariableExpr withVar = null;
+ Expression expr = null;
+ VariableExpr decorVar = null;
+ Expression decorExpr = null;
+}
+{
+ {
+ Scope newScope = extendCurrentScopeNoPush(true);
+ // extendCurrentScope(true);
+ }
+ <GROUP>
+ {
+ String hint = getHint(token);
+ if (hint != null && hint.equals(HASH_GROUP_BY_HINT)) {
+ gbc.setHashGroupByHint(true);
+ }
+ }
+ <BY> (LOOKAHEAD(2) var = Variable()
+ {
+ newScope.addNewVarSymbolToScope(var.getVar());
+ } <ASSIGN>)?
+ expr = Expression()
+ {
+ GbyVariableExpressionPair pair1 = new GbyVariableExpressionPair(var, expr);
+ vePairList.add(pair1);
+ }
+ (<COMMA> ( LOOKAHEAD(2) var = Variable()
+ {
+ newScope.addNewVarSymbolToScope(var.getVar());
+ } <ASSIGN>)?
+ expr = Expression()
+ {
+ GbyVariableExpressionPair pair2 = new GbyVariableExpressionPair(var, expr);
+ vePairList.add(pair2);
+ }
+ )*
+ (<DECOR> decorVar = Variable() <ASSIGN> decorExpr = Expression()
+ {
+ newScope.addNewVarSymbolToScope(decorVar.getVar());
+ GbyVariableExpressionPair pair3 = new GbyVariableExpressionPair(decorVar, decorExpr);
+ decorPairList.add(pair3);
+ }
+ (<COMMA> <DECOR> decorVar = Variable() <ASSIGN> decorExpr = Expression()
+ {
+ newScope.addNewVarSymbolToScope(decorVar.getVar());
+ GbyVariableExpressionPair pair4 = new GbyVariableExpressionPair(decorVar, decorExpr);
+ decorPairList.add(pair4);
+ }
+ )*
+ )?
+ (<WITH>|<KEEPING>) withVar = VariableRef()
+ {
+ if(withVar.getIsNewVar()==true)
+ throw new ParseException("can't find variable " + withVar.getVar());
+ withVarList.add(withVar);
+ newScope.addNewVarSymbolToScope(withVar.getVar());
+ }
+ (<COMMA> withVar = VariableRef()
+ {
+ if(withVar.getIsNewVar()==true)
+ throw new ParseException("can't find variable " + withVar.getVar());
+ withVarList.add(withVar);
+ newScope.addNewVarSymbolToScope(withVar.getVar());
+ })*
+ {
+ gbc.setGbyPairList(vePairList);
+ gbc.setDecorPairList(decorPairList);
+ gbc.setWithVarList(withVarList);
+ replaceCurrentScope(newScope);
+ return gbc;
+ }
+}
+
+
+LimitClause LimitClause() throws ParseException:
+{
+ LimitClause lc = new LimitClause();
+ Expression expr;
+ pushForbiddenScope(getCurrentScope());
+}
+{
+ <LIMIT> expr = Expression() { lc.setLimitExpr(expr); }
+ (<OFFSET> expr = Expression() { lc.setOffset(expr); })?
+
+ {
+ popForbiddenScope();
+ return lc;
+ }
+}
+
+DistinctClause DistinctClause() throws ParseException:
+{
+ List<Expression> exprs = new ArrayList<Expression>();
+ Expression expr;
+}
+{
+ <DISTINCT> <BY> expr = Expression()
+ {
+ exprs.add(expr);
+ }
+ (<COMMA> expr = Expression()
+ {
+ exprs.add(expr);
+ }
+ )*
+ {
+ return new DistinctClause(exprs);
+ }
+}
+
+QuantifiedExpression QuantifiedExpression()throws ParseException:
+{
+ QuantifiedExpression qc = new QuantifiedExpression();
+ List<QuantifiedPair> quantifiedList = new ArrayList<QuantifiedPair>();
+ Expression satisfiesExpr;
+ VariableExpr var;
+ Expression inExpr;
+ QuantifiedPair pair;
+}
+{
+ {
+ createNewScope();
+ }
+
+ ( (<SOME> { qc.setQuantifier(QuantifiedExpression.Quantifier.SOME); })
+ | (<EVERY> { qc.setQuantifier(QuantifiedExpression.Quantifier.EVERY); }))
+ var = Variable() <IN> inExpr = Expression()
+ {
+ pair = new QuantifiedPair(var, inExpr);
+ getCurrentScope().addNewVarSymbolToScope(var.getVar());
+ quantifiedList.add(pair);
+ }
+ (
+ <COMMA> var = Variable() <IN> inExpr = Expression()
+ {
+ pair = new QuantifiedPair(var, inExpr);
+ getCurrentScope().addNewVarSymbolToScope(var.getVar());
+ quantifiedList.add(pair);
+ }
+ )*
+ <SATISFIES> satisfiesExpr = Expression()
+ {
+ qc.setSatisfiesExpr(satisfiesExpr);
+ qc.setQuantifiedList(quantifiedList);
+ removeCurrentScope();
+ return qc;
+ }
+}
+
+TOKEN_MGR_DECLS:
+{
+ public int commentDepth = 0;
+ public IntStack lexerStateStack = new IntStack();
+
+ public void pushState() {
+ lexerStateStack.push( curLexState );
+ }
+
+ public void popState(String token) {
+ if (lexerStateStack.size() > 0) {
+ SwitchTo( lexerStateStack.pop() );
+ } else {
+ int errorLine = input_stream.getEndLine();
+ int errorColumn = input_stream.getEndColumn();
+ String msg = "Lexical error at line " + errorLine + ", column " + errorColumn + ". Encountered \"" + token
+ + "\" but state stack is empty.";
+ throw new TokenMgrError(msg, -1);
+ }
+ }
+}
+
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+ <ASC : "asc">
+ | <AT : "at">
+ | <BY : "by">
+ | <DATASET : "dataset">
+ | <DECOR : "decor">
+ | <DESC : "desc">
+ | <DISTINCT : "distinct">
+ | <ELSE : "else">
+ | <EVERY : "every">
+ | <FOR : "for">
+ | <FROM : "from">
+ | <GROUP : "group">
+ | <IF : "if">
+ | <IN : "in">
+ | <LET : "let">
+ | <LIMIT : "limit">
+ | <OFFSET : "offset">
+ | <ORDER : "order">
+ | <RETURN : "return">
+ | <SATISFIES : "satisfies">
+ | <SELECT : "select">
+ | <SOME : "some">
+ | <THEN : "then">
+ | <UNION : "union">
+ | <WHERE : "where">
+ | <WITH : "with">
+ | <KEEPING : "keeping">
+}
+
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+ <CARET : "^">
+ | <DIV : "/">
+ | <IDIV : "idiv">
+ | <MINUS : "-">
+ | <MOD : "%">
+ | <MUL : "*">
+ | <PLUS : "+">
+
+ | <LEFTPAREN : "(">
+ | <RIGHTPAREN : ")">
+ | <LEFTBRACKET : "[">
+ | <RIGHTBRACKET : "]">
+
+ | <COLON : ":">
+ | <COMMA : ",">
+ | <DOT : ".">
+ | <QUES : "?">
+
+ | <LT : "<">
+ | <GT : ">">
+ | <LE : "<=">
+ | <GE : ">=">
+ | <EQ : "=">
+ | <NE : "!=">
+ | <SIMILAR : "~=">
+ | <ASSIGN : ":=">
+
+ | <AND : "and">
+ | <OR : "or">
+}
+
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+ <LEFTBRACE : "{"> { pushState(); } : DEFAULT
+}
+
+<DEFAULT>
+TOKEN :
+{
+ <RIGHTBRACE : "}"> { popState("}"); }
+}
+
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+ <LEFTDBLBRACE : "{{"> { pushState(); } : IN_DBL_BRACE
+}
+
+<IN_DBL_BRACE>
+TOKEN :
+{
+ <RIGHTDBLBRACE : "}}"> { popState("}}"); }
+}
+
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+ <INTEGER_LITERAL : (<DIGIT>)+ >
+}
+
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+ <NULL : "null">
+ | <TRUE : "true">
+ | <FALSE : "false">
+}
+
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+ <#DIGIT : ["0" - "9"]>
+}
+
+<DEFAULT,IN_DBL_BRACE>
+TOKEN:
+{
+ < DOUBLE_LITERAL: <DIGITS>
+ | <DIGITS> ( "." <DIGITS> )?
+ | "." <DIGITS>
+ >
+ | < FLOAT_LITERAL: <DIGITS> ( "f" | "F" )
+ | <DIGITS> ( "." <DIGITS> ( "f" | "F" ) )?
+ | "." <DIGITS> ( "f" | "F" )
+ >
+ | <DIGITS : (<DIGIT>)+ >
+}
+
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+ <#LETTER : ["A" - "Z", "a" - "z"]>
+ | <SPECIALCHARS : ["$", "_", "-"]>
+}
+
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+ // backslash u + 4 hex digits escapes are handled in the underlying JavaCharStream
+ <STRING_LITERAL : ("\"" (
+ <EscapeQuot>
+ | <EscapeBslash>
+ | <EscapeSlash>
+ | <EscapeBspace>
+ | <EscapeFormf>
+ | <EscapeNl>
+ | <EscapeCr>
+ | <EscapeTab>
+ | ~["\"","\\"])* "\"")
+ | ("\'"(
+ <EscapeApos>
+ | <EscapeBslash>
+ | <EscapeSlash>
+ | <EscapeBspace>
+ | <EscapeFormf>
+ | <EscapeNl>
+ | <EscapeCr>
+ | <EscapeTab>
+ | ~["\'","\\"])* "\'")>
+ | < #EscapeQuot: "\\\"" >
+ | < #EscapeApos: "\\\'" >
+ | < #EscapeBslash: "\\\\" >
+ | < #EscapeSlash: "\\/" >
+ | < #EscapeBspace: "\\b" >
+ | < #EscapeFormf: "\\f" >
+ | < #EscapeNl: "\\n" >
+ | < #EscapeCr: "\\r" >
+ | < #EscapeTab: "\\t" >
+}
+
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+ <IDENTIFIER : <LETTER> (<LETTER> | <DIGIT> | <SPECIALCHARS>)*>
+}
+
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+ <VARIABLE : "$" <LETTER> (<LETTER> | <DIGIT> | "_")*>
+}
+
+<DEFAULT,IN_DBL_BRACE>
+SKIP:
+{
+ " "
+ | "\t"
+ | "\r"
+ | "\n"
+}
+
+<DEFAULT,IN_DBL_BRACE>
+SKIP:
+{
+ <"//" (~["\n"])* "\n">
+}
+
+<DEFAULT,IN_DBL_BRACE>
+SKIP:
+{
+ <"//" (~["\n","\r"])* ("\n"|"\r"|"\r\n")?>
+}
+
+<DEFAULT,IN_DBL_BRACE>
+SKIP:
+{
+ <"/*"> { pushState(); } : INSIDE_COMMENT
+}
+
+<INSIDE_COMMENT>
+SPECIAL_TOKEN:
+{
+ <"+"(" ")*(~["*"])*>
+}
+
+<INSIDE_COMMENT>
+SKIP:
+{
+ <"/*"> { pushState(); }
+}
+
+<INSIDE_COMMENT>
+SKIP:
+{
+ <"*/"> { popState("*/"); }
+ | <~[]>
+}