SQL++ parser:
1. refactored asterix-aql to become asterix-lang-common and asterix-lang-aql, where the former is the common part for different languages;
2. added asterix-lang-sqlpp on top of asterix-lang-common;
3. ported parser tests, optimizer tests and runtime tests in asterix-app to their sql++ version, and added parser tests for all the queries.
Change-Id: Ie5af4e3b692ca017ec047a1ba3b404a51beb3a2e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/466
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <tillw@apache.org>
diff --git a/asterix-lang-aql/src/main/javacc/AQL.jj b/asterix-lang-aql/src/main/javacc/AQL.jj
new file mode 100644
index 0000000..bb574d0
--- /dev/null
+++ b/asterix-lang-aql/src/main/javacc/AQL.jj
@@ -0,0 +1,2684 @@
+options {
+
+
+ STATIC = false;
+
+}
+
+
+PARSER_BEGIN(AQLParser)
+
+package org.apache.asterix.lang.aql.parser;
+
+// For AQLParserTokenManager
+import org.apache.xerces.util.IntStack;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.StringReader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.asterix.common.annotations.AutoDataGen;
+import org.apache.asterix.common.annotations.DateBetweenYearsDataGen;
+import org.apache.asterix.common.annotations.DatetimeAddRandHoursDataGen;
+import org.apache.asterix.common.annotations.DatetimeBetweenYearsDataGen;
+import org.apache.asterix.common.annotations.FieldIntervalDataGen;
+import org.apache.asterix.common.annotations.FieldValFileDataGen;
+import org.apache.asterix.common.annotations.FieldValFileSameIndexDataGen;
+import org.apache.asterix.common.annotations.IRecordFieldDataGen;
+import org.apache.asterix.common.annotations.InsertRandIntDataGen;
+import org.apache.asterix.common.annotations.ListDataGen;
+import org.apache.asterix.common.annotations.ListValFileDataGen;
+import org.apache.asterix.common.annotations.SkipSecondaryIndexSearchExpressionAnnotation;
+import org.apache.asterix.common.annotations.TypeDataGen;
+import org.apache.asterix.common.annotations.UndeclaredFieldsDataGen;
+import org.apache.asterix.common.config.DatasetConfig.DatasetType;
+import org.apache.asterix.common.config.DatasetConfig.IndexType;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.lang.aql.clause.DistinctClause;
+import org.apache.asterix.lang.aql.clause.ForClause;
+import org.apache.asterix.lang.aql.expression.FLWOGRExpression;
+import org.apache.asterix.lang.aql.expression.UnionExpr;
+import org.apache.asterix.lang.aql.util.RangeMapBuilder;
+import org.apache.asterix.lang.common.base.Clause;
+import org.apache.asterix.lang.common.base.Expression;
+import org.apache.asterix.lang.common.base.IParser;
+import org.apache.asterix.lang.common.base.Literal;
+import org.apache.asterix.lang.common.base.Statement;
+import org.apache.asterix.lang.common.clause.GroupbyClause;
+import org.apache.asterix.lang.common.clause.LetClause;
+import org.apache.asterix.lang.common.clause.LimitClause;
+import org.apache.asterix.lang.common.clause.OrderbyClause;
+import org.apache.asterix.lang.common.clause.UpdateClause;
+import org.apache.asterix.lang.common.clause.WhereClause;
+import org.apache.asterix.lang.common.context.RootScopeFactory;
+import org.apache.asterix.lang.common.context.Scope;
+import org.apache.asterix.lang.common.expression.AbstractAccessor;
+import org.apache.asterix.lang.common.expression.CallExpr;
+import org.apache.asterix.lang.common.expression.FieldAccessor;
+import org.apache.asterix.lang.common.expression.FieldBinding;
+import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair;
+import org.apache.asterix.lang.common.expression.IfExpr;
+import org.apache.asterix.lang.common.expression.IndexAccessor;
+import org.apache.asterix.lang.common.expression.ListConstructor;
+import org.apache.asterix.lang.common.expression.LiteralExpr;
+import org.apache.asterix.lang.common.expression.OperatorExpr;
+import org.apache.asterix.lang.common.expression.OrderedListTypeDefinition;
+import org.apache.asterix.lang.common.expression.QuantifiedExpression;
+import org.apache.asterix.lang.common.expression.RecordConstructor;
+import org.apache.asterix.lang.common.expression.RecordTypeDefinition;
+import org.apache.asterix.lang.common.expression.TypeExpression;
+import org.apache.asterix.lang.common.expression.TypeReferenceExpression;
+import org.apache.asterix.lang.common.expression.UnaryExpr;
+import org.apache.asterix.lang.common.expression.UnorderedListTypeDefinition;
+import org.apache.asterix.lang.common.expression.VariableExpr;
+import org.apache.asterix.lang.common.expression.UnaryExpr.Sign;
+import org.apache.asterix.lang.common.literal.DoubleLiteral;
+import org.apache.asterix.lang.common.literal.FalseLiteral;
+import org.apache.asterix.lang.common.literal.FloatLiteral;
+import org.apache.asterix.lang.common.literal.LongIntegerLiteral;
+import org.apache.asterix.lang.common.literal.NullLiteral;
+import org.apache.asterix.lang.common.literal.StringLiteral;
+import org.apache.asterix.lang.common.literal.TrueLiteral;
+import org.apache.asterix.lang.common.parser.ScopeChecker;
+import org.apache.asterix.lang.common.statement.CompactStatement;
+import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
+import org.apache.asterix.lang.common.statement.CreateDataverseStatement;
+import org.apache.asterix.lang.common.statement.CreateFeedPolicyStatement;
+import org.apache.asterix.lang.common.statement.CreateFeedStatement;
+import org.apache.asterix.lang.common.statement.CreateFunctionStatement;
+import org.apache.asterix.lang.common.statement.CreateIndexStatement;
+import org.apache.asterix.lang.common.statement.CreatePrimaryFeedStatement;
+import org.apache.asterix.lang.common.statement.CreateSecondaryFeedStatement;
+import org.apache.asterix.lang.common.statement.DatasetDecl;
+import org.apache.asterix.lang.common.statement.DataverseDecl;
+import org.apache.asterix.lang.common.statement.DataverseDropStatement;
+import org.apache.asterix.lang.common.statement.DeleteStatement;
+import org.apache.asterix.lang.common.statement.DisconnectFeedStatement;
+import org.apache.asterix.lang.common.statement.DropStatement;
+import org.apache.asterix.lang.common.statement.ExternalDetailsDecl;
+import org.apache.asterix.lang.common.statement.FeedDropStatement;
+import org.apache.asterix.lang.common.statement.FunctionDecl;
+import org.apache.asterix.lang.common.statement.FunctionDropStatement;
+import org.apache.asterix.lang.common.statement.IndexDropStatement;
+import org.apache.asterix.lang.common.statement.InsertStatement;
+import org.apache.asterix.lang.common.statement.InternalDetailsDecl;
+import org.apache.asterix.lang.common.statement.LoadStatement;
+import org.apache.asterix.lang.common.statement.NodeGroupDropStatement;
+import org.apache.asterix.lang.common.statement.NodegroupDecl;
+import org.apache.asterix.lang.common.statement.Query;
+import org.apache.asterix.lang.common.statement.RefreshExternalDatasetStatement;
+import org.apache.asterix.lang.common.statement.RunStatement;
+import org.apache.asterix.lang.common.statement.SetStatement;
+import org.apache.asterix.lang.common.statement.TypeDecl;
+import org.apache.asterix.lang.common.statement.TypeDropStatement;
+import org.apache.asterix.lang.common.statement.UpdateStatement;
+import org.apache.asterix.lang.common.statement.WriteStatement;
+import org.apache.asterix.lang.common.struct.Identifier;
+import org.apache.asterix.lang.common.struct.QuantifiedPair;
+import org.apache.asterix.lang.common.struct.VarIdentifier;
+import org.apache.asterix.metadata.bootstrap.MetadataConstants;
+import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.algebricks.common.utils.Triple;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionAnnotation;
+import org.apache.hyracks.algebricks.core.algebra.expressions.IndexedNLJoinExpressionAnnotation;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+
+
+public class AQLParser extends ScopeChecker implements IParser {
+
+ // optimizer hints
+ private static final String AUTO_HINT = "auto";
+ private static final String BROADCAST_JOIN_HINT = "bcast";
+ private static final String COMPOSE_VAL_FILES_HINT = "compose-val-files";
+ private static final String DATE_BETWEEN_YEARS_HINT = "date-between-years";
+ private static final String DATETIME_ADD_RAND_HOURS_HINT = "datetime-add-rand-hours";
+ private static final String DATETIME_BETWEEN_YEARS_HINT = "datetime-between-years";
+ private static final String HASH_GROUP_BY_HINT = "hash";
+ private static final String INDEXED_NESTED_LOOP_JOIN_HINT = "indexnl";
+ private static final String INMEMORY_HINT = "inmem";
+ private static final String INSERT_RAND_INT_HINT = "insert-rand-int";
+ private static final String INTERVAL_HINT = "interval";
+ private static final String LIST_HINT = "list";
+ private static final String LIST_VAL_FILE_HINT = "list-val-file";
+ private static final String RANGE_HINT = "range";
+ private static final String SKIP_SECONDARY_INDEX_SEARCH_HINT = "skip-index";
+ private static final String VAL_FILE_HINT = "val-files";
+ private static final String VAL_FILE_SAME_INDEX_HINT = "val-file-same-idx";
+
+ private static final String GEN_FIELDS_HINT = "gen-fields";
+
+ // data generator hints
+ private static final String DGEN_HINT = "dgen";
+
+ private static class IndexParams {
+ public IndexType type;
+ public int gramLength;
+
+ public IndexParams(IndexType type, int gramLength) {
+ this.type = type;
+ this.gramLength = gramLength;
+ }
+ };
+
+ private static class FunctionName {
+ public String dataverse = null;
+ public String library = null;
+ public String function = null;
+ public String hint = null;
+ }
+
+ private static String getHint(Token t) {
+ if (t.specialToken == null) {
+ return null;
+ }
+ String s = t.specialToken.image;
+ int n = s.length();
+ if (n < 2) {
+ return null;
+ }
+ return s.substring(1).trim();
+ }
+
+ private static IRecordFieldDataGen parseFieldDataGen(String hint) throws ParseException {
+ IRecordFieldDataGen rfdg = null;
+ String splits[] = hint.split(" +");
+ if (splits[0].equals(VAL_FILE_HINT)) {
+ File[] valFiles = new File[splits.length - 1];
+ for (int k=1; k<splits.length; k++) {
+ valFiles[k-1] = new File(splits[k]);
+ }
+ rfdg = new FieldValFileDataGen(valFiles);
+ } else if (splits[0].equals(VAL_FILE_SAME_INDEX_HINT)) {
+ rfdg = new FieldValFileSameIndexDataGen(new File(splits[1]), splits[2]);
+ } else if (splits[0].equals(LIST_VAL_FILE_HINT)) {
+ rfdg = new ListValFileDataGen(new File(splits[1]), Integer.parseInt(splits[2]), Integer.parseInt(splits[3]));
+ } else if (splits[0].equals(LIST_HINT)) {
+ rfdg = new ListDataGen(Integer.parseInt(splits[1]), Integer.parseInt(splits[2]));
+ } else if (splits[0].equals(INTERVAL_HINT)) {
+ FieldIntervalDataGen.ValueType vt;
+ if (splits[1].equals("int")) {
+ vt = FieldIntervalDataGen.ValueType.INT;
+ } else if (splits[1].equals("long")) {
+ vt = FieldIntervalDataGen.ValueType.LONG;
+ } else if (splits[1].equals("float")) {
+ vt = FieldIntervalDataGen.ValueType.FLOAT;
+ } else if (splits[1].equals("double")) {
+ vt = FieldIntervalDataGen.ValueType.DOUBLE;
+ } else {
+ throw new ParseException("Unknown type for interval data gen: " + splits[1]);
+ }
+ rfdg = new FieldIntervalDataGen(vt, splits[2], splits[3]);
+ } else if (splits[0].equals(INSERT_RAND_INT_HINT)) {
+ rfdg = new InsertRandIntDataGen(splits[1], splits[2]);
+ } else if (splits[0].equals(DATE_BETWEEN_YEARS_HINT)) {
+ rfdg = new DateBetweenYearsDataGen(Integer.parseInt(splits[1]), Integer.parseInt(splits[2]));
+ } else if (splits[0].equals(DATETIME_BETWEEN_YEARS_HINT)) {
+ rfdg = new DatetimeBetweenYearsDataGen(Integer.parseInt(splits[1]), Integer.parseInt(splits[2]));
+ } else if (splits[0].equals(DATETIME_ADD_RAND_HOURS_HINT)) {
+ rfdg = new DatetimeAddRandHoursDataGen(Integer.parseInt(splits[1]), Integer.parseInt(splits[2]), splits[3]);
+ } else if (splits[0].equals(AUTO_HINT)) {
+ rfdg = new AutoDataGen(splits[1]);
+ }
+ return rfdg;
+ }
+
+ public AQLParser(String s){
+ this(new StringReader(s));
+ super.setInput(s);
+ }
+
+ public static void main(String args[]) throws ParseException, TokenMgrError, IOException, FileNotFoundException, AsterixException {
+ File file = new File(args[0]);
+ Reader fis = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
+ AQLParser parser = new AQLParser(fis);
+ List<Statement> st = parser.parse();
+ //st.accept(new AQLPrintVisitor(), 0);
+ }
+
+ public List<Statement> parse() throws AsterixException {
+ try {
+ return Statement();
+ } catch (Error e) {
+ // this is here as the JavaCharStream that's below the lexer somtimes throws Errors that are not handled
+ // by the ANTLR-generated lexer or parser (e.g it does this for invalid backslash u + 4 hex digits escapes)
+ throw new AsterixException(new ParseException(e.getMessage()));
+ } catch (ParseException e){
+ throw new AsterixException(e.getMessage());
+ }
+ }
+}
+
+PARSER_END(AQLParser)
+
+
+List<Statement> Statement() throws ParseException:
+{
+ scopeStack.push(RootScopeFactory.createRootScope(this));
+ List<Statement> decls = new ArrayList<Statement>();
+ Statement stmt = null;
+}
+{
+ ( stmt = SingleStatement() (";") ?
+ {
+ decls.add(stmt);
+ }
+ )*
+ <EOF>
+ {
+ return decls;
+ }
+}
+
+Statement SingleStatement() throws ParseException:
+{
+ Statement stmt = null;
+}
+{
+ (
+ stmt = DataverseDeclaration()
+ | stmt = FunctionDeclaration()
+ | stmt = CreateStatement()
+ | stmt = LoadStatement()
+ | stmt = DropStatement()
+ | stmt = WriteStatement()
+ | stmt = SetStatement()
+ | stmt = InsertStatement()
+ | stmt = DeleteStatement()
+ | stmt = UpdateStatement()
+ | stmt = FeedStatement()
+ | stmt = CompactStatement()
+ | stmt = Query()
+ | stmt = RefreshExternalDatasetStatement()
+ | stmt = RunStatement()
+ )
+ {
+ return stmt;
+ }
+}
+
+DataverseDecl DataverseDeclaration() throws ParseException:
+{
+ String dvName = null;
+}
+{
+ "use" "dataverse" dvName = Identifier()
+ {
+ defaultDataverse = dvName;
+ return new DataverseDecl(new Identifier(dvName));
+ }
+}
+
+Statement CreateStatement() throws ParseException:
+{
+ String hint = null;
+ boolean dgen = false;
+ Statement stmt = null;
+}
+{
+ "create"
+ (
+ {
+ hint = getHint(token);
+ if (hint != null && hint.startsWith(DGEN_HINT)) {
+ dgen = true;
+ }
+ }
+ stmt = TypeSpecification(hint, dgen)
+ | stmt = NodegroupSpecification()
+ | stmt = DatasetSpecification()
+ | stmt = IndexSpecification()
+ | stmt = DataverseSpecification()
+ | stmt = FunctionSpecification()
+ | stmt = FeedSpecification()
+ | stmt = FeedPolicySpecification()
+ )
+ {
+ return stmt;
+ }
+}
+
+TypeDecl TypeSpecification(String hint, boolean dgen) throws ParseException:
+{
+ Pair<Identifier,Identifier> nameComponents = null;
+ boolean ifNotExists = false;
+ TypeExpression typeExpr = null;
+}
+{
+ "type" nameComponents = TypeName() ifNotExists = IfNotExists()
+ "as" typeExpr = TypeExpr()
+ {
+ long numValues = -1;
+ String filename = null;
+ if (dgen) {
+ String splits[] = hint.split(" +");
+ if (splits.length != 3) {
+ throw new ParseException("Expecting /*+ dgen <filename> <numberOfItems> */");
+ }
+ filename = splits[1];
+ numValues = Long.parseLong(splits[2]);
+ }
+ TypeDataGen tddg = new TypeDataGen(dgen, filename, numValues);
+ return new TypeDecl(nameComponents.first, nameComponents.second, typeExpr, tddg, ifNotExists);
+ }
+}
+
+
+NodegroupDecl NodegroupSpecification() throws ParseException:
+{
+ String name = null;
+ String tmp = null;
+ boolean ifNotExists = false;
+ List<Identifier>ncNames = null;
+}
+{
+ "nodegroup" name = Identifier()
+ ifNotExists = IfNotExists() "on" tmp = Identifier()
+ {
+ ncNames = new ArrayList<Identifier>();
+ ncNames.add(new Identifier(tmp));
+ }
+ ( <COMMA> tmp = Identifier()
+ {
+ ncNames.add(new Identifier(tmp));
+ }
+ )*
+ {
+ return new NodegroupDecl(new Identifier(name), ncNames, ifNotExists);
+ }
+}
+
+DatasetDecl DatasetSpecification() throws ParseException:
+{
+ Pair<Identifier,Identifier> nameComponents = null;
+ boolean ifNotExists = false;
+ String typeName = null;
+ String adapterName = null;
+ Map<String,String> properties = null;
+ Map<String,String> compactionPolicyProperties = null;
+ FunctionSignature appliedFunction = null;
+ List<List<String>> primaryKeyFields = null;
+ String nodeGroupName = null;
+ Map<String,String> hints = new HashMap<String,String>();
+ DatasetDecl dsetDecl = null;
+ boolean autogenerated = false;
+ String compactionPolicy = null;
+ boolean temp = false;
+ List<String> filterField = null;
+}
+{
+ (
+ "external" <DATASET> nameComponents = QualifiedName()
+ <LEFTPAREN> typeName = Identifier() <RIGHTPAREN>
+ ifNotExists = IfNotExists()
+ "using" adapterName = AdapterName() properties = Configuration()
+ ("on" nodeGroupName = Identifier() )?
+ ( "hints" hints = Properties() )?
+ ( "using" "compaction" "policy" compactionPolicy = CompactionPolicy() (compactionPolicyProperties = Configuration())? )?
+ {
+ ExternalDetailsDecl edd = new ExternalDetailsDecl();
+ edd.setAdapter(adapterName);
+ edd.setProperties(properties);
+ dsetDecl = new DatasetDecl(nameComponents.first,
+ nameComponents.second,
+ new Identifier(typeName),
+ nodeGroupName != null? new Identifier(nodeGroupName): null,
+ compactionPolicy,
+ compactionPolicyProperties,
+ hints,
+ DatasetType.EXTERNAL,
+ edd,
+ ifNotExists);
+ }
+
+ | ("internal" | "temporary" {
+ temp = token.image.toLowerCase().equals("temporary");
+ }
+ )?
+ <DATASET> nameComponents = QualifiedName()
+ <LEFTPAREN> typeName = Identifier() <RIGHTPAREN>
+ ifNotExists = IfNotExists()
+ primaryKeyFields = PrimaryKey()
+ ("autogenerated" { autogenerated = true; } )?
+ ("on" nodeGroupName = Identifier() )?
+ ( "hints" hints = Properties() )?
+ ( "using" "compaction" "policy" compactionPolicy = CompactionPolicy() (compactionPolicyProperties = Configuration())? )?
+ ( "with filter on" filterField = NestedField() )?
+ {
+ InternalDetailsDecl idd = new InternalDetailsDecl(primaryKeyFields,
+ autogenerated,
+ filterField,
+ temp);
+ dsetDecl = new DatasetDecl(nameComponents.first,
+ nameComponents.second,
+ new Identifier(typeName),
+ nodeGroupName != null ? new Identifier(nodeGroupName) : null,
+ compactionPolicy,
+ compactionPolicyProperties,
+ hints,
+ DatasetType.INTERNAL,
+ idd,
+ ifNotExists);
+ }
+ )
+ {
+ return dsetDecl;
+ }
+}
+
+RefreshExternalDatasetStatement RefreshExternalDatasetStatement() throws ParseException:
+{
+ RefreshExternalDatasetStatement redss = new RefreshExternalDatasetStatement();
+ Pair<Identifier,Identifier> nameComponents = null;
+ String datasetName = null;
+}
+{
+ "refresh external" <DATASET> nameComponents = QualifiedName()
+ {
+ redss.setDataverseName(nameComponents.first);
+ redss.setDatasetName(nameComponents.second);
+ return redss;
+ }
+}
+
+RunStatement RunStatement() throws ParseException:
+{
+ String system = null;
+ String tmp;
+ ArrayList<String> parameters = new ArrayList<String>();
+ Pair<Identifier,Identifier> nameComponentsFrom = null;
+ Pair<Identifier,Identifier> nameComponentsTo = null;
+}
+{
+ "run" system = Identifier()<LEFTPAREN> ( tmp = Identifier() [<COMMA>]
+ {
+ parameters.add(tmp);
+ }
+ )*<RIGHTPAREN>
+ <FROM> <DATASET> nameComponentsFrom = QualifiedName()
+ "to" <DATASET> nameComponentsTo = QualifiedName()
+ {
+ return new RunStatement(system, parameters, nameComponentsFrom.first, nameComponentsFrom.second, nameComponentsTo.first, nameComponentsTo.second);
+ }
+}
+
+CreateIndexStatement IndexSpecification() throws ParseException:
+{
+ CreateIndexStatement cis = new CreateIndexStatement();
+ String indexName = null;
+ boolean ifNotExists = false;
+ Pair<Identifier,Identifier> nameComponents = null;
+ Pair<List<String>, TypeExpression> fieldPair = null;
+ IndexParams indexType = null;
+ boolean enforced = false;
+}
+{
+ "index" indexName = Identifier()
+ ifNotExists = IfNotExists()
+ "on" nameComponents = QualifiedName()
+ <LEFTPAREN> ( fieldPair = OpenField()
+ {
+ cis.addFieldExprPair(fieldPair);
+ }
+ ) (<COMMA> fieldPair = OpenField()
+ {
+ cis.addFieldExprPair(fieldPair);
+ }
+ )* <RIGHTPAREN> ( "type" indexType = IndexType() )? ( "enforced" { enforced = true; } )?
+ {
+ cis.setIndexName(new Identifier(indexName));
+ cis.setIfNotExists(ifNotExists);
+ cis.setDataverseName(nameComponents.first);
+ cis.setDatasetName(nameComponents.second);
+ if (indexType != null) {
+ cis.setIndexType(indexType.type);
+ cis.setGramLength(indexType.gramLength);
+ }
+ cis.setEnforced(enforced);
+ return cis;
+ }
+}
+
+String CompactionPolicy() throws ParseException :
+{
+ String compactionPolicy = null;
+}
+{
+ compactionPolicy = Identifier()
+ {
+ return compactionPolicy;
+ }
+}
+
+String FilterField() throws ParseException :
+{
+ String filterField = null;
+}
+{
+ filterField = Identifier()
+ {
+ return filterField;
+ }
+}
+
+IndexParams IndexType() throws ParseException:
+{
+ IndexType type = null;
+ int gramLength = 0;
+}
+{
+ ("btree"
+ {
+ type = IndexType.BTREE;
+ }
+ | "rtree"
+ {
+ type = IndexType.RTREE;
+ }
+ | "keyword"
+ {
+ type = IndexType.LENGTH_PARTITIONED_WORD_INVIX;
+ }
+ | "ngram" <LEFTPAREN> <INTEGER_LITERAL>
+ {
+ type = IndexType.LENGTH_PARTITIONED_NGRAM_INVIX;
+ gramLength = Integer.valueOf(token.image);
+ }
+ <RIGHTPAREN>)
+ {
+ return new IndexParams(type, gramLength);
+ }
+}
+
+CreateDataverseStatement DataverseSpecification() throws ParseException :
+{
+ String dvName = null;
+ boolean ifNotExists = false;
+ String format = null;
+}
+{
+ "dataverse" dvName = Identifier()
+ ifNotExists = IfNotExists()
+ ( "with format" format = StringLiteral() )?
+ {
+ return new CreateDataverseStatement(new Identifier(dvName), format, ifNotExists);
+ }
+}
+
+CreateFunctionStatement FunctionSpecification() throws ParseException:
+{
+ FunctionSignature signature;
+ boolean ifNotExists = false;
+ List<VarIdentifier> paramList = new ArrayList<VarIdentifier>();
+ String functionBody;
+ VarIdentifier var = null;
+ Expression functionBodyExpr;
+ Token beginPos;
+ Token endPos;
+ FunctionName fctName = null;
+
+ createNewScope();
+}
+{
+ "function" fctName = FunctionName()
+ ifNotExists = IfNotExists()
+ paramList = ParameterList()
+ <LEFTBRACE>
+ {
+ beginPos = token;
+ }
+ functionBodyExpr = Expression() <RIGHTBRACE>
+ {
+ endPos = token;
+ functionBody = extractFragment(beginPos.beginLine, beginPos.beginColumn, endPos.beginLine, endPos.beginColumn);
+ // TODO use fctName.library
+ signature = new FunctionSignature(fctName.dataverse, fctName.function, paramList.size());
+ getCurrentScope().addFunctionDescriptor(signature, false);
+ removeCurrentScope();
+ return new CreateFunctionStatement(signature, paramList, functionBody, ifNotExists);
+ }
+}
+
+CreateFeedStatement FeedSpecification() throws ParseException:
+{
+ Pair<Identifier,Identifier> nameComponents = null;
+ boolean ifNotExists = false;
+ String adapterName = null;
+ Map<String,String> properties = null;
+ FunctionSignature appliedFunction = null;
+ CreateFeedStatement cfs = null;
+ Pair<Identifier,Identifier> sourceNameComponents = null;
+
+}
+{
+ (
+ "secondary" "feed" nameComponents = QualifiedName() ifNotExists = IfNotExists()
+ <FROM> "feed" sourceNameComponents = QualifiedName() (appliedFunction = ApplyFunction())?
+ {
+ cfs = new CreateSecondaryFeedStatement(nameComponents,
+ sourceNameComponents, appliedFunction, ifNotExists);
+ }
+ |
+ ("primary")? "feed" nameComponents = QualifiedName() ifNotExists = IfNotExists()
+ "using" adapterName = AdapterName() properties = Configuration() (appliedFunction = ApplyFunction())?
+ {
+ cfs = new CreatePrimaryFeedStatement(nameComponents,
+ adapterName, properties, appliedFunction, ifNotExists);
+ }
+ )
+ {
+ return cfs;
+ }
+}
+
+CreateFeedPolicyStatement FeedPolicySpecification() throws ParseException:
+{
+ String policyName = null;
+ String basePolicyName = null;
+ String sourcePolicyFile = null;
+ String definition = null;
+ boolean ifNotExists = false;
+ Map<String,String> properties = null;
+ CreateFeedPolicyStatement cfps = null;
+}
+{
+ (
+ "ingestion" "policy" policyName = Identifier() ifNotExists = IfNotExists()
+ <FROM>
+ ("policy" basePolicyName = Identifier() properties = Configuration() ("definition" definition = StringLiteral())?
+ {
+ cfps = new CreateFeedPolicyStatement(policyName,
+ basePolicyName, properties, definition, ifNotExists);
+ }
+ | "path" sourcePolicyFile = Identifier() ("definition" definition = StringLiteral())?
+ {
+ cfps = new CreateFeedPolicyStatement(policyName, sourcePolicyFile, definition, ifNotExists);
+ }
+ )
+
+ )
+ {
+ return cfps;
+ }
+}
+
+
+
+List<VarIdentifier> ParameterList() throws ParseException:
+{
+ List<VarIdentifier> paramList = new ArrayList<VarIdentifier>();
+ VarIdentifier var = null;
+}
+{
+ <LEFTPAREN> (<VARIABLE>
+ {
+ var = new VarIdentifier();
+ var.setValue(token.image);
+ paramList.add(var);
+ getCurrentScope().addNewVarSymbolToScope(var);
+ }
+ (<COMMA> <VARIABLE>
+ {
+ var = new VarIdentifier();
+ var.setValue(token.image);
+ paramList.add(var);
+ getCurrentScope().addNewVarSymbolToScope(var);
+ }
+ )*)? <RIGHTPAREN>
+ {
+ return paramList;
+ }
+}
+
+boolean IfNotExists() throws ParseException:
+{
+}
+{
+ ( "if not exists"
+ {
+ return true;
+ }
+ )?
+ {
+ return false;
+ }
+}
+
+FunctionSignature ApplyFunction() throws ParseException:
+{
+ FunctionName functioName = null;
+ FunctionSignature funcSig = null;
+}
+{
+ "apply" "function" functioName = FunctionName()
+ {
+ String fqFunctionName = functioName.library == null ? functioName.function : functioName.library + "#" + functioName.function;
+ return new FunctionSignature(functioName.dataverse, fqFunctionName, 1);
+ }
+}
+
+String GetPolicy() throws ParseException:
+{
+ String policy = null;
+}
+{
+ "using" "policy" policy = Identifier()
+ {
+ return policy;
+ }
+
+}
+
+FunctionSignature FunctionSignature() throws ParseException:
+{
+ FunctionName fctName = null;
+ int arity = 0;
+}
+{
+ fctName = FunctionName() "@" <INTEGER_LITERAL>
+ {
+ arity = new Integer(token.image);
+ if (arity < 0 && arity != FunctionIdentifier.VARARGS) {
+ throw new ParseException(" invalid arity:" + arity);
+ }
+
+ // TODO use fctName.library
+ String fqFunctionName = fctName.library == null ? fctName.function : fctName.library + "#" + fctName.function;
+ return new FunctionSignature(fctName.dataverse, fqFunctionName, arity);
+ }
+}
+
+List<List<String>> PrimaryKey() throws ParseException:
+{
+ List<String> tmp = null;
+ List<List<String>> primaryKeyFields = new ArrayList<List<String>>();
+}
+{
+ "primary" "key" tmp = NestedField()
+ {
+ primaryKeyFields.add(tmp);
+ }
+ ( <COMMA> tmp = NestedField()
+ {
+ primaryKeyFields.add(tmp);
+ }
+ )*
+ {
+ return primaryKeyFields;
+ }
+}
+
+Statement DropStatement() throws ParseException:
+{
+ String id = null;
+ Pair<Identifier,Identifier> pairId = null;
+ Triple<Identifier,Identifier,Identifier> tripleId = null;
+ FunctionSignature funcSig = null;
+ boolean ifExists = false;
+ Statement stmt = null;
+}
+{
+ "drop"
+ (
+ <DATASET> pairId = QualifiedName() ifExists = IfExists()
+ {
+ stmt = new DropStatement(pairId.first, pairId.second, ifExists);
+ }
+ | "index" tripleId = DoubleQualifiedName() ifExists = IfExists()
+ {
+ stmt = new IndexDropStatement(tripleId.first, tripleId.second, tripleId.third, ifExists);
+ }
+ | "nodegroup" id = Identifier() ifExists = IfExists()
+ {
+ stmt = new NodeGroupDropStatement(new Identifier(id), ifExists);
+ }
+ | "type" pairId = TypeName() ifExists = IfExists()
+ {
+ stmt = new TypeDropStatement(pairId.first, pairId.second, ifExists);
+ }
+ | "dataverse" id = Identifier() ifExists = IfExists()
+ {
+ stmt = new DataverseDropStatement(new Identifier(id), ifExists);
+ }
+ | "function" funcSig = FunctionSignature() ifExists = IfExists()
+ {
+ stmt = new FunctionDropStatement(funcSig, ifExists);
+ }
+ | "feed" pairId = QualifiedName() ifExists = IfExists()
+ {
+ stmt = new FeedDropStatement(pairId.first, pairId.second, ifExists);
+ }
+ )
+ {
+ return stmt;
+ }
+}
+
+boolean IfExists() throws ParseException :
+{
+}
+{
+ ( <IF> "exists"
+ {
+ return true;
+ }
+ )?
+ {
+ return false;
+ }
+}
+
+InsertStatement InsertStatement() throws ParseException:
+{
+ Pair<Identifier,Identifier> nameComponents = null;
+ Query query;
+}
+{
+ "insert" "into" <DATASET> nameComponents = QualifiedName() query = Query()
+ {
+ query.setTopLevel(false);
+ return new InsertStatement(nameComponents.first, nameComponents.second, query, getVarCounter());
+ }
+}
+
+DeleteStatement DeleteStatement() throws ParseException:
+{
+ VariableExpr var = null;
+ Expression condition = null;
+ Pair<Identifier, Identifier> nameComponents;
+ // This is related to the new metadata lock management
+ setDataverses(new ArrayList<String>());
+ setDatasets(new ArrayList<String>());
+
+}
+{
+ "delete" var = Variable()
+ {
+ getCurrentScope().addNewVarSymbolToScope(var.getVar());
+ }
+ <FROM> <DATASET> nameComponents = QualifiedName()
+ (<WHERE> condition = Expression())?
+ {
+ // First we get the dataverses and datasets that we want to lock
+ List<String> dataverses = getDataverses();
+ List<String> datasets = getDatasets();
+ // we remove the pointer to the dataverses and datasets
+ setDataverses(null);
+ setDatasets(null);
+ return new DeleteStatement(var, nameComponents.first, nameComponents.second,
+ condition, getVarCounter(), dataverses, datasets);
+ }
+}
+
+UpdateStatement UpdateStatement() throws ParseException:
+{
+ VariableExpr vars;
+ Expression target;
+ Expression condition;
+ UpdateClause uc;
+ List<UpdateClause> ucs = new ArrayList<UpdateClause>();
+}
+{
+ "update" vars = Variable() <IN> target = Expression()
+ <WHERE> condition = Expression()
+ <LEFTPAREN> (uc = UpdateClause()
+ {
+ ucs.add(uc);
+ }
+ (<COMMA> uc = UpdateClause()
+ {
+ ucs.add(uc);
+ }
+ )*) <RIGHTPAREN>
+ {
+ return new UpdateStatement(vars, target, condition, ucs);
+ }
+}
+
+UpdateClause UpdateClause() throws ParseException:
+{
+ Expression target = null;
+ Expression value = null ;
+ InsertStatement is = null;
+ DeleteStatement ds = null;
+ UpdateStatement us = null;
+ Expression condition = null;
+ UpdateClause ifbranch = null;
+ UpdateClause elsebranch = null;
+}
+{
+ ("set" target = Expression() <ASSIGN> value = Expression()
+ | is = InsertStatement()
+ | ds = DeleteStatement()
+ | us = UpdateStatement()
+ | <IF> <LEFTPAREN> condition = Expression() <RIGHTPAREN>
+ <THEN> ifbranch = UpdateClause()
+ [LOOKAHEAD(1) <ELSE> elsebranch = UpdateClause()]
+ {
+ return new UpdateClause(target, value, is, ds, us, condition, ifbranch, elsebranch);
+ }
+ )
+}
+
+Statement SetStatement() throws ParseException:
+{
+ String pn = null;
+ String pv = null;
+}
+{
+ "set" pn = Identifier() pv = StringLiteral()
+ {
+ return new SetStatement(pn, pv);
+ }
+}
+
+Statement WriteStatement() throws ParseException:
+{
+ String nodeName = null;
+ String fileName = null;
+ Query query;
+ String writerClass = null;
+ Pair<Identifier,Identifier> nameComponents = null;
+}
+{
+ "write" "output" "to" nodeName = Identifier() <COLON> fileName = StringLiteral()
+ ( "using" writerClass = StringLiteral() )?
+ {
+ return new WriteStatement(new Identifier(nodeName), fileName, writerClass);
+ }
+}
+
+LoadStatement LoadStatement() throws ParseException:
+{
+ Identifier dataverseName = null;
+ Identifier datasetName = null;
+ boolean alreadySorted = false;
+ String adapterName;
+ Map<String,String> properties;
+ Pair<Identifier,Identifier> nameComponents = null;
+}
+{
+ "load" <DATASET> nameComponents = QualifiedName()
+ {
+ dataverseName = nameComponents.first;
+ datasetName = nameComponents.second;
+ }
+ "using" adapterName = AdapterName() properties = Configuration()
+ ("pre-sorted"
+ {
+ alreadySorted = true;
+ }
+ )?
+ {
+ return new LoadStatement(dataverseName, datasetName, adapterName, properties, alreadySorted);
+ }
+}
+
+
+String AdapterName() throws ParseException :
+{
+ String adapterName = null;
+}
+{
+ adapterName = Identifier()
+ {
+ return adapterName;
+ }
+}
+
+Statement CompactStatement() throws ParseException:
+{
+ Pair<Identifier,Identifier> nameComponents = null;
+ Statement stmt = null;
+}
+{
+ "compact" <DATASET> nameComponents = QualifiedName()
+ {
+ stmt = new CompactStatement(nameComponents.first, nameComponents.second);
+ }
+ {
+ return stmt;
+ }
+}
+
+Statement FeedStatement() throws ParseException:
+{
+ Pair<Identifier,Identifier> feedNameComponents = null;
+ Pair<Identifier,Identifier> datasetNameComponents = null;
+
+ Map<String,String> configuration = null;
+ Statement stmt = null;
+ String policy = null;
+}
+{
+ (
+ "connect" "feed" feedNameComponents = QualifiedName() "to" <DATASET> datasetNameComponents = QualifiedName() (policy = GetPolicy())?
+ {
+ stmt = new ConnectFeedStatement(feedNameComponents, datasetNameComponents, policy, getVarCounter());
+ }
+ | "disconnect" "feed" feedNameComponents = QualifiedName() <FROM> <DATASET> datasetNameComponents = QualifiedName()
+ {
+ stmt = new DisconnectFeedStatement(feedNameComponents, datasetNameComponents);
+ }
+ )
+ {
+ return stmt;
+ }
+}
+
+Map<String,String> Configuration() throws ParseException :
+{
+ Map<String,String> configuration = new LinkedHashMap<String,String>();
+ Pair<String, String> keyValuePair = null;
+}
+{
+ <LEFTPAREN> ( keyValuePair = KeyValuePair()
+ {
+ configuration.put(keyValuePair.first, keyValuePair.second);
+ }
+ ( <COMMA> keyValuePair = KeyValuePair()
+ {
+ configuration.put(keyValuePair.first, keyValuePair.second);
+ }
+ )* )? <RIGHTPAREN>
+ {
+ return configuration;
+ }
+}
+
+Pair<String, String> KeyValuePair() throws ParseException:
+{
+ String key;
+ String value;
+}
+{
+ <LEFTPAREN> key = StringLiteral() <EQ> value = StringLiteral() <RIGHTPAREN>
+ {
+ return new Pair<String, String>(key, value);
+ }
+}
+
+Map<String,String> Properties() throws ParseException:
+{
+ Map<String,String> properties = new HashMap<String,String>();
+ Pair<String, String> property;
+}
+{
+ ( <LEFTPAREN> property = Property()
+ {
+ properties.put(property.first, property.second);
+ }
+ ( <COMMA> property = Property()
+ {
+ properties.put(property.first, property.second);
+ }
+ )* <RIGHTPAREN> )?
+ {
+ return properties;
+ }
+}
+
+Pair<String, String> Property() throws ParseException:
+{
+ String key;
+ String value;
+}
+{
+ key = Identifier() <EQ> ( value = StringLiteral() | <INTEGER_LITERAL>
+ {
+ try {
+ value = "" + Long.valueOf(token.image);
+ } catch (NumberFormatException nfe) {
+ throw new ParseException("inapproriate value: " + token.image);
+ }
+ }
+ )
+ {
+ return new Pair<String, String>(key.toUpperCase(), value);
+ }
+}
+
+TypeExpression IndexedTypeExpr() throws ParseException:
+{
+ TypeExpression typeExpr = null;
+}
+{
+ (
+ typeExpr = TypeReference()
+ | typeExpr = OrderedListTypeDef()
+ | typeExpr = UnorderedListTypeDef()
+ )
+ {
+ return typeExpr;
+ }
+}
+
+TypeExpression TypeExpr() throws ParseException:
+{
+ TypeExpression typeExpr = null;
+}
+{
+ (
+ typeExpr = RecordTypeDef()
+ | typeExpr = TypeReference()
+ | typeExpr = OrderedListTypeDef()
+ | typeExpr = UnorderedListTypeDef()
+ )
+ {
+ return typeExpr;
+ }
+}
+
+RecordTypeDefinition RecordTypeDef() throws ParseException:
+{
+ RecordTypeDefinition recType = new RecordTypeDefinition();
+ RecordTypeDefinition.RecordKind recordKind = null;
+}
+{
+ ( "closed" { recordKind = RecordTypeDefinition.RecordKind.CLOSED; }
+ | "open" { recordKind = RecordTypeDefinition.RecordKind.OPEN; } )?
+ <LEFTBRACE>
+ {
+ String hint = getHint(token);
+ if (hint != null) {
+ String splits[] = hint.split(" +");
+ if (splits[0].equals(GEN_FIELDS_HINT)) {
+ if (splits.length != 5) {
+ throw new ParseException("Expecting: /*+ gen-fields <type> <min> <max> <prefix>*/");
+ }
+ if (!splits[1].equals("int")) {
+ throw new ParseException("The only supported type for gen-fields is int.");
+ }
+ UndeclaredFieldsDataGen ufdg = new UndeclaredFieldsDataGen(UndeclaredFieldsDataGen.Type.INT,
+ Integer.parseInt(splits[2]), Integer.parseInt(splits[3]), splits[4]);
+ recType.setUndeclaredFieldsDataGen(ufdg);
+ }
+ }
+
+ }
+ (
+ RecordField(recType)
+ ( <COMMA> RecordField(recType) )*
+ )?
+ <RIGHTBRACE>
+ {
+ if (recordKind == null) {
+ recordKind = RecordTypeDefinition.RecordKind.OPEN;
+ }
+ recType.setRecordKind(recordKind);
+ return recType;
+ }
+}
+
+void RecordField(RecordTypeDefinition recType) throws ParseException:
+{
+ String fieldName;
+ TypeExpression type = null;
+ boolean nullable = false;
+}
+{
+ fieldName = Identifier()
+ {
+ String hint = getHint(token);
+ IRecordFieldDataGen rfdg = hint != null ? parseFieldDataGen(hint) : null;
+ }
+ <COLON> type = TypeExpr() (<QUES> { nullable = true; } )?
+ {
+ recType.addField(fieldName, type, nullable, rfdg);
+ }
+}
+
+TypeReferenceExpression TypeReference() throws ParseException:
+{
+ String id = null;
+}
+{
+ id = Identifier()
+ {
+ if (id.equalsIgnoreCase("int")) {
+ id = "int64";
+ }
+
+ return new TypeReferenceExpression(new Identifier(id));
+ }
+}
+
+OrderedListTypeDefinition OrderedListTypeDef() throws ParseException:
+{
+ TypeExpression type = null;
+}
+{
+ <LEFTBRACKET>
+ ( type = TypeExpr() )
+ <RIGHTBRACKET>
+ {
+ return new OrderedListTypeDefinition(type);
+ }
+}
+
+
+UnorderedListTypeDefinition UnorderedListTypeDef() throws ParseException:
+{
+ TypeExpression type = null;
+}
+{
+ <LEFTDBLBRACE>
+ ( type = TypeExpr() )
+ <RIGHTDBLBRACE>
+ {
+ return new UnorderedListTypeDefinition(type);
+ }
+}
+
+FunctionName FunctionName() throws ParseException:
+{
+ String first = null;
+ String second = null;
+ String third = null;
+ boolean secondAfterDot = false;
+}
+{
+ first = Identifier()
+ {
+ FunctionName result = new FunctionName();
+ result.hint = getHint(token);
+ }
+ ( <DOT> second = Identifier()
+ {
+ secondAfterDot = true;
+ }
+ ("#" third = Identifier())? | "#" second = Identifier() )?
+ {
+ if (second == null) {
+ result.dataverse = defaultDataverse;
+ result.library = null;
+ result.function = first;
+ } else if (third == null) {
+ if (secondAfterDot) {
+ result.dataverse = first;
+ result.library = null;
+ result.function = second;
+ } else {
+ result.dataverse = defaultDataverse;
+ result.library = first;
+ result.function = second;
+ }
+ } else {
+ result.dataverse = first;
+ result.library = second;
+ result.function = third;
+ }
+
+ if (result.function.equalsIgnoreCase("int")) {
+ result.function = "int64";
+ }
+ return result;
+ }
+}
+
+
+Pair<Identifier,Identifier> TypeName() throws ParseException:
+{
+ Pair<Identifier,Identifier> name = null;
+}
+{
+ name = QualifiedName()
+ {
+ if (name.first == null) {
+ name.first = new Identifier(defaultDataverse);
+ }
+ return name;
+ }
+}
+
+String Identifier() throws ParseException:
+{
+ String lit = null;
+}
+{
+ (<IDENTIFIER>
+ {
+ return token.image;
+ }
+ | lit = StringLiteral()
+ {
+ return lit;
+ }
+ )
+}
+
+Pair<List<String>, TypeExpression> OpenField() throws ParseException:
+{
+ TypeExpression fieldType = null;
+ List<String> fieldList = null;
+}
+{
+ fieldList = NestedField()
+ ( <COLON> fieldType = IndexedTypeExpr() )?
+ {
+ return new Pair<List<String>, TypeExpression>(fieldList, fieldType);
+ }
+}
+
+List<String> NestedField() throws ParseException:
+{
+ List<String> exprList = new ArrayList<String>();
+ String lit = null;
+}
+{
+ lit = Identifier()
+ {
+ exprList.add(lit);
+ }
+ (<DOT>
+ lit = Identifier()
+ {
+ exprList.add(lit);
+ }
+ )*
+ {
+ return exprList;
+ }
+}
+
+
+
+String StringLiteral() throws ParseException:
+{
+}
+{
+ <STRING_LITERAL>
+ {
+ return removeQuotesAndEscapes(token.image);
+ }
+}
+
+Pair<Identifier,Identifier> QualifiedName() throws ParseException:
+{
+ String first = null;
+ String second = null;
+}
+{
+ first = Identifier() (<DOT> second = Identifier())?
+ {
+ Identifier id1 = null;
+ Identifier id2 = null;
+ if (second == null) {
+ id2 = new Identifier(first);
+ } else
+ {
+ id1 = new Identifier(first);
+ id2 = new Identifier(second);
+ }
+ return new Pair<Identifier,Identifier>(id1, id2);
+ }
+}
+
+Triple<Identifier,Identifier,Identifier> DoubleQualifiedName() throws ParseException:
+{
+ String first = null;
+ String second = null;
+ String third = null;
+}
+{
+ first = Identifier() <DOT> second = Identifier() (<DOT> third = Identifier())?
+ {
+ Identifier id1 = null;
+ Identifier id2 = null;
+ Identifier id3 = null;
+ if (third == null) {
+ id2 = new Identifier(first);
+ id3 = new Identifier(second);
+ } else {
+ id1 = new Identifier(first);
+ id2 = new Identifier(second);
+ id3 = new Identifier(third);
+ }
+ return new Triple<Identifier,Identifier,Identifier>(id1, id2, id3);
+ }
+}
+
+FunctionDecl FunctionDeclaration() throws ParseException:
+{
+ FunctionDecl funcDecl;
+ FunctionSignature signature;
+ String functionName;
+ List<VarIdentifier> paramList = new ArrayList<VarIdentifier>();
+ Expression funcBody;
+ createNewScope();
+}
+{
+ "declare" "function" functionName = Identifier()
+ paramList = ParameterList()
+ <LEFTBRACE> funcBody = Expression() <RIGHTBRACE>
+ {
+ signature = new FunctionSignature(defaultDataverse, functionName, paramList.size());
+ getCurrentScope().addFunctionDescriptor(signature, false);
+ funcDecl = new FunctionDecl(signature, paramList, funcBody);
+ removeCurrentScope();
+ return funcDecl;
+ }
+}
+
+
+Query Query() throws ParseException:
+{
+ Query query = new Query();
+ // we set the pointers to the dataverses and datasets lists to fill them with entities to be locked
+ setDataverses(query.getDataverses());
+ setDatasets(query.getDatasets());
+ Expression expr;
+}
+{
+ expr = Expression()
+ {
+ query.setBody(expr);
+ query.setVarCounter(getVarCounter());
+ // we remove the pointers to the locked entities before we return the query object
+ setDataverses(null);
+ setDatasets(null);
+ return query;
+ }
+
+}
+
+
+
+Expression Expression():
+{
+ Expression expr = null;
+ Expression exprP = null;
+}
+{
+(
+
+//OperatorExpr | IfThenElse | FLWOGRExpression | QuantifiedExpression
+ expr = OperatorExpr()
+ | expr = IfThenElse()
+ | expr = FLWOGR()
+ | expr = QuantifiedExpression()
+
+
+)
+ {
+ return (exprP==null) ? expr : exprP;
+ }
+}
+
+
+
+Expression OperatorExpr()throws ParseException:
+{
+ OperatorExpr op = null;
+ Expression operand = null;
+}
+{
+ operand = AndExpr()
+ (
+
+ <OR>
+ {
+ if (op == null) {
+ op = new OperatorExpr();
+ op.addOperand(operand);
+ op.setCurrentop(true);
+ }
+ op.addOperator(token.image);
+ }
+
+ operand = AndExpr()
+ {
+ op.addOperand(operand);
+ }
+
+ )*
+
+ {
+ return op==null? operand: op;
+ }
+}
+
+Expression AndExpr()throws ParseException:
+{
+ OperatorExpr op = null;
+ Expression operand = null;
+}
+{
+ operand = RelExpr()
+ (
+
+ <AND>
+ {
+ if (op == null) {
+ op = new OperatorExpr();
+ op.addOperand(operand);
+ op.setCurrentop(true);
+ }
+ op.addOperator(token.image);
+ }
+
+ operand = RelExpr()
+ {
+ op.addOperand(operand);
+ }
+
+ )*
+
+ {
+ return op==null? operand: op;
+ }
+}
+
+
+
+Expression RelExpr()throws ParseException:
+{
+ OperatorExpr op = null;
+ Expression operand = null;
+ boolean broadcast = false;
+ IExpressionAnnotation annotation = null;
+}
+{
+ operand = AddExpr()
+ {
+ if (operand instanceof VariableExpr) {
+ String hint = getHint(token);
+ if (hint != null && hint.equals(BROADCAST_JOIN_HINT)) {
+ broadcast = true;
+ }
+ }
+ }
+
+ (
+ LOOKAHEAD(2)( <LT> | <GT> | <LE> | <GE> | <EQ> | <NE> |<SIMILAR>)
+ {
+ String mhint = getHint(token);
+ if (mhint != null) {
+ if (mhint.equals(INDEXED_NESTED_LOOP_JOIN_HINT)) {
+ annotation = IndexedNLJoinExpressionAnnotation.INSTANCE;
+ } else if (mhint.equals(SKIP_SECONDARY_INDEX_SEARCH_HINT)) {
+ annotation = SkipSecondaryIndexSearchExpressionAnnotation.INSTANCE;
+ }
+ }
+ if (op == null) {
+ op = new OperatorExpr();
+ op.addOperand(operand, broadcast);
+ op.setCurrentop(true);
+ broadcast = false;
+ }
+ op.addOperator(token.image);
+ }
+
+ operand = AddExpr()
+ {
+ broadcast = false;
+ if (operand instanceof VariableExpr) {
+ String hint = getHint(token);
+ if (hint != null && hint.equals(BROADCAST_JOIN_HINT)) {
+ broadcast = true;
+ }
+ }
+ op.addOperand(operand, broadcast);
+ }
+ )?
+
+ {
+ if (annotation != null) {
+ op.addHint(annotation);
+ }
+ return op==null? operand: op;
+ }
+}
+
+Expression AddExpr()throws ParseException:
+{
+ OperatorExpr op = null;
+ Expression operand = null;
+}
+{
+ operand = MultExpr()
+
+ ( (<PLUS> | <MINUS>)
+ {
+ if (op == null) {
+ op = new OperatorExpr();
+ op.addOperand(operand);
+ op.setCurrentop(true);
+ }
+ ((OperatorExpr)op).addOperator(token.image);
+ }
+
+ operand = MultExpr()
+ {
+ op.addOperand(operand);
+ }
+ )*
+
+ {
+ return op==null? operand: op;
+ }
+}
+
+Expression MultExpr()throws ParseException:
+{
+ OperatorExpr op = null;
+ Expression operand = null;
+}
+{
+ operand = UnionExpr()
+
+ (( <MUL> | <DIV> | <MOD> | <CARET> | <IDIV>)
+ {
+ if (op == null) {
+ op = new OperatorExpr();
+ op.addOperand(operand);
+ op.setCurrentop(true);
+ }
+ op.addOperator(token.image);
+ }
+ operand = UnionExpr()
+ {
+ op.addOperand(operand);
+ }
+ )*
+
+ {
+ return op==null?operand:op;
+ }
+}
+
+Expression UnionExpr() throws ParseException:
+{
+ UnionExpr union = null;
+ Expression operand1 = null;
+ Expression operand2 = null;
+}
+{
+ operand1 = UnaryExpr()
+ (<UNION>
+ (operand2 = UnaryExpr()) {
+ if (union == null) {
+ union = new UnionExpr();
+ union.addExpr(operand1);
+ }
+ union.addExpr(operand2);
+ } )*
+ {
+ return (union == null)? operand1: union;
+ }
+}
+
+Expression UnaryExpr() throws ParseException:
+{
+ Expression uexpr = null;
+ Expression expr = null;
+}
+{
+ ( (<PLUS> | <MINUS>)
+ {
+ uexpr = new UnaryExpr();
+ if("+".equals(token.image))
+ ((UnaryExpr)uexpr).setSign(Sign.POSITIVE);
+ else if("-".equals(token.image))
+ ((UnaryExpr)uexpr).setSign(Sign.NEGATIVE);
+ else
+ throw new ParseException();
+ }
+ )?
+
+ expr = ValueExpr()
+ {
+ if(uexpr!=null){
+ ((UnaryExpr)uexpr).setExpr(expr);
+ return uexpr;
+ }
+ else{
+ return expr;
+ }
+ }
+}
+
+Expression ValueExpr()throws ParseException:
+{
+ Expression expr = null;
+ Identifier ident = null;
+ AbstractAccessor fa = null;
+ Expression indexExpr = null;
+}
+{
+ expr = PrimaryExpr() ( ident = Field()
+ {
+ fa = (fa == null ? new FieldAccessor(expr, ident)
+ : new FieldAccessor(fa, ident));
+ }
+ | indexExpr = Index()
+ {
+ fa = (fa == null ? new IndexAccessor(expr, indexExpr)
+ : new IndexAccessor(fa, indexExpr));
+ }
+ )*
+ {
+ return fa == null ? expr : fa;
+ }
+}
+
+Identifier Field() throws ParseException:
+{
+ String ident = null;
+}
+{
+ <DOT> ident = Identifier()
+ {
+ return new Identifier(ident);
+ }
+}
+
+Expression Index() throws ParseException:
+{
+ Expression expr = null;
+}
+{
+ <LEFTBRACKET> ( expr = Expression()
+ {
+ if(expr.getKind() == Expression.Kind.LITERAL_EXPRESSION)
+ {
+ Literal lit = ((LiteralExpr)expr).getValue();
+ if(lit.getLiteralType() != Literal.Type.INTEGER &&
+ lit.getLiteralType() != Literal.Type.LONG) {
+ throw new ParseException("Index should be an INTEGER");
+ }
+ }
+ }
+
+ | <QUES> // ANY
+
+ )
+
+ <RIGHTBRACKET>
+ {
+ return expr;
+ }
+}
+
+
+Expression PrimaryExpr()throws ParseException:
+{
+ Expression expr = null;
+}
+{
+ ( LOOKAHEAD(2)
+ expr = FunctionCallExpr()
+ | expr = Literal()
+ | expr = DatasetAccessExpression()
+ | expr = VariableRef()
+ {
+ if(((VariableExpr)expr).getIsNewVar() == true)
+ throw new ParseException("can't find variable " + ((VariableExpr)expr).getVar());
+ }
+ | expr = ListConstructor()
+ | expr = RecordConstructor()
+ | expr = ParenthesizedExpression()
+ )
+ {
+ return expr;
+ }
+}
+
+Expression Literal() throws ParseException:
+{
+ LiteralExpr lit = new LiteralExpr();
+ String str = null;
+}
+{
+ ( str = StringLiteral()
+ {
+ lit.setValue(new StringLiteral(str));
+ }
+ | <INTEGER_LITERAL>
+ {
+ lit.setValue(new LongIntegerLiteral(new Long(token.image)));
+ }
+ | <FLOAT_LITERAL>
+ {
+ lit.setValue(new FloatLiteral(new Float(token.image)));
+ }
+ | <DOUBLE_LITERAL>
+ {
+ lit.setValue(new DoubleLiteral(new Double(token.image)));
+ }
+ | <NULL>
+ {
+ lit.setValue(NullLiteral.INSTANCE);
+ }
+ | <TRUE>
+ {
+ lit.setValue(TrueLiteral.INSTANCE);
+ }
+ | <FALSE>
+ {
+ lit.setValue(FalseLiteral.INSTANCE);
+ }
+ )
+ {
+ return lit;
+ }
+}
+
+
+VariableExpr VariableRef() throws ParseException:
+{
+ VariableExpr varExp = new VariableExpr();
+ VarIdentifier var = new VarIdentifier();
+}
+{
+ <VARIABLE>
+ {
+ String varName = token.image;
+ Identifier ident = lookupSymbol(varName);
+ if (isInForbiddenScopes(varName)) {
+ throw new ParseException("Inside limit clauses, it is disallowed to reference a variable having the same name as any variable bound in the same scope as the limit clause.");
+ }
+ if(ident != null) { // exist such ident
+ varExp.setIsNewVar(false);
+ varExp.setVar((VarIdentifier)ident);
+ } else {
+ varExp.setVar(var);
+ }
+ var.setValue(varName);
+ return varExp;
+ }
+}
+
+
+VariableExpr Variable() throws ParseException:
+{
+ VariableExpr varExp = new VariableExpr();
+ VarIdentifier var = new VarIdentifier();
+}
+{
+ <VARIABLE>
+ {
+ Identifier ident = lookupSymbol(token.image);
+ if(ident != null) { // exist such ident
+ varExp.setIsNewVar(false);
+ }
+ varExp.setVar(var);
+ var.setValue(token.image);
+ return varExp;
+ }
+}
+
+Expression ListConstructor() throws ParseException:
+{
+ Expression expr = null;
+}
+{
+ (
+ expr = OrderedListConstructor() | expr = UnorderedListConstructor()
+ )
+
+ {
+ return expr;
+ }
+}
+
+
+ListConstructor OrderedListConstructor() throws ParseException:
+{
+ ListConstructor expr = new ListConstructor();
+ List<Expression> exprList = null;
+ expr.setType(ListConstructor.Type.ORDERED_LIST_CONSTRUCTOR);
+}
+{
+ <LEFTBRACKET> exprList = ExpressionList() <RIGHTBRACKET>
+ {
+ expr.setExprList(exprList);
+ return expr;
+ }
+}
+
+ListConstructor UnorderedListConstructor() throws ParseException:
+{
+ ListConstructor expr = new ListConstructor();
+ List<Expression> exprList = null;
+ expr.setType(ListConstructor.Type.UNORDERED_LIST_CONSTRUCTOR);
+}
+{
+ <LEFTDBLBRACE> exprList = ExpressionList() <RIGHTDBLBRACE>
+ {
+ expr.setExprList(exprList);
+ return expr;
+ }
+}
+
+List<Expression> ExpressionList() throws ParseException:
+{
+ Expression expr = null;
+ List<Expression> list = null;
+ List<Expression> exprList = new ArrayList<Expression>();
+}
+{
+ (
+ expr = Expression() { exprList.add(expr); }
+ (LOOKAHEAD(1) <COMMA> list = ExpressionList() { exprList.addAll(list); })?
+ )?
+ (LOOKAHEAD(1) Comma())?
+ {
+ return exprList;
+ }
+}
+
+void Comma():
+{}
+{
+ <COMMA>
+}
+
+RecordConstructor RecordConstructor() throws ParseException:
+{
+ RecordConstructor expr = new RecordConstructor();
+ FieldBinding tmp = null;
+ List<FieldBinding> fbList = new ArrayList<FieldBinding>();
+}
+{
+ <LEFTBRACE> (tmp = FieldBinding()
+ {
+ fbList.add(tmp);
+ }
+ (<COMMA> tmp = FieldBinding() { fbList.add(tmp); })*)? <RIGHTBRACE>
+ {
+ expr.setFbList(fbList);
+ return expr;
+ }
+}
+
+FieldBinding FieldBinding() throws ParseException:
+{
+ FieldBinding fb = new FieldBinding();
+ Expression left, right;
+}
+{
+ left = Expression() <COLON> right = Expression()
+ {
+ fb.setLeftExpr(left);
+ fb.setRightExpr(right);
+ return fb;
+ }
+}
+
+
+Expression FunctionCallExpr() throws ParseException:
+{
+ CallExpr callExpr;
+ List<Expression> argList = new ArrayList<Expression>();
+ Expression tmp;
+ int arity = 0;
+ FunctionName funcName = null;
+ String hint = null;
+}
+{
+ funcName = FunctionName()
+ {
+ hint = funcName.hint;
+ }
+ <LEFTPAREN> (tmp = Expression()
+ {
+ argList.add(tmp);
+ arity ++;
+ }
+ (<COMMA> tmp = Expression()
+ {
+ argList.add(tmp);
+ arity++;
+ }
+ )*)? <RIGHTPAREN>
+ {
+ // TODO use funcName.library
+ String fqFunctionName = funcName.library == null ? funcName.function : funcName.library + "#" + funcName.function;
+ FunctionSignature signature
+ = lookupFunctionSignature(funcName.dataverse, fqFunctionName, arity);
+ if (signature == null) {
+ signature = new FunctionSignature(funcName.dataverse, fqFunctionName, arity);
+ }
+ callExpr = new CallExpr(signature,argList);
+ if (hint != null) {
+ if (hint.startsWith(INDEXED_NESTED_LOOP_JOIN_HINT)) {
+ callExpr.addHint(IndexedNLJoinExpressionAnnotation.INSTANCE);
+ } else if (hint.startsWith(SKIP_SECONDARY_INDEX_SEARCH_HINT)) {
+ callExpr.addHint(SkipSecondaryIndexSearchExpressionAnnotation.INSTANCE);
+ }
+ }
+ return callExpr;
+ }
+}
+
+
+Expression DatasetAccessExpression() throws ParseException:
+{
+ String funcName;
+ String arg1 = null;
+ String arg2 = null;
+ Expression nameArg;
+}
+{
+ <DATASET>
+ {
+ funcName = token.image;
+ }
+ ( ( arg1 = Identifier() ( <DOT> arg2 = Identifier() )? )
+ {
+ String name = arg2 == null ? arg1 : arg1 + "." + arg2;
+ LiteralExpr ds = new LiteralExpr();
+ ds.setValue( new StringLiteral(name) );
+ nameArg = ds;
+ if(arg2 != null){
+ addDataverse(arg1.toString());
+ addDataset(name);
+ } else {
+ addDataset(defaultDataverse + "." + name);
+ }
+ }
+ | ( <LEFTPAREN> nameArg = Expression() <RIGHTPAREN> ) )
+ {
+ String dataverse = MetadataConstants.METADATA_DATAVERSE_NAME;
+ FunctionSignature signature = lookupFunctionSignature(dataverse, funcName, 1);
+ if (signature == null) {
+ signature = new FunctionSignature(dataverse, funcName, 1);
+ }
+ List<Expression> argList = new ArrayList<Expression>();
+ argList.add(nameArg);
+ return new CallExpr(signature, argList);
+ }
+}
+
+Expression ParenthesizedExpression() throws ParseException:
+{
+ Expression expr;
+}
+{
+ <LEFTPAREN> expr = Expression() <RIGHTPAREN>
+ {
+ return expr;
+ }
+}
+
+Expression IfThenElse() throws ParseException:
+{
+ Expression condExpr;
+ Expression thenExpr;
+ Expression elseExpr;
+ IfExpr ifExpr = new IfExpr();
+}
+{
+ <IF> <LEFTPAREN> condExpr = Expression() <RIGHTPAREN> <THEN> thenExpr = Expression() <ELSE> elseExpr = Expression()
+
+ {
+ ifExpr.setCondExpr(condExpr);
+ ifExpr.setThenExpr(thenExpr);
+ ifExpr.setElseExpr(elseExpr);
+ return ifExpr;
+ }
+}
+
+Expression FLWOGR() throws ParseException:
+{
+ FLWOGRExpression flworg = new FLWOGRExpression();
+ List<Clause> clauseList = new ArrayList<Clause>();
+ Expression returnExpr;
+ Clause tmp;
+ createNewScope();
+}
+{
+ (tmp = ForClause() {clauseList.add(tmp);} | tmp = LetClause() {clauseList.add(tmp);})
+ (tmp = Clause() {clauseList.add(tmp);})* (<RETURN>|<SELECT>) returnExpr = Expression()
+
+ {
+ flworg.setClauseList(clauseList);
+ flworg.setReturnExpr(returnExpr);
+ removeCurrentScope();
+ return flworg;
+ }
+}
+
+Clause Clause()throws ParseException :
+{
+ Clause clause;
+}
+{
+ (
+ clause = ForClause()
+ | clause = LetClause()
+ | clause = WhereClause()
+ | clause = OrderbyClause()
+ | clause = GroupClause()
+ | clause = LimitClause()
+ | clause = DistinctClause()
+ )
+ {
+ return clause;
+ }
+}
+
+Clause ForClause()throws ParseException :
+{
+ ForClause fc = new ForClause();
+ VariableExpr varExp;
+ VariableExpr varPos = null;
+ Expression inExp;
+ extendCurrentScope();
+}
+{
+ (<FOR>|<FROM>) varExp = Variable() (<AT> varPos = Variable())? <IN> ( inExp = Expression() )
+ {
+ fc.setVarExpr(varExp);
+ getCurrentScope().addNewVarSymbolToScope(varExp.getVar());
+ fc.setInExpr(inExp);
+ if (varPos != null) {
+ fc.setPosExpr(varPos);
+ getCurrentScope().addNewVarSymbolToScope(varPos.getVar());
+ }
+ return fc;
+ }
+}
+
+Clause LetClause() throws ParseException:
+{
+ LetClause lc = new LetClause();
+ VariableExpr varExp;
+ Expression beExp;
+ extendCurrentScope();
+}
+{
+ (<LET>|<WITH>) varExp = Variable() <ASSIGN> beExp = Expression()
+ {
+ getCurrentScope().addNewVarSymbolToScope(varExp.getVar());
+ lc.setVarExpr(varExp);
+ lc.setBindingExpr(beExp);
+ return lc;
+ }
+}
+
+Clause WhereClause()throws ParseException :
+{
+ WhereClause wc = new WhereClause();
+ Expression whereExpr;
+}
+{
+ <WHERE> whereExpr = Expression()
+ {
+ wc.setWhereExpr(whereExpr);
+ return wc;
+ }
+}
+
+Clause OrderbyClause()throws ParseException :
+{
+ OrderbyClause oc = new OrderbyClause();
+ Expression orderbyExpr;
+ List<Expression> orderbyList = new ArrayList<Expression>();
+ List<OrderbyClause.OrderModifier> modifierList = new ArrayList<OrderbyClause.OrderModifier >();
+ int numOfOrderby = 0;
+}
+{
+ (
+ <ORDER>
+ {
+ String hint = getHint(token);
+ if (hint != null) {
+ if (hint.startsWith(INMEMORY_HINT)) {
+ String splits[] = hint.split(" +");
+ int numFrames = Integer.parseInt(splits[1]);
+ int numTuples = Integer.parseInt(splits[2]);
+ oc.setNumFrames(numFrames);
+ oc.setNumTuples(numTuples);
+ }
+ if (hint.startsWith(RANGE_HINT)) {
+ try{
+ oc.setRangeMap(RangeMapBuilder.parseHint(hint.substring(RANGE_HINT.length())));
+ } catch (AsterixException e) {
+ throw new ParseException(e.getMessage());
+ }
+ }
+ }
+ }
+ <BY> orderbyExpr = Expression()
+ {
+ orderbyList.add(orderbyExpr);
+ OrderbyClause.OrderModifier modif = OrderbyClause.OrderModifier.ASC;
+ }
+ ( (<ASC> { modif = OrderbyClause.OrderModifier.ASC; })
+ | (<DESC> { modif = OrderbyClause.OrderModifier.DESC; }))?
+ {
+ modifierList.add(modif);
+ }
+
+ (<COMMA> orderbyExpr = Expression()
+ {
+ orderbyList.add(orderbyExpr);
+ modif = OrderbyClause.OrderModifier.ASC;
+ }
+ ( (<ASC> { modif = OrderbyClause.OrderModifier.ASC; })
+ | (<DESC> { modif = OrderbyClause.OrderModifier.DESC; }))?
+ {
+ modifierList.add(modif);
+ }
+ )*
+)
+ {
+ oc.setModifierList(modifierList);
+ oc.setOrderbyList(orderbyList);
+ return oc;
+ }
+}
+Clause GroupClause()throws ParseException :
+{
+ GroupbyClause gbc = new GroupbyClause();
+ // GbyVariableExpressionPair pair = new GbyVariableExpressionPair();
+ List<GbyVariableExpressionPair> vePairList = new ArrayList<GbyVariableExpressionPair>();
+ List<GbyVariableExpressionPair> decorPairList = new ArrayList<GbyVariableExpressionPair>();
+ List<VariableExpr> withVarList= new ArrayList<VariableExpr>();
+ VariableExpr var = null;
+ VariableExpr withVar = null;
+ Expression expr = null;
+ VariableExpr decorVar = null;
+ Expression decorExpr = null;
+}
+{
+ {
+ Scope newScope = extendCurrentScopeNoPush(true);
+ // extendCurrentScope(true);
+ }
+ <GROUP>
+ {
+ String hint = getHint(token);
+ if (hint != null && hint.equals(HASH_GROUP_BY_HINT)) {
+ gbc.setHashGroupByHint(true);
+ }
+ }
+ <BY> (LOOKAHEAD(2) var = Variable()
+ {
+ newScope.addNewVarSymbolToScope(var.getVar());
+ } <ASSIGN>)?
+ expr = Expression()
+ {
+ GbyVariableExpressionPair pair1 = new GbyVariableExpressionPair(var, expr);
+ vePairList.add(pair1);
+ }
+ (<COMMA> ( LOOKAHEAD(2) var = Variable()
+ {
+ newScope.addNewVarSymbolToScope(var.getVar());
+ } <ASSIGN>)?
+ expr = Expression()
+ {
+ GbyVariableExpressionPair pair2 = new GbyVariableExpressionPair(var, expr);
+ vePairList.add(pair2);
+ }
+ )*
+ (<DECOR> decorVar = Variable() <ASSIGN> decorExpr = Expression()
+ {
+ newScope.addNewVarSymbolToScope(decorVar.getVar());
+ GbyVariableExpressionPair pair3 = new GbyVariableExpressionPair(decorVar, decorExpr);
+ decorPairList.add(pair3);
+ }
+ (<COMMA> <DECOR> decorVar = Variable() <ASSIGN> decorExpr = Expression()
+ {
+ newScope.addNewVarSymbolToScope(decorVar.getVar());
+ GbyVariableExpressionPair pair4 = new GbyVariableExpressionPair(decorVar, decorExpr);
+ decorPairList.add(pair4);
+ }
+ )*
+ )?
+ (<WITH>|<KEEPING>) withVar = VariableRef()
+ {
+ if(withVar.getIsNewVar()==true)
+ throw new ParseException("can't find variable " + withVar.getVar());
+ withVarList.add(withVar);
+ newScope.addNewVarSymbolToScope(withVar.getVar());
+ }
+ (<COMMA> withVar = VariableRef()
+ {
+ if(withVar.getIsNewVar()==true)
+ throw new ParseException("can't find variable " + withVar.getVar());
+ withVarList.add(withVar);
+ newScope.addNewVarSymbolToScope(withVar.getVar());
+ })*
+ {
+ gbc.setGbyPairList(vePairList);
+ gbc.setDecorPairList(decorPairList);
+ gbc.setWithVarList(withVarList);
+ replaceCurrentScope(newScope);
+ return gbc;
+ }
+}
+
+
+LimitClause LimitClause() throws ParseException:
+{
+ LimitClause lc = new LimitClause();
+ Expression expr;
+ pushForbiddenScope(getCurrentScope());
+}
+{
+ <LIMIT> expr = Expression() { lc.setLimitExpr(expr); }
+ (<OFFSET> expr = Expression() { lc.setOffset(expr); })?
+
+ {
+ popForbiddenScope();
+ return lc;
+ }
+}
+
+DistinctClause DistinctClause() throws ParseException:
+{
+ List<Expression> exprs = new ArrayList<Expression>();
+ Expression expr;
+}
+{
+ <DISTINCT> <BY> expr = Expression()
+ {
+ exprs.add(expr);
+ }
+ (<COMMA> expr = Expression()
+ {
+ exprs.add(expr);
+ }
+ )*
+ {
+ return new DistinctClause(exprs);
+ }
+}
+
+QuantifiedExpression QuantifiedExpression()throws ParseException:
+{
+ QuantifiedExpression qc = new QuantifiedExpression();
+ List<QuantifiedPair> quantifiedList = new ArrayList<QuantifiedPair>();
+ Expression satisfiesExpr;
+ VariableExpr var;
+ Expression inExpr;
+ QuantifiedPair pair;
+}
+{
+ {
+ createNewScope();
+ }
+
+ ( (<SOME> { qc.setQuantifier(QuantifiedExpression.Quantifier.SOME); })
+ | (<EVERY> { qc.setQuantifier(QuantifiedExpression.Quantifier.EVERY); }))
+ var = Variable() <IN> inExpr = Expression()
+ {
+ pair = new QuantifiedPair(var, inExpr);
+ getCurrentScope().addNewVarSymbolToScope(var.getVar());
+ quantifiedList.add(pair);
+ }
+ (
+ <COMMA> var = Variable() <IN> inExpr = Expression()
+ {
+ pair = new QuantifiedPair(var, inExpr);
+ getCurrentScope().addNewVarSymbolToScope(var.getVar());
+ quantifiedList.add(pair);
+ }
+ )*
+ <SATISFIES> satisfiesExpr = Expression()
+ {
+ qc.setSatisfiesExpr(satisfiesExpr);
+ qc.setQuantifiedList(quantifiedList);
+ removeCurrentScope();
+ return qc;
+ }
+}
+
+TOKEN_MGR_DECLS:
+{
+ public int commentDepth = 0;
+ public IntStack lexerStateStack = new IntStack();
+
+ public void pushState() {
+ lexerStateStack.push( curLexState );
+ }
+
+ public void popState(String token) {
+ if (lexerStateStack.size() > 0) {
+ SwitchTo( lexerStateStack.pop() );
+ } else {
+ int errorLine = input_stream.getEndLine();
+ int errorColumn = input_stream.getEndColumn();
+ String msg = "Lexical error at line " + errorLine + ", column " + errorColumn + ". Encountered \"" + token
+ + "\" but state stack is empty.";
+ throw new TokenMgrError(msg, -1);
+ }
+ }
+}
+
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+ <ASC : "asc">
+ | <AT : "at">
+ | <BY : "by">
+ | <DATASET : "dataset">
+ | <DECOR : "decor">
+ | <DESC : "desc">
+ | <DISTINCT : "distinct">
+ | <ELSE : "else">
+ | <EVERY : "every">
+ | <FOR : "for">
+ | <FROM : "from">
+ | <GROUP : "group">
+ | <IF : "if">
+ | <IN : "in">
+ | <LET : "let">
+ | <LIMIT : "limit">
+ | <OFFSET : "offset">
+ | <ORDER : "order">
+ | <RETURN : "return">
+ | <SATISFIES : "satisfies">
+ | <SELECT : "select">
+ | <SOME : "some">
+ | <THEN : "then">
+ | <UNION : "union">
+ | <WHERE : "where">
+ | <WITH : "with">
+ | <KEEPING : "keeping">
+}
+
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+ <CARET : "^">
+ | <DIV : "/">
+ | <IDIV : "idiv">
+ | <MINUS : "-">
+ | <MOD : "%">
+ | <MUL : "*">
+ | <PLUS : "+">
+
+ | <LEFTPAREN : "(">
+ | <RIGHTPAREN : ")">
+ | <LEFTBRACKET : "[">
+ | <RIGHTBRACKET : "]">
+
+ | <COLON : ":">
+ | <COMMA : ",">
+ | <DOT : ".">
+ | <QUES : "?">
+
+ | <LT : "<">
+ | <GT : ">">
+ | <LE : "<=">
+ | <GE : ">=">
+ | <EQ : "=">
+ | <NE : "!=">
+ | <SIMILAR : "~=">
+ | <ASSIGN : ":=">
+
+ | <AND : "and">
+ | <OR : "or">
+}
+
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+ <LEFTBRACE : "{"> { pushState(); } : DEFAULT
+}
+
+<DEFAULT>
+TOKEN :
+{
+ <RIGHTBRACE : "}"> { popState("}"); }
+}
+
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+ <LEFTDBLBRACE : "{{"> { pushState(); } : IN_DBL_BRACE
+}
+
+<IN_DBL_BRACE>
+TOKEN :
+{
+ <RIGHTDBLBRACE : "}}"> { popState("}}"); }
+}
+
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+ <INTEGER_LITERAL : (<DIGIT>)+ >
+}
+
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+ <NULL : "null">
+ | <TRUE : "true">
+ | <FALSE : "false">
+}
+
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+ <#DIGIT : ["0" - "9"]>
+}
+
+<DEFAULT,IN_DBL_BRACE>
+TOKEN:
+{
+ < DOUBLE_LITERAL: <DIGITS>
+ | <DIGITS> ( "." <DIGITS> )?
+ | "." <DIGITS>
+ >
+ | < FLOAT_LITERAL: <DIGITS> ( "f" | "F" )
+ | <DIGITS> ( "." <DIGITS> ( "f" | "F" ) )?
+ | "." <DIGITS> ( "f" | "F" )
+ >
+ | <DIGITS : (<DIGIT>)+ >
+}
+
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+ <#LETTER : ["A" - "Z", "a" - "z"]>
+ | <SPECIALCHARS : ["$", "_", "-"]>
+}
+
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+ // backslash u + 4 hex digits escapes are handled in the underlying JavaCharStream
+ <STRING_LITERAL : ("\"" (
+ <EscapeQuot>
+ | <EscapeBslash>
+ | <EscapeSlash>
+ | <EscapeBspace>
+ | <EscapeFormf>
+ | <EscapeNl>
+ | <EscapeCr>
+ | <EscapeTab>
+ | ~["\"","\\"])* "\"")
+ | ("\'"(
+ <EscapeApos>
+ | <EscapeBslash>
+ | <EscapeSlash>
+ | <EscapeBspace>
+ | <EscapeFormf>
+ | <EscapeNl>
+ | <EscapeCr>
+ | <EscapeTab>
+ | ~["\'","\\"])* "\'")>
+ | < #EscapeQuot: "\\\"" >
+ | < #EscapeApos: "\\\'" >
+ | < #EscapeBslash: "\\\\" >
+ | < #EscapeSlash: "\\/" >
+ | < #EscapeBspace: "\\b" >
+ | < #EscapeFormf: "\\f" >
+ | < #EscapeNl: "\\n" >
+ | < #EscapeCr: "\\r" >
+ | < #EscapeTab: "\\t" >
+}
+
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+ <IDENTIFIER : <LETTER> (<LETTER> | <DIGIT> | <SPECIALCHARS>)*>
+}
+
+<DEFAULT,IN_DBL_BRACE>
+TOKEN :
+{
+ <VARIABLE : "$" <LETTER> (<LETTER> | <DIGIT> | "_")*>
+}
+
+<DEFAULT,IN_DBL_BRACE>
+SKIP:
+{
+ " "
+ | "\t"
+ | "\r"
+ | "\n"
+}
+
+<DEFAULT,IN_DBL_BRACE>
+SKIP:
+{
+ <"//" (~["\n"])* "\n">
+}
+
+<DEFAULT,IN_DBL_BRACE>
+SKIP:
+{
+ <"//" (~["\n","\r"])* ("\n"|"\r"|"\r\n")?>
+}
+
+<DEFAULT,IN_DBL_BRACE>
+SKIP:
+{
+ <"/*"> { pushState(); } : INSIDE_COMMENT
+}
+
+<INSIDE_COMMENT>
+SPECIAL_TOKEN:
+{
+ <"+"(" ")*(~["*"])*>
+}
+
+<INSIDE_COMMENT>
+SKIP:
+{
+ <"/*"> { pushState(); }
+}
+
+<INSIDE_COMMENT>
+SKIP:
+{
+ <"*/"> { popState("*/"); }
+ | <~[]>
+}