asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj - asterixdb - Gitiles

 //
 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
 // distributed with this work for additional information
 // regarding copyright ownership.  The ASF licenses this file
 // to you under the Apache License, Version 2.0 (the
 // "License"); you may not use this file except in compliance
 // with the License.  You may obtain a copy of the License at
 //
 //   http://www.apache.org/licenses/LICENSE-2.0
 //
 // Unless required by applicable law or agreed to in writing,
 // software distributed under the License is distributed on an
 // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
 //
 options {
   COMMON_TOKEN_ACTION = true;
   STATIC = false;
   TOKEN_EXTENDS = "org.apache.asterix.lang.sqlpp.parser.SqlppToken";
 }

 PARSER_BEGIN(SQLPPParser)

 package org.apache.asterix.lang.sqlpp.parser;

 // For SQL++ ParserTokenManager
 import java.util.ArrayDeque;

 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStreamReader;
 import java.io.Reader;
 import java.io.StringReader;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.Iterator;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;

 import org.apache.asterix.common.annotations.AutoDataGen;
 import org.apache.asterix.common.annotations.DateBetweenYearsDataGen;
 import org.apache.asterix.common.annotations.DatetimeAddRandHoursDataGen;
 import org.apache.asterix.common.annotations.DatetimeBetweenYearsDataGen;
 import org.apache.asterix.common.annotations.ExternalSubpathAnnotation;
 import org.apache.asterix.common.annotations.FieldIntervalDataGen;
 import org.apache.asterix.common.annotations.FieldValFileDataGen;
 import org.apache.asterix.common.annotations.FieldValFileSameIndexDataGen;
 import org.apache.asterix.common.annotations.IRecordFieldDataGen;
 import org.apache.asterix.common.annotations.IndexedNLJoinExpressionAnnotation;
 import org.apache.asterix.common.annotations.InsertRandIntDataGen;
 import org.apache.asterix.common.annotations.ListDataGen;
 import org.apache.asterix.common.annotations.ListValFileDataGen;
 import org.apache.asterix.common.annotations.RangeAnnotation;
 import org.apache.asterix.common.annotations.SecondaryIndexSearchPreferenceAnnotation;
 import org.apache.asterix.common.annotations.SkipSecondaryIndexSearchExpressionAnnotation;
 import org.apache.asterix.common.annotations.SpatialJoinAnnotation;
 import org.apache.asterix.common.annotations.TypeDataGen;
 import org.apache.asterix.common.annotations.UndeclaredFieldsDataGen;
 import org.apache.asterix.common.config.DatasetConfig.DatasetType;
 import org.apache.asterix.common.config.DatasetConfig.IndexType;
 import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.common.exceptions.CompilationException;
 import org.apache.asterix.common.exceptions.ErrorCode;
 import org.apache.asterix.common.exceptions.WarningCollector;
 import org.apache.asterix.common.functions.FunctionConstants;
 import org.apache.asterix.common.functions.FunctionSignature;
 import org.apache.asterix.common.metadata.DataverseName;
 import org.apache.asterix.common.metadata.DatasetFullyQualifiedName;
 import org.apache.asterix.common.metadata.MetadataConstants;
 import org.apache.asterix.external.dataset.adapter.AdapterIdentifier;
 import org.apache.asterix.lang.common.base.AbstractClause;
 import org.apache.asterix.lang.common.base.AbstractLangExpression;
 import org.apache.asterix.lang.common.base.AbstractStatement;
 import org.apache.asterix.lang.common.base.Expression;
 import org.apache.asterix.lang.common.base.Literal;
 import org.apache.asterix.lang.common.base.IParser;
 import org.apache.asterix.lang.common.base.ILangExpression;
 import org.apache.asterix.lang.common.base.IParserFactory;
 import org.apache.asterix.lang.common.base.Statement;
 import org.apache.asterix.lang.common.clause.GroupbyClause;
 import org.apache.asterix.lang.common.clause.LetClause;
 import org.apache.asterix.lang.common.clause.LimitClause;
 import org.apache.asterix.lang.common.clause.OrderbyClause;
 import org.apache.asterix.lang.common.clause.UpdateClause;
 import org.apache.asterix.lang.common.clause.WhereClause;
 import org.apache.asterix.lang.common.context.RootScopeFactory;
 import org.apache.asterix.lang.common.context.Scope;
 import org.apache.asterix.lang.common.expression.AbstractAccessor;
 import org.apache.asterix.lang.common.expression.CallExpr;
 import org.apache.asterix.lang.common.expression.FieldAccessor;
 import org.apache.asterix.lang.common.expression.FieldBinding;
 import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair;
 import org.apache.asterix.lang.common.expression.IfExpr;
 import org.apache.asterix.lang.common.expression.IndexAccessor;
 import org.apache.asterix.lang.common.expression.IndexedTypeExpression;
 import org.apache.asterix.lang.common.expression.ListConstructor;
 import org.apache.asterix.lang.common.expression.ListSliceExpression;
 import org.apache.asterix.lang.common.expression.LiteralExpr;
 import org.apache.asterix.lang.common.expression.OperatorExpr;
 import org.apache.asterix.lang.common.expression.OrderedListTypeDefinition;
 import org.apache.asterix.lang.common.expression.QuantifiedExpression;
 import org.apache.asterix.lang.common.expression.RecordConstructor;
 import org.apache.asterix.lang.common.expression.RecordTypeDefinition;
 import org.apache.asterix.lang.common.expression.TypeExpression;
 import org.apache.asterix.lang.common.expression.TypeReferenceExpression;
 import org.apache.asterix.lang.common.expression.UnaryExpr;
 import org.apache.asterix.lang.common.expression.UnorderedListTypeDefinition;
 import org.apache.asterix.lang.common.expression.VariableExpr;
 import org.apache.asterix.lang.common.literal.DoubleLiteral;
 import org.apache.asterix.lang.common.literal.FalseLiteral;
 import org.apache.asterix.lang.common.literal.FloatLiteral;
 import org.apache.asterix.lang.common.literal.LongIntegerLiteral;
 import org.apache.asterix.lang.common.literal.MissingLiteral;
 import org.apache.asterix.lang.common.literal.NullLiteral;
 import org.apache.asterix.lang.common.literal.StringLiteral;
 import org.apache.asterix.lang.common.literal.TrueLiteral;
 import org.apache.asterix.lang.common.parser.ScopeChecker;
 import org.apache.asterix.lang.common.statement.AdapterDropStatement;
 import org.apache.asterix.lang.common.statement.AnalyzeStatement;
 import org.apache.asterix.lang.common.statement.AnalyzeDropStatement;
 import org.apache.asterix.lang.common.statement.CompactStatement;
 import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
 import org.apache.asterix.lang.common.statement.StartFeedStatement;
 import org.apache.asterix.lang.common.statement.StopFeedStatement;
 import org.apache.asterix.lang.common.statement.CreateAdapterStatement;
 import org.apache.asterix.lang.common.statement.CreateDatabaseStatement;
 import org.apache.asterix.lang.common.statement.CreateDataverseStatement;
 import org.apache.asterix.lang.common.statement.CreateFeedPolicyStatement;
 import org.apache.asterix.lang.common.statement.CreateFeedStatement;
 import org.apache.asterix.lang.common.statement.CreateFunctionStatement;
 import org.apache.asterix.lang.common.statement.CreateIndexStatement;
 import org.apache.asterix.lang.common.statement.CreateSynonymStatement;
 import org.apache.asterix.lang.common.statement.CreateFullTextFilterStatement;
 import org.apache.asterix.lang.common.statement.CreateFullTextConfigStatement;
 import org.apache.asterix.lang.common.statement.CreateViewStatement;
 import org.apache.asterix.lang.common.statement.DatabaseDropStatement;
 import org.apache.asterix.lang.common.statement.DatasetDecl;
 import org.apache.asterix.lang.common.statement.DataverseDecl;
 import org.apache.asterix.lang.common.statement.DataverseDropStatement;
 import org.apache.asterix.lang.common.statement.DeleteStatement;
 import org.apache.asterix.lang.common.statement.DisconnectFeedStatement;
 import org.apache.asterix.lang.common.statement.DropDatasetStatement;
 import org.apache.asterix.lang.common.statement.ExternalDetailsDecl;
 import org.apache.asterix.lang.common.statement.FeedDropStatement;
 import org.apache.asterix.lang.common.statement.FeedPolicyDropStatement;
 import org.apache.asterix.lang.common.statement.FunctionDecl;
 import org.apache.asterix.lang.common.statement.FunctionDropStatement;
 import org.apache.asterix.lang.common.statement.IndexDropStatement;
 import org.apache.asterix.lang.common.statement.FullTextFilterDropStatement;
 import org.apache.asterix.lang.common.statement.FullTextConfigDropStatement;
 import org.apache.asterix.lang.common.statement.InsertStatement;
 import org.apache.asterix.lang.common.statement.InternalDetailsDecl;
 import org.apache.asterix.lang.common.statement.LoadStatement;
 import org.apache.asterix.lang.common.statement.CopyStatement;
 import org.apache.asterix.lang.common.statement.NodeGroupDropStatement;
 import org.apache.asterix.lang.common.statement.NodegroupDecl;
 import org.apache.asterix.lang.common.statement.Query;
 import org.apache.asterix.lang.common.statement.SetStatement;
 import org.apache.asterix.lang.common.statement.SynonymDropStatement;
 import org.apache.asterix.lang.common.statement.TypeDecl;
 import org.apache.asterix.lang.common.statement.TypeDropStatement;
 import org.apache.asterix.lang.common.statement.UpdateStatement;
 import org.apache.asterix.lang.common.statement.UpsertStatement;
 import org.apache.asterix.lang.common.statement.ViewDecl;
 import org.apache.asterix.lang.common.statement.ViewDropStatement;
 import org.apache.asterix.lang.common.struct.Identifier;
 import org.apache.asterix.lang.common.struct.OperatorType;
 import org.apache.asterix.lang.common.struct.QuantifiedPair;
 import org.apache.asterix.lang.common.struct.VarIdentifier;
 import org.apache.asterix.lang.common.util.DatasetDeclParametersUtil;
 import org.apache.asterix.lang.common.util.ExpressionUtils;
 import org.apache.asterix.lang.common.util.RangeMapBuilder;
 import org.apache.asterix.lang.sqlpp.clause.AbstractBinaryCorrelateClause;
 import org.apache.asterix.lang.sqlpp.clause.FromClause;
 import org.apache.asterix.lang.sqlpp.clause.FromTerm;
 import org.apache.asterix.lang.sqlpp.clause.HavingClause;
 import org.apache.asterix.lang.sqlpp.clause.JoinClause;
 import org.apache.asterix.lang.sqlpp.clause.Projection;
 import org.apache.asterix.lang.sqlpp.clause.SelectBlock;
 import org.apache.asterix.lang.sqlpp.clause.SelectClause;
 import org.apache.asterix.lang.sqlpp.clause.SelectElement;
 import org.apache.asterix.lang.sqlpp.clause.SelectRegular;
 import org.apache.asterix.lang.sqlpp.clause.SelectSetOperation;
 import org.apache.asterix.lang.sqlpp.clause.UnnestClause;
 import org.apache.asterix.lang.common.clause.WhereClause;
 import org.apache.asterix.lang.sqlpp.expression.CaseExpression;
 import org.apache.asterix.lang.sqlpp.expression.SelectExpression;
 import org.apache.asterix.lang.sqlpp.expression.WindowExpression;
 import org.apache.asterix.lang.sqlpp.optype.JoinType;
 import org.apache.asterix.lang.sqlpp.optype.SetOpType;
 import org.apache.asterix.lang.sqlpp.optype.UnnestType;
 import org.apache.asterix.lang.sqlpp.parser.SqlppGroupingSetsParser;
 import org.apache.asterix.lang.sqlpp.parser.SqlppGroupingSetsParser.GroupingElement;
 import org.apache.asterix.lang.sqlpp.parser.SqlppGroupingSetsParser.GroupingSet;
 import org.apache.asterix.lang.sqlpp.parser.SqlppGroupingSetsParser.GroupingSets;
 import org.apache.asterix.lang.sqlpp.parser.SqlppGroupingSetsParser.RollupCube;
 import org.apache.asterix.lang.sqlpp.parser.SqlppHint;
 import org.apache.asterix.lang.sqlpp.struct.SetOperationInput;
 import org.apache.asterix.lang.sqlpp.struct.SetOperationRight;
 import org.apache.asterix.lang.sqlpp.util.ExpressionToVariableUtil;
 import org.apache.asterix.lang.sqlpp.util.FunctionMapUtil;
 import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil;
 import org.apache.asterix.metadata.bootstrap.MetadataBuiltinEntities;
 import org.apache.asterix.om.exceptions.TypeMismatchException;
 import org.apache.asterix.om.functions.BuiltinFunctions;
 import org.apache.asterix.om.types.BuiltinType;
 import org.apache.commons.lang3.ArrayUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.hyracks.algebricks.common.utils.Pair;
 import org.apache.hyracks.algebricks.common.utils.Triple;
 import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
 import org.apache.hyracks.algebricks.core.algebra.expressions.BroadcastExpressionAnnotation;
 import org.apache.hyracks.algebricks.core.algebra.expressions.HashJoinExpressionAnnotation;
 import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionAnnotation;
 import org.apache.hyracks.algebricks.core.algebra.expressions.JoinProductivityAnnotation;
 import org.apache.hyracks.algebricks.core.algebra.expressions.PredicateCardinalityAnnotation;
 import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
 import org.apache.hyracks.api.exceptions.IWarningCollector;
 import org.apache.hyracks.api.exceptions.SourceLocation;
 import org.apache.hyracks.api.exceptions.Warning;
 import org.apache.hyracks.dataflow.common.data.partition.range.RangeMap;
 import org.apache.hyracks.util.LogRedactionUtil;
 import org.apache.hyracks.util.StringUtil;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;

 class SQLPPParser extends ScopeChecker implements IParser {

     // tokens parsed as identifiers
     private static final String CUBE = "CUBE";
     private static final String CURRENT = "CURRENT";
     private static final String DEFAULT = "DEFAULT";
     private static final String EXCLUDE = "EXCLUDE";
     private static final String INCLUDE = "INCLUDE";
     private static final String FIRST = "FIRST";
     private static final String FOLLOWING = "FOLLOWING";
     private static final String FOREIGN = "FOREIGN";
     private static final String GROUPING = "GROUPING";
     private static final String GROUPS = "GROUPS";
     private static final String IGNORE = "IGNORE";
     private static final String LAST = "LAST";
     private static final String META = "META";
     private static final String NO = "NO";
     private static final String NULLS = "NULLS";
     private static final String OTHERS = "OTHERS";
     private static final String PARTITION = "PARTITION";
     private static final String PRECEDING = "PRECEDING";
     private static final String RANGE = "RANGE";
     private static final String REFERENCES = "REFERENCES";
     private static final String RESPECT = "RESPECT";
     private static final String ROLLUP = "ROLLUP";
     private static final String ROW = "ROW";
     private static final String ROWS = "ROWS";
     private static final String SETS = "SETS";
     private static final String TIES = "TIES";
     private static final String UNBOUNDED = "UNBOUNDED";
     private static final String REPLACE = "REPLACE";
     private static final String RETURNS = "RETURNS";
     private static final String CONFIG = "CONFIG";
     private static final String STATISTICS = "STATISTICS";


     private static final String INT_TYPE_NAME = "int";
     private static final String UDF_VARARGS_PARAM_NAME = "args"; // Note: this value is stored in the function metadata

     // error configuration
     protected static final boolean REPORT_EXPECTED_TOKENS = false;

     private int externalVarCounter;

     private DataverseName defaultDataverse;

     private SqlppGroupingSetsParser groupingSetsParser;

     private final WarningCollector warningCollector = new WarningCollector();

     private final Map<SourceLocation, String> hintCollector = new HashMap<SourceLocation, String>();

     private static class IndexParams {
       public IndexType type;
       public int gramLength;
       public String fullTextConfig;

       public IndexParams(IndexType type, int gramLength, String fullTextConfig) {
         this.type = type;
         this.gramLength = gramLength;
         this.fullTextConfig = fullTextConfig;
       }
     }

     private static class FunctionName {
        public DataverseName dataverse;
        public String library;
        public String function;
        public Token hintToken;
        public SourceLocation sourceLoc;
     }

     private IRecordFieldDataGen parseFieldDataGen(Token hintToken) throws ParseException {
       String[] splits = hintToken.hintParams != null ? hintToken.hintParams.split("\\s+") : null;
       switch (hintToken.hint) {
         case VAL_FILE_HINT:
           File[] valFiles = new File[splits.length];
           for (int k=0; k<splits.length; k++) {
             valFiles[k] = new File(splits[k]);
           }
           return new FieldValFileDataGen(valFiles);
         case VAL_FILE_SAME_INDEX_HINT:
           return new FieldValFileSameIndexDataGen(new File(splits[0]), splits[1]);
         case LIST_VAL_FILE_HINT:
           return new ListValFileDataGen(new File(splits[0]), Integer.parseInt(splits[1]), Integer.parseInt(splits[2]));
         case LIST_HINT:
           return new ListDataGen(Integer.parseInt(splits[0]), Integer.parseInt(splits[1]));
         case INTERVAL_HINT:
           FieldIntervalDataGen.ValueType vt;
           switch (splits[0]) {
             case "int":
               vt = FieldIntervalDataGen.ValueType.INT;
               break;
             case "long":
               vt = FieldIntervalDataGen.ValueType.LONG;
               break;
             case "float":
               vt = FieldIntervalDataGen.ValueType.FLOAT;
               break;
             case "double":
               vt = FieldIntervalDataGen.ValueType.DOUBLE;
               break;
             default:
               throw new SqlppParseException(getSourceLocation(hintToken),
                 "Unknown type for interval data gen: " + splits[0]);
           }
           return new FieldIntervalDataGen(vt, splits[1], splits[2]);
         case INSERT_RAND_INT_HINT:
           return new InsertRandIntDataGen(splits[0], splits[1]);
         case DATE_BETWEEN_YEARS_HINT:
           return new DateBetweenYearsDataGen(Integer.parseInt(splits[0]), Integer.parseInt(splits[1]));
         case DATETIME_BETWEEN_YEARS_HINT:
           return new DatetimeBetweenYearsDataGen(Integer.parseInt(splits[0]), Integer.parseInt(splits[1]));
         case DATETIME_ADD_RAND_HOURS_HINT:
           return new DatetimeAddRandHoursDataGen(Integer.parseInt(splits[0]), Integer.parseInt(splits[1]), splits[2]);
         case AUTO_HINT:
           return new AutoDataGen(splits[0]);
         default:
           return null;
       }
     }

     public SQLPPParser(String s) {
         this(new StringReader(s));
         super.setInput(s);
     }

     public static void main(String[] args) throws ParseException, TokenMgrError, IOException, FileNotFoundException, CompilationException {
         File file = new File(args[0]);
         Reader fis = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
         SQLPPParser parser = new SQLPPParser(fis);
         List<Statement> st = parser.parse();
         //st.accept(new SQLPPPrintVisitor(), 0);
     }

     @Override
     public List<Statement> parse() throws CompilationException {
         return parseImpl(new ParseFunction<List<Statement>>() {
             @Override
             public List<Statement> parse() throws ParseException {
                 return SQLPPParser.this.Statement();
             }
         });
     }

     @Override
     public Expression parseExpression() throws CompilationException {
         return parseImpl(new ParseFunction<Expression>() {
             @Override
             public Expression parse() throws ParseException {
                 return SQLPPParser.this.Expression();
             }
         });
     }

     private static Expression parseExpression(String text) throws CompilationException {
         return new SQLPPParser(text).parseExpression();
     }

     @Override
     public List<String> parseMultipartIdentifier() throws CompilationException {
         return parseImpl(new ParseFunction<List<String>>() {
             @Override
             public List<String> parse() throws ParseException {
                 return SQLPPParser.this.MultipartIdentifier().first;
             }
         });
     }

     private List<String> parseParenthesizedIdentifierList() throws CompilationException {
         return parseImpl(new ParseFunction<List<String>>() {
             @Override
             public  List<String> parse() throws ParseException {
                 return SQLPPParser.this.ParenthesizedIdentifierList();
             }
         });
     }

     private static List<String> parseParenthesizedIdentifierList(String text) throws CompilationException {
         return new SQLPPParser(text).parseParenthesizedIdentifierList();
     }

     private Pair<HashJoinExpressionAnnotation.BuildOrProbe, String> parseHashJoinParams() throws CompilationException {
         return parseImpl(new ParseFunction<Pair<HashJoinExpressionAnnotation.BuildOrProbe, String>>() {
             @Override
             public  Pair<HashJoinExpressionAnnotation.BuildOrProbe, String> parse() throws ParseException {
                 return SQLPPParser.this.buildOrProbeParenthesizedIdentifier();
             }
         });
     }

     private static Pair<HashJoinExpressionAnnotation.BuildOrProbe, String> parseHashJoinParams(String text) throws CompilationException {
         return new SQLPPParser(text).parseHashJoinParams();
     }

     private String parseBroadcastJoinParams() throws CompilationException {
             return parseImpl(new ParseFunction<String>() {
                 @Override
                 public  String parse() throws ParseException {
                     return SQLPPParser.this.parenthesizedIdentifier();
                 }
             });
         }

     private static String parseBroadcastJoinParams(String text) throws CompilationException {
         return new SQLPPParser(text).parseBroadcastJoinParams();
     }

     private List<Literal> parseParenthesizedLiteralList() throws CompilationException {
         return parseImpl(new ParseFunction<List<Literal>>() {
             @Override
             public  List<Literal> parse() throws ParseException {
                 return SQLPPParser.this.ParenthesizedLiteralList();
             }
         });
     }

     private static List<Literal> parseParenthesizedLiteralList(String text) throws CompilationException {
         return new SQLPPParser(text).parseParenthesizedLiteralList();
     }

     @Override
     public FunctionDecl parseFunctionBody(FunctionSignature signature, List<String> paramNames, boolean isStored)
       throws CompilationException {
         return parseImpl(new ParseFunction<FunctionDecl>() {
             @Override
             public FunctionDecl parse() throws ParseException {
                 DataverseName dataverse = defaultDataverse;
                 defaultDataverse = signature.getDataverseName();
                 createNewScope();
                 List<VarIdentifier> paramVars = new ArrayList<VarIdentifier>(paramNames.size());
                 for (String paramName : paramNames) {
                     paramVars.add(SqlppVariableUtil.toInternalVariableIdentifier(paramName));
                 }
                 Expression functionBodyExpr = SQLPPParser.this.FunctionBody();
                 removeCurrentScope();
                 defaultDataverse = dataverse;
                 return new FunctionDecl(signature, paramVars, functionBodyExpr, isStored);
             }
         });
     }

     @Override
     public ViewDecl parseViewBody(DatasetFullyQualifiedName viewName) throws CompilationException {
         return parseImpl(new ParseFunction<ViewDecl>() {
             @Override
             public ViewDecl parse() throws ParseException {
                 DataverseName dataverse = defaultDataverse;
                 defaultDataverse = viewName.getDataverseName();
                 createNewScope();
                 Expression viewBodyExpr = SQLPPParser.this.ViewBody();
                 removeCurrentScope();
                 defaultDataverse = dataverse;
                 return new ViewDecl(viewName, viewBodyExpr);
             }
         });
     }

     private <T> T parseImpl(ParseFunction<T> parseFunction) throws CompilationException {
         warningCollector.clear();
         hintCollector.clear();
         token_source.hintCollector = hintCollector;
         try {
             return parseFunction.parse();
         } catch (SqlppParseException e) {
             if (e.getCause() instanceof AlgebricksException) {
               AlgebricksException cause = (AlgebricksException) e.getCause();
               if (cause.getError().isPresent() && cause.getError().get() instanceof ErrorCode) {
                 throw new CompilationException((ErrorCode) cause.getError().get(), e.getSourceLocation(),
                   cause.getParams());
               }
             }
             throw new CompilationException(ErrorCode.PARSE_ERROR, e.getSourceLocation(),
               LogRedactionUtil.userData(getMessage(e)));
         } catch (ParseException e) {
             throw new CompilationException(ErrorCode.PARSE_ERROR, LogRedactionUtil.userData(getMessage(e)));
         } catch (Error e) {
             // this is here as the JavaCharStream that's below the lexer sometimes throws Errors that are not handled
             // by the generated lexer or parser (e.g it does this for invalid backslash u + 4 hex digits escapes)
             final String msg = e.getClass().getSimpleName() + (e.getMessage() != null ? ": " + e.getMessage() : "");
             throw new CompilationException(ErrorCode.PARSE_ERROR, LogRedactionUtil.userData(msg));
         } finally {
             reportUnclaimedHints();
         }
     }

     @FunctionalInterface
     private interface ParseFunction<T> {
         T parse() throws ParseException;
     }

     @Override
     public void getWarnings(IWarningCollector outWarningCollector) {
         warningCollector.getWarnings(outWarningCollector);
     }

     @Override
     public void getWarnings(Collection<? super Warning> outWarnings, long maxWarnings) {
         warningCollector.getWarnings(outWarnings, maxWarnings);
     }

     @Override
     public long getTotalWarningsCount() {
         return warningCollector.getTotalWarningsCount();
     }

     protected String getMessage(ParseException pe) {
         Token currentToken = pe.currentToken;
         if (currentToken == null) {
             return pe.getMessage();
         }
         int[][] expectedTokenSequences = pe.expectedTokenSequences;
         String[] tokenImage = pe.tokenImage;
         String sep = REPORT_EXPECTED_TOKENS ? eol : " ";
         StringBuilder expected = REPORT_EXPECTED_TOKENS ? new StringBuilder() : null;
         int maxSize = appendExpected(expected, expectedTokenSequences, tokenImage);
         Token tok = currentToken.next;
         int line = tok.beginLine;
         StringBuilder message = new StringBuilder(128);
         message.append("In line ").append(line).append(" >>").append(getLine(line)).append("<<").append(sep).append("Encountered ");
         for (int i = 0; i < maxSize; i++) {
             if (i != 0) {
                 message.append(' ');
             }
             if (tok.kind == 0) {
                 message.append(fixQuotes(tokenImage[0]));
                 break;
             }
             final String fixedTokenImage = tokenImage[tok.kind];
             if (! tok.image.equalsIgnoreCase(stripQuotes(fixedTokenImage))) {
                 message.append(fixQuotes(fixedTokenImage)).append(' ');
             }
             message.append(quot).append(addEscapes(tok.image)).append(quot);
             tok = tok.next;
         }
         message.append(" at column ").append(currentToken.next.beginColumn).append('.').append(sep);
         if (REPORT_EXPECTED_TOKENS) {
             if (expectedTokenSequences.length == 1) {
                 message.append("Was expecting:").append(sep).append("    ");
             } else {
                 message.append("Was expecting one of:").append(sep).append("    ");
             }
             message.append(expected);
         }
         return message.toString();
     }

     protected static SourceLocation getSourceLocation(Token token) {
         return
           token == null ? null :
           token.sourceLocation != null ? token.sourceLocation :
           new SourceLocation(token.beginLine, token.beginColumn);
     }

     protected static <T extends AbstractLangExpression> T addSourceLocation(T expr, Token token) {
         expr.setSourceLocation(getSourceLocation(token));
         return expr;
     }

     private boolean isToken(String image) {
       return isToken(token, image);
     }

     private static boolean isToken(Token token, String image) {
       return token.image.equalsIgnoreCase(image);
     }

     private void expectToken(String image) throws SqlppParseException {
       expectToken(token, image);
     }

     private static void expectToken(Token token, String image) throws SqlppParseException {
       if (!isToken(token, image)) {
         throw createUnexpectedTokenError(token);
       }
     }

     private SqlppParseException createUnexpectedTokenError() {
       return createUnexpectedTokenError(token, null);
     }

     private static SqlppParseException createUnexpectedTokenError(Token t) {
       return createUnexpectedTokenError(t, null);
     }

     private SqlppParseException createUnexpectedTokenError(String expected) {
       return createUnexpectedTokenError(token, expected);
     }

     private static SqlppParseException createUnexpectedTokenError(Token t, String expected) {
       String message = "Unexpected token: " + LogRedactionUtil.userData(t.image) +
         (expected == null ? "" : ". Expected: " + LogRedactionUtil.userData(expected));
       return new SqlppParseException(getSourceLocation(t), message);
     }

     private boolean laToken(int idx, int kind) {
       Token t = getToken(idx);
       return t.kind == kind;
     }

     private boolean laToken(int idx, int kind, String image) {
       Token t = getToken(idx);
       return t.kind == kind && t.image.equalsIgnoreCase(image);
     }

     private boolean laIdentifier(int idx, String image) {
       return laToken(idx, IDENTIFIER, image);
     }

     private boolean laIdentifier(String image) {
       return laIdentifier(1, image);
     }

     private Token fetchHint(Token token, SqlppHint... expectedHints) {
       Token hintToken = token.specialToken;
       if (hintToken == null) {
         return null;
       }
       SourceLocation sourceLoc = getSourceLocation(hintToken);
       hintCollector.remove(sourceLoc);
       if (hintToken.hint == null) {
         warnUnexpectedHint(hintToken.hintParams, sourceLoc, expectedHints);
         return null;
       } else if (!ArrayUtils.contains(expectedHints, hintToken.hint)) {
         warnUnexpectedHint(hintToken.hint.getIdentifier(), sourceLoc, expectedHints);
         return null;
       } else {
         return hintToken;
       }
     }

     private void reportUnclaimedHints() {
         for (Map.Entry<SourceLocation, String> me : hintCollector.entrySet()) {
             warnUnexpectedHint(me.getValue(), me.getKey(), "None");
         }
     }

     private void warnUnexpectedHint(String actualHint, SourceLocation sourceLoc, SqlppHint... expectedHints) {
         warnUnexpectedHint(actualHint, sourceLoc, StringUtil.join(expectedHints, ", ", "\""));
     }

     private void warnUnexpectedHint(String actualHint, SourceLocation sourceLoc, String expectedHints) {
         if (warningCollector.shouldWarn()) {
             warningCollector.warn(Warning.of(sourceLoc, ErrorCode.UNEXPECTED_HINT, actualHint, expectedHints));
         }
     }

     private IExpressionAnnotation parseExpressionAnnotation(Token hintToken) {
       // placeholder for the annotation that should be returned if this hint's parameters cannot be parsed
       IExpressionAnnotation onParseErrorReturn = null;
       double selectivity, cardinality, productivity;
       Pattern number = Pattern.compile("\\d+\\.\\d+");
       Pattern stringNumber = Pattern.compile("\\w+\\s+\\d+\\.\\d+");
       Pattern lessThanOnePat = Pattern.compile("0\\.\\d+");
       try {
         switch (hintToken.hint) {
            case SINGLE_DATASET_PREDICATE_SELECTIVITY_HINT:
              if (hintToken.hintParams == null) {
                throw new SqlppParseException(getSourceLocation(hintToken), "Expected selectivity value");
              }
              else {
                selectivity = 1.0; // uninitialized
                Matcher mat = lessThanOnePat.matcher(hintToken.hintParams);
                if (mat.find()) {
                  selectivity = Double.parseDouble (mat.group());
                }
                else {
                  throw new SqlppParseException(getSourceLocation(hintToken), "Selectivity has to be a decimal value greater than 0 and less than 1");
                }
                return new PredicateCardinalityAnnotation(selectivity);
              }
            case JOIN_PREDICATE_PRODUCTIVITY_HINT:
              if (hintToken.hintParams == null) {
                throw new SqlppParseException(getSourceLocation(hintToken), "Expected productivity collection name and value");
              }
              else {
                productivity = 1.0; // uninitialized
                String leftSideDataSet = null;
                Matcher StringNum = stringNumber.matcher(hintToken.hintParams);
                if (StringNum.find()) {
                  String matchedGroup = StringNum.group();
                  Pattern var = Pattern.compile("[a-zA-Z]\\w*"); // any word character [a-zA-Z_0-9]
                  Matcher matVar = var.matcher(matchedGroup);
                  if (matVar.find()) {
                    leftSideDataSet = matVar.group();
                  }
                  else {
                     throw new SqlppParseException(getSourceLocation(hintToken), "Expected productivity collection name");
                  }
                  Matcher numMat = number.matcher(matchedGroup);
                  if (numMat.find()) {
                    productivity = Double.parseDouble (numMat.group());
                  }
                  else {
                    throw new SqlppParseException(getSourceLocation(hintToken), "Productivity has to be a decimal value greater than 0");
                  }
                }
                else {
                  throw new SqlppParseException(getSourceLocation(hintToken), "Invalid format for productivity values");
                }
                // attach hint to global scope
                return new JoinProductivityAnnotation (productivity, leftSideDataSet);
              }
           case HASH_BROADCAST_JOIN_HINT:
             if (hintToken.hintParams == null) {
               return new BroadcastExpressionAnnotation(BroadcastExpressionAnnotation.BroadcastSide.RIGHT);
             }
             else {
               // if parameter parsing fails then ignore this hint.
               String name = parseBroadcastJoinParams(hintToken.hintParams);
               return new BroadcastExpressionAnnotation(name);
             }
           case HASH_JOIN_HINT:
             if (hintToken.hintParams == null) {
               throw new SqlppParseException(getSourceLocation(hintToken), "Expected hash join build/probe collection name");
             }
             else {
               // if parameter parsing fails then ignore this hint.
               Pair<HashJoinExpressionAnnotation.BuildOrProbe, String> pair = parseHashJoinParams(hintToken.hintParams);
               return new HashJoinExpressionAnnotation(pair);
             }
           case INDEXED_NESTED_LOOP_JOIN_HINT:
             if (hintToken.hintParams == null) {
               return IndexedNLJoinExpressionAnnotation.INSTANCE_ANY_INDEX;
             } else {
               // if parameter parsing fails then return hint annotation without parameters
               onParseErrorReturn = IndexedNLJoinExpressionAnnotation.INSTANCE_ANY_INDEX;
               List<String> indexNames = parseParenthesizedIdentifierList(hintToken.hintParams);
               return IndexedNLJoinExpressionAnnotation.newInstance(indexNames);
             }
           case RANGE_HINT:
             Expression rangeExpr = parseExpression(hintToken.hintParams);
             RangeMap rangeMap = RangeMapBuilder.parseHint(rangeExpr);
             return new RangeAnnotation(rangeMap);
           case SKIP_SECONDARY_INDEX_SEARCH_HINT:
             if (hintToken.hintParams == null) {
               return SkipSecondaryIndexSearchExpressionAnnotation.INSTANCE_ANY_INDEX;
             } else {
               // if parameter parsing fails then ignore this hint
               List<String> indexNames = parseParenthesizedIdentifierList(hintToken.hintParams);
               return SkipSecondaryIndexSearchExpressionAnnotation.newInstance(indexNames);
             }
           case SPATIAL_JOIN_HINT:
             List<Literal> hintValues = parseParenthesizedLiteralList(hintToken.hintParams);

             // Handle exceptions
             if (hintValues.size() != 6) {
               throw new SqlppParseException(getSourceLocation(hintToken), String.format("Unexpected hint: %s. 6 arguments are required.",
                                 hintToken.hint.toString()));
             }

             for (int i = 0; i < 4; i++) {
               Literal lit = hintValues.get(i);
               if ((lit.getLiteralType() != Literal.Type.DOUBLE)
                     && (lit.getLiteralType() != Literal.Type.FLOAT)
                     && (lit.getLiteralType() != Literal.Type.LONG)
                     && (lit.getLiteralType() != Literal.Type.INTEGER)){
                 throw new SqlppParseException(getSourceLocation(hintToken), String.format("Unexpected hint: %s. Numeric value is required for first 4 arguments.",
                                 hintToken.hint.toString()));
               }
             }

             for (int i = 4; i < 6; i++) {
               Literal lit = hintValues.get(i);
               if ((lit.getLiteralType() != Literal.Type.LONG)
                     && (lit.getLiteralType() != Literal.Type.INTEGER)) {
                 throw new SqlppParseException(getSourceLocation(hintToken), String.format("Unexpected hint: %s. Long/int is required for last 2 arguments.",
                               hintToken.hint.toString()));
               }
             }

             try {
               double minX = ExpressionUtils.getDoubleValue(hintValues.get(0));
               double minY = ExpressionUtils.getDoubleValue(hintValues.get(1));
               double maxX = ExpressionUtils.getDoubleValue(hintValues.get(2));
               double maxY = ExpressionUtils.getDoubleValue(hintValues.get(3));
               int numRows = (int) ExpressionUtils.getLongValue(hintValues.get(4));
               int numColumns = (int) ExpressionUtils.getLongValue(hintValues.get(5));
               SpatialJoinAnnotation spatialJoinAnn = new SpatialJoinAnnotation(minX, minY, maxX, maxY, numRows, numColumns);
               return spatialJoinAnn;
             } catch (TypeMismatchException e) {
               throw new SqlppParseException(getSourceLocation(hintToken), e.getMessage());
             }
           case USE_SECONDARY_INDEX_SEARCH_HINT:
             if (hintToken.hintParams == null) {
               throw new SqlppParseException(getSourceLocation(hintToken), "Expected index name(s)");
             } else {
               // if parameter parsing fails then ignore this hint
               List<String> indexNames = parseParenthesizedIdentifierList(hintToken.hintParams);
               return SecondaryIndexSearchPreferenceAnnotation.newInstance(indexNames);
             }
           default:
             throw new SqlppParseException(getSourceLocation(hintToken), "Unexpected hint");
         }
       } catch (SqlppParseException e) {
         if (warningCollector.shouldWarn()) {
             warningCollector.warn(Warning.of(getSourceLocation(hintToken), ErrorCode.INVALID_HINT,
               hintToken.hint.toString(), e.getMessage()));
         }
         return onParseErrorReturn;
       } catch (CompilationException e) {
         if (warningCollector.shouldWarn()) {
             warningCollector.warn(Warning.of(getSourceLocation(hintToken), ErrorCode.INVALID_HINT,
               hintToken.hint.toString(), e.getMessage()));
         }
         return onParseErrorReturn;
       }
     }

     private void ensureNoTypeDeclsInFunction(String fnName, List<Pair<VarIdentifier, TypeExpression>> paramList,
       TypeExpression returnType, Token startToken) throws SqlppParseException {
       for (Pair<VarIdentifier, TypeExpression> p : paramList) {
         if (p.second != null) {
           String paramName = SqlppVariableUtil.toUserDefinedName(p.first.getValue());
           throw new SqlppParseException(getSourceLocation(startToken),
             "Unexpected type declaration for parameter " + paramName + " in function " + fnName);
         }
       }
       if (returnType != null) {
         throw new SqlppParseException(getSourceLocation(startToken),
           "Unexpected return type declaration for function " + fnName);
       }
     }

     private void ensureIntegerLiteral(LiteralExpr expr, String errorPrefix)  throws SqlppParseException {
         Literal lit = expr.getValue();
         if (lit.getLiteralType() != Literal.Type.INTEGER && lit.getLiteralType() != Literal.Type.LONG) {
             throw new SqlppParseException(expr.getSourceLocation(), errorPrefix + " should be an INTEGER");
         }
     }

     private DataverseName createDataverseName(List<String> parts, int fromIndex, int toIndex, Token startToken)
       throws SqlppParseException {
       try {
         return DataverseName.create(parts, fromIndex, toIndex);
       } catch (AsterixException e) {
         SqlppParseException pe = new SqlppParseException(getSourceLocation(startToken), e.getMessage());
         pe.initCause(e);
         throw pe;
       }
     }

     private void validatePKFields(Object primaryKeyFields, Token startToken) throws SqlppParseException {
       if (primaryKeyFields == null) {
         throw new SqlppParseException(getSourceLocation(startToken), "Invalid primary key specification.");
       }
     }
 }

 PARSER_END(SQLPPParser)

 List<Statement> Statement() throws ParseException:
 {
   scopeStack.push(RootScopeFactory.createRootScope(this));
   List<Statement> decls = new ArrayList<Statement>();
   Statement stmt = null;
 }
 {
   (
     (stmt = ExplainStatement()
       {
         decls.add(stmt);
       }
     )?
     (<SEMICOLON>)+
   )*
   <EOF>
   {
     return decls;
   }
 }

 Statement ExplainStatement() throws ParseException:
 {
   Statement stmt = null;
   Token explainToken = null;
 }
 {
    ( <EXPLAIN> { explainToken = token; } )?
    stmt = SingleStatement()
    {
       if (explainToken != null) {
         if (stmt.getKind() == Statement.Kind.QUERY) {
           ((Query)stmt).setExplain(true);
         } else {
           throw new SqlppParseException(getSourceLocation(explainToken),
             "EXPLAIN is not supported for this kind of statement");
         }
       }
       return stmt;
    }
 }

 Statement SingleStatement() throws ParseException:
 {
   Statement stmt = null;
 }
 {
   (
     stmt = DataverseDeclaration()
     | stmt = FunctionDeclaration()
     | stmt = CreateStatement()
     | stmt = LoadStatement()
     | stmt = CopyStatement()
     | stmt = DropStatement()
     | stmt = SetStatement()
     | stmt = InsertStatement()
     | stmt = DeleteStatement()
     | stmt = UpdateStatement()
     | stmt = UpsertStatement()
     | stmt = ConnectionStatement()
     | stmt = CompactStatement()
     | stmt = AnalyzeStatement()
     | stmt = Query()
   )
   {
     return stmt;
   }
 }

 DataverseDecl DataverseDeclaration() throws ParseException:
 {
   Token startToken = null;
   DataverseName dvName = null;
 }
 {
   <USE> { startToken = token; } dvName = DataverseName()
     {
       defaultDataverse = dvName;
       DataverseDecl dvDecl = new DataverseDecl(defaultDataverse);
       return addSourceLocation(dvDecl, startToken);
     }
 }

 Statement CreateStatement() throws ParseException:
 {
   Token startToken = null;
   Statement stmt = null;
 }
 {
   <CREATE> { startToken = token; }
   (
     stmt = CreateOrReplaceStatement(startToken)
     | stmt = CreateTypeStatement(startToken)
     | stmt = CreateNodegroupStatement(startToken)
     | stmt = CreateDatasetStatement(startToken)
     | stmt = CreateIndexStatement(startToken)
     | stmt = CreateDatabaseStatement(startToken)
     | stmt = CreateDataverseStatement(startToken)
     | stmt = CreateFunctionStatement(startToken, false)
     | stmt = CreateAdapterStatement(startToken)
     | stmt = CreateSynonymStatement(startToken)
     | stmt = CreateFeedStatement(startToken)
     | stmt = CreateFeedPolicyStatement(startToken)
     | stmt = CreateFullTextStatement(startToken)
     | stmt = CreateViewStatement(startToken, false)
   )
   {
     return stmt;
   }
 }

 Statement CreateOrReplaceStatement(Token startStmtToken) throws ParseException:
 {
   Statement stmt = null;
   Token replaceToken = null;
 }
 {
   <OR> <IDENTIFIER> { replaceToken = token; }
   (
     stmt = CreateFunctionStatement(startStmtToken, true)
     | stmt = CreateViewStatement(startStmtToken, true)
   )
   {
     // check expected token here to make the grammar extension plugin happy
     expectToken(replaceToken, REPLACE);
     return stmt;
   }
 }

 TypeDecl CreateTypeStatement(Token startStmtToken) throws ParseException:
 {
   TypeDecl stmt = null;
 }
 {
   <TYPE> stmt = TypeSpecification(startStmtToken)
   {
     return stmt;
   }
 }

 TypeDecl TypeSpecification(Token startStmtToken) throws ParseException:
 {
   Pair<DataverseName,Identifier> nameComponents = null;
   boolean ifNotExists = false;
   TypeExpression typeExpr = null;
 }
 {
   nameComponents = TypeName() ifNotExists = IfNotExists()
   <AS> typeExpr = RecordTypeDef()
   {
     boolean dgen = false;
     long numValues = -1;
     String filename = null;
     Token hintToken = fetchHint(startStmtToken, SqlppHint.DGEN_HINT);
     if (hintToken != null) {
         String hintParams = hintToken.hintParams;
         String[] splits = hintParams != null ? hintParams.split("\\s+") : null;
         if (splits == null || splits.length != 2) {
           throw new SqlppParseException(getSourceLocation(hintToken),
             "Expecting /*+ dgen <filename> <numberOfItems> */");
         }
         dgen = true;
         filename = splits[0];
         numValues = Long.parseLong(splits[1]);
     }
     TypeDataGen tddg = new TypeDataGen(dgen, filename, numValues);
     TypeDecl stmt = new TypeDecl(nameComponents.first, nameComponents.second, typeExpr, tddg, ifNotExists);
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 NodegroupDecl CreateNodegroupStatement(Token startStmtToken) throws ParseException:
 {
   NodegroupDecl stmt = null;
 }
 {
   <NODEGROUP> stmt = NodegroupSpecification(startStmtToken)
   {
     return stmt;
   }
 }

 NodegroupDecl NodegroupSpecification(Token startStmtToken) throws ParseException:
 {
   String name = null;
   String tmp = null;
   boolean ifNotExists = false;
   List<Identifier> ncNames = new ArrayList<Identifier>();
 }
 {
   name = Identifier() ifNotExists = IfNotExists()
   <ON> tmp = Identifier()
   {
     ncNames.add(new Identifier(tmp));
   }
   ( <COMMA> tmp = Identifier()
     {
       ncNames.add(new Identifier(tmp));
     }
   )*
   {
     NodegroupDecl stmt = new NodegroupDecl(new Identifier(name), ncNames, ifNotExists);
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 void Dataset() throws ParseException:
 {
 }
 {
     (<DATASET>|<COLLECTION>)
 }

 void DatasetToken() throws ParseException:
 {
 }
 {
     Dataset()
 }

 DatasetDecl CreateDatasetStatement(Token startStmtToken) throws ParseException:
 {
   DatasetDecl stmt = null;
 }
 {
   (
     (<INTERNAL>)? Dataset() stmt = DatasetSpecification(startStmtToken)
     | <EXTERNAL> Dataset() stmt = ExternalDatasetSpecification(startStmtToken)
   )
   {
     return stmt;
   }
 }

 DatasetDecl DatasetSpecification(Token startStmtToken) throws ParseException:
 {
   Pair<DataverseName,Identifier> nameComponents = null;
   boolean ifNotExists = false;
   TypeExpression typeExpr = null;
   TypeExpression metaTypeExpr = null;
   Pair<List<Integer>, List<List<String>>> primaryKeyFields = null;
   Triple<List<Integer>, List<List<String>>, List<TypeExpression>> primaryKeyFieldsWithTypes = null;
   Map<String,String> hints = new HashMap<String,String>();
   DatasetDecl stmt = null;
   boolean autogenerated = false;
   Pair<Integer, List<String>> filterField = null;
   RecordConstructor withRecord = null;
   SelectExpression selectExpr = null;
   Query query = null;
 }
 {
   nameComponents = QualifiedName()
   (typeExpr = DatasetTypeSpecification())?
   (
     { String name; }
     <WITH>
     name = Identifier()
     {
         if (!name.equalsIgnoreCase("meta")){
             throw new SqlppParseException(getSourceLocation(startStmtToken),
                 "We can only support one additional associated field called \"meta\".");
         }
     }
     metaTypeExpr = DatasetTypeSpecification()
   )?
   ifNotExists = IfNotExists()
   (LOOKAHEAD(3) primaryKeyFieldsWithTypes = PrimaryKeyWithType()
     | primaryKeyFields = PrimaryKey())
   (<AUTOGENERATED> { autogenerated = true; } )?
   ( <HINTS> hints = Properties() )?
   ( LOOKAHEAD(2) <WITH> <FILTER> <ON>  filterField = NestedField() )?
   ( <WITH> withRecord = RecordConstructor() )?
   ( <AS> selectExpr = SelectExpression(false) )?
   {
     try {
       if (selectExpr != null) {
         query = new Query();
         query.setBody(selectExpr);
         query.setSourceLocation(selectExpr.getSourceLocation());
       }

       if (typeExpr == null) {
         validatePKFields(primaryKeyFieldsWithTypes, startStmtToken);
         InternalDetailsDecl idd = new InternalDetailsDecl(primaryKeyFieldsWithTypes.second,
           primaryKeyFieldsWithTypes.first, autogenerated, filterField == null? null : filterField.first,
           filterField == null? null : filterField.second, primaryKeyFieldsWithTypes.third);
         final TypeReferenceExpression anyObjectReference = new TypeReferenceExpression(
           new Pair(MetadataBuiltinEntities.ANY_OBJECT_DATATYPE.getDataverseName(),
             new Identifier(MetadataBuiltinEntities.ANY_OBJECT_DATATYPE.getDatatypeName())));
         stmt = new DatasetDecl(nameComponents.first, nameComponents.second, anyObjectReference, null, hints,
           DatasetType.INTERNAL, idd, withRecord, ifNotExists, query);
         return addSourceLocation(stmt, startStmtToken);
       } else {
         validatePKFields(primaryKeyFields, startStmtToken);
         InternalDetailsDecl idd = new InternalDetailsDecl(primaryKeyFields.second, primaryKeyFields.first, autogenerated,
           filterField == null? null : filterField.first, filterField == null? null : filterField.second);
         DatasetDeclParametersUtil.adjustInlineTypeDecl(typeExpr, primaryKeyFields.second, primaryKeyFields.first, false);
         if (metaTypeExpr != null) {
           DatasetDeclParametersUtil.adjustInlineTypeDecl(metaTypeExpr, primaryKeyFields.second, primaryKeyFields.first,
             true);
         }
         stmt = new DatasetDecl(nameComponents.first, nameComponents.second, typeExpr, metaTypeExpr, hints,
           DatasetType.INTERNAL, idd, withRecord, ifNotExists, query);
         return addSourceLocation(stmt, startStmtToken);
       }
     } catch (CompilationException e) {
        throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage());
     }
   }
 }

 DatasetDecl ExternalDatasetSpecification(Token startStmtToken) throws ParseException:
 {
   Pair<DataverseName,Identifier> nameComponents = null;
   TypeExpression typeExpr = null;
   boolean ifNotExists = false;
   String adapterName = null;
   Map<String,String> properties = null;
   Map<String,String> hints = new HashMap<String,String>();
   DatasetDecl stmt = null;
   RecordConstructor withRecord = null;
 }
 {
   nameComponents = QualifiedName()
   typeExpr = DatasetTypeSpecification()
   ifNotExists = IfNotExists()
   <USING> adapterName = AdapterName() properties = Configuration()
   ( <HINTS> hints = Properties() )?
   ( <WITH> withRecord = RecordConstructor() )?
   {
     ExternalDetailsDecl edd = new ExternalDetailsDecl();
     edd.setAdapter(adapterName);
     edd.setProperties(properties);
     try {
       stmt = new DatasetDecl(nameComponents.first, nameComponents.second, typeExpr, null, hints, DatasetType.EXTERNAL,
         edd, withRecord, ifNotExists);
       return addSourceLocation(stmt, startStmtToken);
     } catch (CompilationException e) {
        throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage());
     }
   }
 }

 TypeExpression DatasetTypeSpecification() throws ParseException:
 {
   TypeExpression typeExpr = null;
 }
 {
   (
     LOOKAHEAD(3) typeExpr = DatasetRecordTypeSpecification(true)
     | typeExpr = DatasetReferenceTypeSpecification()
   )
   {
     return typeExpr;
   }
 }

 TypeExpression DatasetReferenceTypeSpecification() throws ParseException:
 {
   TypeExpression typeExpr = null;
 }
 {
   <LEFTPAREN> typeExpr = TypeReference() <RIGHTPAREN>
   {
     return typeExpr;
   }
 }

 RecordTypeDefinition DatasetRecordTypeSpecification(boolean allowRecordKindModifier) throws ParseException:
 {
   RecordTypeDefinition recordTypeDef = null;
   RecordTypeDefinition.RecordKind recordKind = null;
   Token startToken = null, recordKindToken = null;
 }
 {
    <LEFTPAREN> { startToken = token; } recordTypeDef = DatasetRecordTypeDef() <RIGHTPAREN>
    ( recordKind = RecordTypeKind() { recordKindToken = token; } <TYPE> )?
    {
      if (recordKind == null) {
        recordKind = RecordTypeDefinition.RecordKind.CLOSED;
      } else if (!allowRecordKindModifier) {
        throw createUnexpectedTokenError(recordKindToken);
      }
      recordTypeDef.setRecordKind(recordKind);
      return addSourceLocation(recordTypeDef, startToken);
    }
 }

 RecordTypeDefinition DatasetRecordTypeDef() throws ParseException:
 {
   RecordTypeDefinition recType = new RecordTypeDefinition();
 }
 {
   DatasetRecordField(recType) ( <COMMA> DatasetRecordField(recType) )*
   {
     return recType;
   }
 }

 void DatasetRecordField(RecordTypeDefinition recType) throws ParseException:
 {
   String fieldName;
   TypeExpression type = null;
   boolean nullable = true, missable = true;
 }
 {
   fieldName = Identifier()
   type = TypeReference() ( <NOT> <UNKNOWN> { nullable = false; missable = false; } )?
   {
     recType.addField(fieldName, type, nullable, missable);
   }
 }

 CreateIndexStatement CreateIndexStatement(Token startStmtToken) throws ParseException:
 {
   CreateIndexStatement stmt = null;
 }
 {
   (
     <INDEX> stmt = IndexSpecification(startStmtToken)
     | <PRIMARY> <INDEX> stmt = PrimaryIndexSpecification(startStmtToken)
   )
   {
     return stmt;
   }
 }

 CreateIndexStatement IndexSpecification(Token startStmtToken) throws ParseException:
 {
   Pair<DataverseName,Identifier> nameComponents = null;
   String indexName = null;
   IndexParams indexParams = null;
   CreateIndexStatement.IndexedElement indexedElement = null;
   List<CreateIndexStatement.IndexedElement> indexedElementList = new ArrayList<CreateIndexStatement.IndexedElement>();
   boolean enforced = false;
   boolean ifNotExists = false;
   boolean hasUnnest = false;
   String fullTextConfigName = null;
   Token startElementToken = null;
   Boolean excludeUnknown = null;
   Pair<Map<String, String>, Boolean> castConfigDefaultNull;
   Boolean castDefaultNull = null;
   Map<String, String> castConfig = null;
 }
 {
   (
     indexName = Identifier() ifNotExists = IfNotExists()
     <ON> nameComponents = QualifiedName()
     <LEFTPAREN> { startElementToken  = token; }
       indexedElement = IndexedElement(startElementToken) {
         indexedElementList.add(indexedElement);
         hasUnnest |= indexedElement.hasUnnest();
       }
       (<COMMA> { startElementToken = token; }
         indexedElement = IndexedElement(startElementToken) {
           indexedElementList.add(indexedElement);
           hasUnnest |= indexedElement.hasUnnest();
         }
       )*
     <RIGHTPAREN>
     ( <TYPE> indexParams = IndexType() )? ( <ENFORCED> { enforced = true; } )?
     ( LOOKAHEAD({laIdentifier(EXCLUDE) || laIdentifier(INCLUDE)}) <IDENTIFIER>
     {
       if (isToken(EXCLUDE)) {
         excludeUnknown = true;
       } else if (isToken(INCLUDE)) {
         excludeUnknown = false;
       } else {
         throw createUnexpectedTokenError();
       }
     } <UNKNOWN> <KEY>
     )?

     ( <CAST><LEFTPAREN> castConfigDefaultNull = CastDefaultNull() <RIGHTPAREN>
       {
         castConfig = castConfigDefaultNull.first;
         castDefaultNull = castConfigDefaultNull.second;
       }
     )?
   )
   {
     IndexType indexType;
     int gramLength;
     if (indexParams != null) {
       indexType = indexParams.type;
       gramLength = indexParams.gramLength;
       fullTextConfigName = indexParams.fullTextConfig;
     } else {
       indexType = hasUnnest ? IndexType.ARRAY : IndexType.BTREE;
       gramLength = -1;
       fullTextConfigName = null;
     }
     CreateIndexStatement stmt = new CreateIndexStatement(nameComponents.first, nameComponents.second,
       new Identifier(indexName), indexType, indexedElementList, enforced, gramLength, fullTextConfigName, ifNotExists,
       excludeUnknown, castDefaultNull, castConfig);
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 CreateIndexStatement.IndexedElement IndexedElement(Token startElementToken) throws ParseException:
 {
   Triple<Integer, List<List<String>>, List<Pair<List<String>, IndexedTypeExpression>>> element = null;
   Pair<List<String>, IndexedTypeExpression> elementSimple = null;
   int elementSimpleSource = 0;
 }
 {
   (
     element = IndexedElementUnnestSelect()
     | (
         LOOKAHEAD({ laIdentifier(META) && laToken(2, LEFTPAREN) && laToken(3, RIGHTPAREN) })
         <IDENTIFIER> { expectToken(META); } <LEFTPAREN> <RIGHTPAREN>
         <DOT> elementSimple = IndexedField()
         { elementSimpleSource = 1; }
       )
     | elementSimple = IndexedField()
     | <LEFTPAREN> ( element = IndexedElementUnnestSelect() | elementSimple = IndexedField() ) <RIGHTPAREN>
   )
   {
     int source;
     List<List<String>> unnestList;
     List<Pair<List<String>, IndexedTypeExpression>> projectList;
     if (elementSimple != null) {
       source = elementSimpleSource;
       unnestList = null;
       projectList = Collections.singletonList(elementSimple);
     } else {
       source = element.first;
       unnestList = element.second;
       projectList = element.third;
     }
     CreateIndexStatement.IndexedElement ie = new CreateIndexStatement.IndexedElement(source, unnestList, projectList);
     ie.setSourceLocation(getSourceLocation(startElementToken));
     return ie;
   }
 }

 Triple<Integer, List<List<String>>, List<Pair<List<String>, IndexedTypeExpression>>> IndexedElementUnnestSelect()
   throws ParseException:
 {
   int source = 0;
   Pair<List<List<String>>, List<Pair<List<String>, IndexedTypeExpression>>> element = null;
 }
 {
   <UNNEST>
   (
     (
        LOOKAHEAD({ laIdentifier(META) && laToken(2, LEFTPAREN) && laToken(3, RIGHTPAREN) })
        <IDENTIFIER> { expectToken(META); } <LEFTPAREN> <RIGHTPAREN>
        <DOT> element = IndexedElementUnnestSelectBody() { source = 1; }
     ) | element = IndexedElementUnnestSelectBody()
   )
   {
     return new Triple<Integer, List<List<String>>, List<Pair<List<String>, IndexedTypeExpression>>>(
       source, element.first, element.second
     );
   }
 }

 Pair<List<List<String>>, List<Pair<List<String>, IndexedTypeExpression>>> IndexedElementUnnestSelectBody()
   throws ParseException:
 {
   Triple<List<String>, Token, Token> path = null;
   IndexedTypeExpression type = null;
   List<List<String>> unnestList = new ArrayList();
   List<Pair<List<String>, IndexedTypeExpression>> projectList = new ArrayList();
 }
 {
   path = MultipartIdentifier() { unnestList.add(path.first); }
   ( <UNNEST> path = MultipartIdentifier() { unnestList.add(path.first); })*
   (
     ( <COLON> type = IndexedTypeExpr(false)
       {
         projectList.add(new Pair<List<String>, IndexedTypeExpression>(null, type));
       }
     ) |
     (
       <SELECT> path = MultipartIdentifier() ( <COLON> type = IndexedTypeExpr(false) )?
       {
         projectList.add(new Pair<List<String>, IndexedTypeExpression>(path.first, type));
       }
       ( <COMMA> path = MultipartIdentifier() ( <COLON> type = IndexedTypeExpr(false) )?
         {
           projectList.add(new Pair<List<String>, IndexedTypeExpression>(path.first, type));
         }
       )*
     )
   )?
   {
     if (projectList.isEmpty()) {
       // To support the case (<UNNEST> IDENTIFIER)* IDENTIFIER w/o any type specification.
       projectList.add(new Pair<List<String>, IndexedTypeExpression>(null, null));
     }

     return new Pair<List<List<String>>, List<Pair<List<String>, IndexedTypeExpression>>>(unnestList, projectList);
   }
 }

 Pair<List<String>, IndexedTypeExpression> IndexedField() throws ParseException:
 {
   Triple<List<String>, Token, Token> path = null;
   IndexedTypeExpression type = null;
 }
 {
   path = MultipartIdentifier() ( <COLON> type = IndexedTypeExpr(true) )?
   {
     return new Pair<List<String>, IndexedTypeExpression>(path.first, type);
   }
 }

 CreateIndexStatement PrimaryIndexSpecification(Token startStmtToken) throws ParseException:
 {
   Pair<DataverseName,Identifier> nameComponents = null;
   String indexName = null;
   boolean ifNotExists = false;
 }
 {
   (indexName = Identifier())? ifNotExists = IfNotExists()
   <ON> nameComponents = QualifiedName() (<TYPE> <BTREE>)?
   {
     if (indexName == null) {
       indexName = MetadataConstants.PRIMARY_INDEX_PREFIX + nameComponents.second;
     }
     CreateIndexStatement stmt = new CreateIndexStatement(nameComponents.first, nameComponents.second,
       new Identifier(indexName), IndexType.BTREE, Collections.emptyList(), false, -1, null, ifNotExists, null, null,
       Collections.emptyMap());
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 String FilterField() throws ParseException :
 {
   String filterField = null;
 }
 {
   filterField = Identifier()
     {
       return filterField;
     }
 }

 IndexParams IndexType() throws ParseException:
 {
   IndexType type = null;
   int gramLength = 0;
   String fullTextConfig = null;
 }
 {
   (<BTREE>
     {
       type = IndexType.BTREE;
     }
   | <RTREE>
     {
       type = IndexType.RTREE;
     }
   | <KEYWORD>
     {
       type = IndexType.LENGTH_PARTITIONED_WORD_INVIX;
     }
   | <FULLTEXT>
     {
       type = IndexType.SINGLE_PARTITION_WORD_INVIX;
     }
     // For now we don't allow inverted index creation using a full-text config in another data verse.
     // We may want to support corss-dataverse full-text config access later
     // If so, replace the Identifier() with QualifiedName() to get the dataverse name
     ( <USING> Identifier()
       {
         fullTextConfig = token.image;
       }
     )?
   | <NGRAM> <LEFTPAREN> <INTEGER_LITERAL>
     {
       type = IndexType.LENGTH_PARTITIONED_NGRAM_INVIX;
       gramLength = Integer.valueOf(token.image);
     }
   <RIGHTPAREN>)
     {
       return new IndexParams(type, gramLength, fullTextConfig);
     }
 }

 CreateDatabaseStatement CreateDatabaseStatement(Token startStmtToken) throws ParseException :
 {
   CreateDatabaseStatement stmt = null;
 }
 {
   <DATABASE> stmt = DatabaseSpecification(startStmtToken)
   {
     return stmt;
   }
 }

 CreateDatabaseStatement DatabaseSpecification(Token startStmtToken) throws ParseException :
 {
   String dbName = null;
   boolean ifNotExists = false;
 }
 {
   dbName = Identifier() ifNotExists = IfNotExists()
   {
     CreateDatabaseStatement stmt = new CreateDatabaseStatement(new Identifier(dbName), ifNotExists);
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 CreateDataverseStatement CreateDataverseStatement(Token startStmtToken) throws ParseException :
 {
   CreateDataverseStatement stmt = null;
 }
 {
   <DATAVERSE> stmt = DataverseSpecification(startStmtToken)
   {
     return stmt;
   }
 }

 CreateDataverseStatement DataverseSpecification(Token startStmtToken) throws ParseException :
 {
   DataverseName dvName = null;
   boolean ifNotExists = false;
 }
 {
   dvName = DataverseName() ifNotExists = IfNotExists()
   {
     CreateDataverseStatement stmt = new CreateDataverseStatement(dvName, null, ifNotExists);
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 CreateAdapterStatement CreateAdapterStatement(Token startStmtToken) throws ParseException:
 {
   CreateAdapterStatement stmt = null;
 }
 {
   <ADAPTER> stmt = AdapterSpecification(startStmtToken)
   {
     return stmt;
   }
 }

 CreateAdapterStatement AdapterSpecification(Token startStmtToken) throws ParseException:
 {
   Pair<DataverseName,Identifier> adapterName = null;
   Pair<DataverseName,Identifier> libraryName = null;
   List<String> externalIdentifier = null;
   boolean ifNotExists = false;
 }
 {
   adapterName = QualifiedName()
   ifNotExists = IfNotExists()
   <AS> externalIdentifier = FunctionExternalIdentifier()
   <AT> libraryName = QualifiedName()
   {
     CreateAdapterStatement stmt = new CreateAdapterStatement(adapterName.first, adapterName.second.getValue(),
       libraryName.first, libraryName.second.getValue(), externalIdentifier, ifNotExists);
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 CreateViewStatement CreateViewStatement(Token startStmtToken, boolean orReplace) throws ParseException:
 {
   CreateViewStatement stmt = null;
 }
 {
   <VIEW> stmt = ViewSpecification(startStmtToken, orReplace)
   {
     return stmt;
   }
 }

 CreateViewStatement ViewSpecification(Token startStmtToken, boolean orReplace) throws ParseException:
 {
   Pair<DataverseName, Identifier> nameComponents = null;
   TypeExpression typeExpr = null;
   boolean ifNotExists = false;
   Token beginPos = null, endPos = null;
   Expression viewBodyExpr = null;
   Pair<Map<String, String>, Boolean> viewConfigDefaultNull;
   Boolean defaultNull = null;
   Map<String, String> viewConfig = null;
   Pair<List<Integer>, List<List<String>>> primaryKeyFields = null;
   Pair<List<Integer>, List<List<String>>> foreignKeyFields = null;
   Pair<DataverseName, Identifier> refNameComponents = null;
   List<CreateViewStatement.ForeignKeyDecl> foreignKeyDecls = null;
   DataverseName currentDataverse = defaultDataverse;
 }
 {
   nameComponents = QualifiedName()
   (
       (
         typeExpr = DatasetTypeSpecification()
         ifNotExists = IfNotExists()
         viewConfigDefaultNull = CastDefaultNull()
         {
           viewConfig = viewConfigDefaultNull.first;
           defaultNull = viewConfigDefaultNull.second;
         }
         (
           <PRIMARY> <KEY> <LEFTPAREN> primaryKeyFields = PrimaryKeyFields() <RIGHTPAREN>
           <NOT> <ENFORCED>
         )?
         (
           <IDENTIFIER> { expectToken(FOREIGN); } <KEY> <LEFTPAREN> foreignKeyFields = PrimaryKeyFields() <RIGHTPAREN>
           <IDENTIFIER> { expectToken(REFERENCES); } refNameComponents = QualifiedName()
           <NOT> <ENFORCED>
           {
             if (foreignKeyDecls == null) {
               foreignKeyDecls = new ArrayList<CreateViewStatement.ForeignKeyDecl>();
             }
             foreignKeyDecls.add(new CreateViewStatement.ForeignKeyDecl(foreignKeyFields.second,
               foreignKeyFields.first, refNameComponents.first, refNameComponents.second));
           }
         )*
       )
       |
       ( ifNotExists = IfNotExists() )
   )
   {
     if (orReplace && ifNotExists) {
       throw new SqlppParseException(getSourceLocation(startStmtToken), "Unexpected IF NOT EXISTS");
     }
   }
   <AS>
   {
     beginPos = token;
     createNewScope();
     if (nameComponents.first != null) {
       defaultDataverse = nameComponents.first;
     }
   }
   viewBodyExpr = ViewBody()
   {
     endPos = token;
     String viewBody = extractFragment(beginPos.beginLine, beginPos.beginColumn + 1, endPos.endLine,
       endPos.endColumn + 1);
     removeCurrentScope();
     defaultDataverse = currentDataverse;
     if (typeExpr != null && primaryKeyFields != null) {
       DatasetDeclParametersUtil.adjustInlineTypeDecl(typeExpr, primaryKeyFields.second, primaryKeyFields.first, false);
     }
     CreateViewStatement.KeyDecl primaryKeyDecl = primaryKeyFields != null ?
       new CreateViewStatement.KeyDecl(primaryKeyFields.second, primaryKeyFields.first) : null;
     CreateViewStatement stmt = new CreateViewStatement(nameComponents.first, nameComponents.second.getValue(),
       typeExpr, viewBody, viewBodyExpr, defaultNull, viewConfig, primaryKeyDecl, foreignKeyDecls, orReplace,
       ifNotExists);
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 Expression ViewBody() throws ParseException:
 {
   Expression viewBodyExpr = null;
 }
 {
   (
     ( viewBodyExpr = VariableRef() ( viewBodyExpr = FieldAccessor(viewBodyExpr) )* )
     | viewBodyExpr = SelectExpression(true)
   )
   {
     return viewBodyExpr;
   }
 }

 Pair<Map<String, String>, Boolean> CastDefaultNull() throws ParseException:
 {
   Map<String, String> castConfig = null;
   Boolean defaultNull = null;
   String propertyName = null, propertyValue = null;
 }
 {
   <IDENTIFIER> { expectToken(DEFAULT); } <NULL> { defaultNull = true; }
     (
       LOOKAHEAD(2) <IDENTIFIER> { propertyName = token.image.toLowerCase(); } propertyValue = StringLiteral()
       {
         if (castConfig == null) {
           castConfig = new HashMap<String, String>();
         }
         castConfig.put(propertyName, propertyValue);
       }
     )*
   {
     return new Pair<Map<String, String>, Boolean>(castConfig, defaultNull);
   }
 }

 CreateFunctionStatement CreateFunctionStatement(Token startStmtToken, boolean orReplace) throws ParseException:
 {
   CreateFunctionStatement stmt = null;
 }
 {
   <FUNCTION> stmt = FunctionSpecification(startStmtToken, orReplace)
   {
     return stmt;
   }
 }

 CreateFunctionStatement FunctionSpecification(Token startStmtToken, boolean orReplace) throws ParseException:
 {
   FunctionSignature signature = null;
   FunctionName fctName = null;
   Pair<Integer, List<Pair<VarIdentifier,TypeExpression>>> paramsWithArity = null;
   List<Pair<VarIdentifier,TypeExpression>> params = null;
   int arity = 0;
   TypeExpression returnType = null;
   Token beginPos = null, endPos = null;
   Expression functionBodyExpr = null;
   Pair<DataverseName,Identifier> libraryName = null;
   List<String> externalIdentifier = null;
   RecordConstructor withOptions = null;
   boolean ifNotExists = false;
   CreateFunctionStatement stmt = null;
   DataverseName currentDataverse = defaultDataverse;
 }
 {
   fctName = FunctionName()
   {
      defaultDataverse = fctName.dataverse;
   }
   paramsWithArity = FunctionParameters()
   {
     arity = paramsWithArity.first;
     params = paramsWithArity.second;
     signature = new FunctionSignature(fctName.dataverse, fctName.function, arity);
   }
   ifNotExists = IfNotExists()
   {
     if (orReplace && ifNotExists) {
       throw new SqlppParseException(getSourceLocation(token), "Unexpected IF NOT EXISTS");
     }
   }
   returnType = FunctionReturnType()
   (
     (
       <LEFTBRACE>
       {
         beginPos = token;
         createNewScope();
       }
       functionBodyExpr = FunctionBody()
       <RIGHTBRACE>
       {
         endPos = token;
         String functionBody = extractFragment(beginPos.beginLine, beginPos.beginColumn, endPos.beginLine,
           endPos.beginColumn);
         getCurrentScope().addFunctionDescriptor(signature, false);
         removeCurrentScope();
         ensureNoTypeDeclsInFunction(fctName.function, params, returnType, startStmtToken);
         stmt = new CreateFunctionStatement(signature, params, functionBody, functionBodyExpr, orReplace, ifNotExists);
       }
     )
   |
     (
       <AS> externalIdentifier = FunctionExternalIdentifier()
       <AT> libraryName = QualifiedName()
       (<WITH> withOptions = RecordConstructor())?
       {
         try {
           stmt = new CreateFunctionStatement(signature, params, returnType, libraryName.first,
             libraryName.second.getValue(), externalIdentifier, withOptions, orReplace, ifNotExists);
         } catch (AlgebricksException e) {
             throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage());
         }
       }
     )
   )
   {
     defaultDataverse = currentDataverse;
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 Pair<Integer, List<Pair<VarIdentifier, TypeExpression>>> FunctionParameters() :
 {
   Pair<Integer, List<Pair<VarIdentifier, TypeExpression>>> params = null;
 }
 {
   <LEFTPAREN> (params = FunctionParameterList())? <RIGHTPAREN>
   {
     if (params == null) {
       params = new Pair<Integer, List<Pair<VarIdentifier, TypeExpression>>>(
         0, Collections.<Pair<VarIdentifier, TypeExpression>>emptyList()
       );
     }
     return params;
   }
 }

 Pair<Integer, List<Pair<VarIdentifier, TypeExpression>>> FunctionParameterList() :
 {
   List<Pair<VarIdentifier, TypeExpression>> paramList = null;
   Pair<VarIdentifier, TypeExpression> param = null;
   int arity = 0;
 }
 {
   (
     (
       <DOT> <DOT> <DOT>
       {
         param = new Pair<VarIdentifier, TypeExpression>(
           SqlppVariableUtil.toInternalVariableIdentifier(UDF_VARARGS_PARAM_NAME), null
         );
         paramList = Collections.<Pair<VarIdentifier, TypeExpression>>singletonList(param);
         arity = FunctionIdentifier.VARARGS;
       }
     )
     |
     (
       param = FunctionParameter()
       {
         paramList = new ArrayList<Pair<VarIdentifier, TypeExpression>>();
         paramList.add(param);
       }
       ( <COMMA> param = FunctionParameter() { paramList.add(param); } )*
       {
         arity = paramList.size();
       }
     )
   )
   {
     return new Pair<Integer, List<Pair<VarIdentifier, TypeExpression>>>(arity, paramList);
   }
 }

 Pair<VarIdentifier,TypeExpression> FunctionParameter() throws ParseException:
 {
   String name = null;
   TypeExpression type = null;
 }
 {
   name = VariableIdentifier()
   ( LOOKAHEAD(3) (<COLON>)? type = TypeExpr(false) )?
   {
     return new Pair<VarIdentifier,TypeExpression>(new VarIdentifier(name), type);
   }
 }

 TypeExpression FunctionReturnType() throws ParseException:
 {
   TypeExpression returnType = null;
 }
 {
   ( LOOKAHEAD({laIdentifier(RETURNS)}) <IDENTIFIER> returnType = TypeExpr(false) )?
   {
     return returnType;
   }
 }

 Expression FunctionBody() throws ParseException:
 {
   Expression functionBodyExpr = null;
 }
 {
   ( functionBodyExpr = SelectExpression(true) | functionBodyExpr = Expression() )
   {
     return functionBodyExpr;
   }
 }

 List<String> FunctionExternalIdentifier() throws ParseException:
 {
   String ident = null;
   List<String> identList = new ArrayList(2);
 }
 {
   ident = StringLiteral() { identList.add(ident.trim()); }
   ( <COMMA> ident = StringLiteral() { identList.add(ident.trim()); } )*
   {
     return identList;
   }
 }

 CreateFeedStatement CreateFeedStatement(Token startStmtToken) throws ParseException:
 {
   CreateFeedStatement stmt = null;
 }
 {
   <FEED> stmt = FeedSpecification(startStmtToken)
   {
     return stmt;
   }
 }

 CreateFeedStatement FeedSpecification(Token startStmtToken) throws ParseException:
 {
   Pair<DataverseName,Identifier> nameComponents = null;
   boolean ifNotExists = false;
   String adapterName = null;
   Map<String,String> properties = null;
   CreateFeedStatement stmt = null;
   Pair<Identifier,Identifier> sourceNameComponents = null;
   RecordConstructor withRecord = null;
 }
 {
   nameComponents = QualifiedName() ifNotExists = IfNotExists()
   <WITH> withRecord = RecordConstructor()
   {
     try {
       stmt = new CreateFeedStatement(nameComponents, withRecord, ifNotExists);
       return addSourceLocation(stmt, startStmtToken);
     } catch (AlgebricksException e) {
       throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage());
     }
   }
 }

 CreateFeedPolicyStatement CreateFeedPolicyStatement(Token startStmtToken) throws ParseException:
 {
   CreateFeedPolicyStatement stmt = null;
 }
 {
   <INGESTION> <POLICY> stmt = FeedPolicySpecification(startStmtToken)
   {
     return stmt;
   }
 }

 CreateFeedPolicyStatement FeedPolicySpecification(Token startStmtToken) throws ParseException:
 {
   String policyName = null;
   String basePolicyName = null;
   String sourcePolicyFile = null;
   String definition = null;
   boolean ifNotExists = false;
   Map<String,String> properties = null;
   CreateFeedPolicyStatement stmt = null;
 }
 {
   policyName = Identifier() ifNotExists = IfNotExists()
   <FROM>
   (<POLICY> basePolicyName = Identifier() properties = Configuration() (<DEFINITION> definition = ConstantString())?
     {
       stmt = new CreateFeedPolicyStatement(policyName, basePolicyName, properties, definition, ifNotExists);
     }
   | <PATH> sourcePolicyFile = ConstantString() (<DEFINITION> definition = ConstantString())?
     {
       stmt = new CreateFeedPolicyStatement(policyName, sourcePolicyFile, definition, ifNotExists);
     }
   )
   {
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 Statement CreateFullTextStatement(Token startStmtToken) throws ParseException:
 {
   Statement stmt = null;
 }
 {
   (
     <FULLTEXT>
     (
       <FILTER> stmt = CreateFullTextFilterSpec(startStmtToken)
       | (<IDENTIFIER> { expectToken(CONFIG); } stmt = CreateFullTextConfigSpec(startStmtToken))
     )
   )
   {
     return stmt;
   }
 }

 CreateFullTextFilterStatement CreateFullTextFilterSpec(Token startStmtToken) throws ParseException:
 {
   CreateFullTextFilterStatement stmt = null;
   Pair<DataverseName,Identifier> nameComponents = null;
   boolean ifNotExists = false;
   RecordConstructor expr = null;
 }
 {
   (
     nameComponents = QualifiedName() ifNotExists = IfNotExists()
     <AS>
     expr = RecordConstructor()
   )
   {
     try {
       stmt = new CreateFullTextFilterStatement(nameComponents.first, nameComponents.second.getValue(), ifNotExists, expr);
       return addSourceLocation(stmt, startStmtToken);
     } catch (CompilationException e) {
        throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage());
     }
   }
 }

 CreateFullTextConfigStatement CreateFullTextConfigSpec(Token startStmtToken) throws ParseException:
 {
   CreateFullTextConfigStatement stmt = null;
   Pair<DataverseName,Identifier> nameComponents = null;
   boolean ifNotExists = false;
   RecordConstructor expr = null;
 }
 {
   (
     nameComponents = QualifiedName() ifNotExists = IfNotExists()
     <AS>
     expr = RecordConstructor()
   )
   {
     try {
       stmt = new CreateFullTextConfigStatement(nameComponents.first, nameComponents.second.getValue(), ifNotExists, expr);
       return addSourceLocation(stmt, startStmtToken);
     } catch (CompilationException e) {
        throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage());
     }
   }
 }

 CreateSynonymStatement CreateSynonymStatement(Token startStmtToken) throws ParseException:
 {
   CreateSynonymStatement stmt = null;
 }
 {
   <SYNONYM> stmt = SynonymSpecification(startStmtToken)
   {
     return stmt;
   }
 }

 CreateSynonymStatement SynonymSpecification(Token startStmtToken) throws ParseException:
 {
   Pair<DataverseName,Identifier> nameComponents = null;
   Pair<DataverseName,Identifier> objectNameComponents = null;
   boolean ifNotExists = false;
 }
 {
   nameComponents = QualifiedName() ifNotExists = IfNotExists()
   <FOR> objectNameComponents = QualifiedName()
   {
     CreateSynonymStatement stmt = new CreateSynonymStatement(nameComponents.first, nameComponents.second.getValue(),
       objectNameComponents.first, objectNameComponents.second.getValue(), ifNotExists);
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 boolean IfNotExists() throws ParseException:
 {
 }
 {
   ( LOOKAHEAD(1) <IF> <NOT> <EXISTS>
     {
       return true;
     }
   )?
     {
       return false;
     }
 }

 void ApplyFunction(List<FunctionSignature> funcSigs) throws ParseException:
 {
   FunctionName functionName = null;
   String fqFunctionName = null;
 }
 {
   <APPLY> <FUNCTION> functionName = FunctionName()
   {
      fqFunctionName = functionName.library == null ? functionName.function : functionName.library + "#" + functionName.function;
      funcSigs.add(new FunctionSignature(functionName.dataverse, fqFunctionName, 1));
   }
   (
       <COMMA> functionName = FunctionName()
       {
         fqFunctionName = functionName.library == null ? functionName.function : functionName.library + "#" + functionName.function;
         funcSigs.add(new FunctionSignature(functionName.dataverse, fqFunctionName, 1));
       }
   )*
 }

 String GetPolicy() throws ParseException:
 {
   String policy = null;
 }
 {
    <USING> <POLICY> policy = Identifier()
    {
      return policy;
    }

 }

 FunctionSignature FunctionSignature() throws ParseException:
 {
   FunctionName fctName = null;
   Pair<Integer, List<Pair<VarIdentifier,TypeExpression>>> params = null;
   int arity = 0;
 }
 {
   fctName = FunctionName()
   (
     LOOKAHEAD(2) params = FunctionParameters() { arity = params.first; }
   | ( <LEFTPAREN> arity = FunctionArity() <RIGHTPAREN> )
   | ( <ATT> arity = FunctionArity() ) // back-compat
   )
   {
     return new FunctionSignature(fctName.dataverse, fctName.function, arity);
   }
 }

 int FunctionArity() throws ParseException:
 {
   int arity;
 }
 {
   <INTEGER_LITERAL>
   {
     try {
       arity = Integer.parseInt(token.image);
     } catch (NumberFormatException e) {
       throw new SqlppParseException(getSourceLocation(token), "Invalid arity: " + token.image);
     }
     if (arity < 0 && arity != FunctionIdentifier.VARARGS) {
       throw new SqlppParseException(getSourceLocation(token), "Invalid arity: " + arity);
     }
     return arity;
   }
 }

 Triple<List<Integer>, List<List<String>>, List<TypeExpression>> PrimaryKeyWithType() throws ParseException:
 {
   Triple<List<Integer>, List<List<String>>, List<TypeExpression>> primaryKeyFieldsWithTypes = null;
 }
 {
   <PRIMARY> <KEY> <LEFTPAREN> primaryKeyFieldsWithTypes = PrimaryKeyFieldsWithType() <RIGHTPAREN>
   {
     return primaryKeyFieldsWithTypes;
   }
 }

 Triple<List<Integer>, List<List<String>>, List<TypeExpression>> PrimaryKeyFieldsWithType() throws ParseException:
 {
   Pair<Integer, List<String>> tmp = null;
   List<Integer> keyFieldSourceIndicators = new ArrayList<Integer>();
   List<List<String>> primaryKeyFields = new ArrayList<List<String>>();
   List<TypeExpression> primaryKeyFieldTypes = new ArrayList<TypeExpression>();
   TypeExpression type = null;
 }
 {
   tmp = NestedField() <COLON> type = TypeReference()
     {
       keyFieldSourceIndicators.add(tmp.first);
       primaryKeyFields.add(tmp.second);
       primaryKeyFieldTypes.add(type);
     }
   ( <COMMA> tmp = NestedField() <COLON> type = TypeReference()
     {
       keyFieldSourceIndicators.add(tmp.first);
       primaryKeyFields.add(tmp.second);
       primaryKeyFieldTypes.add(type);
     }
   )*
     {
       return new Triple<List<Integer>, List<List<String>>, List<TypeExpression>> (keyFieldSourceIndicators,
         primaryKeyFields, primaryKeyFieldTypes);
     }
 }

 Pair<List<Integer>, List<List<String>>> PrimaryKey() throws ParseException:
 {
   Pair<List<Integer>, List<List<String>>> primaryKeyFields = null;
 }
 {
   <PRIMARY> <KEY> primaryKeyFields = PrimaryKeyFields()
   {
     return primaryKeyFields;
   }
 }

 Pair<List<Integer>, List<List<String>>> PrimaryKeyFields() throws ParseException:
 {
   Pair<Integer, List<String>> tmp = null;
   List<Integer> keyFieldSourceIndicators = new ArrayList<Integer>();
   List<List<String>> primaryKeyFields = new ArrayList<List<String>>();
 }
 {
   tmp = NestedField()
     {
       keyFieldSourceIndicators.add(tmp.first);
       primaryKeyFields.add(tmp.second);
     }
   ( <COMMA> tmp = NestedField()
     {
       keyFieldSourceIndicators.add(tmp.first);
       primaryKeyFields.add(tmp.second);
     }
   )*
     {
       return new Pair<List<Integer>, List<List<String>>> (keyFieldSourceIndicators, primaryKeyFields);
     }
 }

 Statement DropStatement() throws ParseException:
 {
   Token startToken = null;
   Statement stmt = null;
 }
 {
   <DROP> { startToken = token; }
   (
     stmt = DropDatasetStatement(startToken)
     | stmt = DropIndexStatement(startToken)
     | stmt = DropNodeGroupStatement(startToken)
     | stmt = DropTypeStatement(startToken)
     | stmt = DropDatabaseStatement(startToken)
     | stmt = DropDataverseStatement(startToken)
     | stmt = DropAdapterStatement(startToken)
     | stmt = DropFunctionStatement(startToken)
     | stmt = DropFeedStatement(startToken)
     | stmt = DropFeedPolicyStatement(startToken)
     | stmt = DropSynonymStatement(startToken)
     | stmt = DropFullTextStatement(startToken)
     | stmt = DropViewStatement(startToken)
   )
   {
     return stmt;
   }
 }

 DropDatasetStatement DropDatasetStatement(Token startStmtToken) throws ParseException:
 {
   DropDatasetStatement stmt = null;
 }
 {
   Dataset() stmt = DropDatasetSpecification(startStmtToken)
   {
     return stmt;
   }
 }

 DropDatasetStatement DropDatasetSpecification(Token startStmtToken) throws ParseException:
 {
   Pair<DataverseName,Identifier> pairId = null;
   boolean ifExists = false;
 }
 {
   pairId = QualifiedName() ifExists = IfExists()
   {
     DropDatasetStatement stmt = new DropDatasetStatement(pairId.first, pairId.second, ifExists);
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 ViewDropStatement DropViewStatement(Token startStmtToken) throws ParseException:
 {
   ViewDropStatement stmt = null;
 }
 {
   <VIEW> stmt = DropViewSpecification(startStmtToken)
   {
     return stmt;
   }
 }

 ViewDropStatement DropViewSpecification(Token startStmtToken) throws ParseException:
 {
   Pair<DataverseName,Identifier> pairId = null;
   boolean ifExists = false;
 }
 {
   pairId = QualifiedName() ifExists = IfExists()
   {
     ViewDropStatement stmt = new ViewDropStatement(pairId.first, pairId.second, ifExists);
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 IndexDropStatement DropIndexStatement(Token startStmtToken) throws ParseException:
 {
   IndexDropStatement stmt = null;
 }
 {
   <INDEX> stmt = DropIndexSpecification(startStmtToken)
   {
     return stmt;
   }
 }

 IndexDropStatement DropIndexSpecification(Token startStmtToken) throws ParseException:
 {
   Triple<DataverseName,Identifier,Identifier> tripleId = null;
   boolean ifExists = false;
 }
 {
   tripleId = DoubleQualifiedName() ifExists = IfExists()
   {
     IndexDropStatement stmt = new IndexDropStatement(tripleId.first, tripleId.second, tripleId.third, ifExists);
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 Statement DropFullTextStatement(Token startStmtToken) throws ParseException:
 {
   Statement stmt = null;
 }
 {
   <FULLTEXT>
   (
     <FILTER> stmt = DropFullTextFilterSpec(startStmtToken)
      | (<IDENTIFIER> { expectToken(CONFIG); } stmt = DropFullTextConfigSpec(startStmtToken))
   )
   {
     return stmt;
   }
 }

 FullTextFilterDropStatement DropFullTextFilterSpec(Token startStmtToken) throws ParseException:
 {
   FullTextFilterDropStatement stmt = null;
   Pair<DataverseName,Identifier> nameComponents = null;
   boolean ifExists = false;
 }
 {
   nameComponents = QualifiedName() ifExists = IfExists()
   {
     stmt = new FullTextFilterDropStatement(nameComponents.first, nameComponents.second.getValue(), ifExists);
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 FullTextConfigDropStatement DropFullTextConfigSpec(Token startStmtToken) throws ParseException:
 {
   FullTextConfigDropStatement stmt = null;
   Pair<DataverseName,Identifier> nameComponents = null;
   boolean ifExists = false;
 }
 {
   nameComponents = QualifiedName() ifExists = IfExists()
   {
     stmt = new FullTextConfigDropStatement(nameComponents.first, nameComponents.second.getValue(), ifExists);
     return addSourceLocation(stmt, startStmtToken);
   }
 }


 NodeGroupDropStatement DropNodeGroupStatement(Token startStmtToken) throws ParseException:
 {
   NodeGroupDropStatement stmt = null;
 }
 {
   <NODEGROUP> stmt = DropNodeGroupSpecification(startStmtToken)
   {
     return stmt;
   }
 }

 NodeGroupDropStatement DropNodeGroupSpecification(Token startStmtToken) throws ParseException:
 {
   String id = null;
   boolean ifExists = false;
 }
 {
   id = Identifier() ifExists = IfExists()
   {
     NodeGroupDropStatement stmt = new NodeGroupDropStatement(new Identifier(id), ifExists);
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 TypeDropStatement DropTypeStatement(Token startStmtToken) throws ParseException:
 {
   TypeDropStatement stmt = null;
 }
 {
   <TYPE> stmt = DropTypeSpecification(startStmtToken)
   {
     return stmt;
   }
 }

 TypeDropStatement DropTypeSpecification(Token startStmtToken) throws ParseException:
 {
   Pair<DataverseName,Identifier> pairId = null;
   boolean ifExists = false;
 }
 {
   pairId = TypeName() ifExists = IfExists()
   {
     TypeDropStatement stmt = new TypeDropStatement(pairId.first, pairId.second, ifExists);
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 DatabaseDropStatement DropDatabaseStatement(Token startStmtToken) throws ParseException:
 {
   DatabaseDropStatement stmt = null;
 }
 {
   <DATABASE> stmt = DropDatabaseSpecification(startStmtToken)
   {
     return stmt;
   }
 }

 DatabaseDropStatement DropDatabaseSpecification(Token startStmtToken) throws ParseException:
 {
   String dbName = null;
   boolean ifExists = false;
 }
 {
   dbName = Identifier() ifExists = IfExists()
   {
     DatabaseDropStatement stmt = new DatabaseDropStatement(new Identifier(dbName), ifExists);
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 DataverseDropStatement DropDataverseStatement(Token startStmtToken) throws ParseException:
 {
   DataverseDropStatement stmt = null;
 }
 {
   <DATAVERSE> stmt = DropDataverseSpecification(startStmtToken)
   {
     return stmt;
   }
 }

 DataverseDropStatement DropDataverseSpecification(Token startStmtToken) throws ParseException:
 {
   DataverseName dvName = null;
   boolean ifExists = false;
 }
 {
   dvName = DataverseName() ifExists = IfExists()
   {
     DataverseDropStatement stmt = new DataverseDropStatement(dvName, ifExists);
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 AdapterDropStatement DropAdapterStatement(Token startStmtToken) throws ParseException:
 {
   AdapterDropStatement stmt = null;
 }
 {
   <ADAPTER> stmt = DropAdapterSpecification(startStmtToken)
   {
     return stmt;
   }
 }

 AdapterDropStatement DropAdapterSpecification(Token startStmtToken) throws ParseException:
 {
   Pair<DataverseName,Identifier> adapterName = null;
   boolean ifExists = false;
 }
 {
   adapterName = QualifiedName() ifExists = IfExists()
   {
     AdapterDropStatement stmt = new AdapterDropStatement(adapterName.first, adapterName.second.getValue(), ifExists);
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 FunctionDropStatement DropFunctionStatement(Token startStmtToken) throws ParseException:
 {
   FunctionDropStatement stmt = null;
 }
 {
   <FUNCTION> stmt = DropFunctionSpecification(startStmtToken)
   {
     return stmt;
   }
 }

 FunctionDropStatement DropFunctionSpecification(Token startStmtToken) throws ParseException:
 {
   FunctionSignature funcSig = null;
   boolean ifExists = false;
 }
 {
   funcSig = FunctionSignature() ifExists = IfExists()
   {
     FunctionDropStatement stmt = new FunctionDropStatement(funcSig, ifExists);
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 FeedDropStatement DropFeedStatement(Token startStmtToken) throws ParseException:
 {
   FeedDropStatement stmt = null;
 }
 {
   <FEED> stmt = DropFeedSpecification(startStmtToken)
   {
     return stmt;
   }
 }

 FeedDropStatement DropFeedSpecification(Token startStmtToken) throws ParseException:
 {
   Pair<DataverseName,Identifier> pairId = null;
   boolean ifExists = false;
 }
 {
   pairId = QualifiedName() ifExists = IfExists()
   {
     FeedDropStatement stmt = new FeedDropStatement(pairId.first, pairId.second, ifExists);
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 FeedPolicyDropStatement DropFeedPolicyStatement(Token startStmtToken) throws ParseException:
 {
   FeedPolicyDropStatement stmt = null;
 }
 {
   <INGESTION> <POLICY> stmt = DropFeedPolicySpecification(startStmtToken)
   {
     return stmt;
   }
 }

 FeedPolicyDropStatement DropFeedPolicySpecification(Token startStmtToken) throws ParseException:
 {
   Pair<DataverseName,Identifier> pairId = null;
   boolean ifExists = false;
 }
 {
   pairId = QualifiedName() ifExists = IfExists()
   {
     FeedPolicyDropStatement stmt = new FeedPolicyDropStatement(pairId.first, pairId.second, ifExists);
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 SynonymDropStatement DropSynonymStatement(Token startStmtToken) throws ParseException:
 {
   SynonymDropStatement stmt = null;
 }
 {
   <SYNONYM> stmt = DropSynonymSpecification(startStmtToken)
   {
     return stmt;
   }
 }

 SynonymDropStatement DropSynonymSpecification(Token startStmtToken) throws ParseException:
 {
   Pair<DataverseName,Identifier> pairId = null;
   boolean ifExists = false;
 }
 {
   pairId = QualifiedName() ifExists = IfExists()
   {
     SynonymDropStatement stmt = new SynonymDropStatement(pairId.first, pairId.second.getValue(), ifExists);
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 boolean IfExists() throws ParseException :
 {
 }
 {
   ( LOOKAHEAD(1) <IF> <EXISTS>
     {
       return true;
     }
   )?
     {
       return false;
     }
 }

 InsertStatement InsertStatement() throws ParseException:
 {
   Token startToken = null;
   Pair<DataverseName,Identifier> nameComponents = null;
   VariableExpr var = null;
   Query query = null;
   Expression returnExpression = null;
 }
 {
   <INSERT> { startToken = token; } <INTO> nameComponents = QualifiedName() (<AS> var = Variable())?
     query = Query()
     ( <RETURNING> returnExpression = Expression())?
     {
       if (var == null && returnExpression != null) {
         var = new VariableExpr(SqlppVariableUtil.toInternalVariableIdentifier(nameComponents.second.getValue()));
         addSourceLocation(var, startToken);
       }
       query.setTopLevel(true);
       InsertStatement stmt = new InsertStatement(nameComponents.first, nameComponents.second.getValue(), query,
         getVarCounter(), var, returnExpression);
       return addSourceLocation(stmt, startToken);
     }
 }

 UpsertStatement UpsertStatement() throws ParseException:
 {
   Token startToken = null;
   Pair<DataverseName,Identifier> nameComponents = null;
   VariableExpr var = null;
   Query query = null;
   Expression returnExpression = null;
 }
 {
   <UPSERT> { startToken = token; } <INTO> nameComponents = QualifiedName() (<AS> var = Variable())?
     query = Query()
     ( <RETURNING> returnExpression = Expression())?
     {
       if (var == null && returnExpression != null) {
           var = new VariableExpr(SqlppVariableUtil.toInternalVariableIdentifier(nameComponents.second.getValue()));
           addSourceLocation(var, startToken);
       }
       query.setTopLevel(true);
       UpsertStatement stmt = new UpsertStatement(nameComponents.first, nameComponents.second.getValue(), query,
         getVarCounter(), var, returnExpression);
       return addSourceLocation(stmt, startToken);
     }
 }

 DeleteStatement DeleteStatement() throws ParseException:
 {
   Token startToken = null;
   VariableExpr var = null;
   Expression condition = null;
   Pair<DataverseName, Identifier> nameComponents;
 }
 {
   <DELETE> { startToken = token; }
   <FROM> nameComponents  = QualifiedName() ((<AS>)? var = Variable())?
   (<WHERE> condition = Expression())?
   {
       if (var == null) {
         var = new VariableExpr(SqlppVariableUtil.toInternalVariableIdentifier(nameComponents.second.getValue()));
         addSourceLocation(var, startToken);
       }
       DeleteStatement stmt = new DeleteStatement(var, nameComponents.first, nameComponents.second.getValue(),
           condition, getVarCounter());
       return addSourceLocation(stmt, startToken);
   }
 }

 UpdateStatement UpdateStatement() throws ParseException:
 {
   Token startToken = null;
   VariableExpr vars;
   Expression target;
   Expression condition;
   UpdateClause uc;
   List<UpdateClause> ucs = new ArrayList<UpdateClause>();
 }
 {
   <UPDATE> { startToken = token; } vars = Variable() <IN> target = Expression()
   <WHERE> condition = Expression()
   <LEFTPAREN> (uc = UpdateClause()
     {
       ucs.add(uc);
     }
   (<COMMA> uc = UpdateClause()
     {
       ucs.add(uc);
     }
   )*) <RIGHTPAREN>
     {
       UpdateStatement stmt = new UpdateStatement(vars, target, condition, ucs);
       return addSourceLocation(stmt, startToken);
     }
 }

 UpdateClause UpdateClause() throws ParseException:
 {
   Expression target = null;
   Expression value = null ;
   InsertStatement is = null;
   DeleteStatement ds = null;
   UpdateStatement us = null;
   Expression condition = null;
   UpdateClause ifbranch = null;
   UpdateClause elsebranch = null;
 }
 {
    (<SET> target = Expression() <EQ> value = Expression()
    | is = InsertStatement()
    | ds = DeleteStatement()
    | us = UpdateStatement()
    | <IF> <LEFTPAREN> condition = Expression() <RIGHTPAREN>
      <THEN> ifbranch = UpdateClause()
      [LOOKAHEAD(1) <ELSE> elsebranch = UpdateClause()]
      {
        return new UpdateClause(target, value, is, ds, us, condition, ifbranch, elsebranch);
      }
    )
 }

 Statement SetStatement() throws ParseException:
 {
   Token startToken = null;
   String pn = null;
   String pv = null;
 }
 {
   <SET> { startToken = token; } pn = Identifier() pv = ConstantString()
     {
       SetStatement stmt = new SetStatement(pn, pv);
       return addSourceLocation(stmt, startToken);
     }
 }

 CopyStatement CopyStatement() throws ParseException:
 {
   Token startToken = null;
   DataverseName dataverseName = null;
   Identifier datasetName = null;
   TypeExpression typeExpr = null;
   boolean alreadySorted = false;
   String adapterName;
   Map<String,String> properties;
   Pair<DataverseName,Identifier> nameComponents = null;
 }
 {
   <COPY> (<INTO>)? { startToken = token; } nameComponents = QualifiedName()
     {
       dataverseName = nameComponents.first;
       datasetName = nameComponents.second;
     }
   (<AS> typeExpr = DatasetTypeSpecification())?
   <USING> adapterName = AdapterName() properties = Configuration()
     {
        ExternalDetailsDecl edd = new ExternalDetailsDecl();
        edd.setAdapter(adapterName);
        edd.setProperties(properties);
        try {
          CopyStatement stmt = new CopyStatement(dataverseName, datasetName.getValue(), typeExpr, edd, null);
          return addSourceLocation(stmt, startToken);
        } catch (CompilationException e){
            throw new SqlppParseException(getSourceLocation(startToken), e.getMessage());
        }
     }
 }

 LoadStatement LoadStatement() throws ParseException:
 {
   Token startToken = null;
   DataverseName dataverseName = null;
   Identifier datasetName = null;
   boolean alreadySorted = false;
   String adapterName;
   Map<String,String> properties;
   Pair<DataverseName,Identifier> nameComponents = null;
 }
 {
   <LOAD> { startToken = token; } Dataset() nameComponents = QualifiedName()
     {
       dataverseName = nameComponents.first;
       datasetName = nameComponents.second;
     }
   <USING> adapterName = AdapterName() properties = Configuration()
   (<PRESORTED>
     {
       alreadySorted = true;
     }
   )?
     {
       LoadStatement stmt = new LoadStatement(dataverseName, datasetName.getValue(), adapterName, properties,
         alreadySorted);
       return addSourceLocation(stmt, startToken);
     }
 }

 String AdapterName() throws ParseException :
 {
   String adapterName = null;
 }
 {
   adapterName = Identifier()
     {
       return adapterName;
     }
 }

 Statement AnalyzeStatement() throws ParseException:
 {
   Token startToken = null;
   Statement stmt = null;
   Pair<DataverseName,Identifier> nameComponents = null;
 }
 {
   <ANALYZE> { startToken = token; } DatasetToken() nameComponents = QualifiedName()
   (
     stmt = AnalyzeDatasetDropStatement(startToken, nameComponents.first, nameComponents.second)
     | stmt = AnalyzeDatasetStatement(startToken, nameComponents.first, nameComponents.second)
   )
   {
     return stmt;
   }
 }

 Statement AnalyzeDatasetStatement(Token startToken, DataverseName dvName, Identifier identifier) throws ParseException:
 {
   RecordConstructor withRecord = null;
 }
 {
   ( <WITH> withRecord = RecordConstructor() )?
   {
     try {
     AnalyzeStatement stmt = new AnalyzeStatement(dvName, identifier.getValue(), withRecord);
     return addSourceLocation(stmt, startToken);
     } catch (CompilationException e) {
        throw new SqlppParseException(getSourceLocation(startToken), e.getMessage());
     }
   }
 }

 Statement AnalyzeDatasetDropStatement(Token startToken, DataverseName dvName, Identifier identifier) throws ParseException:
 {
 }
 {
   <DROP> <IDENTIFIER> { expectToken(STATISTICS); }
   {
     AnalyzeDropStatement stmt = new AnalyzeDropStatement(dvName, identifier.getValue());
     return addSourceLocation(stmt, startToken);
   }
 }

 Statement CompactStatement() throws ParseException:
 {
   Token startToken = null;
   Pair<DataverseName,Identifier> nameComponents = null;
 }
 {
   <COMPACT> { startToken = token; } Dataset() nameComponents = QualifiedName()
     {
       CompactStatement stmt = new CompactStatement(nameComponents.first, nameComponents.second);
       return addSourceLocation(stmt, startToken);
     }
 }

 Statement ConnectionStatement() throws ParseException:
 {
   Token startToken = null;
   Statement stmt = null;
 }
 {
   (
     <CONNECT> { startToken = token; } stmt = ConnectStatement(startToken)
   | <DISCONNECT> { startToken = token; } stmt = DisconnectStatement(startToken)
   | <START> { startToken = token; } stmt = StartStatement(startToken)
   | <STOP> { startToken = token; } stmt = StopStatement(startToken)
   )
   {
     return stmt;
   }
 }

 Statement StartStatement(Token startStmtToken) throws ParseException:
 {
   Pair<DataverseName,Identifier> feedNameComponents = null;
   AbstractStatement stmt = null;
 }
 {
   <FEED> feedNameComponents = QualifiedName()
   {
     stmt = new StartFeedStatement (feedNameComponents);
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 AbstractStatement StopStatement(Token startStmtToken) throws ParseException:
 {
   Pair<DataverseName,Identifier> feedNameComponents = null;
   AbstractStatement stmt = null;
 }
 {
   <FEED> feedNameComponents = QualifiedName()
   {
     stmt = new StopFeedStatement (feedNameComponents);
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 AbstractStatement DisconnectStatement(Token startStmtToken) throws ParseException:
 {
   Pair<DataverseName,Identifier> feedNameComponents = null;
   Pair<DataverseName,Identifier> datasetNameComponents = null;

   AbstractStatement stmt = null;
 }
 {
   (
     <FEED> feedNameComponents = QualifiedName() <FROM> Dataset() datasetNameComponents = QualifiedName()
       {
         stmt = new DisconnectFeedStatement(feedNameComponents, datasetNameComponents);
       }
   )
   {
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 AbstractStatement ConnectStatement(Token startStmtToken) throws ParseException:
 {
   Pair<DataverseName,Identifier> feedNameComponents = null;
   Pair<DataverseName,Identifier> datasetNameComponents = null;

   Map<String,String> configuration = null;
   List<FunctionSignature> appliedFunctions = new ArrayList<FunctionSignature>();
   AbstractStatement stmt = null;
   String policy = null;
   String whereClauseBody = null;
   WhereClause whereClause = null;
   Token beginPos = null;
   Token endPos = null;
 }
 {
   (
     <FEED> feedNameComponents = QualifiedName() <TO> Dataset() datasetNameComponents = QualifiedName()
     (ApplyFunction(appliedFunctions))?
     (policy = GetPolicy())?
     (
       <WHERE>
       {
         beginPos = token;
         whereClause = new WhereClause();
         Expression whereExpr;
       }
       whereExpr = Expression()
       {
         whereClause.setWhereExpr(whereExpr);
       }
     )?
     {
       if (whereClause != null) {
         endPos = token;
         whereClauseBody = extractFragment(beginPos.endLine, beginPos.endColumn, endPos.endLine, endPos.endColumn + 1);
       }
     }
       {
         stmt = new ConnectFeedStatement(feedNameComponents, datasetNameComponents, appliedFunctions,
          policy, whereClauseBody, getVarCounter());
       }
   )
   {
     return addSourceLocation(stmt, startStmtToken);
   }
 }

 Map<String,String> Configuration()  throws ParseException :
 {
     Map<String,String> configuration = new LinkedHashMap<String,String>();
     Pair<String, String> keyValuePair = null;
 }
 {
   <LEFTPAREN> ( keyValuePair = KeyValuePair()
     {
       configuration.put(keyValuePair.first, keyValuePair.second);
     }
   ( <COMMA> keyValuePair = KeyValuePair()
     {
       configuration.put(keyValuePair.first, keyValuePair.second);
     }
   )* )? <RIGHTPAREN>
     {
       return configuration;
     }
 }

 Pair<String, String> KeyValuePair() throws ParseException:
 {
   String key;
   String value;
 }
 {
   <LEFTPAREN> key = ConstantString()
   <EQ> ( value = ConstantString() | (<TRUE> | <FALSE>) {value = token.image.toLowerCase();} )
   <RIGHTPAREN>
     {
       return new Pair<String, String>(key, value);
     }
 }

 Map<String,String> Properties() throws ParseException:
 {
   Map<String,String> properties = new HashMap<String,String>();
   Pair<String, String> property;
 }
 {
   (LOOKAHEAD(1) <LEFTPAREN> property = Property()
     {
       properties.put(property.first, property.second);
     }
   ( <COMMA> property = Property()
     {
       properties.put(property.first, property.second);
     }
   )* <RIGHTPAREN> )?
     {
       return properties;
     }
 }

 Pair<String, String> Property() throws ParseException:
 {
   String key = null;
   String value = null;
 }
 {
   (key = Identifier() | key = StringLiteral())
   <EQ>
   ( value = ConstantString() | <INTEGER_LITERAL>
     {
       try {
         value = String.valueOf(Long.parseLong(token.image));
       } catch (NumberFormatException nfe) {
         throw new SqlppParseException(getSourceLocation(token), "inapproriate value: " + token.image);
       }
     }
   )
     {
       return new Pair<String, String>(key.toUpperCase(), value);
     }
 }

 IndexedTypeExpression IndexedTypeExpr(boolean allowQues) throws ParseException:
 {
   TypeExpression typeExpr = null;
   boolean isUnknownable = !allowQues;
 }
 {
   typeExpr = TypeExpr(false)
   ( <QUES>
     {
       if (!allowQues) {
         throw new SqlppParseException(getSourceLocation(token), "'?' quantifier is illegal for the type expression.");
       }
       isUnknownable = true;
     }
   )?
   {
     return new IndexedTypeExpression(typeExpr, isUnknownable);
   }
 }

 TypeExpression TypeExpr(boolean allowRecordTypeDef) throws ParseException:
 {
   TypeExpression typeExpr = null;
 }
 {
   (
     typeExpr = TypeReference()
     | typeExpr = OrderedListTypeDef(allowRecordTypeDef)
     | typeExpr = UnorderedListTypeDef(allowRecordTypeDef)
     | typeExpr = RecordTypeDef() {
       if (!allowRecordTypeDef) {
         throw new SqlppParseException(typeExpr.getSourceLocation(), "Unexpected record type declaration");
       }
     }
   )
   {
     return typeExpr;
   }
 }

 RecordTypeDefinition.RecordKind RecordTypeKind() throws ParseException:
 {
   RecordTypeDefinition.RecordKind recordKind = null;
 }
 {
   (
     <CLOSED> { recordKind = RecordTypeDefinition.RecordKind.CLOSED; }
     | <OPEN> { recordKind = RecordTypeDefinition.RecordKind.OPEN; }
   )
   {
     return recordKind;
   }
 }

 RecordTypeDefinition RecordTypeDef() throws ParseException:
 {
   Token startToken = null;
   RecordTypeDefinition recType = new RecordTypeDefinition();
   RecordTypeDefinition.RecordKind recordKind = RecordTypeDefinition.RecordKind.OPEN;
 }
 {
    ( recordKind = RecordTypeKind() )?
    <LEFTBRACE>
     {
       startToken = token;
       Token hintToken = fetchHint(token, SqlppHint.GEN_FIELDS_HINT);
       if (hintToken != null) {
         String hintParams = hintToken.hintParams;
         String[] splits = hintParams != null ? hintParams.split("\\s+") : null;
         if (splits == null || splits.length != 4) {
           throw new SqlppParseException(getSourceLocation(hintToken),
             "Expecting: /*+ gen-fields <type> <min> <max> <prefix>*/");
         }
         if (!splits[0].equals("int")) {
           throw new SqlppParseException(getSourceLocation(hintToken),
             "The only supported type for gen-fields is int.");
         }
         UndeclaredFieldsDataGen ufdg = new UndeclaredFieldsDataGen(UndeclaredFieldsDataGen.Type.INT,
           Integer.parseInt(splits[1]), Integer.parseInt(splits[2]), splits[3]);
           recType.setUndeclaredFieldsDataGen(ufdg);
       }
     }
     (
       RecordField(recType)
       ( <COMMA>  RecordField(recType) )*
     )?
    <RIGHTBRACE>
    {
       recType.setRecordKind(recordKind);
       return addSourceLocation(recType, startToken);
    }
 }

 void RecordField(RecordTypeDefinition recType) throws ParseException:
 {
   String fieldName;
   TypeExpression type = null;
   boolean nullable = false, missable = false;
 }
 {
   fieldName = Identifier()
     {
       Token hintToken = fetchHint(token, SqlppHint.VAL_FILE_HINT, SqlppHint.VAL_FILE_SAME_INDEX_HINT,
         SqlppHint.LIST_VAL_FILE_HINT, SqlppHint.LIST_HINT, SqlppHint.INTERVAL_HINT, SqlppHint.INSERT_RAND_INT_HINT,
         SqlppHint.DATE_BETWEEN_YEARS_HINT, SqlppHint.DATETIME_ADD_RAND_HOURS_HINT, SqlppHint.AUTO_HINT);
       IRecordFieldDataGen rfdg = hintToken != null ? parseFieldDataGen(hintToken) : null;
     }
   <COLON> type = TypeExpr(true) ( <QUES> { nullable = true; missable = true; } )?
     {
       recType.addField(fieldName, type, nullable, missable, rfdg);
     }
 }

 TypeReferenceExpression TypeReference() throws ParseException:
 {
   Pair<DataverseName,Identifier> id = null;
 }
 {
   id = QualifiedName()
   {
     if (id.first == null && id.second.getValue().equalsIgnoreCase(INT_TYPE_NAME)) {
       id.second = new Identifier(BuiltinType.AINT64.getTypeName());
     }

     TypeReferenceExpression typeRef = new TypeReferenceExpression(id);
     return addSourceLocation(typeRef, token);
   }
 }

 OrderedListTypeDefinition OrderedListTypeDef(boolean allowRecordTypeDef) throws ParseException:
 {
   Token startToken = null;
   TypeExpression type = null;
 }
 {
   <LEFTBRACKET> { startToken = token; }
     ( type = TypeExpr(allowRecordTypeDef) )
   <RIGHTBRACKET>
   {
     OrderedListTypeDefinition typeDef = new OrderedListTypeDefinition(type);
     return addSourceLocation(typeDef, startToken);
   }
 }

 UnorderedListTypeDefinition UnorderedListTypeDef(boolean allowRecordTypeDef) throws ParseException:
 {
   Token startToken = null;
   TypeExpression type = null;
 }
 {
   <LEFTDBLBRACE> { startToken = token; }
     ( type = TypeExpr(allowRecordTypeDef) )
   <RIGHTDBLBRACE>
   {
     UnorderedListTypeDefinition typeDef = new UnorderedListTypeDefinition(type);
     return addSourceLocation(typeDef, startToken);
   }
 }

 FunctionName FunctionName() throws ParseException:
 {
   Triple<List<String>, Token, Token> prefix = null;
   String suffix = null;
 }
 {
   // Note: there's a copy of this production in PrimaryExpr() (LOOKAHEAD for FunctionCallExpr())
   //       that copy must be kept in sync with this code
   prefix = MultipartIdentifierWithHints(
     SqlppHint.INDEXED_NESTED_LOOP_JOIN_HINT, SqlppHint.RANGE_HINT, SqlppHint.HASH_JOIN_HINT, SqlppHint.SKIP_SECONDARY_INDEX_SEARCH_HINT,
     SqlppHint.SPATIAL_JOIN_HINT, SqlppHint.USE_SECONDARY_INDEX_SEARCH_HINT
   )
   (<SHARP> suffix = Identifier())?
   {
     Token startToken = prefix.second;
     FunctionName result = new FunctionName();
     result.sourceLoc = getSourceLocation(startToken);
     result.hintToken = prefix.third;
     List<String> list = prefix.first;
     int ln = list.size();
     String last = list.get(ln - 1);
     if (suffix == null) {
       // prefix = (dv_part1.dv_part2...dv_partN.)?func_name
       // no library name
       result.function = last;
     } else {
       // prefix = (dv_part1.dv_part2...dv_partN.)?lib_name
       // suffix = func_name
       result.library = last;
       result.function = suffix;
     }
     if (ln > 1) {
       result.dataverse = createDataverseName(list, 0, ln - 1, startToken);
     } else {
       result.dataverse = defaultDataverse;
     }

     if (result.function.equalsIgnoreCase(INT_TYPE_NAME)) {
        result.function = BuiltinType.AINT64.getTypeName();
     }
     return result;
   }
 }

 Pair<DataverseName,Identifier> TypeName() throws ParseException:
 {
   Pair<DataverseName,Identifier> name = null;
 }
 {
   name = QualifiedName()
     {
       if (name.first == null) {
         name.first = defaultDataverse;
       }
       return name;
     }
 }

 String Identifier() throws ParseException:
 {
   String lit = null;
 }
 {
   (<IDENTIFIER>
     {
       return token.image;
     }
   | lit = QuotedString()
     {
       return lit;
     }
   )
 }

 List<String> ParenthesizedIdentifierList() throws ParseException:
 {
   List<String> list = new ArrayList<String>();
   String ident = null;
 }
 {
   <LEFTPAREN>
   ident = Identifier() { list.add(ident); }
   ( <COMMA> ident = Identifier() { list.add(ident); } )*
   <RIGHTPAREN>
   {
     return list;
   }
 }

 Pair<HashJoinExpressionAnnotation.BuildOrProbe, String> buildOrProbeParenthesizedIdentifier() throws ParseException:
 {
   String ident1 = null;
   String ident2 = null;
 }
 {
   ident1 = Identifier() <LEFTPAREN> ident2 = Identifier() <RIGHTPAREN>
   {
     // check
     if (ident1.equals("build")) {
         return new Pair<HashJoinExpressionAnnotation.BuildOrProbe, String>(HashJoinExpressionAnnotation.BuildOrProbe.BUILD, ident2);
     }
     else if (ident1.equals("probe")) {
             return new Pair<HashJoinExpressionAnnotation.BuildOrProbe, String>(HashJoinExpressionAnnotation.BuildOrProbe.PROBE, ident2);
     }
     else {
         throw new SqlppParseException(getSourceLocation(token), "The string after hashjoin has to be \"build\" or \"probe\".");
     }
     return null;
   }
 }

 String parenthesizedIdentifier() throws ParseException:
 {
   String ident = null;
 }
 {
   <LEFTPAREN> ident = Identifier() <RIGHTPAREN>
   {
     return ident;
   }
 }

 List<Literal> ParenthesizedLiteralList() throws ParseException:
 {
   List<Literal> list = new ArrayList<Literal>();
   boolean minus = false;
   Expression litExpr = null;
   Literal lit = null;
 }
 {
   <LEFTPAREN>
   (<MINUS> { minus = true; })? litExpr = Literal()
   {
     lit = ((LiteralExpr) litExpr).getValue();
     if (minus)
     {
       try {
         lit = ExpressionUtils.reverseSign(lit);
       } catch (TypeMismatchException e) {
         throw new SqlppParseException(getSourceLocation(token), e.getMessage());
       }
       minus = false;
     }
     list.add(lit);
   }
   ( <COMMA> (<MINUS> { minus = true; })? litExpr = Literal()
   {
     lit = ((LiteralExpr) litExpr).getValue();
     if (minus)
     {
       try {
         lit = ExpressionUtils.reverseSign(lit);
       } catch (TypeMismatchException e) {
         throw new SqlppParseException(getSourceLocation(token), e.getMessage());
       }
       minus = false;
     }
     list.add(lit);
   } )*
   <RIGHTPAREN>
   {
     return list;
   }
 }

 Pair<Integer, Pair<List<String>, IndexedTypeExpression>> OpenField() throws ParseException:
 {
   IndexedTypeExpression fieldType = null;
   Pair<Integer, List<String>> fieldList = null;
 }
 {
   fieldList = NestedField()
   ( <COLON> fieldType = IndexedTypeExpr(true) )?
   {
     return new Pair<Integer, Pair<List<String>, IndexedTypeExpression>>
             (fieldList.first, new Pair<List<String>, IndexedTypeExpression>(fieldList.second, fieldType));
   }
 }

 Pair<Integer, List<String>> NestedField() throws ParseException:
 {
   List<String> exprList = new ArrayList<String>();
   String lit = null;
   Token litToken = null;
   int source = 0;
 }
 {
   lit = Identifier()
   {
     boolean meetParens = false;
     litToken = token;
   }
   (
     LOOKAHEAD(1)
     <LEFTPAREN><RIGHTPAREN>
     {
         if(!lit.equalsIgnoreCase("meta")){
             throw new SqlppParseException(getSourceLocation(litToken), "The string before () has to be \"meta\".");
         }
         meetParens = true;
         source = 1;
     }
   )?
   {
     if(!meetParens){
         exprList.add(lit);
     }
   }
   (<DOT>
     lit = Identifier()
     {
       exprList.add(lit);
     }
   )*
   {
     return new Pair<Integer, List<String>>(source, exprList);
   }
 }

 String ConstantString() throws ParseException:
 {
   String value = null;
 }
 {
   (value = QuotedString() | value = StringLiteral())
   {
      return value;
   }
 }

 String QuotedString() throws ParseException:
 {
 }
 {
   <QUOTED_STRING>
     {
       return removeQuotesAndEscapes(token.image);
     }
 }

 String StringLiteral() throws ParseException:
 {
   StringBuilder str = null;
   char quote = 0;
   boolean ext = false;
   Token litToken = null;
   String lit = null;
 }
 {
   ( <STRING_LITERAL>
     {
       String quoted = token.image;
       char q = quoted.charAt(0);
       boolean e = q == 'E';
       if (e) {
         q = quoted.charAt(1);
         quoted = quoted.substring(1);
       }
       if (lit == null) {
         quote = q;
         ext = e;
       } else {
         boolean isAdjacent = litToken.endLine == token.beginLine && litToken.endColumn + 1 == token.beginColumn;
         if (!isAdjacent || ext || e || (q != quote)) {
           throw new SqlppParseException(getSourceLocation(token), "Invalid string literal");
         }
         if (str == null) {
           str = new StringBuilder();
         }
         str.append(lit);
         str.append(quote);
       }
       lit = removeQuotesAndEscapes(quoted);
       litToken = token;
     }
   )+
   {
     if (str == null) {
       return lit;
     } else {
       str.append(lit);
       return str.toString();
     }
   }
 }

 Triple<List<String>, Token, Token> MultipartIdentifier() throws ParseException:
 {
   Triple<List<String>, Token, Token> result = null;
 }
 {
   result = MultipartIdentifierWithHints(null)
   {
     return result;
   }
 }

 Triple<List<String>, Token, Token> MultipartIdentifierWithHints(SqlppHint... expectedHints)
   throws ParseException:
 {
   List<String> list = new ArrayList<String>();
   SourceLocation sourceLoc = null;
   Token startToken, hintToken = null;
   String item = null;
 }
 {
   item = Identifier()
   {
     list.add(item);
     startToken = token;
     if (expectedHints != null && expectedHints.length > 0) {
       hintToken = fetchHint(token, expectedHints);
     }
   }
   (<DOT> item = Identifier() { list.add(item); } )*
   {
     return new Triple<List<String>, Token, Token>(list, startToken, hintToken);
   }
 }

 DataverseName DataverseName() throws ParseException:
 {
   Triple<List<String>, Token, Token> ident = null;
 }
 {
   ident = MultipartIdentifier()
   {
     List<String> list = ident.first;
     Token startToken = ident.second;
     return createDataverseName(list, 0, list.size(), startToken);
   }
 }

 Pair<DataverseName,Identifier> QualifiedName() throws ParseException:
 {
   Triple<List<String>, Token, Token> ident = null;
 }
 {
   ident = MultipartIdentifier()
   {
     List<String> list = ident.first;
     Token startToken = ident.second;
     int len = list.size();
     DataverseName id1 = len > 1 ? createDataverseName(list, 0, len - 1, startToken) : null;
     Identifier id2 = new Identifier(list.get(len - 1));
     return new Pair<DataverseName,Identifier>(id1, id2);
   }
 }

 Triple<DataverseName, Identifier, Identifier> DoubleQualifiedName() throws ParseException:
 {
   List<String> list = new ArrayList<String>();
   String item = null;
   Token startToken = null;
 }
 {
   item = Identifier() { list.add(item); startToken = token; }
   (<DOT> item = Identifier() { list.add(item); } )+
   {
     int len = list.size();
     DataverseName id1 = len > 2 ? createDataverseName(list, 0, len - 2, startToken) : null;
     Identifier id2 = new Identifier(list.get(len - 2));
     Identifier id3 = new Identifier(list.get(len - 1));
     return new Triple<DataverseName,Identifier,Identifier>(id1, id2, id3);
   }
 }

 FunctionDecl FunctionDeclaration() throws ParseException:
 {
   Token startToken = null;
   String functionName;
   Pair<Integer, List<Pair<VarIdentifier,TypeExpression>>> paramsWithArity = null;
   Expression funcBody;
   createNewScope();
 }
 {
   <DECLARE> { startToken = token; } <FUNCTION>
     functionName = Identifier()
     paramsWithArity = FunctionParameters()
   <LEFTBRACE>
     funcBody = FunctionBody()
   <RIGHTBRACE>
   {
     int arity = paramsWithArity.first;
     List<Pair<VarIdentifier,TypeExpression>> paramList = paramsWithArity.second;
     FunctionSignature signature = new FunctionSignature(defaultDataverse, functionName, arity);
     getCurrentScope().addFunctionDescriptor(signature, false);
     ensureNoTypeDeclsInFunction(functionName, paramList, null, startToken);
     List<VarIdentifier> params = new ArrayList<VarIdentifier>(paramList.size());
     for (Pair<VarIdentifier,TypeExpression> p: paramList) {
         params.add(p.getFirst());
     }
     FunctionDecl stmt = new FunctionDecl(signature, params, funcBody, false);
     removeCurrentScope();
     return addSourceLocation(stmt, startToken);
   }
 }

 Query Query() throws ParseException:
 {
   Query query = new Query();
   Expression expr;
 }
 {
   (
   expr = Expression()
   |
   expr = SelectExpression(false)
   )
   {
     query.setBody(expr);
     query.setSourceLocation(expr.getSourceLocation());
     return query;
   }
 }


 Expression Expression():
 {
   Expression expr = null;
   Expression exprP = null;
 }
 {
 (
     LOOKAHEAD(2)
     expr = OperatorExpr()
     | expr = QuantifiedExpression()
 )
     {
       return (exprP==null) ? expr : exprP;
     }
 }

 Expression OperatorExpr() throws ParseException:
 {
   OperatorExpr op = null;
   Expression operand = null;
 }
 {
     operand = AndExpr()
     (
       <OR>
       {
         if (op == null) {
           op = new OperatorExpr();
           op.addOperand(operand);
           op.setCurrentop(true);
           addSourceLocation(op, token);
         }
         try{
             op.addOperator(token.image.toLowerCase());
         } catch (Exception e){
             throw new SqlppParseException(getSourceLocation(token), e.getMessage());
         }
       }

       operand = AndExpr()
       {
         op.addOperand(operand);
       }

     )*

     {
       return op==null? operand: op;
     }
 }

 Expression AndExpr() throws ParseException:
 {
   OperatorExpr op = null;
   Expression operand = null;
 }
 {
     operand = NotExpr()
     (
       <AND>
       {
         if (op == null) {
           op = new OperatorExpr();
           op.addOperand(operand);
           op.setCurrentop(true);
           addSourceLocation(op, token);
         }
         try{
            op.addOperator(token.image.toLowerCase());
         } catch (CompilationException e){
            throw new SqlppParseException(getSourceLocation(token), e.getMessage());
         }
     }

     operand = NotExpr()
     {
       op.addOperand(operand);
     }

     )*

     {
       return op==null ? operand: op;
     }
 }

 Expression NotExpr() throws ParseException:
 {
    Expression inputExpr;
    boolean not = false;
    Token startToken = null;
 }
 {
   (LOOKAHEAD(2) <NOT> { not = true; startToken = token; } )? inputExpr = RelExpr()
   {
     if(not) {
         FunctionSignature signature = new FunctionSignature(BuiltinFunctions.NOT);
         CallExpr callExpr = new CallExpr(signature, new ArrayList<Expression>(Collections.singletonList(inputExpr)));
         return addSourceLocation(callExpr, startToken);
     } else {
         return inputExpr;
     }
   }
 }

 Expression RelExpr() throws ParseException:
 {
   boolean not = false;
   OperatorExpr op = null;
   Token opToken = null;
   Expression operand = null;
   IExpressionAnnotation annotation = null;
   List<IExpressionAnnotation> annotationList = new ArrayList<IExpressionAnnotation>();
 }
 {
     operand = BetweenExpr()

     (
       LOOKAHEAD(3)( <LT> | <GT> | <LE> | <GE> | <EQ> | <NE> | <LG> |<SIMILAR> | (<NOT> { not = true; })? <IN> |
                     <IS> (<NOT> { not = true; })? <DISTINCT> { opToken = token; } <FROM> )
         {
           if (opToken == null) {
             opToken = token;
           }
           Token hintToken = fetchHint(token,
             SqlppHint.HASH_BROADCAST_JOIN_HINT, SqlppHint.INDEXED_NESTED_LOOP_JOIN_HINT,
             SqlppHint.HASH_JOIN_HINT, SqlppHint.SKIP_SECONDARY_INDEX_SEARCH_HINT, SqlppHint.USE_SECONDARY_INDEX_SEARCH_HINT,
             SqlppHint.SINGLE_DATASET_PREDICATE_SELECTIVITY_HINT, SqlppHint.JOIN_PREDICATE_PRODUCTIVITY_HINT
           );
           while (hintToken != null) {
             annotation = parseExpressionAnnotation(hintToken);
             if (annotation != null) {
                 // annotation may be null if hints are malformed
                 annotationList.add(annotation);
             }
             hintToken = hintToken.specialToken;
             if (hintToken != null) {
                 SourceLocation sourceLoc = getSourceLocation(hintToken);
                 hintCollector.remove(sourceLoc);
             }
           }
           String operator = opToken.image.toLowerCase();
           if (operator.equals("<>")){
               operator = "!=";
           }
           if (not) {
             operator = "not_" + operator;
           }
           if (op == null) {
             op = new OperatorExpr();
             op.addOperand(operand);
             op.setCurrentop(true);
             addSourceLocation(op, token);
           }
           try{
             op.addOperator(operator);
           } catch (CompilationException e){
             throw new SqlppParseException(getSourceLocation(token), e.getMessage());
           }
         }

        operand = BetweenExpr()
        {
          op.addOperand(operand);
        }
     )?

      {
        if (annotation != null) {
          op.addHints(annotationList);
        }
        return op==null? operand: op;
      }
 }

 Expression BetweenExpr() throws ParseException:
 {
   boolean not = false;
   OperatorExpr op = null;
   Expression operand = null;
   IExpressionAnnotation annotation = null;
 }
 {
     operand = IsExpr()
     (
       LOOKAHEAD(2)
       (<NOT> { not = true; })? <BETWEEN>
         {
           Token hintToken = fetchHint(token,
             SqlppHint.INDEXED_NESTED_LOOP_JOIN_HINT, SqlppHint.SKIP_SECONDARY_INDEX_SEARCH_HINT, SqlppHint.HASH_JOIN_HINT,
             SqlppHint.USE_SECONDARY_INDEX_SEARCH_HINT, SqlppHint.SINGLE_DATASET_PREDICATE_SELECTIVITY_HINT
           );
           if (hintToken != null) {
             annotation = parseExpressionAnnotation(hintToken);
           }
           String operator = token.image.toLowerCase();
           if(not){
             operator = "not_" + operator;
           }
           if (op == null) {
             op = new OperatorExpr();
             op.addOperand(operand);
             op.setCurrentop(true);
             addSourceLocation(op, token);
           }
           try{
             op.addOperator(operator);
           } catch (CompilationException e){
             throw new SqlppParseException(getSourceLocation(token), e.getMessage());
           }
         }

        operand = IsExpr()
        {
          op.addOperand(operand);
        }

        <AND>
        operand = IsExpr()
        {
          op.addOperator(OperatorType.AND);
          op.addOperand(operand);
        }
     )?

     {
        if (annotation != null) {
          op.addHint(annotation);
        }
        return op==null ? operand: op;
     }
 }

 Expression IsExpr() throws ParseException:
 {
     Token notToken = null;
     CallExpr expr = null;
     Expression operand = null;
     boolean not = false;
     FunctionIdentifier fn = null;
 }
 {
     operand = LikeExpr()
     (
       LOOKAHEAD(3)
       <IS>
         (<NOT> { not = true; notToken = token; })?
         (
             <NULL> { fn = BuiltinFunctions.IS_NULL; } |
             <MISSING> { fn = BuiltinFunctions.IS_MISSING; } |
             <UNKNOWN> { fn = BuiltinFunctions.IS_UNKNOWN; } |
             (<KNOWN> | <VALUED>) { not = !not; fn = BuiltinFunctions.IS_UNKNOWN; }
         )
       {
         FunctionSignature signature = new FunctionSignature(fn);
         expr = new CallExpr(signature, new ArrayList<Expression>(Collections.singletonList(operand)));
         addSourceLocation(expr, token);
         if (not) {
            FunctionSignature notSignature = new FunctionSignature(BuiltinFunctions.NOT);
            expr = new CallExpr(notSignature, new ArrayList<Expression>(Collections.singletonList(expr)));
            addSourceLocation(expr, notToken);
         }
       }
     )?
     {
         return expr == null ? operand : expr;
     }
 }

 Expression LikeExpr() throws ParseException:
 {
   boolean not = false;
   OperatorExpr op = null;
   Expression operand = null;
   IExpressionAnnotation annotation = null;
 }
 {
     operand = ConcatExpr()
     (
         LOOKAHEAD(2)
         (<NOT> { not = true; })? <LIKE>
         {
           Token hintToken = fetchHint(token, SqlppHint.SINGLE_DATASET_PREDICATE_SELECTIVITY_HINT);
           if (hintToken != null) {
                annotation = parseExpressionAnnotation(hintToken);
           }
           op = new OperatorExpr();
           op.addOperand(operand);
           op.setCurrentop(true);
           addSourceLocation(op, token);

           String operator = token.image.toLowerCase();
           if (not) {
             operator = "not_" + operator;
           }
           try {
             op.addOperator(operator);
           } catch (CompilationException e){
             throw new SqlppParseException(getSourceLocation(token), e.getMessage());
           }
           if (annotation != null) {
              op.addHint(annotation);
           }
         }

         operand = ConcatExpr()
         {
           op.addOperand(operand);
         }
      )?

      {
        return op == null ? operand : op;
      }
 }

 Expression ConcatExpr() throws ParseException:
 {
   OperatorExpr op = null;
   Expression operand = null;
 }
 {
     operand = AddExpr()
     (
       LOOKAHEAD(1)
       <CONCAT>
       {
         if (op == null) {
           op = new OperatorExpr();
           op.addOperand(operand);
           op.setCurrentop(true);
           addSourceLocation(op, token);
         }
         op.addOperator(OperatorType.CONCAT);
     }
     operand = AddExpr()
     {
       op.addOperand(operand);
     }
     )*

     {
        return op == null ? operand : op;
     }
 }

 Expression AddExpr() throws ParseException:
 {
   OperatorExpr op = null;
   OperatorType opType = null;
   Expression operand = null;
 }
 {
     operand = MultExpr()
     (
       LOOKAHEAD(1)
       (<PLUS> { opType = OperatorType.PLUS; } | <MINUS> { opType = OperatorType.MINUS; } )
       {
         if (op == null) {
           op = new OperatorExpr();
           op.addOperand(operand);
           op.setCurrentop(true);
           addSourceLocation(op, token);
         }
         op.addOperator(opType);
     }

     operand = MultExpr()
     {
       op.addOperand(operand);
     }
     )*

     {
        return op == null ? operand : op;
     }
 }

 Expression MultExpr() throws ParseException:
 {
   OperatorExpr op = null;
   OperatorType opType = null;
   Expression operand = null;
 }
 {
     operand = ExponentExpr()

     ( (
         <MUL> { opType = OperatorType.MUL; } |
         <DIVIDE> { opType = OperatorType.DIVIDE; } |
         <DIV> { opType = OperatorType.DIV; } |
         ( <MOD> | <PERCENT> ) { opType = OperatorType.MOD; }
       )
       {
         if (op == null) {
           op = new OperatorExpr();
           op.addOperand(operand);
           op.setCurrentop(true);
           addSourceLocation(op, token);
         }
         op.addOperator(opType);
     }
     operand = ExponentExpr()
     {
        op.addOperand(operand);
     }
     )*

      {
        return op == null ? operand : op;
      }
 }

 Expression ExponentExpr() throws ParseException:
 {
   OperatorExpr op = null;
   Expression operand = null;
 }
 {
     operand = UnaryExpr()
     (<CARET>
     {
         if (op == null) {
           op = new OperatorExpr();
           op.addOperand(operand);
           op.setCurrentop(true);
           addSourceLocation(op, token);
         }
         op.addOperator(OperatorType.CARET);
     }
     operand = UnaryExpr()
     {
        op.addOperand(operand);
     }
     )?
     {
        return op == null ? operand : op;
     }
 }

 Expression UnaryExpr() throws ParseException:
 {
     boolean not = false;
     UnaryExpr uexpr = null;
     Expression expr = null;
 }
 {
     ( (<PLUS> | <MINUS> | (<NOT> { not = true; } )? <EXISTS> )
     {
         String exprType = token.image.toLowerCase();
         if (not) {
            exprType = "not_" + exprType;
         }
         uexpr = new UnaryExpr();
         addSourceLocation(uexpr, token);
         try {
             uexpr.setExprType(exprType);
         } catch (CompilationException e){
             throw new SqlppParseException(getSourceLocation(token), e.getMessage());
         }
     }
     )?

     expr = ValueExpr()
     {
        if (uexpr == null) {
          return expr;
        } else {
          uexpr.setExpr(expr);
          return uexpr;
        }
     }
 }

 Expression ValueExpr() throws ParseException:
 {
   Expression expr = null;
   AbstractAccessor accessor = null;
 }
 {
   expr = PrimaryExpr()
   (
     accessor = FieldAccessor(accessor != null ? accessor : expr)
     |
     accessor = IndexAccessor(accessor != null ? accessor : expr)
   )*
   {
     return accessor == null ? expr : accessor;
   }
 }

 FieldAccessor FieldAccessor(Expression inputExpr) throws ParseException:
 {
   Token startToken = null;
   String ident = null;
 }
 {
    <DOT> { startToken = token; } ident = Identifier()
     {
       FieldAccessor fa = new FieldAccessor(inputExpr, new Identifier(ident));
       return addSourceLocation(fa, startToken);
     }
 }

 AbstractAccessor IndexAccessor(Expression inputExpr) throws ParseException:
 {
   Token startToken = null;
   boolean star = false, slice = false;
   Expression expr1 = null, expr2 = null;
 }
 {
   <LEFTBRACKET> { startToken = token; }
   (
     <MUL> { star = true; }
     |
     ( expr1 = Expression() ( <COLON> { slice = true; } ( expr2 = Expression() )? )? )
   )
   <RIGHTBRACKET>
   {
     if (expr1 != null && expr1.getKind() == Expression.Kind.LITERAL_EXPRESSION) {
       ensureIntegerLiteral( (LiteralExpr) expr1, "Index");
     }
     if (expr2 != null && expr2.getKind() == Expression.Kind.LITERAL_EXPRESSION) {
       ensureIntegerLiteral( (LiteralExpr) expr2, "Index");
     }
     AbstractAccessor resultAccessor;
     if (slice) {
       resultAccessor = new ListSliceExpression(inputExpr, expr1, expr2);
     } else if (star) {
       resultAccessor = new IndexAccessor(inputExpr, IndexAccessor.IndexKind.STAR, null);
     } else {
       resultAccessor = new IndexAccessor(inputExpr, IndexAccessor.IndexKind.ELEMENT, expr1);
     }
     return addSourceLocation(resultAccessor, startToken);
   }
 }

 Expression PrimaryExpr() throws ParseException:
 {
   Expression expr = null;
 }
 {
   (
     LOOKAHEAD(Identifier() (<DOT> Identifier())* (<SHARP> Identifier())? <LEFTPAREN>) expr = FunctionCallExpr()
   | expr = CaseExpr()
   | expr = Literal()
   | expr = VariableRef()
   | expr = ExternalVariableRef()
   | expr = ListConstructor()
   | expr = RecordConstructor()
   | expr = ParenthesizedExpression()
   )
     {
       return expr;
     }
 }

 Expression Literal() throws ParseException:
 {
   LiteralExpr lit = new LiteralExpr();
   String str = null;
 }
 {
   ( str = StringLiteral()
     {
       lit.setValue(new StringLiteral(str));
     }
   | <INTEGER_LITERAL>
     {
         try {
             lit.setValue(new LongIntegerLiteral(Long.valueOf(token.image)));
         } catch (NumberFormatException e) {
             throw new SqlppParseException(getSourceLocation(token), "Could not parse numeric literal \"" + LogRedactionUtil.userData(token.image) +'"');
         }
     }
   | <FLOAT_LITERAL>
     {
         try {
             lit.setValue(new FloatLiteral(Float.valueOf(token.image)));
         } catch (NumberFormatException e) {
             throw new SqlppParseException(getSourceLocation(token), "Could not parse numeric literal \"" + LogRedactionUtil.userData(token.image) +'"');
         }
     }
   | <DOUBLE_LITERAL>
     {
         try {
             lit.setValue(new DoubleLiteral(Double.valueOf(token.image)));
         } catch (NumberFormatException e) {
             throw new SqlppParseException(getSourceLocation(token), "Could not parse numeric literal \"" + LogRedactionUtil.userData(token.image) +'"');
         }
     }
   | <MISSING>
     {
       lit.setValue(MissingLiteral.INSTANCE);
     }
   | <NULL>
     {
       lit.setValue(NullLiteral.INSTANCE);
     }
   | <TRUE>
     {
       lit.setValue(TrueLiteral.INSTANCE);
     }
   | <FALSE>
     {
       lit.setValue(FalseLiteral.INSTANCE);
     }
   )
     {
       return addSourceLocation(lit, token);
     }
 }

 VariableExpr VariableRef() throws ParseException:
 {
     String id = null;
 }
 {
     id = VariableIdentifier()
     {
      Identifier ident = lookupSymbol(id);
      if (isInForbiddenScopes(id)) {
        throw new SqlppParseException(getSourceLocation(token),
         "Inside limit clauses, it is disallowed to reference a variable having the same name as any variable bound in the same scope as the limit clause.");
      }
      VariableExpr varExp;
      if (ident != null) { // exist such ident
        varExp = new VariableExpr((VarIdentifier)ident);
      } else {
        varExp = new VariableExpr(new VarIdentifier(id));
        varExp.setIsNewVar(false);
      }
      return addSourceLocation(varExp, token);
     }
 }

 VariableExpr Variable() throws ParseException:
 {
     String id = null;
 }
 {
     id = VariableIdentifier()
     {
      Identifier ident = lookupSymbol(id);
      VariableExpr varExp = new VariableExpr(new VarIdentifier(id));
      if (ident != null) { // exist such ident
        varExp.setIsNewVar(false);
      }
      return addSourceLocation(varExp, token);
     }
 }

 String VariableIdentifier() throws ParseException:
 {
   String id = null;
 }
 {
   (<IDENTIFIER> { id = token.image; } | id = QuotedString())
   {
     return SqlppVariableUtil.toInternalVariableName(id); // prefix user-defined variables with "$".
   }
 }

 Pair<VariableExpr, List<Pair<Expression, Identifier>>> VariableWithFieldMap() throws ParseException:
 {
   VariableExpr var = null;
   List<Pair<Expression, Identifier>> fieldList = new ArrayList<Pair<Expression, Identifier>>();
 }
 {
   var = Variable()
   ( LOOKAHEAD(1)
     {
       VariableExpr fieldVarExpr = null;
       String fieldIdentifierStr = null;
     }
     <LEFTPAREN>
     fieldVarExpr = VariableRef() <AS> fieldIdentifierStr = Identifier()
     {
       fieldList.add(new Pair<Expression, Identifier>(fieldVarExpr, new Identifier(fieldIdentifierStr)));
     }
     (<COMMA>
       fieldVarExpr = VariableRef() <AS> fieldIdentifierStr = Identifier()
       {
         fieldList.add(new Pair<Expression, Identifier>(fieldVarExpr, new Identifier(fieldIdentifierStr)));
       }
     )*
     <RIGHTPAREN>
   )?
   {
     return new Pair<VariableExpr, List<Pair<Expression, Identifier>>>(var, fieldList);
   }
 }

 VariableExpr ExternalVariableRef() throws ParseException:
 {
   String name = null;
 }
 {
   (
       <DOLLAR_IDENTIFIER> { name = token.image.substring(1); }
     | <DOLLAR_INTEGER_LITERAL> { name = token.image.substring(1); }
     | <DOLLAR_QUOTED_STRING> { name = removeQuotesAndEscapes(token.image.substring(1)); }
     | <QUES> { name = String.valueOf(++externalVarCounter); }
   )
   {
      String idName = SqlppVariableUtil.toExternalVariableName(name);
      VarIdentifier id = new VarIdentifier(idName);
      VariableExpr varExp = new VariableExpr(id);
      return addSourceLocation(varExp, token);
   }
 }

 Expression ListConstructor() throws ParseException:
 {
     Expression expr = null;
 }
 {
     (
       expr = OrderedListConstructor() |
       expr = UnorderedListConstructor()
     )
     {
       return expr;
     }
 }

 ListConstructor OrderedListConstructor() throws ParseException:
 {
   Token startToken = null;
   List<Expression> exprList = null;
 }
 {
   <LEFTBRACKET> { startToken = token; }
   exprList = ExpressionList()
   <RIGHTBRACKET>
   {
     ListConstructor expr = new ListConstructor(ListConstructor.Type.ORDERED_LIST_CONSTRUCTOR, exprList);
     return addSourceLocation(expr, startToken);
   }
 }

 ListConstructor UnorderedListConstructor() throws ParseException:
 {
   Token startToken = null;
   List<Expression> exprList = null;
 }
 {
   <LEFTDBLBRACE> { startToken = token; }
   exprList = ExpressionList()
   <RIGHTDBLBRACE>
   {
     ListConstructor expr = new ListConstructor(ListConstructor.Type.UNORDERED_LIST_CONSTRUCTOR, exprList);
     return addSourceLocation(expr, startToken);
   }
 }

 List<Expression> ExpressionList() throws ParseException:
 {
   Expression expr = null;
   List<Expression> exprList = new ArrayList<Expression>();
 }
 {
   (
     expr = Expression()
     {
       exprList.add(expr);
     }
     ( <COMMA> expr = Expression()
       {
         exprList.add(expr);
       }
     )*
   )?
   {
     return exprList;
   }
 }

 RecordConstructor RecordConstructor() throws ParseException:
 {
   Token startToken = null;
   FieldBinding fb = null;
   List<FieldBinding> fbList = new ArrayList<FieldBinding>();
 }
 {
   <LEFTBRACE> { startToken = token; }
   (
     fb = FieldBinding() { fbList.add(fb); }
     (<COMMA> fb = FieldBinding() { fbList.add(fb); })*
   )?
   <RIGHTBRACE>
   {
     RecordConstructor expr = new RecordConstructor(fbList);
     return addSourceLocation(expr, startToken);
   }
 }

 FieldBinding FieldBinding() throws ParseException:
 {
   Expression left, right = null;
 }
 {
   left = Expression() ( <COLON> right = Expression() )?
   {
     if (right == null) {
         String generatedIdentifier = ExpressionToVariableUtil.getGeneratedIdentifier(left, false);
         if (generatedIdentifier == null) {
           throw new SqlppParseException(getSourceLocation(token), "Cannot infer field name");
         }
         String generatedName = SqlppVariableUtil.toUserDefinedName(generatedIdentifier);
         LiteralExpr generatedNameExpr = new LiteralExpr(new StringLiteral(generatedName));
         generatedNameExpr.setSourceLocation(left.getSourceLocation());
         return new FieldBinding(generatedNameExpr, left);
     } else {
       return new FieldBinding(left, right);
     }
   }
 }

 Expression FunctionCallExpr() throws ParseException:
 {
   List<Expression> argList = new ArrayList<Expression>();
   Expression argExpr = null;
   FunctionName funcName = null;
   boolean star = false;
   boolean distinct = false;
   Expression filterExpr = null;
   WindowExpression windowExpr = null;
 }
 {
   funcName = FunctionName()
   <LEFTPAREN> (
     ( <DISTINCT> { distinct = true; } )?
     ( argExpr = Expression() | <MUL> { star = true; } )
     {
       if (star) {
         if (funcName.function.equalsIgnoreCase(BuiltinFunctions.SCALAR_COUNT.getName())) {
           argExpr = new LiteralExpr(new LongIntegerLiteral(1L));
         } else {
           throw new SqlppParseException(getSourceLocation(token),
             "The parameter * can only be used in " + BuiltinFunctions.SCALAR_COUNT.getName() + "().");
         }
       }
       argList.add(argExpr);
     }
   (<COMMA> argExpr = Expression()
     {
       argList.add(argExpr);
     }
   )*)? <RIGHTPAREN>

   {
     String name = funcName.function;
     if (distinct) {
       name += FunctionMapUtil.DISTINCT_AGGREGATE_SUFFIX;
     }
     String fqFunctionName = funcName.library == null ? name : funcName.library + "#" + name;
     int arity = argList.size();
     FunctionSignature signature = lookupFunctionSignature(funcName.dataverse, fqFunctionName, arity);
     if (signature == null) {
       signature = new FunctionSignature(funcName.dataverse, fqFunctionName, arity);
     }
   }

   ( <FILTER> <LEFTPAREN> <WHERE> filterExpr = Expression() <RIGHTPAREN> )?

   ( LOOKAHEAD(5) windowExpr = WindowExpr(signature, argList, filterExpr) )?

   {
     if (windowExpr != null) {
       return windowExpr;
     } else {
       CallExpr callExpr = new CallExpr(signature, argList, filterExpr);
       if (funcName.hintToken != null) {
         IExpressionAnnotation annotation = parseExpressionAnnotation(funcName.hintToken);
         if (annotation != null) {
           callExpr.addHint(annotation);
         }
       }
       FunctionMapUtil.normalizedListInputFunctions(callExpr);
       callExpr.setSourceLocation(funcName.sourceLoc);
       return callExpr;
     }
   }
 }

 WindowExpression WindowExpr(FunctionSignature signature, List<Expression> argList, Expression aggFilterExpr)
   throws ParseException:
 {
   WindowExpression windowExpr = null;
   Boolean fromLast = null, ignoreNulls = null;
 }
 {
   (
     // FROM ( FIRST | LAST ) ( ( RESPECT | IGNORE ) NULLS )? OVER
     LOOKAHEAD(5, <FROM> <IDENTIFIER> ( <IDENTIFIER> <IDENTIFIER> )? <OVER>)
     <FROM> <IDENTIFIER>
     {
       if (isToken(FIRST)) {
         fromLast = false;
       } else if (isToken(LAST)) {
         fromLast = true;
       } else {
         throw createUnexpectedTokenError();
       }
     }
   )?
   (
     // ( RESPECT | IGNORE ) NULLS OVER
     LOOKAHEAD(3, <IDENTIFIER> <IDENTIFIER> <OVER>)
     <IDENTIFIER>
     {
       if (isToken(RESPECT)) {
         ignoreNulls = false;
       } else if (isToken(IGNORE)) {
         ignoreNulls = true;
       } else {
         throw createUnexpectedTokenError();
       }
     }
     <IDENTIFIER>
     {
       if (!isToken(NULLS)) {
         throw createUnexpectedTokenError();
       }
     }
   )?
   <OVER> windowExpr = OverClause(signature, argList, aggFilterExpr, token, fromLast, ignoreNulls)
   {
     return windowExpr;
   }
 }

 WindowExpression OverClause(FunctionSignature signature, List<Expression> argList, Expression aggFilterExpr,
     Token startToken, Boolean fromLast, Boolean ignoreNulls) throws ParseException:
 {
   Expression partitionExpr = null;
   List<Expression> partitionExprs = new ArrayList<Expression>();
   OrderbyClause orderByClause = null;
   List<Expression> orderbyList = null;
   List<OrderbyClause.OrderModifier> orderbyModifierList = null;
   List<OrderbyClause.NullOrderModifier> orderbyNullModifierList = null;
   WindowExpression.FrameMode frameMode = null;
   Pair<WindowExpression.FrameBoundaryKind, Expression> frameStart = null, frameEnd = null;
   WindowExpression.FrameBoundaryKind frameStartKind = null, frameEndKind = null;
   Expression frameStartExpr = null, frameEndExpr = null;
   WindowExpression.FrameExclusionKind frameExclusionKind = null;
   Pair<VariableExpr, List<Pair<Expression, Identifier>>> windowVarWithFieldList = null;
   VariableExpr windowVar = null;
   List<Pair<Expression, Identifier>> windowFieldList = null;
 }
 {
   (
     windowVarWithFieldList = VariableWithFieldMap() <AS>
     {
       windowVar = windowVarWithFieldList.first;
       windowFieldList = windowVarWithFieldList.second;
     }
   )?
   <LEFTPAREN>
   (
     <IDENTIFIER> { expectToken(PARTITION); } <BY>
     partitionExpr = Expression() { partitionExprs.add(partitionExpr); }
     ( <COMMA> partitionExpr = Expression() { partitionExprs.add(partitionExpr); } )*
   )?
   (
     orderByClause = OrderbyClause()
     {
       orderbyList = orderByClause.getOrderbyList();
       orderbyModifierList = orderByClause.getModifierList();
       orderbyNullModifierList = orderByClause.getNullModifierList();
     }
     (
       frameMode = WindowFrameMode()
       (
         frameStart = WindowFrameBoundary() |
         ( <BETWEEN> frameStart = WindowFrameBoundary() <AND> frameEnd = WindowFrameBoundary() )
       )
       ( frameExclusionKind = WindowFrameExclusion() )?
       {
         frameStartKind = frameStart.first;
         frameStartExpr = frameStart.second;
         if (frameEnd == null) {
           frameEndKind = WindowExpression.FrameBoundaryKind.CURRENT_ROW;
         } else {
           frameEndKind = frameEnd.first;
           frameEndExpr = frameEnd.second;
         }
         if (frameExclusionKind == null) {
           frameExclusionKind = WindowExpression.FrameExclusionKind.NO_OTHERS;
         }
       }
     )?
   )?
   <RIGHTPAREN>
   {
     WindowExpression winExpr = new WindowExpression(signature, argList, aggFilterExpr, partitionExprs, orderbyList,
       orderbyModifierList, orderbyNullModifierList, frameMode, frameStartKind, frameStartExpr, frameEndKind,
       frameEndExpr, frameExclusionKind, windowVar, windowFieldList, ignoreNulls, fromLast);
     return addSourceLocation(winExpr, startToken);
   }
 }

 WindowExpression.FrameMode WindowFrameMode() throws ParseException:
 {
 }
 {
   <IDENTIFIER>
   {
     if (isToken(RANGE)) {
       return WindowExpression.FrameMode.RANGE;
     } else if (isToken(ROWS)) {
       return WindowExpression.FrameMode.ROWS;
     } else if (isToken(GROUPS)) {
       return WindowExpression.FrameMode.GROUPS;
     } else {
       throw createUnexpectedTokenError();
     }
   }
 }

 Pair<WindowExpression.FrameBoundaryKind, Expression> WindowFrameBoundary() throws ParseException:
 {
   boolean current = false;
   Expression expr = null;
 }
 {
   (
     LOOKAHEAD({ laIdentifier(CURRENT) || laIdentifier(UNBOUNDED) }) <IDENTIFIER> { current = isToken(CURRENT); }
     | expr = Expression()
   )
   <IDENTIFIER>
   {
     WindowExpression.FrameBoundaryKind kind;
     if (current && isToken(ROW)) {
       kind = WindowExpression.FrameBoundaryKind.CURRENT_ROW;
     } else if (!current && isToken(PRECEDING)) {
       kind = expr == null
         ? WindowExpression.FrameBoundaryKind.UNBOUNDED_PRECEDING
         : WindowExpression.FrameBoundaryKind.BOUNDED_PRECEDING;
     } else if (!current && isToken(FOLLOWING)) {
       kind = expr == null
         ? WindowExpression.FrameBoundaryKind.UNBOUNDED_FOLLOWING
         : WindowExpression.FrameBoundaryKind.BOUNDED_FOLLOWING;
     } else {
       throw createUnexpectedTokenError();
     }
     return new Pair<WindowExpression.FrameBoundaryKind, Expression>(kind, expr);
   }
 }

 WindowExpression.FrameExclusionKind WindowFrameExclusion() throws ParseException:
 {
   boolean current = false, no = false;
 }
 {
   <IDENTIFIER>
   {
     expectToken(EXCLUDE);
   }
   (
     <GROUP>
     {
       return WindowExpression.FrameExclusionKind.GROUP;
     }
     |
     (
       <IDENTIFIER>
       {
         if (isToken(TIES)) {
           return WindowExpression.FrameExclusionKind.TIES;
         } else if (isToken(CURRENT)) {
           current = true;
         } else if (isToken(NO)) {
           no = true;
         } else {
           throw createUnexpectedTokenError();
         }
       }
       <IDENTIFIER>
       {
         if (current && isToken(ROW)) {
           return WindowExpression.FrameExclusionKind.CURRENT_ROW;
         } else if (no && isToken(OTHERS)) {
           return WindowExpression.FrameExclusionKind.NO_OTHERS;
         } else {
           throw createUnexpectedTokenError();
         }
       }
     )
   )
 }

 Expression ParenthesizedExpression() throws ParseException:
 {
   Token startToken = null;
   Expression expr = null, expr2 = null;
   List<Expression> exprList = null;
 }
 {
     (
     LOOKAHEAD(2)
     <LEFTPAREN> { startToken = token; } expr = Expression()
     (
       <COMMA> expr2 = Expression()
       {
         if (exprList == null) {
           exprList = new ArrayList<Expression>();
           exprList.add(expr);
         }
         exprList.add(expr2);
       }
     )*
     <RIGHTPAREN>
     |
     expr = Subquery()
     )
     {
       if (exprList != null) {
         ListConstructor listExpr = new ListConstructor(ListConstructor.Type.ORDERED_LIST_CONSTRUCTOR, exprList);
         return addSourceLocation(listExpr, startToken);
       } else {
         return expr;
       }
     }
 }

 Expression CaseExpr() throws ParseException:
 {
    Token startToken = null;
    Expression conditionExpr = null;
    List<Expression> whenExprs = new ArrayList<Expression>();
    List<Expression> thenExprs = new ArrayList<Expression>();
    Expression elseExpr = null;
    Expression whenExpr = null;
    Expression thenExpr = null;
 }
 {
    <CASE> { startToken = token; }
    ( conditionExpr = Expression() )?
    (
      <WHEN> whenExpr = Expression()
      {
         whenExprs.add(whenExpr);
      }
      <THEN> thenExpr = Expression()
      {
         thenExprs.add(thenExpr);
      }
    )*
    (<ELSE> elseExpr = Expression() )?
    <END>
    {
      if (conditionExpr == null) {
         LiteralExpr litExpr = new LiteralExpr(TrueLiteral.INSTANCE);
         conditionExpr = addSourceLocation(litExpr, startToken);
      }
      CaseExpression caseExpr = new CaseExpression(conditionExpr, whenExprs, thenExprs, elseExpr);
      return addSourceLocation(caseExpr, startToken);
    }
 }

 SelectExpression SelectExpression(boolean subquery) throws ParseException:
 {
   List<LetClause> letClauses = new ArrayList<LetClause>();
   SelectSetOperation selectSetOperation;
   OrderbyClause orderbyClause = null;
   LimitClause limitClause = null;
   createNewScope();
 }
 {
     ( letClauses = LetClause() )?
     selectSetOperation = SelectSetOperation()
     (orderbyClause = OrderbyClause() {})?
     (limitClause = LimitClause() {})?
     {
       SelectExpression selectExpr =
         new SelectExpression(letClauses, selectSetOperation, orderbyClause, limitClause, subquery);
       selectExpr.setSourceLocation((!letClauses.isEmpty() ? letClauses.get(0) : selectSetOperation).getSourceLocation());
       return selectExpr;
     }
 }

 SelectSetOperation SelectSetOperation() throws ParseException:
 {
   SetOperationInput setOperationInputLeft;
   List<SetOperationRight> setOperationRights = new ArrayList<SetOperationRight>();
 }
 {
   {
       SelectBlock selectBlockLeft = null;
       SelectExpression subqueryLeft = null;
       Expression expr = null;
   }
   selectBlockLeft = SelectBlock()
   {
      setOperationInputLeft = new SetOperationInput(selectBlockLeft, subqueryLeft);
   }
   (
     {
       SetOpType opType = SetOpType.UNION;
       boolean setSemantics = true;
       SelectBlock selectBlockRight = null;
       SelectExpression subqueryRight = null;
     }
     (<UNION> {opType = SetOpType.UNION;} |<INTERSECT> {opType = SetOpType.INTERSECT;} |<EXCEPT> {opType = SetOpType.EXCEPT;}) (<ALL> {setSemantics = false;} )?
     (selectBlockRight = SelectBlock()| subqueryRight = Subquery())
     {
         setOperationRights.add(new SetOperationRight(opType, setSemantics, new SetOperationInput(selectBlockRight, subqueryRight)));
     }
   )*
   {
     SelectSetOperation selectSetOp = new SelectSetOperation(setOperationInputLeft, setOperationRights);
     selectSetOp.setSourceLocation(selectBlockLeft.getSourceLocation());
     return selectSetOp;
   }
 }

 SelectExpression Subquery() throws ParseException:
 {
    SelectExpression selectExpr = null;
 }
 {
   <LEFTPAREN> selectExpr = SelectExpression(true) {} <RIGHTPAREN>
   {
     return selectExpr;
   }
 }

 SelectBlock SelectBlock() throws ParseException:
 {
   SelectClause selectClause = null;
   FromClause fromClause = null;
   List<LetClause> fromLetClauses = null;
   WhereClause whereClause = null;
   List<AbstractClause> fromLetWhereClauses = new ArrayList<AbstractClause>();
   GroupbyClause groupbyClause = null;
   List<LetClause> gbyLetClauses = null;
   HavingClause havingClause = null;
   List<AbstractClause> gbyLetHavingClauses = new ArrayList<AbstractClause>();
   SourceLocation startSrcLoc = null;
 }
 {
   (
     (
      selectClause = SelectClause() { startSrcLoc = selectClause.getSourceLocation(); }
      (
         (
           fromClause = FromClause()
           (
               fromLetClauses = LetClause()
           )?
           (whereClause = WhereClause())?
           (
             groupbyClause = GroupbyClause()
             (
                 gbyLetClauses = LetClause()
             )?
             (havingClause = HavingClause())?
           )?
         )
         |
         (
           fromLetClauses = LetClause()
           {
             // LET without FROM -> create dummy FROM clause: FROM {{missing}} AS #0
             SourceLocation sourceLoc = getSourceLocation(token);
             LiteralExpr missingExpr = new LiteralExpr(MissingLiteral.INSTANCE);
             missingExpr.setSourceLocation(sourceLoc);
             List<Expression> list = new ArrayList<Expression>(1);
             list.add(missingExpr);
             ListConstructor listExpr = new ListConstructor(ListConstructor.Type.ORDERED_LIST_CONSTRUCTOR, list);
             listExpr.setSourceLocation(sourceLoc);
             List<FromTerm> fromTerms = new ArrayList<FromTerm>(1);
             VariableExpr fromVar = new VariableExpr(new VarIdentifier("#0"));
             fromVar.setSourceLocation(sourceLoc);
             fromTerms.add(new FromTerm(listExpr, fromVar, null, new ArrayList<AbstractBinaryCorrelateClause>()));
             fromClause = new FromClause(fromTerms);
           }
           (whereClause = WhereClause())?
         )
      )?
     )
     |
     (
      fromClause = FromClause() { startSrcLoc = fromClause.getSourceLocation(); }
      (
         fromLetClauses = LetClause()
      )?
      (whereClause = WhereClause())?
      (
         groupbyClause = GroupbyClause()
         (
             gbyLetClauses = LetClause()
         )?
         (havingClause = HavingClause())?
      )?
      selectClause = SelectClause()
     )
   )
   {
     if (fromLetClauses != null) {
       fromLetWhereClauses.addAll(fromLetClauses);
     }
     if (whereClause != null) {
       fromLetWhereClauses.add(whereClause);
     }
     if (gbyLetClauses != null) {
       gbyLetHavingClauses.addAll(gbyLetClauses);
     }
     if (havingClause != null) {
       gbyLetHavingClauses.add(havingClause);
     }
     SelectBlock selectBlock = new SelectBlock(selectClause, fromClause, fromLetWhereClauses, groupbyClause,
       gbyLetHavingClauses);
     selectBlock.setSourceLocation(startSrcLoc);
     return selectBlock;
   }
 }

 SelectClause SelectClause() throws ParseException:
 {
   Token startToken = null;
   SelectRegular selectRegular = null;
   SelectElement selectElement = null;
   List<List<String>> fieldExclusions = new ArrayList<List<String>>();
   List<String> nestedField = new ArrayList<String>();
   String identifier;
   boolean distinct = false;
 }
 {
   <SELECT> { startToken = token; } (<ALL>|<DISTINCT> { distinct = true; } )?
   (
     (
       selectRegular = SelectRegular()
       ( LOOKAHEAD({laIdentifier(EXCLUDE)}) <IDENTIFIER>
         identifier = Identifier() { nestedField.add(identifier); }
         ( <DOT> identifier = Identifier() { nestedField.add(identifier); } )*
         {
           fieldExclusions.add(nestedField);
           nestedField = new ArrayList<String>();
         }
         ( LOOKAHEAD(1) // Force <COMMA> to be recognized for a nested field in our EXCLUDE list.
           <COMMA> identifier = Identifier() { nestedField.add(identifier); }
           ( <DOT> identifier = Identifier() { nestedField.add(identifier); } )*
           {
             fieldExclusions.add(nestedField);
             nestedField = new ArrayList<String>();
           }
         )*
       )?
     )
     |
     selectElement = SelectElement()
   )?
   {
     SourceLocation sourceLoc = getSourceLocation(startToken);
     if (selectRegular == null && selectElement == null){
         Projection projection = new Projection(Projection.Kind.STAR, null, null);
         projection.setSourceLocation(sourceLoc);
         List<Projection> projections = new ArrayList<Projection>();
         projections.add(projection);
         selectRegular = new SelectRegular(projections);
         selectRegular.setSourceLocation(sourceLoc);
     }
     SelectClause selectClause = new SelectClause(selectElement, selectRegular, fieldExclusions, distinct);
     selectClause.setSourceLocation(sourceLoc);
     return selectClause;
   }
 }

 SelectRegular SelectRegular() throws ParseException:
 {
   SourceLocation startSrcLoc = null;
   List<Projection> projections = new ArrayList<Projection>();
   Projection projection = null;
 }
 {
   projection = Projection()
   {
     projections.add(projection);
     startSrcLoc = projection.getSourceLocation();
   }
   ( LOOKAHEAD(2) <COMMA> projection = Projection()
     {
       projections.add(projection);
     }
   )*
   {
     SelectRegular selectRegular = new SelectRegular(projections);
     selectRegular.setSourceLocation(startSrcLoc);
     return selectRegular;
   }
 }

 SelectElement SelectElement() throws ParseException:
 {
   Token startToken = null;
   Expression expr = null;
   String name = null;
 }
 {
   (<RAW>|<ELEMENT>|<VALUE>) { startToken = token; } expr = Expression()
   {
     SelectElement selectElement = new SelectElement(expr);
     return addSourceLocation(selectElement, startToken);
   }
 }

 Projection Projection() throws ParseException :
 {
   SourceLocation startSrcLoc = null;
   Expression expr = null;
   Identifier identifier = null;
   String name = null;
   Projection.Kind kind = null;
   boolean star = false;
   boolean varStar = false;
 }
 {
   (
     <MUL> { kind = Projection.Kind.STAR; startSrcLoc = getSourceLocation(token); }
     | LOOKAHEAD(3) expr = VariableRef() <DOT> <MUL> { kind = Projection.Kind.VAR_STAR; }
     | expr = Expression()
       ( // EXCLUDE is a soft-keyword-- we want to avoid mistaking EXCLUDE as an identifier here.
         LOOKAHEAD({ getToken(1).kind == AS || getToken(1).kind == QUOTED_STRING
                     || (getToken(1).kind == IDENTIFIER && !laIdentifier(1, EXCLUDE)) })
         (<AS>)? name = Identifier()
       )?
       {
         kind = Projection.Kind.NAMED_EXPR;
         if (name == null) {
           String generatedColumnIdentifier = ExpressionToVariableUtil.getGeneratedIdentifier(expr, false);
           if (generatedColumnIdentifier != null) {
             name = SqlppVariableUtil.toUserDefinedName(generatedColumnIdentifier);
           }
         }
       }
   )
   {
     Projection projection = new Projection(kind, expr, name);
     projection.setSourceLocation(expr != null ? expr.getSourceLocation() : startSrcLoc);
     return projection;
   }
 }

 FromClause FromClause() throws ParseException :
 {
   Token startToken = null;
   List<FromTerm> fromTerms = new ArrayList<FromTerm>();
   extendCurrentScope();
 }
 {
   {
     FromTerm fromTerm = null;
   }
     <FROM> { startToken = token; } fromTerm = FromTerm() { fromTerms.add(fromTerm); }
     (LOOKAHEAD(2) <COMMA> fromTerm = FromTerm() { fromTerms.add(fromTerm); } )*
   {
     FromClause fromClause = new FromClause(fromTerms);
     return addSourceLocation(fromClause, startToken);
   }
 }

 FromTerm FromTerm() throws ParseException :
 {
   Expression leftExpr = null;
   VariableExpr leftVar = null;
   VariableExpr posVar = null;
   AbstractBinaryCorrelateClause correlateClause = null;
   List<AbstractBinaryCorrelateClause> correlateClauses = new ArrayList<AbstractBinaryCorrelateClause>();
 }
 {
   leftExpr = Expression()
   {
     if (leftExpr.getKind() == Expression.Kind.VARIABLE_EXPRESSION) {
       Token hintToken = fetchHint(token, SqlppHint.SUBPATH_HINT);
       if (hintToken != null) {
         String subPath = hintToken.hintParams;
         ((VariableExpr) leftExpr).addHint(new ExternalSubpathAnnotation(subPath));
       }
     }
   } ((<AS>)? leftVar = Variable())? (<AT> posVar = Variable())?
   (
      (
       correlateClause = JoinOrUnnestClause(JoinType.INNER, UnnestType.INNER)
       | ( <INNER> correlateClause = JoinOrUnnestClause(JoinType.INNER, UnnestType.INNER) )
       | ( <LEFT> ( <OUTER> )? correlateClause = JoinOrUnnestClause(JoinType.LEFTOUTER, UnnestType.LEFTOUTER) )
       | ( <RIGHT> ( <OUTER> )? correlateClause = JoinClause(JoinType.RIGHTOUTER) )
       | ( <CROSS> correlateClause = CrossJoinClause() )
      )
      {
         correlateClauses.add(correlateClause);
      }
   )*
   {
     if (leftVar == null) {
         leftVar = ExpressionToVariableUtil.getGeneratedVariable(leftExpr, true);
     }
     FromTerm fromTerm = new FromTerm(leftExpr, leftVar, posVar, correlateClauses);
     fromTerm.setSourceLocation(leftExpr.getSourceLocation());
     return fromTerm;
   }
 }

 AbstractBinaryCorrelateClause JoinOrUnnestClause(JoinType joinType, UnnestType unnestType) throws ParseException :
 {
   AbstractBinaryCorrelateClause correlateClause = null;
 }
 {
   ( correlateClause = JoinClause(joinType) | correlateClause = UnnestClause(unnestType) )
   {
     return correlateClause;
   }
 }

 JoinClause JoinClause(JoinType joinType) throws ParseException :
 {
     Token startToken = null;
     Triple<Expression, VariableExpr, VariableExpr> rightInput = null;
     Expression conditionExpr = null;
 }
 {
   <JOIN> { startToken = token; } rightInput = JoinClauseRightInput() <ON> conditionExpr = Expression()
   {
     JoinClause joinClause = new JoinClause(joinType, rightInput.first, rightInput.second, rightInput.third,
       conditionExpr, joinType == JoinType.INNER ? null : Literal.Type.MISSING);
     return addSourceLocation(joinClause, startToken);
   }
 }

 JoinClause CrossJoinClause() throws ParseException :
 {
     Token startToken = null;
     Triple<Expression, VariableExpr, VariableExpr> rightInput = null;
     Expression conditionExpr = null;
 }
 {
   <JOIN> { startToken = token; } rightInput = JoinClauseRightInput()
   {
     JoinClause joinClause = new JoinClause(JoinType.INNER, rightInput.first, rightInput.second, rightInput.third,
       new LiteralExpr(TrueLiteral.INSTANCE), null);
     return addSourceLocation(joinClause, startToken);
   }
 }

 Triple<Expression, VariableExpr, VariableExpr> JoinClauseRightInput() throws ParseException :
 {
     Expression rightExpr = null;
     VariableExpr rightVar = null;
     VariableExpr posVar = null;
 }
 {
   rightExpr = Expression()
   {
     if (rightExpr.getKind() == Expression.Kind.VARIABLE_EXPRESSION) {
         Token hintToken = fetchHint(token, SqlppHint.SUBPATH_HINT);
         if (hintToken != null) {
           String subPath = hintToken.hintParams;
           ((VariableExpr) rightExpr).addHint(new ExternalSubpathAnnotation(subPath));
         }
     }
   } ((<AS>)? rightVar = Variable())? (<AT> posVar = Variable())?
   {
     if (rightVar == null) {
       rightVar = ExpressionToVariableUtil.getGeneratedVariable(rightExpr, true);
     }
     return new Triple<Expression, VariableExpr, VariableExpr>(rightExpr, rightVar, posVar);
   }
 }

 UnnestClause UnnestClause(UnnestType unnestType) throws ParseException :
 {
     Token startToken = null;
     Expression rightExpr = null;
     VariableExpr rightVar = null;
     VariableExpr posVar = null;
 }
 {
   (<UNNEST>|<CORRELATE>|<FLATTEN>) { startToken = token; } rightExpr = Expression() ((<AS>)? rightVar = Variable())? (<AT> posVar = Variable())?
   {
     if (rightVar == null) {
       rightVar = ExpressionToVariableUtil.getGeneratedVariable(rightExpr, true);
     }
     UnnestClause unnestClause = new UnnestClause(unnestType, rightExpr, rightVar, posVar,
       unnestType == UnnestType.INNER ? null : Literal.Type.MISSING);
     return addSourceLocation(unnestClause, startToken);
   }
 }

 List<LetClause> LetClause() throws ParseException:
 {
     List<LetClause> letList = new ArrayList<LetClause>();
     LetClause letClause;
 }
 {
     (
      (<LET>|<LETTING>) letClause = LetElement() { letList.add(letClause); } (LOOKAHEAD(1) <COMMA> letClause = LetElement() { letList.add(letClause); })*
      |
      <WITH> letClause = WithElement() { letList.add(letClause); } (LOOKAHEAD(1) <COMMA> letClause = WithElement() { letList.add(letClause); })*
     )
     {
       return letList;
     }
 }

 WhereClause WhereClause() throws ParseException :
 {
   Token startToken = null;
   Expression whereExpr;
 }
 {
     <WHERE> { startToken = token; } whereExpr = Expression()
     {
       WhereClause wc = new WhereClause(whereExpr);
       return addSourceLocation(wc, startToken);
     }
 }

 OrderbyClause OrderbyClause() throws ParseException :
 {
     Token startToken = null;
     OrderbyClause oc = new OrderbyClause();
     Triple<Expression, OrderbyClause.OrderModifier, OrderbyClause.NullOrderModifier> orderbyExpr = null;
     List<Expression> orderbyList = new ArrayList<Expression>();
     List<OrderbyClause.OrderModifier> modifierList = new ArrayList<OrderbyClause.OrderModifier>();
     List<OrderbyClause.NullOrderModifier> nullModifierList = new ArrayList<OrderbyClause.NullOrderModifier>();
 }
 {
   <ORDER>
   {
     startToken = token;
     Token hintToken = fetchHint(token, SqlppHint.INMEMORY_HINT, SqlppHint.RANGE_HINT);
     if (hintToken != null) {
       switch (hintToken.hint) {
         case INMEMORY_HINT:
           String[] splits = hintToken.hintParams.split("\\s+");
           int numFrames = Integer.parseInt(splits[0]);
           int numTuples = Integer.parseInt(splits[1]);
           oc.setNumFrames(numFrames);
           oc.setNumTuples(numTuples);
           break;
         case RANGE_HINT:
           try {
             Expression rangeExpr = parseExpression(hintToken.hintParams);
             RangeMap rangeMap = RangeMapBuilder.parseHint(rangeExpr);
             oc.setRangeMap(rangeMap);
           } catch (CompilationException e) {
             throw new SqlppParseException(getSourceLocation(hintToken), e.getMessage());
           }
           break;
       }
     }
   }
   <BY> orderbyExpr = OrderByExpression()
   {
     orderbyList.add(orderbyExpr.first);
     modifierList.add(orderbyExpr.second);
     nullModifierList.add(orderbyExpr.third);
   }
   (
     LOOKAHEAD(2) <COMMA> orderbyExpr = OrderByExpression()
     {
       orderbyList.add(orderbyExpr.first);
       modifierList.add(orderbyExpr.second);
       nullModifierList.add(orderbyExpr.third);
     }
   )*
   {
     oc.setOrderbyList(orderbyList);
     oc.setModifierList(modifierList);
     oc.setNullModifierList(nullModifierList);
     return addSourceLocation(oc, startToken);
   }
 }

 Triple<Expression, OrderbyClause.OrderModifier, OrderbyClause.NullOrderModifier> OrderByExpression()
   throws ParseException:
 {
   Expression orderbyExpr = null;
   OrderbyClause.OrderModifier modif = OrderbyClause.OrderModifier.ASC;
   OrderbyClause.NullOrderModifier nullModif = null;
 }
 {
   orderbyExpr = Expression()
   (
     <ASC> { modif = OrderbyClause.OrderModifier.ASC; }
     |
     <DESC> { modif = OrderbyClause.OrderModifier.DESC; }
   )?
   (
     LOOKAHEAD({ laIdentifier(NULLS) }) <IDENTIFIER> { expectToken(NULLS); } <IDENTIFIER>
     {
       if (isToken(FIRST)) {
         nullModif = OrderbyClause.NullOrderModifier.FIRST;
       } else if (isToken(LAST)) {
         nullModif = OrderbyClause.NullOrderModifier.LAST;
       } else {
         throw createUnexpectedTokenError();
       }
     }
   )?
   {
     return new Triple<Expression, OrderbyClause.OrderModifier, OrderbyClause.NullOrderModifier>(orderbyExpr, modif,
       nullModif);
   }
 }

 GroupbyClause GroupbyClause()throws ParseException :
 {
     Token startToken = null;
     GroupbyClause gbc = new GroupbyClause();
     List<List<GbyVariableExpressionPair>> gbyList = null;
     List<GroupingElement> groupingElementList = null;
     Pair<VariableExpr, List<Pair<Expression, Identifier>>> groupVarWithFieldList = null;
     VariableExpr groupVar = null;
     List<Pair<Expression, Identifier>> groupFieldList = null;
 }
 {
     {
       Scope newScope = extendCurrentScopeNoPush(true);
       // extendCurrentScope(true);
     }
     <GROUP>
     {
       startToken = token;
       Token hintToken = fetchHint(token, SqlppHint.HASH_GROUP_BY_HINT);
       if (hintToken != null) {
         gbc.setHashGroupByHint(true);
       }
     }
     <BY> groupingElementList = GroupingElementList()
     (
       <GROUP> <AS> groupVarWithFieldList = VariableWithFieldMap()
       {
         groupVar = groupVarWithFieldList.first;
         groupFieldList = groupVarWithFieldList.second;
       }
     )?
     {
       if (groupingSetsParser == null) {
         groupingSetsParser = new SqlppGroupingSetsParser();
       }
       SourceLocation sourceLoc = getSourceLocation(startToken);
       try {
         gbyList = groupingSetsParser.parse(groupingElementList, sourceLoc);
       } catch (CompilationException e) {
         throw new SqlppParseException(sourceLoc, e.getMessage());
       }
       gbc.setGbyPairList(gbyList);
       gbc.setDecorPairList(new ArrayList<GbyVariableExpressionPair>());
       gbc.setWithVarMap(new HashMap<Expression, VariableExpr>());
       gbc.setGroupVar(groupVar);
       gbc.setGroupFieldList(groupFieldList);
       replaceCurrentScope(newScope);
       return addSourceLocation(gbc, startToken);
     }
 }

 List<GroupingElement> GroupingElementList() throws ParseException:
 {
   List<GroupingElement> groupingElementList = new ArrayList<GroupingElement>();
   GroupingElement groupingElement = null;
 }
 {
   groupingElement = GroupingElement() { groupingElementList.add(groupingElement); }
   ( LOOKAHEAD(1) <COMMA> groupingElement = GroupingElement() { groupingElementList.add(groupingElement); } )*
   {
     return groupingElementList;
   }
 }

 GroupingElement GroupingElement() throws ParseException:
 {
   GroupingElement groupingElement = null;
   List<GroupingSet> groupingSets = null;
   List<GroupingElement> groupingElements = null;
 }
 {
   (
     LOOKAHEAD(2)
     <LEFTPAREN> <RIGHTPAREN>
     {
       groupingElement = GroupingSet.EMPTY;
     }
     |
     LOOKAHEAD({ laIdentifier(ROLLUP) && laToken(2, LEFTPAREN) })
     <IDENTIFIER> { expectToken(ROLLUP); }
     <LEFTPAREN> groupingSets = OrdinaryGroupingSetList() <RIGHTPAREN>
     {
       groupingElement = new RollupCube(groupingSets, false);
     }
     |
     LOOKAHEAD({ laIdentifier(CUBE) && laToken(2, LEFTPAREN) })
     <IDENTIFIER> { expectToken(CUBE); }
     <LEFTPAREN> groupingSets = OrdinaryGroupingSetList() <RIGHTPAREN>
     {
       groupingElement = new RollupCube(groupingSets, true);
     }
     |
     LOOKAHEAD({ laIdentifier(GROUPING) && laIdentifier(2, SETS) && laToken(3, LEFTPAREN) })
     <IDENTIFIER> { expectToken(GROUPING); } <IDENTIFIER> { expectToken(SETS); }
     <LEFTPAREN> groupingElements = GroupingElementList() <RIGHTPAREN>
     {
       groupingElement = new GroupingSets(groupingElements);
     }
     |
     groupingElement = OrdinaryGroupingSet()
   )
   {
     return groupingElement;
   }
 }

 GroupingSet OrdinaryGroupingSet() throws ParseException:
 {
   GbyVariableExpressionPair gbyExprPair = null;
   List<GbyVariableExpressionPair> items = null;
 }
 {
   (
     LOOKAHEAD(1) <LEFTPAREN> items = GbyVariableExpressionPairList() <RIGHTPAREN>
     | gbyExprPair = GbyVariableExpressionPair() { items = Collections.singletonList(gbyExprPair); }
   )
   {
     return new GroupingSet(items);
   }
 }

 List<GroupingSet> OrdinaryGroupingSetList() throws ParseException:
 {
   GroupingSet groupingSet = null;
   List<GroupingSet> items = new ArrayList<GroupingSet>();
 }
 {
   groupingSet = OrdinaryGroupingSet() { items.add(groupingSet); }
   ( LOOKAHEAD(1) <COMMA> groupingSet = OrdinaryGroupingSet() { items.add(groupingSet); } )*
   {
     return items;
   }
 }

 List<GbyVariableExpressionPair> GbyVariableExpressionPairList() throws ParseException:
 {
   GbyVariableExpressionPair gbyExprPair = null;
   List<GbyVariableExpressionPair> items = new ArrayList<GbyVariableExpressionPair>();
 }
 {
   gbyExprPair = GbyVariableExpressionPair() { items.add(gbyExprPair); }
   ( LOOKAHEAD(1) <COMMA> gbyExprPair = GbyVariableExpressionPair() { items.add(gbyExprPair); } )*
   {
     return items;
   }
 }

 GbyVariableExpressionPair GbyVariableExpressionPair() throws ParseException:
 {
   Expression expr = null;
   VariableExpr var = null;
 }
 {
   expr = Expression() ( (<AS>)? var = Variable() )?
   {
     return new GbyVariableExpressionPair(var, expr);
   }
 }

 HavingClause HavingClause() throws ParseException:
 {
    Token startToken = null;
    Expression filterExpr = null;
 }
 {
     <HAVING> { startToken = token; } filterExpr = Expression()
     {
        HavingClause havingClause = new HavingClause(filterExpr);
        return addSourceLocation(havingClause, startToken);
     }
 }

 LimitClause LimitClause() throws ParseException:
 {
     Token startToken = null;
     Expression limitExpr = null, offsetExpr = null;
 }
 {
   (
     (
       <LIMIT> { startToken = token; pushForbiddenScope(getCurrentScope()); } limitExpr = Expression()
       ( <OFFSET> offsetExpr = Expression() )?
       { popForbiddenScope(); }
     )
     |
     (
       <OFFSET> { startToken = token; pushForbiddenScope(getCurrentScope()); } offsetExpr = Expression()
       { popForbiddenScope(); }
     )
   )
   {
     LimitClause lc = new LimitClause(limitExpr, offsetExpr);
     return addSourceLocation(lc, startToken);
   }
 }

 QuantifiedExpression QuantifiedExpression()throws ParseException:
 {
   Token startToken = null;
   QuantifiedExpression qc = new QuantifiedExpression();
   List<QuantifiedPair> quantifiedList = new ArrayList<QuantifiedPair>();
   Expression satisfiesExpr;
   VariableExpr var;
   Expression inExpr;
   QuantifiedPair pair;
 }
 {
   {
     createNewScope();
   }

      ( LOOKAHEAD(2)
        (<ANY>|<SOME>)<AND><EVERY> { startToken = token; qc.setQuantifier(QuantifiedExpression.Quantifier.SOME_AND_EVERY); }
        | (<ANY>|<SOME>) { startToken = token; qc.setQuantifier(QuantifiedExpression.Quantifier.SOME); }
        | <EVERY> {  startToken = token; qc.setQuantifier(QuantifiedExpression.Quantifier.EVERY); } )
     var = Variable() <IN> inExpr = Expression()
     {
       pair = new QuantifiedPair(var, inExpr);
       quantifiedList.add(pair);
     }
     (
     <COMMA> var = Variable() <IN> inExpr = Expression()
     {
       pair = new QuantifiedPair(var, inExpr);
       quantifiedList.add(pair);
     }
     )*
      <SATISFIES> satisfiesExpr = Expression() (<END>)?
      {
        qc.setSatisfiesExpr(satisfiesExpr);
        qc.setQuantifiedList(quantifiedList);
        removeCurrentScope();
        return addSourceLocation(qc, startToken);
      }
 }

 LetClause LetElement() throws ParseException:
 {
     VariableExpr varExp;
     Expression beExp;
     extendCurrentScope();
 }
 {
     varExp = Variable() <EQ> beExp = Expression()
     {
       LetClause lc = new LetClause(varExp, beExp);
       lc.setSourceLocation(varExp.getSourceLocation());
       return lc;
     }
 }

 LetClause WithElement() throws ParseException:
 {
     VariableExpr varExp;
     Expression beExp;
     extendCurrentScope();
 }
 {
     varExp = Variable() <AS> beExp = Expression()
     {
       LetClause lc = new LetClause(varExp, beExp);
       lc.setSourceLocation(varExp.getSourceLocation());
       return lc;
     }
 }

 TOKEN_MGR_DECLS:
 {
     public int commentDepth = 0;
     public ArrayDeque<Integer> lexerStateStack = new ArrayDeque<Integer>();
     public Map<SourceLocation, String> hintCollector;

     public void pushState() {
       lexerStateStack.push( curLexState );
     }

     public void popState(String token) {
       if (lexerStateStack.size() > 0) {
          SwitchTo( lexerStateStack.pop() );
       } else {
          int errorLine = input_stream.getEndLine();
          int errorColumn = input_stream.getEndColumn();
          String msg = "Lexical error at line " + errorLine + ", column " + errorColumn + ". Encountered \"" + token
              + "\" but state stack is empty.";
          throw new TokenMgrError(msg, -1);
       }
     }

     void CommonTokenAction(Token token) {
       Token hintToken = token.specialToken;
       while (hintToken != null) { // make this a while loop
         hintToken.sourceLocation = new SourceLocation(hintToken.beginLine, hintToken.beginColumn);
         String text = hintToken.image.substring(1).trim();
         boolean hintFound = hintToken.parseHint(text);
         hintCollector.put(hintToken.sourceLocation, hintFound ? hintToken.hint.getIdentifier() : hintToken.hintParams);
         hintToken = hintToken.specialToken;
       }
     }
 }

 <DEFAULT,IN_DBL_BRACE>
 TOKEN [IGNORE_CASE]:
 {
   <ADAPTER: "adapter">
   | <ALL : "all">
   | <ANALYZE: "analyze">
   | <AND : "and">
   | <ANY : "any">
   | <APPLY : "apply">
   | <AS : "as">
   | <ASC : "asc">
   | <AT : "at">
   | <AUTOGENERATED : "autogenerated">
   | <BETWEEN : "between">
   | <BTREE : "btree">
   | <BY : "by">
   | <CASE : "case">
   | <CAST : "cast">
   | <CLOSED : "closed">
   | <CREATE : "create">
   | <CROSS : "cross">
   | <COMPACTION : "compaction"> // no longer used
   | <COMPACT : "compact">
   | <CONNECT : "connect">
   | <CORRELATE : "correlate">
   | <DATASET : "dataset">
   | <COLLECTION : "collection">
   | <DATABASE : "database">
   | <DATAVERSE : "dataverse">
   | <DECLARE : "declare">
   | <DEFINITION : "definition">
   | <DELETE : "delete">
   | <DESC : "desc">
   | <DISCONNECT : "disconnect">
   | <DISTINCT : "distinct">
   | <DIV : "div">
   | <DROP : "drop">
   | <ELEMENT : "element">
   | <EXPLAIN : "explain">
   | <ELSE : "else">
   | <ENFORCED : "enforced">
   | <END : "end">
   | <EVERY : "every">
   | <EXCEPT : "except">
   | <EXISTS : "exists">
   | <EXTERNAL : "external">
   | <FALSE : "false">
   | <FEED : "feed">
   | <FILTER : "filter">
   | <FLATTEN : "flatten">
   | <FOR : "for">
   | <FROM : "from">
   | <FULL : "full">
   | <FULLTEXT : "fulltext">
   | <FUNCTION : "function">
   | <GROUP : "group">
   | <HAVING : "having">
   | <HINTS : "hints">
   | <IF : "if">
   | <INTO : "into">
   | <IN : "in">
   | <INDEX : "index">
   | <INGESTION : "ingestion">
   | <INNER : "inner">
   | <INSERT : "insert">
   | <INTERNAL : "internal">
   | <INTERSECT : "intersect">
   | <IS : "is">
   | <JOIN : "join">
   | <KEYWORD : "keyword">
   | <KEY : "key">
   | <KNOWN : "known">
   | <LEFT : "left">
   | <LETTING : "letting">
   | <LET : "let">
   | <LIKE : "like">
   | <LIMIT : "limit">
   | <LOAD : "load">
   | <MISSING : "missing">
   | <MOD : "mod">
   | <NODEGROUP : "nodegroup">
   | <NGRAM : "ngram">
   | <NOT : "not">
   | <NULL : "null">
   | <OFFSET : "offset">
   | <ON : "on">
   | <OPEN : "open">
   | <OR : "or">
   | <ORDER : "order">
   | <OUTER : "outer">
   | <OUTPUT : "output">
   | <OVER: "over">
   | <PATH : "path">
   | <POLICY : "policy">
   | <PRESORTED : "pre-sorted">
   | <PRIMARY : "primary">
   | <RAW : "raw">
   | <RETURN : "return">
   | <RETURNING : "returning">
   | <RIGHT : "right">
   | <RTREE : "rtree">
   | <RUN : "run">
   | <SATISFIES : "satisfies">
   | <SECONDARY : "secondary">
   | <SELECT : "select">
   | <SET : "set">
   | <SOME : "some">
   | <START : "start">
   | <STOP : "stop">
   | <SYNONYM : "synonym">
   | <TEMPORARY : "temporary"> // intentionally not used but reserved for future usage
   | <THEN : "then">
   | <TO : "to">
   | <TRUE : "true">
   | <TYPE : "type">
   | <UNION : "union">
   | <UNKNOWN : "unknown">
   | <UNNEST : "unnest">
   | <UPDATE : "update">
   | <UPSERT : "upsert">
   | <USE : "use">
   | <USING : "using">
   | <VALUE : "value">
   | <VALUED : "valued">
   | <VIEW : "view">
   | <WHEN : "when">
   | <WHERE : "where">
   | <WITH : "with">
   | <WRITE : "write">
   | <COPY : "copy">
 }

 <DEFAULT,IN_DBL_BRACE>
 TOKEN :
 {
     <CARET : "^">
   | <CONCAT : "||">
   | <DIVIDE : "/">
   | <MINUS : "-">
   | <MUL : "*">
   | <PLUS : "+">

   | <LEFTPAREN : "(">
   | <RIGHTPAREN : ")">
   | <LEFTBRACKET : "[">
   | <RIGHTBRACKET : "]">

   | <ATT : "@">
   | <COLON : ":">
   | <COMMA : ",">
   | <DOT : ".">
   | <PERCENT: "%">
   | <QUES : "?">
   | <SEMICOLON : ";">
   | <SHARP : "#">

   | <LT : "<">
   | <GT : ">">
   | <LE : "<=">
   | <GE : ">=">
   | <EQ : "=">
   | <NE : "!=">
   | <LG : "<>">
   | <SIMILAR : "~=">
 }

 <DEFAULT,IN_DBL_BRACE>
 TOKEN :
 {
     <LEFTBRACE : "{"> { pushState(); } : DEFAULT
 }

 <DEFAULT>
 TOKEN :
 {
     <RIGHTBRACE : "}"> { popState("}"); }
 }

 <DEFAULT,IN_DBL_BRACE>
 TOKEN :
 {
     <LEFTDBLBRACE : "{{"> { pushState(); } : IN_DBL_BRACE
 }

 <IN_DBL_BRACE>
 TOKEN :
 {
     <RIGHTDBLBRACE : "}}"> { popState("}}"); }
 }

 <DEFAULT,IN_DBL_BRACE>
 TOKEN :
 {
     <#DIGIT : ["0" - "9"]>
 }

 <DEFAULT,IN_DBL_BRACE>
 TOKEN:
 {
     <INTEGER_LITERAL : <DIGITS> >
   | <DOUBLE_LITERAL: <DIGITS> ( "." <DIGITS> ) (("e"|"E") ("+"|"-")? <DIGITS>)?
                       | <DIGITS> (("e"|"E") ("+"|"-")? <DIGITS>)
                       | "." <DIGITS> (("e"|"E") ("+"|"-")? <DIGITS>)?
     >
   | <FLOAT_LITERAL:  <DIGITS> ( "f" | "F" )
                       | <DIGITS> ( "." <DIGITS> ( "f" | "F" ) )?
                       | "." <DIGITS> ( "f" | "F" )
     >
   | <#DIGITS : (<DIGIT>)+ >
 }

 <DEFAULT,IN_DBL_BRACE>
 TOKEN :
 {
     <#LETTER : ["A" - "Z", "a" - "z"]>
   | <#IDENTIFIER_START_SPECIALCHAR : ["_"]>
   | <#IDENTIFIER_REST_SPECIALCHAR : ["$"]>
   | <#IDENTIFIER_START : <LETTER> | <IDENTIFIER_START_SPECIALCHAR> >
   | <#IDENTIFIER_REST  : <LETTER> | <DIGIT> | <IDENTIFIER_START_SPECIALCHAR> | <IDENTIFIER_REST_SPECIALCHAR> >
   | <IDENTIFIER : <IDENTIFIER_START> (<IDENTIFIER_REST>)* >
 }

 <DEFAULT,IN_DBL_BRACE>
 TOKEN :
 {
     // backslash u + 4 hex digits escapes are handled in the underlying JavaCharStream
     <QUOTED_STRING : "`" (
           <EscapeQuot>
         | <EscapeBslash>
         | <EscapeSlash>
         | <EscapeBspace>
         | <EscapeFormf>
         | <EscapeNl>
         | <EscapeCr>
         | <EscapeTab>
         | <EscapeBTickWithBslash>
         | <EscapeBtickWithBtick>
         | ~["`","\\"])* "`">
   | <STRING_LITERAL : ( ("E")? "\"" (
           <EscapeQuot>
         | <EscapeBslash>
         | <EscapeSlash>
         | <EscapeBspace>
         | <EscapeFormf>
         | <EscapeNl>
         | <EscapeBTickWithBslash>
         | <EscapeCr>
         | <EscapeTab>
         | ~["\"","\\"])* "\"")
       | ( ("E")? "\'" (
           <EscapeApos>
         | <EscapeBslash>
         | <EscapeSlash>
         | <EscapeBspace>
         | <EscapeFormf>
         | <EscapeNl>
         | <EscapeBTickWithBslash>
         | <EscapeCr>
         | <EscapeTab>
         | ~["\'","\\"])* "\'")>
   | < #EscapeBTickWithBslash: "\\`" >
   | < #EscapeBtickWithBtick: "``" >
   | < #EscapeQuot: "\\\"" >
   | < #EscapeApos: "\\\'" >
   | < #EscapeBslash: "\\\\" >
   | < #EscapeSlash: "\\/" >
   | < #EscapeBspace: "\\b" >
   | < #EscapeFormf: "\\f" >
   | < #EscapeNl: "\\n" >
   | < #EscapeCr: "\\r" >
   | < #EscapeTab: "\\t" >
 }

 <DEFAULT,IN_DBL_BRACE>
 TOKEN :
 {
     <DOLLAR_INTEGER_LITERAL : "$" <INTEGER_LITERAL> >
   | <DOLLAR_IDENTIFIER : "$" <IDENTIFIER> >
   | <DOLLAR_QUOTED_STRING: "$" <QUOTED_STRING> >
 }

 <DEFAULT,IN_DBL_BRACE>
 SKIP:
 {
     " "
   | "\t"
   | "\r"
   | "\n"
 }

 <DEFAULT,IN_DBL_BRACE>
 SKIP:
 {
     <"//" (~["\n"])* "\n">
 }

 <DEFAULT,IN_DBL_BRACE>
 SKIP:
 {
     <"//" (~["\n","\r"])* ("\n"|"\r"|"\r\n")?>
 }

 <DEFAULT,IN_DBL_BRACE>
 SKIP:
 {
     <"--" (~["\n"])* "\n">
 }


 <DEFAULT,IN_DBL_BRACE>
 SKIP:
 {
     <"--" (~["\n","\r"])* ("\n"|"\r"|"\r\n")?>
 }

 <DEFAULT,IN_DBL_BRACE>
 SKIP:
 {
     <"/*"> { pushState(); } : INSIDE_COMMENT
 }

 <INSIDE_COMMENT>
 SPECIAL_TOKEN:
 {
     <"+"(" ")*(~["*"])*>
 }

 <INSIDE_COMMENT>
 SKIP:
 {
     <"/*"> { pushState(); }
 }

 <INSIDE_COMMENT>
 SKIP:
 {
     <"*/"> { popState("*/"); }
   | <~[]>
 }