asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj

//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
//
options {
  COMMON_TOKEN_ACTION = true;
  STATIC = false;
  TOKEN_EXTENDS = "org.apache.asterix.lang.sqlpp.parser.SqlppToken";
}

PARSER_BEGIN(SQLPPParser)

package org.apache.asterix.lang.sqlpp.parser;

// For SQL++ ParserTokenManager
import java.util.ArrayDeque;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

import org.apache.asterix.common.annotations.AutoDataGen;
import org.apache.asterix.common.annotations.DateBetweenYearsDataGen;
import org.apache.asterix.common.annotations.DatetimeAddRandHoursDataGen;
import org.apache.asterix.common.annotations.DatetimeBetweenYearsDataGen;
import org.apache.asterix.common.annotations.FieldIntervalDataGen;
import org.apache.asterix.common.annotations.FieldValFileDataGen;
import org.apache.asterix.common.annotations.FieldValFileSameIndexDataGen;
import org.apache.asterix.common.annotations.IRecordFieldDataGen;
import org.apache.asterix.common.annotations.IndexedNLJoinExpressionAnnotation;
import org.apache.asterix.common.annotations.InsertRandIntDataGen;
import org.apache.asterix.common.annotations.ListDataGen;
import org.apache.asterix.common.annotations.ListValFileDataGen;
import org.apache.asterix.common.annotations.RangeAnnotation;
import org.apache.asterix.common.annotations.SecondaryIndexSearchPreferenceAnnotation;
import org.apache.asterix.common.annotations.SkipSecondaryIndexSearchExpressionAnnotation;
import org.apache.asterix.common.annotations.TypeDataGen;
import org.apache.asterix.common.annotations.UndeclaredFieldsDataGen;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.WarningCollector;
import org.apache.asterix.common.exceptions.WarningUtil;
import org.apache.asterix.common.functions.FunctionConstants;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.lang.common.base.AbstractClause;
import org.apache.asterix.lang.common.base.AbstractLangExpression;
import org.apache.asterix.lang.common.base.AbstractStatement;
import org.apache.asterix.lang.common.base.Expression;
import org.apache.asterix.lang.common.base.Literal;
import org.apache.asterix.lang.common.base.IParser;
import org.apache.asterix.lang.common.base.ILangExpression;
import org.apache.asterix.lang.common.base.IParserFactory;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.lang.common.clause.GroupbyClause;
import org.apache.asterix.lang.common.clause.LetClause;
import org.apache.asterix.lang.common.clause.LimitClause;
import org.apache.asterix.lang.common.clause.OrderbyClause;
import org.apache.asterix.lang.common.clause.UpdateClause;
import org.apache.asterix.lang.common.clause.WhereClause;
import org.apache.asterix.lang.common.context.RootScopeFactory;
import org.apache.asterix.lang.common.context.Scope;
import org.apache.asterix.lang.common.expression.AbstractAccessor;
import org.apache.asterix.lang.common.expression.CallExpr;
import org.apache.asterix.lang.common.expression.FieldAccessor;
import org.apache.asterix.lang.common.expression.FieldBinding;
import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair;
import org.apache.asterix.lang.common.expression.IfExpr;
import org.apache.asterix.lang.common.expression.IndexAccessor;
import org.apache.asterix.lang.common.expression.IndexedTypeExpression;
import org.apache.asterix.lang.common.expression.ListConstructor;
import org.apache.asterix.lang.common.expression.ListSliceExpression;
import org.apache.asterix.lang.common.expression.LiteralExpr;
import org.apache.asterix.lang.common.expression.OperatorExpr;
import org.apache.asterix.lang.common.expression.OrderedListTypeDefinition;
import org.apache.asterix.lang.common.expression.QuantifiedExpression;
import org.apache.asterix.lang.common.expression.RecordConstructor;
import org.apache.asterix.lang.common.expression.RecordTypeDefinition;
import org.apache.asterix.lang.common.expression.TypeExpression;
import org.apache.asterix.lang.common.expression.TypeReferenceExpression;
import org.apache.asterix.lang.common.expression.UnaryExpr;
import org.apache.asterix.lang.common.expression.UnorderedListTypeDefinition;
import org.apache.asterix.lang.common.expression.VariableExpr;
import org.apache.asterix.lang.common.literal.DoubleLiteral;
import org.apache.asterix.lang.common.literal.FalseLiteral;
import org.apache.asterix.lang.common.literal.FloatLiteral;
import org.apache.asterix.lang.common.literal.LongIntegerLiteral;
import org.apache.asterix.lang.common.literal.MissingLiteral;
import org.apache.asterix.lang.common.literal.NullLiteral;
import org.apache.asterix.lang.common.literal.StringLiteral;
import org.apache.asterix.lang.common.literal.TrueLiteral;
import org.apache.asterix.lang.common.parser.ScopeChecker;
import org.apache.asterix.lang.common.statement.AdapterDropStatement;
import org.apache.asterix.lang.common.statement.CompactStatement;
import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
import org.apache.asterix.lang.common.statement.StartFeedStatement;
import org.apache.asterix.lang.common.statement.StopFeedStatement;
import org.apache.asterix.lang.common.statement.CreateAdapterStatement;
import org.apache.asterix.lang.common.statement.CreateDataverseStatement;
import org.apache.asterix.lang.common.statement.CreateFeedPolicyStatement;
import org.apache.asterix.lang.common.statement.CreateFeedStatement;
import org.apache.asterix.lang.common.statement.CreateFunctionStatement;
import org.apache.asterix.lang.common.statement.CreateIndexStatement;
import org.apache.asterix.lang.common.statement.CreateSynonymStatement;
import org.apache.asterix.lang.common.statement.DatasetDecl;
import org.apache.asterix.lang.common.statement.DataverseDecl;
import org.apache.asterix.lang.common.statement.DataverseDropStatement;
import org.apache.asterix.lang.common.statement.DeleteStatement;
import org.apache.asterix.lang.common.statement.DisconnectFeedStatement;
import org.apache.asterix.lang.common.statement.DropDatasetStatement;
import org.apache.asterix.lang.common.statement.ExternalDetailsDecl;
import org.apache.asterix.lang.common.statement.FeedDropStatement;
import org.apache.asterix.lang.common.statement.FeedPolicyDropStatement;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.statement.FunctionDropStatement;
import org.apache.asterix.lang.common.statement.IndexDropStatement;
import org.apache.asterix.lang.common.statement.InsertStatement;
import org.apache.asterix.lang.common.statement.InternalDetailsDecl;
import org.apache.asterix.lang.common.statement.LoadStatement;
import org.apache.asterix.lang.common.statement.NodeGroupDropStatement;
import org.apache.asterix.lang.common.statement.NodegroupDecl;
import org.apache.asterix.lang.common.statement.Query;
import org.apache.asterix.lang.common.statement.RefreshExternalDatasetStatement;
import org.apache.asterix.lang.common.statement.SetStatement;
import org.apache.asterix.lang.common.statement.SynonymDropStatement;
import org.apache.asterix.lang.common.statement.TypeDecl;
import org.apache.asterix.lang.common.statement.TypeDropStatement;
import org.apache.asterix.lang.common.statement.UpdateStatement;
import org.apache.asterix.lang.common.statement.UpsertStatement;
import org.apache.asterix.lang.common.statement.WriteStatement;
import org.apache.asterix.lang.common.struct.Identifier;
import org.apache.asterix.lang.common.struct.OperatorType;
import org.apache.asterix.lang.common.struct.QuantifiedPair;
import org.apache.asterix.lang.common.struct.VarIdentifier;
import org.apache.asterix.lang.common.util.RangeMapBuilder;
import org.apache.asterix.lang.sqlpp.clause.AbstractBinaryCorrelateClause;
import org.apache.asterix.lang.sqlpp.clause.FromClause;
import org.apache.asterix.lang.sqlpp.clause.FromTerm;
import org.apache.asterix.lang.sqlpp.clause.HavingClause;
import org.apache.asterix.lang.sqlpp.clause.JoinClause;
import org.apache.asterix.lang.sqlpp.clause.Projection;
import org.apache.asterix.lang.sqlpp.clause.SelectBlock;
import org.apache.asterix.lang.sqlpp.clause.SelectClause;
import org.apache.asterix.lang.sqlpp.clause.SelectElement;
import org.apache.asterix.lang.sqlpp.clause.SelectRegular;
import org.apache.asterix.lang.sqlpp.clause.SelectSetOperation;
import org.apache.asterix.lang.sqlpp.clause.UnnestClause;
import org.apache.asterix.lang.common.clause.WhereClause;
import org.apache.asterix.lang.sqlpp.expression.CaseExpression;
import org.apache.asterix.lang.sqlpp.expression.SelectExpression;
import org.apache.asterix.lang.sqlpp.expression.WindowExpression;
import org.apache.asterix.lang.sqlpp.optype.JoinType;
import org.apache.asterix.lang.sqlpp.optype.SetOpType;
import org.apache.asterix.lang.sqlpp.optype.UnnestType;
import org.apache.asterix.lang.sqlpp.parser.SqlppGroupingSetsParser;
import org.apache.asterix.lang.sqlpp.parser.SqlppGroupingSetsParser.GroupingElement;
import org.apache.asterix.lang.sqlpp.parser.SqlppGroupingSetsParser.GroupingSet;
import org.apache.asterix.lang.sqlpp.parser.SqlppGroupingSetsParser.GroupingSets;
import org.apache.asterix.lang.sqlpp.parser.SqlppGroupingSetsParser.RollupCube;
import org.apache.asterix.lang.sqlpp.parser.SqlppHint;
import org.apache.asterix.lang.sqlpp.struct.SetOperationInput;
import org.apache.asterix.lang.sqlpp.struct.SetOperationRight;
import org.apache.asterix.lang.sqlpp.util.ExpressionToVariableUtil;
import org.apache.asterix.lang.sqlpp.util.FunctionMapUtil;
import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil;
import org.apache.asterix.external.dataset.adapter.AdapterIdentifier;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.expressions.BroadcastExpressionAnnotation;
import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionAnnotation;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.api.exceptions.Warning;
import org.apache.hyracks.dataflow.common.data.partition.range.RangeMap;
import org.apache.hyracks.util.LogRedactionUtil;
import org.apache.hyracks.util.StringUtil;

class SQLPPParser extends ScopeChecker implements IParser {

    // tokens parsed as identifiers
    private static final String CUBE = "CUBE";
    private static final String CURRENT = "CURRENT";
    private static final String EXCLUDE = "EXCLUDE";
    private static final String FIRST = "FIRST";
    private static final String FOLLOWING = "FOLLOWING";
    private static final String GROUPING = "GROUPING";
    private static final String GROUPS = "GROUPS";
    private static final String IGNORE = "IGNORE";
    private static final String LAST = "LAST";
    private static final String NO = "NO";
    private static final String NULLS = "NULLS";
    private static final String OTHERS = "OTHERS";
    private static final String PARTITION = "PARTITION";
    private static final String PRECEDING = "PRECEDING";
    private static final String RANGE = "RANGE";
    private static final String RESPECT = "RESPECT";
    private static final String ROLLUP = "ROLLUP";
    private static final String ROW = "ROW";
    private static final String ROWS = "ROWS";
    private static final String SETS = "SETS";
    private static final String TIES = "TIES";
    private static final String UNBOUNDED = "UNBOUNDED";
    private static final String REPLACE = "REPLACE";
    private static final String RETURNS = "RETURNS";

    private static final String INT_TYPE_NAME = "int";
    private static final String UDF_VARARGS_PARAM_NAME = "args"; // Note: this value is stored in the function metadata

    // error configuration
    protected static final boolean REPORT_EXPECTED_TOKENS = false;

    private int externalVarCounter;

    private DataverseName defaultDataverse;

    private SqlppGroupingSetsParser groupingSetsParser;

    private final WarningCollector warningCollector = new WarningCollector();

    private final Map<SourceLocation, String> hintCollector = new HashMap<SourceLocation, String>();

    private static class IndexParams {
      public IndexType type;
      public int gramLength;

      public IndexParams(IndexType type, int gramLength) {
        this.type = type;
        this.gramLength = gramLength;
      }
    };

    private static class FunctionName {
       public DataverseName dataverse;
       public String library;
       public String function;
       public Token hintToken;
       public SourceLocation sourceLoc;
    }

    private IRecordFieldDataGen parseFieldDataGen(Token hintToken) throws ParseException {
      String[] splits = hintToken.hintParams != null ? hintToken.hintParams.split("\\s+") : null;
      switch (hintToken.hint) {
        case VAL_FILE_HINT:
          File[] valFiles = new File[splits.length];
          for (int k=0; k<splits.length; k++) {
            valFiles[k] = new File(splits[k]);
          }
          return new FieldValFileDataGen(valFiles);
        case VAL_FILE_SAME_INDEX_HINT:
          return new FieldValFileSameIndexDataGen(new File(splits[0]), splits[1]);
        case LIST_VAL_FILE_HINT:
          return new ListValFileDataGen(new File(splits[0]), Integer.parseInt(splits[1]), Integer.parseInt(splits[2]));
        case LIST_HINT:
          return new ListDataGen(Integer.parseInt(splits[0]), Integer.parseInt(splits[1]));
        case INTERVAL_HINT:
          FieldIntervalDataGen.ValueType vt;
          switch (splits[0]) {
            case "int":
              vt = FieldIntervalDataGen.ValueType.INT;
              break;
            case "long":
              vt = FieldIntervalDataGen.ValueType.LONG;
              break;
            case "float":
              vt = FieldIntervalDataGen.ValueType.FLOAT;
              break;
            case "double":
              vt = FieldIntervalDataGen.ValueType.DOUBLE;
              break;
            default:
              throw new SqlppParseException(getSourceLocation(hintToken),
                "Unknown type for interval data gen: " + splits[0]);
          }
          return new FieldIntervalDataGen(vt, splits[1], splits[2]);
        case INSERT_RAND_INT_HINT:
          return new InsertRandIntDataGen(splits[0], splits[1]);
        case DATE_BETWEEN_YEARS_HINT:
          return new DateBetweenYearsDataGen(Integer.parseInt(splits[0]), Integer.parseInt(splits[1]));
        case DATETIME_BETWEEN_YEARS_HINT:
          return new DatetimeBetweenYearsDataGen(Integer.parseInt(splits[0]), Integer.parseInt(splits[1]));
        case DATETIME_ADD_RAND_HOURS_HINT:
          return new DatetimeAddRandHoursDataGen(Integer.parseInt(splits[0]), Integer.parseInt(splits[1]), splits[2]);
        case AUTO_HINT:
          return new AutoDataGen(splits[0]);
        default:
          return null;
      }
    }

    public SQLPPParser(String s) {
        this(new StringReader(s));
        super.setInput(s);
    }

    public static void main(String args[]) throws ParseException, TokenMgrError, IOException, FileNotFoundException, CompilationException {
        File file = new File(args[0]);
        Reader fis = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
        SQLPPParser parser = new SQLPPParser(fis);
        List<Statement> st = parser.parse();
        //st.accept(new SQLPPPrintVisitor(), 0);
    }

    @Override
    public List<Statement> parse() throws CompilationException {
        return parseImpl(new ParseFunction<List<Statement>>() {
            @Override
            public List<Statement> parse() throws ParseException {
                return SQLPPParser.this.Statement();
            }
        });
    }

    @Override
    public Expression parseExpression() throws CompilationException {
        return parseImpl(new ParseFunction<Expression>() {
            @Override
            public Expression parse() throws ParseException {
                return SQLPPParser.this.Expression();
            }
        });
    }

    private static Expression parseExpression(String text) throws CompilationException {
        return new SQLPPParser(text).parseExpression();
    }

    @Override
    public List<String> parseMultipartIdentifier() throws CompilationException {
        return parseImpl(new ParseFunction<List<String>>() {
            @Override
            public List<String> parse() throws ParseException {
                return SQLPPParser.this.MultipartIdentifier();
            }
        });
    }

    private List<String> parseParenthesizedIdentifierList() throws CompilationException {
        return parseImpl(new ParseFunction<List<String>>() {
            @Override
            public  List<String> parse() throws ParseException {
                return SQLPPParser.this.ParenthesizedIdentifierList();
            }
        });
    }

    private static List<String> parseParenthesizedIdentifierList(String text) throws CompilationException {
        return new SQLPPParser(text).parseParenthesizedIdentifierList();
    }

    @Override
    public FunctionDecl parseFunctionBody(FunctionSignature signature, List<String> paramNames)
      throws CompilationException {
        return parseImpl(new ParseFunction<FunctionDecl>() {
            @Override
            public FunctionDecl parse() throws ParseException {
                DataverseName dataverse = defaultDataverse;
                defaultDataverse = signature.getDataverseName();
                createNewScope();
                List<VarIdentifier> paramVars = new ArrayList<VarIdentifier>(paramNames.size());
                for (String paramName : paramNames) {
                    paramVars.add(SqlppVariableUtil.toInternalVariableIdentifier(paramName));
                }
                Expression functionBodyExpr = SQLPPParser.this.FunctionBody();
                removeCurrentScope();
                defaultDataverse = dataverse;
                return new FunctionDecl(signature, paramVars, functionBodyExpr);
            }
        });
    }

    private <T> T parseImpl(ParseFunction<T> parseFunction) throws CompilationException {
        warningCollector.clear();
        hintCollector.clear();
        token_source.hintCollector = hintCollector;
        try {
            return parseFunction.parse();
        } catch (SqlppParseException e) {
            throw new CompilationException(ErrorCode.PARSE_ERROR, e.getSourceLocation(), LogRedactionUtil.userData(getMessage(e)));
        } catch (ParseException e) {
            throw new CompilationException(ErrorCode.PARSE_ERROR, LogRedactionUtil.userData(getMessage(e)));
        } catch (Error e) {
            // this is here as the JavaCharStream that's below the lexer sometimes throws Errors that are not handled
            // by the generated lexer or parser (e.g it does this for invalid backslash u + 4 hex digits escapes)
            final String msg = e.getClass().getSimpleName() + (e.getMessage() != null ? ": " + e.getMessage() : "");
            throw new CompilationException(ErrorCode.PARSE_ERROR, LogRedactionUtil.userData(msg));
        } finally {
            reportUnclaimedHints();
        }
    }

    @FunctionalInterface
    private interface ParseFunction<T> {
        T parse() throws ParseException;
    }

    @Override
    public void getWarnings(IWarningCollector outWarningCollector) {
        warningCollector.getWarnings(outWarningCollector);
    }

    @Override
    public void getWarnings(Collection<? super Warning> outWarnings, long maxWarnings) {
        warningCollector.getWarnings(outWarnings, maxWarnings);
    }

    @Override
    public long getTotalWarningsCount() {
        return warningCollector.getTotalWarningsCount();
    }

    protected String getMessage(ParseException pe) {
        Token currentToken = pe.currentToken;
        if (currentToken == null) {
            return pe.getMessage();
        }
        int[][] expectedTokenSequences = pe.expectedTokenSequences;
        String[] tokenImage = pe.tokenImage;
        String sep = REPORT_EXPECTED_TOKENS ? eol : " ";
        StringBuilder expected = REPORT_EXPECTED_TOKENS ? new StringBuilder() : null;
        int maxSize = appendExpected(expected, expectedTokenSequences, tokenImage);
        Token tok = currentToken.next;
        int line = tok.beginLine;
        StringBuilder message = new StringBuilder(128);
        message.append("In line ").append(line).append(" >>").append(getLine(line)).append("<<").append(sep).append("Encountered ");
        for (int i = 0; i < maxSize; i++) {
            if (i != 0) {
                message.append(' ');
            }
            if (tok.kind == 0) {
                message.append(fixQuotes(tokenImage[0]));
                break;
            }
            final String fixedTokenImage = tokenImage[tok.kind];
            if (! tok.image.equalsIgnoreCase(stripQuotes(fixedTokenImage))) {
                message.append(fixQuotes(fixedTokenImage)).append(' ');
            }
            message.append(quot).append(addEscapes(tok.image)).append(quot);
            tok = tok.next;
        }
        message.append(" at column ").append(currentToken.next.beginColumn).append('.').append(sep);
        if (REPORT_EXPECTED_TOKENS) {
            if (expectedTokenSequences.length == 1) {
                message.append("Was expecting:").append(sep).append("    ");
            } else {
                message.append("Was expecting one of:").append(sep).append("    ");
            }
            message.append(expected);
        }
        return message.toString();
    }

    protected static SourceLocation getSourceLocation(Token token) {
        return
          token == null ? null :
          token.sourceLocation != null ? token.sourceLocation :
          new SourceLocation(token.beginLine, token.beginColumn);
    }

    protected static <T extends AbstractLangExpression> T addSourceLocation(T expr, Token token) {
        expr.setSourceLocation(getSourceLocation(token));
        return expr;
    }

    private boolean isToken(String image) {
      return isToken(token, image);
    }

    private static boolean isToken(Token token, String image) {
      return token.image.equalsIgnoreCase(image);
    }

    private void expectToken(String image) throws SqlppParseException {
      expectToken(token, image);
    }

    private static void expectToken(Token token, String image) throws SqlppParseException {
      if (!isToken(token, image)) {
        throw createUnexpectedTokenError(token);
      }
    }

    private SqlppParseException createUnexpectedTokenError() {
      return createUnexpectedTokenError(token, null);
    }

    private static SqlppParseException createUnexpectedTokenError(Token t) {
      return createUnexpectedTokenError(t, null);
    }

    private SqlppParseException createUnexpectedTokenError(String expected) {
      return createUnexpectedTokenError(token, expected);
    }

    private static SqlppParseException createUnexpectedTokenError(Token t, String expected) {
      String message = "Unexpected token: " + LogRedactionUtil.userData(t.image) +
        (expected == null ? "" : ". Expected: " + LogRedactionUtil.userData(expected));
      return new SqlppParseException(getSourceLocation(t), message);
    }

    private boolean laToken(int idx, int kind) {
      Token t = getToken(idx);
      return t.kind == kind;
    }

    private boolean laToken(int idx, int kind, String image) {
      Token t = getToken(idx);
      return t.kind == kind && t.image.equalsIgnoreCase(image);
    }

    private boolean laIdentifier(int idx, String image) {
      return laToken(idx, IDENTIFIER, image);
    }

    private boolean laIdentifier(String image) {
      return laIdentifier(1, image);
    }

    private Token fetchHint(Token token, SqlppHint... expectedHints) {
      Token hintToken = token.specialToken;
      if (hintToken == null) {
        return null;
      }
      SourceLocation sourceLoc = getSourceLocation(hintToken);
      hintCollector.remove(sourceLoc);
      if (hintToken.hint == null) {
        warnUnexpectedHint(hintToken.hintParams, sourceLoc, expectedHints);
        return null;
      } else if (!ArrayUtils.contains(expectedHints, hintToken.hint)) {
        warnUnexpectedHint(hintToken.hint.getIdentifier(), sourceLoc, expectedHints);
        return null;
      } else {
        return hintToken;
      }
    }

    private void reportUnclaimedHints() {
        for (Map.Entry<SourceLocation, String> me : hintCollector.entrySet()) {
            warnUnexpectedHint(me.getValue(), me.getKey(), "None");
        }
    }

    private void warnUnexpectedHint(String actualHint, SourceLocation sourceLoc, SqlppHint... expectedHints) {
        warnUnexpectedHint(actualHint, sourceLoc, StringUtil.join(expectedHints, ", ", "\""));
    }

    private void warnUnexpectedHint(String actualHint, SourceLocation sourceLoc, String expectedHints) {
        if (warningCollector.shouldWarn()) {
            warningCollector.warn(WarningUtil.forAsterix(sourceLoc, ErrorCode.UNEXPECTED_HINT, actualHint, expectedHints));
        }
    }

    private IExpressionAnnotation parseExpressionAnnotation(Token hintToken) {
      // placeholder for the annotation that should be returned if this hint's parameters cannot be parsed
      IExpressionAnnotation onParseErrorReturn = null;
      try {
        switch (hintToken.hint) {
          case HASH_BROADCAST_JOIN_HINT:
            return new BroadcastExpressionAnnotation(BroadcastExpressionAnnotation.BroadcastSide.RIGHT);
          case INDEXED_NESTED_LOOP_JOIN_HINT:
            if (hintToken.hintParams == null) {
              return IndexedNLJoinExpressionAnnotation.INSTANCE_ANY_INDEX;
            } else {
              // if parameter parsing fails then return hint annotation without parameters
              onParseErrorReturn = IndexedNLJoinExpressionAnnotation.INSTANCE_ANY_INDEX;
              List<String> indexNames = parseParenthesizedIdentifierList(hintToken.hintParams);
              return IndexedNLJoinExpressionAnnotation.newInstance(indexNames);
            }
          case RANGE_HINT:
            Expression rangeExpr = parseExpression(hintToken.hintParams);
            RangeMap rangeMap = RangeMapBuilder.parseHint(rangeExpr);
            return new RangeAnnotation(rangeMap);
          case SKIP_SECONDARY_INDEX_SEARCH_HINT:
            if (hintToken.hintParams == null) {
              return SkipSecondaryIndexSearchExpressionAnnotation.INSTANCE_ANY_INDEX;
            } else {
              // if parameter parsing fails then ignore this hint
              List<String> indexNames = parseParenthesizedIdentifierList(hintToken.hintParams);
              return SkipSecondaryIndexSearchExpressionAnnotation.newInstance(indexNames);
            }
          case USE_SECONDARY_INDEX_SEARCH_HINT:
            if (hintToken.hintParams == null) {
              throw new SqlppParseException(getSourceLocation(hintToken), "Expected index name(s)");
            } else {
              // if parameter parsing fails then ignore this hint
              List<String> indexNames = parseParenthesizedIdentifierList(hintToken.hintParams);
              return SecondaryIndexSearchPreferenceAnnotation.newInstance(indexNames);
            }
          default:
            throw new SqlppParseException(getSourceLocation(hintToken), "Unexpected hint");
        }
      } catch (SqlppParseException e) {
        if (warningCollector.shouldWarn()) {
            warningCollector.warn(WarningUtil.forAsterix(getSourceLocation(hintToken), ErrorCode.INVALID_HINT,
              hintToken.hint.toString(), e.getMessage()));
        }
        return onParseErrorReturn;
      } catch (CompilationException e) {
        if (warningCollector.shouldWarn()) {
            warningCollector.warn(WarningUtil.forAsterix(getSourceLocation(hintToken), ErrorCode.INVALID_HINT,
              hintToken.hint.toString(), e.getMessage()));
        }
        return onParseErrorReturn;
      }
    }

    private void ensureNoTypeDeclsInFunction(String fnName, List<Pair<VarIdentifier, TypeExpression>> paramList,
      TypeExpression returnType, Token startToken) throws SqlppParseException {
      for (Pair<VarIdentifier, TypeExpression> p : paramList) {
        if (p.second != null) {
          String paramName = SqlppVariableUtil.toUserDefinedName(p.first.getValue());
          throw new SqlppParseException(getSourceLocation(startToken),
            "Unexpected type declaration for parameter " + paramName + " in function " + fnName);
        }
      }
      if (returnType != null) {
        throw new SqlppParseException(getSourceLocation(startToken),
          "Unexpected return type declaration for function " + fnName);
      }
    }

    private void ensureIntegerLiteral(LiteralExpr expr, String errorPrefix)  throws SqlppParseException {
        Literal lit = expr.getValue();
        if (lit.getLiteralType() != Literal.Type.INTEGER && lit.getLiteralType() != Literal.Type.LONG) {
            throw new SqlppParseException(expr.getSourceLocation(), errorPrefix + " should be an INTEGER");
        }
    }
}

PARSER_END(SQLPPParser)

List<Statement> Statement() throws ParseException:
{
  scopeStack.push(RootScopeFactory.createRootScope(this));
  List<Statement> decls = new ArrayList<Statement>();
  Statement stmt = null;
}
{
  (
    (stmt = ExplainStatement()
      {
        decls.add(stmt);
      }
    )?
    (<SEMICOLON>)+
  )*
  <EOF>
  {
    return decls;
  }
}

Statement ExplainStatement() throws ParseException:
{
  Statement stmt = null;
  Token explainToken = null;
}
{
   ( <EXPLAIN> { explainToken = token; } )?
   stmt = SingleStatement()
   {
      if (explainToken != null) {
        if (stmt.getKind() == Statement.Kind.QUERY) {
          ((Query)stmt).setExplain(true);
        } else {
          throw new SqlppParseException(getSourceLocation(explainToken),
            "EXPLAIN is not supported for this kind of statement");
        }
      }
      return stmt;
   }
}

Statement SingleStatement() throws ParseException:
{
  Statement stmt = null;
}
{
  (
    stmt = DataverseDeclaration()
    | stmt = FunctionDeclaration()
    | stmt = CreateStatement()
    | stmt = LoadStatement()
    | stmt = DropStatement()
    | stmt = WriteStatement()
    | stmt = SetStatement()
    | stmt = InsertStatement()
    | stmt = DeleteStatement()
    | stmt = UpdateStatement()
    | stmt = UpsertStatement()
    | stmt = ConnectionStatement()
    | stmt = CompactStatement()
    | stmt = Query()
    | stmt = RefreshExternalDatasetStatement()
  )
  {
    return stmt;
  }
}

DataverseDecl DataverseDeclaration() throws ParseException:
{
  Token startToken = null;
  List<String> dvName = null;
}
{
  <USE> { startToken = token; } dvName = MultipartIdentifier()
    {
      defaultDataverse = DataverseName.create(dvName);
      DataverseDecl dvDecl = new DataverseDecl(defaultDataverse);
      return addSourceLocation(dvDecl, startToken);
    }
}

Statement CreateStatement() throws ParseException:
{
  Token startToken = null;
  Statement stmt = null;
}
{
  <CREATE> { startToken = token; }
  (
    stmt = CreateOrReplaceStatement(startToken)
    | stmt = CreateTypeStatement(startToken)
    | stmt = CreateNodegroupStatement(startToken)
    | stmt = CreateDatasetStatement(startToken)
    | stmt = CreateIndexStatement(startToken)
    | stmt = CreateDataverseStatement(startToken)
    | stmt = CreateFunctionStatement(startToken, false)
    | stmt = CreateAdapterStatement(startToken)
    | stmt = CreateSynonymStatement(startToken)
    | stmt = CreateFeedStatement(startToken)
    | stmt = CreateFeedPolicyStatement(startToken)
  )
  {
    return stmt;
  }
}

Statement CreateOrReplaceStatement(Token startStmtToken) throws ParseException:
{
  Statement stmt = null;
  Token replaceToken = null;
}
{
  <OR> <IDENTIFIER> { replaceToken = token; }
  (
    stmt = CreateFunctionStatement(startStmtToken, true)
  )
  {
    // check expected token here to make the grammar extension plugin happy
    expectToken(replaceToken, REPLACE);
    return stmt;
  }
}

TypeDecl CreateTypeStatement(Token startStmtToken) throws ParseException:
{
  TypeDecl stmt = null;
}
{
  <TYPE> stmt = TypeSpecification(startStmtToken)
  {
    return stmt;
  }
}

TypeDecl TypeSpecification(Token startStmtToken) throws ParseException:
{
  Pair<DataverseName,Identifier> nameComponents = null;
  boolean ifNotExists = false;
  TypeExpression typeExpr = null;
}
{
  nameComponents = TypeName() ifNotExists = IfNotExists()
  <AS> typeExpr = RecordTypeDef()
  {
    boolean dgen = false;
    long numValues = -1;
    String filename = null;
    Token hintToken = fetchHint(startStmtToken, SqlppHint.DGEN_HINT);
    if (hintToken != null) {
        String hintParams = hintToken.hintParams;
        String[] splits = hintParams != null ? hintParams.split("\\s+") : null;
        if (splits == null || splits.length != 2) {
          throw new SqlppParseException(getSourceLocation(hintToken),
            "Expecting /*+ dgen <filename> <numberOfItems> */");
        }
        dgen = true;
        filename = splits[0];
        numValues = Long.parseLong(splits[1]);
    }
    TypeDataGen tddg = new TypeDataGen(dgen, filename, numValues);
    TypeDecl stmt = new TypeDecl(nameComponents.first, nameComponents.second, typeExpr, tddg, ifNotExists);
    return addSourceLocation(stmt, startStmtToken);
  }
}

NodegroupDecl CreateNodegroupStatement(Token startStmtToken) throws ParseException:
{
  NodegroupDecl stmt = null;
}
{
  <NODEGROUP> stmt = NodegroupSpecification(startStmtToken)
  {
    return stmt;
  }
}

NodegroupDecl NodegroupSpecification(Token startStmtToken) throws ParseException:
{
  String name = null;
  String tmp = null;
  boolean ifNotExists = false;
  List<Identifier> ncNames = new ArrayList<Identifier>();
}
{
  name = Identifier() ifNotExists = IfNotExists()
  <ON> tmp = Identifier()
  {
    ncNames.add(new Identifier(tmp));
  }
  ( <COMMA> tmp = Identifier()
    {
      ncNames.add(new Identifier(tmp));
    }
  )*
  {
    NodegroupDecl stmt = new NodegroupDecl(new Identifier(name), ncNames, ifNotExists);
    return addSourceLocation(stmt, startStmtToken);
  }
}

void Dataset() throws ParseException:
{
}
{
    (<DATASET>|<COLLECTION>)
}

DatasetDecl CreateDatasetStatement(Token startStmtToken) throws ParseException:
{
  DatasetDecl stmt = null;
}
{
  (
    (<INTERNAL>)? Dataset() stmt = DatasetSpecification(startStmtToken)
    | <EXTERNAL> Dataset() stmt = ExternalDatasetSpecification(startStmtToken)
  )
  {
    return stmt;
  }
}

DatasetDecl DatasetSpecification(Token startStmtToken) throws ParseException:
{
  Pair<DataverseName,Identifier> nameComponents = null;
  boolean ifNotExists = false;
  TypeExpression typeExpr = null;
  TypeExpression metaTypeExpr = null;
  Pair<List<Integer>, List<List<String>>> primaryKeyFields = null;
  Map<String,String> hints = new HashMap<String,String>();
  DatasetDecl stmt = null;
  boolean autogenerated = false;
  Pair<Integer, List<String>> filterField = null;
  RecordConstructor withRecord = null;
}
{
  nameComponents = QualifiedName()
  typeExpr = DatasetTypeSpecification()
  (
    { String name; }
    <WITH>
    name = Identifier()
    {
        if (!name.equalsIgnoreCase("meta")){
            throw new SqlppParseException(getSourceLocation(startStmtToken),
                "We can only support one additional associated field called \"meta\".");
        }
    }
    metaTypeExpr = DatasetTypeSpecification()
  )?
  ifNotExists = IfNotExists()
  primaryKeyFields = PrimaryKey()
  (<AUTOGENERATED> { autogenerated = true; } )?
  ( <HINTS> hints = Properties() )?
  ( LOOKAHEAD(2) <WITH> <FILTER> <ON>  filterField = NestedField() )?
  ( <WITH> withRecord = RecordConstructor() )?
  {
    try {
      InternalDetailsDecl idd = new InternalDetailsDecl(primaryKeyFields.second, primaryKeyFields.first, autogenerated,
        filterField == null? null : filterField.first, filterField == null? null : filterField.second);
      stmt = new DatasetDecl(nameComponents.first, nameComponents.second, typeExpr, metaTypeExpr, hints,
        DatasetType.INTERNAL, idd, withRecord, ifNotExists);
      return addSourceLocation(stmt, startStmtToken);
    } catch (CompilationException e) {
       throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage());
    }
  }
}

DatasetDecl ExternalDatasetSpecification(Token startStmtToken) throws ParseException:
{
  Pair<DataverseName,Identifier> nameComponents = null;
  TypeExpression typeExpr = null;
  boolean ifNotExists = false;
  String adapterName = null;
  Map<String,String> properties = null;
  Map<String,String> hints = new HashMap<String,String>();
  DatasetDecl stmt = null;
  RecordConstructor withRecord = null;
}
{
  nameComponents = QualifiedName()
  typeExpr = DatasetTypeSpecification()
  ifNotExists = IfNotExists()
  <USING> adapterName = AdapterName() properties = Configuration()
  ( <HINTS> hints = Properties() )?
  ( <WITH> withRecord = RecordConstructor() )?
  {
    ExternalDetailsDecl edd = new ExternalDetailsDecl();
    edd.setAdapter(adapterName);
    edd.setProperties(properties);
    try {
      stmt = new DatasetDecl(nameComponents.first, nameComponents.second, typeExpr, null, hints, DatasetType.EXTERNAL,
        edd, withRecord, ifNotExists);
      return addSourceLocation(stmt, startStmtToken);
    } catch (CompilationException e) {
       throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage());
    }
  }
}

TypeExpression DatasetTypeSpecification() throws ParseException:
{
  TypeExpression typeExpr = null;
}
{
  (
    LOOKAHEAD(3) typeExpr = DatasetRecordTypeSpecification(true)
    | typeExpr = DatasetReferenceTypeSpecification()
  )
  {
    return typeExpr;
  }
}

TypeExpression DatasetReferenceTypeSpecification() throws ParseException:
{
  TypeExpression typeExpr = null;
}
{
  <LEFTPAREN> typeExpr = TypeReference() <RIGHTPAREN>
  {
    return typeExpr;
  }
}

TypeExpression DatasetRecordTypeSpecification(boolean allowRecordKindModifier) throws ParseException:
{
  RecordTypeDefinition recordTypeDef = null;
  RecordTypeDefinition.RecordKind recordKind = null;
  Token startToken = null, recordKindToken = null;
}
{
   <LEFTPAREN> { startToken = token; } recordTypeDef = DatasetRecordTypeDef() <RIGHTPAREN>
   ( recordKind = RecordTypeKind() { recordKindToken = token; } <TYPE> )?
   {
     if (recordKind == null) {
       recordKind = RecordTypeDefinition.RecordKind.CLOSED;
     } else if (!allowRecordKindModifier) {
       throw createUnexpectedTokenError(recordKindToken);
     }
     recordTypeDef.setRecordKind(recordKind);
     return addSourceLocation(recordTypeDef, startToken);
   }
}

RecordTypeDefinition DatasetRecordTypeDef() throws ParseException:
{
  RecordTypeDefinition recType = new RecordTypeDefinition();
}
{
  DatasetRecordField(recType) ( <COMMA> DatasetRecordField(recType) )*
  {
    return recType;
  }
}

void DatasetRecordField(RecordTypeDefinition recType) throws ParseException:
{
  String fieldName;
  TypeExpression type = null;
  boolean nullable = true, missable = true;
}
{
  fieldName = Identifier()
  type = TypeReference() ( <NOT> <UNKNOWN> { nullable = false; missable = false; } )?
  {
    recType.addField(fieldName, type, nullable, missable);
  }
}

RefreshExternalDatasetStatement RefreshExternalDatasetStatement() throws ParseException:
{
  Token startToken = null;
  Pair<DataverseName,Identifier> nameComponents = null;
  String datasetName = null;
}
{
  <REFRESH> { startToken = token; } <EXTERNAL> Dataset() nameComponents = QualifiedName()
  {
    RefreshExternalDatasetStatement stmt = new RefreshExternalDatasetStatement();
    stmt.setDataverseName(nameComponents.first);
    stmt.setDatasetName(nameComponents.second);
    return addSourceLocation(stmt, startToken);
  }
}

CreateIndexStatement CreateIndexStatement(Token startStmtToken) throws ParseException:
{
  CreateIndexStatement stmt = null;
}
{
  (
    <INDEX> stmt = IndexSpecification(startStmtToken)
    | <PRIMARY> <INDEX> stmt = PrimaryIndexSpecification(startStmtToken)
  )
  {
    return stmt;
  }
}

CreateIndexStatement IndexSpecification(Token startStmtToken) throws ParseException:
{
  CreateIndexStatement stmt = new CreateIndexStatement();
  String indexName = null;
  boolean ifNotExists = false;
  Pair<DataverseName,Identifier> nameComponents = null;
  Pair<Integer, Pair<List<String>, IndexedTypeExpression>> fieldPair = null;
  IndexParams indexType = null;
  boolean enforced = false;
}
{
  (
    indexName = Identifier() ifNotExists = IfNotExists()
    <ON> nameComponents = QualifiedName()
    <LEFTPAREN> ( fieldPair = OpenField()
      {
        stmt.addFieldExprPair(fieldPair.second);
        stmt.addFieldIndexIndicator(fieldPair.first);
      }
    ) (<COMMA> fieldPair = OpenField()
      {
        stmt.addFieldExprPair(fieldPair.second);
        stmt.addFieldIndexIndicator(fieldPair.first);
      }
    )* <RIGHTPAREN> ( <TYPE> indexType = IndexType() )? ( <ENFORCED> { enforced = true; } )?
  )
  {
    stmt.setIndexName(new Identifier(indexName));
    stmt.setIfNotExists(ifNotExists);
    stmt.setDataverseName(nameComponents.first);
    stmt.setDatasetName(nameComponents.second);
    if (indexType != null) {
      stmt.setIndexType(indexType.type);
      stmt.setGramLength(indexType.gramLength);
    }
    stmt.setEnforced(enforced);
    return addSourceLocation(stmt, startStmtToken);
  }
}

CreateIndexStatement PrimaryIndexSpecification(Token startStmtToken) throws ParseException:
{
  CreateIndexStatement stmt = new CreateIndexStatement();
  String indexName = null;
  boolean ifNotExists = false;
  Pair<DataverseName,Identifier> nameComponents = null;
}
{
  (indexName = Identifier())? ifNotExists = IfNotExists()
  <ON> nameComponents = QualifiedName() (<TYPE> <BTREE>)?
  {
    if (indexName == null) {
      indexName = "primary_idx_" + nameComponents.second;
    }
    stmt.setIndexName(new Identifier(indexName));
    stmt.setIfNotExists(ifNotExists);
    stmt.setDataverseName(nameComponents.first);
    stmt.setDatasetName(nameComponents.second);
    return addSourceLocation(stmt, startStmtToken);
  }
}

String FilterField() throws ParseException :
{
  String filterField = null;
}
{
  filterField = Identifier()
    {
      return filterField;
    }
}

IndexParams IndexType() throws ParseException:
{
  IndexType type = null;
  int gramLength = 0;
}
{
  (<BTREE>
    {
      type = IndexType.BTREE;
    }
  | <RTREE>
    {
      type = IndexType.RTREE;
    }
  | <KEYWORD>
    {
      type = IndexType.LENGTH_PARTITIONED_WORD_INVIX;
    }
  |<FULLTEXT>
    {
      type = IndexType.SINGLE_PARTITION_WORD_INVIX;
    }
  | <NGRAM> <LEFTPAREN> <INTEGER_LITERAL>
    {
      type = IndexType.LENGTH_PARTITIONED_NGRAM_INVIX;
      gramLength = Integer.valueOf(token.image);
    }
  <RIGHTPAREN>)
    {
      return new IndexParams(type, gramLength);
    }
}

CreateDataverseStatement CreateDataverseStatement(Token startStmtToken) throws ParseException :
{
  CreateDataverseStatement stmt = null;
}
{
  <DATAVERSE> stmt = DataverseSpecification(startStmtToken)
  {
    return stmt;
  }
}

CreateDataverseStatement DataverseSpecification(Token startStmtToken) throws ParseException :
{
  List<String> dvName = null;
  boolean ifNotExists = false;
}
{
  dvName = MultipartIdentifier() ifNotExists = IfNotExists()
  {
    CreateDataverseStatement stmt = new CreateDataverseStatement(DataverseName.create(dvName), null, ifNotExists);
    return addSourceLocation(stmt, startStmtToken);
  }
}

CreateAdapterStatement CreateAdapterStatement(Token startStmtToken) throws ParseException:
{
  CreateAdapterStatement stmt = null;
}
{
  <ADAPTER> stmt = AdapterSpecification(startStmtToken)
  {
    return stmt;
  }
}

CreateAdapterStatement AdapterSpecification(Token startStmtToken) throws ParseException:
{
  Pair<DataverseName,Identifier> adapterName = null;
  Pair<DataverseName,Identifier> libraryName = null;
  List<String> externalIdentifier = null;
  boolean ifNotExists = false;
}
{
  adapterName = QualifiedName()
  ifNotExists = IfNotExists()
  <AS> externalIdentifier = FunctionExternalIdentifier()
  <AT> libraryName = QualifiedName()
  {
    CreateAdapterStatement stmt = new CreateAdapterStatement(adapterName.first, adapterName.second.getValue(),
      libraryName.first, libraryName.second.getValue(), externalIdentifier, ifNotExists);
    return addSourceLocation(stmt, startStmtToken);
  }
}

CreateFunctionStatement CreateFunctionStatement(Token startStmtToken, boolean orReplace) throws ParseException:
{
  CreateFunctionStatement stmt = null;
}
{
  <FUNCTION> stmt = FunctionSpecification(startStmtToken, orReplace)
  {
    return stmt;
  }
}

CreateFunctionStatement FunctionSpecification(Token startStmtToken, boolean orReplace) throws ParseException:
{
  FunctionSignature signature = null;
  FunctionName fctName = null;
  Pair<Integer, List<Pair<VarIdentifier,TypeExpression>>> paramsWithArity = null;
  List<Pair<VarIdentifier,TypeExpression>> params = null;
  int arity = 0;
  TypeExpression returnType = null;
  Token beginPos = null, endPos = null;
  Expression functionBodyExpr = null;
  Pair<DataverseName,Identifier> libraryName = null;
  List<String> externalIdentifier = null;
  RecordConstructor withOptions = null;
  boolean ifNotExists = false;
  CreateFunctionStatement stmt = null;
  DataverseName currentDataverse = defaultDataverse;
}
{
  fctName = FunctionName()
  {
     defaultDataverse = fctName.dataverse;
  }
  ifNotExists = IfNotExists()
  paramsWithArity = FunctionParameters()
  {
    arity = paramsWithArity.first;
    params = paramsWithArity.second;
  }
  returnType = FunctionReturnType()
  (
    (
      <LEFTBRACE>
      {
        beginPos = token;
        createNewScope();
      }
      functionBodyExpr = FunctionBody()
      <RIGHTBRACE>
      {
        endPos = token;
        String functionBody = extractFragment(beginPos.beginLine, beginPos.beginColumn, endPos.beginLine,
          endPos.beginColumn);
        signature = new FunctionSignature(fctName.dataverse, fctName.function, arity);
        getCurrentScope().addFunctionDescriptor(signature, false);
        removeCurrentScope();
        defaultDataverse = currentDataverse;
        ensureNoTypeDeclsInFunction(fctName.function, params, returnType, startStmtToken);
        stmt = new CreateFunctionStatement(signature, params, functionBody, functionBodyExpr, orReplace, ifNotExists);
        return addSourceLocation(stmt, startStmtToken);
      }
    )
  |
    (
      <AS> externalIdentifier = FunctionExternalIdentifier()
      <AT> libraryName = QualifiedName()
      (<WITH> withOptions = RecordConstructor())?
      {
        signature = new FunctionSignature(fctName.dataverse, fctName.function, arity);
        defaultDataverse = currentDataverse;
        try {
          stmt = new CreateFunctionStatement(signature, params, returnType, libraryName.first,
            libraryName.second.getValue(), externalIdentifier, withOptions, orReplace, ifNotExists);
        } catch (AlgebricksException e) {
            throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage());
        }
        return addSourceLocation(stmt, startStmtToken);
      }
    )
  )
}

Pair<Integer, List<Pair<VarIdentifier, TypeExpression>>> FunctionParameters() :
{
  Pair<Integer, List<Pair<VarIdentifier, TypeExpression>>> params = null;
}
{
  <LEFTPAREN> (params = FunctionParameterList())? <RIGHTPAREN>
  {
    if (params == null) {
      params = new Pair<Integer, List<Pair<VarIdentifier, TypeExpression>>>(
        0, Collections.<Pair<VarIdentifier, TypeExpression>>emptyList()
      );
    }
    return params;
  }
}

Pair<Integer, List<Pair<VarIdentifier, TypeExpression>>> FunctionParameterList() :
{
  List<Pair<VarIdentifier, TypeExpression>> paramList = null;
  Pair<VarIdentifier, TypeExpression> param = null;
  int arity = 0;
}
{
  (
    (
      <DOT> <DOT> <DOT>
      {
        param = new Pair<VarIdentifier, TypeExpression>(
          SqlppVariableUtil.toInternalVariableIdentifier(UDF_VARARGS_PARAM_NAME), null
        );
        paramList = Collections.<Pair<VarIdentifier, TypeExpression>>singletonList(param);
        arity = FunctionIdentifier.VARARGS;
      }
    )
    |
    (
      param = FunctionParameter()
      {
        paramList = new ArrayList<Pair<VarIdentifier, TypeExpression>>();
        paramList.add(param);
      }
      ( <COMMA> param = FunctionParameter() { paramList.add(param); } )*
      {
        arity = paramList.size();
      }
    )
  )
  {
    return new Pair<Integer, List<Pair<VarIdentifier, TypeExpression>>>(arity, paramList);
  }
}

Pair<VarIdentifier,TypeExpression> FunctionParameter() throws ParseException:
{
  String name = null;
  TypeExpression type = null;
}
{
  name = VariableIdentifier()
  ( LOOKAHEAD(3) (<COLON>)? type = TypeExpr(false) )?
  {
    return new Pair<VarIdentifier,TypeExpression>(new VarIdentifier(name), type);
  }
}

TypeExpression FunctionReturnType() throws ParseException:
{
  TypeExpression returnType = null;
}
{
  ( LOOKAHEAD({laIdentifier(RETURNS)}) <IDENTIFIER> returnType = TypeExpr(false) )?
  {
    return returnType;
  }
}

Expression FunctionBody() throws ParseException:
{
  Expression functionBodyExpr = null;
}
{
  ( functionBodyExpr = SelectExpression(true) | functionBodyExpr = Expression() )
  {
    return functionBodyExpr;
  }
}

List<String> FunctionExternalIdentifier() throws ParseException:
{
  String ident = null;
  List<String> identList = new ArrayList(2);
}
{
  ident = StringLiteral() { identList.add(ident.trim()); }
  ( <COMMA> ident = StringLiteral() { identList.add(ident.trim()); } )*
  {
    return identList;
  }
}

CreateFeedStatement CreateFeedStatement(Token startStmtToken) throws ParseException:
{
  CreateFeedStatement stmt = null;
}
{
  <FEED> stmt = FeedSpecification(startStmtToken)
  {
    return stmt;
  }
}

CreateFeedStatement FeedSpecification(Token startStmtToken) throws ParseException:
{
  Pair<DataverseName,Identifier> nameComponents = null;
  boolean ifNotExists = false;
  String adapterName = null;
  Map<String,String> properties = null;
  CreateFeedStatement stmt = null;
  Pair<Identifier,Identifier> sourceNameComponents = null;
  RecordConstructor withRecord = null;
}
{
  nameComponents = QualifiedName() ifNotExists = IfNotExists()
  <WITH> withRecord = RecordConstructor()
  {
    try {
      stmt = new CreateFeedStatement(nameComponents, withRecord, ifNotExists);
      return addSourceLocation(stmt, startStmtToken);
    } catch (AlgebricksException e) {
      throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage());
    }
  }
}

CreateFeedPolicyStatement CreateFeedPolicyStatement(Token startStmtToken) throws ParseException:
{
  CreateFeedPolicyStatement stmt = null;
}
{
  <INGESTION> <POLICY> stmt = FeedPolicySpecification(startStmtToken)
  {
    return stmt;
  }
}

CreateFeedPolicyStatement FeedPolicySpecification(Token startStmtToken) throws ParseException:
{
  String policyName = null;
  String basePolicyName = null;
  String sourcePolicyFile = null;
  String definition = null;
  boolean ifNotExists = false;
  Map<String,String> properties = null;
  CreateFeedPolicyStatement stmt = null;
}
{
  policyName = Identifier() ifNotExists = IfNotExists()
  <FROM>
  (<POLICY> basePolicyName = Identifier() properties = Configuration() (<DEFINITION> definition = ConstantString())?
    {
      stmt = new CreateFeedPolicyStatement(policyName, basePolicyName, properties, definition, ifNotExists);
    }
  | <PATH> sourcePolicyFile = ConstantString() (<DEFINITION> definition = ConstantString())?
    {
      stmt = new CreateFeedPolicyStatement(policyName, sourcePolicyFile, definition, ifNotExists);
    }
  )
  {
    return addSourceLocation(stmt, startStmtToken);
  }
}

CreateSynonymStatement CreateSynonymStatement(Token startStmtToken) throws ParseException:
{
  CreateSynonymStatement stmt = null;
}
{
  <SYNONYM> stmt = SynonymSpecification(startStmtToken)
  {
    return stmt;
  }
}

CreateSynonymStatement SynonymSpecification(Token startStmtToken) throws ParseException:
{
  Pair<DataverseName,Identifier> nameComponents = null;
  Pair<DataverseName,Identifier> objectNameComponents = null;
  boolean ifNotExists = false;
}
{
  nameComponents = QualifiedName() ifNotExists = IfNotExists()
  <FOR> objectNameComponents = QualifiedName()
  {
    CreateSynonymStatement stmt = new CreateSynonymStatement(nameComponents.first, nameComponents.second.getValue(),
      objectNameComponents.first, objectNameComponents.second.getValue(), ifNotExists);
    return addSourceLocation(stmt, startStmtToken);
  }
}

boolean IfNotExists() throws ParseException:
{
}
{
  ( LOOKAHEAD(1) <IF> <NOT> <EXISTS>
    {
      return true;
    }
  )?
    {
      return false;
    }
}

void ApplyFunction(List<FunctionSignature> funcSigs) throws ParseException:
{
  FunctionName functionName = null;
  String fqFunctionName = null;
}
{
  <APPLY> <FUNCTION> functionName = FunctionName()
  {
     fqFunctionName = functionName.library == null ? functionName.function : functionName.library + "#" + functionName.function;
     funcSigs.add(new FunctionSignature(functionName.dataverse, fqFunctionName, 1));
  }
  (
      <COMMA> functionName = FunctionName()
      {
        fqFunctionName = functionName.library == null ? functionName.function : functionName.library + "#" + functionName.function;
        funcSigs.add(new FunctionSignature(functionName.dataverse, fqFunctionName, 1));
      }
  )*
}

String GetPolicy() throws ParseException:
{
  String policy = null;
}
{
   <USING> <POLICY> policy = Identifier()
   {
     return policy;
   }

}

FunctionSignature FunctionSignature() throws ParseException:
{
  FunctionName fctName = null;
  Pair<Integer, List<Pair<VarIdentifier,TypeExpression>>> params = null;
  int arity = 0;
}
{
  fctName = FunctionName()
  (
    LOOKAHEAD(2) params = FunctionParameters() { arity = params.first; }
  | ( <LEFTPAREN> arity = FunctionArity() <RIGHTPAREN> )
  | ( <ATT> arity = FunctionArity() ) // back-compat
  )
  {
    return new FunctionSignature(fctName.dataverse, fctName.function, arity);
  }
}

int FunctionArity() throws ParseException:
{
  int arity;
}
{
  <INTEGER_LITERAL>
  {
    try {
      arity = Integer.parseInt(token.image);
    } catch (NumberFormatException e) {
      throw new SqlppParseException(getSourceLocation(token), "Invalid arity: " + token.image);
    }
    if (arity < 0 && arity != FunctionIdentifier.VARARGS) {
      throw new SqlppParseException(getSourceLocation(token), "Invalid arity: " + arity);
    }
    return arity;
  }
}

Pair<List<Integer>, List<List<String>>> PrimaryKey() throws ParseException:
{
  Pair<Integer, List<String>> tmp = null;
  List<Integer> keyFieldSourceIndicators = new ArrayList<Integer>();
  List<List<String>> primaryKeyFields = new ArrayList<List<String>>();
}
{
  <PRIMARY> <KEY> tmp = NestedField()
    {
      keyFieldSourceIndicators.add(tmp.first);
      primaryKeyFields.add(tmp.second);
    }
  ( <COMMA> tmp = NestedField()
    {
      keyFieldSourceIndicators.add(tmp.first);
      primaryKeyFields.add(tmp.second);
    }
  )*
    {
      return new Pair<List<Integer>, List<List<String>>> (keyFieldSourceIndicators, primaryKeyFields);
    }
}

Statement DropStatement() throws ParseException:
{
  Token startToken = null;
  Statement stmt = null;
}
{
  <DROP> { startToken = token; }
  (
    stmt = DropDatasetStatement(startToken)
    | stmt = DropIndexStatement(startToken)
    | stmt = DropNodeGroupStatement(startToken)
    | stmt = DropTypeStatement(startToken)
    | stmt = DropDataverseStatement(startToken)
    | stmt = DropAdapterStatement(startToken)
    | stmt = DropFunctionStatement(startToken)
    | stmt = DropFeedStatement(startToken)
    | stmt = DropFeedPolicyStatement(startToken)
    | stmt = DropSynonymStatement(startToken)
  )
  {
    return stmt;
  }
}

DropDatasetStatement DropDatasetStatement(Token startStmtToken) throws ParseException:
{
  DropDatasetStatement stmt = null;
}
{
  Dataset() stmt = DropDatasetSpecification(startStmtToken)
  {
    return stmt;
  }
}

DropDatasetStatement DropDatasetSpecification(Token startStmtToken) throws ParseException:
{
  Pair<DataverseName,Identifier> pairId = null;
  boolean ifExists = false;
}
{
  pairId = QualifiedName() ifExists = IfExists()
  {
    DropDatasetStatement stmt = new DropDatasetStatement(pairId.first, pairId.second, ifExists);
    return addSourceLocation(stmt, startStmtToken);
  }
}

IndexDropStatement DropIndexStatement(Token startStmtToken) throws ParseException:
{
  IndexDropStatement stmt = null;
}
{
  <INDEX> stmt = DropIndexSpecification(startStmtToken)
  {
    return stmt;
  }
}

IndexDropStatement DropIndexSpecification(Token startStmtToken) throws ParseException:
{
  Triple<DataverseName,Identifier,Identifier> tripleId = null;
  boolean ifExists = false;
}
{
  tripleId = DoubleQualifiedName() ifExists = IfExists()
  {
    IndexDropStatement stmt = new IndexDropStatement(tripleId.first, tripleId.second, tripleId.third, ifExists);
    return addSourceLocation(stmt, startStmtToken);
  }
}

NodeGroupDropStatement DropNodeGroupStatement(Token startStmtToken) throws ParseException:
{
  NodeGroupDropStatement stmt = null;
}
{
  <NODEGROUP> stmt = DropNodeGroupSpecification(startStmtToken)
  {
    return stmt;
  }
}

NodeGroupDropStatement DropNodeGroupSpecification(Token startStmtToken) throws ParseException:
{
  String id = null;
  boolean ifExists = false;
}
{
  id = Identifier() ifExists = IfExists()
  {
    NodeGroupDropStatement stmt = new NodeGroupDropStatement(new Identifier(id), ifExists);
    return addSourceLocation(stmt, startStmtToken);
  }
}

TypeDropStatement DropTypeStatement(Token startStmtToken) throws ParseException:
{
  TypeDropStatement stmt = null;
}
{
  <TYPE> stmt = DropTypeSpecification(startStmtToken)
  {
    return stmt;
  }
}

TypeDropStatement DropTypeSpecification(Token startStmtToken) throws ParseException:
{
  Pair<DataverseName,Identifier> pairId = null;
  boolean ifExists = false;
}
{
  pairId = TypeName() ifExists = IfExists()
  {
    TypeDropStatement stmt = new TypeDropStatement(pairId.first, pairId.second, ifExists);
    return addSourceLocation(stmt, startStmtToken);
  }
}

DataverseDropStatement DropDataverseStatement(Token startStmtToken) throws ParseException:
{
  DataverseDropStatement stmt = null;
}
{
  <DATAVERSE> stmt = DropDataverseSpecification(startStmtToken)
  {
    return stmt;
  }
}

DataverseDropStatement DropDataverseSpecification(Token startStmtToken) throws ParseException:
{
  List<String> multipartId = null;
  boolean ifExists = false;
}
{
  multipartId = MultipartIdentifier() ifExists = IfExists()
  {
    DataverseDropStatement stmt = new DataverseDropStatement(DataverseName.create(multipartId), ifExists);
    return addSourceLocation(stmt, startStmtToken);
  }
}

AdapterDropStatement DropAdapterStatement(Token startStmtToken) throws ParseException:
{
  AdapterDropStatement stmt = null;
}
{
  <ADAPTER> stmt = DropAdapterSpecification(startStmtToken)
  {
    return stmt;
  }
}

AdapterDropStatement DropAdapterSpecification(Token startStmtToken) throws ParseException:
{
  Pair<DataverseName,Identifier> adapterName = null;
  boolean ifExists = false;
}
{
  adapterName = QualifiedName() ifExists = IfExists()
  {
    AdapterDropStatement stmt = new AdapterDropStatement(adapterName.first, adapterName.second.getValue(), ifExists);
    return addSourceLocation(stmt, startStmtToken);
  }
}

FunctionDropStatement DropFunctionStatement(Token startStmtToken) throws ParseException:
{
  FunctionDropStatement stmt = null;
}
{
  <FUNCTION> stmt = DropFunctionSpecification(startStmtToken)
  {
    return stmt;
  }
}

FunctionDropStatement DropFunctionSpecification(Token startStmtToken) throws ParseException:
{
  FunctionSignature funcSig = null;
  boolean ifExists = false;
}
{
  funcSig = FunctionSignature() ifExists = IfExists()
  {
    FunctionDropStatement stmt = new FunctionDropStatement(funcSig, ifExists);
    return addSourceLocation(stmt, startStmtToken);
  }
}

FeedDropStatement DropFeedStatement(Token startStmtToken) throws ParseException:
{
  FeedDropStatement stmt = null;
}
{
  <FEED> stmt = DropFeedSpecification(startStmtToken)
  {
    return stmt;
  }
}

FeedDropStatement DropFeedSpecification(Token startStmtToken) throws ParseException:
{
  Pair<DataverseName,Identifier> pairId = null;
  boolean ifExists = false;
}
{
  pairId = QualifiedName() ifExists = IfExists()
  {
    FeedDropStatement stmt = new FeedDropStatement(pairId.first, pairId.second, ifExists);
    return addSourceLocation(stmt, startStmtToken);
  }
}

FeedPolicyDropStatement DropFeedPolicyStatement(Token startStmtToken) throws ParseException:
{
  FeedPolicyDropStatement stmt = null;
}
{
  <INGESTION> <POLICY> stmt = DropFeedPolicySpecification(startStmtToken)
  {
    return stmt;
  }
}

FeedPolicyDropStatement DropFeedPolicySpecification(Token startStmtToken) throws ParseException:
{
  Pair<DataverseName,Identifier> pairId = null;
  boolean ifExists = false;
}
{
  pairId = QualifiedName() ifExists = IfExists()
  {
    FeedPolicyDropStatement stmt = new FeedPolicyDropStatement(pairId.first, pairId.second, ifExists);
    return addSourceLocation(stmt, startStmtToken);
  }
}

SynonymDropStatement DropSynonymStatement(Token startStmtToken) throws ParseException:
{
  SynonymDropStatement stmt = null;
}
{
  <SYNONYM> stmt = DropSynonymSpecification(startStmtToken)
  {
    return stmt;
  }
}

SynonymDropStatement DropSynonymSpecification(Token startStmtToken) throws ParseException:
{
  Pair<DataverseName,Identifier> pairId = null;
  boolean ifExists = false;
}
{
  pairId = QualifiedName() ifExists = IfExists()
  {
    SynonymDropStatement stmt = new SynonymDropStatement(pairId.first, pairId.second.getValue(), ifExists);
    return addSourceLocation(stmt, startStmtToken);
  }
}

boolean IfExists() throws ParseException :
{
}
{
  ( LOOKAHEAD(1) <IF> <EXISTS>
    {
      return true;
    }
  )?
    {
      return false;
    }
}

InsertStatement InsertStatement() throws ParseException:
{
  Token startToken = null;
  Pair<DataverseName,Identifier> nameComponents = null;
  VariableExpr var = null;
  Query query = null;
  Expression returnExpression = null;
}
{
  <INSERT> { startToken = token; } <INTO> nameComponents = QualifiedName() (<AS> var = Variable())?
    query = Query()
    ( <RETURNING> returnExpression = Expression())?
    {
      if (var == null && returnExpression != null) {
        var = new VariableExpr(SqlppVariableUtil.toInternalVariableIdentifier(nameComponents.second.getValue()));
        addSourceLocation(var, startToken);
      }
      query.setTopLevel(true);
      InsertStatement stmt = new InsertStatement(nameComponents.first, nameComponents.second.getValue(), query,
        getVarCounter(), var, returnExpression);
      return addSourceLocation(stmt, startToken);
    }
}

UpsertStatement UpsertStatement() throws ParseException:
{
  Token startToken = null;
  Pair<DataverseName,Identifier> nameComponents = null;
  VariableExpr var = null;
  Query query = null;
  Expression returnExpression = null;
}
{
  <UPSERT> { startToken = token; } <INTO> nameComponents = QualifiedName() (<AS> var = Variable())?
    query = Query()
    ( <RETURNING> returnExpression = Expression())?
    {
      if (var == null && returnExpression != null) {
          var = new VariableExpr(SqlppVariableUtil.toInternalVariableIdentifier(nameComponents.second.getValue()));
          addSourceLocation(var, startToken);
      }
      query.setTopLevel(true);
      UpsertStatement stmt = new UpsertStatement(nameComponents.first, nameComponents.second.getValue(), query,
        getVarCounter(), var, returnExpression);
      return addSourceLocation(stmt, startToken);
    }
}

DeleteStatement DeleteStatement() throws ParseException:
{
  Token startToken = null;
  VariableExpr var = null;
  Expression condition = null;
  Pair<DataverseName, Identifier> nameComponents;
}
{
  <DELETE> { startToken = token; }
  <FROM> nameComponents  = QualifiedName() ((<AS>)? var = Variable())?
  (<WHERE> condition = Expression())?
  {
      if (var == null) {
        var = new VariableExpr(SqlppVariableUtil.toInternalVariableIdentifier(nameComponents.second.getValue()));
        addSourceLocation(var, startToken);
      }
      DeleteStatement stmt = new DeleteStatement(var, nameComponents.first, nameComponents.second.getValue(),
          condition, getVarCounter());
      return addSourceLocation(stmt, startToken);
  }
}

UpdateStatement UpdateStatement() throws ParseException:
{
  Token startToken = null;
  VariableExpr vars;
  Expression target;
  Expression condition;
  UpdateClause uc;
  List<UpdateClause> ucs = new ArrayList<UpdateClause>();
}
{
  <UPDATE> { startToken = token; } vars = Variable() <IN> target = Expression()
  <WHERE> condition = Expression()
  <LEFTPAREN> (uc = UpdateClause()
    {
      ucs.add(uc);
    }
  (<COMMA> uc = UpdateClause()
    {
      ucs.add(uc);
    }
  )*) <RIGHTPAREN>
    {
      UpdateStatement stmt = new UpdateStatement(vars, target, condition, ucs);
      return addSourceLocation(stmt, startToken);
    }
}

UpdateClause UpdateClause() throws ParseException:
{
  Expression target = null;
  Expression value = null ;
  InsertStatement is = null;
  DeleteStatement ds = null;
  UpdateStatement us = null;
  Expression condition = null;
  UpdateClause ifbranch = null;
  UpdateClause elsebranch = null;
}
{
   (<SET> target = Expression() <EQ> value = Expression()
   | is = InsertStatement()
   | ds = DeleteStatement()
   | us = UpdateStatement()
   | <IF> <LEFTPAREN> condition = Expression() <RIGHTPAREN>
     <THEN> ifbranch = UpdateClause()
     [LOOKAHEAD(1) <ELSE> elsebranch = UpdateClause()]
     {
       return new UpdateClause(target, value, is, ds, us, condition, ifbranch, elsebranch);
     }
   )
}

Statement SetStatement() throws ParseException:
{
  Token startToken = null;
  String pn = null;
  String pv = null;
}
{
  <SET> { startToken = token; } pn = Identifier() pv = ConstantString()
    {
      SetStatement stmt = new SetStatement(pn, pv);
      return addSourceLocation(stmt, startToken);
    }
}

Statement WriteStatement() throws ParseException:
{
  Token startToken = null;
  String nodeName = null;
  String fileName = null;
  Query query;
  String writerClass = null;
  Pair<Identifier,Identifier> nameComponents = null;
}
{
  <WRITE> { startToken = token; } <OUTPUT> <TO> nodeName = Identifier() <COLON> fileName = ConstantString()
    ( <USING> writerClass = ConstantString() )?
    {
      WriteStatement stmt = new WriteStatement(new Identifier(nodeName), fileName, writerClass);
      return addSourceLocation(stmt, startToken);
    }
}

LoadStatement LoadStatement() throws ParseException:
{
  Token startToken = null;
  DataverseName dataverseName = null;
  Identifier datasetName = null;
  boolean alreadySorted = false;
  String adapterName;
  Map<String,String> properties;
  Pair<DataverseName,Identifier> nameComponents = null;
}
{
  <LOAD> { startToken = token; } Dataset() nameComponents = QualifiedName()
    {
      dataverseName = nameComponents.first;
      datasetName = nameComponents.second;
    }
  <USING> adapterName = AdapterName() properties = Configuration()
  (<PRESORTED>
    {
      alreadySorted = true;
    }
  )?
    {
      LoadStatement stmt = new LoadStatement(dataverseName, datasetName.getValue(), adapterName, properties,
        alreadySorted);
      return addSourceLocation(stmt, startToken);
    }
}


String AdapterName() throws ParseException :
{
  String adapterName = null;
}
{
  adapterName = Identifier()
    {
      return adapterName;
    }
}

Statement CompactStatement() throws ParseException:
{
  Token startToken = null;
  Pair<DataverseName,Identifier> nameComponents = null;
}
{
  <COMPACT> { startToken = token; } Dataset() nameComponents = QualifiedName()
    {
      CompactStatement stmt = new CompactStatement(nameComponents.first, nameComponents.second);
      return addSourceLocation(stmt, startToken);
    }
}

Statement ConnectionStatement() throws ParseException:
{
  Token startToken = null;
  Statement stmt = null;
}
{
  (
    <CONNECT> { startToken = token; } stmt = ConnectStatement(startToken)
  | <DISCONNECT> { startToken = token; } stmt = DisconnectStatement(startToken)
  | <START> { startToken = token; } stmt = StartStatement(startToken)
  | <STOP> { startToken = token; } stmt = StopStatement(startToken)
  )
  {
    return stmt;
  }
}

Statement StartStatement(Token startStmtToken) throws ParseException:
{
  Pair<DataverseName,Identifier> feedNameComponents = null;
  AbstractStatement stmt = null;
}
{
  <FEED> feedNameComponents = QualifiedName()
  {
    stmt = new StartFeedStatement (feedNameComponents);
    return addSourceLocation(stmt, startStmtToken);
  }
}

AbstractStatement StopStatement(Token startStmtToken) throws ParseException:
{
  Pair<DataverseName,Identifier> feedNameComponents = null;
  AbstractStatement stmt = null;
}
{
  <FEED> feedNameComponents = QualifiedName()
  {
    stmt = new StopFeedStatement (feedNameComponents);
    return addSourceLocation(stmt, startStmtToken);
  }
}

AbstractStatement DisconnectStatement(Token startStmtToken) throws ParseException:
{
  Pair<DataverseName,Identifier> feedNameComponents = null;
  Pair<DataverseName,Identifier> datasetNameComponents = null;

  AbstractStatement stmt = null;
}
{
  (
    <FEED> feedNameComponents = QualifiedName() <FROM> Dataset() datasetNameComponents = QualifiedName()
      {
        stmt = new DisconnectFeedStatement(feedNameComponents, datasetNameComponents);
      }
  )
  {
    return addSourceLocation(stmt, startStmtToken);
  }
}

AbstractStatement ConnectStatement(Token startStmtToken) throws ParseException:
{
  Pair<DataverseName,Identifier> feedNameComponents = null;
  Pair<DataverseName,Identifier> datasetNameComponents = null;

  Map<String,String> configuration = null;
  List<FunctionSignature> appliedFunctions = new ArrayList<FunctionSignature>();
  AbstractStatement stmt = null;
  String policy = null;
  String whereClauseBody = null;
  WhereClause whereClause = null;
  Token beginPos = null;
  Token endPos = null;
}
{
  (
    <FEED> feedNameComponents = QualifiedName() <TO> Dataset() datasetNameComponents = QualifiedName()
    (ApplyFunction(appliedFunctions))?
    (policy = GetPolicy())?
    (
      <WHERE>
      {
        beginPos = token;
        whereClause = new WhereClause();
        Expression whereExpr;
      }
      whereExpr = Expression()
      {
        whereClause.setWhereExpr(whereExpr);
      }
    )?
    {
      if (whereClause != null) {
        endPos = token;
        whereClauseBody = extractFragment(beginPos.endLine, beginPos.endColumn, endPos.endLine, endPos.endColumn + 1);
      }
    }
      {
        stmt = new ConnectFeedStatement(feedNameComponents, datasetNameComponents, appliedFunctions,
         policy, whereClauseBody, getVarCounter());
      }
  )
  {
    return addSourceLocation(stmt, startStmtToken);
  }
}

Map<String,String> Configuration()  throws ParseException :
{
    Map<String,String> configuration = new LinkedHashMap<String,String>();
    Pair<String, String> keyValuePair = null;
}
{
  <LEFTPAREN> ( keyValuePair = KeyValuePair()
    {
      configuration.put(keyValuePair.first, keyValuePair.second);
    }
  ( <COMMA> keyValuePair = KeyValuePair()
    {
      configuration.put(keyValuePair.first, keyValuePair.second);
    }
  )* )? <RIGHTPAREN>
    {
      return configuration;
    }
}

Pair<String, String> KeyValuePair() throws ParseException:
{
  String key;
  String value;
}
{
  <LEFTPAREN> key = ConstantString()
  <EQ> ( value = ConstantString() | (<TRUE> | <FALSE>) {value = token.image.toLowerCase();} )
  <RIGHTPAREN>
    {
      return new Pair<String, String>(key, value);
    }
}

Map<String,String> Properties() throws ParseException:
{
  Map<String,String> properties = new HashMap<String,String>();
  Pair<String, String> property;
}
{
  (LOOKAHEAD(1) <LEFTPAREN> property = Property()
    {
      properties.put(property.first, property.second);
    }
  ( <COMMA> property = Property()
    {
      properties.put(property.first, property.second);
    }
  )* <RIGHTPAREN> )?
    {
      return properties;
    }
}

Pair<String, String> Property() throws ParseException:
{
  String key = null;
  String value = null;
}
{
  (key = Identifier() | key = StringLiteral())
  <EQ>
  ( value = ConstantString() | <INTEGER_LITERAL>
    {
      try {
        value = String.valueOf(Long.parseLong(token.image));
      } catch (NumberFormatException nfe) {
        throw new SqlppParseException(getSourceLocation(token), "inapproriate value: " + token.image);
      }
    }
  )
    {
      return new Pair<String, String>(key.toUpperCase(), value);
    }
}

IndexedTypeExpression IndexedTypeExpr() throws ParseException:
{
  TypeExpression typeExpr = null;
  boolean isUnknownable = false;
}
{
  typeExpr = TypeExpr(false)
  ( <QUES> { isUnknownable = true; } )?
  {
    return new IndexedTypeExpression(typeExpr, isUnknownable);
  }
}

TypeExpression TypeExpr(boolean allowRecordTypeDef) throws ParseException:
{
  TypeExpression typeExpr = null;
}
{
  (
    typeExpr = TypeReference()
    | typeExpr = OrderedListTypeDef(allowRecordTypeDef)
    | typeExpr = UnorderedListTypeDef(allowRecordTypeDef)
    | typeExpr = RecordTypeDef() {
      if (!allowRecordTypeDef) {
        throw new SqlppParseException(typeExpr.getSourceLocation(), "Unexpected record type declaration");
      }
    }
  )
  {
    return typeExpr;
  }
}

RecordTypeDefinition.RecordKind RecordTypeKind() throws ParseException:
{
  RecordTypeDefinition.RecordKind recordKind = null;
}
{
  (
    <CLOSED> { recordKind = RecordTypeDefinition.RecordKind.CLOSED; }
    | <OPEN> { recordKind = RecordTypeDefinition.RecordKind.OPEN; }
  )
  {
    return recordKind;
  }
}

RecordTypeDefinition RecordTypeDef() throws ParseException:
{
  Token startToken = null;
  RecordTypeDefinition recType = new RecordTypeDefinition();
  RecordTypeDefinition.RecordKind recordKind = RecordTypeDefinition.RecordKind.OPEN;
}
{
   ( recordKind = RecordTypeKind() )?
   <LEFTBRACE>
    {
      startToken = token;
      Token hintToken = fetchHint(token, SqlppHint.GEN_FIELDS_HINT);
      if (hintToken != null) {
        String hintParams = hintToken.hintParams;
        String[] splits = hintParams != null ? hintParams.split("\\s+") : null;
        if (splits == null || splits.length != 4) {
          throw new SqlppParseException(getSourceLocation(hintToken),
            "Expecting: /*+ gen-fields <type> <min> <max> <prefix>*/");
        }
        if (!splits[0].equals("int")) {
          throw new SqlppParseException(getSourceLocation(hintToken),
            "The only supported type for gen-fields is int.");
        }
        UndeclaredFieldsDataGen ufdg = new UndeclaredFieldsDataGen(UndeclaredFieldsDataGen.Type.INT,
          Integer.parseInt(splits[1]), Integer.parseInt(splits[2]), splits[3]);
          recType.setUndeclaredFieldsDataGen(ufdg);
      }
    }
    (
      RecordField(recType)
      ( <COMMA>  RecordField(recType) )*
    )?
   <RIGHTBRACE>
   {
      recType.setRecordKind(recordKind);
      return addSourceLocation(recType, startToken);
   }
}

void RecordField(RecordTypeDefinition recType) throws ParseException:
{
  String fieldName;
  TypeExpression type = null;
  boolean nullable = false, missable = false;
}
{
  fieldName = Identifier()
    {
      Token hintToken = fetchHint(token, SqlppHint.VAL_FILE_HINT, SqlppHint.VAL_FILE_SAME_INDEX_HINT,
        SqlppHint.LIST_VAL_FILE_HINT, SqlppHint.LIST_HINT, SqlppHint.INTERVAL_HINT, SqlppHint.INSERT_RAND_INT_HINT,
        SqlppHint.DATE_BETWEEN_YEARS_HINT, SqlppHint.DATETIME_ADD_RAND_HOURS_HINT, SqlppHint.AUTO_HINT);
      IRecordFieldDataGen rfdg = hintToken != null ? parseFieldDataGen(hintToken) : null;
    }
  <COLON> type = TypeExpr(true) ( <QUES> { nullable = true; missable = true; } )?
    {
      recType.addField(fieldName, type, nullable, missable, rfdg);
    }
}

TypeReferenceExpression TypeReference() throws ParseException:
{
  Pair<DataverseName,Identifier> id = null;
}
{
  id = QualifiedName()
  {
    if (id.first == null && id.second.getValue().equalsIgnoreCase(INT_TYPE_NAME)) {
      id.second = new Identifier(BuiltinType.AINT64.getTypeName());
    }

    TypeReferenceExpression typeRef = new TypeReferenceExpression(id);
    return addSourceLocation(typeRef, token);
  }
}

OrderedListTypeDefinition OrderedListTypeDef(boolean allowRecordTypeDef) throws ParseException:
{
  Token startToken = null;
  TypeExpression type = null;
}
{
  <LEFTBRACKET> { startToken = token; }
    ( type = TypeExpr(allowRecordTypeDef) )
  <RIGHTBRACKET>
  {
    OrderedListTypeDefinition typeDef = new OrderedListTypeDefinition(type);
    return addSourceLocation(typeDef, startToken);
  }
}

UnorderedListTypeDefinition UnorderedListTypeDef(boolean allowRecordTypeDef) throws ParseException:
{
  Token startToken = null;
  TypeExpression type = null;
}
{
  <LEFTDBLBRACE> { startToken = token; }
    ( type = TypeExpr(allowRecordTypeDef) )
  <RIGHTDBLBRACE>
  {
    UnorderedListTypeDefinition typeDef = new UnorderedListTypeDefinition(type);
    return addSourceLocation(typeDef, startToken);
  }
}

FunctionName FunctionName() throws ParseException:
{
  Triple<List<String>, SourceLocation, Token> prefix = null;
  String suffix = null;
}
{
  // Note: there's a copy of this production in PrimaryExpr() (LOOKAHEAD for FunctionCallExpr())
  //       that copy must be kept in sync with this code
  prefix = MultipartIdentifierWithHints(
    SqlppHint.INDEXED_NESTED_LOOP_JOIN_HINT, SqlppHint.RANGE_HINT,
    SqlppHint.SKIP_SECONDARY_INDEX_SEARCH_HINT, SqlppHint.USE_SECONDARY_INDEX_SEARCH_HINT
  )
  (<SHARP> suffix = Identifier())?
  {
    FunctionName result = new FunctionName();
    result.sourceLoc = prefix.second;
    result.hintToken = prefix.third;
    List<String> list = prefix.first;
    int ln = list.size();
    String last = list.get(ln - 1);
    if (suffix == null) {
      // prefix = (dv_part1.dv_part2...dv_partN.)?func_name
      // no library name
      result.function = last;
    } else {
      // prefix = (dv_part1.dv_part2...dv_partN.)?lib_name
      // suffix = func_name
      result.library = last;
      result.function = suffix;
    }
    if (ln > 1) {
      result.dataverse = DataverseName.create(list, 0, ln - 1);
    } else {
      result.dataverse = defaultDataverse;
    }

    if (result.function.equalsIgnoreCase(INT_TYPE_NAME)) {
       result.function = BuiltinType.AINT64.getTypeName();
    }
    return result;
  }
}

Pair<DataverseName,Identifier> TypeName() throws ParseException:
{
  Pair<DataverseName,Identifier> name = null;
}
{
  name = QualifiedName()
    {
      if (name.first == null) {
        name.first = defaultDataverse;
      }
      return name;
    }
}

String Identifier() throws ParseException:
{
  String lit = null;
}
{
  (<IDENTIFIER>
    {
      return token.image;
    }
  | lit = QuotedString()
    {
      return lit;
    }
  )
}

List<String> ParenthesizedIdentifierList() throws ParseException:
{
  List<String> list = new ArrayList<String>();
  String ident = null;
}
{
  <LEFTPAREN>
  ident = Identifier() { list.add(ident); }
  ( <COMMA> ident = Identifier() { list.add(ident); } )*
  <RIGHTPAREN>
  {
    return list;
  }
}

Pair<Integer, Pair<List<String>, IndexedTypeExpression>> OpenField() throws ParseException:
{
  IndexedTypeExpression fieldType = null;
  Pair<Integer, List<String>> fieldList = null;
}
{
  fieldList = NestedField()
  ( <COLON> fieldType = IndexedTypeExpr() )?
  {
    return new Pair<Integer, Pair<List<String>, IndexedTypeExpression>>
            (fieldList.first, new Pair<List<String>, IndexedTypeExpression>(fieldList.second, fieldType));
  }
}

Pair<Integer, List<String>> NestedField() throws ParseException:
{
  List<String> exprList = new ArrayList<String>();
  String lit = null;
  Token litToken = null;
  int source = 0;
}
{
  lit = Identifier()
  {
    boolean meetParens = false;
    litToken = token;
  }
  (
    LOOKAHEAD(1)
    <LEFTPAREN><RIGHTPAREN>
    {
        if(!lit.equalsIgnoreCase("meta")){
            throw new SqlppParseException(getSourceLocation(litToken), "The string before () has to be \"meta\".");
        }
        meetParens = true;
        source = 1;
    }
  )?
  {
    if(!meetParens){
        exprList.add(lit);
    }
  }
  (<DOT>
    lit = Identifier()
    {
      exprList.add(lit);
    }
  )*
  {
    return new Pair<Integer, List<String>>(source, exprList);
  }
}

String ConstantString() throws ParseException:
{
  String value = null;
}
{
  (value = QuotedString() | value = StringLiteral())
  {
     return value;
  }
}

String QuotedString() throws ParseException:
{
}
{
  <QUOTED_STRING>
    {
      return removeQuotesAndEscapes(token.image);
    }
}

String StringLiteral() throws ParseException:
{
}
{
  <STRING_LITERAL>
    {
      return removeQuotesAndEscapes(token.image);
    }
}

List<String> MultipartIdentifier() throws ParseException:
{
  Triple<List<String>, SourceLocation, Token> result = null;
}
{
  result = MultipartIdentifierWithHints(null)
  {
    return result.first;
  }
}

Triple<List<String>, SourceLocation, Token> MultipartIdentifierWithHints(SqlppHint... expectedHints)
  throws ParseException:
{
  List<String> list = new ArrayList<String>();
  SourceLocation sourceLoc = null;
  Token hint = null;
  String item = null;
}
{
  item = Identifier()
  {
    list.add(item);
    sourceLoc = getSourceLocation(token);
    if (expectedHints != null && expectedHints.length > 0) {
      hint = fetchHint(token, expectedHints);
    }
  }
  (<DOT> item = Identifier() { list.add(item); } )*
  {
    return new Triple<List<String>, SourceLocation, Token>(list, sourceLoc, hint);
  }
}

Pair<DataverseName,Identifier> QualifiedName() throws ParseException:
{
  List<String> list = null;
}
{
  list = MultipartIdentifier()
  {
    int len = list.size();
    DataverseName id1 = len > 1 ? DataverseName.create(list, 0, len - 1) : null;
    Identifier id2 = new Identifier(list.get(len - 1));
    return new Pair<DataverseName,Identifier>(id1, id2);
  }
}

Triple<DataverseName, Identifier, Identifier> DoubleQualifiedName() throws ParseException:
{
  List<String> list = new ArrayList<String>();
  String item = null;
}
{
  item = Identifier() { list.add(item); }
  (<DOT> item = Identifier() { list.add(item); } )+
  {
    int len = list.size();
    DataverseName id1 = len > 2 ? DataverseName.create(list, 0, len - 2) : null;
    Identifier id2 = new Identifier(list.get(len - 2));
    Identifier id3 = new Identifier(list.get(len - 1));
    return new Triple<DataverseName,Identifier,Identifier>(id1, id2, id3);
  }
}

FunctionDecl FunctionDeclaration() throws ParseException:
{
  Token startToken = null;
  String functionName;
  Pair<Integer, List<Pair<VarIdentifier,TypeExpression>>> paramsWithArity = null;
  Expression funcBody;
  createNewScope();
}
{
  <DECLARE> { startToken = token; } <FUNCTION>
    functionName = Identifier()
    paramsWithArity = FunctionParameters()
  <LEFTBRACE>
    funcBody = FunctionBody()
  <RIGHTBRACE>
  {
    int arity = paramsWithArity.first;
    List<Pair<VarIdentifier,TypeExpression>> paramList = paramsWithArity.second;
    FunctionSignature signature = new FunctionSignature(defaultDataverse, functionName, arity);
    getCurrentScope().addFunctionDescriptor(signature, false);
    ensureNoTypeDeclsInFunction(functionName, paramList, null, startToken);
    List<VarIdentifier> params = new ArrayList<VarIdentifier>(paramList.size());
    for (Pair<VarIdentifier,TypeExpression> p: paramList) {
        params.add(p.getFirst());
    }
    FunctionDecl stmt = new FunctionDecl(signature, params, funcBody);
    removeCurrentScope();
    return addSourceLocation(stmt, startToken);
  }
}

Query Query() throws ParseException:
{
  Query query = new Query();
  Expression expr;
}
{
  (
  expr = Expression()
  |
  expr = SelectExpression(false)
  )
  {
    query.setBody(expr);
    query.setSourceLocation(expr.getSourceLocation());
    return query;
  }
}


Expression Expression():
{
  Expression expr = null;
  Expression exprP = null;
}
{
(
    LOOKAHEAD(2)
    expr = OperatorExpr()
    | expr = QuantifiedExpression()
)
    {
      return (exprP==null) ? expr : exprP;
    }
}

Expression OperatorExpr() throws ParseException:
{
  OperatorExpr op = null;
  Expression operand = null;
}
{
    operand = AndExpr()
    (
      <OR>
      {
        if (op == null) {
          op = new OperatorExpr();
          op.addOperand(operand);
          op.setCurrentop(true);
          addSourceLocation(op, token);
        }
        try{
            op.addOperator(token.image.toLowerCase());
        } catch (Exception e){
            throw new SqlppParseException(getSourceLocation(token), e.getMessage());
        }
      }

      operand = AndExpr()
      {
        op.addOperand(operand);
      }

    )*

    {
      return op==null? operand: op;
    }
}

Expression AndExpr() throws ParseException:
{
  OperatorExpr op = null;
  Expression operand = null;
}
{
    operand = NotExpr()
    (
      <AND>
      {
        if (op == null) {
          op = new OperatorExpr();
          op.addOperand(operand);
          op.setCurrentop(true);
          addSourceLocation(op, token);
        }
        try{
           op.addOperator(token.image.toLowerCase());
        } catch (CompilationException e){
           throw new SqlppParseException(getSourceLocation(token), e.getMessage());
        }
    }

    operand = NotExpr()
    {
      op.addOperand(operand);
    }

    )*

    {
      return op==null ? operand: op;
    }
}

Expression NotExpr() throws ParseException:
{
   Expression inputExpr;
   boolean not = false;
   Token startToken = null;
}
{
  (LOOKAHEAD(2) <NOT> { not = true; startToken = token; } )? inputExpr = RelExpr()
  {
    if(not) {
        FunctionSignature signature = new FunctionSignature(BuiltinFunctions.NOT);
        CallExpr callExpr = new CallExpr(signature, new ArrayList<Expression>(Collections.singletonList(inputExpr)));
        return addSourceLocation(callExpr, startToken);
    } else {
        return inputExpr;
    }
  }
}

Expression RelExpr() throws ParseException:
{
  boolean not = false;
  OperatorExpr op = null;
  Expression operand = null;
  IExpressionAnnotation annotation = null;
}
{
    operand = BetweenExpr()

    (
      LOOKAHEAD(2)( <LT> | <GT> | <LE> | <GE> | <EQ> | <NE> | <LG> |<SIMILAR> | (<NOT> { not = true; })? <IN>)
        {
          Token hintToken = fetchHint(token,
            SqlppHint.HASH_BROADCAST_JOIN_HINT, SqlppHint.INDEXED_NESTED_LOOP_JOIN_HINT,
            SqlppHint.SKIP_SECONDARY_INDEX_SEARCH_HINT, SqlppHint.USE_SECONDARY_INDEX_SEARCH_HINT
          );
          if (hintToken != null) {
            annotation = parseExpressionAnnotation(hintToken);
          }
          String operator = token.image.toLowerCase();
          if (operator.equals("<>")){
              operator = "!=";
          }
          if (not) {
            operator = "not_" + operator;
          }
          if (op == null) {
            op = new OperatorExpr();
            op.addOperand(operand);
            op.setCurrentop(true);
            addSourceLocation(op, token);
          }
          try{
            op.addOperator(operator);
          } catch (CompilationException e){
            throw new SqlppParseException(getSourceLocation(token), e.getMessage());
          }
        }

       operand = BetweenExpr()
       {
         op.addOperand(operand);
       }
    )?

     {
       if (annotation != null) {
         op.addHint(annotation);
       }
       return op==null? operand: op;
     }
}

Expression BetweenExpr() throws ParseException:
{
  boolean not = false;
  OperatorExpr op = null;
  Expression operand = null;
  IExpressionAnnotation annotation = null;
}
{
    operand = IsExpr()
    (
      LOOKAHEAD(2)
      (<NOT> { not = true; })? <BETWEEN>
        {
          Token hintToken = fetchHint(token,
            SqlppHint.INDEXED_NESTED_LOOP_JOIN_HINT, SqlppHint.SKIP_SECONDARY_INDEX_SEARCH_HINT,
            SqlppHint.USE_SECONDARY_INDEX_SEARCH_HINT
          );
          if (hintToken != null) {
            annotation = parseExpressionAnnotation(hintToken);
          }
          String operator = token.image.toLowerCase();
          if(not){
            operator = "not_" + operator;
          }
          if (op == null) {
            op = new OperatorExpr();
            op.addOperand(operand);
            op.setCurrentop(true);
            addSourceLocation(op, token);
          }
          try{
            op.addOperator(operator);
          } catch (CompilationException e){
            throw new SqlppParseException(getSourceLocation(token), e.getMessage());
          }
        }

       operand = IsExpr()
       {
         op.addOperand(operand);
       }

       <AND>
       operand = IsExpr()
       {
         op.addOperator(OperatorType.AND);
         op.addOperand(operand);
       }
    )?

    {
       if (annotation != null) {
         op.addHint(annotation);
       }
       return op==null ? operand: op;
    }
}

Expression IsExpr() throws ParseException:
{
    Token notToken = null;
    CallExpr expr = null;
    Expression operand = null;
    boolean not = false;
    FunctionIdentifier fn = null;
}
{
    operand = LikeExpr()
    ( <IS>
        (<NOT> { not = true; notToken = token; })?
        (
            <NULL> { fn = BuiltinFunctions.IS_NULL; } |
            <MISSING> { fn = BuiltinFunctions.IS_MISSING; } |
            <UNKNOWN> { fn = BuiltinFunctions.IS_UNKNOWN; } |
            (<KNOWN> | <VALUED>) { not = !not; fn = BuiltinFunctions.IS_UNKNOWN; }
        )
      {
        FunctionSignature signature = new FunctionSignature(fn);
        expr = new CallExpr(signature, new ArrayList<Expression>(Collections.singletonList(operand)));
        addSourceLocation(expr, token);
        if (not) {
           FunctionSignature notSignature = new FunctionSignature(BuiltinFunctions.NOT);
           expr = new CallExpr(notSignature, new ArrayList<Expression>(Collections.singletonList(expr)));
           addSourceLocation(expr, notToken);
        }
      }
    )?
    {
        return expr == null ? operand : expr;
    }
}

Expression LikeExpr() throws ParseException:
{
  boolean not = false;
  OperatorExpr op = null;
  Expression operand = null;
}
{
    operand = ConcatExpr()
    (
        LOOKAHEAD(2)
        (<NOT> { not = true; })? <LIKE>
        {
          op = new OperatorExpr();
          op.addOperand(operand);
          op.setCurrentop(true);
          addSourceLocation(op, token);

          String operator = token.image.toLowerCase();
          if (not) {
            operator = "not_" + operator;
          }
          try {
            op.addOperator(operator);
          } catch (CompilationException e){
            throw new SqlppParseException(getSourceLocation(token), e.getMessage());
          }
        }

        operand = ConcatExpr()
        {
          op.addOperand(operand);
        }
     )?

     {
       return op == null ? operand : op;
     }
}

Expression ConcatExpr() throws ParseException:
{
  OperatorExpr op = null;
  Expression operand = null;
}
{
    operand = AddExpr()
    (
      LOOKAHEAD(1)
      <CONCAT>
      {
        if (op == null) {
          op = new OperatorExpr();
          op.addOperand(operand);
          op.setCurrentop(true);
          addSourceLocation(op, token);
        }
        op.addOperator(OperatorType.CONCAT);
    }
    operand = AddExpr()
    {
      op.addOperand(operand);
    }
    )*

    {
       return op == null ? operand : op;
    }
}

Expression AddExpr() throws ParseException:
{
  OperatorExpr op = null;
  OperatorType opType = null;
  Expression operand = null;
}
{
    operand = MultExpr()
    (
      LOOKAHEAD(1)
      (<PLUS> { opType = OperatorType.PLUS; } | <MINUS> { opType = OperatorType.MINUS; } )
      {
        if (op == null) {
          op = new OperatorExpr();
          op.addOperand(operand);
          op.setCurrentop(true);
          addSourceLocation(op, token);
        }
        op.addOperator(opType);
    }

    operand = MultExpr()
    {
      op.addOperand(operand);
    }
    )*

    {
       return op == null ? operand : op;
    }
}

Expression MultExpr() throws ParseException:
{
  OperatorExpr op = null;
  OperatorType opType = null;
  Expression operand = null;
}
{
    operand = ExponentExpr()

    ( (
        <MUL> { opType = OperatorType.MUL; } |
        <DIVIDE> { opType = OperatorType.DIVIDE; } |
        <DIV> { opType = OperatorType.DIV; } |
        ( <MOD> | <PERCENT> ) { opType = OperatorType.MOD; }
      )
      {
        if (op == null) {
          op = new OperatorExpr();
          op.addOperand(operand);
          op.setCurrentop(true);
          addSourceLocation(op, token);
        }
        op.addOperator(opType);
    }
    operand = ExponentExpr()
    {
       op.addOperand(operand);
    }
    )*

     {
       return op == null ? operand : op;
     }
}

Expression ExponentExpr() throws ParseException:
{
  OperatorExpr op = null;
  Expression operand = null;
}
{
    operand = UnaryExpr()
    (<CARET>
    {
        if (op == null) {
          op = new OperatorExpr();
          op.addOperand(operand);
          op.setCurrentop(true);
          addSourceLocation(op, token);
        }
        op.addOperator(OperatorType.CARET);
    }
    operand = UnaryExpr()
    {
       op.addOperand(operand);
    }
    )?
    {
       return op == null ? operand : op;
    }
}

Expression UnaryExpr() throws ParseException:
{
    boolean not = false;
    UnaryExpr uexpr = null;
    Expression expr = null;
}
{
    ( (<PLUS> | <MINUS> | (<NOT> { not = true; } )? <EXISTS> )
    {
        String exprType = token.image.toLowerCase();
        if (not) {
           exprType = "not_" + exprType;
        }
        uexpr = new UnaryExpr();
        addSourceLocation(uexpr, token);
        try {
            uexpr.setExprType(exprType);
        } catch (CompilationException e){
            throw new SqlppParseException(getSourceLocation(token), e.getMessage());
        }
    }
    )?

    expr = ValueExpr()
    {
       if (uexpr == null) {
         return expr;
       } else {
         uexpr.setExpr(expr);
         return uexpr;
       }
    }
}

Expression ValueExpr() throws ParseException:
{
  Expression expr = null;
  AbstractAccessor accessor = null;
}
{
  expr = PrimaryExpr()
  (
    accessor = FieldAccessor(accessor != null ? accessor : expr)
    |
    accessor = IndexAccessor(accessor != null ? accessor : expr)
  )*
  {
    return accessor == null ? expr : accessor;
  }
}

FieldAccessor FieldAccessor(Expression inputExpr) throws ParseException:
{
  Token startToken = null;
  String ident = null;
}
{
   <DOT> { startToken = token; } ident = Identifier()
    {
      FieldAccessor fa = new FieldAccessor(inputExpr, new Identifier(ident));
      return addSourceLocation(fa, startToken);
    }
}

AbstractAccessor IndexAccessor(Expression inputExpr) throws ParseException:
{
  Token startToken = null;
  boolean star = false, slice = false;
  Expression expr1 = null, expr2 = null;
}
{
  <LEFTBRACKET> { startToken = token; }
  (
    <MUL> { star = true; }
    |
    ( expr1 = Expression() ( <COLON> { slice = true; } ( expr2 = Expression() )? )? )
  )
  <RIGHTBRACKET>
  {
    if (expr1 != null && expr1.getKind() == Expression.Kind.LITERAL_EXPRESSION) {
      ensureIntegerLiteral( (LiteralExpr) expr1, "Index");
    }
    if (expr2 != null && expr2.getKind() == Expression.Kind.LITERAL_EXPRESSION) {
      ensureIntegerLiteral( (LiteralExpr) expr2, "Index");
    }
    AbstractAccessor resultAccessor;
    if (slice) {
      resultAccessor = new ListSliceExpression(inputExpr, expr1, expr2);
    } else if (star) {
      resultAccessor = new IndexAccessor(inputExpr, IndexAccessor.IndexKind.STAR, null);
    } else {
      resultAccessor = new IndexAccessor(inputExpr, IndexAccessor.IndexKind.ELEMENT, expr1);
    }
    return addSourceLocation(resultAccessor, startToken);
  }
}

Expression PrimaryExpr() throws ParseException:
{
  Expression expr = null;
}
{
  (
    LOOKAHEAD(Identifier() (<DOT> Identifier())* (<SHARP> Identifier())? <LEFTPAREN>) expr = FunctionCallExpr()
  | expr = CaseExpr()
  | expr = Literal()
  | expr = VariableRef()
  | expr = ExternalVariableRef()
  | expr = ListConstructor()
  | expr = RecordConstructor()
  | expr = ParenthesizedExpression()
  )
    {
      return expr;
    }
}

Expression Literal() throws ParseException:
{
  LiteralExpr lit = new LiteralExpr();
  String str = null;
}
{
  ( str = StringLiteral()
    {
      lit.setValue(new StringLiteral(str));
    }
  | <INTEGER_LITERAL>
    {
        try {
            lit.setValue(new LongIntegerLiteral(Long.valueOf(token.image)));
        } catch (NumberFormatException e) {
            throw new SqlppParseException(getSourceLocation(token), "Could not parse numeric literal \"" + LogRedactionUtil.userData(token.image) +'"');
        }
    }
  | <FLOAT_LITERAL>
    {
        try {
            lit.setValue(new FloatLiteral(Float.valueOf(token.image)));
        } catch (NumberFormatException e) {
            throw new SqlppParseException(getSourceLocation(token), "Could not parse numeric literal \"" + LogRedactionUtil.userData(token.image) +'"');
        }
    }
  | <DOUBLE_LITERAL>
    {
        try {
            lit.setValue(new DoubleLiteral(Double.valueOf(token.image)));
        } catch (NumberFormatException e) {
            throw new SqlppParseException(getSourceLocation(token), "Could not parse numeric literal \"" + LogRedactionUtil.userData(token.image) +'"');
        }
    }
  | <MISSING>
    {
      lit.setValue(MissingLiteral.INSTANCE);
    }
  | <NULL>
    {
      lit.setValue(NullLiteral.INSTANCE);
    }
  | <TRUE>
    {
      lit.setValue(TrueLiteral.INSTANCE);
    }
  | <FALSE>
    {
      lit.setValue(FalseLiteral.INSTANCE);
    }
  )
    {
      return addSourceLocation(lit, token);
    }
}

VariableExpr VariableRef() throws ParseException:
{
    String id = null;
}
{
    id = VariableIdentifier()
    {
     Identifier ident = lookupSymbol(id);
     if (isInForbiddenScopes(id)) {
       throw new SqlppParseException(getSourceLocation(token),
        "Inside limit clauses, it is disallowed to reference a variable having the same name as any variable bound in the same scope as the limit clause.");
     }
     VariableExpr varExp;
     if (ident != null) { // exist such ident
       varExp = new VariableExpr((VarIdentifier)ident);
     } else {
       varExp = new VariableExpr(new VarIdentifier(id));
       varExp.setIsNewVar(false);
     }
     return addSourceLocation(varExp, token);
    }
}

VariableExpr Variable() throws ParseException:
{
    String id = null;
}
{
    id = VariableIdentifier()
    {
     Identifier ident = lookupSymbol(id);
     VariableExpr varExp = new VariableExpr(new VarIdentifier(id));
     if (ident != null) { // exist such ident
       varExp.setIsNewVar(false);
     }
     return addSourceLocation(varExp, token);
    }
}

String VariableIdentifier() throws ParseException:
{
  String id = null;
}
{
  (<IDENTIFIER> { id = token.image; } | id = QuotedString())
  {
    return SqlppVariableUtil.toInternalVariableName(id); // prefix user-defined variables with "$".
  }
}

Pair<VariableExpr, List<Pair<Expression, Identifier>>> VariableWithFieldMap() throws ParseException:
{
  VariableExpr var = null;
  List<Pair<Expression, Identifier>> fieldList = new ArrayList<Pair<Expression, Identifier>>();
}
{
  var = Variable()
  ( LOOKAHEAD(1)
    {
      VariableExpr fieldVarExpr = null;
      String fieldIdentifierStr = null;
    }
    <LEFTPAREN>
    fieldVarExpr = VariableRef() <AS> fieldIdentifierStr = Identifier()
    {
      fieldList.add(new Pair<Expression, Identifier>(fieldVarExpr, new Identifier(fieldIdentifierStr)));
    }
    (<COMMA>
      fieldVarExpr = VariableRef() <AS> fieldIdentifierStr = Identifier()
      {
        fieldList.add(new Pair<Expression, Identifier>(fieldVarExpr, new Identifier(fieldIdentifierStr)));
      }
    )*
    <RIGHTPAREN>
  )?
  {
    return new Pair<VariableExpr, List<Pair<Expression, Identifier>>>(var, fieldList);
  }
}

VariableExpr ExternalVariableRef() throws ParseException:
{
  String name = null;
}
{
  (
      <DOLLAR_IDENTIFIER> { name = token.image.substring(1); }
    | <DOLLAR_INTEGER_LITERAL> { name = token.image.substring(1); }
    | <DOLLAR_QUOTED_STRING> { name = removeQuotesAndEscapes(token.image.substring(1)); }
    | <QUES> { name = String.valueOf(++externalVarCounter); }
  )
  {
     String idName = SqlppVariableUtil.toExternalVariableName(name);
     VarIdentifier id = new VarIdentifier(idName);
     VariableExpr varExp = new VariableExpr(id);
     return addSourceLocation(varExp, token);
  }
}

Expression ListConstructor() throws ParseException:
{
    Expression expr = null;
}
{
    (
      expr = OrderedListConstructor() |
      expr = UnorderedListConstructor()
    )
    {
      return expr;
    }
}

ListConstructor OrderedListConstructor() throws ParseException:
{
  Token startToken = null;
  List<Expression> exprList = null;
}
{
  <LEFTBRACKET> { startToken = token; }
  exprList = ExpressionList()
  <RIGHTBRACKET>
  {
    ListConstructor expr = new ListConstructor(ListConstructor.Type.ORDERED_LIST_CONSTRUCTOR, exprList);
    return addSourceLocation(expr, startToken);
  }
}

ListConstructor UnorderedListConstructor() throws ParseException:
{
  Token startToken = null;
  List<Expression> exprList = null;
}
{
  <LEFTDBLBRACE> { startToken = token; }
  exprList = ExpressionList()
  <RIGHTDBLBRACE>
  {
    ListConstructor expr = new ListConstructor(ListConstructor.Type.UNORDERED_LIST_CONSTRUCTOR, exprList);
    return addSourceLocation(expr, startToken);
  }
}

List<Expression> ExpressionList() throws ParseException:
{
  Expression expr = null;
  List<Expression> exprList = new ArrayList<Expression>();
}
{
  (
    expr = Expression()
    {
      exprList.add(expr);
    }
    ( <COMMA> expr = Expression()
      {
        exprList.add(expr);
      }
    )*
  )?
  {
    return exprList;
  }
}

RecordConstructor RecordConstructor() throws ParseException:
{
  Token startToken = null;
  FieldBinding fb = null;
  List<FieldBinding> fbList = new ArrayList<FieldBinding>();
}
{
  <LEFTBRACE> { startToken = token; }
  (
    fb = FieldBinding() { fbList.add(fb); }
    (<COMMA> fb = FieldBinding() { fbList.add(fb); })*
  )?
  <RIGHTBRACE>
  {
    RecordConstructor expr = new RecordConstructor(fbList);
    return addSourceLocation(expr, startToken);
  }
}

FieldBinding FieldBinding() throws ParseException:
{
  Expression left, right = null;
}
{
  left = Expression() ( <COLON> right = Expression() )?
  {
    if (right == null) {
        String generatedIdentifier = ExpressionToVariableUtil.getGeneratedIdentifier(left, false);
        if (generatedIdentifier == null) {
          throw new SqlppParseException(getSourceLocation(token), "Cannot infer field name");
        }
        String generatedName = SqlppVariableUtil.toUserDefinedName(generatedIdentifier);
        LiteralExpr generatedNameExpr = new LiteralExpr(new StringLiteral(generatedName));
        generatedNameExpr.setSourceLocation(left.getSourceLocation());
        return new FieldBinding(generatedNameExpr, left);
    } else {
      return new FieldBinding(left, right);
    }
  }
}

Expression FunctionCallExpr() throws ParseException:
{
  List<Expression> argList = new ArrayList<Expression>();
  Expression argExpr = null;
  FunctionName funcName = null;
  boolean star = false;
  boolean distinct = false;
  Expression filterExpr = null;
  WindowExpression windowExpr = null;
}
{
  funcName = FunctionName()
  <LEFTPAREN> (
    ( <DISTINCT> { distinct = true; } )?
    ( argExpr = Expression() | <MUL> { star = true; } )
    {
      if (star) {
        if (funcName.function.equalsIgnoreCase(BuiltinFunctions.SCALAR_COUNT.getName())) {
          argExpr = new LiteralExpr(new LongIntegerLiteral(1L));
        } else {
          throw new SqlppParseException(getSourceLocation(token),
            "The parameter * can only be used in " + BuiltinFunctions.SCALAR_COUNT.getName() + "().");
        }
      }
      argList.add(argExpr);
    }
  (<COMMA> argExpr = Expression()
    {
      argList.add(argExpr);
    }
  )*)? <RIGHTPAREN>

  {
    String name = funcName.function;
    if (distinct) {
      name += FunctionMapUtil.DISTINCT_AGGREGATE_SUFFIX;
    }
    String fqFunctionName = funcName.library == null ? name : funcName.library + "#" + name;
    int arity = argList.size();
    FunctionSignature signature = lookupFunctionSignature(funcName.dataverse, fqFunctionName, arity);
    if (signature == null) {
      signature = new FunctionSignature(funcName.dataverse, fqFunctionName, arity);
    }
  }

  ( <FILTER> <LEFTPAREN> <WHERE> filterExpr = Expression() <RIGHTPAREN> )?

  ( LOOKAHEAD(5) windowExpr = WindowExpr(signature, argList, filterExpr) )?

  {
    if (windowExpr != null) {
      return windowExpr;
    } else {
      CallExpr callExpr = new CallExpr(signature, argList, filterExpr);
      if (funcName.hintToken != null) {
        IExpressionAnnotation annotation = parseExpressionAnnotation(funcName.hintToken);
        if (annotation != null) {
          callExpr.addHint(annotation);
        }
      }
      FunctionMapUtil.normalizedListInputFunctions(callExpr);
      callExpr.setSourceLocation(funcName.sourceLoc);
      return callExpr;
    }
  }
}

WindowExpression WindowExpr(FunctionSignature signature, List<Expression> argList, Expression aggFilterExpr)
  throws ParseException:
{
  Boolean fromLast = null, ignoreNulls = null;
}
{
  (
    // FROM ( FIRST | LAST ) ( ( RESPECT | IGNORE ) NULLS )? OVER
    LOOKAHEAD(5, <FROM> <IDENTIFIER> ( <IDENTIFIER> <IDENTIFIER> )? <OVER>)
    <FROM> <IDENTIFIER>
    {
      if (isToken(FIRST)) {
        fromLast = false;
      } else if (isToken(LAST)) {
        fromLast = true;
      } else {
        throw createUnexpectedTokenError();
      }
    }
  )?
  (
    // ( RESPECT | IGNORE ) NULLS OVER
    LOOKAHEAD(3, <IDENTIFIER> <IDENTIFIER> <OVER>)
    <IDENTIFIER>
    {
      if (isToken(RESPECT)) {
        ignoreNulls = false;
      } else if (isToken(IGNORE)) {
        ignoreNulls = true;
      } else {
        throw createUnexpectedTokenError();
      }
    }
    <IDENTIFIER>
    {
      if (!isToken(NULLS)) {
        throw createUnexpectedTokenError();
      }
    }
  )?
  <OVER>
  {
    return OverClause(signature, argList, aggFilterExpr, token, fromLast, ignoreNulls);
  }
}

WindowExpression OverClause(FunctionSignature signature, List<Expression> argList, Expression aggFilterExpr,
    Token startToken, Boolean fromLast, Boolean ignoreNulls) throws ParseException:
{
  Expression partitionExpr = null;
  List<Expression> partitionExprs = new ArrayList<Expression>();
  OrderbyClause orderByClause = null;
  List<Expression> orderbyList = null;
  List<OrderbyClause.OrderModifier> orderbyModifierList = null;
  WindowExpression.FrameMode frameMode = null;
  Pair<WindowExpression.FrameBoundaryKind, Expression> frameStart = null, frameEnd = null;
  WindowExpression.FrameBoundaryKind frameStartKind = null, frameEndKind = null;
  Expression frameStartExpr = null, frameEndExpr = null;
  WindowExpression.FrameExclusionKind frameExclusionKind = null;
  Pair<VariableExpr, List<Pair<Expression, Identifier>>> windowVarWithFieldList = null;
  VariableExpr windowVar = null;
  List<Pair<Expression, Identifier>> windowFieldList = null;
}
{
  (
    windowVarWithFieldList = VariableWithFieldMap() <AS>
    {
      windowVar = windowVarWithFieldList.first;
      windowFieldList = windowVarWithFieldList.second;
    }
  )?
  <LEFTPAREN>
  (
    <IDENTIFIER> { expectToken(PARTITION); } <BY>
    partitionExpr = Expression() { partitionExprs.add(partitionExpr); }
    ( <COMMA> partitionExpr = Expression() { partitionExprs.add(partitionExpr); } )*
  )?
  (
    orderByClause = OrderbyClause()
    {
      orderbyList = orderByClause.getOrderbyList();
      orderbyModifierList = orderByClause.getModifierList();
    }
    (
      frameMode = WindowFrameMode()
      (
        frameStart = WindowFrameBoundary() |
        ( <BETWEEN> frameStart = WindowFrameBoundary() <AND> frameEnd = WindowFrameBoundary() )
      )
      ( frameExclusionKind = WindowFrameExclusion() )?
      {
        frameStartKind = frameStart.first;
        frameStartExpr = frameStart.second;
        if (frameEnd == null) {
          frameEndKind = WindowExpression.FrameBoundaryKind.CURRENT_ROW;
        } else {
          frameEndKind = frameEnd.first;
          frameEndExpr = frameEnd.second;
        }
        if (frameExclusionKind == null) {
          frameExclusionKind = WindowExpression.FrameExclusionKind.NO_OTHERS;
        }
      }
    )?
  )?
  <RIGHTPAREN>
  {
    WindowExpression winExpr = new WindowExpression(signature, argList, aggFilterExpr, partitionExprs, orderbyList,
      orderbyModifierList, frameMode, frameStartKind, frameStartExpr, frameEndKind, frameEndExpr, frameExclusionKind,
      windowVar, windowFieldList, ignoreNulls, fromLast);
    return addSourceLocation(winExpr, startToken);
  }
}

WindowExpression.FrameMode WindowFrameMode() throws ParseException:
{
}
{
  <IDENTIFIER>
  {
    if (isToken(RANGE)) {
      return WindowExpression.FrameMode.RANGE;
    } else if (isToken(ROWS)) {
      return WindowExpression.FrameMode.ROWS;
    } else if (isToken(GROUPS)) {
      return WindowExpression.FrameMode.GROUPS;
    } else {
      throw createUnexpectedTokenError();
    }
  }
}

Pair<WindowExpression.FrameBoundaryKind, Expression> WindowFrameBoundary() throws ParseException:
{
  boolean current = false;
  Expression expr = null;
}
{
  (
    LOOKAHEAD({ laIdentifier(CURRENT) || laIdentifier(UNBOUNDED) }) <IDENTIFIER> { current = isToken(CURRENT); }
    | expr = Expression()
  )
  <IDENTIFIER>
  {
    WindowExpression.FrameBoundaryKind kind;
    if (current && isToken(ROW)) {
      kind = WindowExpression.FrameBoundaryKind.CURRENT_ROW;
    } else if (!current && isToken(PRECEDING)) {
      kind = expr == null
        ? WindowExpression.FrameBoundaryKind.UNBOUNDED_PRECEDING
        : WindowExpression.FrameBoundaryKind.BOUNDED_PRECEDING;
    } else if (!current && isToken(FOLLOWING)) {
      kind = expr == null
        ? WindowExpression.FrameBoundaryKind.UNBOUNDED_FOLLOWING
        : WindowExpression.FrameBoundaryKind.BOUNDED_FOLLOWING;
    } else {
      throw createUnexpectedTokenError();
    }
    return new Pair<WindowExpression.FrameBoundaryKind, Expression>(kind, expr);
  }
}

WindowExpression.FrameExclusionKind WindowFrameExclusion() throws ParseException:
{
  boolean current = false, no = false;
}
{
  <IDENTIFIER>
  {
    expectToken(EXCLUDE);
  }
  (
    <GROUP>
    {
      return WindowExpression.FrameExclusionKind.GROUP;
    }
    |
    (
      <IDENTIFIER>
      {
        if (isToken(TIES)) {
          return WindowExpression.FrameExclusionKind.TIES;
        } else if (isToken(CURRENT)) {
          current = true;
        } else if (isToken(NO)) {
          no = true;
        } else {
          throw createUnexpectedTokenError();
        }
      }
      <IDENTIFIER>
      {
        if (current && isToken(ROW)) {
          return WindowExpression.FrameExclusionKind.CURRENT_ROW;
        } else if (no && isToken(OTHERS)) {
          return WindowExpression.FrameExclusionKind.NO_OTHERS;
        } else {
          throw createUnexpectedTokenError();
        }
      }
    )
  )
}

Expression ParenthesizedExpression() throws ParseException:
{
  Expression expr;
}
{
    (
    LOOKAHEAD(2)
    <LEFTPAREN> expr = Expression() <RIGHTPAREN>
    |
    expr = Subquery()
    )
    {
      return expr;
    }
}

Expression CaseExpr() throws ParseException:
{
   Token startToken = null;
   Expression conditionExpr = null;
   List<Expression> whenExprs = new ArrayList<Expression>();
   List<Expression> thenExprs = new ArrayList<Expression>();
   Expression elseExpr = null;
   Expression whenExpr = null;
   Expression thenExpr = null;
}
{
   <CASE> { startToken = token; }
   ( conditionExpr = Expression() )?
   (
     <WHEN> whenExpr = Expression()
     {
        whenExprs.add(whenExpr);
     }
     <THEN> thenExpr = Expression()
     {
        thenExprs.add(thenExpr);
     }
   )*
   (<ELSE> elseExpr = Expression() )?
   <END>
   {
     if (conditionExpr == null) {
        LiteralExpr litExpr = new LiteralExpr(TrueLiteral.INSTANCE);
        conditionExpr = addSourceLocation(litExpr, startToken);
     }
     CaseExpression caseExpr = new CaseExpression(conditionExpr, whenExprs, thenExprs, elseExpr);
     return addSourceLocation(caseExpr, startToken);
   }
}

SelectExpression SelectExpression(boolean subquery) throws ParseException:
{
  List<LetClause> letClauses = new ArrayList<LetClause>();
  SelectSetOperation selectSetOperation;
  OrderbyClause orderbyClause = null;
  LimitClause limitClause = null;
  createNewScope();
}
{
    ( letClauses = LetClause() )?
    selectSetOperation = SelectSetOperation()
    (orderbyClause = OrderbyClause() {})?
    (limitClause = LimitClause() {})?
    {
      SelectExpression selectExpr =
        new SelectExpression(letClauses, selectSetOperation, orderbyClause, limitClause, subquery);
      selectExpr.setSourceLocation((!letClauses.isEmpty() ? letClauses.get(0) : selectSetOperation).getSourceLocation());
      return selectExpr;
    }
}

SelectSetOperation SelectSetOperation() throws ParseException:
{
  SetOperationInput setOperationInputLeft;
  List<SetOperationRight> setOperationRights = new ArrayList<SetOperationRight>();
}
{
  {
      SelectBlock selectBlockLeft = null;
      SelectExpression subqueryLeft = null;
      Expression expr = null;
  }
  selectBlockLeft = SelectBlock()
  {
     setOperationInputLeft = new SetOperationInput(selectBlockLeft, subqueryLeft);
  }
  (
    {
      SetOpType opType = SetOpType.UNION;
      boolean setSemantics = true;
      SelectBlock selectBlockRight = null;
      SelectExpression subqueryRight = null;
    }
    (<UNION> {opType = SetOpType.UNION;} |<INTERSECT> {opType = SetOpType.INTERSECT;} |<EXCEPT> {opType = SetOpType.EXCEPT;}) (<ALL> {setSemantics = false;} )?
    (selectBlockRight = SelectBlock()| subqueryRight = Subquery())
    {
        setOperationRights.add(new SetOperationRight(opType, setSemantics, new SetOperationInput(selectBlockRight, subqueryRight)));
    }
  )*
  {
    SelectSetOperation selectSetOp = new SelectSetOperation(setOperationInputLeft, setOperationRights);
    selectSetOp.setSourceLocation(selectBlockLeft.getSourceLocation());
    return selectSetOp;
  }
}

SelectExpression Subquery() throws ParseException:
{
   SelectExpression selectExpr = null;
}
{
  <LEFTPAREN> selectExpr = SelectExpression(true) {} <RIGHTPAREN>
  {
    return selectExpr;
  }
}

SelectBlock SelectBlock() throws ParseException:
{
  SelectClause selectClause = null;
  FromClause fromClause = null;
  List<LetClause> fromLetClauses = null;
  WhereClause whereClause = null;
  List<AbstractClause> fromLetWhereClauses = new ArrayList<AbstractClause>();
  GroupbyClause groupbyClause = null;
  List<LetClause> gbyLetClauses = null;
  HavingClause havingClause = null;
  List<AbstractClause> gbyLetHavingClauses = new ArrayList<AbstractClause>();
  SourceLocation startSrcLoc = null;
}
{
  (
     selectClause = SelectClause() { startSrcLoc = selectClause.getSourceLocation(); }
     (
        (
          fromClause = FromClause()
          (
              fromLetClauses = LetClause()
          )?
        )
        |
        (
          fromLetClauses = LetClause()
          {
            // LET without FROM -> create dummy FROM clause: FROM {{missing}} AS #0
            SourceLocation sourceLoc = getSourceLocation(token);
            LiteralExpr missingExpr = new LiteralExpr(MissingLiteral.INSTANCE);
            missingExpr.setSourceLocation(sourceLoc);
            List<Expression> list = new ArrayList<Expression>(1);
            list.add(missingExpr);
            ListConstructor listExpr = new ListConstructor(ListConstructor.Type.ORDERED_LIST_CONSTRUCTOR, list);
            listExpr.setSourceLocation(sourceLoc);
            List<FromTerm> fromTerms = new ArrayList<FromTerm>(1);
            VariableExpr fromVar = new VariableExpr(new VarIdentifier("#0"));
            fromVar.setSourceLocation(sourceLoc);
            fromTerms.add(new FromTerm(listExpr, fromVar, null, new ArrayList<AbstractBinaryCorrelateClause>()));
            fromClause = new FromClause(fromTerms);
          }
        )
     )?
     (whereClause = WhereClause())?
     (
        groupbyClause = GroupbyClause()
        (
            gbyLetClauses = LetClause()
        )?
        (havingClause = HavingClause())?
     )?
    |
     fromClause = FromClause() { startSrcLoc = fromClause.getSourceLocation(); }
     (
        fromLetClauses = LetClause()
     )?
     (whereClause = WhereClause())?
     (
        groupbyClause = GroupbyClause()
        (
            gbyLetClauses = LetClause()
        )?
        (havingClause = HavingClause())?
     )?
     selectClause = SelectClause()
  )
  {
    if (fromLetClauses != null) {
      fromLetWhereClauses.addAll(fromLetClauses);
    }
    if (whereClause != null) {
      fromLetWhereClauses.add(whereClause);
    }
    if (gbyLetClauses != null) {
      gbyLetHavingClauses.addAll(gbyLetClauses);
    }
    if (havingClause != null) {
      gbyLetHavingClauses.add(havingClause);
    }
    SelectBlock selectBlock = new SelectBlock(selectClause, fromClause, fromLetWhereClauses, groupbyClause,
      gbyLetHavingClauses);
    selectBlock.setSourceLocation(startSrcLoc);
    return selectBlock;
  }
}

SelectClause SelectClause() throws ParseException:
{
  Token startToken = null;
  SelectRegular selectRegular = null;
  SelectElement selectElement = null;
  boolean distinct = false;
}
{
  <SELECT> { startToken = token; } (<ALL>|<DISTINCT> { distinct = true; } )?
  (
    selectRegular = SelectRegular()
    |
    selectElement = SelectElement()
  )?
  {
    SourceLocation sourceLoc = getSourceLocation(startToken);
    if (selectRegular == null && selectElement == null){
        Projection projection = new Projection(null, null, true, false);
        projection.setSourceLocation(sourceLoc);
        List<Projection> projections = new ArrayList<Projection>();
        projections.add(projection);
        selectRegular = new SelectRegular(projections);
        selectRegular.setSourceLocation(sourceLoc);
    }
    SelectClause selectClause = new SelectClause(selectElement, selectRegular, distinct);
    selectClause.setSourceLocation(sourceLoc);
    return selectClause;
  }
}

SelectRegular SelectRegular() throws ParseException:
{
  SourceLocation startSrcLoc = null;
  List<Projection> projections = new ArrayList<Projection>();
  Projection projection = null;
}
{
  projection = Projection()
  {
    projections.add(projection);
    startSrcLoc = projection.getSourceLocation();
  }
  ( LOOKAHEAD(2) <COMMA> projection = Projection()
    {
      projections.add(projection);
    }
  )*
  {
    SelectRegular selectRegular = new SelectRegular(projections);
    selectRegular.setSourceLocation(startSrcLoc);
    return selectRegular;
  }
}

SelectElement SelectElement() throws ParseException:
{
  Token startToken = null;
  Expression expr = null;
  String name = null;
}
{
  (<RAW>|<ELEMENT>|<VALUE>) { startToken = token; } expr = Expression()
  {
    SelectElement selectElement = new SelectElement(expr);
    return addSourceLocation(selectElement, startToken);
  }
}

Projection Projection() throws ParseException :
{
  SourceLocation startSrcLoc = null;
  Expression expr = null;
  Identifier identifier = null;
  String name = null;
  boolean star = false;
  boolean varStar = false;
}
{
  (
    <MUL> { star = true; startSrcLoc = getSourceLocation(token); }
    | LOOKAHEAD(3) expr = VariableRef() <DOT> <MUL> { varStar = true; }
    | expr = Expression() ((<AS>)? name = Identifier())?
      {
        if (name == null) {
          String generatedColumnIdentifier = ExpressionToVariableUtil.getGeneratedIdentifier(expr, false);
          if (generatedColumnIdentifier != null) {
            name = SqlppVariableUtil.toUserDefinedName(generatedColumnIdentifier);
          }
        }
      }
  )
  {
    Projection projection = new Projection(expr, name, star, varStar);
    projection.setSourceLocation(expr != null ? expr.getSourceLocation() : startSrcLoc);
    return projection;
  }
}

FromClause FromClause() throws ParseException :
{
  Token startToken = null;
  List<FromTerm> fromTerms = new ArrayList<FromTerm>();
  extendCurrentScope();
}
{
  {
    FromTerm fromTerm = null;
  }
    <FROM> { startToken = token; } fromTerm = FromTerm() { fromTerms.add(fromTerm); }
    (LOOKAHEAD(2) <COMMA> fromTerm = FromTerm() { fromTerms.add(fromTerm); } )*
  {
    FromClause fromClause = new FromClause(fromTerms);
    return addSourceLocation(fromClause, startToken);
  }
}

FromTerm FromTerm() throws ParseException :
{
  Expression leftExpr = null;
  VariableExpr leftVar = null;
  VariableExpr posVar = null;
  AbstractBinaryCorrelateClause correlateClause = null;
  List<AbstractBinaryCorrelateClause> correlateClauses = new ArrayList<AbstractBinaryCorrelateClause>();
}
{
  leftExpr = Expression() ((<AS>)? leftVar = Variable())? (<AT> posVar = Variable())?
  (
     (
      correlateClause = JoinOrUnnestClause(JoinType.INNER, UnnestType.INNER)
      | ( <INNER> correlateClause = JoinOrUnnestClause(JoinType.INNER, UnnestType.INNER) )
      | ( <LEFT> ( <OUTER> )? correlateClause = JoinOrUnnestClause(JoinType.LEFTOUTER, UnnestType.LEFTOUTER) )
      | ( <RIGHT> ( <OUTER> )? correlateClause = JoinClause(JoinType.RIGHTOUTER) )
     )
     {
        correlateClauses.add(correlateClause);
     }
  )*
  {
    if (leftVar == null) {
        leftVar = ExpressionToVariableUtil.getGeneratedVariable(leftExpr, true);
    }
    FromTerm fromTerm = new FromTerm(leftExpr, leftVar, posVar, correlateClauses);
    fromTerm.setSourceLocation(leftExpr.getSourceLocation());
    return fromTerm;
  }
}

AbstractBinaryCorrelateClause JoinOrUnnestClause(JoinType joinType, UnnestType unnestType) throws ParseException :
{
  AbstractBinaryCorrelateClause correlateClause = null;
}
{
  ( correlateClause = JoinClause(joinType) | correlateClause = UnnestClause(unnestType) )
  {
    return correlateClause;
  }
}

JoinClause JoinClause(JoinType joinType) throws ParseException :
{
    Token startToken = null;
    Expression rightExpr = null;
    VariableExpr rightVar = null;
    VariableExpr posVar = null;
    Expression conditionExpr = null;
}
{
  <JOIN> { startToken = token; } rightExpr = Expression() ((<AS>)? rightVar = Variable())? (<AT> posVar = Variable())? <ON> conditionExpr = Expression()
  {
    if (rightVar == null) {
      rightVar = ExpressionToVariableUtil.getGeneratedVariable(rightExpr, true);
    }
    JoinClause joinClause = new JoinClause(joinType, rightExpr, rightVar, posVar, conditionExpr);
    return addSourceLocation(joinClause, startToken);
  }
}

UnnestClause UnnestClause(UnnestType unnestType) throws ParseException :
{
    Token startToken = null;
    Expression rightExpr;
    VariableExpr rightVar;
    VariableExpr posVar = null;
}
{
  (<UNNEST>|<CORRELATE>|<FLATTEN>) { startToken = token; } rightExpr = Expression() ((<AS>)? rightVar = Variable()) (<AT> posVar = Variable())?
  {
    if (rightVar == null) {
      rightVar = ExpressionToVariableUtil.getGeneratedVariable(rightExpr, true);
    }
    UnnestClause unnestClause = new UnnestClause(unnestType, rightExpr, rightVar, posVar);
    return addSourceLocation(unnestClause, startToken);
  }
}

List<LetClause> LetClause() throws ParseException:
{
    List<LetClause> letList = new ArrayList<LetClause>();
    LetClause letClause;
}
{
    (
     (<LET>|<LETTING>) letClause = LetElement() { letList.add(letClause); } (LOOKAHEAD(1) <COMMA> letClause = LetElement() { letList.add(letClause); })*
     |
     <WITH> letClause = WithElement() { letList.add(letClause); } (LOOKAHEAD(1) <COMMA> letClause = WithElement() { letList.add(letClause); })*
    )
    {
      return letList;
    }
}

WhereClause WhereClause() throws ParseException :
{
  Token startToken = null;
  Expression whereExpr;
}
{
    <WHERE> { startToken = token; } whereExpr = Expression()
    {
      WhereClause wc = new WhereClause(whereExpr);
      return addSourceLocation(wc, startToken);
    }
}

OrderbyClause OrderbyClause() throws ParseException :
{
    Token startToken = null;
    OrderbyClause oc = new OrderbyClause();
    Expression orderbyExpr;
    List<Expression> orderbyList = new ArrayList<Expression>();
    List<OrderbyClause.OrderModifier> modifierList = new ArrayList<OrderbyClause.OrderModifier>();
    int numOfOrderby = 0;
}
{
    <ORDER>
      {
        startToken = token;
        Token hintToken = fetchHint(token, SqlppHint.INMEMORY_HINT, SqlppHint.RANGE_HINT);
        if (hintToken != null) {
          switch (hintToken.hint) {
            case INMEMORY_HINT:
              String[] splits = hintToken.hintParams.split("\\s+");
              int numFrames = Integer.parseInt(splits[0]);
              int numTuples = Integer.parseInt(splits[1]);
              oc.setNumFrames(numFrames);
              oc.setNumTuples(numTuples);
              break;
            case RANGE_HINT:
              try {
                Expression rangeExpr = parseExpression(hintToken.hintParams);
                RangeMap rangeMap = RangeMapBuilder.parseHint(rangeExpr);
                oc.setRangeMap(rangeMap);
              } catch (CompilationException e) {
                throw new SqlppParseException(getSourceLocation(hintToken), e.getMessage());
              }
              break;
          }
        }
      }
    <BY> orderbyExpr = Expression()
    {
      orderbyList.add(orderbyExpr);
      OrderbyClause.OrderModifier modif = OrderbyClause.OrderModifier.ASC;
    }
    ( (<ASC> { modif = OrderbyClause.OrderModifier.ASC; })
    | (<DESC> { modif = OrderbyClause.OrderModifier.DESC; }))?
    {
      modifierList.add(modif);
    }

    (LOOKAHEAD(2) <COMMA> orderbyExpr = Expression()
    {
      orderbyList.add(orderbyExpr);
      modif = OrderbyClause.OrderModifier.ASC;
    }
    ( (<ASC> { modif = OrderbyClause.OrderModifier.ASC; })
    | (<DESC> { modif = OrderbyClause.OrderModifier.DESC; }))?
    {
      modifierList.add(modif);
    }
    )*

    {
      oc.setModifierList(modifierList);
      oc.setOrderbyList(orderbyList);
      return addSourceLocation(oc, startToken);
    }
}

GroupbyClause GroupbyClause()throws ParseException :
{
    Token startToken = null;
    GroupbyClause gbc = new GroupbyClause();
    List<List<GbyVariableExpressionPair>> gbyList = null;
    List<GroupingElement> groupingElementList = null;
    Pair<VariableExpr, List<Pair<Expression, Identifier>>> groupVarWithFieldList = null;
    VariableExpr groupVar = null;
    List<Pair<Expression, Identifier>> groupFieldList = null;
}
{
    {
      Scope newScope = extendCurrentScopeNoPush(true);
      // extendCurrentScope(true);
    }
    <GROUP>
    {
      startToken = token;
      Token hintToken = fetchHint(token, SqlppHint.HASH_GROUP_BY_HINT);
      if (hintToken != null) {
        gbc.setHashGroupByHint(true);
      }
    }
    <BY> groupingElementList = GroupingElementList()
    (
      <GROUP> <AS> groupVarWithFieldList = VariableWithFieldMap()
      {
        groupVar = groupVarWithFieldList.first;
        groupFieldList = groupVarWithFieldList.second;
      }
    )?
    {
      if (groupingSetsParser == null) {
        groupingSetsParser = new SqlppGroupingSetsParser();
      }
      SourceLocation sourceLoc = getSourceLocation(startToken);
      try {
        gbyList = groupingSetsParser.parse(groupingElementList, sourceLoc);
      } catch (CompilationException e) {
        throw new SqlppParseException(sourceLoc, e.getMessage());
      }
      gbc.setGbyPairList(gbyList);
      gbc.setDecorPairList(new ArrayList<GbyVariableExpressionPair>());
      gbc.setWithVarMap(new HashMap<Expression, VariableExpr>());
      gbc.setGroupVar(groupVar);
      gbc.setGroupFieldList(groupFieldList);
      replaceCurrentScope(newScope);
      return addSourceLocation(gbc, startToken);
    }
}

List<GroupingElement> GroupingElementList() throws ParseException:
{
  List<GroupingElement> groupingElementList = new ArrayList<GroupingElement>();
  GroupingElement groupingElement = null;
}
{
  groupingElement = GroupingElement() { groupingElementList.add(groupingElement); }
  ( LOOKAHEAD(1) <COMMA> groupingElement = GroupingElement() { groupingElementList.add(groupingElement); } )*
  {
    return groupingElementList;
  }
}

GroupingElement GroupingElement() throws ParseException:
{
  GroupingElement groupingElement = null;
  List<GroupingSet> groupingSets = null;
  List<GroupingElement> groupingElements = null;
}
{
  (
    LOOKAHEAD(2)
    <LEFTPAREN> <RIGHTPAREN>
    {
      groupingElement = GroupingSet.EMPTY;
    }
    |
    LOOKAHEAD({ laIdentifier(ROLLUP) && laToken(2, LEFTPAREN) })
    <IDENTIFIER> { expectToken(ROLLUP); }
    <LEFTPAREN> groupingSets = OrdinaryGroupingSetList() <RIGHTPAREN>
    {
      groupingElement = new RollupCube(groupingSets, false);
    }
    |
    LOOKAHEAD({ laIdentifier(CUBE) && laToken(2, LEFTPAREN) })
    <IDENTIFIER> { expectToken(CUBE); }
    <LEFTPAREN> groupingSets = OrdinaryGroupingSetList() <RIGHTPAREN>
    {
      groupingElement = new RollupCube(groupingSets, true);
    }
    |
    LOOKAHEAD({ laIdentifier(GROUPING) && laIdentifier(2, SETS) && laToken(3, LEFTPAREN) })
    <IDENTIFIER> { expectToken(GROUPING); } <IDENTIFIER> { expectToken(SETS); }
    <LEFTPAREN> groupingElements = GroupingElementList() <RIGHTPAREN>
    {
      groupingElement = new GroupingSets(groupingElements);
    }
    |
    groupingElement = OrdinaryGroupingSet()
  )
  {
    return groupingElement;
  }
}

GroupingSet OrdinaryGroupingSet() throws ParseException:
{
  GbyVariableExpressionPair gbyExprPair = null;
  List<GbyVariableExpressionPair> items = null;
}
{
  (
    LOOKAHEAD(1) <LEFTPAREN> items = GbyVariableExpressionPairList() <RIGHTPAREN>
    | gbyExprPair = GbyVariableExpressionPair() { items = Collections.singletonList(gbyExprPair); }
  )
  {
    return new GroupingSet(items);
  }
}

List<GroupingSet> OrdinaryGroupingSetList() throws ParseException:
{
  GroupingSet groupingSet = null;
  List<GroupingSet> items = new ArrayList<GroupingSet>();
}
{
  groupingSet = OrdinaryGroupingSet() { items.add(groupingSet); }
  ( LOOKAHEAD(1) <COMMA> groupingSet = OrdinaryGroupingSet() { items.add(groupingSet); } )*
  {
    return items;
  }
}

List<GbyVariableExpressionPair> GbyVariableExpressionPairList() throws ParseException:
{
  GbyVariableExpressionPair gbyExprPair = null;
  List<GbyVariableExpressionPair> items = new ArrayList<GbyVariableExpressionPair>();
}
{
  gbyExprPair = GbyVariableExpressionPair() { items.add(gbyExprPair); }
  ( LOOKAHEAD(1) <COMMA> gbyExprPair = GbyVariableExpressionPair() { items.add(gbyExprPair); } )*
  {
    return items;
  }
}

GbyVariableExpressionPair GbyVariableExpressionPair() throws ParseException:
{
  Expression expr = null;
  VariableExpr var = null;
}
{
  expr = Expression() ( (<AS>)? var = Variable() )?
  {
    return new GbyVariableExpressionPair(var, expr);
  }
}

HavingClause HavingClause() throws ParseException:
{
   Token startToken = null;
   Expression filterExpr = null;
}
{
    <HAVING> { startToken = token; } filterExpr = Expression()
    {
       HavingClause havingClause = new HavingClause(filterExpr);
       return addSourceLocation(havingClause, startToken);
    }
}

LimitClause LimitClause() throws ParseException:
{
    Token startToken = null;
    Expression limitExpr = null, offsetExpr = null;
}
{
  (
    (
      <LIMIT> { startToken = token; pushForbiddenScope(getCurrentScope()); } limitExpr = Expression()
      ( <OFFSET> offsetExpr = Expression() )?
      { popForbiddenScope(); }
    )
    |
    (
      <OFFSET> { startToken = token; pushForbiddenScope(getCurrentScope()); } offsetExpr = Expression()
      { popForbiddenScope(); }
    )
  )
  {
    LimitClause lc = new LimitClause(limitExpr, offsetExpr);
    return addSourceLocation(lc, startToken);
  }
}

QuantifiedExpression QuantifiedExpression()throws ParseException:
{
  Token startToken = null;
  QuantifiedExpression qc = new QuantifiedExpression();
  List<QuantifiedPair> quantifiedList = new ArrayList<QuantifiedPair>();
  Expression satisfiesExpr;
  VariableExpr var;
  Expression inExpr;
  QuantifiedPair pair;
}
{
  {
    createNewScope();
  }

   ( ((<ANY>|<SOME>) { startToken = token; qc.setQuantifier(QuantifiedExpression.Quantifier.SOME); })
     | (<EVERY> {  startToken = token; qc.setQuantifier(QuantifiedExpression.Quantifier.EVERY); }))
    var = Variable() <IN> inExpr = Expression()
    {
      pair = new QuantifiedPair(var, inExpr);
      quantifiedList.add(pair);
    }
    (
    <COMMA> var = Variable() <IN> inExpr = Expression()
    {
      pair = new QuantifiedPair(var, inExpr);
      quantifiedList.add(pair);
    }
    )*
     <SATISFIES> satisfiesExpr = Expression() (<END>)?
     {
       qc.setSatisfiesExpr(satisfiesExpr);
       qc.setQuantifiedList(quantifiedList);
       removeCurrentScope();
       return addSourceLocation(qc, startToken);
     }
}

LetClause LetElement() throws ParseException:
{
    VariableExpr varExp;
    Expression beExp;
    extendCurrentScope();
}
{
    varExp = Variable() <EQ> beExp = Expression()
    {
      LetClause lc = new LetClause(varExp, beExp);
      lc.setSourceLocation(varExp.getSourceLocation());
      return lc;
    }
}

LetClause WithElement() throws ParseException:
{
    VariableExpr varExp;
    Expression beExp;
    extendCurrentScope();
}
{
    varExp = Variable() <AS> beExp = Expression()
    {
      LetClause lc = new LetClause(varExp, beExp);
      lc.setSourceLocation(varExp.getSourceLocation());
      return lc;
    }
}

TOKEN_MGR_DECLS:
{
    public int commentDepth = 0;
    public ArrayDeque<Integer> lexerStateStack = new ArrayDeque<Integer>();
    public Map<SourceLocation, String> hintCollector;

    public void pushState() {
      lexerStateStack.push( curLexState );
    }

    public void popState(String token) {
      if (lexerStateStack.size() > 0) {
         SwitchTo( lexerStateStack.pop() );
      } else {
         int errorLine = input_stream.getEndLine();
         int errorColumn = input_stream.getEndColumn();
         String msg = "Lexical error at line " + errorLine + ", column " + errorColumn + ". Encountered \"" + token
             + "\" but state stack is empty.";
         throw new TokenMgrError(msg, -1);
      }
    }

    void CommonTokenAction(Token token) {
      Token hintToken = token.specialToken;
      if (hintToken != null) {
        hintToken.sourceLocation = new SourceLocation(hintToken.beginLine, hintToken.beginColumn);
        String text = hintToken.image.substring(1).trim();
        boolean hintFound = hintToken.parseHint(text);
        hintCollector.put(hintToken.sourceLocation, hintFound ? hintToken.hint.getIdentifier() : hintToken.hintParams);
      }
    }
}

<DEFAULT,IN_DBL_BRACE>
TOKEN [IGNORE_CASE]:
{
  <ADAPTER: "adapter">
  | <ALL : "all">
  | <AND : "and">
  | <ANY : "any">
  | <APPLY : "apply">
  | <AS : "as">
  | <ASC : "asc">
  | <AT : "at">
  | <AUTOGENERATED : "autogenerated">
  | <BETWEEN : "between">
  | <BTREE : "btree">
  | <BY : "by">
  | <CASE : "case">
  | <CLOSED : "closed">
  | <CREATE : "create">
  | <COMPACTION : "compaction"> // no longer used
  | <COMPACT : "compact">
  | <CONNECT : "connect">
  | <CORRELATE : "correlate">
  | <DATASET : "dataset">
  | <COLLECTION : "collection">
  | <DATAVERSE : "dataverse">
  | <DECLARE : "declare">
  | <DEFINITION : "definition">
  | <DELETE : "delete">
  | <DESC : "desc">
  | <DISCONNECT : "disconnect">
  | <DISTINCT : "distinct">
  | <DIV : "div">
  | <DROP : "drop">
  | <ELEMENT : "element">
  | <EXPLAIN : "explain">
  | <ELSE : "else">
  | <ENFORCED : "enforced">
  | <END : "end">
  | <EVERY : "every">
  | <EXCEPT : "except">
  | <EXISTS : "exists">
  | <EXTERNAL : "external">
  | <FALSE : "false">
  | <FEED : "feed">
  | <FILTER : "filter">
  | <FLATTEN : "flatten">
  | <FOR : "for">
  | <FROM : "from">
  | <FULL : "full">
  | <FULLTEXT : "fulltext">
  | <FUNCTION : "function">
  | <GROUP : "group">
  | <HAVING : "having">
  | <HINTS : "hints">
  | <IF : "if">
  | <INTO : "into">
  | <IN : "in">
  | <INDEX : "index">
  | <INGESTION : "ingestion">
  | <INNER : "inner">
  | <INSERT : "insert">
  | <INTERNAL : "internal">
  | <INTERSECT : "intersect">
  | <IS : "is">
  | <JOIN : "join">
  | <KEYWORD : "keyword">
  | <KEY : "key">
  | <KNOWN : "known">
  | <LEFT : "left">
  | <LETTING : "letting">
  | <LET : "let">
  | <LIKE : "like">
  | <LIMIT : "limit">
  | <LOAD : "load">
  | <MISSING : "missing">
  | <MOD : "mod">
  | <NODEGROUP : "nodegroup">
  | <NGRAM : "ngram">
  | <NOT : "not">
  | <NULL : "null">
  | <OFFSET : "offset">
  | <ON : "on">
  | <OPEN : "open">
  | <OR : "or">
  | <ORDER : "order">
  | <OUTER : "outer">
  | <OUTPUT : "output">
  | <OVER: "over">
  | <PATH : "path">
  | <POLICY : "policy">
  | <PRESORTED : "pre-sorted">
  | <PRIMARY : "primary">
  | <RAW : "raw">
  | <REFRESH : "refresh">
  | <RETURN : "return">
  | <RETURNING : "returning">
  | <RIGHT : "right">
  | <RTREE : "rtree">
  | <RUN : "run">
  | <SATISFIES : "satisfies">
  | <SECONDARY : "secondary">
  | <SELECT : "select">
  | <SET : "set">
  | <SOME : "some">
  | <START : "start">
  | <STOP : "stop">
  | <SYNONYM : "synonym">
  | <TEMPORARY : "temporary"> // intentionally not used but reserved for future usage
  | <THEN : "then">
  | <TO : "to">
  | <TRUE : "true">
  | <TYPE : "type">
  | <UNION : "union">
  | <UNKNOWN : "unknown">
  | <UNNEST : "unnest">
  | <UPDATE : "update">
  | <UPSERT : "upsert">
  | <USE : "use">
  | <USING : "using">
  | <VALUE : "value">
  | <VALUED : "valued">
  | <WHEN : "when">
  | <WHERE : "where">
  | <WITH : "with">
  | <WRITE : "write">
}

<DEFAULT,IN_DBL_BRACE>
TOKEN :
{
    <CARET : "^">
  | <CONCAT : "||">
  | <DIVIDE : "/">
  | <MINUS : "-">
  | <MUL : "*">
  | <PLUS : "+">

  | <LEFTPAREN : "(">
  | <RIGHTPAREN : ")">
  | <LEFTBRACKET : "[">
  | <RIGHTBRACKET : "]">

  | <ATT : "@">
  | <COLON : ":">
  | <COMMA : ",">
  | <DOT : ".">
  | <PERCENT: "%">
  | <QUES : "?">
  | <SEMICOLON : ";">
  | <SHARP : "#">

  | <LT : "<">
  | <GT : ">">
  | <LE : "<=">
  | <GE : ">=">
  | <EQ : "=">
  | <NE : "!=">
  | <LG : "<>">
  | <SIMILAR : "~=">
}

<DEFAULT,IN_DBL_BRACE>
TOKEN :
{
    <LEFTBRACE : "{"> { pushState(); } : DEFAULT
}

<DEFAULT>
TOKEN :
{
    <RIGHTBRACE : "}"> { popState("}"); }
}

<DEFAULT,IN_DBL_BRACE>
TOKEN :
{
    <LEFTDBLBRACE : "{{"> { pushState(); } : IN_DBL_BRACE
}

<IN_DBL_BRACE>
TOKEN :
{
    <RIGHTDBLBRACE : "}}"> { popState("}}"); }
}

<DEFAULT,IN_DBL_BRACE>
TOKEN :
{
    <#DIGIT : ["0" - "9"]>
}

<DEFAULT,IN_DBL_BRACE>
TOKEN:
{
    <INTEGER_LITERAL : <DIGITS> >
  | <DOUBLE_LITERAL: <DIGITS> ( "." <DIGITS> ) (("e"|"E") ("+"|"-")? <DIGITS>)?
                      | <DIGITS> (("e"|"E") ("+"|"-")? <DIGITS>)
                      | "." <DIGITS> (("e"|"E") ("+"|"-")? <DIGITS>)?
    >
  | <FLOAT_LITERAL:  <DIGITS> ( "f" | "F" )
                      | <DIGITS> ( "." <DIGITS> ( "f" | "F" ) )?
                      | "." <DIGITS> ( "f" | "F" )
    >
  | <#DIGITS : (<DIGIT>)+ >
}

<DEFAULT,IN_DBL_BRACE>
TOKEN :
{
    <#LETTER : ["A" - "Z", "a" - "z"]>
  | <#IDENTIFIER_START_SPECIALCHAR : ["_"]>
  | <#IDENTIFIER_REST_SPECIALCHAR : ["$"]>
  | <#IDENTIFIER_START : <LETTER> | <IDENTIFIER_START_SPECIALCHAR> >
  | <#IDENTIFIER_REST  : <LETTER> | <DIGIT> | <IDENTIFIER_START_SPECIALCHAR> | <IDENTIFIER_REST_SPECIALCHAR> >
  | <IDENTIFIER : <IDENTIFIER_START> (<IDENTIFIER_REST>)* >
}

<DEFAULT,IN_DBL_BRACE>
TOKEN :
{
    // backslash u + 4 hex digits escapes are handled in the underlying JavaCharStream
    <QUOTED_STRING : "`" (
          <EscapeQuot>
        | <EscapeBslash>
        | <EscapeSlash>
        | <EscapeBspace>
        | <EscapeFormf>
        | <EscapeNl>
        | <EscapeCr>
        | <EscapeTab>
        | ~["`","\\"])* "`">
  | <STRING_LITERAL : ("\"" (
          <EscapeQuot>
        | <EscapeBslash>
        | <EscapeSlash>
        | <EscapeBspace>
        | <EscapeFormf>
        | <EscapeNl>
        | <EscapeCr>
        | <EscapeTab>
        | ~["\"","\\"])* "\"")
      | ("\'"(
          <EscapeApos>
        | <EscapeBslash>
        | <EscapeSlash>
        | <EscapeBspace>
        | <EscapeFormf>
        | <EscapeNl>
        | <EscapeCr>
        | <EscapeTab>
        | ~["\'","\\"])* "\'")>
  | < #EscapeQuot: "\\\"" >
  | < #EscapeApos: "\\\'" >
  | < #EscapeBslash: "\\\\" >
  | < #EscapeSlash: "\\/" >
  | < #EscapeBspace: "\\b" >
  | < #EscapeFormf: "\\f" >
  | < #EscapeNl: "\\n" >
  | < #EscapeCr: "\\r" >
  | < #EscapeTab: "\\t" >
}

<DEFAULT,IN_DBL_BRACE>
TOKEN :
{
    <DOLLAR_INTEGER_LITERAL : "$" <INTEGER_LITERAL> >
  | <DOLLAR_IDENTIFIER : "$" <IDENTIFIER> >
  | <DOLLAR_QUOTED_STRING: "$" <QUOTED_STRING> >
}

<DEFAULT,IN_DBL_BRACE>
SKIP:
{
    " "
  | "\t"
  | "\r"
  | "\n"
}

<DEFAULT,IN_DBL_BRACE>
SKIP:
{
    <"//" (~["\n"])* "\n">
}

<DEFAULT,IN_DBL_BRACE>
SKIP:
{
    <"//" (~["\n","\r"])* ("\n"|"\r"|"\r\n")?>
}

<DEFAULT,IN_DBL_BRACE>
SKIP:
{
    <"--" (~["\n"])* "\n">
}


<DEFAULT,IN_DBL_BRACE>
SKIP:
{
    <"--" (~["\n","\r"])* ("\n"|"\r"|"\r\n")?>
}

<DEFAULT,IN_DBL_BRACE>
SKIP:
{
    <"/*"> { pushState(); } : INSIDE_COMMENT
}

<INSIDE_COMMENT>
SPECIAL_TOKEN:
{
    <"+"(" ")*(~["*"])*>
}

<INSIDE_COMMENT>
SKIP:
{
    <"/*"> { pushState(); }
}

<INSIDE_COMMENT>
SKIP:
{
    <"*/"> { popState("*/"); }
  | <~[]>
}