asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj

//
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements.  See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership.  The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License.  You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied.  See the License for the
// specific language governing permissions and limitations
// under the License.
//
options {


       STATIC = false;

}


PARSER_BEGIN(SQLPPParser)

package org.apache.asterix.lang.sqlpp.parser;

// For SQL++ ParserTokenManager
import java.util.ArrayDeque;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.Reader;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;

import org.apache.asterix.common.annotations.AutoDataGen;
import org.apache.asterix.common.annotations.DateBetweenYearsDataGen;
import org.apache.asterix.common.annotations.DatetimeAddRandHoursDataGen;
import org.apache.asterix.common.annotations.DatetimeBetweenYearsDataGen;
import org.apache.asterix.common.annotations.FieldIntervalDataGen;
import org.apache.asterix.common.annotations.FieldValFileDataGen;
import org.apache.asterix.common.annotations.FieldValFileSameIndexDataGen;
import org.apache.asterix.common.annotations.IRecordFieldDataGen;
import org.apache.asterix.common.annotations.InsertRandIntDataGen;
import org.apache.asterix.common.annotations.ListDataGen;
import org.apache.asterix.common.annotations.ListValFileDataGen;
import org.apache.asterix.common.annotations.SkipSecondaryIndexSearchExpressionAnnotation;
import org.apache.asterix.common.annotations.TypeDataGen;
import org.apache.asterix.common.annotations.UndeclaredFieldsDataGen;
import org.apache.asterix.common.config.DatasetConfig.DatasetType;
import org.apache.asterix.common.config.DatasetConfig.IndexType;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.functions.FunctionConstants;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.lang.common.base.AbstractClause;
import org.apache.asterix.lang.common.base.AbstractLangExpression;
import org.apache.asterix.lang.common.base.AbstractStatement;
import org.apache.asterix.lang.common.base.Expression;
import org.apache.asterix.lang.common.base.Literal;
import org.apache.asterix.lang.common.base.IParser;
import org.apache.asterix.lang.common.base.ILangExpression;
import org.apache.asterix.lang.common.base.IParserFactory;
import org.apache.asterix.lang.common.base.Statement;
import org.apache.asterix.lang.common.clause.GroupbyClause;
import org.apache.asterix.lang.common.clause.LetClause;
import org.apache.asterix.lang.common.clause.LimitClause;
import org.apache.asterix.lang.common.clause.OrderbyClause;
import org.apache.asterix.lang.common.clause.UpdateClause;
import org.apache.asterix.lang.common.clause.WhereClause;
import org.apache.asterix.lang.common.context.RootScopeFactory;
import org.apache.asterix.lang.common.context.Scope;
import org.apache.asterix.lang.common.expression.AbstractAccessor;
import org.apache.asterix.lang.common.expression.CallExpr;
import org.apache.asterix.lang.common.expression.FieldAccessor;
import org.apache.asterix.lang.common.expression.FieldBinding;
import org.apache.asterix.lang.common.expression.GbyVariableExpressionPair;
import org.apache.asterix.lang.common.expression.IfExpr;
import org.apache.asterix.lang.common.expression.IndexAccessor;
import org.apache.asterix.lang.common.expression.IndexedTypeExpression;
import org.apache.asterix.lang.common.expression.ListConstructor;
import org.apache.asterix.lang.common.expression.ListSliceExpression;
import org.apache.asterix.lang.common.expression.LiteralExpr;
import org.apache.asterix.lang.common.expression.OperatorExpr;
import org.apache.asterix.lang.common.expression.OrderedListTypeDefinition;
import org.apache.asterix.lang.common.expression.QuantifiedExpression;
import org.apache.asterix.lang.common.expression.RecordConstructor;
import org.apache.asterix.lang.common.expression.RecordTypeDefinition;
import org.apache.asterix.lang.common.expression.TypeExpression;
import org.apache.asterix.lang.common.expression.TypeReferenceExpression;
import org.apache.asterix.lang.common.expression.UnaryExpr;
import org.apache.asterix.lang.common.expression.UnorderedListTypeDefinition;
import org.apache.asterix.lang.common.expression.VariableExpr;
import org.apache.asterix.lang.common.literal.DoubleLiteral;
import org.apache.asterix.lang.common.literal.FalseLiteral;
import org.apache.asterix.lang.common.literal.FloatLiteral;
import org.apache.asterix.lang.common.literal.LongIntegerLiteral;
import org.apache.asterix.lang.common.literal.MissingLiteral;
import org.apache.asterix.lang.common.literal.NullLiteral;
import org.apache.asterix.lang.common.literal.StringLiteral;
import org.apache.asterix.lang.common.literal.TrueLiteral;
import org.apache.asterix.lang.common.parser.ScopeChecker;
import org.apache.asterix.lang.common.statement.CompactStatement;
import org.apache.asterix.lang.common.statement.ConnectFeedStatement;
import org.apache.asterix.lang.common.statement.StartFeedStatement;
import org.apache.asterix.lang.common.statement.StopFeedStatement;
import org.apache.asterix.lang.common.statement.CreateDataverseStatement;
import org.apache.asterix.lang.common.statement.CreateFeedPolicyStatement;
import org.apache.asterix.lang.common.statement.CreateFeedStatement;
import org.apache.asterix.lang.common.statement.CreateFunctionStatement;
import org.apache.asterix.lang.common.statement.CreateIndexStatement;
import org.apache.asterix.lang.common.statement.DatasetDecl;
import org.apache.asterix.lang.common.statement.DataverseDecl;
import org.apache.asterix.lang.common.statement.DataverseDropStatement;
import org.apache.asterix.lang.common.statement.DeleteStatement;
import org.apache.asterix.lang.common.statement.DisconnectFeedStatement;
import org.apache.asterix.lang.common.statement.DropDatasetStatement;
import org.apache.asterix.lang.common.statement.ExternalDetailsDecl;
import org.apache.asterix.lang.common.statement.FeedDropStatement;
import org.apache.asterix.lang.common.statement.FeedPolicyDropStatement;
import org.apache.asterix.lang.common.statement.FunctionDecl;
import org.apache.asterix.lang.common.statement.FunctionDropStatement;
import org.apache.asterix.lang.common.statement.IndexDropStatement;
import org.apache.asterix.lang.common.statement.InsertStatement;
import org.apache.asterix.lang.common.statement.InternalDetailsDecl;
import org.apache.asterix.lang.common.statement.LoadStatement;
import org.apache.asterix.lang.common.statement.NodeGroupDropStatement;
import org.apache.asterix.lang.common.statement.NodegroupDecl;
import org.apache.asterix.lang.common.statement.Query;
import org.apache.asterix.lang.common.statement.RefreshExternalDatasetStatement;
import org.apache.asterix.lang.common.statement.SetStatement;
import org.apache.asterix.lang.common.statement.TypeDecl;
import org.apache.asterix.lang.common.statement.TypeDropStatement;
import org.apache.asterix.lang.common.statement.UpdateStatement;
import org.apache.asterix.lang.common.statement.UpsertStatement;
import org.apache.asterix.lang.common.statement.WriteStatement;
import org.apache.asterix.lang.common.struct.Identifier;
import org.apache.asterix.lang.common.struct.OperatorType;
import org.apache.asterix.lang.common.struct.QuantifiedPair;
import org.apache.asterix.lang.common.struct.VarIdentifier;
import org.apache.asterix.lang.common.util.RangeMapBuilder;
import org.apache.asterix.lang.sqlpp.clause.AbstractBinaryCorrelateClause;
import org.apache.asterix.lang.sqlpp.clause.FromClause;
import org.apache.asterix.lang.sqlpp.clause.FromTerm;
import org.apache.asterix.lang.sqlpp.clause.HavingClause;
import org.apache.asterix.lang.sqlpp.clause.JoinClause;
import org.apache.asterix.lang.sqlpp.clause.Projection;
import org.apache.asterix.lang.sqlpp.clause.SelectBlock;
import org.apache.asterix.lang.sqlpp.clause.SelectClause;
import org.apache.asterix.lang.sqlpp.clause.SelectElement;
import org.apache.asterix.lang.sqlpp.clause.SelectRegular;
import org.apache.asterix.lang.sqlpp.clause.SelectSetOperation;
import org.apache.asterix.lang.sqlpp.clause.UnnestClause;
import org.apache.asterix.lang.common.clause.WhereClause;
import org.apache.asterix.lang.sqlpp.expression.CaseExpression;
import org.apache.asterix.lang.sqlpp.expression.SelectExpression;
import org.apache.asterix.lang.sqlpp.expression.WindowExpression;
import org.apache.asterix.lang.sqlpp.optype.JoinType;
import org.apache.asterix.lang.sqlpp.optype.SetOpType;
import org.apache.asterix.lang.sqlpp.struct.SetOperationInput;
import org.apache.asterix.lang.sqlpp.struct.SetOperationRight;
import org.apache.asterix.lang.sqlpp.util.ExpressionToVariableUtil;
import org.apache.asterix.lang.sqlpp.util.FunctionMapUtil;
import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil;
import org.apache.asterix.om.functions.BuiltinFunctions;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.algebricks.common.utils.Triple;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionAnnotation;
import org.apache.hyracks.algebricks.core.algebra.expressions.IndexedNLJoinExpressionAnnotation;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.util.LogRedactionUtil;

class SQLPPParser extends ScopeChecker implements IParser {

    // tokens parsed as identifiers
    private static final String CURRENT = "CURRENT";
    private static final String EXCLUDE = "EXCLUDE";
    private static final String FOLLOWING = "FOLLOWING";
    private static final String GROUPS = "GROUPS";
    private static final String NO = "NO";
    private static final String OTHERS = "OTHERS";
    private static final String PARTITION = "PARTITION";
    private static final String PRECEDING = "PRECEDING";
    private static final String RANGE = "RANGE";
    private static final String ROW = "ROW";
    private static final String ROWS = "ROWS";
    private static final String TIES = "TIES";
    private static final String UNBOUNDED = "UNBOUNDED";

    // optimizer hints
    private static final String AUTO_HINT = "auto";
    private static final String BROADCAST_JOIN_HINT = "bcast";
    private static final String COMPOSE_VAL_FILES_HINT = "compose-val-files";
    private static final String DATE_BETWEEN_YEARS_HINT = "date-between-years";
    private static final String DATETIME_ADD_RAND_HOURS_HINT = "datetime-add-rand-hours";
    private static final String DATETIME_BETWEEN_YEARS_HINT = "datetime-between-years";
    private static final String HASH_GROUP_BY_HINT = "hash";
    private static final String INDEXED_NESTED_LOOP_JOIN_HINT = "indexnl";
    private static final String INMEMORY_HINT = "inmem";
    private static final String INSERT_RAND_INT_HINT = "insert-rand-int";
    private static final String INTERVAL_HINT = "interval";
    private static final String LIST_HINT = "list";
    private static final String LIST_VAL_FILE_HINT = "list-val-file";
    private static final String RANGE_HINT = "range";
    private static final String SKIP_SECONDARY_INDEX_SEARCH_HINT = "skip-index";
    private static final String VAL_FILE_HINT = "val-files";
    private static final String VAL_FILE_SAME_INDEX_HINT = "val-file-same-idx";
    private static final String GEN_FIELDS_HINT = "gen-fields";

    // data generator hints
    private static final String DGEN_HINT = "dgen";

    // error configuration
    protected static final boolean REPORT_EXPECTED_TOKENS = false;

    private int externalVarCounter;

    private static class IndexParams {
      public IndexType type;
      public int gramLength;

      public IndexParams(IndexType type, int gramLength) {
        this.type = type;
        this.gramLength = gramLength;
      }
    };

    private static class FunctionName {
       public String dataverse;
       public String library;
       public String function;
       public String hint;
       public SourceLocation sourceLoc;
    }

    private String getHint(Token t) {
        if (t.specialToken == null) {
            return null;
        }
        String s = t.specialToken.image;
        int n = s.length();
        if (n < 2) {
            return null;
        }
        return s.substring(1).trim();
    }

    private static IParser createNewParser(String statement) {
        return new SQLPPParser(statement);
    }

    private Token getHintToken(Token t) {
        return t.specialToken;
    }

    private IRecordFieldDataGen parseFieldDataGen(String hint, Token hintToken) throws ParseException {
      IRecordFieldDataGen rfdg = null;
      String splits[] = hint.split(" +");
      if (splits[0].equals(VAL_FILE_HINT)) {
        File[] valFiles = new File[splits.length - 1];
        for (int k=1; k<splits.length; k++) {
          valFiles[k-1] = new File(splits[k]);
        }
        rfdg = new FieldValFileDataGen(valFiles);
      } else if (splits[0].equals(VAL_FILE_SAME_INDEX_HINT)) {
        rfdg = new FieldValFileSameIndexDataGen(new File(splits[1]), splits[2]);
      } else if (splits[0].equals(LIST_VAL_FILE_HINT)) {
        rfdg = new ListValFileDataGen(new File(splits[1]), Integer.parseInt(splits[2]), Integer.parseInt(splits[3]));
      } else if (splits[0].equals(LIST_HINT)) {
        rfdg = new ListDataGen(Integer.parseInt(splits[1]), Integer.parseInt(splits[2]));
      } else if (splits[0].equals(INTERVAL_HINT)) {
        FieldIntervalDataGen.ValueType vt;
        if (splits[1].equals("int")) {
          vt = FieldIntervalDataGen.ValueType.INT;
        } else if (splits[1].equals("long")) {
          vt = FieldIntervalDataGen.ValueType.LONG;
        } else if (splits[1].equals("float")) {
          vt = FieldIntervalDataGen.ValueType.FLOAT;
        } else if (splits[1].equals("double")) {
          vt = FieldIntervalDataGen.ValueType.DOUBLE;
        } else {
          throw new SqlppParseException(getSourceLocation(hintToken), "Unknown type for interval data gen: " + splits[1]);
        }
        rfdg = new FieldIntervalDataGen(vt, splits[2], splits[3]);
      } else if (splits[0].equals(INSERT_RAND_INT_HINT)) {
        rfdg = new InsertRandIntDataGen(splits[1], splits[2]);
      } else if (splits[0].equals(DATE_BETWEEN_YEARS_HINT)) {
        rfdg = new DateBetweenYearsDataGen(Integer.parseInt(splits[1]), Integer.parseInt(splits[2]));
      } else if (splits[0].equals(DATETIME_BETWEEN_YEARS_HINT)) {
        rfdg = new DatetimeBetweenYearsDataGen(Integer.parseInt(splits[1]), Integer.parseInt(splits[2]));
      } else if (splits[0].equals(DATETIME_ADD_RAND_HOURS_HINT)) {
        rfdg = new DatetimeAddRandHoursDataGen(Integer.parseInt(splits[1]), Integer.parseInt(splits[2]), splits[3]);
      } else if (splits[0].equals(AUTO_HINT)) {
        rfdg = new AutoDataGen(splits[1]);
      }
      return rfdg;
    }

    public SQLPPParser(String s) {
        this(new StringReader(s));
        super.setInput(s);
    }

    public static void main(String args[]) throws ParseException, TokenMgrError, IOException, FileNotFoundException, CompilationException {
        File file = new File(args[0]);
        Reader fis = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
        SQLPPParser parser = new SQLPPParser(fis);
        List<Statement> st = parser.parse();
        //st.accept(new SQLPPPrintVisitor(), 0);
    }

    public List<Statement> parse() throws CompilationException {
        try {
            return Statement();
        } catch (Error e) {
            // this is here as the JavaCharStream that's below the lexer sometimes throws Errors that are not handled
            // by the ANTLR-generated lexer or parser (e.g it does this for invalid backslash u + 4 hex digits escapes)
            final String msg = e.getClass().getSimpleName() + (e.getMessage() != null ? ": " + e.getMessage() : "");
            throw new CompilationException(ErrorCode.PARSE_ERROR, msg);
        } catch (SqlppParseException e) {
            throw new CompilationException(ErrorCode.PARSE_ERROR, e.getSourceLocation(), getMessage(e));
        } catch (ParseException e) {
            throw new CompilationException(ErrorCode.PARSE_ERROR, getMessage(e));
        }
    }

    protected String getMessage(ParseException pe) {
        Token currentToken = pe.currentToken;
        if (currentToken == null) {
            return pe.getMessage();
        }
        int[][] expectedTokenSequences = pe.expectedTokenSequences;
        String[] tokenImage = pe.tokenImage;
        String sep = REPORT_EXPECTED_TOKENS ? eol : " ";
        StringBuilder expected = REPORT_EXPECTED_TOKENS ? new StringBuilder() : null;
        int maxSize = appendExpected(expected, expectedTokenSequences, tokenImage);
        Token tok = currentToken.next;
        int line = tok.beginLine;
        StringBuilder message = new StringBuilder(128);
        message.append("In line ").append(line).append(" >>").append(getLine(line)).append("<<").append(sep).append("Encountered ");
        for (int i = 0; i < maxSize; i++) {
            if (i != 0) {
                message.append(' ');
            }
            if (tok.kind == 0) {
                message.append(fixQuotes(tokenImage[0]));
                break;
            }
            final String fixedTokenImage = tokenImage[tok.kind];
            if (! tok.image.equalsIgnoreCase(stripQuotes(fixedTokenImage))) {
                message.append(fixQuotes(fixedTokenImage)).append(' ');
            }
            message.append(quot).append(addEscapes(tok.image)).append(quot);
            tok = tok.next;
        }
        message.append(" at column ").append(currentToken.next.beginColumn).append('.').append(sep);
        if (REPORT_EXPECTED_TOKENS) {
            if (expectedTokenSequences.length == 1) {
                message.append("Was expecting:").append(sep).append("    ");
            } else {
                message.append("Was expecting one of:").append(sep).append("    ");
            }
            message.append(expected);
        }
        return message.toString();
    }

    protected static SourceLocation getSourceLocation(Token token) {
        return token != null ? new SourceLocation(token.beginLine, token.beginColumn) : null;
    }

    protected static <T extends AbstractLangExpression> T addSourceLocation(T expr, Token token) {
        expr.setSourceLocation(getSourceLocation(token));
        return expr;
    }

    private boolean isToken(String image) {
      return token.image.equalsIgnoreCase(image);
    }

    private void expectToken(String image) throws SqlppParseException {
      if (!isToken(image)) {
        throw createUnexpectedTokenError();
      }
    }

    private SqlppParseException createUnexpectedTokenError() {
      return new SqlppParseException(getSourceLocation(token), "Unexpected token: " + LogRedactionUtil.userData(token.image));
    }

    private boolean laToken(int idx, int kind, String image) {
      Token t = getToken(idx);
      return t.kind == kind && t.image.equalsIgnoreCase(image);
    }

    private boolean laIdentifier(int idx, String image) {
      return laToken(idx, IDENTIFIER, image);
    }

    private boolean laIdentifier(String image) {
      return laIdentifier(1, image);
    }
}

PARSER_END(SQLPPParser)


List<Statement> Statement() throws ParseException:
{
  scopeStack.push(RootScopeFactory.createRootScope(this));
  List<Statement> decls = new ArrayList<Statement>();
  Statement stmt = null;
}
{
  (
    (stmt = SingleStatement()
      {
        decls.add(stmt);
      }
    )?
    (<SEMICOLON>)+
  )*
  <EOF>
  {
    return decls;
  }
}

Statement SingleStatement() throws ParseException:
{
  Statement stmt = null;
}
{
  (
    stmt = DataverseDeclaration()
    | stmt = FunctionDeclaration()
    | stmt = CreateStatement()
    | stmt = LoadStatement()
    | stmt = DropStatement()
    | stmt = WriteStatement()
    | stmt = SetStatement()
    | stmt = InsertStatement()
    | stmt = DeleteStatement()
    | stmt = UpdateStatement()
    | stmt = UpsertStatement()
    | stmt = ConnectionStatement()
    | stmt = CompactStatement()
    | stmt = ExplainStatement()
    | stmt = Query(false)
    | stmt = RefreshExternalDatasetStatement()
  )
  {
    return stmt;
  }
}

DataverseDecl DataverseDeclaration() throws ParseException:
{
  Token startToken = null;
  String dvName = null;
}
{
  <USE> { startToken = token; } dvName = Identifier()
    {
      defaultDataverse = dvName;
      DataverseDecl dvDecl = new DataverseDecl(new Identifier(dvName));
      return addSourceLocation(dvDecl, startToken);
    }
}

Statement CreateStatement() throws ParseException:
{
  Token startToken = null;
  String hint = null;
  Token hintToken = null;
  boolean hintDGen = false;
  Statement stmt = null;
}
{
  <CREATE> { startToken = token; }
  (
    {
      hint = getHint(token);
      if (hint != null) {
        hintToken = getHintToken(token);
        hintDGen = hint.startsWith(DGEN_HINT);
      }
    }
    stmt = TypeSpecification(startToken, hint, hintDGen, hintToken)
    | stmt = NodegroupSpecification(startToken)
    | stmt = DatasetSpecification(startToken)
    | stmt = IndexSpecification(startToken)
    | stmt = DataverseSpecification(startToken)
    | stmt = FunctionSpecification(startToken)
    | stmt = FeedSpecification(startToken)
    | stmt = FeedPolicySpecification(startToken)
  )
  {
    return stmt;
  }
}

TypeDecl TypeSpecification(Token startStmtToken, String hint, boolean dgen, Token hintToken) throws ParseException:
{
  Pair<Identifier,Identifier> nameComponents = null;
  boolean ifNotExists = false;
  TypeExpression typeExpr = null;
}
{
  <TYPE> nameComponents = TypeName() ifNotExists = IfNotExists()
  <AS> typeExpr = RecordTypeDef()
    {
      long numValues = -1;
      String filename = null;
      if (dgen) {
        String splits[] = hint.split(" +");
        if (splits.length != 3) {
          throw new SqlppParseException(getSourceLocation(hintToken), "Expecting /*+ dgen <filename> <numberOfItems> */");
        }
        filename = splits[1];
        numValues = Long.parseLong(splits[2]);
      }
      TypeDataGen tddg = new TypeDataGen(dgen, filename, numValues);
      TypeDecl stmt = new TypeDecl(nameComponents.first, nameComponents.second, typeExpr, tddg, ifNotExists);
      return addSourceLocation(stmt, startStmtToken);
    }
}

NodegroupDecl NodegroupSpecification(Token startStmtToken) throws ParseException:
{
  String name = null;
  String tmp = null;
  boolean ifNotExists = false;
  List<Identifier>ncNames = null;
}
{
  <NODEGROUP> name = Identifier()
  ifNotExists = IfNotExists() <ON> tmp = Identifier()
    {
      ncNames = new ArrayList<Identifier>();
      ncNames.add(new Identifier(tmp));
    }
  ( <COMMA> tmp = Identifier()
    {
      ncNames.add(new Identifier(tmp));
    }
  )*
    {
      NodegroupDecl stmt = new NodegroupDecl(new Identifier(name), ncNames, ifNotExists);
      return addSourceLocation(stmt, startStmtToken);
    }
}

DatasetDecl DatasetSpecification(Token startStmtToken) throws ParseException:
{
  Pair<Identifier,Identifier> nameComponents = null;
  boolean ifNotExists = false;
  Pair<Identifier,Identifier> typeComponents = null;
  String adapterName = null;
  Map<String,String> properties = null;
  FunctionSignature appliedFunction = null;
  Pair<List<Integer>, List<List<String>>> primaryKeyFields = null;
  String nodeGroupName = null;
  Map<String,String> hints = new HashMap<String,String>();
  DatasetDecl stmt = null;
  boolean autogenerated = false;
  Pair<Integer, List<String>> filterField = null;
  Pair<Identifier,Identifier> metaTypeComponents = new Pair<Identifier, Identifier>(null, null);
  RecordConstructor withRecord = null;
}
{
  (
    <EXTERNAL> Dataset() nameComponents = QualifiedName()
    <LEFTPAREN> typeComponents = TypeName() <RIGHTPAREN>
    ifNotExists = IfNotExists()
    <USING> adapterName = AdapterName() properties = Configuration()
    ( <ON> nodeGroupName = Identifier() )?
    ( <HINTS> hints = Properties() )?
    ( <WITH> withRecord = RecordConstructor() )?
      {
        ExternalDetailsDecl edd = new ExternalDetailsDecl();
        edd.setAdapter(adapterName);
        edd.setProperties(properties);
        try{
        stmt = new DatasetDecl(nameComponents.first,
                                   nameComponents.second,
                                   typeComponents.first,
                                   typeComponents.second,
                                   metaTypeComponents.first,
                                   metaTypeComponents.second,
                                   nodeGroupName != null? new Identifier(nodeGroupName): null,
                                   hints,
                                   DatasetType.EXTERNAL,
                                   edd,
                                   withRecord,
                                   ifNotExists);
        } catch (CompilationException e){
           throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage());
        }
      }

    | ( <INTERNAL> )?
    Dataset() nameComponents = QualifiedName()
    <LEFTPAREN> typeComponents = TypeName() <RIGHTPAREN>
    (
        { String name; }
        <WITH>
        name = Identifier()
        {
            if (!name.equalsIgnoreCase("meta")){
                throw new SqlppParseException(getSourceLocation(startStmtToken),
                    "We can only support one additional associated field called \"meta\".");
            }
        }
        <LEFTPAREN> metaTypeComponents = TypeName() <RIGHTPAREN>
    )?
    ifNotExists = IfNotExists()
    primaryKeyFields = PrimaryKey()
    (<AUTOGENERATED> { autogenerated = true; } )?
    (<ON> nodeGroupName = Identifier() )?
    ( <HINTS> hints = Properties() )?
    ( LOOKAHEAD(2) <WITH> <FILTER> <ON>  filterField = NestedField() )?
    ( <WITH> withRecord = RecordConstructor() )?
      {
        if(filterField!=null && filterField.first!=0){
          throw new SqlppParseException(getSourceLocation(startStmtToken),
            "A filter field can only be a field in the main record of the dataset.");
        }
        InternalDetailsDecl idd = new InternalDetailsDecl(primaryKeyFields.second,
                                                          primaryKeyFields.first,
                                                          autogenerated,
                                                          filterField == null? null : filterField.second);
        try{
        stmt = new DatasetDecl(nameComponents.first,
                                   nameComponents.second,
                                   typeComponents.first,
                                   typeComponents.second,
                                   metaTypeComponents.first,
                                   metaTypeComponents.second,
                                   nodeGroupName != null ? new Identifier(nodeGroupName) : null,
                                   hints,
                                   DatasetType.INTERNAL,
                                   idd,
                                   withRecord,
                                   ifNotExists);
        } catch (CompilationException e){
           throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage());
        }
      }
  )
    {
      return addSourceLocation(stmt, startStmtToken);
    }
}

RefreshExternalDatasetStatement RefreshExternalDatasetStatement() throws ParseException:
{
  Token startToken = null;
  Pair<Identifier,Identifier> nameComponents = null;
  String datasetName = null;
}
{
    <REFRESH> { startToken = token; } <EXTERNAL> Dataset() nameComponents = QualifiedName()
    {
      RefreshExternalDatasetStatement stmt = new RefreshExternalDatasetStatement();
      stmt.setDataverseName(nameComponents.first);
      stmt.setDatasetName(nameComponents.second);
      return addSourceLocation(stmt, startToken);
    }
}

CreateIndexStatement IndexSpecification(Token startStmtToken) throws ParseException:
{
  CreateIndexStatement stmt = new CreateIndexStatement();
  String indexName = null;
  boolean ifNotExists = false;
  Pair<Identifier,Identifier> nameComponents = null;
  Pair<Integer, Pair<List<String>, IndexedTypeExpression>> fieldPair = null;
  IndexParams indexType = null;
  boolean enforced = false;
  boolean isPrimaryIdx = false;
}
{
  (
    (<INDEX> indexName = Identifier()
    ifNotExists = IfNotExists()
    <ON> nameComponents = QualifiedName()
    <LEFTPAREN> ( fieldPair = OpenField()
      {
        stmt.addFieldExprPair(fieldPair.second);
        stmt.addFieldIndexIndicator(fieldPair.first);
      }
    ) (<COMMA> fieldPair = OpenField()
      {
        stmt.addFieldExprPair(fieldPair.second);
        stmt.addFieldIndexIndicator(fieldPair.first);
      }
    )* <RIGHTPAREN> ( <TYPE> indexType = IndexType() )? ( <ENFORCED> { enforced = true; } )?)
    |
    (<PRIMARY> <INDEX> {isPrimaryIdx = true;}
      (
        (indexName = Identifier())? ifNotExists = IfNotExists()
      )
      <ON> nameComponents = QualifiedName() (<TYPE> <BTREE>)?
    )
  )
  {
    if (isPrimaryIdx && indexName == null) {
      indexName = "primary_idx_" + nameComponents.second;
    }
    stmt.setIndexName(new Identifier(indexName));
    stmt.setIfNotExists(ifNotExists);
    stmt.setDataverseName(nameComponents.first);
    stmt.setDatasetName(nameComponents.second);
    if (indexType != null) {
      stmt.setIndexType(indexType.type);
      stmt.setGramLength(indexType.gramLength);
    }
    stmt.setEnforced(enforced);
    return addSourceLocation(stmt, startStmtToken);
  }
}

String CompactionPolicy() throws ParseException :
{
  String compactionPolicy = null;
}
{
  compactionPolicy = Identifier()
    {
      return compactionPolicy;
    }
}

String FilterField() throws ParseException :
{
  String filterField = null;
}
{
  filterField = Identifier()
    {
      return filterField;
    }
}

IndexParams IndexType() throws ParseException:
{
  IndexType type = null;
  int gramLength = 0;
}
{
  (<BTREE>
    {
      type = IndexType.BTREE;
    }
  | <RTREE>
    {
      type = IndexType.RTREE;
    }
  | <KEYWORD>
    {
      type = IndexType.LENGTH_PARTITIONED_WORD_INVIX;
    }
  |<FULLTEXT>
    {
      type = IndexType.SINGLE_PARTITION_WORD_INVIX;
    }
  | <NGRAM> <LEFTPAREN> <INTEGER_LITERAL>
    {
      type = IndexType.LENGTH_PARTITIONED_NGRAM_INVIX;
      gramLength = Integer.valueOf(token.image);
    }
  <RIGHTPAREN>)
    {
      return new IndexParams(type, gramLength);
    }
}

CreateDataverseStatement DataverseSpecification(Token startStmtToken) throws ParseException :
{
  String dvName = null;
  boolean ifNotExists = false;
}
{
  <DATAVERSE> dvName = Identifier()
  ifNotExists = IfNotExists()
    {
      CreateDataverseStatement stmt = new CreateDataverseStatement(new Identifier(dvName), null, ifNotExists);
      return addSourceLocation(stmt, startStmtToken);
    }
}

CreateFunctionStatement FunctionSpecification(Token startStmtToken) throws ParseException:
{
  FunctionSignature signature;
  boolean ifNotExists = false;
  List<VarIdentifier> paramList = new ArrayList<VarIdentifier>();
  String functionBody;
  VarIdentifier var = null;
  Expression functionBodyExpr;
  Token beginPos;
  Token endPos;
  FunctionName fctName = null;
  String currentDataverse = defaultDataverse;

  createNewScope();
}
{
  <FUNCTION> fctName = FunctionName()
  {
     defaultDataverse = fctName.dataverse;
  }
  ifNotExists = IfNotExists()
  paramList = ParameterList()
  <LEFTBRACE>
  {
     beginPos = token;
  }
  (functionBodyExpr = SelectExpression(true) | functionBodyExpr = Expression())
  <RIGHTBRACE>
    {
      endPos = token;
      functionBody = extractFragment(beginPos.beginLine, beginPos.beginColumn, endPos.beginLine, endPos.beginColumn);
      // TODO use fctName.library
      signature = new FunctionSignature(fctName.dataverse, fctName.function, paramList.size());
      getCurrentScope().addFunctionDescriptor(signature, false);
      removeCurrentScope();
      defaultDataverse = currentDataverse;
      CreateFunctionStatement stmt = new CreateFunctionStatement(signature, paramList, functionBody, functionBodyExpr, ifNotExists);
      return addSourceLocation(stmt, startStmtToken);
    }
}

CreateFeedStatement FeedSpecification(Token startStmtToken) throws ParseException:
{
  Pair<Identifier,Identifier> nameComponents = null;
  boolean ifNotExists = false;
  String adapterName = null;
  Map<String,String> properties = null;
  CreateFeedStatement stmt = null;
  Pair<Identifier,Identifier> sourceNameComponents = null;
  RecordConstructor withRecord = null;
}
{
  <FEED> nameComponents = QualifiedName() ifNotExists = IfNotExists()
  <WITH> withRecord = RecordConstructor()
  {
    try {
      stmt = new CreateFeedStatement(nameComponents, withRecord, ifNotExists);
      return addSourceLocation(stmt, startStmtToken);
    } catch (AlgebricksException e) {
      throw new SqlppParseException(getSourceLocation(startStmtToken), e.getMessage());
    }
  }
}

CreateFeedPolicyStatement FeedPolicySpecification(Token startStmtToken) throws ParseException:
{
  String policyName = null;
  String basePolicyName = null;
  String sourcePolicyFile = null;
  String definition = null;
  boolean ifNotExists = false;
  Map<String,String> properties = null;
  CreateFeedPolicyStatement stmt = null;
}
{
  (
    <INGESTION> <POLICY>  policyName = Identifier() ifNotExists = IfNotExists()
      <FROM>
      (<POLICY> basePolicyName = Identifier() properties = Configuration() (<DEFINITION> definition = ConstantString())?
      {
        stmt = new CreateFeedPolicyStatement(policyName,
                                   basePolicyName, properties, definition, ifNotExists);
      }
     | <PATH> sourcePolicyFile = ConstantString() (<DEFINITION> definition = ConstantString())?
       {
        stmt = new CreateFeedPolicyStatement(policyName, sourcePolicyFile, definition, ifNotExists);
       }
     )
  )
    {
      return addSourceLocation(stmt, startStmtToken);
    }
}

List<VarIdentifier> ParameterList() throws ParseException:
{
  List<VarIdentifier> paramList = new ArrayList<VarIdentifier>();
  VarIdentifier var = null;
}
{
  <LEFTPAREN> (<IDENTIFIER>
    {
      var = SqlppVariableUtil.toInternalVariableIdentifier(token.image);
      paramList.add(var);
    }
  (<COMMA> <IDENTIFIER>
    {
      var = SqlppVariableUtil.toInternalVariableIdentifier(token.image);
      paramList.add(var);
    }
  )*)? <RIGHTPAREN>
    {
      return paramList;
    }
}

boolean IfNotExists() throws ParseException:
{
}
{
  ( LOOKAHEAD(1) <IF> <NOT> <EXISTS>
    {
      return true;
    }
  )?
    {
      return false;
    }
}

void ApplyFunction(List<FunctionSignature> funcSigs) throws ParseException:
{
  FunctionName functioName = null;
  String fqFunctionName = null;
}
{
  <APPLY> <FUNCTION> functioName = FunctionName()
    {
       fqFunctionName = functioName.library == null ? functioName.function : functioName.library + "#" + functioName.function;
       funcSigs.add(new FunctionSignature(functioName.dataverse, fqFunctionName, 1));
    }
  (
      <COMMA> functioName = FunctionName()
      {
        fqFunctionName = functioName.library == null ? functioName.function : functioName.library + "#" + functioName.function;
        funcSigs.add(new FunctionSignature(functioName.dataverse, fqFunctionName, 1));
      }
  )*
}

String GetPolicy() throws ParseException:
{
  String policy = null;
}
{
   <USING> <POLICY> policy = Identifier()
   {
     return policy;
   }

}

FunctionSignature FunctionSignature() throws ParseException:
{
  FunctionName fctName = null;
  int arity = 0;
}
{
  fctName = FunctionName() <ATT> <INTEGER_LITERAL>
    {
      arity = new Integer(token.image);
      if (arity < 0 && arity != FunctionIdentifier.VARARGS) {
        throw new SqlppParseException(getSourceLocation(token), "Invalid arity:" + arity);
      }

      // TODO use fctName.library
      String fqFunctionName = fctName.library == null ? fctName.function : fctName.library + "#" + fctName.function;
      return new FunctionSignature(fctName.dataverse, fqFunctionName, arity);
    }
}

Pair<List<Integer>, List<List<String>>> PrimaryKey() throws ParseException:
{
  Pair<Integer, List<String>> tmp = null;
  List<Integer> keyFieldSourceIndicators = new ArrayList<Integer>();
  List<List<String>> primaryKeyFields = new ArrayList<List<String>>();
}
{
  <PRIMARY> <KEY> tmp = NestedField()
    {
      keyFieldSourceIndicators.add(tmp.first);
      primaryKeyFields.add(tmp.second);
    }
  ( <COMMA> tmp = NestedField()
    {
      keyFieldSourceIndicators.add(tmp.first);
      primaryKeyFields.add(tmp.second);
    }
  )*
    {
      return new Pair<List<Integer>, List<List<String>>> (keyFieldSourceIndicators, primaryKeyFields);
    }
}

Statement DropStatement() throws ParseException:
{
  Token startToken = null;
  String id = null;
  Pair<Identifier,Identifier> pairId = null;
  Triple<Identifier,Identifier,Identifier> tripleId = null;
  FunctionSignature funcSig = null;
  boolean ifExists = false;
  AbstractStatement stmt = null;
}
{
  <DROP> { startToken = token; }
  (
    Dataset() pairId = QualifiedName() ifExists = IfExists()
      {
        stmt = new DropDatasetStatement(pairId.first, pairId.second, ifExists);
      }
    | <INDEX> tripleId = DoubleQualifiedName() ifExists = IfExists()
      {
        stmt = new IndexDropStatement(tripleId.first, tripleId.second, tripleId.third, ifExists);
      }
    | <NODEGROUP> id = Identifier() ifExists = IfExists()
      {
        stmt = new NodeGroupDropStatement(new Identifier(id), ifExists);
      }
    | <TYPE> pairId = TypeName() ifExists = IfExists()
      {
        stmt = new TypeDropStatement(pairId.first, pairId.second, ifExists);
      }
    | <DATAVERSE> id = Identifier() ifExists = IfExists()
      {
        stmt = new DataverseDropStatement(new Identifier(id), ifExists);
      }
    | <FUNCTION> funcSig = FunctionSignature() ifExists = IfExists()
      {
        stmt = new FunctionDropStatement(funcSig, ifExists);
      }
    | <FEED> pairId = QualifiedName() ifExists = IfExists()
      {
        stmt = new FeedDropStatement(pairId.first, pairId.second, ifExists);
      }
    | <INGESTION> <POLICY> pairId = QualifiedName() ifExists = IfExists()
      {
        stmt = new FeedPolicyDropStatement(pairId.first, pairId.second, ifExists);
      }
  )
  {
    return addSourceLocation(stmt, startToken);
  }
}

boolean IfExists() throws ParseException :
{
}
{
  ( LOOKAHEAD(1) <IF> <EXISTS>
    {
      return true;
    }
  )?
    {
      return false;
    }
}

InsertStatement InsertStatement() throws ParseException:
{
  Token startToken = null;
  Pair<Identifier,Identifier> nameComponents = null;
  VariableExpr var = null;
  Query query = null;
  Expression returnExpression = null;
}
{
  <INSERT> { startToken = token; } <INTO> nameComponents = QualifiedName() (<AS> var = Variable())?
    query = Query(false)
    ( <RETURNING> returnExpression = Expression())?
    {
      if (returnExpression != null && var == null) {
        var = ExpressionToVariableUtil.getGeneratedVariable(query.getBody(), true);
      }
      query.setTopLevel(true);
      InsertStatement stmt = new InsertStatement(nameComponents.first, nameComponents.second, query, getVarCounter(),
        var, returnExpression);
      return addSourceLocation(stmt, startToken);
    }
}

UpsertStatement UpsertStatement() throws ParseException:
{
  Token startToken = null;
  Pair<Identifier,Identifier> nameComponents = null;
  VariableExpr var = null;
  Query query = null;
  Expression returnExpression = null;
}
{
  <UPSERT> { startToken = token; } <INTO> nameComponents = QualifiedName() (<AS> var = Variable())?
    query = Query(false)
    ( <RETURNING> returnExpression = Expression())?
    {
      if (returnExpression != null && var == null) {
        var = ExpressionToVariableUtil.getGeneratedVariable(query.getBody(), true);
      }
      query.setTopLevel(true);
      UpsertStatement stmt = new UpsertStatement(nameComponents.first, nameComponents.second, query, getVarCounter(),
        var, returnExpression);
      return addSourceLocation(stmt, startToken);
    }
}

DeleteStatement DeleteStatement() throws ParseException:
{
  Token startToken = null;
  VariableExpr varExpr = null;
  Expression condition = null;
  Pair<Identifier, Identifier> nameComponents;
}
{
  <DELETE> { startToken = token; }
  <FROM> nameComponents  = QualifiedName()
         ((<AS>)? varExpr = Variable())?
  (<WHERE> condition = Expression())?
  {
      if(varExpr == null){
        varExpr = new VariableExpr();
        VarIdentifier var = SqlppVariableUtil.toInternalVariableIdentifier(nameComponents.second.getValue());
        varExpr.setVar(var);
        addSourceLocation(varExpr, startToken);
      }
      DeleteStatement stmt = new DeleteStatement(varExpr, nameComponents.first, nameComponents.second,
          condition, getVarCounter());
      return addSourceLocation(stmt, startToken);
  }
}

UpdateStatement UpdateStatement() throws ParseException:
{
  Token startToken = null;
  VariableExpr vars;
  Expression target;
  Expression condition;
  UpdateClause uc;
  List<UpdateClause> ucs = new ArrayList<UpdateClause>();
}
{
  <UPDATE> { startToken = token; } vars = Variable() <IN> target = Expression()
  <WHERE> condition = Expression()
  <LEFTPAREN> (uc = UpdateClause()
    {
      ucs.add(uc);
    }
  (<COMMA> uc = UpdateClause()
    {
      ucs.add(uc);
    }
  )*) <RIGHTPAREN>
    {
      UpdateStatement stmt = new UpdateStatement(vars, target, condition, ucs);
      return addSourceLocation(stmt, startToken);
    }
}

UpdateClause UpdateClause() throws ParseException:
{
  Expression target = null;
  Expression value = null ;
  InsertStatement is = null;
  DeleteStatement ds = null;
  UpdateStatement us = null;
  Expression condition = null;
  UpdateClause ifbranch = null;
  UpdateClause elsebranch = null;
}
{
   (<SET> target = Expression() <EQ> value = Expression()
   | is = InsertStatement()
   | ds = DeleteStatement()
   | us = UpdateStatement()
   | <IF> <LEFTPAREN> condition = Expression() <RIGHTPAREN>
     <THEN> ifbranch = UpdateClause()
     [LOOKAHEAD(1) <ELSE> elsebranch = UpdateClause()]
     {
       return new UpdateClause(target, value, is, ds, us, condition, ifbranch, elsebranch);
     }
   )
}

Statement SetStatement() throws ParseException:
{
  Token startToken = null;
  String pn = null;
  String pv = null;
}
{
  <SET> { startToken = token; } pn = Identifier() pv = ConstantString()
    {
      SetStatement stmt = new SetStatement(pn, pv);
      return addSourceLocation(stmt, startToken);
    }
}

Statement WriteStatement() throws ParseException:
{
  Token startToken = null;
  String nodeName = null;
  String fileName = null;
  Query query;
  String writerClass = null;
  Pair<Identifier,Identifier> nameComponents = null;
}
{
  <WRITE> { startToken = token; } <OUTPUT> <TO> nodeName = Identifier() <COLON> fileName = ConstantString()
    ( <USING> writerClass = ConstantString() )?
    {
      WriteStatement stmt = new WriteStatement(new Identifier(nodeName), fileName, writerClass);
      return addSourceLocation(stmt, startToken);
    }
}

LoadStatement LoadStatement() throws ParseException:
{
  Token startToken = null;
  Identifier dataverseName = null;
  Identifier datasetName = null;
  boolean alreadySorted = false;
  String adapterName;
  Map<String,String> properties;
  Pair<Identifier,Identifier> nameComponents = null;
}
{
  <LOAD> { startToken = token; } Dataset() nameComponents = QualifiedName()
    {
      dataverseName = nameComponents.first;
      datasetName = nameComponents.second;
    }
  <USING> adapterName = AdapterName() properties = Configuration()
  (<PRESORTED>
    {
      alreadySorted = true;
    }
  )?
    {
      LoadStatement stmt = new LoadStatement(dataverseName, datasetName, adapterName, properties, alreadySorted);
      return addSourceLocation(stmt, startToken);
    }
}


String AdapterName() throws ParseException :
{
  String adapterName = null;
}
{
  adapterName = Identifier()
    {
      return adapterName;
    }
}

Statement CompactStatement() throws ParseException:
{
  Token startToken = null;
  Pair<Identifier,Identifier> nameComponents = null;
}
{
  <COMPACT> { startToken = token; } Dataset() nameComponents = QualifiedName()
    {
      CompactStatement stmt = new CompactStatement(nameComponents.first, nameComponents.second);
      return addSourceLocation(stmt, startToken);
    }
}

Statement ConnectionStatement() throws ParseException:
{
  Token startToken = null;
  Statement stmt = null;
}
{
  (
    <CONNECT> { startToken = token; } stmt = ConnectStatement(startToken)
  | <DISCONNECT> { startToken = token; } stmt = DisconnectStatement(startToken)
  | <START> { startToken = token; } stmt = StartStatement(startToken)
  | <STOP> { startToken = token; } stmt = StopStatement(startToken)
  )
  {
    return stmt;
  }
}

Statement StartStatement(Token startStmtToken) throws ParseException:
{
  Pair<Identifier,Identifier> feedNameComponents = null;

  AbstractStatement stmt = null;
}
{
  <FEED> feedNameComponents = QualifiedName()
  {
    stmt = new StartFeedStatement (feedNameComponents);
    return addSourceLocation(stmt, startStmtToken);
  }
}

AbstractStatement StopStatement(Token startStmtToken) throws ParseException:
{
  Pair<Identifier,Identifier> feedNameComponents = null;

  AbstractStatement stmt = null;
}
{
  <FEED> feedNameComponents = QualifiedName()
  {
    stmt = new StopFeedStatement (feedNameComponents);
    return addSourceLocation(stmt, startStmtToken);
  }
}

AbstractStatement DisconnectStatement(Token startStmtToken) throws ParseException:
{
  Pair<Identifier,Identifier> feedNameComponents = null;
  Pair<Identifier,Identifier> datasetNameComponents = null;

  AbstractStatement stmt = null;
}
{
  (
    <FEED> feedNameComponents = QualifiedName() <FROM> Dataset() datasetNameComponents = QualifiedName()
      {
        stmt = new DisconnectFeedStatement(feedNameComponents, datasetNameComponents);
      }
  )
  {
    return addSourceLocation(stmt, startStmtToken);
  }
}

AbstractStatement ConnectStatement(Token startStmtToken) throws ParseException:
{
  Pair<Identifier,Identifier> feedNameComponents = null;
  Pair<Identifier,Identifier> datasetNameComponents = null;

  Map<String,String> configuration = null;
  List<FunctionSignature> appliedFunctions = new ArrayList<FunctionSignature>();
  AbstractStatement stmt = null;
  String policy = null;
  String whereClauseBody = null;
  WhereClause whereClause = null;
  Token beginPos = null;
  Token endPos = null;
}
{
  (
    <FEED> feedNameComponents = QualifiedName() <TO> Dataset() datasetNameComponents = QualifiedName()
    (ApplyFunction(appliedFunctions))?
    (policy = GetPolicy())?
    (
      <WHERE>
      {
        beginPos = token;
        whereClause = new WhereClause();
        Expression whereExpr;
      }
      whereExpr = Expression()
      {
        whereClause.setWhereExpr(whereExpr);
      }
    )?
    {
      if (whereClause != null) {
        endPos = token;
        whereClauseBody = extractFragment(beginPos.endLine, beginPos.endColumn, endPos.endLine, endPos.endColumn + 1);
      }
    }
      {
        stmt = new ConnectFeedStatement(feedNameComponents, datasetNameComponents, appliedFunctions,
         policy, whereClauseBody, getVarCounter());
      }
  )
  {
    return addSourceLocation(stmt, startStmtToken);
  }
}

Map<String,String> Configuration()  throws ParseException :
{
    Map<String,String> configuration = new LinkedHashMap<String,String>();
    Pair<String, String> keyValuePair = null;
}
{
  <LEFTPAREN> ( keyValuePair = KeyValuePair()
    {
      configuration.put(keyValuePair.first, keyValuePair.second);
    }
  ( <COMMA> keyValuePair = KeyValuePair()
    {
      configuration.put(keyValuePair.first, keyValuePair.second);
    }
  )* )? <RIGHTPAREN>
    {
      return configuration;
    }
}

Pair<String, String> KeyValuePair() throws ParseException:
{
  String key;
  String value;
}
{
  <LEFTPAREN> key = ConstantString() <EQ> value = ConstantString()  <RIGHTPAREN>
    {
      return new Pair<String, String>(key, value);
    }
}

Map<String,String> Properties() throws ParseException:
{
  Map<String,String> properties = new HashMap<String,String>();
  Pair<String, String> property;
}
{
  (LOOKAHEAD(1) <LEFTPAREN> property = Property()
    {
      properties.put(property.first, property.second);
    }
  ( <COMMA> property = Property()
    {
      properties.put(property.first, property.second);
    }
  )* <RIGHTPAREN> )?
    {
      return properties;
    }
}

Pair<String, String> Property() throws ParseException:
{
  String key = null;
  String value = null;
}
{
  (key = Identifier() | key = StringLiteral())
  <EQ>
  ( value = ConstantString() | <INTEGER_LITERAL>
    {
      try {
        value = String.valueOf(Long.parseLong(token.image));
      } catch (NumberFormatException nfe) {
        throw new SqlppParseException(getSourceLocation(token), "inapproriate value: " + token.image);
      }
    }
  )
    {
      return new Pair<String, String>(key.toUpperCase(), value);
    }
}

IndexedTypeExpression IndexedTypeExpr() throws ParseException:
{
  TypeExpression typeExpr = null;
  boolean isUnknownable = false;
}
{
  (
      typeExpr = TypeReference()
    | typeExpr = OrderedListTypeDef()
    | typeExpr = UnorderedListTypeDef()
  )
  ( <QUES> { isUnknownable = true; } )?
  {
    return new IndexedTypeExpression(typeExpr, isUnknownable);
  }
}

TypeExpression TypeExpr() throws ParseException:
{
  TypeExpression typeExpr = null;
}
{
  (
      typeExpr = RecordTypeDef()
    | typeExpr = TypeReference()
    | typeExpr = OrderedListTypeDef()
    | typeExpr = UnorderedListTypeDef()
  )
  {
    return typeExpr;
  }
}

RecordTypeDefinition RecordTypeDef() throws ParseException:
{
  Token startToken = null;
  RecordTypeDefinition recType = new RecordTypeDefinition();
  RecordTypeDefinition.RecordKind recordKind = null;
}
{
  ( <CLOSED> { recordKind = RecordTypeDefinition.RecordKind.CLOSED; }
    | <OPEN> { recordKind = RecordTypeDefinition.RecordKind.OPEN; } )?
   <LEFTBRACE>
    {
      startToken = token;
      String hint = getHint(token);
      if (hint != null) {
        String splits[] = hint.split(" +");
        if (splits[0].equals(GEN_FIELDS_HINT)) {
          if (splits.length != 5) {
            throw new SqlppParseException(getSourceLocation(getHintToken(token)),
                "Expecting: /*+ gen-fields <type> <min> <max> <prefix>*/");
          }
          if (!splits[1].equals("int")) {
            throw new SqlppParseException(getSourceLocation(getHintToken(token)),
                "The only supported type for gen-fields is int.");
          }
          UndeclaredFieldsDataGen ufdg = new UndeclaredFieldsDataGen(UndeclaredFieldsDataGen.Type.INT,
             Integer.parseInt(splits[2]), Integer.parseInt(splits[3]), splits[4]);
          recType.setUndeclaredFieldsDataGen(ufdg);
        }
      }

    }
        (
          RecordField(recType)
          ( <COMMA>  RecordField(recType) )*
        )?
   <RIGHTBRACE>
   {
      if (recordKind == null) {
        recordKind = RecordTypeDefinition.RecordKind.OPEN;
      }
      recType.setRecordKind(recordKind);
      return addSourceLocation(recType, startToken);
   }
}

void RecordField(RecordTypeDefinition recType) throws ParseException:
{
  String fieldName;
  TypeExpression type = null;
  boolean nullable = false;
}
{
  fieldName = Identifier()
    {
      String hint = getHint(token);
      IRecordFieldDataGen rfdg = hint != null ? parseFieldDataGen(hint, token.specialToken) : null;
    }
  <COLON> type =  TypeExpr() (<QUES> { nullable = true; } )?
    {
      recType.addField(fieldName, type, nullable, rfdg);
    }
}

TypeReferenceExpression TypeReference() throws ParseException:
{
  Pair<Identifier,Identifier> id = null;
}
{
  id = QualifiedName()
  {
    if (id.first == null && id.second.getValue().equalsIgnoreCase("int")) {
      id.second.setValue("int64");
    }

    TypeReferenceExpression typeRef = new TypeReferenceExpression(id);
    return addSourceLocation(typeRef, token);
  }
}

OrderedListTypeDefinition OrderedListTypeDef() throws ParseException:
{
  Token startToken = null;
  TypeExpression type = null;
}
{
  <LEFTBRACKET> { startToken = token; }
    ( type = TypeExpr() )
  <RIGHTBRACKET>
  {
    OrderedListTypeDefinition typeDef = new OrderedListTypeDefinition(type);
    return addSourceLocation(typeDef, startToken);
  }
}

UnorderedListTypeDefinition UnorderedListTypeDef() throws ParseException:
{
  Token startToken = null;
  TypeExpression type = null;
}
{
  <LEFTDBLBRACE> { startToken = token; }
    ( type =  TypeExpr() )
  <RIGHTDBLBRACE>
  {
    UnorderedListTypeDefinition typeDef = new UnorderedListTypeDefinition(type);
    return addSourceLocation(typeDef, startToken);
  }
}

FunctionName FunctionName() throws ParseException:
{
  String first = null;
  String second = null;
  String third = null;
  boolean secondAfterDot = false;
}
{
  first = Identifier()
  {
    FunctionName result = new FunctionName();
    result.hint = getHint(token);
    result.sourceLoc = getSourceLocation(token);
  }
  ( <DOT> second = Identifier()
    {
      secondAfterDot = true;
    }
  (<SHARP> third = Identifier())? | <SHARP> second = Identifier() )?
    {
      if (second == null) {
        result.dataverse = defaultDataverse;
        result.library = null;
        result.function = first;
      } else if (third == null) {
        if (secondAfterDot) {
          result.dataverse = first;
          result.library   = null;
          result.function = second;
        } else {
          result.dataverse = defaultDataverse;
          result.library   = first;
          result.function = second;
        }
      } else {
        result.dataverse = first;
        result.library   = second;
        result.function  = third;
      }

      if (result.function.equalsIgnoreCase("int")) {
            result.function = "int64";
      }
      return result;
    }
}

Pair<Identifier,Identifier> TypeName() throws ParseException:
{
  Pair<Identifier,Identifier> name = null;
}
{
  name = QualifiedName()
    {
      if (name.first == null) {
        name.first = new Identifier(defaultDataverse);
      }
      return name;
    }
}

String Identifier() throws ParseException:
{
  String lit = null;
}
{
  (<IDENTIFIER>
    {
      return token.image;
    }
  | lit = QuotedString()
    {
      return lit;
    }
  )
}

void Dataset() throws ParseException:
{
}
{
    (<DATASET>|<COLLECTION>)
}

Pair<Integer, Pair<List<String>, IndexedTypeExpression>> OpenField() throws ParseException:
{
  IndexedTypeExpression fieldType = null;
  Pair<Integer, List<String>> fieldList = null;
}
{
  fieldList = NestedField()
  ( <COLON> fieldType = IndexedTypeExpr() )?
  {
    return new Pair<Integer, Pair<List<String>, IndexedTypeExpression>>
            (fieldList.first, new Pair<List<String>, IndexedTypeExpression>(fieldList.second, fieldType));
  }
}

Pair<Integer, List<String>> NestedField() throws ParseException:
{
  List<String> exprList = new ArrayList<String>();
  String lit = null;
  Token litToken = null;
  int source = 0;
}
{
  lit = Identifier()
  {
    boolean meetParens = false;
    litToken = token;
  }
  (
    LOOKAHEAD(1)
    <LEFTPAREN><RIGHTPAREN>
    {
        if(!lit.equalsIgnoreCase("meta")){
            throw new SqlppParseException(getSourceLocation(litToken), "The string before () has to be \"meta\".");
        }
        meetParens = true;
        source = 1;
    }
  )?
  {
    if(!meetParens){
        exprList.add(lit);
    }
  }
  (<DOT>
    lit = Identifier()
    {
      exprList.add(lit);
    }
  )*
  {
    return new Pair<Integer, List<String>>(source, exprList);
  }
}

String ConstantString() throws ParseException:
{
  String value = null;
}
{
  (value = QuotedString() | value = StringLiteral())
  {
     return value;
  }
}

String QuotedString() throws ParseException:
{
}
{
  <QUOTED_STRING>
    {
      return removeQuotesAndEscapes(token.image);
    }
}

String StringLiteral() throws ParseException:
{
}
{
  <STRING_LITERAL>
    {
      return removeQuotesAndEscapes(token.image);
    }
}

Pair<Identifier,Identifier> QualifiedName() throws ParseException:
{
  String first = null;
  String second = null;
}
{
  first = Identifier() (<DOT> second = Identifier())?
  {
    Identifier id1 = null;
    Identifier id2 = null;
    if (second == null) {
      id2 = new Identifier(first);
    } else
    {
      id1 = new Identifier(first);
      id2 = new Identifier(second);
    }
    return new Pair<Identifier,Identifier>(id1, id2);
  }
}

Triple<Identifier,Identifier,Identifier> DoubleQualifiedName() throws ParseException:
{
  String first = null;
  String second = null;
  String third = null;
}
{
  first = Identifier() <DOT> second = Identifier() (<DOT> third = Identifier())?
  {
    Identifier id1 = null;
    Identifier id2 = null;
    Identifier id3 = null;
    if (third == null) {
      id2 = new Identifier(first);
      id3 = new Identifier(second);
    } else {
      id1 = new Identifier(first);
      id2 = new Identifier(second);
      id3 = new Identifier(third);
    }
    return new Triple<Identifier,Identifier,Identifier>(id1, id2, id3);
  }
}

FunctionDecl FunctionDeclaration() throws ParseException:
{
  Token startToken = null;
  String functionName;
  List<VarIdentifier> paramList = new ArrayList<VarIdentifier>();
  Expression funcBody;
  createNewScope();
}
{
  <DECLARE> { startToken = token; } <FUNCTION>
    functionName = Identifier()
    paramList = ParameterList()
  <LEFTBRACE>
    (funcBody = SelectExpression(true) | funcBody = Expression())
  <RIGHTBRACE>
  {
    FunctionSignature signature = new FunctionSignature(defaultDataverse, functionName, paramList.size());
    getCurrentScope().addFunctionDescriptor(signature, false);
    FunctionDecl stmt = new FunctionDecl(signature, paramList, funcBody);
    removeCurrentScope();
    return addSourceLocation(stmt, startToken);
  }
}

Query ExplainStatement() throws ParseException:
{
  Query query;
}
{
  <EXPLAIN> query = Query(true)
  {
    return query;
  }
}

Query Query(boolean explain) throws ParseException:
{
  Query query = new Query(explain);
  Expression expr;
}
{
  (
  expr = Expression()
  |
  expr = SelectExpression(false)
  )
  {
    query.setBody(expr);
    query.setSourceLocation(expr.getSourceLocation());
    return query;
  }
}


Expression Expression():
{
  Expression expr = null;
  Expression exprP = null;
}
{
(
    LOOKAHEAD(2)
    expr = OperatorExpr()
    | expr = QuantifiedExpression()
)
    {
      return (exprP==null) ? expr : exprP;
    }
}

Expression OperatorExpr() throws ParseException:
{
  OperatorExpr op = null;
  Expression operand = null;
}
{
    operand = AndExpr()
    (
      <OR>
      {
        if (op == null) {
          op = new OperatorExpr();
          op.addOperand(operand);
          op.setCurrentop(true);
          addSourceLocation(op, token);
        }
        try{
            op.addOperator(token.image.toLowerCase());
        } catch (Exception e){
            throw new SqlppParseException(getSourceLocation(token), e.getMessage());
        }
      }

      operand = AndExpr()
      {
        op.addOperand(operand);
      }

    )*

    {
      return op==null? operand: op;
    }
}

Expression AndExpr() throws ParseException:
{
  OperatorExpr op = null;
  Expression operand = null;
}
{
    operand = NotExpr()
    (
      <AND>
      {
        if (op == null) {
          op = new OperatorExpr();
          op.addOperand(operand);
          op.setCurrentop(true);
          addSourceLocation(op, token);
        }
        try{
           op.addOperator(token.image.toLowerCase());
        } catch (CompilationException e){
           throw new SqlppParseException(getSourceLocation(token), e.getMessage());
        }
    }

    operand = NotExpr()
    {
      op.addOperand(operand);
    }

    )*

    {
      return op==null ? operand: op;
    }
}

Expression NotExpr() throws ParseException:
{
   Expression inputExpr;
   boolean not = false;
   Token startToken = null;
}
{
  (<NOT> { not = true; startToken = token; } )? inputExpr = RelExpr()
  {
    if(not) {
        FunctionSignature signature = new FunctionSignature(BuiltinFunctions.NOT);
        CallExpr callExpr = new CallExpr(signature, new ArrayList<Expression>(Collections.singletonList(inputExpr)));
        return addSourceLocation(callExpr, startToken);
    } else {
        return inputExpr;
    }
  }
}

Expression RelExpr() throws ParseException:
{
  boolean not = false;
  OperatorExpr op = null;
  Expression operand = null;
  boolean broadcast = false;
  IExpressionAnnotation annotation = null;
}
{
    operand = BetweenExpr()

    (
      LOOKAHEAD(2)( <LT> | <GT> | <LE> | <GE> | <EQ> | <NE> | <LG> |<SIMILAR> | (<NOT> { not = true; })? <IN>)
        {
          String mhint = getHint(token);
          if (mhint != null) {
            if (mhint.equals(INDEXED_NESTED_LOOP_JOIN_HINT)) {
                annotation = IndexedNLJoinExpressionAnnotation.INSTANCE;
            } else if (mhint.equals(SKIP_SECONDARY_INDEX_SEARCH_HINT)) {
                annotation = SkipSecondaryIndexSearchExpressionAnnotation.INSTANCE;
            } else if (mhint.equals(BROADCAST_JOIN_HINT)) {
                broadcast = true;
            }
          }

          String operator = token.image.toLowerCase();
          if (operator.equals("<>")){
              operator = "!=";
          }
          if (not) {
            operator = "not_" + operator;
          }
          if (op == null) {
            op = new OperatorExpr();
            op.addOperand(operand, false); // broadcast is always for the right branch
            op.setCurrentop(true);
            addSourceLocation(op, token);
          }
          try{
            op.addOperator(operator);
          } catch (CompilationException e){
            throw new SqlppParseException(getSourceLocation(token), e.getMessage());
          }
        }

       operand = BetweenExpr()
       {
         op.addOperand(operand, broadcast);
       }
    )?

     {
       if (annotation != null) {
         op.addHint(annotation);
       }
       return op==null? operand: op;
     }
}

Expression BetweenExpr() throws ParseException:
{
  boolean not = false;
  OperatorExpr op = null;
  Expression operand = null;
  IExpressionAnnotation annotation = null;
}
{
    operand = IsExpr()
    (
      LOOKAHEAD(2)
      (<NOT> { not = true; })? <BETWEEN>
        {
          String mhint = getHint(token);
          if (mhint != null) {
            if (mhint.equals(INDEXED_NESTED_LOOP_JOIN_HINT)) {
                annotation = IndexedNLJoinExpressionAnnotation.INSTANCE;
            } else if (mhint.equals(SKIP_SECONDARY_INDEX_SEARCH_HINT)) {
                annotation = SkipSecondaryIndexSearchExpressionAnnotation.INSTANCE;
            }
          }
          String operator = token.image.toLowerCase();
          if(not){
            operator = "not_" + operator;
          }
          if (op == null) {
            op = new OperatorExpr();
            op.addOperand(operand);
            op.setCurrentop(true);
            addSourceLocation(op, token);
          }
          try{
            op.addOperator(operator);
          } catch (CompilationException e){
            throw new SqlppParseException(getSourceLocation(token), e.getMessage());
          }
        }

       operand = IsExpr()
       {
         op.addOperand(operand);
       }

       <AND>
       operand = IsExpr()
       {
         op.addOperator(OperatorType.AND);
         op.addOperand(operand);
       }
    )?

    {
       if (annotation != null) {
         op.addHint(annotation);
       }
       return op==null ? operand: op;
    }
}

Expression IsExpr() throws ParseException:
{
    Token notToken = null;
    CallExpr expr = null;
    Expression operand = null;
    boolean not = false;
    FunctionIdentifier fn = null;
}
{
    operand = LikeExpr()
    ( <IS>
        (<NOT> { not = true; notToken = token; })?
        (
            <NULL> { fn = BuiltinFunctions.IS_NULL; } |
            <MISSING> { fn = BuiltinFunctions.IS_MISSING; } |
            <UNKNOWN> { fn = BuiltinFunctions.IS_UNKNOWN; } |
            (<KNOWN> | <VALUED>) { not = !not; fn = BuiltinFunctions.IS_UNKNOWN; }
        )
      {
        FunctionSignature signature = new FunctionSignature(fn);
        expr = new CallExpr(signature, new ArrayList<Expression>(Collections.singletonList(operand)));
        addSourceLocation(expr, token);
        if (not) {
           FunctionSignature notSignature = new FunctionSignature(BuiltinFunctions.NOT);
           expr = new CallExpr(notSignature, new ArrayList<Expression>(Collections.singletonList(expr)));
           addSourceLocation(expr, notToken);
        }
      }
    )?
    {
        return expr == null ? operand : expr;
    }
}

Expression LikeExpr() throws ParseException:
{
  boolean not = false;
  OperatorExpr op = null;
  Expression operand = null;
}
{
    operand = ConcatExpr()
    (
        LOOKAHEAD(2)
        (<NOT> { not = true; })? <LIKE>
        {
          op = new OperatorExpr();
          op.addOperand(operand);
          op.setCurrentop(true);
          addSourceLocation(op, token);

          String operator = token.image.toLowerCase();
          if (not) {
            operator = "not_" + operator;
          }
          try {
            op.addOperator(operator);
          } catch (CompilationException e){
            throw new SqlppParseException(getSourceLocation(token), e.getMessage());
          }
        }

        operand = ConcatExpr()
        {
          op.addOperand(operand);
        }
     )?

     {
       return op == null ? operand : op;
     }
}

Expression ConcatExpr() throws ParseException:
{
  OperatorExpr op = null;
  Expression operand = null;
}
{
    operand = AddExpr()
    (
      LOOKAHEAD(1)
      <CONCAT>
      {
        if (op == null) {
          op = new OperatorExpr();
          op.addOperand(operand);
          op.setCurrentop(true);
          addSourceLocation(op, token);
        }
        op.addOperator(OperatorType.CONCAT);
    }
    operand = AddExpr()
    {
      op.addOperand(operand);
    }
    )*

    {
       return op == null ? operand : op;
    }
}

Expression AddExpr() throws ParseException:
{
  OperatorExpr op = null;
  OperatorType opType = null;
  Expression operand = null;
}
{
    operand = MultExpr()
    (
      LOOKAHEAD(1)
      (<PLUS> { opType = OperatorType.PLUS; } | <MINUS> { opType = OperatorType.MINUS; } )
      {
        if (op == null) {
          op = new OperatorExpr();
          op.addOperand(operand);
          op.setCurrentop(true);
          addSourceLocation(op, token);
        }
        op.addOperator(opType);
    }

    operand = MultExpr()
    {
      op.addOperand(operand);
    }
    )*

    {
       return op == null ? operand : op;
    }
}

Expression MultExpr() throws ParseException:
{
  OperatorExpr op = null;
  OperatorType opType = null;
  Expression operand = null;
}
{
    operand = ExponentExpr()

    ( (
        <MUL> { opType = OperatorType.MUL; } |
        <DIVIDE> { opType = OperatorType.DIVIDE; } |
        <DIV> { opType = OperatorType.DIV; } |
        ( <MOD> | <PERCENT> ) { opType = OperatorType.MOD; }
      )
      {
        if (op == null) {
          op = new OperatorExpr();
          op.addOperand(operand);
          op.setCurrentop(true);
          addSourceLocation(op, token);
        }
        op.addOperator(opType);
    }
    operand = ExponentExpr()
    {
       op.addOperand(operand);
    }
    )*

     {
       return op == null ? operand : op;
     }
}

Expression ExponentExpr() throws ParseException:
{
  OperatorExpr op = null;
  Expression operand = null;
}
{
    operand = UnaryExpr()
    (<CARET>
    {
        if (op == null) {
          op = new OperatorExpr();
          op.addOperand(operand);
          op.setCurrentop(true);
          addSourceLocation(op, token);
        }
        op.addOperator(OperatorType.CARET);
    }
    operand = UnaryExpr()
    {
       op.addOperand(operand);
    }
    )?
    {
       return op == null ? operand : op;
    }
}

Expression UnaryExpr() throws ParseException:
{
    boolean not = false;
    UnaryExpr uexpr = null;
    Expression expr = null;
}
{
    ( (<PLUS> | <MINUS> | (<NOT> { not = true; } )? <EXISTS> )
    {
        String exprType = token.image.toLowerCase();
        if (not) {
           exprType = "not_" + exprType;
        }
        uexpr = new UnaryExpr();
        addSourceLocation(uexpr, token);
        try {
            uexpr.setExprType(exprType);
        } catch (CompilationException e){
            throw new SqlppParseException(getSourceLocation(token), e.getMessage());
        }
    }
    )?

    expr = ValueExpr()
    {
       if (uexpr == null) {
         return expr;
       } else {
         uexpr.setExpr(expr);
         return uexpr;
       }
    }
}

Expression ValueExpr() throws ParseException:
{
  Expression expr = null;
  AbstractAccessor accessor = null;
}
{
  expr = PrimaryExpr()
  (
    accessor = FieldAccessor(accessor != null ? accessor : expr)
    |
    accessor = IndexAccessor(accessor != null ? accessor : expr)
  )*
  {
    return accessor == null ? expr : accessor;
  }
}

FieldAccessor FieldAccessor(Expression inputExpr) throws ParseException:
{
  Token startToken = null;
  String ident = null;
}
{
   <DOT> { startToken = token; } ident = Identifier()
    {
      FieldAccessor fa = new FieldAccessor(inputExpr, new Identifier(ident));
      return addSourceLocation(fa, startToken);
    }
}

AbstractAccessor IndexAccessor(Expression inputExpr) throws ParseException:
{
  Token startToken = null;
  boolean isListSliceExpression = false;
  AbstractAccessor resultExpression = null;
  Expression argumentExpression1 = null;
  Expression argumentExpression2 = null;
}
{
  <LEFTBRACKET> { startToken = token; }
  ( argumentExpression1 = Expression()
    {
        if (argumentExpression1.getKind() == Expression.Kind.LITERAL_EXPRESSION)
        {
            Literal lit = ((LiteralExpr)argumentExpression1).getValue();
            if (lit.getLiteralType() != Literal.Type.INTEGER &&
                lit.getLiteralType() != Literal.Type.LONG) {
                throw new SqlppParseException(argumentExpression1.getSourceLocation(), "Index should be an INTEGER");
            }
        }
    }
  )
  (<COLON>
  {
    isListSliceExpression = true;
  }
  ( argumentExpression2 = Expression()
    {
      if (argumentExpression2.getKind() == Expression.Kind.LITERAL_EXPRESSION) {
          Literal lit = ((LiteralExpr)argumentExpression2).getValue();
          if (lit.getLiteralType() != Literal.Type.INTEGER &&
              lit.getLiteralType() != Literal.Type.LONG) {
                  throw new SqlppParseException(argumentExpression2.getSourceLocation(), "Index should be an INTEGER");
          }
      }
    })?
  )?
  <RIGHTBRACKET>
  {
    if (!isListSliceExpression) {
        resultExpression = new IndexAccessor(inputExpr, argumentExpression1);
    } else {
        resultExpression = new ListSliceExpression(inputExpr, argumentExpression1, argumentExpression2);
    }

    return addSourceLocation(resultExpression, startToken);
  }
}

Expression PrimaryExpr() throws ParseException:
{
  Expression expr = null;
}
{
  ( LOOKAHEAD(4)
    expr = FunctionCallExpr()
  | expr = CaseExpr()
  | expr = Literal()
  | expr = VariableRef()
  | expr = ExternalVariableRef()
  | expr = ListConstructor()
  | expr = RecordConstructor()
  | expr = ParenthesizedExpression()
  )
    {
      return expr;
    }
}

Expression Literal() throws ParseException:
{
  LiteralExpr lit = new LiteralExpr();
  String str = null;
}
{
  ( str = StringLiteral()
    {
      lit.setValue(new StringLiteral(str));
    }
  | <INTEGER_LITERAL>
    {
        try {
            lit.setValue(new LongIntegerLiteral(Long.valueOf(token.image)));
        } catch (NumberFormatException e) {
            throw new SqlppParseException(getSourceLocation(token), "Could not parse numeric literal \"" + LogRedactionUtil.userData(token.image) +'"');
        }
    }
  | <FLOAT_LITERAL>
    {
        try {
            lit.setValue(new FloatLiteral(Float.valueOf(token.image)));
        } catch (NumberFormatException e) {
            throw new SqlppParseException(getSourceLocation(token), "Could not parse numeric literal \"" + LogRedactionUtil.userData(token.image) +'"');
        }
    }
  | <DOUBLE_LITERAL>
    {
        try {
            lit.setValue(new DoubleLiteral(Double.valueOf(token.image)));
        } catch (NumberFormatException e) {
            throw new SqlppParseException(getSourceLocation(token), "Could not parse numeric literal \"" + LogRedactionUtil.userData(token.image) +'"');
        }
    }
  | <MISSING>
    {
      lit.setValue(MissingLiteral.INSTANCE);
    }
  | <NULL>
    {
      lit.setValue(NullLiteral.INSTANCE);
    }
  | <TRUE>
    {
      lit.setValue(TrueLiteral.INSTANCE);
    }
  | <FALSE>
    {
      lit.setValue(FalseLiteral.INSTANCE);
    }
  )
    {
      return addSourceLocation(lit, token);
    }
}

VariableExpr VariableRef() throws ParseException:
{
    VarIdentifier var = new VarIdentifier();
    String id = null;
}
{
    (<IDENTIFIER> { id = token.image; } | id = QuotedString())
    {
     id = SqlppVariableUtil.toInternalVariableName(id); // Prefix user-defined variables with "$"
     Identifier ident = lookupSymbol(id);
     if (isInForbiddenScopes(id)) {
       throw new SqlppParseException(getSourceLocation(token),
        "Inside limit clauses, it is disallowed to reference a variable having the same name as any variable bound in the same scope as the limit clause.");
     }
     VariableExpr varExp = new VariableExpr();
     if (ident != null) { // exist such ident
       varExp.setVar((VarIdentifier)ident);
     } else {
       varExp.setVar(var);
       varExp.setIsNewVar(false);
       var.setValue(id);
     }
     return addSourceLocation(varExp, token);
    }
}

VariableExpr Variable() throws ParseException:
{
    VarIdentifier var = new VarIdentifier();
    String id = null;
}
{
    (<IDENTIFIER> { id = token.image; } | id = QuotedString())
    {
     id = SqlppVariableUtil.toInternalVariableName(id); // prefix user-defined variables with "$".
     Identifier ident = lookupSymbol(id);
     VariableExpr varExp = new VariableExpr();
     if(ident != null) { // exist such ident
       varExp.setIsNewVar(false);
     }
     varExp.setVar(var);
     var.setValue(id);
     return addSourceLocation(varExp, token);
    }
}

Pair<VariableExpr, List<Pair<Expression, Identifier>>> VariableWithFieldMap() throws ParseException:
{
  VariableExpr var = null;
  List<Pair<Expression, Identifier>> fieldList = new ArrayList<Pair<Expression, Identifier>>();
}
{
  var = Variable()
  ( LOOKAHEAD(1)
    {
      VariableExpr fieldVarExpr = null;
      String fieldIdentifierStr = null;
    }
    <LEFTPAREN>
    fieldVarExpr = VariableRef() <AS> fieldIdentifierStr = Identifier()
    {
      fieldList.add(new Pair<Expression, Identifier>(fieldVarExpr, new Identifier(fieldIdentifierStr)));
    }
    (<COMMA>
      fieldVarExpr = VariableRef() <AS> fieldIdentifierStr = Identifier()
      {
        fieldList.add(new Pair<Expression, Identifier>(fieldVarExpr, new Identifier(fieldIdentifierStr)));
      }
    )*
    <RIGHTPAREN>
  )?
  {
    return new Pair<VariableExpr, List<Pair<Expression, Identifier>>>(var, fieldList);
  }
}

VariableExpr ExternalVariableRef() throws ParseException:
{
  String name = null;
}
{
  (
    (
      <DOLLAR>
      (
        <INTEGER_LITERAL> { name = token.image; } |
        <IDENTIFIER> { name = token.image; } |
        name = QuotedString()
      )
    )
    |
    (
      <QUES> { name = String.valueOf(++externalVarCounter); }
    )
  )
  {
     String idName = SqlppVariableUtil.toExternalVariableName(name);
     VarIdentifier id = new VarIdentifier(idName);
     VariableExpr varExp = new VariableExpr(id);
     return addSourceLocation(varExp, token);
  }
}

Expression ListConstructor() throws ParseException:
{
    Expression expr = null;
}
{
    (
      expr = OrderedListConstructor() |
      expr = UnorderedListConstructor()
    )
    {
      return expr;
    }
}

ListConstructor OrderedListConstructor() throws ParseException:
{
  Token startToken = null;
  List<Expression> exprList = null;
}
{
  <LEFTBRACKET> { startToken = token; }
  exprList = ExpressionList()
  <RIGHTBRACKET>
  {
    ListConstructor expr = new ListConstructor(ListConstructor.Type.ORDERED_LIST_CONSTRUCTOR, exprList);
    return addSourceLocation(expr, startToken);
  }
}

ListConstructor UnorderedListConstructor() throws ParseException:
{
  Token startToken = null;
  List<Expression> exprList = null;
}
{
  <LEFTDBLBRACE> { startToken = token; }
  exprList = ExpressionList()
  <RIGHTDBLBRACE>
  {
    ListConstructor expr = new ListConstructor(ListConstructor.Type.UNORDERED_LIST_CONSTRUCTOR, exprList);
    return addSourceLocation(expr, startToken);
  }
}

List<Expression> ExpressionList() throws ParseException:
{
  Expression expr = null;
  List<Expression> exprList = new ArrayList<Expression>();
}
{
  (
    expr = Expression()
    {
      exprList.add(expr);
    }
    ( <COMMA> expr = Expression()
      {
        exprList.add(expr);
      }
    )*
  )?
  {
    return exprList;
  }
}

RecordConstructor RecordConstructor() throws ParseException:
{
  Token startToken = null;
  FieldBinding fb = null;
  List<FieldBinding> fbList = new ArrayList<FieldBinding>();
}
{
  <LEFTBRACE> { startToken = token; }
  (
    fb = FieldBinding() { fbList.add(fb); }
    (<COMMA> fb = FieldBinding() { fbList.add(fb); })*
  )?
  <RIGHTBRACE>
  {
    RecordConstructor expr = new RecordConstructor(fbList);
    return addSourceLocation(expr, startToken);
  }
}

FieldBinding FieldBinding() throws ParseException:
{
  Expression left, right;
}
{
  left = Expression() <COLON> right = Expression()
  {
    return new FieldBinding(left, right);
  }
}

Expression FunctionCallExpr() throws ParseException:
{
  Expression resultExpr;
  CallExpr callExpr;
  List<Expression> argList = new ArrayList<Expression>();
  Expression tmp = null;
  int arity = 0;
  FunctionName funcName = null;
  String hint = null;
  boolean star = false;
  boolean distinct = false;
}
{
  funcName = FunctionName()
  {
    hint = funcName.hint;
  }
  <LEFTPAREN> (
    ( <DISTINCT> { distinct = true; } )?
    ( tmp = Expression() | <MUL> { star = true; } )
    {
      if(star){
        if(!funcName.function.equalsIgnoreCase("count")){
           throw new SqlppParseException(getSourceLocation(token), "The parameter * can only be used in COUNT().");
        }
        argList.add(new LiteralExpr(new LongIntegerLiteral(1L)));
      } else {
         argList.add(tmp);
      }
      arity ++;
    }
  (<COMMA> tmp = Expression()
    {
      argList.add(tmp);
      arity++;
    }
  )*)? <RIGHTPAREN>
    {
      String name = funcName.function;
      if (distinct) {
        name += FunctionMapUtil.DISTINCT_AGGREGATE_SUFFIX;
      }
      // TODO use funcName.library
      String fqFunctionName = funcName.library == null ? name : funcName.library + "#" + name;
      FunctionSignature signature
        = lookupFunctionSignature(funcName.dataverse, fqFunctionName, arity);
      if (signature == null) {
        signature = new FunctionSignature(funcName.dataverse, fqFunctionName, arity);
      }
      callExpr = FunctionMapUtil.normalizedListInputFunctions(new CallExpr(signature,argList));
      if (hint != null) {
        if (hint.startsWith(INDEXED_NESTED_LOOP_JOIN_HINT)) {
          callExpr.addHint(IndexedNLJoinExpressionAnnotation.INSTANCE);
        } else if (hint.startsWith(SKIP_SECONDARY_INDEX_SEARCH_HINT)) {
          callExpr.addHint(SkipSecondaryIndexSearchExpressionAnnotation.INSTANCE);
        }
      }
      callExpr.setSourceLocation(funcName.sourceLoc);
      resultExpr = callExpr;
    }

  ( <OVER> resultExpr = WindowExpr(callExpr.getFunctionSignature(), callExpr.getExprList(), token) )?

  {
     return resultExpr;
  }
}

WindowExpression WindowExpr(FunctionSignature signature, List<Expression> argList, Token startToken) throws ParseException:
{
  Expression partitionExpr = null;
  List<Expression> partitionExprs = new ArrayList<Expression>();
  OrderbyClause orderByClause = null;
  List<Expression> orderbyList = null;
  List<OrderbyClause.OrderModifier> orderbyModifierList = null;
  WindowExpression.FrameMode frameMode = null;
  Pair<WindowExpression.FrameBoundaryKind, Expression> frameStart = null, frameEnd = null;
  WindowExpression.FrameBoundaryKind frameStartKind = null, frameEndKind = null;
  Expression frameStartExpr = null, frameEndExpr = null;
  WindowExpression.FrameExclusionKind frameExclusionKind = null;
  Pair<VariableExpr, List<Pair<Expression, Identifier>>> windowVarWithFieldList = null;
  VariableExpr windowVar = null;
  List<Pair<Expression, Identifier>> windowFieldList = null;
}
{
  (
    windowVarWithFieldList = VariableWithFieldMap() <AS>
    {
      windowVar = windowVarWithFieldList.first;
      windowFieldList = windowVarWithFieldList.second;
    }
  )?
  <LEFTPAREN>
  (
    <IDENTIFIER> { expectToken(PARTITION); } <BY>
    partitionExpr = Expression() { partitionExprs.add(partitionExpr); }
    ( <COMMA> partitionExpr = Expression() { partitionExprs.add(partitionExpr); } )*
  )?
  (
    orderByClause = OrderbyClause()
    {
      orderbyList = orderByClause.getOrderbyList();
      orderbyModifierList = orderByClause.getModifierList();
    }
    (
      frameMode = WindowFrameMode()
      (
        frameStart = WindowFrameBoundary() |
        ( <BETWEEN> frameStart = WindowFrameBoundary() <AND> frameEnd = WindowFrameBoundary() )
      )
      ( frameExclusionKind = WindowFrameExclusion() )?
      {
        frameStartKind = frameStart.first;
        frameStartExpr = frameStart.second;
        if (frameEnd == null) {
          frameEndKind = WindowExpression.FrameBoundaryKind.CURRENT_ROW;
        } else {
          frameEndKind = frameEnd.first;
          frameEndExpr = frameEnd.second;
        }
        if (frameExclusionKind == null) {
          frameExclusionKind = WindowExpression.FrameExclusionKind.NO_OTHERS;
        }
      }
    )?
  )?
  <RIGHTPAREN>
  {
    WindowExpression winExp = new WindowExpression(signature, argList, partitionExprs, orderbyList, orderbyModifierList,
      frameMode, frameStartKind, frameStartExpr, frameEndKind, frameEndExpr, frameExclusionKind, windowVar,
      windowFieldList);
    return addSourceLocation(winExp, startToken);
  }
}

WindowExpression.FrameMode WindowFrameMode() throws ParseException:
{
}
{
  <IDENTIFIER>
  {
    if (isToken(RANGE)) {
      return WindowExpression.FrameMode.RANGE;
    } else if (isToken(ROWS)) {
      return WindowExpression.FrameMode.ROWS;
    } else if (isToken(GROUPS)) {
      return WindowExpression.FrameMode.GROUPS;
    } else {
      throw createUnexpectedTokenError();
    }
  }
}

Pair<WindowExpression.FrameBoundaryKind, Expression> WindowFrameBoundary() throws ParseException:
{
  boolean current = false;
  Expression expr = null;
}
{
  (
    LOOKAHEAD({ laIdentifier(CURRENT) }) <IDENTIFIER> { current = true; }
    | LOOKAHEAD({ laIdentifier(UNBOUNDED) }) <IDENTIFIER>
    | expr = Expression()
  )
  <IDENTIFIER>
  {
    WindowExpression.FrameBoundaryKind kind;
    if (current && isToken(ROW)) {
      kind = WindowExpression.FrameBoundaryKind.CURRENT_ROW;
    } else if (!current && isToken(PRECEDING)) {
      kind = expr == null
        ? WindowExpression.FrameBoundaryKind.UNBOUNDED_PRECEDING
        : WindowExpression.FrameBoundaryKind.BOUNDED_PRECEDING;
    } else if (!current && isToken(FOLLOWING)) {
      kind = expr == null
        ? WindowExpression.FrameBoundaryKind.UNBOUNDED_FOLLOWING
        : WindowExpression.FrameBoundaryKind.BOUNDED_FOLLOWING;
    } else {
      throw createUnexpectedTokenError();
    }
    return new Pair<WindowExpression.FrameBoundaryKind, Expression>(kind, expr);
  }
}

WindowExpression.FrameExclusionKind WindowFrameExclusion() throws ParseException:
{
  boolean current = false, no = false;
}
{
  <IDENTIFIER>
  {
    expectToken(EXCLUDE);
  }
  (
    <GROUP>
    {
      return WindowExpression.FrameExclusionKind.GROUP;
    }
    |
    (
      <IDENTIFIER>
      {
        if (isToken(TIES)) {
          return WindowExpression.FrameExclusionKind.TIES;
        } else if (isToken(CURRENT)) {
          current = true;
        } else if (isToken(NO)) {
          no = true;
        } else {
          throw createUnexpectedTokenError();
        }
      }
      <IDENTIFIER>
      {
        if (current && isToken(ROW)) {
          return WindowExpression.FrameExclusionKind.CURRENT_ROW;
        } else if (no && isToken(OTHERS)) {
          return WindowExpression.FrameExclusionKind.NO_OTHERS;
        } else {
          throw createUnexpectedTokenError();
        }
      }
    )
  )
}

Expression ParenthesizedExpression() throws ParseException:
{
  Expression expr;
}
{
    (
    LOOKAHEAD(2)
    <LEFTPAREN> expr = Expression() <RIGHTPAREN>
    |
    expr = Subquery()
    )
    {
      return expr;
    }
}

Expression CaseExpr() throws ParseException:
{
   Token startToken = null;
   Expression conditionExpr = null;
   List<Expression> whenExprs = new ArrayList<Expression>();
   List<Expression> thenExprs = new ArrayList<Expression>();
   Expression elseExpr = null;
   Expression whenExpr = null;
   Expression thenExpr = null;
}
{
   <CASE> { startToken = token; }
   ( conditionExpr = Expression() )?
   (
     <WHEN> whenExpr = Expression()
     {
        whenExprs.add(whenExpr);
     }
     <THEN> thenExpr = Expression()
     {
        thenExprs.add(thenExpr);
     }
   )*
   (<ELSE> elseExpr = Expression() )?
   <END>
   {
     if (conditionExpr == null) {
        LiteralExpr litExpr = new LiteralExpr(TrueLiteral.INSTANCE);
        conditionExpr = addSourceLocation(litExpr, startToken);
     }
     CaseExpression caseExpr = new CaseExpression(conditionExpr, whenExprs, thenExprs, elseExpr);
     return addSourceLocation(caseExpr, startToken);
   }
}

SelectExpression SelectExpression(boolean subquery) throws ParseException:
{
  List<LetClause> letClauses = new ArrayList<LetClause>();
  SelectSetOperation selectSetOperation;
  OrderbyClause orderbyClause = null;
  LimitClause limitClause = null;
  createNewScope();
}
{
    ( letClauses = LetClause() )?
    selectSetOperation = SelectSetOperation()
    (orderbyClause = OrderbyClause() {})?
    (limitClause = LimitClause() {})?
    {
      SelectExpression selectExpr =
        new SelectExpression(letClauses, selectSetOperation, orderbyClause, limitClause, subquery);
      selectExpr.setSourceLocation((!letClauses.isEmpty() ? letClauses.get(0) : selectSetOperation).getSourceLocation());
      return selectExpr;
    }
}

SelectSetOperation SelectSetOperation() throws ParseException:
{
  SetOperationInput setOperationInputLeft;
  List<SetOperationRight> setOperationRights = new ArrayList<SetOperationRight>();
}
{
  {
      SelectBlock selectBlockLeft = null;
      SelectExpression subqueryLeft = null;
      Expression expr = null;
  }
  selectBlockLeft = SelectBlock()
  {
     setOperationInputLeft = new SetOperationInput(selectBlockLeft, subqueryLeft);
  }
  (
    {
      SetOpType opType = SetOpType.UNION;
      boolean setSemantics = true;
      SelectBlock selectBlockRight = null;
      SelectExpression subqueryRight = null;
    }
    (<UNION> {opType = SetOpType.UNION;} |<INTERSECT> {opType = SetOpType.INTERSECT;} |<EXCEPT> {opType = SetOpType.EXCEPT;}) (<ALL> {setSemantics = false;} )?
    (selectBlockRight = SelectBlock()| subqueryRight = Subquery())
    {
        setOperationRights.add(new SetOperationRight(opType, setSemantics, new SetOperationInput(selectBlockRight, subqueryRight)));
    }
  )*
  {
    SelectSetOperation selectSetOp = new SelectSetOperation(setOperationInputLeft, setOperationRights);
    selectSetOp.setSourceLocation(selectBlockLeft.getSourceLocation());
    return selectSetOp;
  }
}

SelectExpression Subquery() throws ParseException:
{
   SelectExpression selectExpr = null;
}
{
  <LEFTPAREN> selectExpr = SelectExpression(true) {} <RIGHTPAREN>
  {
    return selectExpr;
  }
}

SelectBlock SelectBlock() throws ParseException:
{
  SelectClause selectClause = null;
  FromClause fromClause = null;
  List<LetClause> fromLetClauses = null;
  WhereClause whereClause = null;
  List<AbstractClause> fromLetWhereClauses = new ArrayList<AbstractClause>();
  GroupbyClause groupbyClause = null;
  List<LetClause> gbyLetClauses = null;
  HavingClause havingClause = null;
  List<AbstractClause> gbyLetHavingClauses = new ArrayList<AbstractClause>();
  SourceLocation startSrcLoc = null;
}
{
  (
     selectClause = SelectClause() { startSrcLoc = selectClause.getSourceLocation(); }
     (
        LOOKAHEAD(1)
        fromClause = FromClause()
        (
            LOOKAHEAD(1)
            fromLetClauses = LetClause()
        )?
     )?
     (whereClause = WhereClause())?
     (
        groupbyClause = GroupbyClause()
        (
            LOOKAHEAD(1)
            gbyLetClauses = LetClause()
        )?
        (havingClause = HavingClause())?
     )?
    |
     fromClause = FromClause() { startSrcLoc = fromClause.getSourceLocation(); }
     (
        LOOKAHEAD(1)
        fromLetClauses = LetClause()
     )?
     (whereClause = WhereClause())?
     (
        groupbyClause = GroupbyClause()
        (
            gbyLetClauses = LetClause()
        )?
        (havingClause = HavingClause())?
     )?
     selectClause = SelectClause()
  )
  {
    if (fromLetClauses != null) {
      fromLetWhereClauses.addAll(fromLetClauses);
    }
    if (whereClause != null) {
      fromLetWhereClauses.add(whereClause);
    }
    if (gbyLetClauses != null) {
      gbyLetHavingClauses.addAll(gbyLetClauses);
    }
    if (havingClause != null) {
      gbyLetHavingClauses.add(havingClause);
    }
    SelectBlock selectBlock = new SelectBlock(selectClause, fromClause, fromLetWhereClauses, groupbyClause,
      gbyLetHavingClauses);
    selectBlock.setSourceLocation(startSrcLoc);
    return selectBlock;
  }
}

SelectClause SelectClause() throws ParseException:
{
  Token startToken = null;
  SelectRegular selectRegular = null;
  SelectElement selectElement = null;
  boolean distinct = false;
}
{
  <SELECT> { startToken = token; } (<ALL>|<DISTINCT> { distinct = true; } )?
  (
    selectRegular = SelectRegular()
    |
    selectElement = SelectElement()
  )?
  {
    SourceLocation sourceLoc = getSourceLocation(startToken);
    if (selectRegular == null && selectElement == null){
        Projection projection = new Projection(null, null, true, false);
        projection.setSourceLocation(sourceLoc);
        List<Projection> projections = new ArrayList<Projection>();
        projections.add(projection);
        selectRegular = new SelectRegular(projections);
        selectRegular.setSourceLocation(sourceLoc);
    }
    SelectClause selectClause = new SelectClause(selectElement, selectRegular, distinct);
    selectClause.setSourceLocation(sourceLoc);
    return selectClause;
  }
}

SelectRegular SelectRegular() throws ParseException:
{
  SourceLocation startSrcLoc = null;
  List<Projection> projections = new ArrayList<Projection>();
  Projection projection = null;
}
{
  projection = Projection()
  {
    projections.add(projection);
    startSrcLoc = projection.getSourceLocation();
  }
  ( LOOKAHEAD(2) <COMMA> projection = Projection()
    {
      projections.add(projection);
    }
  )*
  {
    SelectRegular selectRegular = new SelectRegular(projections);
    selectRegular.setSourceLocation(startSrcLoc);
    return selectRegular;
  }
}

SelectElement SelectElement() throws ParseException:
{
  Token startToken = null;
  Expression expr = null;
  String name = null;
}
{
  (<RAW>|<ELEMENT>|<VALUE>) { startToken = token; } expr = Expression()
  {
    SelectElement selectElement = new SelectElement(expr);
    return addSourceLocation(selectElement, startToken);
  }
}

Projection Projection() throws ParseException :
{
  SourceLocation startSrcLoc = null;
  Expression expr = null;
  Identifier identifier = null;
  String name = null;
  boolean star = false;
  boolean varStar = false;
}
{
  (
    <MUL> { star = true; startSrcLoc = getSourceLocation(token); }
    | LOOKAHEAD(3) expr = VariableRef() <DOT> <MUL> { varStar = true; }
    | expr = Expression() ((<AS>)? name = Identifier())?
      {
        if (name == null) {
          String generatedColumnIdentifier = ExpressionToVariableUtil.getGeneratedIdentifier(expr, false);
          if (generatedColumnIdentifier != null) {
            name = SqlppVariableUtil.toUserDefinedName(generatedColumnIdentifier);
          }
        }
      }
  )
  {
    Projection projection = new Projection(expr, name, star, varStar);
    projection.setSourceLocation(expr != null ? expr.getSourceLocation() : startSrcLoc);
    return projection;
  }
}

FromClause FromClause() throws ParseException :
{
  Token startToken = null;
  List<FromTerm> fromTerms = new ArrayList<FromTerm>();
  extendCurrentScope();
}
{
  {
    FromTerm fromTerm = null;
  }
    <FROM> { startToken = token; } fromTerm = FromTerm() { fromTerms.add(fromTerm); }
    (LOOKAHEAD(2) <COMMA> fromTerm = FromTerm() { fromTerms.add(fromTerm); } )*
  {
    FromClause fromClause = new FromClause(fromTerms);
    return addSourceLocation(fromClause, startToken);
  }
}

FromTerm FromTerm() throws ParseException :
{
  Expression leftExpr = null;
  VariableExpr leftVar = null;
  VariableExpr posVar = null;
  List<AbstractBinaryCorrelateClause> correlateClauses = new ArrayList<AbstractBinaryCorrelateClause>();
}
{
  leftExpr = Expression() ((<AS>)? leftVar = Variable())? (<AT> posVar = Variable())?
  (
     {JoinType joinType = JoinType.INNER; }
     (joinType = JoinType())?
     {
       AbstractBinaryCorrelateClause correlateClause = null;
     }
     (correlateClause = JoinClause(joinType)
      | correlateClause = UnnestClause(joinType)
     )
     {
        correlateClauses.add(correlateClause);
     }
  )*
  {
    if (leftVar == null) {
        leftVar = ExpressionToVariableUtil.getGeneratedVariable(leftExpr, true);
    }
    FromTerm fromTerm = new FromTerm(leftExpr, leftVar, posVar, correlateClauses);
    fromTerm.setSourceLocation(leftExpr.getSourceLocation());
    return fromTerm;
  }
}

JoinClause JoinClause(JoinType joinType) throws ParseException :
{
    Token startToken = null;
    Expression rightExpr = null;
    VariableExpr rightVar = null;
    VariableExpr posVar = null;
    Expression conditionExpr = null;
}
{
  <JOIN> { startToken = token; } rightExpr = Expression() ((<AS>)? rightVar = Variable())? (<AT> posVar = Variable())? <ON> conditionExpr = Expression()
  {
    if(rightVar==null){
        rightVar = ExpressionToVariableUtil.getGeneratedVariable(rightExpr, true);
    }
    JoinClause joinClause = new JoinClause(joinType, rightExpr, rightVar, posVar, conditionExpr);
    return addSourceLocation(joinClause, startToken);
  }
}

UnnestClause UnnestClause(JoinType joinType) throws ParseException :
{
    Token startToken = null;
    Expression rightExpr;
    VariableExpr rightVar;
    VariableExpr posVar = null;
}
{
  (<UNNEST>|<CORRELATE>|<FLATTEN>) { startToken = token; } rightExpr = Expression() ((<AS>)? rightVar = Variable()) (<AT> posVar = Variable())?
  {
    if (rightVar == null) {
      rightVar = ExpressionToVariableUtil.getGeneratedVariable(rightExpr, true);
    }
    UnnestClause unnestClause = new UnnestClause(joinType, rightExpr, rightVar, posVar);
    return addSourceLocation(unnestClause, startToken);
  }
}

JoinType JoinType() throws ParseException :
{
   JoinType joinType = JoinType.INNER;
}
{
     (<INNER>|<LEFT> (<OUTER>)? {joinType = JoinType.LEFTOUTER; })
     {
       return joinType;
     }
}

List<LetClause> LetClause() throws ParseException:
{
    List<LetClause> letList = new ArrayList<LetClause>();
    LetClause letClause;
}
{
    (
     (<LET>|<LETTING>) letClause = LetElement() { letList.add(letClause); } (LOOKAHEAD(1) <COMMA> letClause = LetElement() { letList.add(letClause); })*
     |
     <WITH> letClause = WithElement() { letList.add(letClause); } (LOOKAHEAD(1) <COMMA> letClause = WithElement() { letList.add(letClause); })*
    )
    {
      return letList;
    }
}

WhereClause WhereClause() throws ParseException :
{
  Token startToken = null;
  Expression whereExpr;
}
{
    <WHERE> { startToken = token; } whereExpr = Expression()
    {
      WhereClause wc = new WhereClause(whereExpr);
      return addSourceLocation(wc, startToken);
    }
}

OrderbyClause OrderbyClause() throws ParseException :
{
    Token startToken = null;
    OrderbyClause oc = new OrderbyClause();
    Expression orderbyExpr;
    List<Expression> orderbyList = new ArrayList<Expression>();
    List<OrderbyClause.OrderModifier> modifierList = new ArrayList<OrderbyClause.OrderModifier>();
    int numOfOrderby = 0;
}
{
    <ORDER>
      {
        startToken = token;
        String hint = getHint(token);
        if (hint != null) {
          if (hint.startsWith(INMEMORY_HINT)) {
            String splits[] = hint.split(" +");
            int numFrames = Integer.parseInt(splits[1]);
            int numTuples = Integer.parseInt(splits[2]);
            oc.setNumFrames(numFrames);
            oc.setNumTuples(numTuples);
          }
          if (hint.startsWith(RANGE_HINT)) {
            try {
              oc.setRangeMap(RangeMapBuilder.parseHint(createNewParser(hint.substring(RANGE_HINT.length()))));
            } catch (CompilationException e) {
              throw new SqlppParseException(getSourceLocation(getHintToken(token)), e.getMessage());
            }
          }
        }
      }
    <BY> orderbyExpr = Expression()
    {
      orderbyList.add(orderbyExpr);
      OrderbyClause.OrderModifier modif = OrderbyClause.OrderModifier.ASC;
    }
    ( (<ASC> { modif = OrderbyClause.OrderModifier.ASC; })
    | (<DESC> { modif = OrderbyClause.OrderModifier.DESC; }))?
    {
      modifierList.add(modif);
    }

    (LOOKAHEAD(2) <COMMA> orderbyExpr = Expression()
    {
      orderbyList.add(orderbyExpr);
      modif = OrderbyClause.OrderModifier.ASC;
    }
    ( (<ASC> { modif = OrderbyClause.OrderModifier.ASC; })
    | (<DESC> { modif = OrderbyClause.OrderModifier.DESC; }))?
    {
      modifierList.add(modif);
    }
    )*

    {
      oc.setModifierList(modifierList);
      oc.setOrderbyList(orderbyList);
      return addSourceLocation(oc, startToken);
    }
}

GroupbyClause GroupbyClause()throws ParseException :
{
    Token startToken = null;
    GroupbyClause gbc = new GroupbyClause();
    List<GbyVariableExpressionPair> vePairList = new ArrayList<GbyVariableExpressionPair>();
    VariableExpr var = null;
    Expression expr = null;
    VariableExpr decorVar = null;
    Expression decorExpr = null;
    Pair<VariableExpr, List<Pair<Expression, Identifier>>> groupVarWithFieldList = null;
    VariableExpr groupVar = null;
    List<Pair<Expression, Identifier>> groupFieldList = null;
}
{
      {
        Scope newScope = extendCurrentScopeNoPush(true);
        // extendCurrentScope(true);
      }
    <GROUP>
      {
         startToken = token;
         String hint = getHint(token);
         if (hint != null && hint.equals(HASH_GROUP_BY_HINT)) {
           gbc.setHashGroupByHint(true);
         }
      }
    <BY> (
       expr = Expression()
       (LOOKAHEAD(1) (<AS>)?
        var = Variable()
        )?
        {
            if(var==null){
                var = ExpressionToVariableUtil.getGeneratedVariable(expr, false);
            }
            GbyVariableExpressionPair pair1 = new GbyVariableExpressionPair(var, expr);
            vePairList.add(pair1);
        }
       ( LOOKAHEAD(1) <COMMA>
         {
            var = null;
         }
         expr = Expression()
         (LOOKAHEAD(1)  (<AS>)?
         var = Variable()
         )?
         {
             if(var==null){
                var = ExpressionToVariableUtil.getGeneratedVariable(expr, false);
             }
             GbyVariableExpressionPair pair2 = new GbyVariableExpressionPair(var, expr);
             vePairList.add(pair2);
         }
        )*
    )
    (<GROUP> <AS> groupVarWithFieldList = VariableWithFieldMap()
      {
        groupVar = groupVarWithFieldList.first;
        groupFieldList = groupVarWithFieldList.second;
      }
    )?
    {
      gbc.setGbyPairList(vePairList);
      gbc.setDecorPairList(new ArrayList<GbyVariableExpressionPair>());
      gbc.setWithVarMap(new HashMap<Expression, VariableExpr>());
      gbc.setGroupVar(groupVar);
      gbc.setGroupFieldList(groupFieldList);
      replaceCurrentScope(newScope);
      return addSourceLocation(gbc, startToken);
    }
}

HavingClause HavingClause() throws ParseException:
{
   Token startToken = null;
   Expression filterExpr = null;
}
{
    <HAVING> { startToken = token; } filterExpr = Expression()
    {
       HavingClause havingClause = new HavingClause(filterExpr);
       return addSourceLocation(havingClause, startToken);
    }
}

LimitClause LimitClause() throws ParseException:
{
    Token startToken = null;
    LimitClause lc = new LimitClause();
    Expression expr;
    pushForbiddenScope(getCurrentScope());
}
{
    <LIMIT> { startToken = token; } expr = Expression() { lc.setLimitExpr(expr); }
    (<OFFSET> expr = Expression() { lc.setOffset(expr); })?

  {
    popForbiddenScope();
    return addSourceLocation(lc, startToken);
  }
}

QuantifiedExpression QuantifiedExpression()throws ParseException:
{
  Token startToken = null;
  QuantifiedExpression qc = new QuantifiedExpression();
  List<QuantifiedPair> quantifiedList = new ArrayList<QuantifiedPair>();
  Expression satisfiesExpr;
  VariableExpr var;
  Expression inExpr;
  QuantifiedPair pair;
}
{
  {
    createNewScope();
  }

   ( ((<ANY>|<SOME>) { startToken = token; qc.setQuantifier(QuantifiedExpression.Quantifier.SOME); })
     | (<EVERY> {  startToken = token; qc.setQuantifier(QuantifiedExpression.Quantifier.EVERY); }))
    var = Variable() <IN> inExpr = Expression()
    {
      pair = new QuantifiedPair(var, inExpr);
      quantifiedList.add(pair);
    }
    (
    <COMMA> var = Variable() <IN> inExpr = Expression()
    {
      pair = new QuantifiedPair(var, inExpr);
      quantifiedList.add(pair);
    }
    )*
     <SATISFIES> satisfiesExpr = Expression() (<END>)?
     {
       qc.setSatisfiesExpr(satisfiesExpr);
       qc.setQuantifiedList(quantifiedList);
       removeCurrentScope();
       return addSourceLocation(qc, startToken);
     }
}

LetClause LetElement() throws ParseException:
{
    VariableExpr varExp;
    Expression beExp;
    extendCurrentScope();
}
{
    varExp = Variable() <EQ> beExp = Expression()
    {
      LetClause lc = new LetClause(varExp, beExp);
      lc.setSourceLocation(varExp.getSourceLocation());
      return lc;
    }
}

LetClause WithElement() throws ParseException:
{
    VariableExpr varExp;
    Expression beExp;
    extendCurrentScope();
}
{
    varExp = Variable() <AS> beExp = Expression()
    {
      LetClause lc = new LetClause(varExp, beExp);
      lc.setSourceLocation(varExp.getSourceLocation());
      return lc;
    }
}

TOKEN_MGR_DECLS:
{
    public int commentDepth = 0;
    public ArrayDeque<Integer> lexerStateStack = new ArrayDeque<Integer>();

    public void pushState() {
      lexerStateStack.push( curLexState );
    }

    public void popState(String token) {
      if (lexerStateStack.size() > 0) {
         SwitchTo( lexerStateStack.pop() );
      } else {
         int errorLine = input_stream.getEndLine();
         int errorColumn = input_stream.getEndColumn();
         String msg = "Lexical error at line " + errorLine + ", column " + errorColumn + ". Encountered \"" + token
             + "\" but state stack is empty.";
         throw new TokenMgrError(msg, -1);
      }
    }
}

<DEFAULT,IN_DBL_BRACE>
TOKEN [IGNORE_CASE]:
{
  <ALL : "all">
  | <AND : "and">
  | <ANY : "any">
  | <APPLY : "apply">
  | <AS : "as">
  | <ASC : "asc">
  | <AT : "at">
  | <AUTOGENERATED : "autogenerated">
  | <BETWEEN : "between">
  | <BTREE : "btree">
  | <BY : "by">
  | <CASE : "case">
  | <CLOSED : "closed">
  | <CREATE : "create">
  | <COMPACTION : "compaction">
  | <COMPACT : "compact">
  | <CONNECT : "connect">
  | <CORRELATE : "correlate">
  | <DATASET : "dataset">
  | <COLLECTION : "collection">
  | <DATAVERSE : "dataverse">
  | <DECLARE : "declare">
  | <DEFINITION : "definition">
  | <DELETE : "delete">
  | <DESC : "desc">
  | <DISCONNECT : "disconnect">
  | <DISTINCT : "distinct">
  | <DROP : "drop">
  | <ELEMENT : "element">
  | <EXPLAIN : "explain">
  | <ELSE : "else">
  | <ENFORCED : "enforced">
  | <END : "end">
  | <EVERY : "every">
  | <EXCEPT : "except">
  | <EXISTS : "exists">
  | <EXTERNAL : "external">
  | <FEED : "feed">
  | <FILTER : "filter">
  | <FLATTEN : "flatten">
  | <FOR : "for">
  | <FROM : "from">
  | <FULL : "full">
  | <FULLTEXT : "fulltext">
  | <FUNCTION : "function">
  | <GROUP : "group">
  | <HAVING : "having">
  | <HINTS : "hints">
  | <IF : "if">
  | <INTO : "into">
  | <IN : "in">
  | <INDEX : "index">
  | <INGESTION : "ingestion">
  | <INNER : "inner">
  | <INSERT : "insert">
  | <INTERNAL : "internal">
  | <INTERSECT : "intersect">
  | <IS : "is">
  | <JOIN : "join">
  | <KEYWORD : "keyword">
  | <KEY : "key">
  | <KNOWN : "known">
  | <LEFT : "left">
  | <LETTING : "letting">
  | <LET : "let">
  | <LIKE : "like">
  | <LIMIT : "limit">
  | <LOAD : "load">
  | <NODEGROUP : "nodegroup">
  | <NGRAM : "ngram">
  | <NOT : "not">
  | <OFFSET : "offset">
  | <ON : "on">
  | <OPEN : "open">
  | <OR : "or">
  | <ORDER : "order">
  | <OUTER : "outer">
  | <OUTPUT : "output">
  | <OVER: "over">
  | <PATH : "path">
  | <POLICY : "policy">
  | <PRESORTED : "pre-sorted">
  | <PRIMARY : "primary">
  | <RAW : "raw">
  | <REFRESH : "refresh">
  | <RETURN : "return">
  | <RETURNING : "returning">
  | <RTREE : "rtree">
  | <RUN : "run">
  | <SATISFIES : "satisfies">
  | <SECONDARY : "secondary">
  | <SELECT : "select">
  | <SET : "set">
  | <SOME : "some">
  | <START : "start">
  | <STOP : "stop">
  | <TEMPORARY : "temporary"> // intentionally not used but reserved for future usage
  | <THEN : "then">
  | <TYPE : "type">
  | <TO : "to">
  | <UNION : "union">
  | <UNKNOWN : "unknown">
  | <UNNEST : "unnest">
  | <UPDATE : "update">
  | <UPSERT : "upsert">
  | <USE : "use">
  | <USING : "using">
  | <VALUE : "value">
  | <VALUED : "valued">
  | <WHEN : "when">
  | <WHERE : "where">
  | <WITH : "with">
  | <WRITE : "write">
}

<DEFAULT,IN_DBL_BRACE>
TOKEN :
{
    <CARET : "^">
  | <CONCAT : "||">
  | <DIVIDE : "/">
  | <DIV : "div">
  | <MINUS : "-">
  | <MOD : "mod">
  | <MUL : "*">
  | <PLUS : "+">

  | <LEFTPAREN : "(">
  | <RIGHTPAREN : ")">
  | <LEFTBRACKET : "[">
  | <RIGHTBRACKET : "]">

  | <ATT : "@">
  | <COLON : ":">
  | <COMMA : ",">
  | <DOLLAR: "$">
  | <DOT : ".">
  | <PERCENT: "%">
  | <QUES : "?">
  | <SEMICOLON : ";">
  | <SHARP : "#">

  | <LT : "<">
  | <GT : ">">
  | <LE : "<=">
  | <GE : ">=">
  | <EQ : "=">
  | <NE : "!=">
  | <LG : "<>">
  | <SIMILAR : "~=">
}

<DEFAULT,IN_DBL_BRACE>
TOKEN :
{
    <LEFTBRACE : "{"> { pushState(); } : DEFAULT
}

<DEFAULT>
TOKEN :
{
    <RIGHTBRACE : "}"> { popState("}"); }
}

<DEFAULT,IN_DBL_BRACE>
TOKEN :
{
    <LEFTDBLBRACE : "{{"> { pushState(); } : IN_DBL_BRACE
}

<IN_DBL_BRACE>
TOKEN :
{
    <RIGHTDBLBRACE : "}}"> { popState("}}"); }
}

<DEFAULT,IN_DBL_BRACE>
TOKEN :
{
    <INTEGER_LITERAL : (<DIGIT>)+ >
}

<DEFAULT,IN_DBL_BRACE>
TOKEN [IGNORE_CASE]:
{
    <MISSING : "missing">
  | <NULL : "null">
  | <TRUE : "true">
  | <FALSE : "false">
}

<DEFAULT,IN_DBL_BRACE>
TOKEN :
{
    <#DIGIT : ["0" - "9"]>
}

<DEFAULT,IN_DBL_BRACE>
TOKEN:
{
    < DOUBLE_LITERAL: <DIGITS> ( "." <DIGITS> ) (("e"|"E") ("+"|"-")? <DIGITS>)?
                      | <DIGITS> (("e"|"E") ("+"|"-")? <DIGITS>)
                      | "." <DIGITS> (("e"|"E") ("+"|"-")? <DIGITS>)?
    >
  | < FLOAT_LITERAL:  <DIGITS> ( "f" | "F" )
                      | <DIGITS> ( "." <DIGITS> ( "f" | "F" ) )?
                      | "." <DIGITS> ( "f" | "F" )
    >
  | <DIGITS : (<DIGIT>)+ >
}

<DEFAULT,IN_DBL_BRACE>
TOKEN :
{
    <#LETTER : ["A" - "Z", "a" - "z"]>
  | <#IDENTIFIER_SPECIALCHARS_START : ["_"]>
  | <#IDENTIFIER_SPECIALCHARS_REST : ["$"]>
}

<DEFAULT,IN_DBL_BRACE>
TOKEN :
{
    // backslash u + 4 hex digits escapes are handled in the underlying JavaCharStream
    <QUOTED_STRING : "`" (
          <EscapeQuot>
        | <EscapeBslash>
        | <EscapeSlash>
        | <EscapeBspace>
        | <EscapeFormf>
        | <EscapeNl>
        | <EscapeCr>
        | <EscapeTab>
        | ~["`","\\"])* "`">
  | <STRING_LITERAL : ("\"" (
          <EscapeQuot>
        | <EscapeBslash>
        | <EscapeSlash>
        | <EscapeBspace>
        | <EscapeFormf>
        | <EscapeNl>
        | <EscapeCr>
        | <EscapeTab>
        | ~["\"","\\"])* "\"")
      | ("\'"(
          <EscapeApos>
        | <EscapeBslash>
        | <EscapeSlash>
        | <EscapeBspace>
        | <EscapeFormf>
        | <EscapeNl>
        | <EscapeCr>
        | <EscapeTab>
        | ~["\'","\\"])* "\'")>
  | < #EscapeQuot: "\\\"" >
  | < #EscapeApos: "\\\'" >
  | < #EscapeBslash: "\\\\" >
  | < #EscapeSlash: "\\/" >
  | < #EscapeBspace: "\\b" >
  | < #EscapeFormf: "\\f" >
  | < #EscapeNl: "\\n" >
  | < #EscapeCr: "\\r" >
  | < #EscapeTab: "\\t" >
}

<DEFAULT,IN_DBL_BRACE>
TOKEN :
{
    <IDENTIFIER : ( <LETTER> | <IDENTIFIER_SPECIALCHARS_START> )
                  ( <LETTER> | <DIGIT> | <IDENTIFIER_SPECIALCHARS_START> | <IDENTIFIER_SPECIALCHARS_REST> )*>
}

<DEFAULT,IN_DBL_BRACE>
SKIP:
{
    " "
  | "\t"
  | "\r"
  | "\n"
}

<DEFAULT,IN_DBL_BRACE>
SKIP:
{
    <"//" (~["\n"])* "\n">
}

<DEFAULT,IN_DBL_BRACE>
SKIP:
{
    <"//" (~["\n","\r"])* ("\n"|"\r"|"\r\n")?>
}

<DEFAULT,IN_DBL_BRACE>
SKIP:
{
    <"--" (~["\n"])* "\n">
}


<DEFAULT,IN_DBL_BRACE>
SKIP:
{
    <"--" (~["\n","\r"])* ("\n"|"\r"|"\r\n")?>
}

<DEFAULT,IN_DBL_BRACE>
SKIP:
{
    <"/*"> { pushState(); } : INSIDE_COMMENT
}

<INSIDE_COMMENT>
SPECIAL_TOKEN:
{
    <"+"(" ")*(~["*"])*>
}

<INSIDE_COMMENT>
SKIP:
{
    <"/*"> { pushState(); }
}

<INSIDE_COMMENT>
SKIP:
{
    <"*/"> { popState("*/"); }
  | <~[]>
}