asterix-aql/src/main/javacc/AQL.jj

options {

	  
       STATIC = false;
	
}


PARSER_BEGIN(AQLParser)

package edu.uci.ics.asterix.aql.parser;

import java.io.*;
import java.util.List;
import java.util.ArrayList;
import java.util.Stack;

import java.util.Map;
import java.util.HashMap;
import java.util.LinkedHashMap;
import edu.uci.ics.asterix.aql.literal.FloatLiteral;
import edu.uci.ics.asterix.aql.literal.DoubleLiteral;
import edu.uci.ics.asterix.aql.literal.FalseLiteral;
import edu.uci.ics.asterix.aql.base.Literal;
import edu.uci.ics.asterix.aql.literal.IntegerLiteral;
import edu.uci.ics.asterix.aql.literal.LongIntegerLiteral;
import edu.uci.ics.asterix.aql.literal.NullLiteral;
import edu.uci.ics.asterix.aql.literal.StringLiteral;
import edu.uci.ics.asterix.aql.literal.TrueLiteral;
import edu.uci.ics.asterix.metadata.bootstrap.MetadataConstants;

import edu.uci.ics.asterix.aql.base.*;
import edu.uci.ics.asterix.aql.expression.*;
import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
import edu.uci.ics.asterix.aql.expression.visitor.AQLPrintVisitor;
import edu.uci.ics.asterix.aql.expression.UnaryExpr.Sign;
import edu.uci.ics.asterix.aql.base.Statement.Kind;
import edu.uci.ics.asterix.aql.context.Scope;
import edu.uci.ics.asterix.aql.context.RootScopeFactory;
import edu.uci.ics.asterix.common.annotations.*;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.om.functions.AsterixFunction;
import edu.uci.ics.asterix.common.functions.FunctionSignature;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionAnnotation;
import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IndexedNLJoinExpressionAnnotation;
import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.algebricks.common.utils.Triple;




public class AQLParser extends ScopeChecker {

    // optimizer hints
    private static final String HASH_GROUP_BY_HINT = "hash";
    private static final String BROADCAST_JOIN_HINT = "bcast";
    private static final String INDEXED_NESTED_LOOP_JOIN_HINT = "indexnl";
    private static final String INMEMORY_HINT = "inmem";
    private static final String VAL_FILE_HINT = "val-files";
    private static final String VAL_FILE_SAME_INDEX_HINT = "val-file-same-idx";
    private static final String INTERVAL_HINT = "interval";
    private static final String COMPOSE_VAL_FILES_HINT = "compose-val-files";
    private static final String INSERT_RAND_INT_HINT = "insert-rand-int";
    private static final String LIST_VAL_FILE_HINT = "list-val-file";
    private static final String LIST_HINT = "list";
    private static final String DATETIME_BETWEEN_YEARS_HINT = "datetime-between-years";
    private static final String DATE_BETWEEN_YEARS_HINT = "date-between-years";
    private static final String DATETIME_ADD_RAND_HOURS_HINT = "datetime-add-rand-hours";
    private static final String AUTO_HINT = "auto";   
        
    private static final String GEN_FIELDS_HINT = "gen-fields";   
    
    // data generator hints
    private static final String DGEN_HINT = "dgen";
   
    private static String getHint(Token t) {
       if (t.specialToken == null) {
         return null;
       }       
       String s = t.specialToken.image;
       int n = s.length();
       if (n < 2) {
         return null;
       }  
       return s.substring(1).trim();
    }

    public AQLParser(String s){
		this(new StringReader(s));
		super.setInput(s);
	}
	
	public static void main(String args[]) throws ParseException, TokenMgrError, IOException, FileNotFoundException, AsterixException {
			File file = new File(args[0]);
			Reader fis = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
		    AQLParser parser = new AQLParser(fis);
		    List<Statement> st = parser.Statement();
		    //st.accept(new AQLPrintVisitor(), 0);
	}


}

PARSER_END(AQLParser)


List<Statement> Statement() throws ParseException:
{
  scopeStack.push(RootScopeFactory.createRootScope(this));
  List<Statement> decls = new ArrayList<Statement>();
  Query query=null;
}
{
    (
      (
        ( 
          "use"
            {
              decls.add(DataverseDeclaration());
            }           
          | "declare" "function" { 
                              decls.add(FunctionDeclaration()); 
             }
	  	   |  "create" (
	  	   	  {
                String hint = getHint(token);
                boolean dgen = false;
         	   	if (hint != null && hint.startsWith(DGEN_HINT)) {
         	   	  dgen = true;
         	   	}                  
              } 
              "type"     
              	{ 
                              decls.add(TypeDeclaration(dgen, hint)); 
                 }
              | "nodegroup" 
              	{
                              decls.add(NodegroupDeclaration());
                }   
              | "external" <DATASET>
            	{   
              		decls.add(DatasetDeclaration(DatasetType.EXTERNAL));
            	}
              | "feed" <DATASET>
                {
                   decls.add(DatasetDeclaration(DatasetType.FEED)); 	
            	}
              | <DATASET>
          		{
            		decls.add(DatasetDeclaration(DatasetType.INTERNAL));
          		}
              | "index" 
              	{
              				decls.add(CreateIndexStatement());
                 }
	          | "dataverse"
          		{
            		decls.add(CreateDataverseStatement());
          		}
          	  | "function"
          	    {
          	        decls.add(FunctionCreation());
          	    }	
            )        
         	 | "load" {
                       decls.add(LoadStatement());
                   }  
          	                    
          	| "drop"
			(
          	<DATASET>
          	{
            		decls.add(DropStatement());
          	}
        	| "index"
          	{
            		decls.add(IndexDropStatement());
          	}
        	| "nodegroup"
          	{
            		decls.add(NodeGroupDropStatement());
          	}
        	| "type"
          	{
            		decls.add(TypeDropStatement());
          	}
        	| "dataverse"
          	{
            		decls.add(DataverseDropStatement());
          	}
          	| "function"
          	{
            		decls.add(FunctionDropStatement());
          	}
        	)
          | "write" {
                       decls.add(WriteStatement());
                    }              
          | "set" {
                       decls.add(SetStatement());
                    }                                                      
	  	  | "insert" {
		       decls.add(InsertStatement());
			}
          | "delete" {
			decls.add(DeleteStatement());
		    }
          | "update" {
	  		decls.add(UpdateStatement());		
	  	    } 
	  	  | "begin" "feed"  
	  	      {
                Pair<Identifier,Identifier> nameComponents = getDotSeparatedPair();
                decls.add(new BeginFeedStatement(nameComponents.first, nameComponents.second, getVarCounter()));
	  	      } ";"
	  	      
	  	  | "suspend" "feed"  
	  	  	 {
                decls.add(ControlFeedDeclaration(ControlFeedStatement.OperationType.SUSPEND));
             } ";"
	  	   | "resume" "feed"   {
                decls.add(ControlFeedDeclaration(ControlFeedStatement.OperationType.RESUME));
	  	   } ";"
	  	   | "end" "feed"   {
	  	        decls.add(ControlFeedDeclaration(ControlFeedStatement.OperationType.END));
	  	   } ";" 
	  	   | "alter" "feed"  {
      	        decls.add(AlterFeedDeclaration());
           } ";"
           
           | (query = Query()) {
               decls.add(query);
           }
           )*
          //  (query = Query())?
      )

      <EOF>
    )
    {
     return decls;  
    }
}

InsertStatement InsertStatement() throws ParseException:
{
	Identifier dataverseName;
	Identifier datasetName;
	Pair<Identifier,Identifier> nameComponents = null;
	Query query;
}
{
   "into" <DATASET>
   
   {
    nameComponents = getDotSeparatedPair();
    dataverseName = nameComponents.first;
    datasetName = nameComponents.second;
   }
    
    query = Query() (";")?
   {return new InsertStatement(dataverseName, datasetName, query,  getVarCounter());}
}

DeleteStatement DeleteStatement() throws ParseException:
{
	VariableExpr var = null;
    Identifier dataverseName;
    Identifier datasetName = null;
	Expression condition = null;
	Clause dieClause = null;
	Pair<Identifier, Identifier> nameComponents;
}
{
   var = Variable() { getCurrentScope().addNewVarSymbolToScope(var.getVar());  }
   "from" 
   <DATASET> 
   { 
	  nameComponents  = getDotSeparatedPair();
   }
   ("where" condition = Expression())?  (dieClause = DieClause())? (";")?
   {return new DeleteStatement(var, nameComponents.first, nameComponents.second, condition,  dieClause, getVarCounter()); }
}

UpdateStatement UpdateStatement() throws ParseException:
{
	VariableExpr vars;
    Expression target;
	Expression condition;
	UpdateClause uc;
 	List<UpdateClause> ucs = new ArrayList<UpdateClause>();
}
{
   vars = Variable()  "in" target = Expression()
	"where" condition = Expression() 
	<LEFTPAREN> (uc=UpdateClause() {ucs.add(uc); }  ("," uc=UpdateClause() {ucs.add(uc); } )*) <RIGHTPAREN> ";"
   {return new UpdateStatement(vars, target, condition, ucs);}
}



UpdateClause UpdateClause() throws ParseException:
{
	Expression target = null;
	Expression value = null ;	
	InsertStatement is = null;
	DeleteStatement ds = null;
	UpdateStatement us = null;
	Expression condition = null;
	UpdateClause ifbranch = null;
	UpdateClause elsebranch = null;
}
{
   "set" target = Expression() ":=" value = Expression() 
   | "insert" is = InsertStatement()
   | "delete" ds = DeleteStatement()
   | "update" us = UpdateStatement()
   | "if" <LEFTPAREN> condition = Expression() <RIGHTPAREN> "then" ifbranch = UpdateClause() [LOOKAHEAD(1) "else" elsebranch = UpdateClause()] 
   {return new UpdateClause(target, value, is, ds, us, condition, ifbranch, elsebranch);}
}


Statement SetStatement() throws ParseException:
{
  String pn = null;
  Statement stmt = null;
}
{
  <IDENTIFIER>  { pn = token.image; }  
  <STRING_LITERAL>
    { String pv = removeQuotesAndEscapes(token.image); }
    ";"
  {
    return new SetStatement(pn, pv);
  }
}

Statement WriteStatement() throws ParseException:
{
  Identifier nodeName = null;
  String fileName = null;
  Statement stmt = null;
  Query query;
  String writerClass = null;
  Pair<Identifier,Identifier> nameComponents = null;
}
{
  (( "output" "to" 
    <IDENTIFIER> { nodeName = new Identifier(token.image); } 
    ":" <STRING_LITERAL> { fileName = removeQuotesAndEscapes(token.image); }
    ( "using" <STRING_LITERAL> { writerClass = removeQuotesAndEscapes(token.image); } )?
         {                  
             stmt = new WriteStatement(nodeName, fileName, writerClass);         
         } )
    |
   ( "into" 
     <DATASET> 
     
      {
       nameComponents = getDotSeparatedPair();
      }
     
     <LEFTPAREN> query = Query() <RIGHTPAREN>
     {
        stmt = new WriteFromQueryResultStatement(nameComponents.first, nameComponents.second, query, getVarCounter());
     } ))  
        
    ";"
    {
      return stmt;
    }
}

CreateIndexStatement CreateIndexStatement() throws ParseException:
{
  CreateIndexStatement cis = new CreateIndexStatement();
  Pair<Identifier,Identifier> nameComponents = null;
}
{
  <IDENTIFIER> { cis.setIndexName(new Identifier(token.image)); }
  (
    "if not exists"
    {
      cis.setIfNotExists(true);
    }
  )?
  "on"  
  
   {
   nameComponents = getDotSeparatedPair();
   cis.setDataverseName(nameComponents.first);
   cis.setDatasetName(nameComponents.second);
   }
  
  <LEFTPAREN>
  	( <IDENTIFIER> { cis.addFieldExpr(token.image); } )
  	("," <IDENTIFIER> { cis.addFieldExpr(token.image); })*
  <RIGHTPAREN>
    ("type"
  		("btree" { cis.setIndexType(IndexType.BTREE); }  		
  		| "rtree" { cis.setIndexType(IndexType.RTREE); }
  		| "keyword" { cis.setIndexType(IndexType.WORD_INVIX); }
  		| "fuzzy keyword" { cis.setIndexType(IndexType.FUZZY_WORD_INVIX); }
  		| "ngram"
  		  <LEFTPAREN>
  		  (<INTEGER_LITERAL>
  		    {
  		      cis.setIndexType(IndexType.NGRAM_INVIX);
  		      cis.setGramLength(Integer.valueOf(token.image));
  		    }
  		  )
  		  <RIGHTPAREN>
  		| "fuzzy ngram"
  		  <LEFTPAREN>
  		  (<INTEGER_LITERAL>
  		    {
  		      cis.setIndexType(IndexType.FUZZY_NGRAM_INVIX);
  		      cis.setGramLength(Integer.valueOf(token.image));
  		    }
  		  )
  		  <RIGHTPAREN>
		)
  	";"  	
  	| ";"
    )
   {
     return cis;
   }
}

DataverseDecl DataverseDeclaration() throws ParseException:
{
  Identifier dvName = null;  
}
{
  "dataverse" <IDENTIFIER> { defaultDataverse = token.image;}
  ";"
  {
    return new DataverseDecl(new Identifier(defaultDataverse));
  }
}

DropStatement DropStatement() throws ParseException :
{
  Identifier dataverseName = null;
  Identifier datasetName = null;
  boolean ifExists = false;
  Pair<Identifier,Identifier> nameComponents=null;
}
{
   {
   nameComponents = getDotSeparatedPair();
   dataverseName = nameComponents.first;
   datasetName = nameComponents.second;
   }
   
   
  (
    "if exists"
    {
      ifExists = true;
    }
  )? ";"
  {
    return new DropStatement(dataverseName, datasetName, ifExists);
  }
}

IndexDropStatement IndexDropStatement() throws ParseException :
{
  Identifier dataverseName = null;
  Identifier datasetName = null;
  Identifier indexName = null;
  boolean ifExists = false;
  Triple<Identifier,Identifier,Identifier> nameComponents=null;
}
{
  
  {
   nameComponents = getDotSeparatedTriple();
   dataverseName = nameComponents.first;
   datasetName = nameComponents.second;
   indexName = nameComponents.third;
   }
  
  (
    "if exists"
    {
      ifExists = true;
    }
  )? ";"
  {
    return new IndexDropStatement(dataverseName, datasetName, indexName, ifExists);
  }
}

NodeGroupDropStatement NodeGroupDropStatement() throws ParseException :
{
  Identifier groupName = null;
  boolean ifExists = false;
}
{
  < IDENTIFIER >
  {
    groupName = new Identifier(token.image);
  }
  (
    "if exists"
    {
      ifExists = true;
    }
  )? ";"
  {
    return new NodeGroupDropStatement(groupName, ifExists);
  }
}

TypeDropStatement TypeDropStatement() throws ParseException :
{
  Identifier dataverseName = null;
  Identifier typeName = null;
  boolean ifExists = false;
  Pair<Identifier,Identifier> nameComponents;
}
{
  {
    nameComponents = getDotSeparatedPair();
    dataverseName = nameComponents.first == null ? new Identifier(defaultDataverse) : nameComponents.first;
    typeName = nameComponents.second;
  }
  (
    "if exists"
    {
      ifExists = true;
    }
  )? ";"
  {
    return new TypeDropStatement(dataverseName, typeName, ifExists);
  }
}

DataverseDropStatement DataverseDropStatement() throws ParseException :
{
  Identifier dataverseName = null;
  boolean ifExists = false;
}
{
  < IDENTIFIER >
  {
    dataverseName = new Identifier(token.image);
  }
  (
    "if exists"
    {
      ifExists = true;
    }
  )? ";"
  {
    return new DataverseDropStatement(dataverseName, ifExists);
  }
}

CreateDataverseStatement CreateDataverseStatement() throws ParseException :
{
  Identifier dvName = null;
  boolean ifNotExists = false;
  String format = null;
}
{
  < IDENTIFIER >
  {
    dvName = new Identifier(token.image);
  }
  (
    "if not exists"
    {
      ifNotExists = true;
    }
  )?
  (
    "with format" < STRING_LITERAL >
    {
      format = removeQuotesAndEscapes(token.image);
    }
  )?
  ";"
  {
    return new CreateDataverseStatement(dvName, format, ifNotExists);
  }
}


FunctionDropStatement FunctionDropStatement() throws ParseException :
{
  String dataverse;
  String functionName;
  int arity=0;
  boolean ifExists = false;
  Pair<Identifier, Identifier> nameComponents=null;
}
{
  {
     nameComponents = getDotSeparatedPair();
     dataverse = nameComponents.first != null ? nameComponents.first.getValue() : defaultDataverse;
     functionName = nameComponents.second.getValue(); 
  }
  
   "@"
  <INTEGER_LITERAL> 
  {  
     Token t= getToken(0);
	 arity = new Integer(t.image);
  	 if( arity < 0 && arity != FunctionIdentifier.VARARGS){
  	 	throw new ParseException(" invalid arity:" + arity);
  	 } 
  }
  
  (
    "if exists"
    {
      ifExists = true;
    }
  )? ";"
  {
    return new FunctionDropStatement(new FunctionSignature(dataverse, functionName, arity), ifExists);
  }
}


LoadFromFileStatement LoadStatement() throws ParseException:
{
  Identifier dataverseName = null;
  Identifier datasetName = null;
  boolean alreadySorted = false;
  String adapterName;
  Map<String,String> properties;
  Pair<Identifier,Identifier> nameComponents = null;
}
{
   <DATASET> 
   {
   nameComponents = getDotSeparatedPair();
   dataverseName = nameComponents.first;
   datasetName = nameComponents.second;
   }
   
   "using"
  
    {
    	adapterName = getAdapterName();
    }
   
    {
      properties = getConfiguration();
    }
  
    ("pre-sorted" 
      {  alreadySorted = true; }
    )?
        
  ";"
  {
     return new LoadFromFileStatement(dataverseName, datasetName, adapterName, properties, alreadySorted);
  }   
}


String getAdapterName() throws ParseException :
{
	String adapterName = null;
}
{
    ( 
      <IDENTIFIER> {
     	adapterName = (new Identifier(token.image)).getValue();; 
      }
      | 
      <STRING_LITERAL>
      {
        adapterName = removeQuotesAndEscapes(token.image);
      }
    )
    {
	return adapterName;
	}
}


DatasetDecl DatasetDeclaration(DatasetType datasetType) throws ParseException :
{
  DatasetDecl dd = null;
  Identifier datasetName = null;
  Identifier dataverseName = null;
  Identifier itemDataverseName = null;
  Identifier itemTypeName = null;
  String nameComponentFirst = null;
  String nameComponentSecond = null;
  boolean ifNotExists = false;
  IDatasetDetailsDecl datasetDetails = null;
  Pair<Identifier,Identifier> nameComponents = null;
  Map<String,String> hints = new HashMap<String,String>();	
}
{
  {
   nameComponents = getDotSeparatedPair();
   dataverseName = nameComponents.first;
   datasetName = nameComponents.second;
   }
    
  (
    "if not exists"
    {
      ifNotExists = true;
    }
  )?
  (
  	< LEFTPAREN > <IDENTIFIER>
  	{
    	itemTypeName = new Identifier(token.image);
  	}
  	< RIGHTPAREN >
  )
  {
  	  if(datasetType == DatasetType.INTERNAL) {
      	datasetDetails = InternalDatasetDeclaration();
      }
      else if(datasetType == DatasetType.EXTERNAL) {
      	datasetDetails = ExternalDatasetDeclaration();
      }
      else if(datasetType == DatasetType.FEED) {
      	datasetDetails = FeedDatasetDeclaration();
      }
  }
  
  (
  "hints"
  {
      initProperties(hints);
  }
  )?
   ";"
 
  {
   dd = new DatasetDecl(dataverseName, datasetName, itemTypeName, hints, datasetType, datasetDetails,ifNotExists);
   return dd;
  }
}

InternalDetailsDecl InternalDatasetDeclaration() throws ParseException :
{
    InternalDetailsDecl idd = null;
    List<String> primaryKeyFields = new ArrayList<String>();
    Identifier nodeGroupName=null;
}
{
  (
    {
  	  primaryKeyFields = getPrimaryKeyFields();
    }
  )
  
  (
  "on" < IDENTIFIER >
    {
        nodeGroupName = new Identifier(token.image);
    }
  )?
  
  {
    idd = new InternalDetailsDecl(nodeGroupName, primaryKeyFields);
    return idd;
  }
}

ExternalDetailsDecl ExternalDatasetDeclaration() throws ParseException :
{
  ExternalDetailsDecl edd = null;
  String adapterName = null;
  Map < String, String > properties;
}
{
  {
    edd = new ExternalDetailsDecl();
  }
 
    "using"
    {
    	adapterName = getAdapterName();
    }

    {
      properties = getConfiguration();
    }

    {
    	  edd = new ExternalDetailsDecl();
		  edd.setAdapter(adapterName);
   		  edd.setProperties(properties);
    } 
 
  {
    return edd;
  }
}

FeedDetailsDecl FeedDatasetDeclaration() throws ParseException :
{
    FeedDetailsDecl fdd = null;
    String adapterName = null;
    Map < String, String > properties;
	Pair<Identifier,Identifier> nameComponents;
	List<String> primaryKeyFields = new ArrayList<String>();
    Identifier nodeGroupName=null;
    FunctionSignature appliedFunction=null;
	String dataverse;
	String functionName;
	String libraryName;
	int arity;
	Triple<String,String,String> functionCallComponents = null;
}
{
   "using"
    {
    	adapterName = getAdapterName();
    }

    {
      properties = getConfiguration();
    }
  
  ("apply" "function" 
   (
      ( functionCallComponents = getFunctionCallComponents() )
       {
         dataverse = functionCallComponents.first == null ? defaultDataverse : functionCallComponents.first;
         libraryName = functionCallComponents.second;
         functionName = functionCallComponents.third;
         if(libraryName != null){
          functionName = libraryName + ":" + functionName;
         }
       }
     )
     
  ("@" <INTEGER_LITERAL> 
     {
        arity = Integer.parseInt(token.image);
     }
  )
  
  {
    appliedFunction = new FunctionSignature(dataverse, functionName, arity);
  }   
  )?
  
  (
    {
  	  primaryKeyFields  = getPrimaryKeyFields();
    }
  )
  
  (
  "on" < IDENTIFIER >
  {
    	nodeGroupName = new Identifier(token.image);
  }
  )?
  
  {
    fdd = new FeedDetailsDecl(adapterName, properties, appliedFunction, nodeGroupName, primaryKeyFields);
    return fdd;
  }
}

List<String> getPrimaryKeyFields()  throws ParseException :
{
	List<String> primaryKeyFields = new ArrayList<String>();
}
{

  "primary" "key"
  < IDENTIFIER >
  {
    	 primaryKeyFields.add(token.image);
  }
  (
    "," < IDENTIFIER >
    {
    	 primaryKeyFields.add(token.image);
    }
  )*
  {
   	return primaryKeyFields;
  }
  
}





ControlFeedStatement ControlFeedDeclaration(ControlFeedStatement.OperationType operationType) throws ParseException :
{
  Pair<Identifier,Identifier> nameComponents = null;
}
{
    {
    nameComponents = getDotSeparatedPair();
    return new ControlFeedStatement(operationType, nameComponents.first, nameComponents.second);
    }
}


ControlFeedStatement AlterFeedDeclaration() throws ParseException :
{
    Pair<Identifier,Identifier> nameComponents = null;
    Map < String, String > configuration = new HashMap < String, String > ();
}
{
   {
    nameComponents = getDotSeparatedPair();
   }

   "set"
   { 
   configuration = getConfiguration();
   }
  
  {
    return new ControlFeedStatement(ControlFeedStatement.OperationType.ALTER, nameComponents.first, nameComponents.second, configuration);
  }
}

Map<String,String> getConfiguration()  throws ParseException :
{
	Map<String,String> configuration = new LinkedHashMap<String,String>();
	String key;
	String value;
}
{

<LEFTPAREN>
    (
      (
        <LEFTPAREN>
        (
          <STRING_LITERAL>
          {
            key = removeQuotesAndEscapes(token.image);
          }
          "=" <STRING_LITERAL>
          {
            value = removeQuotesAndEscapes(token.image);
          }
        )
        <RIGHTPAREN>
        {
          configuration.put(key, value);
        }
      )
      (
        "," <LEFTPAREN>
        (
          <STRING_LITERAL>
          {
            key = removeQuotesAndEscapes(token.image);
          }
          "=" <STRING_LITERAL>
          {
            value = removeQuotesAndEscapes(token.image);
          }
        )
        <RIGHTPAREN>
        {
          configuration.put(key, value);
        }
      )*
    )?
    <RIGHTPAREN>
     {
     	return configuration;
     }
}

void initProperties(Map<String,String> properties)  throws ParseException :
{
	String key;
	String value;
}
{
    (
      <LEFTPAREN>
        (
          <IDENTIFIER>
          {
            key = (new Identifier(token.image)).getValue();
          }
          "=" 
          (
            (<STRING_LITERAL>
             {
              value = removeQuotesAndEscapes(token.image);
             }
            ) |
            (<INTEGER_LITERAL>
             {
             try{
              value = "" + Long.valueOf(token.image);
              } catch (NumberFormatException nfe){
                  throw new ParseException("inapproriate value: " + token.image); 
              }
             } 
            )
          )
        {
          properties.put(key.toUpperCase(), value);
        }
       ( 
        "," 
        (
          <IDENTIFIER>
          {
            key = (new Identifier(token.image)).getValue();
          }
          "=" 
          (
           (<STRING_LITERAL>
            {
              value = removeQuotesAndEscapes(token.image);
            }
           ) |
           (<INTEGER_LITERAL>
            {
              try{
                value = "" + Long.valueOf(token.image);
              } catch (NumberFormatException nfe){
              	throw new ParseException("inapproriate value: " + token.image); 
              }
            } 
           )
          ) 
        )
        {
          properties.put(key.toUpperCase(), value);
        }
        
       )*
      )
       <RIGHTPAREN>
    )?
}



NodegroupDecl NodegroupDeclaration() throws ParseException :
{
  Identifier name = null;
  List < Identifier > ncNames = new ArrayList < Identifier > ();
  boolean ifNotExists = false;
}
{
  < IDENTIFIER >
  {
    name = new Identifier(token.image);
  }
  (
    "if not exists"
    { 
      ifNotExists = true;
    }
  )?
  "on" < IDENTIFIER >
  {
    ncNames.add(new Identifier(token.image));
  }
  (
    "," < IDENTIFIER >
    {
      ncNames.add(new Identifier(token.image));
    }
  )*
  ";"
  {
    return new NodegroupDecl(name, ncNames, ifNotExists);
  }
}


TypeDecl TypeDeclaration(boolean dgen, String hint) throws ParseException:
{
  Identifier dataverse;
  Identifier ident;
  TypeExpression typeExpr;
  boolean ifNotExists = false;
  Pair<Identifier,Identifier> nameComponents=null;	
}
{
  {
    nameComponents = getDotSeparatedPair();
    dataverse = nameComponents.first;
    ident = nameComponents.second; 
  }
  
  (
    "if not exists"
    {
      ifNotExists = true;
    }
  )?
  "as"
  ( typeExpr = TypeExpr() )
  (";")?
  {
    long numValues = -1;
    String filename = null;
    if (dgen) {       
      String splits[] = hint.split(" +");
      if (splits.length != 3) {
        throw new ParseException("Expecting /*+ dgen <filename> <numberOfItems> */");
      } 
      filename = splits[1];
      numValues = Long.parseLong(splits[2]);
    }  
    TypeDataGen tddg = new TypeDataGen(dgen, filename, numValues);
    return new TypeDecl(dataverse, ident, typeExpr, tddg, ifNotExists);
  }
}

TypeExpression TypeExpr() throws ParseException:
{
  TypeExpression typeExpr = null;
}
{
  (
      typeExpr = RecordTypeDef()
    | typeExpr = TypeReference()
    | typeExpr = OrderedListTypeDef()
    | typeExpr = UnorderedListTypeDef()  
  )  
  {
    return typeExpr;
  }
}

RecordTypeDefinition RecordTypeDef() throws ParseException:
{
  RecordTypeDefinition recType = new RecordTypeDefinition();
  RecordTypeDefinition.RecordKind recordKind = null; 
}
{
  ( "closed" { recordKind = RecordTypeDefinition.RecordKind.CLOSED; } 
    | "open" { recordKind = RecordTypeDefinition.RecordKind.OPEN; } )?
   "{"
    {
      String hint = getHint(token);
      if (hint != null) { 
        String splits[] = hint.split(" +");
        if (splits[0].equals(GEN_FIELDS_HINT)) {
          if (splits.length != 5) {
            throw new ParseException("Expecting: /*+ gen-fields <type> <min> <max> <prefix>*/");
          }
          if (!splits[1].equals("int")) {
            throw new ParseException("The only supported type for gen-fields is int.");
          } 
          UndeclaredFieldsDataGen ufdg = new UndeclaredFieldsDataGen(UndeclaredFieldsDataGen.Type.INT, 
             Integer.parseInt(splits[2]), Integer.parseInt(splits[3]), splits[4]);
          recType.setUndeclaredFieldsDataGen(ufdg);   
        }
      }  
      
    }
		( 
		  RecordField(recType)
		  ( ","  RecordField(recType) )*
		)?
   "}"
   {
      if (recordKind == null) {
        recordKind = RecordTypeDefinition.RecordKind.OPEN;
      }
      recType.setRecordKind(recordKind);
      return recType;
   } 
}

void RecordField(RecordTypeDefinition recType) throws ParseException:
{
        String fieldName;
        TypeExpression type = null; 
        boolean nullable = false;
}
{
      <IDENTIFIER>
      	{
	     Token t = getToken(0);
	     fieldName = t.toString();	     	     
         String hint = getHint(t);
         IRecordFieldDataGen rfdg = null;
         if (hint != null) { 
           String splits[] = hint.split(" +");
           if (splits[0].equals(VAL_FILE_HINT)) { 
             File[] valFiles = new File[splits.length - 1];
             for (int k=1; k<splits.length; k++) {
               valFiles[k-1] = new File(splits[k]);
             } 
             rfdg = new FieldValFileDataGen(valFiles);
           } else if (splits[0].equals(VAL_FILE_SAME_INDEX_HINT)) {
             rfdg = new FieldValFileSameIndexDataGen(new File(splits[1]), splits[2]);
           } else if (splits[0].equals(LIST_VAL_FILE_HINT)) {
             rfdg = new ListValFileDataGen(new File(splits[1]), Integer.parseInt(splits[2]), Integer.parseInt(splits[3]));
           } else if (splits[0].equals(LIST_HINT)) {
             rfdg = new ListDataGen(Integer.parseInt(splits[1]), Integer.parseInt(splits[2]));
           } else if (splits[0].equals(INTERVAL_HINT)) {
             FieldIntervalDataGen.ValueType vt;
             if (splits[1].equals("int")) {
               vt = FieldIntervalDataGen.ValueType.INT;
             } else if (splits[1].equals("long")) {
               vt = FieldIntervalDataGen.ValueType.LONG;
             } else if (splits[1].equals("float")) {
               vt = FieldIntervalDataGen.ValueType.FLOAT;
             } else if (splits[1].equals("double")) {
               vt = FieldIntervalDataGen.ValueType.DOUBLE;
             } else {
               throw new ParseException("Unknown type for interval data gen: " + splits[1]);
             }
             rfdg = new FieldIntervalDataGen(vt, splits[2], splits[3]); 
           } else if (splits[0].equals(INSERT_RAND_INT_HINT)) {
             rfdg = new InsertRandIntDataGen(splits[1], splits[2]);
           } else if (splits[0].equals(DATE_BETWEEN_YEARS_HINT)) {
             rfdg = new DateBetweenYearsDataGen(Integer.parseInt(splits[1]), Integer.parseInt(splits[2]));
           } else if (splits[0].equals(DATETIME_BETWEEN_YEARS_HINT)) {
             rfdg = new DatetimeBetweenYearsDataGen(Integer.parseInt(splits[1]), Integer.parseInt(splits[2]));
           } else if (splits[0].equals(DATETIME_ADD_RAND_HOURS_HINT)) {
             rfdg = new DatetimeAddRandHoursDataGen(Integer.parseInt(splits[1]), Integer.parseInt(splits[2]), splits[3]);
           } else if (splits[0].equals(AUTO_HINT)) {
              rfdg = new AutoDataGen(splits[1]);
           }
         }
	    }
	  ":"
	  ( type =  TypeExpr() )
	  ("?" { nullable = true; } )?
	  {
	     recType.addField(fieldName, type, nullable, rfdg);
	  }   
}

TypeReferenceExpression TypeReference() throws ParseException:
{}
{
 <IDENTIFIER>
 {
     Token t = getToken(0);
     Identifier id = new Identifier(t.toString());
     return new TypeReferenceExpression(id);
 }
}

OrderedListTypeDefinition OrderedListTypeDef() throws ParseException:
{  
  TypeExpression type = null;
}
{
  "["
    ( type =  TypeExpr() )
  "]"
  {
    return new OrderedListTypeDefinition(type);
  }
}


UnorderedListTypeDefinition UnorderedListTypeDef() throws ParseException:
{  
  TypeExpression type = null;
}
{
  "{{"
    ( type =  TypeExpr() )
  "}}"
  {
    return new UnorderedListTypeDefinition(type);
  }
}

Pair<Identifier,Identifier> getDotSeparatedPair() throws ParseException:
{
 Identifier first = null;
 Identifier second = null;
}
{
  < IDENTIFIER >
  {
    first = new Identifier(token.image);
  } 
  ("." <IDENTIFIER>
  {
    second = new Identifier(token.image);
  }
  )?
  
  {
   if(second == null){
   	second = first;
   	first = null;
   } 
   
   return new Pair<Identifier,Identifier>(first,second);
  }  
}  
  
Triple<Identifier,Identifier,Identifier> getDotSeparatedTriple() throws ParseException:
{
 Identifier first = null;
 Identifier second = null;
 Identifier third = null;
}
{
  < IDENTIFIER >
  {
    first = new Identifier(token.image);
  } 
  "." <IDENTIFIER>
  {
    second = new Identifier(token.image);
  }
  (
  "." <IDENTIFIER>
  {
    third = new Identifier(token.image);
  }
  )?
  
  {
   if(third == null){
   	third  = second;
   	second = first;
   	first = null;
   } 
   
   return new Triple<Identifier,Identifier,Identifier>(first,second,third);
  }  
}  


  

FunctionDecl FunctionDeclaration() throws ParseException:
{
  FunctionDecl funcDecl;
  FunctionSignature signature;
  String functionName;
  int arity = 0;
  List<VarIdentifier> paramList = new ArrayList<VarIdentifier>();
  Expression funcBody;
  VarIdentifier var = null;
  createNewScope();
}
{

    <IDENTIFIER>
	{
	  Token t = getToken(0);
	  functionName = t.toString();
	}
    <LEFTPAREN> (<VARIABLE>
    {
      var = new VarIdentifier();
      var.setValue(getToken(0).toString());
      paramList.add(var);
      getCurrentScope().addNewVarSymbolToScope(var);
      arity++;
    }
    ("," <VARIABLE>
    {
      var = new VarIdentifier();
      var.setValue(getToken(0).toString());
      paramList.add(var);
      getCurrentScope().addNewVarSymbolToScope(var);
      arity++;
    })*)? <RIGHTPAREN> "{" funcBody = Expression() "}"
    (";")?
    {
      signature = new FunctionSignature(defaultDataverse, functionName, arity);
      getCurrentScope().addFunctionDescriptor(signature, false);
      funcDecl = new FunctionDecl(signature, paramList, funcBody);
      return funcDecl;
    }
}

CreateFunctionStatement FunctionCreation() throws ParseException:
{
  CreateFunctionStatement cfs = null;
  FunctionSignature signature;
  String dataverse;
  String functionName;
  boolean ifNotExists = false;
  List<VarIdentifier> paramList = new ArrayList<VarIdentifier>();
  String functionBody;
  VarIdentifier var = null;
  createNewScope();
  Expression functionBodyExpr;
  Token beginPos;
  Token endPos;
  Pair<Identifier,Identifier> nameComponents=null;
}
{
    {
      nameComponents = getDotSeparatedPair();
   	  dataverse = nameComponents.first != null ? nameComponents.first.getValue() : defaultDataverse;
   	  functionName= nameComponents.second.getValue();
	}
	
	(
      "if not exists"
       {
         ifNotExists = true;
       }
    )?
	
    <LEFTPAREN> (<VARIABLE>
    {
      var = new VarIdentifier();
      var.setValue(getToken(0).toString());
      paramList.add(var);
      getCurrentScope().addNewVarSymbolToScope(var);
    }
    ("," <VARIABLE>
    {
      var = new VarIdentifier();
      var.setValue(getToken(0).toString());
      paramList.add(var);
      getCurrentScope().addNewVarSymbolToScope(var);
    })*)? <RIGHTPAREN>  "{"
          {
          beginPos = getToken(0);
          } 
          functionBodyExpr = Expression() 
          "}" 
          {
            endPos = getToken(0);
            functionBody = extractFragment(beginPos.beginLine, beginPos.beginColumn, endPos.beginLine, endPos.beginColumn);
          }
          (";")?
    {
      signature = new FunctionSignature(dataverse, functionName, paramList.size());
      getCurrentScope().addFunctionDescriptor(signature, false);
      cfs = new CreateFunctionStatement(signature, paramList, functionBody, ifNotExists);
      return cfs;
    }
}



Query Query()throws ParseException:
{
  Query query = new Query();
  Expression expr;
}
{
    expr = Expression()
    (";")?
    {
      query.setBody(expr);
      query.setVarCounter(getVarCounter());
      return query;
    }
     
}



Expression Expression():
{
  Expression expr = null;
  Expression exprP = null;
}
{
(
  
//OperatorExpr | IfThenElse | FLWOGRExpression | QuantifiedExpression
    expr = OperatorExpr()
    | expr = IfThenElse()
    | expr = FLWOGR()
    | expr = QuantifiedExpression()
   

)
	{
	  return (exprP==null) ? expr : exprP;
	}
}



Expression OperatorExpr()throws ParseException:
{
  OperatorExpr op = null;
  Expression operand = null;
}
{
	operand = AndExpr()
	(
	
	  "or"
  	{
  	  if (op == null) {
  	    op = new OperatorExpr();
  	    op.addOperand(operand);
	    op.setCurrentop(true);  	      	     
  	  }  
  	  Token t = getToken(0);
      op.addOperator(t.toString());
	}

	operand = AndExpr()
	{
	  op.addOperand(operand);
	}

	)*
	
	{
	  return op==null? operand: op;
	}
}

Expression AndExpr()throws ParseException:
{
  OperatorExpr op = null;
  Expression operand = null;
}
{
	operand = RelExpr()
	(
	
	  "and"
  	{
  	  if (op == null) {
  	    op = new OperatorExpr();
  	    op.addOperand(operand);
	    op.setCurrentop(true);  	      	     
  	  }  
  	  Token t = getToken(0);
      op.addOperator(t.toString());
	}

	operand = RelExpr()
	{
	  op.addOperand(operand);
	}

	)*
	
	{
	  return op==null? operand: op;
	}
}



Expression RelExpr()throws ParseException:
{
  OperatorExpr op = null;
  Expression operand = null;
  boolean broadcast = false;
  IExpressionAnnotation annotation = null;
}
{
    operand = AddExpr()
    { 
      if (operand instanceof VariableExpr) {        
        String hint = getHint(token);        
        if (hint != null && hint.equals(BROADCAST_JOIN_HINT)) {
          broadcast = true;
        }
      }
    } 

    (
      LOOKAHEAD(2)( "<" | ">" | "<=" | ">=" | "=" | "!=" |"~=")
  	  {
  	    String mhint = getHint(token);
  	    if (mhint != null && mhint.equals(INDEXED_NESTED_LOOP_JOIN_HINT)) {
          annotation = IndexedNLJoinExpressionAnnotation.INSTANCE;
        }
  	    if (op == null) {
  	      op = new OperatorExpr();
  	      op.addOperand(operand, broadcast);
          op.setCurrentop(true);
          broadcast = false;
  	    }   
  	    Token t = getToken(0);
        op.addOperator(t.toString());
	  }
	  
 	  operand = AddExpr()
	  {
         broadcast = false;	     
         if (operand instanceof VariableExpr) {
           String hint = getHint(token);        
           if (hint != null && hint.equals(BROADCAST_JOIN_HINT)) {
             broadcast = true;
           }
         }
         op.addOperand(operand, broadcast);
      } 
    )?
 	
 	{
 	  if (annotation != null) {
 	    op.addHint(annotation);
 	  }
 	  return op==null? operand: op;
 	}
}

Expression AddExpr()throws ParseException:
{
  OperatorExpr op = null;
  Expression operand = null;  
}
{
	operand = MultExpr()

	( ("+" | "-")
  	{
  	  if (op == null) {
  	    op = new OperatorExpr();
        op.addOperand(operand);  	    
        op.setCurrentop(true);        
  	  } 
  	  Token t = getToken(0);
	  ((OperatorExpr)op).addOperator(t.toString());
	}

	operand = MultExpr()
	{
	  op.addOperand(operand);
	}
	)*
	
	{
 	  return op==null? operand: op;
 	}
}

Expression MultExpr()throws ParseException:
{
  OperatorExpr op = null;
  Expression operand = null;
}
{
	operand = UnionExpr()

	(( "*" | "/" | "%" | <CARET> | "idiv")
  	{
  	  if (op == null) {
  	    op = new OperatorExpr();
        op.addOperand(operand);
        op.setCurrentop(true);          	    
  	  } 
  	  Token t = getToken(0);
	  op.addOperator(t.toString());
	}
	operand = UnionExpr()
	{
	   op.addOperand(operand);
	}
	)*
	
 	{
 	  return op==null?operand:op;
 	}	
}

Expression UnionExpr() throws ParseException:
{
    UnionExpr union = null;
    Expression operand1 = null;
    Expression operand2 = null;
}
{
   operand1 = UnaryExpr() 
   ("union" 
       (operand2 = UnaryExpr()) {
          if (union == null) {
             union = new UnionExpr();
             union.addExpr(operand1); 
          }
          union.addExpr(operand2);   
       } )*
   {
     return (union == null)? operand1: union;
   }
}

Expression UnaryExpr() throws ParseException:
{
	Expression uexpr = null;
	Expression expr = null;
}
{
	(( "+"|"-") 
	{
	  	uexpr = new UnaryExpr(); 
		Token t = getToken(0);
		if("+".equals(t.toString()))
			((UnaryExpr)uexpr).setSign(Sign.POSITIVE);
		else if("-".equals(t.toString()))
			((UnaryExpr)uexpr).setSign(Sign.NEGATIVE);
		else 
			throw new ParseException();
	}
	)?
	
	expr = ValueExpr()
	{
		if(uexpr!=null){
			((UnaryExpr)uexpr).setExpr(expr);
			return uexpr;
		}
		else{
			return expr;
		}
	}
}

Expression ValueExpr() throws ParseException:
{
  Expression expr;
}
{
  expr = FieldOrIndexAccessor()
  {
    return expr;
  }
}


Expression FieldOrIndexAccessor()throws ParseException:
{
  Expression expr = null;
  Identifier ident = null;
  AbstractAccessor fa = null;
  int index;

}
{
	( expr = PrimaryExpr()

	)


	(
	(
	  	ident = Field()
	{
		  if(fa == null)
		  	fa = new FieldAccessor(expr, ident);
		  else
		  	fa = new FieldAccessor(fa, ident);
	}
	)
	| (
		index = Index()
		{
		  if(fa == null)
		  	fa = new IndexAccessor(expr, index);
		  else
		  	fa = new IndexAccessor(fa, index);
		}
	) 
	)*

	
  	{
 	  return fa==null?expr:fa;
 	}
}

Identifier Field() throws ParseException:
{
  Identifier ident = null;

}
{
  "." < IDENTIFIER >
  	{
    
  	ident = new Identifier();
	ident.setValue(getToken(0).toString());

	  return ident;
	}
}

int Index() throws ParseException:
{
	Expression expr = null;
	int idx = -2;
}
{
  "[" ( expr = Expression()
	{
		if(expr.getKind() == Expression.Kind.LITERAL_EXPRESSION)
		{
			Literal lit = ((LiteralExpr)expr).getValue();
			if(lit.getLiteralType() == Literal.Type.INTEGER || 
			   lit.getLiteralType() == Literal.Type.LONG) {
				idx = Integer.valueOf(lit.getStringValue());
			}
			else {
				throw new ParseException("Index should be an INTEGER");				
            }
		}

	}

  	| "?"
	{
		idx = IndexAccessor.ANY;
	  // ANY
	}
 	 
  	)

   "]"
	{
	  return idx;
	}
}


Expression PrimaryExpr()throws ParseException:
{
  Expression expr = null;
}
{
  //Literal | VariableRef | ListConstructor | RecordConstructor | FunctionCallExpr | DatasetAccessExpression | ParenthesizedExpression
	(
	  expr =Literal() 
	   | expr = FunctionCallExpr()
	   | expr = DatasetAccessExpression()
	   | expr =VariableRef() 
	   
    {
      if(((VariableExpr)expr).getIsNewVar() == true)
      	throw new ParseException("can't find variable " + ((VariableExpr)expr).getVar());
    }
    	   | expr = ListConstructor()
	   | expr = RecordConstructor()
	   | expr = ParenthesizedExpression()
	)
 	{
 	  return expr;
 	}
}

Expression Literal() throws ParseException:
{

  LiteralExpr lit = new LiteralExpr();
  Token t;
}
{
(
      <STRING_LITERAL>
	{
	  t= getToken(0);
	  lit.setValue( new StringLiteral(removeQuotesAndEscapes(t.image)));
	}
    
 	 | <INTEGER_LITERAL>
    {
      t= getToken(0);
	  try {
	      lit.setValue(new IntegerLiteral(new Integer(t.image)));
	  } catch(NumberFormatException ex) {
	      lit.setValue(new LongIntegerLiteral(new Long(t.image)));
	  }
	}
     | < FLOAT_LITERAL >
    {
      t= getToken(0);
      lit.setValue(new FloatLiteral(new Float(t.image)));
    }     
	 | < DOUBLE_LITERAL >
    {
      t= getToken(0);
	  lit.setValue(new DoubleLiteral(new Double(t.image)));
	}	  
 	 | <NULL>
	{
	  t= getToken(0);
	  lit.setValue(NullLiteral.INSTANCE);
	}
   	 | <TRUE>
	{
	  t= getToken(0);
	  lit.setValue(TrueLiteral.INSTANCE);
	}   	 
   	 | <FALSE>
	{
	  t= getToken(0);
	  lit.setValue(FalseLiteral.INSTANCE);
	}
)
    {
      return lit;
    }
}


VariableExpr VariableRef() throws ParseException:
{
	VariableExpr varExp = new VariableExpr();
	VarIdentifier var = new VarIdentifier();
	Token t;
}
{
      <VARIABLE>
    {
     t = getToken(0);//get current token
     String varName = t.toString(); 
     Identifier ident = lookupSymbol(varName);
     if (isInForbiddenScopes(varName)) {
       throw new ParseException("Inside limit clauses, it is disallowed to reference a variable having the same name as any variable bound in the same scope as the limit clause.");
     }
     if(ident != null) { // exist such ident
       varExp.setIsNewVar(false);
       varExp.setVar((VarIdentifier)ident);       
     } else {
       varExp.setVar(var);     
     }
     var.setValue(t.toString());        
     return varExp;
    }
}


VariableExpr Variable() throws ParseException:
{
	VariableExpr varExp = new VariableExpr();
	VarIdentifier var = new VarIdentifier();
	Token t;
}
{
      <VARIABLE>
    {
     t = getToken(0);//get current token
     Identifier ident = lookupSymbol(t.toString());
     if(ident != null) { // exist such ident
       varExp.setIsNewVar(false);
     }  
     varExp.setVar(var);     
     var.setValue(t.toString());        
     return varExp;
    }
}

Expression ListConstructor() throws ParseException:
{
	Expression expr = null;
}
{
    (
    	expr = OrderedListConstructor() | expr = UnorderedListConstructor()
    )
    
    {
      return expr;
    }
}


ListConstructor OrderedListConstructor() throws ParseException:
{
  	ListConstructor expr = new ListConstructor();
  	Expression tmp = null;
  	List<Expression> exprList = new ArrayList<Expression>();
  	expr.setType(ListConstructor.Type.ORDERED_LIST_CONSTRUCTOR);
}
{

    "[" 
	    ( tmp = Expression()
			{
			  exprList.add(tmp);
			}
		
		    ("," tmp = Expression() { exprList.add(tmp);  })*
	    )? 
    
    "]"

    {
      expr.setExprList(exprList);
      return expr;
    }    
}

ListConstructor UnorderedListConstructor() throws ParseException:
{
  	ListConstructor expr = new ListConstructor();
  	Expression tmp = null;
  	List<Expression> exprList = new ArrayList<Expression>();
  	expr.setType(ListConstructor.Type.UNORDERED_LIST_CONSTRUCTOR);
}
{

    "{{" ( tmp = Expression()
	{
	  exprList.add(tmp);
	}
    ("," tmp = Expression() { exprList.add(tmp);  })*)? "}}"
    {
      expr.setExprList(exprList);
      return expr;
    }    
}

RecordConstructor RecordConstructor() throws ParseException:
{
  	RecordConstructor expr = new RecordConstructor();
  	FieldBinding tmp = null;
  	List<FieldBinding> fbList = new ArrayList<FieldBinding>();
}
{
    "{" (tmp = FieldBinding()
    {
      fbList.add(tmp);
    }
    ("," tmp = FieldBinding() { fbList.add(tmp);  })*)? "}"
    {
      expr.setFbList(fbList);
      return expr;
    }       
}

FieldBinding FieldBinding() throws ParseException:
{
	FieldBinding fb = new FieldBinding();
	Expression left, right;
}
{
    left = Expression() ":" right = Expression()
    {
      fb.setLeftExpr(left);
      fb.setRightExpr(right);
      return fb;
    }
}


Expression FunctionCallExpr() throws ParseException:
{
  CallExpr callExpr;
  List<Expression> argList = new ArrayList<Expression>();
  Expression tmp;
  int arity = 0;
  String funcName=null;
  String dataverse=null;
  String libraryName=null;
  String hint=null;
  String id1=null;
  String id2=null;
  String first = null;
  String second = null;
  String third = null;
  Triple<String,String,String> functionCallComponents = null;
}
{  
    (
      ( functionCallComponents = getFunctionCallComponents() )
       {
         dataverse = functionCallComponents.first == null ? defaultDataverse : functionCallComponents.first;
         libraryName = functionCallComponents.second;
         funcName = functionCallComponents.third;
         if(libraryName != null){
          funcName = libraryName + ":" + funcName;
         }
       } 
    )

    {
       hint = getHint(token);
    }
     <LEFTPAREN> (tmp = Expression()
     {
       argList.add(tmp);
       arity ++;
     } ("," tmp = Expression() { argList.add(tmp); arity++; })*)? <RIGHTPAREN>

     {
       FunctionSignature signature = lookupFunctionSignature(dataverse, funcName, arity);
             if(signature == null)
             {
                signature = new FunctionSignature(dataverse, funcName, arity);
             }
       callExpr = new CallExpr(signature,argList);
       if (hint != null && hint.startsWith(INDEXED_NESTED_LOOP_JOIN_HINT)) {
          callExpr.addHint(IndexedNLJoinExpressionAnnotation.INSTANCE);
        }
       return callExpr;
     }
}

Triple<String,String,String> getFunctionCallComponents() throws ParseException:
{
   String first = null;
   String second = null;
   String third = null;
}
{
   <IDENTIFIER>{
     first = token.image;
   }
   (
    ("." <IDENTIFIER> {
      second = token.image;
    }
    )? 
    ( ":" <IDENTIFIER> {
            third = token.image;
      }
    )?
   )
   {
     if(second == null) {
       if(third!=null){
         second="" + first;
       } else {
       third = "" + first;
       }
       first = null;
     } else if (third == null) {
       third = "" + second;
       second = null;
     }
     return new Triple<String,String,String>(first,second,third);
   }

}

Expression DatasetAccessExpression() throws ParseException:
{
  CallExpr callExpr;
  List<Expression> argList = new ArrayList<Expression>();
  String funcName;
  String dataverse;
  LiteralExpr ds;
  LiteralExpr dvds;
  Expression nameArg;
  int arity = 0;
}
{  
    <DATASET> {dataverse = MetadataConstants.METADATA_DATAVERSE_NAME; funcName = getToken(0).toString();}
    (
    (<IDENTIFIER> {ds = new LiteralExpr(); ds.setValue( new StringLiteral(token.image) ); argList.add(ds); arity ++;} ("." <IDENTIFIER> { dvds = new LiteralExpr(); dvds.setValue(new StringLiteral(ds.getValue()+"."+token.image)); argList.remove(0); argList.add(dvds);})? ) |  
    (<LEFTPAREN> nameArg = Expression() {argList.add(nameArg); arity ++;} ("," nameArg = Expression() { argList.add(nameArg); arity++; })* <RIGHTPAREN>)
    )  
     
     {
       FunctionSignature signature = lookupFunctionSignature(dataverse, funcName.toString(), arity);
       if(signature == null)
             {
                signature = new FunctionSignature(dataverse, funcName.toString(), arity);
             }
       callExpr = new CallExpr(signature,argList);
       return callExpr;
     }
}

Expression ParenthesizedExpression() throws ParseException:
{
  Expression expr;
}
{
    <LEFTPAREN> expr = Expression() <RIGHTPAREN>
    {
      return expr;
    }
}

Expression IfThenElse() throws ParseException:
{
  Expression condExpr;
  Expression thenExpr;
  Expression elseExpr;
  IfExpr ifExpr = new IfExpr();
}
{
    "if" <LEFTPAREN> condExpr = Expression() <RIGHTPAREN> "then" thenExpr = Expression() "else" elseExpr = Expression()

    {
      ifExpr.setCondExpr(condExpr);
      ifExpr.setThenExpr(thenExpr);
      ifExpr.setElseExpr(elseExpr);
      return ifExpr;
    }
}

Expression  FLWOGR() throws ParseException:
{
	FLWOGRExpression flworg = new FLWOGRExpression();
	List<Clause> clauseList = new ArrayList<Clause>();
	Expression returnExpr;
	Clause tmp;
	createNewScope();
}
{
     (tmp = ForClause()  {clauseList.add(tmp);} | tmp = LetClause() {clauseList.add(tmp);})
      (tmp = Clause() {clauseList.add(tmp);})* "return" returnExpr = Expression()

     {
       flworg.setClauseList(clauseList);
       flworg.setReturnExpr(returnExpr);
       removeCurrentScope();
       return flworg;
     }
}

Clause Clause()throws ParseException :
{
  Clause clause;
}
{
    (
         clause = ForClause() 
       | clause = LetClause() 
       | clause = WhereClause() 
       | clause = OrderbyClause() 
       | clause = GroupClause() 
       | clause = LimitClause()
       | clause = DistinctClause()
       | clause = DieClause()
    )
    {
      return clause;
    }
}

Clause ForClause()throws ParseException :
{
	ForClause fc = new ForClause();
	VariableExpr varExp;
	VariableExpr varPos = null;
	Expression inExp;
	extendCurrentScope();
}
{
    "for" varExp = Variable()
    {
     	getCurrentScope().addNewVarSymbolToScope(varExp.getVar());
	}
	("at" varPos = Variable()
	  {
	     getCurrentScope().addNewVarSymbolToScope(varPos.getVar());
	  } 
	 )? 
      "in" ( inExp = Expression() )
    {
      fc.setVarExpr(varExp);
      fc.setInExpr(inExp);
      if (varPos != null) {
        fc.setPosExpr(varPos);
      }
      return fc;
    }
}

Clause LetClause() throws ParseException:
{
	LetClause lc = new LetClause();
	VariableExpr varExp;
	Expression beExp;
	extendCurrentScope();
}
{
    "let" varExp = Variable() ":=" beExp = Expression()
    {
      getCurrentScope().addNewVarSymbolToScope(varExp.getVar());
      lc.setVarExpr(varExp);
      lc.setBeExpr(beExp);
      return lc;
    }
}

Clause WhereClause()throws ParseException :
{
  WhereClause wc = new WhereClause();
  Expression whereExpr;
}
{
    "where" whereExpr = Expression()
    {
      wc.setWhereExpr(whereExpr);
      return wc;
    }
}

Clause OrderbyClause()throws ParseException :
{
  OrderbyClause oc = new OrderbyClause();
  Expression orderbyExpr;
  List<Expression> orderbyList = new ArrayList<Expression>();
  List<OrderbyClause.OrderModifier> modifierList = new ArrayList<OrderbyClause.OrderModifier >();
  int numOfOrderby = 0;
}
{
  (
    "order" 
      {
         String hint = getHint(token);
         if (hint != null && hint.startsWith(INMEMORY_HINT)) {
           String splits[] = hint.split(" +"); 
           int numFrames = Integer.parseInt(splits[1]);
           int numTuples = Integer.parseInt(splits[2]);
           oc.setNumFrames(numFrames);
           oc.setNumTuples(numTuples);   
         } 
      }     
    "by" orderbyExpr = Expression()
    {
      orderbyList.add(orderbyExpr);
      OrderbyClause.OrderModifier modif = OrderbyClause.OrderModifier.ASC;       
    }
    ( ("asc" { modif = OrderbyClause.OrderModifier.ASC; })
    | ("desc" { modif = OrderbyClause.OrderModifier.DESC; }))?
    {
      modifierList.add(modif);
    }
    
    ("," orderbyExpr = Expression()
    {
      orderbyList.add(orderbyExpr);
      modif = OrderbyClause.OrderModifier.ASC;
    }
    ( ("asc" { modif = OrderbyClause.OrderModifier.ASC; })
    | ("desc" { modif = OrderbyClause.OrderModifier.DESC; }))?
    {
      modifierList.add(modif);
    }    
    )*
)
    {
      oc.setModifierList(modifierList);
      oc.setOrderbyList(orderbyList);
      return oc;
    }
}
Clause GroupClause()throws ParseException :
{
  	GroupbyClause gbc = new GroupbyClause();
  	// GbyVariableExpressionPair pair = new GbyVariableExpressionPair();
 	List<GbyVariableExpressionPair> vePairList = new ArrayList<GbyVariableExpressionPair>();
    List<GbyVariableExpressionPair> decorPairList = new ArrayList<GbyVariableExpressionPair>();
	List<VariableExpr> withVarList= new ArrayList<VariableExpr>();
	VariableExpr var = null;
	VariableExpr withVar = null;
	Expression expr = null;
	VariableExpr decorVar = null;
	Expression decorExpr = null;
}
{
  	{
  	  Scope newScope = extendCurrentScopeNoPush(true); 
  	  // extendCurrentScope(true);
  	}
    "group"
      {
         String hint = getHint(token);
         if (hint != null && hint.equals(HASH_GROUP_BY_HINT)) {
           gbc.setHashGroupByHint(true);   
         } 
      } 
    "by" (LOOKAHEAD(2)  var = Variable()
    {
      newScope.addNewVarSymbolToScope(var.getVar());
    } ":=")?
    expr = Expression() 
       {
         GbyVariableExpressionPair pair1 = new GbyVariableExpressionPair(var, expr);    
         vePairList.add(pair1);
       }
    ("," ( LOOKAHEAD(2) var = Variable()
    {
      newScope.addNewVarSymbolToScope(var.getVar());
    } ":=")?
    	expr = Expression()  
    	 {
           GbyVariableExpressionPair pair2 = new GbyVariableExpressionPair(var, expr);    
           vePairList.add(pair2);
         }
    	)*
    ("decor" decorVar = Variable() ":=" decorExpr = Expression()
       {    
         newScope.addNewVarSymbolToScope(decorVar.getVar()); 
         GbyVariableExpressionPair pair3 = new GbyVariableExpressionPair(decorVar, decorExpr);
         decorPairList.add(pair3);
       }
      ("," "decor" decorVar = Variable() ":=" decorExpr = Expression()
           { 
             newScope.addNewVarSymbolToScope(decorVar.getVar()); 
             GbyVariableExpressionPair pair4 = new GbyVariableExpressionPair(decorVar, decorExpr);
             decorPairList.add(pair4);              
           }
       )*            
    )?	
    "with" withVar = VariableRef()
    {
      if(withVar.getIsNewVar()==true)
      	throw new ParseException("can't find variable " + withVar.getVar());
      withVarList.add(withVar);
      newScope.addNewVarSymbolToScope(withVar.getVar());
    }
    ("," withVar = VariableRef()
    {
      if(withVar.getIsNewVar()==true)
      	throw new ParseException("can't find variable " + withVar.getVar());
      withVarList.add(withVar);
      newScope.addNewVarSymbolToScope(withVar.getVar());
    })*
    {
      gbc.setGbyPairList(vePairList);
      gbc.setDecorPairList(decorPairList);
      gbc.setWithVarList(withVarList);
      replaceCurrentScope(newScope);
      return gbc;
    }
}


LimitClause LimitClause() throws ParseException:
{
	LimitClause lc = new LimitClause();
	Expression expr;
	pushForbiddenScope(getCurrentScope());
}
{
    "limit" expr = Expression()    { lc.setLimitExpr(expr);    }
    ("offset" expr = Expression() { lc.setOffset(expr);    })?

  {
    popForbiddenScope();   
    return lc;
  }
}

DistinctClause DistinctClause() throws ParseException:
{
  List<Expression> exprs = new ArrayList<Expression>();
  Expression expr;
}
{
  "distinct" "by" expr = Expression() 
  {
    exprs.add(expr);
  }
  ("," expr = Expression() 
  	{
  		exprs.add(expr); 
  	} 
  )*
  {
  	return new DistinctClause(exprs);
  }
}

DieClause DieClause() throws ParseException:
{
	DieClause lc = new DieClause();
	Expression expr;
	pushForbiddenScope(getCurrentScope());
}
{
  "die" "after" expr = Expression()    { lc.setDieExpr(expr);    }
  {
    popForbiddenScope();   
    return lc;
  }
}


QuantifiedExpression QuantifiedExpression()throws ParseException:
{
  QuantifiedExpression qc = new QuantifiedExpression();
  List<QuantifiedPair> quantifiedList = new ArrayList<QuantifiedPair>();
  Expression satisfiesExpr;
  VariableExpr var;
  Expression inExpr;
  QuantifiedPair pair;
}
{
  {
    createNewScope();
  }
	
   (      ("some"  {  qc.setQuantifier(QuantifiedExpression.Quantifier.SOME);	})
		| ("every" {  qc.setQuantifier(QuantifiedExpression.Quantifier.EVERY);	}))
	var = Variable() "in" inExpr = Expression() 
	{
	  pair = new QuantifiedPair(var, inExpr);
      getCurrentScope().addNewVarSymbolToScope(var.getVar());
      quantifiedList.add(pair);
	}
	(
	"," var = Variable() "in" inExpr = Expression() 
	{ 
      pair = new QuantifiedPair(var, inExpr);
      getCurrentScope().addNewVarSymbolToScope(var.getVar());
      quantifiedList.add(pair);	
	}
	)*
	 "satisfies" satisfiesExpr = Expression()
	 {
	   qc.setSatisfiesExpr(satisfiesExpr);
	   qc.setQuantifiedList(quantifiedList);
	   removeCurrentScope();
	   return qc;
	 }
}

TOKEN_MGR_DECLS:
{
        public int commentDepth = 0;
}

<DEFAULT>
TOKEN :
{
   <CARET : "^"  >
}

<DEFAULT>
TOKEN :
{
   <DATASET : "dataset"  >
}

<DEFAULT>
TOKEN :
{
   <LEFTPAREN : "("  >
}

<DEFAULT>
TOKEN :
{
   <RIGHTPAREN : ")"  >
}


<DEFAULT>
TOKEN :
{
	<INTEGER_LITERAL : (<DIGIT>)+ >
}


<DEFAULT>
TOKEN :
{
	<NULL : "null">
}

<DEFAULT>
TOKEN :
{
	<TRUE : "true">
}

<DEFAULT>
TOKEN :
{
	<FALSE : "false">
}

<DEFAULT>
TOKEN :
{
	<#DIGIT : ["0" - "9"]>
}


TOKEN:
{
  < DOUBLE_LITERAL: <INTEGER>
        | <INTEGER> ( "." <INTEGER> )?
        | "." <INTEGER>
  >
  |
  < FLOAT_LITERAL: <INTEGER> ( "f" | "F" )
        | <INTEGER> ( "." <INTEGER> ( "f" | "F" ) )?
        | "." <INTEGER> ( "f" | "F" )
  >
  |
  <INTEGER : (<DIGIT>)+ >
}

<DEFAULT>
TOKEN :
{
	<#LETTER : ["A" - "Z", "a" - "z"]>
}

<DEFAULT>
TOKEN :
{
	<SPECIALCHARS : ["$", "_", "-"]  >
}

<DEFAULT>
TOKEN :
{
	<STRING_LITERAL : ("\"" (<EscapeQuot> | ~["\""])* "\"") | ("\'"(<EscapeApos> | ~["\'"])* "\'")>
	|
	< #EscapeQuot: "\\\"" >
	|
    < #EscapeApos: "\\\'" >
}

<DEFAULT>
TOKEN :
{
	<IDENTIFIER : (<LETTER>)+ (<LETTER> | <DIGIT> | <SPECIALCHARS>)*>
}


<DEFAULT>
TOKEN :
{
	<VARIABLE : "$" <IDENTIFIER> >
}

SKIP:
{
    " "
|   "\t"
|   "\r"
|   "\n"
}

SKIP:
{
	<"//" (~["\n"])* "\n">
}

SKIP:
{
	<"//" (~["\n","\r"])* ("\n"|"\r"|"\r\n")?> 
}


SKIP:
{
        <"/*"> {commentDepth=1;}: INSIDE_COMMENT
}

<INSIDE_COMMENT>
SPECIAL_TOKEN:
{
       <"+"(" ")*(~["*"])*>
}

<INSIDE_COMMENT>
SKIP:
{
        <"/*"> {commentDepth++;}
}

<INSIDE_COMMENT>
SKIP:
{
        <"*/"> {commentDepth--; if (commentDepth == 0) SwitchTo(DEFAULT);}
|       <~[]>
}