Merged asterix_lsm_stabilization upto r1547
git-svn-id: https://asterixdb.googlecode.com/svn/trunk@1622 eaa15691-b419-025a-1212-ee371bd00084
diff --git a/asterix/asterix-aql/src/main/javacc/AQL.jj b/asterix/asterix-aql/src/main/javacc/AQL.jj
index 1fa40b8..8e3e203 100644
--- a/asterix/asterix-aql/src/main/javacc/AQL.jj
+++ b/asterix/asterix-aql/src/main/javacc/AQL.jj
@@ -17,14 +17,17 @@
import java.util.Map;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import edu.uci.ics.asterix.aql.literal.FloatLiteral;
import edu.uci.ics.asterix.aql.literal.DoubleLiteral;
import edu.uci.ics.asterix.aql.literal.FalseLiteral;
-import edu.uci.ics.asterix.aql.base.ILiteral;
+import edu.uci.ics.asterix.aql.base.Literal;
import edu.uci.ics.asterix.aql.literal.IntegerLiteral;
+import edu.uci.ics.asterix.aql.literal.LongIntegerLiteral;
import edu.uci.ics.asterix.aql.literal.NullLiteral;
import edu.uci.ics.asterix.aql.literal.StringLiteral;
import edu.uci.ics.asterix.aql.literal.TrueLiteral;
+import edu.uci.ics.asterix.metadata.bootstrap.MetadataConstants;
import edu.uci.ics.asterix.aql.base.*;
import edu.uci.ics.asterix.aql.expression.*;
@@ -38,26 +41,22 @@
import edu.uci.ics.asterix.common.annotations.*;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.om.functions.AsterixFunction;
+import edu.uci.ics.asterix.common.functions.FunctionSignature;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IExpressionAnnotation;
+import edu.uci.ics.hyracks.algebricks.core.algebra.expressions.IndexedNLJoinExpressionAnnotation;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.algebricks.common.utils.Triple;
+
+
public class AQLParser extends ScopeChecker {
-/*
- private void printHints(Token t) {
- //System.err.println("token="+t.image+"\t special="+t.specialToken);
- if (t.specialToken == null) return;
- Token tmp_t = t.specialToken;
- while (tmp_t.specialToken != null) tmp_t = tmp_t.specialToken;
- while (tmp_t != null) {
- System.out.println(tmp_t.image);
- tmp_t = tmp_t.next;
- }
- }
-*/
-
// optimizer hints
private static final String HASH_GROUP_BY_HINT = "hash";
private static final String BROADCAST_JOIN_HINT = "bcast";
+ private static final String INDEXED_NESTED_LOOP_JOIN_HINT = "indexnl";
private static final String INMEMORY_HINT = "inmem";
private static final String VAL_FILE_HINT = "val-files";
private static final String VAL_FILE_SAME_INDEX_HINT = "val-file-same-idx";
@@ -88,12 +87,17 @@
return s.substring(1).trim();
}
+ public AQLParser(String s){
+ this(new StringReader(s));
+ super.setInput(s);
+ }
+
public static void main(String args[]) throws ParseException, TokenMgrError, IOException, FileNotFoundException, AsterixException {
File file = new File(args[0]);
Reader fis = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
AQLParser parser = new AQLParser(fis);
- Statement st = parser.Statement();
- st.accept(new AQLPrintVisitor(), 0);
+ List<Statement> st = parser.Statement();
+ //st.accept(new AQLPrintVisitor(), 0);
}
@@ -102,11 +106,11 @@
PARSER_END(AQLParser)
-Statement Statement() throws ParseException:
+List<Statement> Statement() throws ParseException:
{
- Query query = null;
scopeStack.push(RootScopeFactory.createRootScope(this));
List<Statement> decls = new ArrayList<Statement>();
+ Query query=null;
}
{
(
@@ -186,6 +190,10 @@
{
decls.add(DataverseDropStatement());
}
+ | "function"
+ {
+ decls.add(FunctionDropStatement());
+ }
)
| "write" {
decls.add(WriteStatement());
@@ -202,66 +210,78 @@
| "update" {
decls.add(UpdateStatement());
}
- | "begin" "feed" <IDENTIFIER> {
- Identifier datasetName = new Identifier(token.image);
- decls.add(new BeginFeedStatement(datasetName, getVarCounter()));
+ | "begin" "feed"
+ {
+ Pair<Identifier,Identifier> nameComponents = getDotSeparatedPair();
+ decls.add(new BeginFeedStatement(nameComponents.first, nameComponents.second, getVarCounter()));
+ } ";"
+
+ | "suspend" "feed"
+ {
+ decls.add(ControlFeedDeclaration(ControlFeedStatement.OperationType.SUSPEND));
+ } ";"
+ | "resume" "feed" {
+ decls.add(ControlFeedDeclaration(ControlFeedStatement.OperationType.RESUME));
} ";"
- | "suspend" "feed" <IDENTIFIER> {
- datasetName = new Identifier(token.image);
- decls.add(new ControlFeedStatement(ControlFeedStatement.OperationType.SUSPEND, datasetName));
- } ";"
- | "resume" "feed" <IDENTIFIER> {
- datasetName = new Identifier(token.image);
- decls.add(new ControlFeedStatement(ControlFeedStatement.OperationType.RESUME, datasetName));
- } ";"
- | "end" "feed" <IDENTIFIER> {
- datasetName = new Identifier(token.image);
- decls.add(new ControlFeedStatement(ControlFeedStatement.OperationType.END, datasetName));
+ | "end" "feed" {
+ decls.add(ControlFeedDeclaration(ControlFeedStatement.OperationType.END));
} ";"
- | "alter" "feed" <IDENTIFIER> {
- datasetName = new Identifier(token.image);
- decls.add(AlterFeedDeclaration(datasetName));
- }
-
- )*
- (query = Query())?
+ | "alter" "feed" {
+ decls.add(AlterFeedDeclaration());
+ } ";"
+
+ | (query = Query()) {
+ decls.add(query);
+ }
+ )*
+ // (query = Query())?
)
<EOF>
)
{
- if (query == null) {
- query = new Query(true);
- }
- query.setPrologDeclList(decls);
-
- return query;
+ return decls;
}
}
InsertStatement InsertStatement() throws ParseException:
{
+ Identifier dataverseName;
Identifier datasetName;
+ Pair<Identifier,Identifier> nameComponents = null;
Query query;
}
{
- "into" <DATASET> <IDENTIFIER> { datasetName = new Identifier(token.image); }
- <LEFTPAREN> query = Query() <RIGHTPAREN> ";"
- {return new InsertStatement(datasetName, query, getVarCounter());}
+ "into" <DATASET>
+
+ {
+ nameComponents = getDotSeparatedPair();
+ dataverseName = nameComponents.first;
+ datasetName = nameComponents.second;
+ }
+
+ query = Query() (";")?
+ {return new InsertStatement(dataverseName, datasetName, query, getVarCounter());}
}
DeleteStatement DeleteStatement() throws ParseException:
{
VariableExpr var = null;
+ Identifier dataverseName;
Identifier datasetName = null;
Expression condition = null;
Clause dieClause = null;
+ Pair<Identifier, Identifier> nameComponents;
}
{
var = Variable() { getCurrentScope().addNewVarSymbolToScope(var.getVar()); }
- "from" <DATASET> <IDENTIFIER> { datasetName = new Identifier(token.image); }
- ("where" condition = Expression())? (dieClause = DieClause())? ";"
- {return new DeleteStatement(var, datasetName, condition, dieClause, getVarCounter()); }
+ "from"
+ <DATASET>
+ {
+ nameComponents = getDotSeparatedPair();
+ }
+ ("where" condition = Expression())? (dieClause = DieClause())? (";")?
+ {return new DeleteStatement(var, nameComponents.first, nameComponents.second, condition, dieClause, getVarCounter()); }
}
UpdateStatement UpdateStatement() throws ParseException:
@@ -321,10 +341,10 @@
{
Identifier nodeName = null;
String fileName = null;
- Identifier datasetName = null;
Statement stmt = null;
Query query;
String writerClass = null;
+ Pair<Identifier,Identifier> nameComponents = null;
}
{
(( "output" "to"
@@ -336,10 +356,15 @@
} )
|
( "into"
- <DATASET> <IDENTIFIER> { datasetName = new Identifier(token.image); }
+ <DATASET>
+
+ {
+ nameComponents = getDotSeparatedPair();
+ }
+
<LEFTPAREN> query = Query() <RIGHTPAREN>
{
- stmt = new WriteFromQueryResultStatement(datasetName, query, getVarCounter());
+ stmt = new WriteFromQueryResultStatement(nameComponents.first, nameComponents.second, query, getVarCounter());
} ))
";"
@@ -351,6 +376,7 @@
CreateIndexStatement CreateIndexStatement() throws ParseException:
{
CreateIndexStatement cis = new CreateIndexStatement();
+ Pair<Identifier,Identifier> nameComponents = null;
}
{
<IDENTIFIER> { cis.setIndexName(new Identifier(token.image)); }
@@ -361,17 +387,41 @@
}
)?
"on"
- <IDENTIFIER> { cis.setDatasetName(new Identifier(token.image)); }
+
+ {
+ nameComponents = getDotSeparatedPair();
+ cis.setDataverseName(nameComponents.first);
+ cis.setDatasetName(nameComponents.second);
+ }
+
<LEFTPAREN>
( <IDENTIFIER> { cis.addFieldExpr(token.image); } )
("," <IDENTIFIER> { cis.addFieldExpr(token.image); })*
<RIGHTPAREN>
("type"
- ("btree" { cis.setIndexType(IndexType.BTREE); }
- | "keyword" { cis.setIndexType(IndexType.KEYWORD); }
- | "qgram" { cis.setIndexType(IndexType.QGRAM); }
- | "rtree" { cis.setIndexType(IndexType.RTREE); }
- )
+ ("btree" { cis.setIndexType(IndexType.BTREE); }
+ | "rtree" { cis.setIndexType(IndexType.RTREE); }
+ | "keyword" { cis.setIndexType(IndexType.WORD_INVIX); }
+ | "fuzzy keyword" { cis.setIndexType(IndexType.FUZZY_WORD_INVIX); }
+ | "ngram"
+ <LEFTPAREN>
+ (<INTEGER_LITERAL>
+ {
+ cis.setIndexType(IndexType.NGRAM_INVIX);
+ cis.setGramLength(Integer.valueOf(token.image));
+ }
+ )
+ <RIGHTPAREN>
+ | "fuzzy ngram"
+ <LEFTPAREN>
+ (<INTEGER_LITERAL>
+ {
+ cis.setIndexType(IndexType.FUZZY_NGRAM_INVIX);
+ cis.setGramLength(Integer.valueOf(token.image));
+ }
+ )
+ <RIGHTPAREN>
+ )
";"
| ";"
)
@@ -385,23 +435,28 @@
Identifier dvName = null;
}
{
- "dataverse" <IDENTIFIER> { dvName = new Identifier(token.image); }
+ "dataverse" <IDENTIFIER> { defaultDataverse = token.image;}
";"
{
- return new DataverseDecl(dvName);
+ return new DataverseDecl(new Identifier(defaultDataverse));
}
}
DropStatement DropStatement() throws ParseException :
{
+ Identifier dataverseName = null;
Identifier datasetName = null;
boolean ifExists = false;
+ Pair<Identifier,Identifier> nameComponents=null;
}
{
- < IDENTIFIER >
- {
- datasetName = new Identifier(token.image);
- }
+ {
+ nameComponents = getDotSeparatedPair();
+ dataverseName = nameComponents.first;
+ datasetName = nameComponents.second;
+ }
+
+
(
"if exists"
{
@@ -409,25 +464,27 @@
}
)? ";"
{
- return new DropStatement(datasetName, ifExists);
+ return new DropStatement(dataverseName, datasetName, ifExists);
}
}
IndexDropStatement IndexDropStatement() throws ParseException :
{
+ Identifier dataverseName = null;
Identifier datasetName = null;
Identifier indexName = null;
boolean ifExists = false;
+ Triple<Identifier,Identifier,Identifier> nameComponents=null;
}
{
- < IDENTIFIER >
+
{
- datasetName = new Identifier(token.image);
- }
- "." < IDENTIFIER >
- {
- indexName = new Identifier(token.image);
- }
+ nameComponents = getDotSeparatedTriple();
+ dataverseName = nameComponents.first;
+ datasetName = nameComponents.second;
+ indexName = nameComponents.third;
+ }
+
(
"if exists"
{
@@ -435,7 +492,7 @@
}
)? ";"
{
- return new IndexDropStatement(datasetName, indexName, ifExists);
+ return new IndexDropStatement(dataverseName, datasetName, indexName, ifExists);
}
}
@@ -462,13 +519,16 @@
TypeDropStatement TypeDropStatement() throws ParseException :
{
+ Identifier dataverseName = null;
Identifier typeName = null;
boolean ifExists = false;
+ Pair<Identifier,Identifier> nameComponents;
}
{
- < IDENTIFIER >
{
- typeName = new Identifier(token.image);
+ nameComponents = getDotSeparatedPair();
+ dataverseName = nameComponents.first == null ? new Identifier(defaultDataverse) : nameComponents.first;
+ typeName = nameComponents.second;
}
(
"if exists"
@@ -477,7 +537,7 @@
}
)? ";"
{
- return new TypeDropStatement(typeName, ifExists);
+ return new TypeDropStatement(dataverseName, typeName, ifExists);
}
}
@@ -531,23 +591,67 @@
}
}
-LoadFromFileStatement LoadStatement() throws ParseException:
+
+FunctionDropStatement FunctionDropStatement() throws ParseException :
{
- Identifier datasetName = null;
- boolean alreadySorted = false;
- String adapterClassname;
- Map<String,String> properties;
+ String dataverse;
+ String functionName;
+ int arity=0;
+ boolean ifExists = false;
+ Pair<Identifier, Identifier> nameComponents=null;
}
{
- <DATASET> <IDENTIFIER> { datasetName = new Identifier(token.image); }
-
- "using"
-
- <STRING_LITERAL>
+ {
+ nameComponents = getDotSeparatedPair();
+ dataverse = nameComponents.first != null ? nameComponents.first.getValue() : defaultDataverse;
+ functionName = nameComponents.second.getValue();
+ }
+
+ "@"
+ <INTEGER_LITERAL>
+ {
+ Token t= getToken(0);
+ arity = new Integer(t.image);
+ if( arity < 0 && arity != FunctionIdentifier.VARARGS){
+ throw new ParseException(" invalid arity:" + arity);
+ }
+ }
+
+ (
+ "if exists"
{
- adapterClassname = removeQuotesAndEscapes(token.image);
+ ifExists = true;
}
+ )? ";"
+ {
+ return new FunctionDropStatement(new FunctionSignature(dataverse, functionName, arity), ifExists);
+ }
+}
+
+LoadFromFileStatement LoadStatement() throws ParseException:
+{
+ Identifier dataverseName = null;
+ Identifier datasetName = null;
+ boolean alreadySorted = false;
+ String adapterName;
+ Map<String,String> properties;
+ Pair<Identifier,Identifier> nameComponents = null;
+}
+{
+ <DATASET>
+ {
+ nameComponents = getDotSeparatedPair();
+ dataverseName = nameComponents.first;
+ datasetName = nameComponents.second;
+ }
+
+ "using"
+
+ {
+ adapterName = getAdapterName();
+ }
+
{
properties = getConfiguration();
}
@@ -558,25 +662,53 @@
";"
{
- return new LoadFromFileStatement(datasetName, adapterClassname, properties, alreadySorted);
+ return new LoadFromFileStatement(dataverseName, datasetName, adapterName, properties, alreadySorted);
}
}
+String getAdapterName() throws ParseException :
+{
+ String adapterName = null;
+}
+{
+ (
+ <IDENTIFIER> {
+ adapterName = (new Identifier(token.image)).getValue();;
+ }
+ |
+ <STRING_LITERAL>
+ {
+ adapterName = removeQuotesAndEscapes(token.image);
+ }
+ )
+ {
+ return adapterName;
+ }
+}
+
DatasetDecl DatasetDeclaration(DatasetType datasetType) throws ParseException :
{
DatasetDecl dd = null;
Identifier datasetName = null;
+ Identifier dataverseName = null;
+ Identifier itemDataverseName = null;
Identifier itemTypeName = null;
+ String nameComponentFirst = null;
+ String nameComponentSecond = null;
boolean ifNotExists = false;
- IDatasetDetailsDecl idd = null;
+ IDatasetDetailsDecl datasetDetails = null;
+ Pair<Identifier,Identifier> nameComponents = null;
+ Map<String,String> hints = new HashMap<String,String>();
}
{
- < IDENTIFIER >
{
- datasetName = new Identifier(token.image);
- }
+ nameComponents = getDotSeparatedPair();
+ dataverseName = nameComponents.first;
+ datasetName = nameComponents.second;
+ }
+
(
"if not exists"
{
@@ -584,59 +716,60 @@
}
)?
(
- < LEFTPAREN > < IDENTIFIER >
+ < LEFTPAREN > <IDENTIFIER>
{
itemTypeName = new Identifier(token.image);
}
< RIGHTPAREN >
- )?
+ )
{
if(datasetType == DatasetType.INTERNAL) {
- idd = InternalDatasetDeclaration();
- dd = new DatasetDecl(datasetName, itemTypeName, idd, ifNotExists);
+ datasetDetails = InternalDatasetDeclaration();
}
else if(datasetType == DatasetType.EXTERNAL) {
- idd = ExternalDatasetDeclaration();
- dd = new DatasetDecl(datasetName, itemTypeName, idd,ifNotExists);
+ datasetDetails = ExternalDatasetDeclaration();
}
else if(datasetType == DatasetType.FEED) {
- idd = FeedDatasetDeclaration();
- dd = new DatasetDecl(datasetName, itemTypeName, idd,ifNotExists);
+ datasetDetails = FeedDatasetDeclaration();
}
- dd.setDatasetType(datasetType);
}
+
+ (
+ "hints"
{
- return dd;
+ initProperties(hints);
+ }
+ )?
+ ";"
+
+ {
+ dd = new DatasetDecl(dataverseName, datasetName, itemTypeName, hints, datasetType, datasetDetails,ifNotExists);
+ return dd;
}
}
InternalDetailsDecl InternalDatasetDeclaration() throws ParseException :
{
InternalDetailsDecl idd = null;
+ List<String> primaryKeyFields = new ArrayList<String>();
+ Identifier nodeGroupName=null;
}
{
- {
- idd = new InternalDetailsDecl();
- }
- "partitioned" "by" "key"
- < IDENTIFIER >
- {
- idd.addPartitioningExpr(token.image);
- }
(
- "," < IDENTIFIER >
{
- idd.addPartitioningExpr(token.image);
+ primaryKeyFields = getPrimaryKeyFields();
}
- )*
+ )
+
(
"on" < IDENTIFIER >
{
- idd.setNodegroupName(new Identifier(token.image));
+ nodeGroupName = new Identifier(token.image);
}
)?
- ";"
+
{
+ idd = new InternalDetailsDecl(nodeGroupName, primaryKeyFields);
return idd;
}
}
@@ -644,7 +777,7 @@
ExternalDetailsDecl ExternalDatasetDeclaration() throws ParseException :
{
ExternalDetailsDecl edd = null;
- String adapterClassname = null;
+ String adapterName = null;
Map < String, String > properties;
}
{
@@ -653,10 +786,8 @@
}
"using"
-
- <STRING_LITERAL>
{
- adapterClassname = removeQuotesAndEscapes(token.image);
+ adapterName = getAdapterName();
}
{
@@ -665,10 +796,9 @@
{
edd = new ExternalDetailsDecl();
- edd.setAdapter(adapterClassname);
+ edd.setAdapter(adapterName);
edd.setProperties(properties);
}
- ";"
{
return edd;
@@ -678,19 +808,20 @@
FeedDetailsDecl FeedDatasetDeclaration() throws ParseException :
{
FeedDetailsDecl fdd = null;
- String adapterClassname = null;
+ String adapterName = null;
Map < String, String > properties;
+ Pair<Identifier,Identifier> nameComponents;
+ List<String> primaryKeyFields = new ArrayList<String>();
+ Identifier nodeGroupName=null;
+ FunctionSignature appliedFunction=null;
+ String dataverse;
+ String functionName;
+ int arity;
}
{
- {
- fdd = new FeedDetailsDecl();
- }
-
"using"
-
- <STRING_LITERAL>
{
- adapterClassname = removeQuotesAndEscapes(token.image);
+ adapterName = getAdapterName();
}
{
@@ -698,57 +829,103 @@
}
("apply" "function"
- < IDENTIFIER >
{
- fdd.setFunctionIdentifier(token.image);
+ nameComponents = getDotSeparatedPair();
+ dataverse = nameComponents.first != null ? nameComponents.first.getValue() : defaultDataverse;
+ functionName = nameComponents.second.getValue();
}
+ ("@" <INTEGER_LITERAL>
+ {
+ arity = Integer.parseInt(token.image);
+ }
+ )
+
+ {
+ appliedFunction = new FunctionSignature(dataverse, functionName, arity);
+ }
)?
- "partitioned" "by" "key"
- < IDENTIFIER >
- {
- fdd.addPartitioningExpr(token.image);
- }
(
- "," < IDENTIFIER >
{
- fdd.addPartitioningExpr(token.image);
+ primaryKeyFields = getPrimaryKeyFields();
}
- )*
+ )
+
(
"on" < IDENTIFIER >
{
- fdd.setNodegroupName(new Identifier(token.image));
+ nodeGroupName = new Identifier(token.image);
}
)?
- ";"
+
{
- fdd.setAdapterClassname(adapterClassname);
- fdd.setProperties(properties);
+ fdd = new FeedDetailsDecl(adapterName, properties, appliedFunction, nodeGroupName, primaryKeyFields);
return fdd;
}
}
-ControlFeedStatement AlterFeedDeclaration(Identifier datasetName) throws ParseException :
+List<String> getPrimaryKeyFields() throws ParseException :
{
- String name = null;
- String value = null;
+ List<String> primaryKeyFields = new ArrayList<String>();
+}
+{
+
+ "primary" "key"
+ < IDENTIFIER >
+ {
+ primaryKeyFields.add(token.image);
+ }
+ (
+ "," < IDENTIFIER >
+ {
+ primaryKeyFields.add(token.image);
+ }
+ )*
+ {
+ return primaryKeyFields;
+ }
+
+}
+
+
+
+
+
+ControlFeedStatement ControlFeedDeclaration(ControlFeedStatement.OperationType operationType) throws ParseException :
+{
+ Pair<Identifier,Identifier> nameComponents = null;
+}
+{
+ {
+ nameComponents = getDotSeparatedPair();
+ return new ControlFeedStatement(operationType, nameComponents.first, nameComponents.second);
+ }
+}
+
+
+ControlFeedStatement AlterFeedDeclaration() throws ParseException :
+{
+ Pair<Identifier,Identifier> nameComponents = null;
Map < String, String > configuration = new HashMap < String, String > ();
}
{
+ {
+ nameComponents = getDotSeparatedPair();
+ }
+
"set"
{
configuration = getConfiguration();
}
- ";"
+
{
- return new ControlFeedStatement(ControlFeedStatement.OperationType.ALTER, datasetName, configuration);
+ return new ControlFeedStatement(ControlFeedStatement.OperationType.ALTER, nameComponents.first, nameComponents.second, configuration);
}
}
Map<String,String> getConfiguration() throws ParseException :
{
- Map<String,String> configuration = new HashMap<String,String>();
+ Map<String,String> configuration = new LinkedHashMap<String,String>();
String key;
String value;
}
@@ -797,6 +974,75 @@
}
}
+void initProperties(Map<String,String> properties) throws ParseException :
+{
+ String key;
+ String value;
+}
+{
+ (
+ <LEFTPAREN>
+ (
+ <IDENTIFIER>
+ {
+ key = (new Identifier(token.image)).getValue();
+ }
+ "="
+ (
+ (<STRING_LITERAL>
+ {
+ value = removeQuotesAndEscapes(token.image);
+ }
+ ) |
+ (<INTEGER_LITERAL>
+ {
+ try{
+ value = "" + Long.valueOf(token.image);
+ } catch (NumberFormatException nfe){
+ throw new ParseException("inapproriate value: " + token.image);
+ }
+ }
+ )
+ )
+ {
+ properties.put(key.toUpperCase(), value);
+ }
+ (
+ ","
+ (
+ <IDENTIFIER>
+ {
+ key = (new Identifier(token.image)).getValue();
+ }
+ "="
+ (
+ (<STRING_LITERAL>
+ {
+ value = removeQuotesAndEscapes(token.image);
+ }
+ ) |
+ (<INTEGER_LITERAL>
+ {
+ try{
+ value = "" + Long.valueOf(token.image);
+ } catch (NumberFormatException nfe){
+ throw new ParseException("inapproriate value: " + token.image);
+ }
+ }
+ )
+ )
+ )
+ {
+ properties.put(key.toUpperCase(), value);
+ }
+
+ )*
+ )
+ <RIGHTPAREN>
+ )?
+}
+
+
NodegroupDecl NodegroupDeclaration() throws ParseException :
{
@@ -834,15 +1080,19 @@
TypeDecl TypeDeclaration(boolean dgen, String hint) throws ParseException:
{
+ Identifier dataverse;
Identifier ident;
TypeExpression typeExpr;
boolean ifNotExists = false;
+ Pair<Identifier,Identifier> nameComponents=null;
}
{
- <IDENTIFIER>
{
- ident = new Identifier(token.image.toString());
+ nameComponents = getDotSeparatedPair();
+ dataverse = nameComponents.first;
+ ident = nameComponents.second;
}
+
(
"if not exists"
{
@@ -851,6 +1101,7 @@
)?
"as"
( typeExpr = TypeExpr() )
+ (";")?
{
long numValues = -1;
String filename = null;
@@ -863,7 +1114,7 @@
numValues = Long.parseLong(splits[2]);
}
TypeDataGen tddg = new TypeDataGen(dgen, filename, numValues);
- return new TypeDecl(ident, typeExpr, tddg, ifNotExists);
+ return new TypeDecl(dataverse, ident, typeExpr, tddg, ifNotExists);
}
}
@@ -989,12 +1240,12 @@
TypeReferenceExpression TypeReference() throws ParseException:
{}
{
- <IDENTIFIER>
- {
- Token t = getToken(0);
- Identifier id = new Identifier(t.toString());
- return new TypeReferenceExpression(id);
- }
+ <IDENTIFIER>
+ {
+ Token t = getToken(0);
+ Identifier id = new Identifier(t.toString());
+ return new TypeReferenceExpression(id);
+ }
}
OrderedListTypeDefinition OrderedListTypeDef() throws ParseException:
@@ -1024,11 +1275,72 @@
}
}
+Pair<Identifier,Identifier> getDotSeparatedPair() throws ParseException:
+{
+ Identifier first = null;
+ Identifier second = null;
+}
+{
+ < IDENTIFIER >
+ {
+ first = new Identifier(token.image);
+ }
+ ("." <IDENTIFIER>
+ {
+ second = new Identifier(token.image);
+ }
+ )?
+
+ {
+ if(second == null){
+ second = first;
+ first = null;
+ }
+
+ return new Pair<Identifier,Identifier>(first,second);
+ }
+}
+
+Triple<Identifier,Identifier,Identifier> getDotSeparatedTriple() throws ParseException:
+{
+ Identifier first = null;
+ Identifier second = null;
+ Identifier third = null;
+}
+{
+ < IDENTIFIER >
+ {
+ first = new Identifier(token.image);
+ }
+ "." <IDENTIFIER>
+ {
+ second = new Identifier(token.image);
+ }
+ (
+ "." <IDENTIFIER>
+ {
+ third = new Identifier(token.image);
+ }
+ )?
+
+ {
+ if(third == null){
+ third = second;
+ second = first;
+ first = null;
+ }
+
+ return new Triple<Identifier,Identifier,Identifier>(first,second,third);
+ }
+}
+
+
+
FunctionDecl FunctionDeclaration() throws ParseException:
{
- FunctionDecl func = new FunctionDecl();
- AsterixFunction ident;
+ FunctionDecl funcDecl;
+ FunctionSignature signature;
String functionName;
int arity = 0;
List<VarIdentifier> paramList = new ArrayList<VarIdentifier>();
@@ -1061,33 +1373,34 @@
})*)? <RIGHTPAREN> "{" funcBody = Expression() "}"
{
- ident = new AsterixFunction(functionName,arity);
- getCurrentScope().addFunctionDescriptor(ident, false);
- func.setIdent(ident);
- func.setFuncBody(funcBody);
- func.setParamList(paramList);
- return func;
+ signature = new FunctionSignature(defaultDataverse, functionName, arity);
+ getCurrentScope().addFunctionDescriptor(signature, false);
+ funcDecl = new FunctionDecl(signature, paramList, funcBody);
+ return funcDecl;
}
}
CreateFunctionStatement FunctionCreation() throws ParseException:
{
CreateFunctionStatement cfs = null;
- AsterixFunction ident;
+ FunctionSignature signature;
+ String dataverse;
String functionName;
- int arity = 0;
boolean ifNotExists = false;
List<VarIdentifier> paramList = new ArrayList<VarIdentifier>();
- String funcBody;
+ String functionBody;
VarIdentifier var = null;
createNewScope();
+ Expression functionBodyExpr;
+ Token beginPos;
+ Token endPos;
+ Pair<Identifier,Identifier> nameComponents=null;
}
{
-
- <IDENTIFIER>
- {
- Token t = getToken(0);
- functionName= t.toString();
+ {
+ nameComponents = getDotSeparatedPair();
+ dataverse = nameComponents.first != null ? nameComponents.first.getValue() : defaultDataverse;
+ functionName= nameComponents.second.getValue();
}
(
@@ -1103,7 +1416,6 @@
var.setValue(getToken(0).toString());
paramList.add(var);
getCurrentScope().addNewVarSymbolToScope(var);
- arity++;
}
("," <VARIABLE>
{
@@ -1111,16 +1423,20 @@
var.setValue(getToken(0).toString());
paramList.add(var);
getCurrentScope().addNewVarSymbolToScope(var);
- arity++;
- })*)? <RIGHTPAREN> "{" <STRING_LITERAL>
+ })*)? <RIGHTPAREN> "{"
{
- funcBody = removeQuotesAndEscapes(token.image);
- }
+ beginPos = getToken(0);
+ }
+ functionBodyExpr = Expression()
"}"
+ {
+ endPos = getToken(0);
+ functionBody = extractFragment(beginPos.beginLine, beginPos.beginColumn, endPos.beginLine, endPos.beginColumn);
+ }
{
- ident = new AsterixFunction(functionName, arity);
- getCurrentScope().addFunctionDescriptor(ident, false);
- cfs = new CreateFunctionStatement(ident, paramList, funcBody, ifNotExists);
+ signature = new FunctionSignature(dataverse, functionName, paramList.size());
+ getCurrentScope().addFunctionDescriptor(signature, false);
+ cfs = new CreateFunctionStatement(signature, paramList, functionBody, ifNotExists);
return cfs;
}
}
@@ -1134,11 +1450,13 @@
}
{
expr = Expression()
-
+ (";")?
{
query.setBody(expr);
+ query.setVarCounter(getVarCounter());
return query;
}
+
}
@@ -1156,7 +1474,7 @@
| expr = IfThenElse()
| expr = FLWOGR()
| expr = QuantifiedExpression()
-
+
)
{
@@ -1237,14 +1555,15 @@
OperatorExpr op = null;
Expression operand = null;
boolean broadcast = false;
+ IExpressionAnnotation annotation = null;
}
{
operand = AddExpr()
- {
- if (operand instanceof VariableExpr) {
- String hint = getHint(token);
+ {
+ if (operand instanceof VariableExpr) {
+ String hint = getHint(token);
if (hint != null && hint.equals(BROADCAST_JOIN_HINT)) {
- broadcast = true;
+ broadcast = true;
}
}
}
@@ -1252,6 +1571,10 @@
(
LOOKAHEAD(2)( "<" | ">" | "<=" | ">=" | "=" | "!=" |"~=")
{
+ String mhint = getHint(token);
+ if (mhint != null && mhint.equals(INDEXED_NESTED_LOOP_JOIN_HINT)) {
+ annotation = IndexedNLJoinExpressionAnnotation.INSTANCE;
+ }
if (op == null) {
op = new OperatorExpr();
op.addOperand(operand, broadcast);
@@ -1264,18 +1587,21 @@
operand = AddExpr()
{
- broadcast = false;
- if (operand instanceof VariableExpr) {
- String hint = getHint(token);
+ broadcast = false;
+ if (operand instanceof VariableExpr) {
+ String hint = getHint(token);
if (hint != null && hint.equals(BROADCAST_JOIN_HINT)) {
broadcast = true;
}
- }
+ }
op.addOperand(operand, broadcast);
}
)?
{
+ if (annotation != null) {
+ op.addHint(annotation);
+ }
return op==null? operand: op;
}
}
@@ -1470,10 +1796,11 @@
{
if(expr.getKind() == Expression.Kind.LITERAL_EXPRESSION)
{
- ILiteral lit = ((LiteralExpr)expr).getValue();
- if(lit.getLiteralType() == ILiteral.Type.INTEGER) {
+ Literal lit = ((LiteralExpr)expr).getValue();
+ if(lit.getLiteralType() == Literal.Type.INTEGER ||
+ lit.getLiteralType() == Literal.Type.LONG) {
idx = Integer.valueOf(lit.getStringValue());
- }
+ }
else {
throw new ParseException("Index should be an INTEGER");
}
@@ -1501,10 +1828,11 @@
Expression expr = null;
}
{
- //ILiteral | VariableRef | ListConstructor | RecordConstructor | FunctionCallExpr | ParenthesizedExpression
+ //Literal | VariableRef | ListConstructor | RecordConstructor | FunctionCallExpr | DatasetAccessExpression | ParenthesizedExpression
(
expr =Literal()
| expr = FunctionCallExpr()
+ | expr = DatasetAccessExpression()
| expr =VariableRef()
{
@@ -1537,7 +1865,11 @@
| <INTEGER_LITERAL>
{
t= getToken(0);
- lit.setValue(new IntegerLiteral(new Integer(t.image)));
+ try {
+ lit.setValue(new IntegerLiteral(new Integer(t.image)));
+ } catch(NumberFormatException ex) {
+ lit.setValue(new LongIntegerLiteral(new Long(t.image)));
+ }
}
| < FLOAT_LITERAL >
{
@@ -1711,18 +2043,24 @@
}
}
+
Expression FunctionCallExpr() throws ParseException:
{
- CallExpr pf = new CallExpr();
- List<Expression > argList = new ArrayList<Expression >();
+ CallExpr callExpr;
+ List<Expression> argList = new ArrayList<Expression>();
Expression tmp;
int arity = 0;
- Token funcName;
+ String funcName;
+ String dataverse;
+ String hint=null;
+ String id1=null;
+ String id2=null;
}
-{
- ( <IDENTIFIER> | <DATASET> )
+{
+
+ <IDENTIFIER> { dataverse = defaultDataverse; funcName = token.image;} ("." <IDENTIFIER> { dataverse = funcName; funcName = token.image;})?
{
- funcName = getToken(0);
+ hint = getHint(token);
}
<LEFTPAREN> (tmp = Expression()
{
@@ -1731,19 +2069,47 @@
} ("," tmp = Expression() { argList.add(tmp); arity++; })*)? <RIGHTPAREN>
{
- AsterixFunction fd = lookupFunctionSignature(funcName.toString(), arity);
- if(fd == null)
- {
- fd = new AsterixFunction(funcName.toString(), arity);
-// notFoundFunctionList.add(fd);
- }
-// throw new ParseException("can't find function "+ funcName.toString() + "@" + arity);
- pf.setIdent(fd);
- pf.setExprList(argList);
- return pf;
+ FunctionSignature signature = lookupFunctionSignature(dataverse, funcName.toString(), arity);
+ if(signature == null)
+ {
+ signature = new FunctionSignature(dataverse, funcName.toString(), arity);
+ }
+ callExpr = new CallExpr(signature,argList);
+ if (hint != null && hint.startsWith(INDEXED_NESTED_LOOP_JOIN_HINT)) {
+ callExpr.addHint(IndexedNLJoinExpressionAnnotation.INSTANCE);
+ }
+ return callExpr;
}
}
+Expression DatasetAccessExpression() throws ParseException:
+{
+ CallExpr callExpr;
+ List<Expression> argList = new ArrayList<Expression>();
+ String funcName;
+ String dataverse;
+ LiteralExpr ds;
+ LiteralExpr dvds;
+ Expression nameArg;
+ int arity = 0;
+}
+{
+ <DATASET> {dataverse = MetadataConstants.METADATA_DATAVERSE_NAME; funcName = getToken(0).toString();}
+ (
+ (<IDENTIFIER> {ds = new LiteralExpr(); ds.setValue( new StringLiteral(token.image) ); argList.add(ds); arity ++;} ("." <IDENTIFIER> { dvds = new LiteralExpr(); dvds.setValue(new StringLiteral(ds.getValue()+"."+token.image)); argList.remove(0); argList.add(dvds);})? ) |
+ (<LEFTPAREN> nameArg = Expression() {argList.add(nameArg); arity ++;} ("," nameArg = Expression() { argList.add(nameArg); arity++; })* <RIGHTPAREN>)
+ )
+
+ {
+ FunctionSignature signature = lookupFunctionSignature(dataverse, funcName.toString(), arity);
+ if(signature == null)
+ {
+ signature = new FunctionSignature(dataverse, funcName.toString(), arity);
+ }
+ callExpr = new CallExpr(signature,argList);
+ return callExpr;
+ }
+}
Expression ParenthesizedExpression() throws ParseException:
{
@@ -1851,12 +2217,9 @@
extendCurrentScope();
}
{
- "let" varExp = Variable()
+ "let" varExp = Variable() ":=" beExp = Expression()
{
getCurrentScope().addNewVarSymbolToScope(varExp.getVar());
- }
- ":=" beExp = Expression()
- {
lc.setVarExpr(varExp);
lc.setBeExpr(beExp);
return lc;
@@ -2203,6 +2566,7 @@
<IDENTIFIER : (<LETTER>)+ (<LETTER> | <DIGIT> | <SPECIALCHARS>)*>
}
+
<DEFAULT>
TOKEN :
{