merge from master
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
index 49ca5ba..48f9e36 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
@@ -3,7 +3,8 @@
import java.util.ArrayList;
import java.util.List;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.api.AsterixAppContextInfo;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.common.dataflow.IAsterixApplicationContextInfo;
import edu.uci.ics.asterix.metadata.MetadataException;
@@ -196,20 +197,22 @@
IBinaryTokenizerFactory queryTokenizerFactory = InvertedIndexAccessMethod.getBinaryTokenizerFactory(
searchModifierType, searchKeyType, secondaryIndex);
IIndexDataflowHelperFactory dataflowHelperFactory;
+
+ AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
if (!isPartitioned) {
dataflowHelperFactory = new LSMInvertedIndexDataflowHelperFactory(
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages());
} else {
dataflowHelperFactory = new PartitionedLSMInvertedIndexDataflowHelperFactory(
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages());
}
LSMInvertedIndexSearchOperatorDescriptor invIndexSearchOp = new LSMInvertedIndexSearchOperatorDescriptor(
jobSpec, queryField, appContext.getStorageManagerInterface(), secondarySplitsAndConstraint.first,
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceDynamicTypeCastRule.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceDynamicTypeCastRule.java
index 5cff07b..1f242b8 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceDynamicTypeCastRule.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/IntroduceDynamicTypeCastRule.java
@@ -109,8 +109,7 @@
inputRecordVar = usedVariables.get(0);
}
IVariableTypeEnvironment env = oldAssignOperator.computeInputTypeEnvironment(context);
- ARecordType inputRecordType = (ARecordType) env.getVarType(inputRecordVar);
-
+ IAType inputRecordType = (IAType) env.getVarType(inputRecordVar);
boolean needCast = !requiredRecordType.equals(inputRecordType);
if (!needCast)
return false;
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/typecast/StaticTypeCastUtil.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/typecast/StaticTypeCastUtil.java
index 6cb4d5c..6e3ab37 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/typecast/StaticTypeCastUtil.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/optimizer/rules/typecast/StaticTypeCastUtil.java
@@ -17,7 +17,9 @@
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
@@ -160,6 +162,10 @@
changed = changed || rewriteFuncExpr(argFuncExpr, exprType, exprType, env);
}
}
+ if (!compatible(reqType, inputType)) {
+ throw new AlgebricksException("type mistmach, requred: " + reqType.toString() + " actual: "
+ + inputType.toString());
+ }
return changed;
}
}
@@ -489,4 +495,42 @@
exprRef.setValue(cast);
TypeComputerUtilities.setRequiredAndInputTypes(cast, reqType, inputType);
}
+
+ /**
+ * Determine if two types are compatible
+ *
+ * @param reqType
+ * the required type
+ * @param inputType
+ * the input type
+ * @return true if the two types are compatiable; false otherwise
+ */
+ private static boolean compatible(IAType reqType, IAType inputType) {
+ if (reqType.getTypeTag() == ATypeTag.ANY || inputType.getTypeTag() == ATypeTag.ANY) {
+ return true;
+ }
+ if (reqType.getTypeTag() != ATypeTag.UNION && inputType.getTypeTag() != ATypeTag.UNION) {
+ if (reqType.equals(inputType)) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+ Set<IAType> reqTypePossible = new HashSet<IAType>();
+ Set<IAType> inputTypePossible = new HashSet<IAType>();
+ if (reqType.getTypeTag() == ATypeTag.UNION) {
+ AUnionType unionType = (AUnionType) reqType;
+ reqTypePossible.addAll(unionType.getUnionList());
+ } else {
+ reqTypePossible.add(reqType);
+ }
+
+ if (inputType.getTypeTag() == ATypeTag.UNION) {
+ AUnionType unionType = (AUnionType) inputType;
+ inputTypePossible.addAll(unionType.getUnionList());
+ } else {
+ inputTypePossible.add(inputType);
+ }
+ return reqTypePossible.equals(inputTypePossible);
+ }
}
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java
index 854ec9f..e07c1d0 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java
@@ -71,7 +71,8 @@
import edu.uci.ics.asterix.aql.expression.WriteStatement;
import edu.uci.ics.asterix.aql.expression.visitor.IAqlExpressionVisitor;
import edu.uci.ics.asterix.aql.util.FunctionUtils;
-import edu.uci.ics.asterix.common.config.AsterixProperties;
+import edu.uci.ics.asterix.common.api.AsterixAppContextInfo;
+import edu.uci.ics.asterix.common.config.AsterixMetadataProperties;
import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.common.functions.FunctionConstants;
@@ -289,7 +290,8 @@
String outputDir = System.getProperty("java.io.tmpDir");
String filePath = outputDir + System.getProperty("file.separator") + OUTPUT_FILE_PREFIX
+ outputFileID.incrementAndGet();
- return new FileSplit(AsterixProperties.INSTANCE.getMetadataNodeName(), new FileReference(new File(filePath)));
+ AsterixMetadataProperties metadataProperties = AsterixAppContextInfo.getInstance().getMetadataProperties();
+ return new FileSplit(metadataProperties.getMetadataNodeName(), new FileReference(new File(filePath)));
}
@Override
diff --git a/asterix-app/data/fbu-dml-insert-shuffled.adm b/asterix-app/data/fbu-dml-insert-shuffled.adm
new file mode 100644
index 0000000..5e42fd2
--- /dev/null
+++ b/asterix-app/data/fbu-dml-insert-shuffled.adm
@@ -0,0 +1,29 @@
+{"id":11381089,"id-copy":11381089,"alias":"Earlene","name":"EarleneAmmons","user-since":datetime("2010-03-24T05:25:35"),"user-since-copy":datetime("2010-03-24T05:25:35"),"friend-ids":{{25392364,36996951,16110083,9799716,22893553,28551996,7706432,14225386,15633254,39395931,46707062,37226919,8532306,3765988,20939685,31136325,45222021,15355741,8760941,12045616,6890610,13560532,44914868,37744233}},"employment":[{"organization-name":"Roundhex","start-date":date("2000-06-10")}]}
+{"id":10495420,"id-copy":10495420,"alias":"Wendy","name":"WendyMcloskey","user-since":datetime("2011-04-26T23:38:24"),"user-since-copy":datetime("2011-04-26T23:38:24"),"friend-ids":{{16762653,46262691,12313140,20481262,347993,23105127,1680519,20880265,45611347,21907223,46615281,17188244,44019800,46943250,28647738,16792673,29406270,42714079}},"employment":[{"organization-name":"Qvohouse","start-date":date("2008-08-27")}]}
+{"id":10957867,"id-copy":10957867,"alias":"Zach","name":"ZachOppenheimer","user-since":datetime("2012-01-01T14:40:11"),"user-since-copy":datetime("2012-01-01T14:40:11"),"friend-ids":{{27759480,2112389,8560433,10052851,37714587,16717012,36648956,44803993,36030695,5359496,32302980,27143894,19287706}},"employment":[{"organization-name":"Tanzumbam","start-date":date("2003-05-14"),"end-date":date("2004-02-23")}]}
+{"id":9988417,"id-copy":9988417,"alias":"Coline","name":"ColineLane","user-since":datetime("2010-01-01T00:12:39"),"user-since-copy":datetime("2010-01-01T00:12:39"),"friend-ids":{{17656229,42804152}},"employment":[{"organization-name":"Fax-fax","start-date":date("2012-05-01")}]}
+{"id":10272571,"id-copy":10272571,"alias":"Jarrett","name":"JarrettGoldvogel","user-since":datetime("2010-04-28T23:24:22"),"user-since-copy":datetime("2010-04-28T23:24:22"),"friend-ids":{{47024505,36647273,32152567,28239957,11739703,47515825,17408763,41224279,41487670,43339913}},"employment":[{"organization-name":"Transhigh","start-date":date("2004-04-06"),"end-date":date("2010-02-14")}]}
+{"id":11307946,"id-copy":11307946,"alias":"Helga","name":"HelgaStough","user-since":datetime("2007-01-12T21:50:11"),"user-since-copy":datetime("2007-01-12T21:50:11"),"friend-ids":{{22768365}},"employment":[{"organization-name":"subtam","start-date":date("2007-01-04"),"end-date":date("2009-06-25")}]}
+{"id":11061631,"id-copy":11061631,"alias":"Maxene","name":"MaxeneKellogg","user-since":datetime("2005-11-13T01:09:31"),"user-since-copy":datetime("2005-11-13T01:09:31"),"friend-ids":{{31578394,39466620,35741359,14244925,3000582,39031643,5008430,18315325,30440631,37868108,12014032,32314102,42887702,1853960,28022174,2024670,38864358,42073112,16259942,34693959,25315399,37475597,33599283}},"employment":[{"organization-name":"Unijobam","start-date":date("2008-05-13")}]}
+{"id":10874791,"id-copy":10874791,"alias":"Haydee","name":"HaydeeGarratt","user-since":datetime("2007-04-14T00:19:00"),"user-since-copy":datetime("2007-04-14T00:19:00"),"friend-ids":{{12247794,10306863,33161811,43877113,37745696}},"employment":[{"organization-name":"Opeelectronics","start-date":date("2008-03-07"),"end-date":date("2011-12-27")}]}
+{"id":11570326,"id-copy":11570326,"alias":"Linden","name":"LindenFilby","user-since":datetime("2007-08-16T03:11:11"),"user-since-copy":datetime("2007-08-16T03:11:11"),"friend-ids":{{6549689,15243636,3147666}},"employment":[{"organization-name":"Solfix","start-date":date("2010-02-23"),"end-date":date("2010-04-22")}]}
+{"id":10498285,"id-copy":10498285,"alias":"Kiley","name":"KileyBridger","user-since":datetime("2006-05-14T21:55:34"),"user-since-copy":datetime("2006-05-14T21:55:34"),"friend-ids":{{38780484,46190003,905670,35609390,46621151,5099226,24328595,16340411,13326485,13872400,35896828,9196151,8525875,7461206,28379538,46461267,45270205,35718577,5310596,7080391}},"employment":[{"organization-name":"Newcom","start-date":date("2009-11-11"),"end-date":date("2009-06-23")}]}
+{"id":9629395,"id-copy":9629395,"alias":"Julius","name":"JuliusWire","user-since":datetime("2008-03-22T13:36:24"),"user-since-copy":datetime("2008-03-22T13:36:24"),"friend-ids":{{}},"employment":[{"organization-name":"Tranzap","start-date":date("2006-11-19")}]}
+{"id":11447332,"id-copy":11447332,"alias":"Sherisse","name":"SherisseMaugham","user-since":datetime("2012-02-09T14:21:08"),"user-since-copy":datetime("2012-02-09T14:21:08"),"friend-ids":{{}},"employment":[{"organization-name":"Tripplelane","start-date":date("2011-09-16")}]}
+{"id":10179538,"id-copy":10179538,"alias":"Orlando","name":"OrlandoBaxter","user-since":datetime("2006-02-06T08:33:07"),"user-since-copy":datetime("2006-02-06T08:33:07"),"friend-ids":{{6233497,33888281,44259464,19279042,22534429,13084190,38886041,41675566,3155617}},"employment":[{"organization-name":"Ontohothex","start-date":date("2009-07-06")}]}
+{"id":10001080,"id-copy":10001080,"alias":"Garrett","name":"GarrettBode","user-since":datetime("2005-10-25T18:07:35"),"user-since-copy":datetime("2005-10-25T18:07:35"),"friend-ids":{{35858744,16426061,11473961,4769664,29038930,33070686,46271872,42593454,36202882,46642640,22243678,20222041,29014540,7389258,7172909,12787979,146736,21081030,21615179,2936936,44934891}},"employment":[{"organization-name":"Tanzimcare","start-date":date("2007-06-24")}]}
+{"id":11675221,"id-copy":11675221,"alias":"Calanthe","name":"CalantheGearhart","user-since":datetime("2007-06-08T02:44:20"),"user-since-copy":datetime("2007-06-08T02:44:20"),"friend-ids":{{19185575}},"employment":[{"organization-name":"Vivaace","start-date":date("2010-05-21")}]}
+{"id":11140213,"id-copy":11140213,"alias":"Montgomery","name":"MontgomeryWhittier","user-since":datetime("2007-06-19T17:46:13"),"user-since-copy":datetime("2007-06-19T17:46:13"),"friend-ids":{{32831460,6030454,30437362,21866470,17388602,40815157,20000967,47555494,5818137,40634742,21692148,2365521,33290069,46471164,9192561,35768343,7552168,3577338,5346012,31129868}},"employment":[{"organization-name":"Y-geohex","start-date":date("2008-02-24")}]}
+{"id":11954992,"id-copy":11954992,"alias":"Caitlin","name":"CaitlinLangston","user-since":datetime("2007-01-02T01:50:34"),"user-since-copy":datetime("2007-01-02T01:50:34"),"friend-ids":{{23355687,22474136,28513847,32515387,44041844,33706721,10874992,36341753,34431157,16146113,15462591,18188151,29554174,44940738,25888018,42795884,14382632,12734889,11724519,15830341,25725320,37580394,24124411,47984339}},"employment":[{"organization-name":"Kanelectrics","start-date":date("2010-05-26"),"end-date":date("2010-03-28")}]}
+{"id":9510451,"id-copy":9510451,"alias":"Chuck","name":"ChuckFinck","user-since":datetime("2011-09-10T08:27:31"),"user-since-copy":datetime("2011-09-10T08:27:31"),"friend-ids":{{5559039,8997599,8311284,20478562,13734713,21511695,30393493}},"employment":[{"organization-name":"Inchdox","start-date":date("2001-10-12")}]}
+{"id":11068231,"id-copy":11068231,"alias":"Dinah","name":"DinahSwink","user-since":datetime("2012-05-02T04:24:33"),"user-since-copy":datetime("2012-05-02T04:24:33"),"friend-ids":{{31542440,17451543,32642661,27867264,32718667,43042567,7921827}},"employment":[{"organization-name":"highfax","start-date":date("2003-04-10"),"end-date":date("2003-10-03")}]}
+{"id":10361965,"id-copy":10361965,"alias":"Arlen","name":"ArlenFlick","user-since":datetime("2011-07-14T18:38:37"),"user-since-copy":datetime("2011-07-14T18:38:37"),"friend-ids":{{34249140,2887282,47622716,3897801,33692288,14374380,14183995,41311739,6378075,17721901,20807501,8908974,41080464,26497672}},"employment":[{"organization-name":"Medflex","start-date":date("2008-05-18"),"end-date":date("2011-09-18")}]}
+{"id":10423588,"id-copy":10423588,"alias":"Shirlene","name":"ShirleneRuch","user-since":datetime("2006-04-09T05:52:24"),"user-since-copy":datetime("2006-04-09T05:52:24"),"friend-ids":{{15418780,12724265,27282306,13592995,24753166,32824252,40619106,27563604,12337625,45387219,27749581,44912564,37470078,19663516}},"employment":[{"organization-name":"Newphase","start-date":date("2003-06-17")}]}
+{"id":11951098,"id-copy":11951098,"alias":"Tera","name":"TeraByers","user-since":datetime("2012-08-03T19:41:26"),"user-since-copy":datetime("2012-08-03T19:41:26"),"friend-ids":{{15537238,13699967,10587728,23542817,12703626,25024772,19223339,5547239,42576945,27351017,22726496,25268071,4361323,24631578,38669047,44781738,34646381}},"employment":[{"organization-name":"Sublamdox","start-date":date("2008-01-04"),"end-date":date("2011-01-14")}]}
+{"id":9594523,"id-copy":9594523,"alias":"Tam","name":"TamWillcox","user-since":datetime("2011-12-23T11:41:58"),"user-since-copy":datetime("2011-12-23T11:41:58"),"friend-ids":{{27383896,20745988,10063024,8241427,40299998,32408463,25171835,22380586,15344194,25951348,28733234,45421004,2273747,2229862,6241144,6704115,8659430,47431991,47929530,24393021}},"employment":[{"organization-name":"Keytech","start-date":date("2001-07-27")}]}
+{"id":9478720,"id-copy":9478720,"alias":"Angelia","name":"AngeliaKettlewell","user-since":datetime("2005-05-27T06:29:30"),"user-since-copy":datetime("2005-05-27T06:29:30"),"friend-ids":{{42556433,20033025,38112512,19420757,31822717,7116081,39544900,19203395,46787205,32303456,4509345,45558040,42616291,6929369,9272653,37459048,37113569,38942369,47741031,46761451,14163845}},"employment":[{"organization-name":"Alphadax","start-date":date("2012-03-28"),"end-date":date("2012-03-04")}]}
+{"id":9142198,"id-copy":9142198,"alias":"Sherry","name":"SherryFea","user-since":datetime("2011-03-28T23:09:22"),"user-since-copy":datetime("2011-03-28T23:09:22"),"friend-ids":{{6835080,34471872,30942941,34858577,5996593,47293442,43097072,44809621,33969893,26410931,6628186,29944391,35957320,20326929,40284077,11681583,43878314,40265961,16871274,28406169,1349311}},"employment":[{"organization-name":"Mathtech","start-date":date("2004-07-28")}]}
+{"id":9313492,"id-copy":9313492,"alias":"Tera","name":"TeraWolfe","user-since":datetime("2010-12-20T12:47:25"),"user-since-copy":datetime("2010-12-20T12:47:25"),"friend-ids":{{45424983,18345704,14849759,31638064,38670515,48015953,36114769}},"employment":[{"organization-name":"Redelectronics","start-date":date("2001-04-26"),"end-date":date("2004-12-06")}]}
+{"id":10307032,"id-copy":10307032,"alias":"Quentin","name":"QuentinSauter","user-since":datetime("2012-07-11T07:16:43"),"user-since-copy":datetime("2012-07-11T07:16:43"),"friend-ids":{{1926278,42211794,1508832,14973540,6721046,28872485,5047722,7805271,31508326,20891455,38735410,13190567,18209753,44468536,34640135,47290587,25576626}},"employment":[{"organization-name":"Zamcorporation","start-date":date("2012-02-13")}]}
+{"id":10733617,"id-copy":10733617,"alias":"Leonardo","name":"LeonardoKight","user-since":datetime("2008-10-20T17:30:29"),"user-since-copy":datetime("2008-10-20T17:30:29"),"friend-ids":{{39687903,7235506,34696496,25995345,18435380,47473591,15710408,44232442,39520147,36384026,25160887,245860,1195579,4587411,536916,47052672,33953823,13203710}},"employment":[{"organization-name":"tresline","start-date":date("2007-07-12"),"end-date":date("2010-03-16")}]}
+{"id":10394488,"id-copy":10394488,"alias":"Oswald","name":"OswaldRay","user-since":datetime("2006-02-12T17:39:23"),"user-since-copy":datetime("2006-02-12T17:39:23"),"friend-ids":{{14370372,14174983,7749259,39375970,1755409,9056913}},"employment":[{"organization-name":"Canline","start-date":date("2011-12-04"),"end-date":date("2011-06-08")}]}
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java
index 8d65323..e1114eb 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java
@@ -26,8 +26,8 @@
import edu.uci.ics.asterix.aql.expression.Query;
import edu.uci.ics.asterix.aql.expression.visitor.AQLPrintVisitor;
import edu.uci.ics.asterix.aql.rewrites.AqlRewriter;
-import edu.uci.ics.asterix.common.api.AsterixAppContextInfoImpl;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.api.AsterixAppContextInfo;
+import edu.uci.ics.asterix.common.config.AsterixCompilerProperties;
import edu.uci.ics.asterix.common.config.OptimizationConfUtil;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.dataflow.data.common.AqlExpressionTypeComputer;
@@ -258,7 +258,8 @@
}
}
- int frameSize = GlobalConfig.getFrameSize();
+ AsterixCompilerProperties compilerProperties = AsterixAppContextInfo.getInstance().getCompilerProperties();
+ int frameSize = compilerProperties.getFrameSize();
HeuristicCompilerFactoryBuilder builder = new HeuristicCompilerFactoryBuilder(
AqlOptimizationContextFactory.INSTANCE);
@@ -342,7 +343,7 @@
IJobletEventListenerFactory jobEventListenerFactory = new JobEventListenerFactory(asterixJobId,
isWriteTransaction);
- JobSpecification spec = compiler.createJob(AsterixAppContextInfoImpl.getInstance(), jobEventListenerFactory);
+ JobSpecification spec = compiler.createJob(AsterixAppContextInfo.getInstance(), jobEventListenerFactory);
if (pc.isPrintJob()) {
switch (pdf) {
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
index 428781f..66a3980 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
@@ -64,6 +64,7 @@
import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.common.functions.FunctionSignature;
import edu.uci.ics.asterix.file.DatasetOperations;
+import edu.uci.ics.asterix.file.DataverseOperations;
import edu.uci.ics.asterix.file.FeedOperations;
import edu.uci.ics.asterix.file.IndexOperations;
import edu.uci.ics.asterix.formats.base.IDataFormat;
@@ -765,6 +766,7 @@
jobsToExecute.add(DatasetOperations.createDropDatasetJobSpec(cds, metadataProvider));
}
}
+ jobsToExecute.add(DataverseOperations.createDropDataverseJobSpec(dv, metadataProvider));
//#. mark PendingDropOp on the dataverse record by
// first, deleting the dataverse record from the DATAVERSE_DATASET
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
index b9b9524..25a2551 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
@@ -22,6 +22,8 @@
import java.util.logging.Logger;
import edu.uci.ics.asterix.api.common.Job;
+import edu.uci.ics.asterix.common.api.AsterixAppContextInfo;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
import edu.uci.ics.asterix.common.config.GlobalConfig;
import edu.uci.ics.asterix.common.config.OptimizationConfUtil;
@@ -118,14 +120,15 @@
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider
.splitProviderAndPartitionConstraintsForInternalOrFeedDataset(dataset.getDataverseName(), datasetName,
datasetName);
+ AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
IndexDropOperatorDescriptor primaryBtreeDrop = new IndexDropOperatorDescriptor(specPrimary,
AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER, AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
splitsAndConstraint.first, new LSMBTreeDataflowHelperFactory(
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES));
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()));
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(specPrimary, primaryBtreeDrop,
splitsAndConstraint.second);
@@ -163,10 +166,11 @@
}
LOGGER.info("CREATING File Splits: " + sb.toString());
+ AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
//prepare a LocalResourceMetadata which will be stored in NC's local resource repository
ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(typeTraits,
- comparatorFactories, blooFilterKeyFields, true, GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
+ comparatorFactories, blooFilterKeyFields, true, storageProperties.getMemoryComponentPageSize(),
+ storageProperties.getMemoryComponentNumPages());
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMBTreeResource);
@@ -176,9 +180,9 @@
new LSMBTreeDataflowHelperFactory(AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES), localResourceFactoryProvider,
- NoOpOperationCallbackFactory.INSTANCE);
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
+ localResourceFactoryProvider, NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, indexCreateOp,
splitsAndConstraint.second);
spec.addRoot(indexCreateOp);
@@ -258,15 +262,17 @@
numElementsHint = Long.parseLong(dataset.getHints().get("CARDINALITY"));
}
+ AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
TreeIndexBulkLoadOperatorDescriptor btreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER, AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
splitsAndConstraint.first, typeTraits, comparatorFactories, blooFilterKeyFields, fieldPermutation,
- GlobalConfig.DEFAULT_BTREE_FILL_FACTOR, false, numElementsHint, new LSMBTreeDataflowHelperFactory(
+ GlobalConfig.DEFAULT_BTREE_FILL_FACTOR, false, numElementsHint,
+ new LSMBTreeDataflowHelperFactory(AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES), NoOpOperationCallbackFactory.INSTANCE);
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
+ NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, btreeBulkLoad,
splitsAndConstraint.second);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DataverseOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DataverseOperations.java
new file mode 100644
index 0000000..0654ff7
--- /dev/null
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DataverseOperations.java
@@ -0,0 +1,22 @@
+package edu.uci.ics.asterix.file;
+
+import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
+import edu.uci.ics.asterix.metadata.entities.Dataverse;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
+import edu.uci.ics.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper;
+import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
+import edu.uci.ics.hyracks.api.job.JobSpecification;
+import edu.uci.ics.hyracks.dataflow.std.file.FileRemoveOperatorDescriptor;
+import edu.uci.ics.hyracks.dataflow.std.file.IFileSplitProvider;
+
+public class DataverseOperations {
+ public static JobSpecification createDropDataverseJobSpec(Dataverse dataverse, AqlMetadataProvider metadata) {
+ JobSpecification jobSpec = new JobSpecification();
+ Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadata
+ .splitProviderAndPartitionConstraintsForDataverse(dataverse.getDataverseName());
+ FileRemoveOperatorDescriptor frod = new FileRemoveOperatorDescriptor(jobSpec, splitsAndConstraint.first);
+ AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(jobSpec, frod, splitsAndConstraint.second);
+ jobSpec.addRoot(frod);
+ return jobSpec;
+ }
+}
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
index dbbbe44..cec97ad 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
@@ -1,6 +1,7 @@
package edu.uci.ics.asterix.file;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.api.AsterixAppContextInfo;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.config.OptimizationConfUtil;
import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
@@ -48,14 +49,15 @@
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider
.splitProviderAndPartitionConstraintsForInternalOrFeedDataset(dataverseName, datasetName, indexName);
+ AsterixStorageProperties storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
IndexDropOperatorDescriptor btreeDrop = new IndexDropOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER, AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
splitsAndConstraint.first, new LSMBTreeDataflowHelperFactory(
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES));
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()));
AlgebricksPartitionConstraintHelper
.setPartitionConstraintInJobSpec(spec, btreeDrop, splitsAndConstraint.second);
spec.addRoot(btreeDrop);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeCreator.java
index 2a1349b..958f8d1 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeCreator.java
@@ -1,6 +1,7 @@
package edu.uci.ics.asterix.file;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
+import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.transaction.management.resource.ILocalResourceMetadata;
@@ -26,31 +27,33 @@
public class SecondaryBTreeCreator extends SecondaryIndexCreator {
- protected SecondaryBTreeCreator(PhysicalOptimizationConfig physOptConf) {
- super(physOptConf);
+ protected SecondaryBTreeCreator(PhysicalOptimizationConfig physOptConf,
+ IAsterixPropertiesProvider propertiesProvider) {
+ super(physOptConf, propertiesProvider);
}
@Override
public JobSpecification buildCreationJobSpec() throws AsterixException, AlgebricksException {
JobSpecification spec = new JobSpecification();
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
//prepare a LocalResourceMetadata which will be stored in NC's local resource repository
ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(
secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, secondaryBloomFilterKeyFields, false,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages());
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMBTreeResource);
TreeIndexCreateOperatorDescriptor secondaryIndexCreateOp = new TreeIndexCreateOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER, AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
secondaryFileSplitProvider, secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories,
- secondaryBloomFilterKeyFields, new LSMBTreeDataflowHelperFactory(
+ secondaryBloomFilterKeyFields,
+ new LSMBTreeDataflowHelperFactory(AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES), localResourceFactoryProvider,
- NoOpOperationCallbackFactory.INSTANCE);
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
+ localResourceFactoryProvider, NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, secondaryIndexCreateOp,
secondaryPartitionConstraint);
spec.addRoot(secondaryIndexCreateOp);
@@ -80,13 +83,17 @@
// Sort by secondary keys.
ExternalSortOperatorDescriptor sortOp = createSortOp(spec, secondaryComparatorFactories, secondaryRecDesc);
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
// Create secondary BTree bulk load op.
- TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec, numSecondaryKeys,
+ TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(
+ spec,
+ numSecondaryKeys,
new LSMBTreeDataflowHelperFactory(AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES), BTree.DEFAULT_FILL_FACTOR);
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
+ BTree.DEFAULT_FILL_FACTOR);
// Connect the operators.
spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexCreator.java
index e7c2e3e..83c9393 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexCreator.java
@@ -19,8 +19,10 @@
import java.io.IOException;
import java.util.List;
+import edu.uci.ics.asterix.common.api.AsterixAppContextInfo;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.formats.nontagged.AqlBinaryBooleanInspectorImpl;
@@ -103,29 +105,34 @@
protected RecordDescriptor secondaryRecDesc;
protected ICopyEvaluatorFactory[] secondaryFieldAccessEvalFactories;
+ protected IAsterixPropertiesProvider propertiesProvider;
+
// Prevent public construction. Should be created via createIndexCreator().
- protected SecondaryIndexCreator(PhysicalOptimizationConfig physOptConf) {
+ protected SecondaryIndexCreator(PhysicalOptimizationConfig physOptConf,
+ IAsterixPropertiesProvider propertiesProvider) {
this.physOptConf = physOptConf;
+ this.propertiesProvider = propertiesProvider;
}
public static SecondaryIndexCreator createIndexCreator(CompiledCreateIndexStatement createIndexStmt,
AqlMetadataProvider metadataProvider, PhysicalOptimizationConfig physOptConf) throws AsterixException,
AlgebricksException {
+ IAsterixPropertiesProvider asterixPropertiesProvider = AsterixAppContextInfo.getInstance();
SecondaryIndexCreator indexCreator = null;
switch (createIndexStmt.getIndexType()) {
case BTREE: {
- indexCreator = new SecondaryBTreeCreator(physOptConf);
+ indexCreator = new SecondaryBTreeCreator(physOptConf, asterixPropertiesProvider);
break;
}
case RTREE: {
- indexCreator = new SecondaryRTreeCreator(physOptConf);
+ indexCreator = new SecondaryRTreeCreator(physOptConf, asterixPropertiesProvider);
break;
}
case WORD_INVIX:
case NGRAM_INVIX:
case FUZZY_WORD_INVIX:
case FUZZY_NGRAM_INVIX: {
- indexCreator = new SecondaryInvertedIndexCreator(physOptConf);
+ indexCreator = new SecondaryInvertedIndexCreator(physOptConf, asterixPropertiesProvider);
break;
}
default: {
@@ -266,6 +273,7 @@
int[] lowKeyFields = null;
// +Infinity
int[] highKeyFields = null;
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
BTreeSearchOperatorDescriptor primarySearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER, AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
primaryFileSplitProvider, primaryRecDesc.getTypeTraits(), primaryComparatorFactories,
@@ -273,8 +281,9 @@
new LSMBTreeDataflowHelperFactory(AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES), false, NoOpOperationCallbackFactory.INSTANCE);
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()), false,
+ NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, primarySearchOp,
primaryPartitionConstraint);
return primarySearchOp;
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
index 0bf379d..7b53a28 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
@@ -2,8 +2,9 @@
import java.util.List;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
@@ -60,8 +61,9 @@
private RecordDescriptor tokenKeyPairRecDesc;
private boolean isPartitioned;
- protected SecondaryInvertedIndexCreator(PhysicalOptimizationConfig physOptConf) {
- super(physOptConf);
+ protected SecondaryInvertedIndexCreator(PhysicalOptimizationConfig physOptConf,
+ IAsterixPropertiesProvider propertiesProvider) {
+ super(physOptConf, propertiesProvider);
}
@Override
@@ -150,10 +152,12 @@
public JobSpecification buildCreationJobSpec() throws AsterixException, AlgebricksException {
JobSpecification spec = new JobSpecification();
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
//prepare a LocalResourceMetadata which will be stored in NC's local resource repository
ILocalResourceMetadata localResourceMetadata = new LSMInvertedIndexLocalResourceMetadata(invListsTypeTraits,
primaryComparatorFactories, tokenTypeTraits, tokenComparatorFactories, tokenizerFactory,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES, isPartitioned);
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages(),
+ isPartitioned);
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMInvertedIndexResource);
@@ -259,20 +263,21 @@
}
private IIndexDataflowHelperFactory createDataflowHelperFactory() {
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
if (!isPartitioned) {
return new LSMInvertedIndexDataflowHelperFactory(
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages());
} else {
return new PartitionedLSMInvertedIndexDataflowHelperFactory(
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages());
}
}
}
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java
index 6f400f9..1a5e0da 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java
@@ -2,7 +2,8 @@
import java.util.List;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
+import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.dataflow.data.nontagged.valueproviders.AqlPrimitiveValueProviderFactory;
@@ -49,20 +50,22 @@
protected int numNestedSecondaryKeyFields;
protected ATypeTag keyType;
- protected SecondaryRTreeCreator(PhysicalOptimizationConfig physOptConf) {
- super(physOptConf);
+ protected SecondaryRTreeCreator(PhysicalOptimizationConfig physOptConf,
+ IAsterixPropertiesProvider propertiesProvider) {
+ super(physOptConf, propertiesProvider);
}
@Override
public JobSpecification buildCreationJobSpec() throws AsterixException, AlgebricksException {
JobSpecification spec = new JobSpecification();
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
//prepare a LocalResourceMetadata which will be stored in NC's local resource repository
ILocalResourceMetadata localResourceMetadata = new LSMRTreeLocalResourceMetadata(
secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, primaryComparatorFactories,
valueProviderFactories, RTreePolicyType.RTREE, AqlMetadataProvider.proposeLinearizer(keyType,
- secondaryComparatorFactories.length), GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
+ secondaryComparatorFactories.length), storageProperties.getMemoryComponentPageSize(),
+ storageProperties.getMemoryComponentNumPages());
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMRTreeResource);
@@ -74,8 +77,8 @@
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER, AqlMetadataProvider.proposeLinearizer(
- keyType, secondaryComparatorFactories.length),
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
+ keyType, secondaryComparatorFactories.length), storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
localResourceFactoryProvider, NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, secondaryIndexCreateOp,
secondaryPartitionConstraint);
@@ -149,6 +152,7 @@
selectOp = createFilterNullsSelectOp(spec, numNestedSecondaryKeyFields);
}
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
// Create secondary RTree bulk load op.
TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(
spec,
@@ -158,8 +162,8 @@
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER, AqlMetadataProvider.proposeLinearizer(
- keyType, secondaryComparatorFactories.length),
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
+ keyType, secondaryComparatorFactories.length), storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
BTree.DEFAULT_FILL_FACTOR);
// Connect the operators.
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
index 1b8990b..01cbca0 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
@@ -13,9 +13,9 @@
import edu.uci.ics.asterix.api.http.servlet.QueryResultAPIServlet;
import edu.uci.ics.asterix.api.http.servlet.QueryStatusAPIServlet;
import edu.uci.ics.asterix.api.http.servlet.UpdateAPIServlet;
-import edu.uci.ics.asterix.common.api.AsterixAppContextInfoImpl;
-import edu.uci.ics.asterix.common.config.AsterixProperties;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.api.AsterixAppContextInfo;
+import edu.uci.ics.asterix.common.config.AsterixExternalProperties;
+import edu.uci.ics.asterix.common.config.AsterixMetadataProperties;
import edu.uci.ics.asterix.metadata.MetadataManager;
import edu.uci.ics.asterix.metadata.api.IAsterixStateProxy;
import edu.uci.ics.asterix.metadata.bootstrap.AsterixStateProxy;
@@ -29,10 +29,6 @@
private static final String HYRACKS_CONNECTION_ATTR = "edu.uci.ics.asterix.HYRACKS_CONNECTION";
- private static final int DEFAULT_WEB_SERVER_PORT = 19001;
-
- private static final int DEFAULT_JSON_API_SERVER_PORT = 19101;
-
private Server webServer;
private Server jsonAPIServer;
private static IAsterixStateProxy proxy;
@@ -45,20 +41,19 @@
LOGGER.info("Starting Asterix cluster controller");
}
+ AsterixAppContextInfo.initialize(appCtx);
+
proxy = AsterixStateProxy.registerRemoteObject();
- proxy.setAsterixProperties(AsterixProperties.INSTANCE);
appCtx.setDistributedState(proxy);
- MetadataManager.INSTANCE = new MetadataManager(proxy);
+ AsterixMetadataProperties metadataProperties = AsterixAppContextInfo.getInstance().getMetadataProperties();
+ MetadataManager.INSTANCE = new MetadataManager(proxy, metadataProperties);
- setupWebServer();
+ AsterixExternalProperties externalProperties = AsterixAppContextInfo.getInstance().getExternalProperties();
+ setupWebServer(externalProperties);
webServer.start();
-
- // Setup and start the web interface
- setupJSONAPIServer();
+ setupJSONAPIServer(externalProperties);
jsonAPIServer.start();
-
- AsterixAppContextInfoImpl.initialize(appCtx);
}
@Override
@@ -78,10 +73,9 @@
return new HyracksConnection(strIP, port);
}
- private void setupWebServer() throws Exception {
- int port = Integer.parseInt((String) AsterixProperties.INSTANCE
- .getProperty(AsterixProperties.AsterixConfigurationKeys.WEB_INTERFACE_PORT));
- webServer = new Server(port);
+ private void setupWebServer(AsterixExternalProperties externalProperties) throws Exception {
+
+ webServer = new Server(externalProperties.getWebInterfacePort());
ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
context.setContextPath("/");
@@ -93,13 +87,8 @@
context.addServlet(new ServletHolder(new APIServlet()), "/*");
}
- private void setupJSONAPIServer() throws Exception {
- String portStr = System.getProperty(GlobalConfig.JSON_API_SERVER_PORT_PROPERTY);
- int port = DEFAULT_JSON_API_SERVER_PORT;
- if (portStr != null) {
- port = Integer.parseInt(portStr);
- }
- jsonAPIServer = new Server(port);
+ private void setupJSONAPIServer(AsterixExternalProperties externalProperties) throws Exception {
+ jsonAPIServer = new Server(externalProperties.getAPIServerPort());
ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
context.setContextPath("/");
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
index 8ad8c67..7dad984 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
@@ -5,6 +5,7 @@
import java.util.logging.Level;
import java.util.logging.Logger;
+import edu.uci.ics.asterix.common.config.AsterixMetadataProperties;
import edu.uci.ics.asterix.common.context.AsterixAppRuntimeContext;
import edu.uci.ics.asterix.metadata.MetadataManager;
import edu.uci.ics.asterix.metadata.MetadataNode;
@@ -86,32 +87,31 @@
@Override
public void notifyStartupComplete() throws Exception {
IAsterixStateProxy proxy = (IAsterixStateProxy) ncApplicationContext.getDistributedState();
+ AsterixMetadataProperties metadataProperties = runtimeContext.getMetadataProperties();
if (systemState == SystemState.NEW_UNIVERSE) {
- PersistentLocalResourceRepository localResourceRepository = (PersistentLocalResourceRepository) runtimeContext
- .getLocalResourceRepository();
-
if (LOGGER.isLoggable(Level.INFO)) {
- LOGGER.info("nodeid" + nodeId);
- LOGGER.info("proxy" + proxy);
- LOGGER.info("stores" + proxy.getAsterixProperties().getStores());
- LOGGER.info("store" + proxy.getAsterixProperties().getStores().get(nodeId)[0]);
+ LOGGER.info("System state: " + SystemState.NEW_UNIVERSE);
+ LOGGER.info("Node ID: " + nodeId);
+ LOGGER.info("Stores: " + metadataProperties.getStores());
+ LOGGER.info("Root Metadata Store: " + metadataProperties.getStores().get(nodeId)[0]);
}
- localResourceRepository.initialize(nodeId, proxy.getAsterixProperties().getStores().get(nodeId)[0], true,
- null);
+ PersistentLocalResourceRepository localResourceRepository = (PersistentLocalResourceRepository) runtimeContext
+ .getLocalResourceRepository();
+ localResourceRepository.initialize(nodeId, metadataProperties.getStores().get(nodeId)[0], true, null);
}
- isMetadataNode = nodeId.equals(proxy.getAsterixProperties().getMetadataNodeName());
+ isMetadataNode = nodeId.equals(metadataProperties.getMetadataNodeName());
if (isMetadataNode) {
registerRemoteMetadataNode(proxy);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Bootstrapping metadata");
}
- MetadataManager.INSTANCE = new MetadataManager(proxy);
+ MetadataManager.INSTANCE = new MetadataManager(proxy, metadataProperties);
MetadataManager.INSTANCE.init();
- MetadataBootstrap.startUniverse(proxy.getAsterixProperties(), ncApplicationContext,
+ MetadataBootstrap.startUniverse(runtimeContext, ncApplicationContext,
systemState == SystemState.NEW_UNIVERSE);
MetadataBootstrap.startDDLRecovery();
}
diff --git a/asterix-app/src/main/resources/asterix-build-configuration.xml b/asterix-app/src/main/resources/asterix-build-configuration.xml
index f2d0992..3e1b4b2 100644
--- a/asterix-app/src/main/resources/asterix-build-configuration.xml
+++ b/asterix-app/src/main/resources/asterix-build-configuration.xml
@@ -10,7 +10,7 @@
</store>
<property>
<name>log.level</name>
- <value>INFO</value>
- <description></description>
+ <value>WARNING</value>
+ <description>Log level for running tests/build</description>
</property>
</asterixConfiguration>
diff --git a/asterix-app/src/test/resources/AQLTS/queries/fieldAccessor.aql b/asterix-app/src/test/resources/AQLTS/queries/fieldAccessor.aql
new file mode 100644
index 0000000..3995374
--- /dev/null
+++ b/asterix-app/src/test/resources/AQLTS/queries/fieldAccessor.aql
@@ -0,0 +1,3 @@
+let $bla := { "name" : "value" }
+return
+ $bla."name" = $bla.name
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/AQLTS/queries/functionDecl3.aql b/asterix-app/src/test/resources/AQLTS/queries/functionDecl3.aql
new file mode 100644
index 0000000..80629be5
--- /dev/null
+++ b/asterix-app/src/test/resources/AQLTS/queries/functionDecl3.aql
@@ -0,0 +1,5 @@
+declare function "function with spaces"($a, $b) {
+ "string with spaces"
+};
+
+"function with spaces" (1, 2)
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/aggregate/query-issue400/query-issue400.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/aggregate/query-issue400/query-issue400.1.ddl.aql
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/aggregate/query-issue400/query-issue400.1.ddl.aql
diff --git a/asterix-app/src/test/resources/runtimets/queries/aggregate/query-issue400/query-issue400.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/aggregate/query-issue400/query-issue400.2.update.aql
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/aggregate/query-issue400/query-issue400.2.update.aql
diff --git a/asterix-app/src/test/resources/runtimets/queries/aggregate/query-issue400/query-issue400.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/aggregate/query-issue400/query-issue400.3.query.aql
new file mode 100644
index 0000000..9819455
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/aggregate/query-issue400/query-issue400.3.query.aql
@@ -0,0 +1,9 @@
+/*
+ * Description : This test case is to verify the fix for issue400
+ : https://code.google.com/p/asterixdb/issues/detail?id=400
+ * Expected Res : Success
+ * Date : 8th May 2013
+ */
+
+let $l := [[1,2,3,4,5],[6,7,8,9]]
+return count(for $i in $l return $i)
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/custord/join_q_04/join_q_04.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/custord/join_q_04/join_q_04.1.ddl.aql
index 2fbbf2f..882fecb 100644
--- a/asterix-app/src/test/resources/runtimets/queries/custord/join_q_04/join_q_04.1.ddl.aql
+++ b/asterix-app/src/test/resources/runtimets/queries/custord/join_q_04/join_q_04.1.ddl.aql
@@ -1,3 +1,10 @@
+/*
+ * Description : This test case is to verify the fix for issue51
+ : https://code.google.com/p/asterixdb/issues/detail?id=51
+ * Expected Res : SUCCESS
+ * Date : 14th May 2013
+ */
+
drop dataverse test if exists;
create dataverse test;
diff --git a/asterix-app/src/test/resources/runtimets/queries/custord/join_q_04/join_q_04.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/custord/join_q_04/join_q_04.2.update.aql
index e69de29..45fba70 100644
--- a/asterix-app/src/test/resources/runtimets/queries/custord/join_q_04/join_q_04.2.update.aql
+++ b/asterix-app/src/test/resources/runtimets/queries/custord/join_q_04/join_q_04.2.update.aql
@@ -0,0 +1,6 @@
+/*
+ * Description : This test case is to verify the fix for issue51
+ : https://code.google.com/p/asterixdb/issues/detail?id=51
+ * Expected Res : SUCCESS
+ * Date : 14th May 2013
+ */
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/custord/join_q_04/join_q_04.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/custord/join_q_04/join_q_04.3.query.aql
index 9996053..9781eb5 100644
--- a/asterix-app/src/test/resources/runtimets/queries/custord/join_q_04/join_q_04.3.query.aql
+++ b/asterix-app/src/test/resources/runtimets/queries/custord/join_q_04/join_q_04.3.query.aql
@@ -1,7 +1,19 @@
+/*
+ * Description : This test case is to verify the fix for issue51
+ : https://code.google.com/p/asterixdb/issues/detail?id=51
+ * Expected Res : SUCCESS
+ * Date : 14th May 2013
+ */
+
use dataverse test;
-for $c in dataset('Customers')
-return {"order_id" :
-for $o in dataset('Orders')
-where $c.cid = $o.cid
-return $o.oid }
+for $c in dataset Customers
+order by $c.name
+return {
+ "cust_name": $c.name,
+ "order_ids":
+ for $o in dataset Orders
+ where $c.cid = $o.cid
+ order by $o.oid
+ return $o.oid
+}
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/query-issue382/query-issue382.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/query-issue382/query-issue382.1.ddl.aql
new file mode 100644
index 0000000..d09e16d
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/query-issue382/query-issue382.1.ddl.aql
@@ -0,0 +1,26 @@
+create dataverse SocialNetworkData;
+
+use dataverse SocialNetworkData;
+
+create type EmploymentType as closed {
+organization-name: string,
+start-date: date,
+end-date: date?
+}
+
+create type FacebookUserType as closed {
+id: int32,
+id-copy: int32,
+alias: string,
+name: string,
+user-since: datetime,
+user-since-copy: datetime,
+friend-ids: {{ int32 }},
+employment: [EmploymentType]
+}
+
+create dataset FacebookUsers(FacebookUserType)
+primary key id;
+
+create dataset HandbookUsers(FacebookUserType)
+primary key id;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/query-issue382/query-issue382.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/query-issue382/query-issue382.2.update.aql
new file mode 100644
index 0000000..72f89e2
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/query-issue382/query-issue382.2.update.aql
@@ -0,0 +1,10 @@
+use dataverse SocialNetworkData;
+
+load dataset FacebookUsers using "edu.uci.ics.asterix.external.dataset.adapter.NCFileSystemAdapter"
+(("path"="nc1://data/fbu-dml-insert-shuffled.adm"),("format"="adm"));
+
+insert into dataset HandbookUsers (
+for $x in dataset FacebookUsers
+where $x.id < 10307032
+return $x
+)
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/query-issue382/query-issue382.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/dml/query-issue382/query-issue382.3.query.aql
new file mode 100644
index 0000000..4a2f1ff
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/query-issue382/query-issue382.3.query.aql
@@ -0,0 +1,3 @@
+use dataverse SocialNetworkData;
+
+count(for $h in dataset HandbookUsers return $h);
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/query-issue433/query-issue433.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/dml/query-issue433/query-issue433.1.ddl.aql
new file mode 100644
index 0000000..e7ab9e8
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/query-issue433/query-issue433.1.ddl.aql
@@ -0,0 +1,23 @@
+/*
+ * Description : This test case is to verify the fix for issue433
+ : https://code.google.com/p/asterixdb/issues/detail?id=433
+ * Expected Res : Success
+ * Date : 3th April 2013
+ */
+
+create dataverse insertIssue;
+use dataverse insertIssue;
+
+create type subElem as closed {
+n: string,
+e: int32?
+}
+
+create type elem as closed {
+id: int32,
+name: string,
+sub: [subElem]
+}
+
+create dataset myDataset(elem)
+primary key id;
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/query-issue433/query-issue433.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/dml/query-issue433/query-issue433.2.update.aql
new file mode 100644
index 0000000..d06b4f5
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/query-issue433/query-issue433.2.update.aql
@@ -0,0 +1,13 @@
+/*
+ * Description : This test case is to verify the fix for issue433
+ : https://code.google.com/p/asterixdb/issues/detail?id=433
+ * Expected Res : Success
+ * Date : 3th April 2013
+ */
+
+use dataverse insertIssue;
+
+insert into dataset myDataset (
+for $t in [ {"id":1, "name":"u1","sub":[{"n":"se1","e":100}]}, {"id":2, "name":"u2","sub":[{"n":"se2","e":200}]} ]
+return $t
+);
diff --git a/asterix-app/src/test/resources/runtimets/queries/dml/query-issue433/query-issue433.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/dml/query-issue433/query-issue433.3.query.aql
new file mode 100644
index 0000000..b863cdf
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/dml/query-issue433/query-issue433.3.query.aql
@@ -0,0 +1,12 @@
+/*
+ * Description : This test case is to verify the fix for issue433
+ : https://code.google.com/p/asterixdb/issues/detail?id=433
+ * Expected Res : Success
+ * Date : 3th April 2013
+ */
+
+use dataverse insertIssue;
+
+for $d in dataset myDataset
+order by $d.id
+return $d;
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/list/query-issue428/query-issue428.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/list/query-issue428/query-issue428.1.ddl.aql
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/list/query-issue428/query-issue428.1.ddl.aql
diff --git a/asterix-app/src/test/resources/runtimets/queries/list/query-issue428/query-issue428.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/list/query-issue428/query-issue428.2.update.aql
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/list/query-issue428/query-issue428.2.update.aql
diff --git a/asterix-app/src/test/resources/runtimets/queries/list/query-issue428/query-issue428.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/list/query-issue428/query-issue428.3.query.aql
new file mode 100644
index 0000000..aa6bf88
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/list/query-issue428/query-issue428.3.query.aql
@@ -0,0 +1,10 @@
+/*
+ * Description : This test case is to verify the fix for issue400
+ : https://code.google.com/p/asterixdb/issues/detail?id=400
+ * Expected Res : Success
+ * Date : 8th May 2013
+ */
+
+for $a in [[1,2],[3,4,5]]
+for $b in [[6,7],[8,9,10]]
+return some $a1 in $a,$b1 in $b satisfies $a1 < $b1
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue377/query-issue377.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue377/query-issue377.1.ddl.aql
new file mode 100644
index 0000000..4722e4f
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue377/query-issue377.1.ddl.aql
@@ -0,0 +1,39 @@
+/*
+ * Description : This test case is to verify the fix for issue377
+ : https://code.google.com/p/asterixdb/issues/detail?id=377
+ * Expected Res : Success
+ * Date : 11th May 2013
+ */
+
+drop dataverse TinySocial if exists;
+create dataverse TinySocial;
+use dataverse TinySocial;
+
+create type TwitterUserType as open {
+ screen-name: string
+}
+
+create type TweetMessageType as open {
+ tweetid: string
+}
+
+create type FacebookUserType as open {
+ id: int32
+}
+
+create type FacebookMessageType as open {
+ message-id: int32
+}
+
+create dataset FacebookUsers(FacebookUserType)
+primary key id;
+
+create dataset FacebookMessages(FacebookMessageType)
+primary key message-id;
+
+create dataset TwitterUsers(TwitterUserType)
+primary key screen-name;
+
+create dataset TweetMessages(TweetMessageType)
+primary key tweetid
+hints(cardinality=100);
diff --git a/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue377/query-issue377.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue377/query-issue377.2.update.aql
new file mode 100644
index 0000000..1d6c5ba
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue377/query-issue377.2.update.aql
@@ -0,0 +1,15 @@
+/*
+ * Description : This test case is to verify the fix for issue377
+ : https://code.google.com/p/asterixdb/issues/detail?id=377
+ * Expected Res : Success
+ * Date : 11th May 2013
+ */
+
+use dataverse TinySocial;
+
+load dataset FacebookUsers using localfs
+(("path"="nc1://data/fbu-dml-insert-shuffled.adm"),("format"="adm"));
+
+load dataset TweetMessages using localfs
+(("path"="nc1://data/twitter/tw_messages.adm"),("format"="adm"));
+
diff --git a/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue377/query-issue377.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue377/query-issue377.3.query.aql
new file mode 100644
index 0000000..81d6cf6
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue377/query-issue377.3.query.aql
@@ -0,0 +1,24 @@
+/*
+ * Description : This test case is to verify the fix for issue377
+ : https://code.google.com/p/asterixdb/issues/detail?id=377
+ * Expected Res : Success
+ * Date : 11th May 2013
+ */
+
+use dataverse TinySocial;
+
+set simfunction "edit-distance";
+set simthreshold "3";
+
+for $fbu in dataset FacebookUsers
+return {
+ "id": $fbu.id,
+ "name": $fbu.name,
+ "similar-users": for $t in dataset TweetMessages
+ let $tu := $t.user
+ where $tu.name ~= $fbu.name
+ return {
+ "twitter-screenname": $tu.screen-name,
+ "twitter-name": $tu.name
+ }
+};
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue410/query-issue410.1.ddl.aql b/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue410/query-issue410.1.ddl.aql
new file mode 100644
index 0000000..0da0aa5
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue410/query-issue410.1.ddl.aql
@@ -0,0 +1,17 @@
+/*
+ * Description : This test case is to verify the fix for issue410
+ : https://code.google.com/p/asterixdb/issues/detail?id=410
+ * Expected Res : Fail
+ * Date : 13th May 2013
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use dataverse test;
+
+create type Emp as open {
+id:int32,
+name:string
+}
+
+create dataset Employee(Emp) primary key id;
diff --git a/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue410/query-issue410.2.update.aql b/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue410/query-issue410.2.update.aql
new file mode 100644
index 0000000..796c5a4
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue410/query-issue410.2.update.aql
@@ -0,0 +1,10 @@
+/*
+ * Description : This test case is to verify the fix for issue410
+ : https://code.google.com/p/asterixdb/issues/detail?id=410
+ * Expected Res : Fail
+ * Date : 11th May 2013
+ */
+
+use dataverse test;
+
+insert into dataset Employee({"id":float("59138237473282.3293"), "name": double("0.01")});
diff --git a/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue410/query-issue410.3.query.aql b/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue410/query-issue410.3.query.aql
new file mode 100644
index 0000000..d76caba
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/queries/open-closed/query-issue410/query-issue410.3.query.aql
@@ -0,0 +1,11 @@
+/*
+ * Description : This test case is to verify the fix for issue410
+ : https://code.google.com/p/asterixdb/issues/detail?id=410
+ * Expected Res : Fail
+ * Date : 11th May 2013
+ */
+
+use dataverse test;
+
+for $x in dataset('Employee')
+return $x
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/aggregate/query-issue400/query-issue400.1.adm b/asterix-app/src/test/resources/runtimets/results/aggregate/query-issue400/query-issue400.1.adm
new file mode 100644
index 0000000..d8263ee
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/aggregate/query-issue400/query-issue400.1.adm
@@ -0,0 +1 @@
+2
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/custord/join_q_04/join_q_04.1.adm b/asterix-app/src/test/resources/runtimets/results/custord/join_q_04/join_q_04.1.adm
index d4721dd..51f998a 100644
--- a/asterix-app/src/test/resources/runtimets/results/custord/join_q_04/join_q_04.1.adm
+++ b/asterix-app/src/test/resources/runtimets/results/custord/join_q_04/join_q_04.1.adm
@@ -1,3 +1,5 @@
-{ "cust_name": "Jodi Rotruck", "orderedlist": [ 1000, 66, 775 ], "unorderedlist": {{ 1000, 66, 775 }}, "ol_item1": 1000, "ol_item2": 66, "ol_item5": null, "ul_item1": 1000 }
-{ "cust_name": "Jodi Alex", "orderedlist": [ 10, 48, 5 ], "unorderedlist": {{ 10, 48, 5 }}, "ol_item1": 10, "ol_item2": 48, "ol_item5": null, "ul_item1": 10 }
-{ "cust_name": "Jodi Rotruck", "orderedlist": [ 10, 66, 775 ], "unorderedlist": {{ 10, 66, 775 }}, "ol_item1": 10, "ol_item2": 66, "ol_item5": null, "ul_item1": 10 }
+{ "cust_name": "Jodi Alex", "order_ids": [ 10 ] }
+{ "cust_name": "Jodi Rotruck", "order_ids": [ 10, 1000 ] }
+{ "cust_name": "Mary Carey", "order_ids": [ ] }
+{ "cust_name": "Mike Carey", "order_ids": [ ] }
+{ "cust_name": "Mike ley", "order_ids": [ ] }
diff --git a/asterix-app/src/test/resources/runtimets/results/dml/query-issue382/query-issue382.1.adm b/asterix-app/src/test/resources/runtimets/results/dml/query-issue382/query-issue382.1.adm
new file mode 100644
index 0000000..9a03714
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/dml/query-issue382/query-issue382.1.adm
@@ -0,0 +1 @@
+10
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/dml/query-issue433/query-issue433.1.adm b/asterix-app/src/test/resources/runtimets/results/dml/query-issue433/query-issue433.1.adm
new file mode 100644
index 0000000..ce89449
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/dml/query-issue433/query-issue433.1.adm
@@ -0,0 +1,2 @@
+{ "id": 1, "name": "u1", "sub": [ { "n": "se1", "e": 100 } ] }
+{ "id": 2, "name": "u2", "sub": [ { "n": "se2", "e": 200 } ] }
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/results/list/query-issue428/query-issue428.1.adm b/asterix-app/src/test/resources/runtimets/results/list/query-issue428/query-issue428.1.adm
new file mode 100644
index 0000000..1140ff5
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/list/query-issue428/query-issue428.1.adm
@@ -0,0 +1,4 @@
+true
+true
+true
+true
diff --git a/asterix-app/src/test/resources/runtimets/results/open-closed/query-issue377/query-issue377.1.adm b/asterix-app/src/test/resources/runtimets/results/open-closed/query-issue377/query-issue377.1.adm
new file mode 100644
index 0000000..c3bb80f
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/open-closed/query-issue377/query-issue377.1.adm
@@ -0,0 +1,29 @@
+{ "id": 9142198, "similar-users": [ ], "name": "SherryFea" }
+{ "id": 9313492, "similar-users": [ ], "name": "TeraWolfe" }
+{ "id": 9478720, "similar-users": [ ], "name": "AngeliaKettlewell" }
+{ "id": 10001080, "similar-users": [ ], "name": "GarrettBode" }
+{ "id": 10179538, "similar-users": [ ], "name": "OrlandoBaxter" }
+{ "id": 10307032, "similar-users": [ ], "name": "QuentinSauter" }
+{ "id": 10394488, "similar-users": [ ], "name": "OswaldRay" }
+{ "id": 10423588, "similar-users": [ ], "name": "ShirleneRuch" }
+{ "id": 10495420, "similar-users": [ ], "name": "WendyMcloskey" }
+{ "id": 11307946, "similar-users": [ ], "name": "HelgaStough" }
+{ "id": 11447332, "similar-users": [ ], "name": "SherisseMaugham" }
+{ "id": 11570326, "similar-users": [ ], "name": "LindenFilby" }
+{ "id": 11951098, "similar-users": [ ], "name": "TeraByers" }
+{ "id": 11954992, "similar-users": [ ], "name": "CaitlinLangston" }
+{ "id": 9510451, "similar-users": [ ], "name": "ChuckFinck" }
+{ "id": 9594523, "similar-users": [ ], "name": "TamWillcox" }
+{ "id": 9629395, "similar-users": [ ], "name": "JuliusWire" }
+{ "id": 9988417, "similar-users": [ ], "name": "ColineLane" }
+{ "id": 10272571, "similar-users": [ ], "name": "JarrettGoldvogel" }
+{ "id": 10361965, "similar-users": [ ], "name": "ArlenFlick" }
+{ "id": 10498285, "similar-users": [ ], "name": "KileyBridger" }
+{ "id": 10733617, "similar-users": [ ], "name": "LeonardoKight" }
+{ "id": 10874791, "similar-users": [ ], "name": "HaydeeGarratt" }
+{ "id": 10957867, "similar-users": [ ], "name": "ZachOppenheimer" }
+{ "id": 11061631, "similar-users": [ ], "name": "MaxeneKellogg" }
+{ "id": 11068231, "similar-users": [ ], "name": "DinahSwink" }
+{ "id": 11140213, "similar-users": [ ], "name": "MontgomeryWhittier" }
+{ "id": 11381089, "similar-users": [ ], "name": "EarleneAmmons" }
+{ "id": 11675221, "similar-users": [ ], "name": "CalantheGearhart" }
diff --git a/asterix-app/src/test/resources/runtimets/results/open-closed/query-issue410/query-issue410.1.adm b/asterix-app/src/test/resources/runtimets/results/open-closed/query-issue410/query-issue410.1.adm
new file mode 100644
index 0000000..ee2dfa4
--- /dev/null
+++ b/asterix-app/src/test/resources/runtimets/results/open-closed/query-issue410/query-issue410.1.adm
@@ -0,0 +1 @@
+{"id":0, "name": ""}
\ No newline at end of file
diff --git a/asterix-app/src/test/resources/runtimets/testsuite.xml b/asterix-app/src/test/resources/runtimets/testsuite.xml
index 6759a6f..41d3922 100644
--- a/asterix-app/src/test/resources/runtimets/testsuite.xml
+++ b/asterix-app/src/test/resources/runtimets/testsuite.xml
@@ -1,6 +1,11 @@
<test-suite xmlns="urn:xml.testframework.asterix.ics.uci.edu" ResultOffsetPath="results" QueryOffsetPath="queries" QueryFileExtension=".aql">
<test-group name="aggregate">
<test-case FilePath="aggregate">
+ <compilation-unit name="query-issue400">
+ <output-dir compare="Text">query-issue400</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="aggregate">
<compilation-unit name="issue395">
<output-dir compare="Text">issue395</output-dir>
</compilation-unit>
@@ -704,13 +709,11 @@
<output-dir compare="Text">join_q_03</output-dir>
</compilation-unit>
</test-case>
- <!--
<test-case FilePath="custord">
<compilation-unit name="join_q_04">
<output-dir compare="Text">join_q_04</output-dir>
</compilation-unit>
</test-case>
- -->
<test-case FilePath="custord">
<compilation-unit name="load-test">
<output-dir compare="Text">load-test</output-dir>
@@ -768,6 +771,16 @@
</test-group>
<test-group name="dml">
<test-case FilePath="dml">
+ <compilation-unit name="query-issue382">
+ <output-dir compare="Text">query-issue382</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
+ <compilation-unit name="query-issue433">
+ <output-dir compare="Text">query-issue433</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
<compilation-unit name="query-issue288">
<output-dir compare="Text">query-issue288</output-dir>
</compilation-unit>
@@ -2125,6 +2138,11 @@
<output-dir compare="Text">unordered-list-constructor_03</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="list">
+ <compilation-unit name="query-issue428">
+ <output-dir compare="Text">query-issue428</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="misc">
<test-case FilePath="misc">
@@ -2719,6 +2737,17 @@
<output-dir compare="Text">query-issue196</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue377">
+ <output-dir compare="Text">query-issue377</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="open-closed">
+ <compilation-unit name="query-issue410">
+ <output-dir compare="Text">query-issue40</output-dir>
+ <expected-error>edu.uci.ics.asterix.common.exceptions.AsterixException</expected-error>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="quantifiers">
<test-case FilePath="quantifiers">
diff --git a/asterix-aql/src/main/javacc/AQL.jj b/asterix-aql/src/main/javacc/AQL.jj
index 68e1497..dc01648 100644
--- a/asterix-aql/src/main/javacc/AQL.jj
+++ b/asterix-aql/src/main/javacc/AQL.jj
@@ -74,33 +74,41 @@
// data generator hints
private static final String DGEN_HINT = "dgen";
+
+ private static class IndexParams {
+ public IndexType type;
+ public int gramLength;
+
+ public IndexParams(IndexType type, int gramLength) {
+ this.type = type;
+ this.gramLength = gramLength;
+ }
+ };
private static String getHint(Token t) {
- if (t.specialToken == null) {
- return null;
- }
- String s = t.specialToken.image;
- int n = s.length();
- if (n < 2) {
- return null;
- }
- return s.substring(1).trim();
+ if (t.specialToken == null) {
+ return null;
+ }
+ String s = t.specialToken.image;
+ int n = s.length();
+ if (n < 2) {
+ return null;
+ }
+ return s.substring(1).trim();
}
public AQLParser(String s){
- this(new StringReader(s));
- super.setInput(s);
- }
-
- public static void main(String args[]) throws ParseException, TokenMgrError, IOException, FileNotFoundException, AsterixException {
- File file = new File(args[0]);
- Reader fis = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
- AQLParser parser = new AQLParser(fis);
- List<Statement> st = parser.Statement();
- //st.accept(new AQLPrintVisitor(), 0);
- }
+ this(new StringReader(s));
+ super.setInput(s);
+ }
-
+ public static void main(String args[]) throws ParseException, TokenMgrError, IOException, FileNotFoundException, AsterixException {
+ File file = new File(args[0]);
+ Reader fis = new BufferedReader(new InputStreamReader(new FileInputStream(file), "UTF-8"));
+ AQLParser parser = new AQLParser(fis);
+ List<Statement> st = parser.Statement();
+ //st.accept(new AQLPrintVisitor(), 0);
+ }
}
PARSER_END(AQLParser)
@@ -110,235 +118,563 @@
{
scopeStack.push(RootScopeFactory.createRootScope(this));
List<Statement> decls = new ArrayList<Statement>();
- Query query=null;
+ Statement stmt = null;
}
{
- (
- (
- (
- "use"
- {
- decls.add(DataverseDeclaration());
- }
- | "declare" "function" {
- decls.add(FunctionDeclaration());
- }
- | "create" (
- {
- String hint = getHint(token);
- boolean dgen = false;
- if (hint != null && hint.startsWith(DGEN_HINT)) {
- dgen = true;
- }
- }
- "type"
- {
- decls.add(TypeDeclaration(dgen, hint));
- }
- | "nodegroup"
- {
- decls.add(NodegroupDeclaration());
- }
- | "external" <DATASET>
- {
- decls.add(DatasetDeclaration(DatasetType.EXTERNAL));
- }
- | "feed" <DATASET>
- {
- decls.add(DatasetDeclaration(DatasetType.FEED));
- }
- | <DATASET>
- {
- decls.add(DatasetDeclaration(DatasetType.INTERNAL));
- }
- | "index"
- {
- decls.add(CreateIndexStatement());
- }
- | "dataverse"
- {
- decls.add(CreateDataverseStatement());
- }
- | "function"
- {
- decls.add(FunctionCreation());
- }
- )
- | "load" {
- decls.add(LoadStatement());
- }
-
- | "drop"
- (
- <DATASET>
- {
- decls.add(DropStatement());
- }
- | "index"
- {
- decls.add(IndexDropStatement());
- }
- | "nodegroup"
- {
- decls.add(NodeGroupDropStatement());
- }
- | "type"
- {
- decls.add(TypeDropStatement());
- }
- | "dataverse"
- {
- decls.add(DataverseDropStatement());
- }
- | "function"
- {
- decls.add(FunctionDropStatement());
- }
- )
- | "write" {
- decls.add(WriteStatement());
- }
- | "set" {
- decls.add(SetStatement());
- }
- | "insert" {
- decls.add(InsertStatement());
- }
- | "delete" {
- decls.add(DeleteStatement());
- }
- | "update" {
- decls.add(UpdateStatement());
- }
- | "begin" "feed"
- {
- Pair<Identifier,Identifier> nameComponents = getDotSeparatedPair();
- decls.add(new BeginFeedStatement(nameComponents.first, nameComponents.second, getVarCounter()));
- } ";"
-
- | "suspend" "feed"
- {
- decls.add(ControlFeedDeclaration(ControlFeedStatement.OperationType.SUSPEND));
- } ";"
- | "resume" "feed" {
- decls.add(ControlFeedDeclaration(ControlFeedStatement.OperationType.RESUME));
- } ";"
- | "end" "feed" {
- decls.add(ControlFeedDeclaration(ControlFeedStatement.OperationType.END));
- } ";"
- | "alter" "feed" {
- decls.add(AlterFeedDeclaration());
- } ";"
-
- | (query = Query()) {
- decls.add(query);
- }
- )*
- // (query = Query())?
- )
-
- <EOF>
- )
+ ( stmt = SingleStatement() (";") ?
{
- return decls;
+ decls.add(stmt);
+ }
+ )*
+ <EOF>
+ {
+ return decls;
+ }
+}
+
+Statement SingleStatement() throws ParseException:
+{
+ Statement stmt = null;
+}
+{
+ (
+ stmt = DataverseDeclaration()
+ | stmt = FunctionDeclaration()
+ | stmt = CreateStatement()
+ | stmt = LoadStatement()
+ | stmt = DropStatement()
+ | stmt = WriteStatement()
+ | stmt = SetStatement()
+ | stmt = InsertStatement()
+ | stmt = DeleteStatement()
+ | stmt = UpdateStatement()
+ | stmt = FeedStatement()
+ | stmt = Query()
+ )
+ {
+ return stmt;
+ }
+}
+
+DataverseDecl DataverseDeclaration() throws ParseException:
+{
+ String dvName = null;
+}
+{
+ "use" "dataverse" dvName = Identifier()
+ {
+ defaultDataverse = dvName;
+ return new DataverseDecl(new Identifier(dvName));
+ }
+}
+
+Statement CreateStatement() throws ParseException:
+{
+ String hint = null;
+ boolean dgen = false;
+ Statement stmt = null;
+}
+{
+ "create"
+ (
+ {
+ hint = getHint(token);
+ if (hint != null && hint.startsWith(DGEN_HINT)) {
+ dgen = true;
+ }
+ }
+ stmt = TypeSpecification(hint, dgen)
+ | stmt = NodegroupSpecification()
+ | stmt = DatasetSpecification()
+ | stmt = IndexSpecification()
+ | stmt = DataverseSpecification()
+ | stmt = FunctionSpecification()
+ )
+ {
+ return stmt;
+ }
+}
+
+TypeDecl TypeSpecification(String hint, boolean dgen) throws ParseException:
+{
+ Pair<Identifier,Identifier> nameComponents = null;
+ boolean ifNotExists = false;
+ TypeExpression typeExpr = null;
+}
+{
+ "type" nameComponents = FunctionOrTypeName() ifNotExists = IfNotExists()
+ "as" typeExpr = TypeExpr()
+ {
+ long numValues = -1;
+ String filename = null;
+ if (dgen) {
+ String splits[] = hint.split(" +");
+ if (splits.length != 3) {
+ throw new ParseException("Expecting /*+ dgen <filename> <numberOfItems> */");
+ }
+ filename = splits[1];
+ numValues = Long.parseLong(splits[2]);
+ }
+ TypeDataGen tddg = new TypeDataGen(dgen, filename, numValues);
+ return new TypeDecl(nameComponents.first, nameComponents.second, typeExpr, tddg, ifNotExists);
+ }
+}
+
+
+NodegroupDecl NodegroupSpecification() throws ParseException:
+{
+ String name = null;
+ String tmp = null;
+ boolean ifNotExists = false;
+ List<Identifier>ncNames = null;
+}
+{
+ "nodegroup" name = Identifier()
+ ifNotExists = IfNotExists() "on" tmp = Identifier()
+ {
+ ncNames = new ArrayList<Identifier>();
+ ncNames.add(new Identifier(tmp));
+ }
+ ( "," tmp = Identifier()
+ {
+ ncNames.add(new Identifier(tmp));
+ }
+ )*
+ {
+ return new NodegroupDecl(new Identifier(name), ncNames, ifNotExists);
+ }
+}
+
+DatasetDecl DatasetSpecification() throws ParseException:
+{
+ Pair<Identifier,Identifier> nameComponents = null;
+ boolean ifNotExists = false;
+ String typeName = null;
+ String adapterName = null;
+ Map<String,String> properties = null;
+ FunctionSignature appliedFunction = null;
+ List<String> primaryKeyFields = null;
+ String nodeGroupName = null;
+ Map<String,String> hints = new HashMap<String,String>();
+ DatasetDecl dsetDecl = null;
+}
+{
+ (
+ "external" <DATASET> nameComponents = QualifiedName()
+ <LEFTPAREN> typeName = Identifier() <RIGHTPAREN>
+ ifNotExists = IfNotExists()
+ "using" adapterName = AdapterName() properties = Configuration()
+ ( "hints" hints = Properties() )?
+ {
+ ExternalDetailsDecl edd = new ExternalDetailsDecl();
+ edd.setAdapter(adapterName);
+ edd.setProperties(properties);
+ dsetDecl = new DatasetDecl(nameComponents.first,
+ nameComponents.second,
+ new Identifier(typeName),
+ hints,
+ DatasetType.EXTERNAL,
+ edd,
+ ifNotExists);
+ }
+
+ | "feed" <DATASET> nameComponents = QualifiedName()
+ <LEFTPAREN> typeName = Identifier() <RIGHTPAREN>
+ ifNotExists = IfNotExists()
+ "using" adapterName = AdapterName() properties = Configuration()
+ (appliedFunction = ApplyFunction())? primaryKeyFields = PrimaryKey()
+ ( "on" nodeGroupName = Identifier() )?
+ ( "hints" hints = Properties() )?
+ {
+ FeedDetailsDecl fdd = new FeedDetailsDecl(adapterName,
+ properties,
+ appliedFunction,
+ nodeGroupName != null
+ ? new Identifier(nodeGroupName)
+ : null,
+ primaryKeyFields);
+ dsetDecl = new DatasetDecl(nameComponents.first,
+ nameComponents.second,
+ new Identifier(typeName),
+ hints,
+ DatasetType.FEED,
+ fdd,
+ ifNotExists);
+ }
+ | <DATASET> nameComponents = QualifiedName()
+ <LEFTPAREN> typeName = Identifier() <RIGHTPAREN>
+ ifNotExists = IfNotExists()
+ primaryKeyFields = PrimaryKey() ("on" nodeGroupName = Identifier() )?
+ ( "hints" hints = Properties() )?
+ {
+ InternalDetailsDecl idd = new InternalDetailsDecl(nodeGroupName != null
+ ? new Identifier(nodeGroupName)
+ : null,
+ primaryKeyFields);
+ dsetDecl = new DatasetDecl(nameComponents.first,
+ nameComponents.second,
+ new Identifier(typeName),
+ hints,
+ DatasetType.INTERNAL,
+ idd,
+ ifNotExists);
+ }
+ )
+ {
+ return dsetDecl;
+ }
+}
+
+CreateIndexStatement IndexSpecification() throws ParseException:
+{
+ CreateIndexStatement cis = new CreateIndexStatement();
+ String indexName = null;
+ String fieldExpr = null;
+ boolean ifNotExists = false;
+ Pair<Identifier,Identifier> nameComponents = null;
+ IndexParams indexType = null;
+}
+{
+ "index" indexName = Identifier()
+ ifNotExists = IfNotExists()
+ "on" nameComponents = QualifiedName()
+ <LEFTPAREN> ( fieldExpr = Identifier()
+ {
+ cis.addFieldExpr(fieldExpr);
+ }
+ ) ("," fieldExpr = Identifier()
+ {
+ cis.addFieldExpr(fieldExpr);
+ }
+ )* <RIGHTPAREN> ( "type" indexType = IndexType() )?
+ {
+ cis.setIndexName(new Identifier(indexName));
+ cis.setIfNotExists(ifNotExists);
+ cis.setDataverseName(nameComponents.first);
+ cis.setDatasetName(nameComponents.second);
+ if (indexType != null) {
+ cis.setIndexType(indexType.type);
+ cis.setGramLength(indexType.gramLength);
+ }
+ return cis;
+ }
+}
+
+IndexParams IndexType() throws ParseException:
+{
+ IndexType type = null;
+ int gramLength = 0;
+}
+{
+ ("btree"
+ {
+ type = IndexType.BTREE;
+ }
+ | "rtree"
+ {
+ type = IndexType.RTREE;
+ }
+ | "keyword"
+ {
+ type = IndexType.WORD_INVIX;
+ }
+ | "fuzzy keyword"
+ {
+ type = IndexType.FUZZY_WORD_INVIX;
+ }
+ | "ngram" <LEFTPAREN> <INTEGER_LITERAL>
+ {
+ type = IndexType.NGRAM_INVIX;
+ gramLength = Integer.valueOf(token.image);
+ }
+ <RIGHTPAREN>
+ | "fuzzy ngram" <LEFTPAREN> <INTEGER_LITERAL>
+ {
+ type = IndexType.FUZZY_NGRAM_INVIX;
+ gramLength = Integer.valueOf(token.image);
+ }
+ <RIGHTPAREN>)
+ {
+ return new IndexParams(type, gramLength);
+ }
+}
+
+CreateDataverseStatement DataverseSpecification() throws ParseException :
+{
+ String dvName = null;
+ boolean ifNotExists = false;
+ String format = null;
+}
+{
+ "dataverse" dvName = Identifier()
+ ifNotExists = IfNotExists()
+ ( "with format" format = StringLiteral() )?
+ {
+ return new CreateDataverseStatement(new Identifier(dvName), format, ifNotExists);
+ }
+}
+
+CreateFunctionStatement FunctionSpecification() throws ParseException:
+{
+ FunctionSignature signature;
+ boolean ifNotExists = false;
+ List<VarIdentifier> paramList = new ArrayList<VarIdentifier>();
+ String functionBody;
+ VarIdentifier var = null;
+ Expression functionBodyExpr;
+ Token beginPos;
+ Token endPos;
+ Pair<Identifier,Identifier> nameComponents=null;
+
+ createNewScope();
+}
+{
+ "function" nameComponents = FunctionOrTypeName()
+ ifNotExists = IfNotExists()
+ <LEFTPAREN> (<VARIABLE>
+ {
+ var = new VarIdentifier();
+ var.setValue(token.image);
+ paramList.add(var);
+ getCurrentScope().addNewVarSymbolToScope(var);
+ }
+ ("," <VARIABLE>
+ {
+ var = new VarIdentifier();
+ var.setValue(token.image);
+ paramList.add(var);
+ getCurrentScope().addNewVarSymbolToScope(var);
+ }
+ )*)? <RIGHTPAREN> "{"
+ {
+ beginPos = token;
+ }
+ functionBodyExpr = Expression() "}"
+ {
+ endPos = token;
+ functionBody = extractFragment(beginPos.beginLine, beginPos.beginColumn, endPos.beginLine, endPos.beginColumn);
+ String dataverse = nameComponents.first.getValue();
+ String functionName = nameComponents.second.getValue();
+ signature = new FunctionSignature(dataverse, functionName, paramList.size());
+ getCurrentScope().addFunctionDescriptor(signature, false);
+ return new CreateFunctionStatement(signature, paramList, functionBody, ifNotExists);
+ }
+}
+
+boolean IfNotExists() throws ParseException:
+{
+}
+{
+ ( "if not exists"
+ {
+ return true;
+ }
+ )?
+ {
+ return false;
+ }
+}
+
+FunctionSignature ApplyFunction() throws ParseException:
+{
+ FunctionSignature funcSig = null;
+}
+{
+ "apply" "function" funcSig = FunctionSignature()
+ {
+ return funcSig;
+ }
+}
+
+FunctionSignature FunctionSignature() throws ParseException:
+{
+ Pair<Identifier,Identifier> pairId = null;
+ int arity = 0;
+}
+{
+ pairId = FunctionOrTypeName() "@" <INTEGER_LITERAL>
+ {
+ arity = new Integer(token.image);
+ if (arity < 0 && arity != FunctionIdentifier.VARARGS) {
+ throw new ParseException(" invalid arity:" + arity);
+ }
+
+ String dataverse = pairId.first.getValue();
+ String functionName = pairId.second.getValue();
+ return new FunctionSignature(dataverse, functionName, arity);
+ }
+}
+
+List<String> PrimaryKey() throws ParseException:
+{
+ String tmp = null;
+ List<String> primaryKeyFields = new ArrayList<String>();
+}
+{
+ "primary" "key" tmp = Identifier()
+ {
+ primaryKeyFields.add(tmp);
+ }
+ ( "," tmp = Identifier()
+ {
+ primaryKeyFields.add(tmp);
+ }
+ )*
+ {
+ return primaryKeyFields;
+ }
+}
+
+Statement DropStatement() throws ParseException:
+{
+ String id = null;
+ Pair<Identifier,Identifier> pairId = null;
+ Triple<Identifier,Identifier,Identifier> tripleId = null;
+ FunctionSignature funcSig = null;
+ boolean ifExists = false;
+ Statement stmt = null;
+}
+{
+ "drop"
+ (
+ <DATASET> pairId = QualifiedName() ifExists = IfExists()
+ {
+ stmt = new DropStatement(pairId.first, pairId.second, ifExists);
+ }
+ | "index" tripleId = DoubleQualifiedName() ifExists = IfExists()
+ {
+ stmt = new IndexDropStatement(tripleId.first, tripleId.second, tripleId.third, ifExists);
+ }
+ | "nodegroup" id = Identifier() ifExists = IfExists()
+ {
+ stmt = new NodeGroupDropStatement(new Identifier(id), ifExists);
+ }
+ | "type" pairId = FunctionOrTypeName() ifExists = IfExists()
+ {
+ stmt = new TypeDropStatement(pairId.first, pairId.second, ifExists);
+ }
+ | "dataverse" id = Identifier() ifExists = IfExists()
+ {
+ stmt = new DataverseDropStatement(new Identifier(id), ifExists);
+ }
+ | "function" funcSig = FunctionSignature() ifExists = IfExists()
+ {
+ stmt = new FunctionDropStatement(funcSig, ifExists);
+ }
+ )
+ {
+ return stmt;
+ }
+}
+
+boolean IfExists() throws ParseException :
+{
+}
+{
+ ( "if" "exists"
+ {
+ return true;
+ }
+ )?
+ {
+ return false;
}
}
InsertStatement InsertStatement() throws ParseException:
{
- Identifier dataverseName;
- Identifier datasetName;
- Pair<Identifier,Identifier> nameComponents = null;
- Query query;
+ Pair<Identifier,Identifier> nameComponents = null;
+ Query query;
}
{
- "into" <DATASET>
-
- {
- nameComponents = getDotSeparatedPair();
- dataverseName = nameComponents.first;
- datasetName = nameComponents.second;
- }
-
- query = Query() (";")?
- {return new InsertStatement(dataverseName, datasetName, query, getVarCounter());}
+ "insert" "into" <DATASET> nameComponents = QualifiedName() query = Query()
+ {
+ return new InsertStatement(nameComponents.first, nameComponents.second, query, getVarCounter());
+ }
}
DeleteStatement DeleteStatement() throws ParseException:
{
- VariableExpr var = null;
- Identifier dataverseName;
- Identifier datasetName = null;
- Expression condition = null;
- Pair<Identifier, Identifier> nameComponents;
+ VariableExpr var = null;
+ Expression condition = null;
+ Pair<Identifier, Identifier> nameComponents;
}
{
- var = Variable() { getCurrentScope().addNewVarSymbolToScope(var.getVar()); }
- "from"
- <DATASET>
- {
- nameComponents = getDotSeparatedPair();
- }
- ("where" condition = Expression())? (";")?
- {return new DeleteStatement(var, nameComponents.first, nameComponents.second, condition, getVarCounter()); }
+ "delete" var = Variable()
+ {
+ getCurrentScope().addNewVarSymbolToScope(var.getVar());
+ }
+ "from" <DATASET> nameComponents = QualifiedName()
+ ("where" condition = Expression())?
+ {
+ return new DeleteStatement(var, nameComponents.first, nameComponents.second, condition, getVarCounter());
+ }
}
UpdateStatement UpdateStatement() throws ParseException:
{
- VariableExpr vars;
- Expression target;
- Expression condition;
- UpdateClause uc;
- List<UpdateClause> ucs = new ArrayList<UpdateClause>();
+ VariableExpr vars;
+ Expression target;
+ Expression condition;
+ UpdateClause uc;
+ List<UpdateClause> ucs = new ArrayList<UpdateClause>();
}
{
- vars = Variable() "in" target = Expression()
- "where" condition = Expression()
- <LEFTPAREN> (uc=UpdateClause() {ucs.add(uc); } ("," uc=UpdateClause() {ucs.add(uc); } )*) <RIGHTPAREN> ";"
- {return new UpdateStatement(vars, target, condition, ucs);}
+ "update" vars = Variable() "in" target = Expression()
+ "where" condition = Expression()
+ <LEFTPAREN> (uc = UpdateClause()
+ {
+ ucs.add(uc);
+ }
+ ("," uc = UpdateClause()
+ {
+ ucs.add(uc);
+ }
+ )*) <RIGHTPAREN>
+ {
+ return new UpdateStatement(vars, target, condition, ucs);
+ }
}
-
-
UpdateClause UpdateClause() throws ParseException:
{
- Expression target = null;
- Expression value = null ;
- InsertStatement is = null;
- DeleteStatement ds = null;
- UpdateStatement us = null;
- Expression condition = null;
- UpdateClause ifbranch = null;
- UpdateClause elsebranch = null;
+ Expression target = null;
+ Expression value = null ;
+ InsertStatement is = null;
+ DeleteStatement ds = null;
+ UpdateStatement us = null;
+ Expression condition = null;
+ UpdateClause ifbranch = null;
+ UpdateClause elsebranch = null;
}
{
"set" target = Expression() ":=" value = Expression()
- | "insert" is = InsertStatement()
- | "delete" ds = DeleteStatement()
- | "update" us = UpdateStatement()
- | "if" <LEFTPAREN> condition = Expression() <RIGHTPAREN> "then" ifbranch = UpdateClause() [LOOKAHEAD(1) "else" elsebranch = UpdateClause()]
- {return new UpdateClause(target, value, is, ds, us, condition, ifbranch, elsebranch);}
+ | is = InsertStatement()
+ | ds = DeleteStatement()
+ | us = UpdateStatement()
+ | "if" <LEFTPAREN> condition = Expression() <RIGHTPAREN>
+ "then" ifbranch = UpdateClause()
+ [LOOKAHEAD(1) "else" elsebranch = UpdateClause()]
+ {
+ return new UpdateClause(target, value, is, ds, us, condition, ifbranch, elsebranch);
+ }
}
-
Statement SetStatement() throws ParseException:
{
String pn = null;
- Statement stmt = null;
+ String pv = null;
}
{
- <IDENTIFIER> { pn = token.image; }
- <STRING_LITERAL>
- { String pv = removeQuotesAndEscapes(token.image); }
- ";"
- {
- return new SetStatement(pn, pv);
- }
+ "set" pn = Identifier() pv = StringLiteral()
+ {
+ return new SetStatement(pn, pv);
+ }
}
Statement WriteStatement() throws ParseException:
{
- Identifier nodeName = null;
+ String nodeName = null;
String fileName = null;
Statement stmt = null;
Query query;
@@ -346,288 +682,27 @@
Pair<Identifier,Identifier> nameComponents = null;
}
{
- (( "output" "to"
- <IDENTIFIER> { nodeName = new Identifier(token.image); }
- ":" <STRING_LITERAL> { fileName = removeQuotesAndEscapes(token.image); }
- ( "using" <STRING_LITERAL> { writerClass = removeQuotesAndEscapes(token.image); } )?
- {
- stmt = new WriteStatement(nodeName, fileName, writerClass);
- } )
- |
- ( "into"
- <DATASET>
-
- {
- nameComponents = getDotSeparatedPair();
+ "write" ((
+ "output" "to" nodeName = Identifier() ":" fileName = StringLiteral()
+ ( "using" writerClass = StringLiteral() )?
+ {
+ stmt = new WriteStatement(new Identifier(nodeName), fileName, writerClass);
}
-
- <LEFTPAREN> query = Query() <RIGHTPAREN>
- {
+ ) | (
+ "into" <DATASET>
+ {
+ nameComponents = QualifiedName();
+ }
+ <LEFTPAREN> query = Query() <RIGHTPAREN>
+ {
stmt = new WriteFromQueryResultStatement(nameComponents.first, nameComponents.second, query, getVarCounter());
- } ))
-
- ";"
+ }
+ ))
{
return stmt;
}
}
-CreateIndexStatement CreateIndexStatement() throws ParseException:
-{
- CreateIndexStatement cis = new CreateIndexStatement();
- Pair<Identifier,Identifier> nameComponents = null;
-}
-{
- <IDENTIFIER> { cis.setIndexName(new Identifier(token.image)); }
- (
- "if not exists"
- {
- cis.setIfNotExists(true);
- }
- )?
- "on"
-
- {
- nameComponents = getDotSeparatedPair();
- cis.setDataverseName(nameComponents.first);
- cis.setDatasetName(nameComponents.second);
- }
-
- <LEFTPAREN>
- ( <IDENTIFIER> { cis.addFieldExpr(token.image); } )
- ("," <IDENTIFIER> { cis.addFieldExpr(token.image); })*
- <RIGHTPAREN>
- ("type"
- ("btree" { cis.setIndexType(IndexType.BTREE); }
- | "rtree" { cis.setIndexType(IndexType.RTREE); }
- | "keyword" { cis.setIndexType(IndexType.WORD_INVIX); }
- | "fuzzy keyword" { cis.setIndexType(IndexType.FUZZY_WORD_INVIX); }
- | "ngram"
- <LEFTPAREN>
- (<INTEGER_LITERAL>
- {
- cis.setIndexType(IndexType.NGRAM_INVIX);
- cis.setGramLength(Integer.valueOf(token.image));
- }
- )
- <RIGHTPAREN>
- | "fuzzy ngram"
- <LEFTPAREN>
- (<INTEGER_LITERAL>
- {
- cis.setIndexType(IndexType.FUZZY_NGRAM_INVIX);
- cis.setGramLength(Integer.valueOf(token.image));
- }
- )
- <RIGHTPAREN>
- )
- ";"
- | ";"
- )
- {
- return cis;
- }
-}
-
-DataverseDecl DataverseDeclaration() throws ParseException:
-{
- Identifier dvName = null;
-}
-{
- "dataverse" <IDENTIFIER> { defaultDataverse = token.image;}
- ";"
- {
- return new DataverseDecl(new Identifier(defaultDataverse));
- }
-}
-
-DropStatement DropStatement() throws ParseException :
-{
- Identifier dataverseName = null;
- Identifier datasetName = null;
- boolean ifExists = false;
- Pair<Identifier,Identifier> nameComponents=null;
-}
-{
- {
- nameComponents = getDotSeparatedPair();
- dataverseName = nameComponents.first;
- datasetName = nameComponents.second;
- }
-
-
- (
- "if exists"
- {
- ifExists = true;
- }
- )? ";"
- {
- return new DropStatement(dataverseName, datasetName, ifExists);
- }
-}
-
-IndexDropStatement IndexDropStatement() throws ParseException :
-{
- Identifier dataverseName = null;
- Identifier datasetName = null;
- Identifier indexName = null;
- boolean ifExists = false;
- Triple<Identifier,Identifier,Identifier> nameComponents=null;
-}
-{
-
- {
- nameComponents = getDotSeparatedTriple();
- dataverseName = nameComponents.first;
- datasetName = nameComponents.second;
- indexName = nameComponents.third;
- }
-
- (
- "if exists"
- {
- ifExists = true;
- }
- )? ";"
- {
- return new IndexDropStatement(dataverseName, datasetName, indexName, ifExists);
- }
-}
-
-NodeGroupDropStatement NodeGroupDropStatement() throws ParseException :
-{
- Identifier groupName = null;
- boolean ifExists = false;
-}
-{
- < IDENTIFIER >
- {
- groupName = new Identifier(token.image);
- }
- (
- "if exists"
- {
- ifExists = true;
- }
- )? ";"
- {
- return new NodeGroupDropStatement(groupName, ifExists);
- }
-}
-
-TypeDropStatement TypeDropStatement() throws ParseException :
-{
- Identifier dataverseName = null;
- Identifier typeName = null;
- boolean ifExists = false;
- Pair<Identifier,Identifier> nameComponents;
-}
-{
- {
- nameComponents = getDotSeparatedPair();
- dataverseName = nameComponents.first == null ? new Identifier(defaultDataverse) : nameComponents.first;
- typeName = nameComponents.second;
- }
- (
- "if exists"
- {
- ifExists = true;
- }
- )? ";"
- {
- return new TypeDropStatement(dataverseName, typeName, ifExists);
- }
-}
-
-DataverseDropStatement DataverseDropStatement() throws ParseException :
-{
- Identifier dataverseName = null;
- boolean ifExists = false;
-}
-{
- < IDENTIFIER >
- {
- dataverseName = new Identifier(token.image);
- }
- (
- "if exists"
- {
- ifExists = true;
- }
- )? ";"
- {
- return new DataverseDropStatement(dataverseName, ifExists);
- }
-}
-
-CreateDataverseStatement CreateDataverseStatement() throws ParseException :
-{
- Identifier dvName = null;
- boolean ifNotExists = false;
- String format = null;
-}
-{
- < IDENTIFIER >
- {
- dvName = new Identifier(token.image);
- }
- (
- "if not exists"
- {
- ifNotExists = true;
- }
- )?
- (
- "with format" < STRING_LITERAL >
- {
- format = removeQuotesAndEscapes(token.image);
- }
- )?
- ";"
- {
- return new CreateDataverseStatement(dvName, format, ifNotExists);
- }
-}
-
-
-FunctionDropStatement FunctionDropStatement() throws ParseException :
-{
- String dataverse;
- String functionName;
- int arity=0;
- boolean ifExists = false;
- Pair<Identifier, Identifier> nameComponents=null;
-}
-{
- {
- nameComponents = getDotSeparatedPair();
- dataverse = nameComponents.first != null ? nameComponents.first.getValue() : defaultDataverse;
- functionName = nameComponents.second.getValue();
- }
-
- "@"
- <INTEGER_LITERAL>
- {
- Token t= getToken(0);
- arity = new Integer(t.image);
- if( arity < 0 && arity != FunctionIdentifier.VARARGS){
- throw new ParseException(" invalid arity:" + arity);
- }
- }
-
- (
- "if exists"
- {
- ifExists = true;
- }
- )? ";"
- {
- return new FunctionDropStatement(new FunctionSignature(dataverse, functionName, arity), ifExists);
- }
-}
-
-
LoadFromFileStatement LoadStatement() throws ParseException:
{
Identifier dataverseName = null;
@@ -638,483 +713,138 @@
Pair<Identifier,Identifier> nameComponents = null;
}
{
- <DATASET>
- {
- nameComponents = getDotSeparatedPair();
- dataverseName = nameComponents.first;
- datasetName = nameComponents.second;
- }
-
- "using"
-
+ "load" <DATASET> nameComponents = QualifiedName()
{
- adapterName = getAdapterName();
+ dataverseName = nameComponents.first;
+ datasetName = nameComponents.second;
}
-
+ "using" adapterName = AdapterName() properties = Configuration()
+ ("pre-sorted"
{
- properties = getConfiguration();
- }
-
- ("pre-sorted"
- { alreadySorted = true; }
- )?
-
- ";"
- {
- return new LoadFromFileStatement(dataverseName, datasetName, adapterName, properties, alreadySorted);
- }
-}
-
-
-String getAdapterName() throws ParseException :
-{
- String adapterName = null;
-}
-{
- (
- <IDENTIFIER> {
- adapterName = (new Identifier(token.image)).getValue();;
- }
- |
- <STRING_LITERAL>
- {
- adapterName = removeQuotesAndEscapes(token.image);
- }
- )
- {
- return adapterName;
- }
-}
-
-
-DatasetDecl DatasetDeclaration(DatasetType datasetType) throws ParseException :
-{
- DatasetDecl dd = null;
- Identifier datasetName = null;
- Identifier dataverseName = null;
- Identifier itemDataverseName = null;
- Identifier itemTypeName = null;
- String nameComponentFirst = null;
- String nameComponentSecond = null;
- boolean ifNotExists = false;
- IDatasetDetailsDecl datasetDetails = null;
- Pair<Identifier,Identifier> nameComponents = null;
- Map<String,String> hints = new HashMap<String,String>();
-}
-{
- {
- nameComponents = getDotSeparatedPair();
- dataverseName = nameComponents.first;
- datasetName = nameComponents.second;
- }
-
- (
- "if not exists"
- {
- ifNotExists = true;
+ alreadySorted = true;
}
)?
- (
- < LEFTPAREN > <IDENTIFIER>
- {
- itemTypeName = new Identifier(token.image);
- }
- < RIGHTPAREN >
- )
- {
- if(datasetType == DatasetType.INTERNAL) {
- datasetDetails = InternalDatasetDeclaration();
- }
- else if(datasetType == DatasetType.EXTERNAL) {
- datasetDetails = ExternalDatasetDeclaration();
- }
- else if(datasetType == DatasetType.FEED) {
- datasetDetails = FeedDatasetDeclaration();
- }
- }
-
- (
- "hints"
- {
- initProperties(hints);
- }
- )?
- ";"
-
- {
- dd = new DatasetDecl(dataverseName, datasetName, itemTypeName, hints, datasetType, datasetDetails,ifNotExists);
- return dd;
- }
+ {
+ return new LoadFromFileStatement(dataverseName, datasetName, adapterName, properties, alreadySorted);
+ }
}
-InternalDetailsDecl InternalDatasetDeclaration() throws ParseException :
-{
- InternalDetailsDecl idd = null;
- List<String> primaryKeyFields = new ArrayList<String>();
- Identifier nodeGroupName=null;
-}
-{
- (
- {
- primaryKeyFields = getPrimaryKeyFields();
- }
- )
-
- (
- "on" < IDENTIFIER >
- {
- nodeGroupName = new Identifier(token.image);
- }
- )?
-
- {
- idd = new InternalDetailsDecl(nodeGroupName, primaryKeyFields);
- return idd;
- }
-}
-ExternalDetailsDecl ExternalDatasetDeclaration() throws ParseException :
+String AdapterName() throws ParseException :
{
- ExternalDetailsDecl edd = null;
String adapterName = null;
- Map < String, String > properties;
}
{
- {
- edd = new ExternalDetailsDecl();
- }
-
- "using"
+ adapterName = Identifier()
{
- adapterName = getAdapterName();
+ return adapterName;
}
-
- {
- properties = getConfiguration();
- }
-
- {
- edd = new ExternalDetailsDecl();
- edd.setAdapter(adapterName);
- edd.setProperties(properties);
- }
-
- {
- return edd;
- }
}
-FeedDetailsDecl FeedDatasetDeclaration() throws ParseException :
-{
- FeedDetailsDecl fdd = null;
- String adapterName = null;
- Map < String, String > properties;
- Pair<Identifier,Identifier> nameComponents;
- List<String> primaryKeyFields = new ArrayList<String>();
- Identifier nodeGroupName=null;
- FunctionSignature appliedFunction=null;
- String dataverse;
- String functionName;
- int arity;
-}
-{
- "using"
- {
- adapterName = getAdapterName();
- }
-
- {
- properties = getConfiguration();
- }
-
- ("apply" "function"
- {
- nameComponents = getDotSeparatedPair();
- dataverse = nameComponents.first != null ? nameComponents.first.getValue() : defaultDataverse;
- functionName = nameComponents.second.getValue();
- }
- ("@" <INTEGER_LITERAL>
- {
- arity = Integer.parseInt(token.image);
- }
- )
-
- {
- appliedFunction = new FunctionSignature(dataverse, functionName, arity);
- }
- )?
-
- (
- {
- primaryKeyFields = getPrimaryKeyFields();
- }
- )
-
- (
- "on" < IDENTIFIER >
- {
- nodeGroupName = new Identifier(token.image);
- }
- )?
-
- {
- fdd = new FeedDetailsDecl(adapterName, properties, appliedFunction, nodeGroupName, primaryKeyFields);
- return fdd;
- }
-}
-
-List<String> getPrimaryKeyFields() throws ParseException :
-{
- List<String> primaryKeyFields = new ArrayList<String>();
-}
-{
-
- "primary" "key"
- < IDENTIFIER >
- {
- primaryKeyFields.add(token.image);
- }
- (
- "," < IDENTIFIER >
- {
- primaryKeyFields.add(token.image);
- }
- )*
- {
- return primaryKeyFields;
- }
-
-}
-
-
-
-
-
-ControlFeedStatement ControlFeedDeclaration(ControlFeedStatement.OperationType operationType) throws ParseException :
+Statement FeedStatement() throws ParseException:
{
Pair<Identifier,Identifier> nameComponents = null;
+ Map<String,String> configuration = null;
+ Statement stmt = null;
}
{
+ (
+ "begin" "feed" nameComponents = QualifiedName()
+ {
+ stmt = new BeginFeedStatement(nameComponents.first, nameComponents.second, getVarCounter());
+ }
+ | "suspend" "feed" nameComponents = QualifiedName()
+ {
+ stmt = new ControlFeedStatement(ControlFeedStatement.OperationType.SUSPEND, nameComponents.first, nameComponents.second);
+ }
+ | "resume" "feed" nameComponents = QualifiedName()
+ {
+ stmt = new ControlFeedStatement(ControlFeedStatement.OperationType.RESUME, nameComponents.first, nameComponents.second);
+ }
+ | "end" "feed" nameComponents = QualifiedName()
+ {
+ stmt = new ControlFeedStatement(ControlFeedStatement.OperationType.END, nameComponents.first, nameComponents.second);
+ }
+ | "alter" "feed" nameComponents = QualifiedName() "set" configuration = Configuration()
+ {
+ stmt = new ControlFeedStatement(ControlFeedStatement.OperationType.ALTER, nameComponents.first, nameComponents.second, configuration);
+ }
+ )
{
- nameComponents = getDotSeparatedPair();
- return new ControlFeedStatement(operationType, nameComponents.first, nameComponents.second);
+ return stmt;
}
}
-
-ControlFeedStatement AlterFeedDeclaration() throws ParseException :
-{
- Pair<Identifier,Identifier> nameComponents = null;
- Map < String, String > configuration = new HashMap < String, String > ();
-}
-{
- {
- nameComponents = getDotSeparatedPair();
- }
-
- "set"
- {
- configuration = getConfiguration();
- }
-
- {
- return new ControlFeedStatement(ControlFeedStatement.OperationType.ALTER, nameComponents.first, nameComponents.second, configuration);
- }
-}
-
-Map<String,String> getConfiguration() throws ParseException :
+Map<String,String> Configuration() throws ParseException :
{
Map<String,String> configuration = new LinkedHashMap<String,String>();
- String key;
- String value;
+ Pair<String, String> keyValuePair = null;
}
{
-
-<LEFTPAREN>
- (
- (
- <LEFTPAREN>
- (
- <STRING_LITERAL>
- {
- key = removeQuotesAndEscapes(token.image);
- }
- "=" <STRING_LITERAL>
- {
- value = removeQuotesAndEscapes(token.image);
- }
- )
- <RIGHTPAREN>
- {
- configuration.put(key, value);
- }
- )
- (
- "," <LEFTPAREN>
- (
- <STRING_LITERAL>
- {
- key = removeQuotesAndEscapes(token.image);
- }
- "=" <STRING_LITERAL>
- {
- value = removeQuotesAndEscapes(token.image);
- }
- )
- <RIGHTPAREN>
- {
- configuration.put(key, value);
- }
- )*
- )?
- <RIGHTPAREN>
- {
- return configuration;
- }
-}
-
-void initProperties(Map<String,String> properties) throws ParseException :
-{
- String key;
- String value;
-}
-{
- (
- <LEFTPAREN>
- (
- <IDENTIFIER>
- {
- key = (new Identifier(token.image)).getValue();
- }
- "="
- (
- (<STRING_LITERAL>
- {
- value = removeQuotesAndEscapes(token.image);
- }
- ) |
- (<INTEGER_LITERAL>
- {
- try{
- value = "" + Long.valueOf(token.image);
- } catch (NumberFormatException nfe){
- throw new ParseException("inapproriate value: " + token.image);
- }
- }
- )
- )
- {
- properties.put(key.toUpperCase(), value);
- }
- (
- ","
- (
- <IDENTIFIER>
- {
- key = (new Identifier(token.image)).getValue();
- }
- "="
- (
- (<STRING_LITERAL>
- {
- value = removeQuotesAndEscapes(token.image);
- }
- ) |
- (<INTEGER_LITERAL>
- {
- try{
- value = "" + Long.valueOf(token.image);
- } catch (NumberFormatException nfe){
- throw new ParseException("inapproriate value: " + token.image);
- }
- }
- )
- )
- )
- {
- properties.put(key.toUpperCase(), value);
- }
-
- )*
- )
- <RIGHTPAREN>
- )?
-}
-
-
-
-NodegroupDecl NodegroupDeclaration() throws ParseException :
-{
- Identifier name = null;
- List < Identifier > ncNames = new ArrayList < Identifier > ();
- boolean ifNotExists = false;
-}
-{
- < IDENTIFIER >
- {
- name = new Identifier(token.image);
- }
- (
- "if not exists"
- {
- ifNotExists = true;
- }
- )?
- "on" < IDENTIFIER >
- {
- ncNames.add(new Identifier(token.image));
- }
- (
- "," < IDENTIFIER >
+ <LEFTPAREN> ( keyValuePair = KeyValuePair()
{
- ncNames.add(new Identifier(token.image));
+ configuration.put(keyValuePair.first, keyValuePair.second);
}
- )*
- ";"
- {
- return new NodegroupDecl(name, ncNames, ifNotExists);
- }
-}
-
-
-TypeDecl TypeDeclaration(boolean dgen, String hint) throws ParseException:
-{
- Identifier dataverse;
- Identifier ident;
- TypeExpression typeExpr;
- boolean ifNotExists = false;
- Pair<Identifier,Identifier> nameComponents=null;
-}
-{
- {
- nameComponents = getDotSeparatedPair();
- dataverse = nameComponents.first;
- ident = nameComponents.second;
- }
-
- (
- "if not exists"
+ ( "," keyValuePair = KeyValuePair()
{
- ifNotExists = true;
+ configuration.put(keyValuePair.first, keyValuePair.second);
}
- )?
- "as"
- ( typeExpr = TypeExpr() )
- (";")?
- {
- long numValues = -1;
- String filename = null;
- if (dgen) {
- String splits[] = hint.split(" +");
- if (splits.length != 3) {
- throw new ParseException("Expecting /*+ dgen <filename> <numberOfItems> */");
- }
- filename = splits[1];
- numValues = Long.parseLong(splits[2]);
+ )* )? <RIGHTPAREN>
+ {
+ return configuration;
+ }
+}
+
+Pair<String, String> KeyValuePair() throws ParseException:
+{
+ String key;
+ String value;
+}
+{
+ <LEFTPAREN> key = StringLiteral() "=" value = StringLiteral() <RIGHTPAREN>
+ {
+ return new Pair<String, String>(key, value);
}
- TypeDataGen tddg = new TypeDataGen(dgen, filename, numValues);
- return new TypeDecl(dataverse, ident, typeExpr, tddg, ifNotExists);
- }
+}
+
+Map<String,String> Properties() throws ParseException:
+{
+ Map<String,String> properties = new HashMap<String,String>();
+ Pair<String, String> property;
+}
+{
+ ( <LEFTPAREN> property = Property()
+ {
+ properties.put(property.first, property.second);
+ }
+ ( "," property = Property()
+ {
+ properties.put(property.first, property.second);
+ }
+ )* <RIGHTPAREN> )?
+ {
+ return properties;
+ }
+}
+
+Pair<String, String> Property() throws ParseException:
+{
+ String key;
+ String value;
+}
+{
+ key = Identifier() "=" ( value = StringLiteral() | <INTEGER_LITERAL>
+ {
+ try {
+ value = "" + Long.valueOf(token.image);
+ } catch (NumberFormatException nfe) {
+ throw new ParseException("inapproriate value: " + token.image);
+ }
+ }
+ )
+ {
+ return new Pair<String, String>(key.toUpperCase(), value);
+ }
}
TypeExpression TypeExpr() throws ParseException:
@@ -1181,11 +911,10 @@
boolean nullable = false;
}
{
- <IDENTIFIER>
+ fieldName = Identifier()
{
- Token t = getToken(0);
- fieldName = t.toString();
- String hint = getHint(t);
+ fieldName = token.image;
+ String hint = getHint(token);
IRecordFieldDataGen rfdg = null;
if (hint != null) {
String splits[] = hint.split(" +");
@@ -1237,14 +966,14 @@
}
TypeReferenceExpression TypeReference() throws ParseException:
-{}
{
- <IDENTIFIER>
- {
- Token t = getToken(0);
- Identifier id = new Identifier(t.toString());
- return new TypeReferenceExpression(id);
- }
+ String id = null;
+}
+{
+ id = Identifier()
+ {
+ return new TypeReferenceExpression(new Identifier(id));
+ }
}
OrderedListTypeDefinition OrderedListTypeDef() throws ParseException:
@@ -1274,68 +1003,91 @@
}
}
-Pair<Identifier,Identifier> getDotSeparatedPair() throws ParseException:
+
+Pair<Identifier,Identifier> FunctionOrTypeName() throws ParseException:
{
- Identifier first = null;
- Identifier second = null;
+ Pair<Identifier,Identifier> name = null;
}
{
- < IDENTIFIER >
+ name = QualifiedName()
+ {
+ if (name.first == null) {
+ name.first = new Identifier(defaultDataverse);
+ }
+ return name;
+ }
+}
+
+String Identifier() throws ParseException:
+{
+ String lit = null;
+}
+{
+ <IDENTIFIER>
+ {
+ return token.image;
+ }
+ | lit = StringLiteral()
+ {
+ return lit;
+ }
+}
+
+String StringLiteral() throws ParseException:
+{
+}
+{
+ <STRING_LITERAL>
+ {
+ return removeQuotesAndEscapes(token.image);
+ }
+}
+
+Pair<Identifier,Identifier> QualifiedName() throws ParseException:
+{
+ String first = null;
+ String second = null;
+}
+{
+ first = Identifier() ("." second = Identifier())?
{
- first = new Identifier(token.image);
- }
- ("." <IDENTIFIER>
- {
- second = new Identifier(token.image);
- }
- )?
-
- {
- if(second == null){
- second = first;
- first = null;
- }
-
- return new Pair<Identifier,Identifier>(first,second);
+ Identifier id1 = null;
+ Identifier id2 = null;
+ if (second == null) {
+ id2 = new Identifier(first);
+ } else
+ {
+ id1 = new Identifier(first);
+ id2 = new Identifier(second);
+ }
+ return new Pair<Identifier,Identifier>(id1, id2);
}
}
-Triple<Identifier,Identifier,Identifier> getDotSeparatedTriple() throws ParseException:
+Triple<Identifier,Identifier,Identifier> DoubleQualifiedName() throws ParseException:
{
- Identifier first = null;
- Identifier second = null;
- Identifier third = null;
+ String first = null;
+ String second = null;
+ String third = null;
}
{
- < IDENTIFIER >
+ first = Identifier() "." second = Identifier() ("." third = Identifier())?
{
- first = new Identifier(token.image);
- }
- "." <IDENTIFIER>
- {
- second = new Identifier(token.image);
- }
- (
- "." <IDENTIFIER>
- {
- third = new Identifier(token.image);
- }
- )?
-
- {
- if(third == null){
- third = second;
- second = first;
- first = null;
- }
-
- return new Triple<Identifier,Identifier,Identifier>(first,second,third);
+ Identifier id1 = null;
+ Identifier id2 = null;
+ Identifier id3 = null;
+ if (third == null) {
+ id2 = new Identifier(first);
+ id3 = new Identifier(second);
+ } else {
+ id1 = new Identifier(first);
+ id2 = new Identifier(second);
+ id3 = new Identifier(third);
+ }
+ return new Triple<Identifier,Identifier,Identifier>(id1, id2, id3);
}
}
-
-
-
FunctionDecl FunctionDeclaration() throws ParseException:
{
FunctionDecl funcDecl;
@@ -1348,29 +1100,23 @@
createNewScope();
}
{
-
- <IDENTIFIER>
- {
- Token t = getToken(0);
- functionName = t.toString();
- }
- <LEFTPAREN> (<VARIABLE>
+ "declare" "function" functionName = Identifier() <LEFTPAREN> (<VARIABLE>
{
var = new VarIdentifier();
- var.setValue(getToken(0).toString());
+ var.setValue(token.image);
paramList.add(var);
getCurrentScope().addNewVarSymbolToScope(var);
arity++;
}
- ("," <VARIABLE>
+ ("," <VARIABLE>
{
var = new VarIdentifier();
- var.setValue(getToken(0).toString());
+ var.setValue(token.image);
paramList.add(var);
getCurrentScope().addNewVarSymbolToScope(var);
arity++;
- })*)? <RIGHTPAREN> "{" funcBody = Expression() "}"
- (";")?
+ }
+ )*)? <RIGHTPAREN> "{" funcBody = Expression() "}"
{
signature = new FunctionSignature(defaultDataverse, functionName, arity);
getCurrentScope().addFunctionDescriptor(signature, false);
@@ -1379,78 +1125,14 @@
}
}
-CreateFunctionStatement FunctionCreation() throws ParseException:
-{
- CreateFunctionStatement cfs = null;
- FunctionSignature signature;
- String dataverse;
- String functionName;
- boolean ifNotExists = false;
- List<VarIdentifier> paramList = new ArrayList<VarIdentifier>();
- String functionBody;
- VarIdentifier var = null;
- createNewScope();
- Expression functionBodyExpr;
- Token beginPos;
- Token endPos;
- Pair<Identifier,Identifier> nameComponents=null;
-}
-{
- {
- nameComponents = getDotSeparatedPair();
- dataverse = nameComponents.first != null ? nameComponents.first.getValue() : defaultDataverse;
- functionName= nameComponents.second.getValue();
- }
-
- (
- "if not exists"
- {
- ifNotExists = true;
- }
- )?
-
- <LEFTPAREN> (<VARIABLE>
- {
- var = new VarIdentifier();
- var.setValue(getToken(0).toString());
- paramList.add(var);
- getCurrentScope().addNewVarSymbolToScope(var);
- }
- ("," <VARIABLE>
- {
- var = new VarIdentifier();
- var.setValue(getToken(0).toString());
- paramList.add(var);
- getCurrentScope().addNewVarSymbolToScope(var);
- })*)? <RIGHTPAREN> "{"
- {
- beginPos = getToken(0);
- }
- functionBodyExpr = Expression()
- "}"
- {
- endPos = getToken(0);
- functionBody = extractFragment(beginPos.beginLine, beginPos.beginColumn, endPos.beginLine, endPos.beginColumn);
- }
- (";")?
- {
- signature = new FunctionSignature(dataverse, functionName, paramList.size());
- getCurrentScope().addFunctionDescriptor(signature, false);
- cfs = new CreateFunctionStatement(signature, paramList, functionBody, ifNotExists);
- return cfs;
- }
-}
-
-
-Query Query()throws ParseException:
+Query Query() throws ParseException:
{
Query query = new Query();
Expression expr;
}
{
- expr = Expression()
- (";")?
+ expr = Expression()
{
query.setBody(expr);
query.setVarCounter(getVarCounter());
@@ -1500,8 +1182,7 @@
op.addOperand(operand);
op.setCurrentop(true);
}
- Token t = getToken(0);
- op.addOperator(t.toString());
+ op.addOperator(token.image);
}
operand = AndExpr()
@@ -1532,8 +1213,7 @@
op.addOperand(operand);
op.setCurrentop(true);
}
- Token t = getToken(0);
- op.addOperator(t.toString());
+ op.addOperator(token.image);
}
operand = RelExpr()
@@ -1580,9 +1260,8 @@
op.addOperand(operand, broadcast);
op.setCurrentop(true);
broadcast = false;
- }
- Token t = getToken(0);
- op.addOperator(t.toString());
+ }
+ op.addOperator(token.image);
}
operand = AddExpr()
@@ -1620,9 +1299,8 @@
op = new OperatorExpr();
op.addOperand(operand);
op.setCurrentop(true);
- }
- Token t = getToken(0);
- ((OperatorExpr)op).addOperator(t.toString());
+ }
+ ((OperatorExpr)op).addOperator(token.image);
}
operand = MultExpr()
@@ -1650,9 +1328,8 @@
op = new OperatorExpr();
op.addOperand(operand);
op.setCurrentop(true);
- }
- Token t = getToken(0);
- op.addOperator(t.toString());
+ }
+ op.addOperator(token.image);
}
operand = UnionExpr()
{
@@ -1694,11 +1371,10 @@
{
(( "+"|"-")
{
- uexpr = new UnaryExpr();
- Token t = getToken(0);
- if("+".equals(t.toString()))
+ uexpr = new UnaryExpr();
+ if("+".equals(token.image))
((UnaryExpr)uexpr).setSign(Sign.POSITIVE);
- else if("-".equals(t.toString()))
+ else if("-".equals(token.image))
((UnaryExpr)uexpr).setSign(Sign.NEGATIVE);
else
throw new ParseException();
@@ -1717,73 +1393,39 @@
}
}
-Expression ValueExpr() throws ParseException:
-{
- Expression expr;
-}
-{
- expr = FieldOrIndexAccessor()
- {
- return expr;
- }
-}
-
-
-Expression FieldOrIndexAccessor()throws ParseException:
+Expression ValueExpr()throws ParseException:
{
Expression expr = null;
Identifier ident = null;
AbstractAccessor fa = null;
int index;
-
}
{
- ( expr = PrimaryExpr()
-
- )
-
-
- (
- (
- ident = Field()
+ expr = PrimaryExpr() ( ident = Field()
{
- if(fa == null)
- fa = new FieldAccessor(expr, ident);
- else
- fa = new FieldAccessor(fa, ident);
- }
- )
- | (
- index = Index()
- {
- if(fa == null)
- fa = new IndexAccessor(expr, index);
- else
- fa = new IndexAccessor(fa, index);
- }
- )
- )*
-
-
- {
- return fa==null?expr:fa;
- }
+ fa = (fa == null ? new FieldAccessor(expr, ident)
+ : new FieldAccessor(fa, ident));
+ }
+ | index = Index()
+ {
+ fa = (fa == null ? new IndexAccessor(expr, index)
+ : new IndexAccessor(fa, index));
+ }
+ )*
+ {
+ return fa == null ? expr : fa;
+ }
}
Identifier Field() throws ParseException:
{
- Identifier ident = null;
-
+ String ident = null;
}
{
- "." < IDENTIFIER >
- {
-
- ident = new Identifier();
- ident.setValue(getToken(0).toString());
-
- return ident;
- }
+ "." ident = Identifier()
+ {
+ return new Identifier(ident);
+ }
}
int Index() throws ParseException:
@@ -1828,75 +1470,63 @@
Expression expr = null;
}
{
- //Literal | VariableRef | ListConstructor | RecordConstructor | FunctionCallExpr | DatasetAccessExpression | ParenthesizedExpression
- (
- expr =Literal()
- | expr = FunctionCallExpr()
- | expr = DatasetAccessExpression()
- | expr =VariableRef()
-
+ ( LOOKAHEAD(2)
+ expr = FunctionCallExpr()
+ | expr = Literal()
+ | expr = DatasetAccessExpression()
+ | expr = VariableRef()
{
if(((VariableExpr)expr).getIsNewVar() == true)
- throw new ParseException("can't find variable " + ((VariableExpr)expr).getVar());
+ throw new ParseException("can't find variable " + ((VariableExpr)expr).getVar());
}
- | expr = ListConstructor()
- | expr = RecordConstructor()
- | expr = ParenthesizedExpression()
- )
- {
- return expr;
- }
+ | expr = ListConstructor()
+ | expr = RecordConstructor()
+ | expr = ParenthesizedExpression()
+ )
+ {
+ return expr;
+ }
}
Expression Literal() throws ParseException:
{
-
LiteralExpr lit = new LiteralExpr();
- Token t;
+ String str = null;
}
{
-(
- <STRING_LITERAL>
- {
- t= getToken(0);
- lit.setValue( new StringLiteral(removeQuotesAndEscapes(t.image)));
- }
-
- | <INTEGER_LITERAL>
+ ( str = StringLiteral()
{
- t= getToken(0);
- try {
- lit.setValue(new IntegerLiteral(new Integer(t.image)));
- } catch(NumberFormatException ex) {
- lit.setValue(new LongIntegerLiteral(new Long(t.image)));
- }
- }
- | < FLOAT_LITERAL >
+ lit.setValue(new StringLiteral(str));
+ }
+ | <INTEGER_LITERAL>
{
- t= getToken(0);
- lit.setValue(new FloatLiteral(new Float(t.image)));
- }
- | < DOUBLE_LITERAL >
+ try {
+ lit.setValue(new IntegerLiteral(new Integer(token.image)));
+ } catch(NumberFormatException ex) {
+ lit.setValue(new LongIntegerLiteral(new Long(token.image)));
+ }
+ }
+ | <FLOAT_LITERAL>
{
- t= getToken(0);
- lit.setValue(new DoubleLiteral(new Double(t.image)));
- }
- | <NULL>
- {
- t= getToken(0);
- lit.setValue(NullLiteral.INSTANCE);
- }
- | <TRUE>
- {
- t= getToken(0);
- lit.setValue(TrueLiteral.INSTANCE);
- }
- | <FALSE>
- {
- t= getToken(0);
- lit.setValue(FalseLiteral.INSTANCE);
- }
-)
+ lit.setValue(new FloatLiteral(new Float(token.image)));
+ }
+ | <DOUBLE_LITERAL>
+ {
+ lit.setValue(new DoubleLiteral(new Double(token.image)));
+ }
+ | <NULL>
+ {
+ lit.setValue(NullLiteral.INSTANCE);
+ }
+ | <TRUE>
+ {
+ lit.setValue(TrueLiteral.INSTANCE);
+ }
+ | <FALSE>
+ {
+ lit.setValue(FalseLiteral.INSTANCE);
+ }
+ )
{
return lit;
}
@@ -1907,13 +1537,11 @@
{
VariableExpr varExp = new VariableExpr();
VarIdentifier var = new VarIdentifier();
- Token t;
}
{
<VARIABLE>
{
- t = getToken(0);//get current token
- String varName = t.toString();
+ String varName = token.image;
Identifier ident = lookupSymbol(varName);
if (isInForbiddenScopes(varName)) {
throw new ParseException("Inside limit clauses, it is disallowed to reference a variable having the same name as any variable bound in the same scope as the limit clause.");
@@ -1924,7 +1552,7 @@
} else {
varExp.setVar(var);
}
- var.setValue(t.toString());
+ var.setValue(varName);
return varExp;
}
}
@@ -1934,18 +1562,16 @@
{
VariableExpr varExp = new VariableExpr();
VarIdentifier var = new VarIdentifier();
- Token t;
}
{
<VARIABLE>
{
- t = getToken(0);//get current token
- Identifier ident = lookupSymbol(t.toString());
+ Identifier ident = lookupSymbol(token.image);
if(ident != null) { // exist such ident
varExp.setIsNewVar(false);
}
varExp.setVar(var);
- var.setValue(t.toString());
+ var.setValue(token.image);
return varExp;
}
}
@@ -2050,36 +1676,42 @@
List<Expression> argList = new ArrayList<Expression>();
Expression tmp;
int arity = 0;
+ Pair<Identifier,Identifier> funcId = null;
String funcName;
String dataverse;
- String hint=null;
- String id1=null;
- String id2=null;
+ String hint = null;
+ String id1 = null;
+ String id2 = null;
}
{
-
- <IDENTIFIER> { dataverse = defaultDataverse; funcName = token.image;} ("." <IDENTIFIER> { dataverse = funcName; funcName = token.image;})?
+ funcId = FunctionOrTypeName()
{
- hint = getHint(token);
+ dataverse = funcId.first.getValue();
+ funcName = funcId.second.getValue();
+ hint = getHint(token);
}
- <LEFTPAREN> (tmp = Expression()
- {
- argList.add(tmp);
- arity ++;
- } ("," tmp = Expression() { argList.add(tmp); arity++; })*)? <RIGHTPAREN>
-
- {
- FunctionSignature signature = lookupFunctionSignature(dataverse, funcName.toString(), arity);
- if(signature == null)
- {
- signature = new FunctionSignature(dataverse, funcName.toString(), arity);
- }
- callExpr = new CallExpr(signature,argList);
- if (hint != null && hint.startsWith(INDEXED_NESTED_LOOP_JOIN_HINT)) {
- callExpr.addHint(IndexedNLJoinExpressionAnnotation.INSTANCE);
- }
- return callExpr;
- }
+ <LEFTPAREN> (tmp = Expression()
+ {
+ argList.add(tmp);
+ arity ++;
+ }
+ ("," tmp = Expression()
+ {
+ argList.add(tmp);
+ arity++;
+ }
+ )*)? <RIGHTPAREN>
+ {
+ FunctionSignature signature = lookupFunctionSignature(dataverse, funcName, arity);
+ if (signature == null) {
+ signature = new FunctionSignature(dataverse, funcName, arity);
+ }
+ callExpr = new CallExpr(signature,argList);
+ if (hint != null && hint.startsWith(INDEXED_NESTED_LOOP_JOIN_HINT)) {
+ callExpr.addHint(IndexedNLJoinExpressionAnnotation.INSTANCE);
+ }
+ return callExpr;
+ }
}
Expression DatasetAccessExpression() throws ParseException:
@@ -2088,27 +1720,45 @@
List<Expression> argList = new ArrayList<Expression>();
String funcName;
String dataverse;
+ String arg1 = null;
+ String arg2 = null;
LiteralExpr ds;
- LiteralExpr dvds;
Expression nameArg;
int arity = 0;
}
{
- <DATASET> {dataverse = MetadataConstants.METADATA_DATAVERSE_NAME; funcName = getToken(0).toString();}
- (
- (<IDENTIFIER> {ds = new LiteralExpr(); ds.setValue( new StringLiteral(token.image) ); argList.add(ds); arity ++;} ("." <IDENTIFIER> { dvds = new LiteralExpr(); dvds.setValue(new StringLiteral(ds.getValue()+"."+token.image)); argList.remove(0); argList.add(dvds);})? ) |
- (<LEFTPAREN> nameArg = Expression() {argList.add(nameArg); arity ++;} ("," nameArg = Expression() { argList.add(nameArg); arity++; })* <RIGHTPAREN>)
- )
-
- {
- FunctionSignature signature = lookupFunctionSignature(dataverse, funcName.toString(), arity);
- if(signature == null)
- {
- signature = new FunctionSignature(dataverse, funcName.toString(), arity);
- }
- callExpr = new CallExpr(signature,argList);
- return callExpr;
+ <DATASET>
+ {
+ dataverse = MetadataConstants.METADATA_DATAVERSE_NAME;
+ funcName = token.image;
+ }
+ ( ( arg1 = Identifier() ( "." arg2 = Identifier() )? )
+ {
+ String name = arg2 == null ? arg1 : arg1 + "." + arg2;
+ ds = new LiteralExpr();
+ ds.setValue( new StringLiteral(name) );
+ argList.add(ds);
+ arity ++;
+ }
+ | ( <LEFTPAREN> nameArg = Expression()
+ {
+ argList.add(nameArg);
+ arity ++;
+ }
+ ( "," nameArg = Expression()
+ {
+ argList.add(nameArg);
+ arity++;
+ }
+ )* <RIGHTPAREN> ) )
+ {
+ FunctionSignature signature = lookupFunctionSignature(dataverse, funcName, arity);
+ if (signature == null) {
+ signature = new FunctionSignature(dataverse, funcName, arity);
}
+ callExpr = new CallExpr(signature,argList);
+ return callExpr;
+ }
}
Expression ParenthesizedExpression() throws ParseException:
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/api/AsterixAppContextInfo.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/api/AsterixAppContextInfo.java
new file mode 100644
index 0000000..06a1c1f
--- /dev/null
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/api/AsterixAppContextInfo.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2009-2012 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package edu.uci.ics.asterix.common.api;
+
+import java.util.logging.Logger;
+
+import edu.uci.ics.asterix.common.config.AsterixCompilerProperties;
+import edu.uci.ics.asterix.common.config.AsterixExternalProperties;
+import edu.uci.ics.asterix.common.config.AsterixMetadataProperties;
+import edu.uci.ics.asterix.common.config.AsterixPropertiesAccessor;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
+import edu.uci.ics.asterix.common.config.AsterixTransactionProperties;
+import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
+import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
+import edu.uci.ics.asterix.common.dataflow.IAsterixApplicationContextInfo;
+import edu.uci.ics.asterix.common.exceptions.AsterixException;
+import edu.uci.ics.hyracks.api.application.ICCApplicationContext;
+import edu.uci.ics.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
+import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
+
+/*
+ * Acts as an holder class for IndexRegistryProvider, AsterixStorageManager
+ * instances that are accessed from the NCs. In addition an instance of ICCApplicationContext
+ * is stored for access by the CC.
+ */
+public class AsterixAppContextInfo implements IAsterixApplicationContextInfo, IAsterixPropertiesProvider {
+
+ private static AsterixAppContextInfo INSTANCE;
+
+ private final ICCApplicationContext appCtx;
+
+ private AsterixCompilerProperties compilerProperties;
+ private AsterixExternalProperties externalProperties;
+ private AsterixMetadataProperties metadataProperties;
+ private AsterixStorageProperties storageProperties;
+ private AsterixTransactionProperties txnProperties;
+
+ public static void initialize(ICCApplicationContext ccAppCtx) throws AsterixException {
+ if (INSTANCE == null) {
+ INSTANCE = new AsterixAppContextInfo(ccAppCtx);
+ }
+ AsterixPropertiesAccessor propertiesAccessor = new AsterixPropertiesAccessor();
+ INSTANCE.compilerProperties = new AsterixCompilerProperties(propertiesAccessor);
+ INSTANCE.externalProperties = new AsterixExternalProperties(propertiesAccessor);
+ INSTANCE.metadataProperties = new AsterixMetadataProperties(propertiesAccessor);
+ INSTANCE.storageProperties = new AsterixStorageProperties(propertiesAccessor);
+ INSTANCE.txnProperties = new AsterixTransactionProperties(propertiesAccessor);
+
+ Logger.getLogger("edu.uci.ics").setLevel(INSTANCE.externalProperties.getLogLevel());
+ }
+
+ private AsterixAppContextInfo(ICCApplicationContext ccAppCtx) {
+ this.appCtx = ccAppCtx;
+ }
+
+ public static AsterixAppContextInfo getInstance() {
+ return INSTANCE;
+ }
+
+ @Override
+ public IStorageManagerInterface getStorageManagerInterface() {
+ return AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER;
+ }
+
+ @Override
+ public ICCApplicationContext getCCApplicationContext() {
+ return appCtx;
+ }
+
+ @Override
+ public IIndexLifecycleManagerProvider getIndexLifecycleManagerProvider() {
+ return AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER;
+ }
+
+ @Override
+ public AsterixStorageProperties getStorageProperties() {
+ return storageProperties;
+ }
+
+ @Override
+ public AsterixTransactionProperties getTransactionProperties() {
+ return txnProperties;
+ }
+
+ @Override
+ public AsterixCompilerProperties getCompilerProperties() {
+ return compilerProperties;
+ }
+
+ @Override
+ public AsterixMetadataProperties getMetadataProperties() {
+ return metadataProperties;
+ }
+
+ @Override
+ public AsterixExternalProperties getExternalProperties() {
+ return externalProperties;
+ }
+}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/api/AsterixAppContextInfoImpl.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/api/AsterixAppContextInfoImpl.java
deleted file mode 100644
index 0f1de56..0000000
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/api/AsterixAppContextInfoImpl.java
+++ /dev/null
@@ -1,62 +0,0 @@
-/*
- * Copyright 2009-2012 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package edu.uci.ics.asterix.common.api;
-
-import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
-import edu.uci.ics.asterix.common.dataflow.IAsterixApplicationContextInfo;
-import edu.uci.ics.hyracks.api.application.ICCApplicationContext;
-import edu.uci.ics.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
-import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
-
-/*
- * Acts as an holder class for IndexRegistryProvider, AsterixStorageManager
- * instances that are accessed from the NCs. In addition an instance of ICCApplicationContext
- * is stored for access by the CC.
- */
-public class AsterixAppContextInfoImpl implements IAsterixApplicationContextInfo {
-
- private static AsterixAppContextInfoImpl INSTANCE;
-
- private final ICCApplicationContext appCtx;
-
- public static void initialize(ICCApplicationContext ccAppCtx) {
- if (INSTANCE == null) {
- INSTANCE = new AsterixAppContextInfoImpl(ccAppCtx);
- }
- }
-
- private AsterixAppContextInfoImpl(ICCApplicationContext ccAppCtx) {
- this.appCtx = ccAppCtx;
- }
-
- public static IAsterixApplicationContextInfo getInstance() {
- return INSTANCE;
- }
-
- @Override
- public IStorageManagerInterface getStorageManagerInterface() {
- return AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER;
- }
-
- @Override
- public ICCApplicationContext getCCApplicationContext() {
- return appCtx;
- }
-
- @Override
- public IIndexLifecycleManagerProvider getIndexLifecycleManagerProvider() {
- return AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER;
- }
-}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AbstractAsterixProperties.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AbstractAsterixProperties.java
new file mode 100644
index 0000000..4968f40
--- /dev/null
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AbstractAsterixProperties.java
@@ -0,0 +1,9 @@
+package edu.uci.ics.asterix.common.config;
+
+public abstract class AbstractAsterixProperties {
+ protected final AsterixPropertiesAccessor accessor;
+
+ public AbstractAsterixProperties(AsterixPropertiesAccessor accessor) {
+ this.accessor = accessor;
+ }
+}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixCompilerProperties.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixCompilerProperties.java
new file mode 100644
index 0000000..b1767b9
--- /dev/null
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixCompilerProperties.java
@@ -0,0 +1,31 @@
+package edu.uci.ics.asterix.common.config;
+
+public class AsterixCompilerProperties extends AbstractAsterixProperties {
+ private static final String COMPILER_SORTMEMORY_KEY = "compiler.sortmemory";
+ private static final long COMPILER_SORTMEMORY_DEFAULT = (512 << 20); // 512MB
+
+ private static final String COMPILER_JOINMEMORY_KEY = "compiler.joinmemory";
+ private static final long COMPILER_JOINMEMORY_DEFAULT = (512 << 20); // 512MB
+
+ private static final String COMPILER_FRAMESIZE_KEY = "compiler.framesize";
+ private static int COMPILER_FRAMESIZE_DEFAULT = (32 << 10); // 32KB
+
+ public AsterixCompilerProperties(AsterixPropertiesAccessor accessor) {
+ super(accessor);
+ }
+
+ public long getSortMemorySize() {
+ return accessor.getProperty(COMPILER_SORTMEMORY_KEY, COMPILER_SORTMEMORY_DEFAULT,
+ PropertyInterpreters.getLongPropertyInterpreter());
+ }
+
+ public long getJoinMemorySize() {
+ return accessor.getProperty(COMPILER_JOINMEMORY_KEY, COMPILER_JOINMEMORY_DEFAULT,
+ PropertyInterpreters.getLongPropertyInterpreter());
+ }
+
+ public int getFrameSize() {
+ return accessor.getProperty(COMPILER_FRAMESIZE_KEY, COMPILER_FRAMESIZE_DEFAULT,
+ PropertyInterpreters.getIntegerPropertyInterpreter());
+ }
+}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixExternalProperties.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixExternalProperties.java
new file mode 100644
index 0000000..cf38932
--- /dev/null
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixExternalProperties.java
@@ -0,0 +1,34 @@
+package edu.uci.ics.asterix.common.config;
+
+import java.util.logging.Level;
+
+public class AsterixExternalProperties extends AbstractAsterixProperties {
+
+ private static final String EXTERNAL_WEBPORT_KEY = "web.port";
+ private static int EXTERNAL_WEBPORT_DEFAULT = 19001;
+
+ private static final String EXTERNAL_LOGLEVEL_KEY = "log.level";
+ private static Level EXTERNAL_LOGLEVEL_DEFAULT = Level.INFO;
+
+ private static final String EXTERNAL_APISERVER_KEY = "api.port";
+ private static int EXTERNAL_APISERVER_DEFAULT = 19101;
+
+ public AsterixExternalProperties(AsterixPropertiesAccessor accessor) {
+ super(accessor);
+ }
+
+ public int getWebInterfacePort() {
+ return accessor.getProperty(EXTERNAL_WEBPORT_KEY, EXTERNAL_WEBPORT_DEFAULT,
+ PropertyInterpreters.getIntegerPropertyInterpreter());
+ }
+
+ public int getAPIServerPort() {
+ return accessor.getProperty(EXTERNAL_APISERVER_KEY, EXTERNAL_APISERVER_DEFAULT,
+ PropertyInterpreters.getIntegerPropertyInterpreter());
+ }
+
+ public Level getLogLevel() {
+ return accessor.getProperty(EXTERNAL_LOGLEVEL_KEY, EXTERNAL_LOGLEVEL_DEFAULT,
+ PropertyInterpreters.getLevelPropertyInterpreter());
+ }
+}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixMetadataProperties.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixMetadataProperties.java
new file mode 100644
index 0000000..6d47e78
--- /dev/null
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixMetadataProperties.java
@@ -0,0 +1,28 @@
+package edu.uci.ics.asterix.common.config;
+
+import java.util.Map;
+import java.util.Set;
+
+public class AsterixMetadataProperties extends AbstractAsterixProperties {
+
+ public AsterixMetadataProperties(AsterixPropertiesAccessor accessor) {
+ super(accessor);
+ }
+
+ public String getMetadataNodeName() {
+ return accessor.getMetadataNodeName();
+ }
+
+ public String getMetadataStore() {
+ return accessor.getMetadataStore();
+ }
+
+ public Map<String, String[]> getStores() {
+ return accessor.getStores();
+ }
+
+ public Set<String> getNodeNames() {
+ return accessor.getNodeNames();
+ }
+
+}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixProperties.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixProperties.java
deleted file mode 100644
index 41ef67c..0000000
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixProperties.java
+++ /dev/null
@@ -1,252 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.asterix.common.config;
-
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.InputStream;
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.Unmarshaller;
-
-import edu.uci.ics.asterix.common.configuration.AsterixConfiguration;
-import edu.uci.ics.asterix.common.configuration.Property;
-import edu.uci.ics.asterix.common.configuration.Store;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-
-/**
- * Holder for Asterix properties values typically set as Java Properties.
- * Intended to live in the AsterixStateProxy so it can be accessed remotely.
- */
-public class AsterixProperties implements Serializable {
-
- private static final long serialVersionUID = 1L;
- private static String metadataNodeName;
- private static HashSet<String> nodeNames;
- private static Map<String, String[]> stores;
- private static Map<String, String> asterixConfigurationParams;
-
- public static AsterixProperties INSTANCE = new AsterixProperties();
-
- public static class AsterixConfigurationKeys {
-
- // JVM parameters for each Node Contoller (NC)
- public static final String NC_JAVA_OPTS = "nc.java.opts";
- public static final String NC_JAVA_OPTS_DEFAULT = "-Xmx1024m";
-
- // JVM parameters for the Cluster Contoller (CC)
- public static final String CC_JAVA_OPTS = "cc.java.opts";
- public static final String CC_JAVA_OPTS_DEFAULT = "-Xmx1024m";
-
- public static final String SIZE_MEMORY_COMPONENT = "size.memory.component";
- public static final String SIZE_MEMORY_COMPONENT_DEFAULT = "512m";
-
- public static final String TOTAL_SIZE_MEMORY_COMPONENT = "total.size.memory.component";
- public static final String TOTAL_SIZE_MEMORY_COMPONENT_DEFAULT = "512m";
-
- public static final String LOG_BUFFER_NUM_PAGES = "log.buffer.num.pages";
- public static final String LOG_BUFFER_NUM_PAGES_DEFAULT = "8";
-
- public static final String LOG_BUFFER_PAGE_SIZE = "log.buffer.page.size";
- public static final String LOG_BUFFER_PAGE_SIZE_DEFAULT = "131072";
-
- public static final String LOG_PARTITION_SIZE = "log.partition.size";
- public static final String LOG_PARTITION_SIZE_DEFAULT = "2147483648";
-
- public static final String GROUP_COMMIT_INTERVAL = "group.commit.interval";
- public static final String GROUP_COMMIT_INTERVAL_DEFAULT = "200ms";
-
- public static final String SORT_OP_MEMORY = "sort.op.memory";
- public static final String SORT_OP_MEMORY_DEFAULT = "512m";
-
- public static final String JOIN_OP_MEMORY = "join.op.memory";
- public static final String JOIN_OP_MEMORY_DEFAULT = "512m";
-
- public static final String WEB_INTERFACE_PORT = "web.interface.port";
- public static final String WEB_INTERFACE_PORT_DEFAULT = "19001";
-
- public static final String NUM_PAGES_BUFFER_CACHE = "num.pages.buffer.cache";
- public static final String NUM_PAGES_BUFFER_CACHE_DEFAULT = "1000";
-
- public static final String LOG_LEVEL = "log.level";
- public static final String LOG_LEVEL_DEFAULT = "INFO";
-
- public static final String LSN_THRESHOLD = "lsn.threshold";
- public static final String LSN_THRESHOLD_DEFAULT = "64m";
-
- public static final String CHECKPOINT_TERMS_IN_SECS = "checkpoint.terms.in.secs";
- public static final String CHECKPOINT_TERMS_IN_SECS_DEFAULT = "120";
-
- public static final String ESCALATE_THRSHOLD_ENTITY_TO_DATASET = "escalate.threshold.entity.to.dataset";
- public static final String ESCALATE_THRSHOLD_ENTITY_TO_DATASET_DEFAULT = "8";
-
- public static final String SHRINK_TIMER_THRESHOLD = "shrink.timer.threshold";
- public static final String SHRINK_TIMER_THRESHOLD_DEFAULT = "120000";
-
- }
-
- private AsterixProperties() {
- try {
- String fileName = System.getProperty(GlobalConfig.CONFIG_FILE_PROPERTY);
- if (fileName == null) {
- fileName = GlobalConfig.DEFAULT_CONFIG_FILE_NAME;
- }
- InputStream is = this.getClass().getClassLoader().getResourceAsStream(fileName);
- if (is == null) {
- try {
- fileName = GlobalConfig.DEFAULT_CONFIG_FILE_NAME;
- is = new FileInputStream(fileName);
- } catch (FileNotFoundException fnf) {
- throw new AlgebricksException("Could not find the configuration file " + fileName);
- }
- }
-
- JAXBContext ctx = JAXBContext.newInstance(AsterixConfiguration.class);
- Unmarshaller unmarshaller = ctx.createUnmarshaller();
- AsterixConfiguration asterixConfiguration = (AsterixConfiguration) unmarshaller.unmarshal(is);
- metadataNodeName = asterixConfiguration.getMetadataNode();
- stores = new HashMap<String, String[]>();
- List<Store> configuredStores = asterixConfiguration.getStore();
- nodeNames = new HashSet<String>();
- for (Store store : configuredStores) {
- String trimmedStoreDirs = store.getStoreDirs().trim();
- stores.put(store.getNcId(), trimmedStoreDirs.split(","));
- nodeNames.add(store.getNcId());
- }
- asterixConfigurationParams = new HashMap<String, String>();
- for (Property p : asterixConfiguration.getProperty()) {
- asterixConfigurationParams.put(p.getName(), p.getValue());
- }
-
- initializeLogLevel(getProperty(AsterixConfigurationKeys.LOG_LEVEL));
- } catch (Exception e) {
- throw new IllegalStateException(e);
- }
- }
-
- public String getMetadataNodeName() {
- return metadataNodeName;
- }
-
- public String getMetadataStore() {
- return stores.get(metadataNodeName)[0];
- }
-
- public Map<String, String[]> getStores() {
- return stores;
- }
-
- public HashSet<String> getNodeNames() {
- return nodeNames;
- }
-
- public String getProperty(String property) {
- String propValue = asterixConfigurationParams.get(property);
- if (propValue == null) {
- switch (property) {
- case AsterixConfigurationKeys.NC_JAVA_OPTS:
- propValue = AsterixConfigurationKeys.NC_JAVA_OPTS_DEFAULT;
- break;
- case AsterixConfigurationKeys.CC_JAVA_OPTS:
- propValue = AsterixConfigurationKeys.CC_JAVA_OPTS_DEFAULT;
- break;
- case AsterixConfigurationKeys.SIZE_MEMORY_COMPONENT:
- propValue = AsterixConfigurationKeys.SIZE_MEMORY_COMPONENT_DEFAULT;
- break;
- case AsterixConfigurationKeys.TOTAL_SIZE_MEMORY_COMPONENT:
- propValue = AsterixConfigurationKeys.TOTAL_SIZE_MEMORY_COMPONENT_DEFAULT;
- break;
- case AsterixConfigurationKeys.LOG_BUFFER_NUM_PAGES:
- propValue = AsterixConfigurationKeys.LOG_BUFFER_NUM_PAGES_DEFAULT;
- break;
- case AsterixConfigurationKeys.LOG_BUFFER_PAGE_SIZE:
- propValue = AsterixConfigurationKeys.LOG_BUFFER_PAGE_SIZE_DEFAULT;
- break;
- case AsterixConfigurationKeys.LOG_PARTITION_SIZE:
- propValue = AsterixConfigurationKeys.LOG_PARTITION_SIZE_DEFAULT;
- break;
- case AsterixConfigurationKeys.GROUP_COMMIT_INTERVAL:
- propValue = AsterixConfigurationKeys.GROUP_COMMIT_INTERVAL_DEFAULT;
- break;
- case AsterixConfigurationKeys.SORT_OP_MEMORY:
- propValue = AsterixConfigurationKeys.SORT_OP_MEMORY_DEFAULT;
- break;
- case AsterixConfigurationKeys.JOIN_OP_MEMORY:
- propValue = AsterixConfigurationKeys.JOIN_OP_MEMORY_DEFAULT;
- break;
- case AsterixConfigurationKeys.WEB_INTERFACE_PORT:
- propValue = AsterixConfigurationKeys.WEB_INTERFACE_PORT_DEFAULT;
- break;
- case AsterixConfigurationKeys.NUM_PAGES_BUFFER_CACHE:
- propValue = AsterixConfigurationKeys.NUM_PAGES_BUFFER_CACHE_DEFAULT;
- break;
- case AsterixConfigurationKeys.LOG_LEVEL:
- propValue = AsterixConfigurationKeys.LOG_LEVEL_DEFAULT;
- break;
- case AsterixConfigurationKeys.LSN_THRESHOLD:
- propValue = AsterixConfigurationKeys.LSN_THRESHOLD_DEFAULT;
- break;
- case AsterixConfigurationKeys.CHECKPOINT_TERMS_IN_SECS:
- propValue = AsterixConfigurationKeys.CHECKPOINT_TERMS_IN_SECS_DEFAULT;
- break;
- case AsterixConfigurationKeys.ESCALATE_THRSHOLD_ENTITY_TO_DATASET:
- propValue = AsterixConfigurationKeys.ESCALATE_THRSHOLD_ENTITY_TO_DATASET_DEFAULT;
- break;
- case AsterixConfigurationKeys.SHRINK_TIMER_THRESHOLD:
- propValue = AsterixConfigurationKeys.SHRINK_TIMER_THRESHOLD_DEFAULT;
- break;
- }
- }
- return propValue;
- }
-
- private void initializeLogLevel(String configuredLogLevel) {
- Level level = null;
- switch (configuredLogLevel.toLowerCase()) {
- case "info":
- level = Level.INFO;
- break;
- case "fine":
- level = Level.FINE;
- break;
- case "finer":
- level = Level.FINER;
- break;
- case "finest":
- level = Level.FINEST;
- break;
- case "severe":
- level = Level.SEVERE;
- break;
- case "off":
- level = Level.OFF;
- break;
- case "warning":
- level = Level.WARNING;
- break;
- default:
- level = Level.ALL;
- }
- Logger.getLogger("edu.uci.ics").setLevel(level);
- }
-}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixPropertiesAccessor.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixPropertiesAccessor.java
new file mode 100644
index 0000000..7b2f2a6
--- /dev/null
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixPropertiesAccessor.java
@@ -0,0 +1,105 @@
+package edu.uci.ics.asterix.common.config;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Unmarshaller;
+
+import edu.uci.ics.asterix.common.configuration.AsterixConfiguration;
+import edu.uci.ics.asterix.common.configuration.Property;
+import edu.uci.ics.asterix.common.configuration.Store;
+import edu.uci.ics.asterix.common.exceptions.AsterixException;
+
+public class AsterixPropertiesAccessor {
+ private static final Logger LOGGER = Logger.getLogger(AsterixPropertiesAccessor.class.getName());
+
+ private final String metadataNodeName;
+ private final Set<String> nodeNames;
+ private final Map<String, String[]> stores;
+ private final Map<String, Property> asterixConfigurationParams;
+
+ public AsterixPropertiesAccessor() throws AsterixException {
+ String fileName = System.getProperty(GlobalConfig.CONFIG_FILE_PROPERTY);
+ if (fileName == null) {
+ fileName = GlobalConfig.DEFAULT_CONFIG_FILE_NAME;
+ }
+ InputStream is = this.getClass().getClassLoader().getResourceAsStream(fileName);
+ if (is == null) {
+ try {
+ fileName = GlobalConfig.DEFAULT_CONFIG_FILE_NAME;
+ is = new FileInputStream(fileName);
+ } catch (FileNotFoundException fnf) {
+ throw new AsterixException("Could not find configuration file " + fileName);
+ }
+ }
+
+ AsterixConfiguration asterixConfiguration = null;
+ try {
+ JAXBContext ctx = JAXBContext.newInstance(AsterixConfiguration.class);
+ Unmarshaller unmarshaller = ctx.createUnmarshaller();
+ asterixConfiguration = (AsterixConfiguration) unmarshaller.unmarshal(is);
+ } catch (JAXBException e) {
+ throw new AsterixException("Failed to read configuration file " + fileName);
+ }
+ metadataNodeName = asterixConfiguration.getMetadataNode();
+ stores = new HashMap<String, String[]>();
+ List<Store> configuredStores = asterixConfiguration.getStore();
+ nodeNames = new HashSet<String>();
+ for (Store store : configuredStores) {
+ String trimmedStoreDirs = store.getStoreDirs().trim();
+ stores.put(store.getNcId(), trimmedStoreDirs.split(","));
+ nodeNames.add(store.getNcId());
+ }
+ asterixConfigurationParams = new HashMap<String, Property>();
+ for (Property p : asterixConfiguration.getProperty()) {
+ asterixConfigurationParams.put(p.getName(), p);
+ }
+ }
+
+ public String getMetadataNodeName() {
+ return metadataNodeName;
+ }
+
+ public String getMetadataStore() {
+ return stores.get(metadataNodeName)[0];
+ }
+
+ public Map<String, String[]> getStores() {
+ return stores;
+ }
+
+ public Set<String> getNodeNames() {
+ return nodeNames;
+ }
+
+ public <T> T getProperty(String property, T defaultValue, IPropertyInterpreter<T> interpreter) {
+ Property p = asterixConfigurationParams.get(property);
+ if (p == null) {
+ return defaultValue;
+ }
+
+ try {
+ return interpreter.interpret(p);
+ } catch (IllegalArgumentException e) {
+ logConfigurationError(p, defaultValue);
+ throw e;
+ }
+ }
+
+ private <T> void logConfigurationError(Property p, T defaultValue) {
+ if (LOGGER.isLoggable(Level.SEVERE)) {
+ LOGGER.severe("Invalid property value '" + p.getValue() + "' for property '" + p.getName()
+ + "'.\n See the description: \n" + p.getDescription() + "\nDefault = " + defaultValue);
+ }
+ }
+}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixStorageProperties.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixStorageProperties.java
new file mode 100644
index 0000000..d34e4ac
--- /dev/null
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixStorageProperties.java
@@ -0,0 +1,72 @@
+package edu.uci.ics.asterix.common.config;
+
+public class AsterixStorageProperties extends AbstractAsterixProperties {
+
+ private static final String STORAGE_BUFFERCACHE_PAGESIZE_KEY = "storage.buffercache.pagesize";
+ private static int STORAGE_BUFFERCACHE_PAGESIZE_DEFAULT = (32 << 10); // 32KB
+
+ private static final String STORAGE_BUFFERCACHE_NUMPAGES_KEY = "storage.buffercache.numpages";
+ private static final int STORAGE_BUFFERCACHE_NUMPAGES_DEFAULT = 1024;
+
+ private static final String STORAGE_BUFFERCACHE_MAXOPENFILES_KEY = "storage.buffercache.maxopenfiles";
+ private static int STORAGE_BUFFERCACHE_MAXOPENFILES_DEFAULT = Integer.MAX_VALUE;
+
+ private static final String STORAGE_MEMORYCOMPONENT_PAGESIZE_KEY = "storage.memorycomponent.pagesize";
+ private static final int STORAGE_MEMORYCOMPONENT_PAGESIZE_DEFAULT = (32 << 10); // 32KB
+
+ private static final String STORAGE_MEMORYCOMPONENT_NUMPAGES_KEY = "storage.memorycomponent.numpages";
+ private static final int STORAGE_MEMORYCOMPONENT_NUMPAGES_DEFAULT = 4096; // ... so 128MB components
+
+ private static final String STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_KEY = "storage.memorycomponent.globalbudget";
+ private static final long STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_DEFAULT = (1 << 30); // 1GB
+
+ private static final String STORAGE_LSM_MERGETHRESHOLD_KEY = "storage.lsm.mergethreshold";
+ private static int STORAGE_LSM_MERGETHRESHOLD_DEFAULT = 3;
+
+ private static final String STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE_KEY = "storage.lsm.bloomfilter.falsepositiverate";
+ private static double STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE_DEFAULT = 0.01;
+
+ public AsterixStorageProperties(AsterixPropertiesAccessor accessor) {
+ super(accessor);
+ }
+
+ public int getBufferCachePageSize() {
+ return accessor.getProperty(STORAGE_BUFFERCACHE_PAGESIZE_KEY, STORAGE_BUFFERCACHE_PAGESIZE_DEFAULT,
+ PropertyInterpreters.getIntegerPropertyInterpreter());
+ }
+
+ public int getBufferCacheNumPages() {
+ return accessor.getProperty(STORAGE_BUFFERCACHE_NUMPAGES_KEY, STORAGE_BUFFERCACHE_NUMPAGES_DEFAULT,
+ PropertyInterpreters.getIntegerPropertyInterpreter());
+ }
+
+ public int getBufferCacheMaxOpenFiles() {
+ return accessor.getProperty(STORAGE_BUFFERCACHE_MAXOPENFILES_KEY, STORAGE_BUFFERCACHE_MAXOPENFILES_DEFAULT,
+ PropertyInterpreters.getIntegerPropertyInterpreter());
+ }
+
+ public int getMemoryComponentPageSize() {
+ return accessor.getProperty(STORAGE_MEMORYCOMPONENT_PAGESIZE_KEY, STORAGE_MEMORYCOMPONENT_PAGESIZE_DEFAULT,
+ PropertyInterpreters.getIntegerPropertyInterpreter());
+ }
+
+ public int getMemoryComponentNumPages() {
+ return accessor.getProperty(STORAGE_MEMORYCOMPONENT_NUMPAGES_KEY, STORAGE_MEMORYCOMPONENT_NUMPAGES_DEFAULT,
+ PropertyInterpreters.getIntegerPropertyInterpreter());
+ }
+
+ public long getMemoryComponentGlobalBudget() {
+ return accessor.getProperty(STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_KEY,
+ STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_DEFAULT, PropertyInterpreters.getLongPropertyInterpreter());
+ }
+
+ public int getLSMIndexMergeThreshold() {
+ return accessor.getProperty(STORAGE_LSM_MERGETHRESHOLD_KEY, STORAGE_LSM_MERGETHRESHOLD_DEFAULT,
+ PropertyInterpreters.getIntegerPropertyInterpreter());
+ }
+
+ public double getBloomFilterFalsePositiveRate() {
+ return accessor.getProperty(STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE_KEY,
+ STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE_DEFAULT, PropertyInterpreters.getDoublePropertyInterpreter());
+ }
+}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixTransactionProperties.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixTransactionProperties.java
new file mode 100644
index 0000000..d97e53b
--- /dev/null
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixTransactionProperties.java
@@ -0,0 +1,72 @@
+package edu.uci.ics.asterix.common.config;
+
+public class AsterixTransactionProperties extends AbstractAsterixProperties {
+ private static final String TXN_LOG_BUFFER_NUMPAGES_KEY = "txn.log.buffer.numpages";
+ private static int TXN_LOG_BUFFER_NUMPAGES_DEFAULT = 8;
+
+ private static final String TXN_LOG_BUFFER_PAGESIZE_KEY = "txn.log.buffer.pagesize";
+ private static final int TXN_LOG_BUFFER_PAGESIZE_DEFAULT = (128 << 10); // 128KB
+
+ private static final String TXN_LOG_PARTITIONSIZE_KEY = "txn.log.partitionsize";
+ private static final long TXN_LOG_PARTITIONSIZE_DEFAULT = (2 << 30); // 2GB
+
+ private static final String TXN_LOG_GROUPCOMMITINTERVAL_KEY = "txn.log.groupcommitinterval";
+ private static int TXN_LOG_GROUPCOMMITINTERVAL_DEFAULT = 200; // 200ms
+
+ private static final String TXN_LOG_CHECKPOINT_LSNTHRESHOLD_KEY = "txn.log.checkpoint.lsnthreshold";
+ private static final int TXN_LOG_CHECKPOINT_LSNTHRESHOLD_DEFAULT = (64 << 20); // 64M
+
+ private static final String TXN_LOG_CHECKPOINT_POLLFREQUENCY_KEY = "txn.log.checkpoint.pollfrequency";
+ private static int TXN_LOG_CHECKPOINT_POLLFREQUENCY_DEFAULT = 120; // 120s
+
+ private static final String TXN_LOCK_ESCALATIONTHRESHOLD_KEY = "txn.lock.escalationthreshold";
+ private static int TXN_LOCK_ESCALATIONTHRESHOLD_DEFAULT = 1000;
+
+ private static final String TXN_LOCK_SHRINKTIMER_KEY = "txn.lock.shrinktimer";
+ private static int TXN_LOCK_SHRINKTIMER_DEFAULT = 120000; // 2m
+
+ public AsterixTransactionProperties(AsterixPropertiesAccessor accessor) {
+ super(accessor);
+ }
+
+ public int getLogBufferNumPages() {
+ return accessor.getProperty(TXN_LOG_BUFFER_NUMPAGES_KEY, TXN_LOG_BUFFER_NUMPAGES_DEFAULT,
+ PropertyInterpreters.getIntegerPropertyInterpreter());
+ }
+
+ public int getLogBufferPageSize() {
+ return accessor.getProperty(TXN_LOG_BUFFER_PAGESIZE_KEY, TXN_LOG_BUFFER_PAGESIZE_DEFAULT,
+ PropertyInterpreters.getIntegerPropertyInterpreter());
+ }
+
+ public long getLogPartitionSize() {
+ return accessor.getProperty(TXN_LOG_PARTITIONSIZE_KEY, TXN_LOG_PARTITIONSIZE_DEFAULT,
+ PropertyInterpreters.getLongPropertyInterpreter());
+ }
+
+ public int getGroupCommitInterval() {
+ return accessor.getProperty(TXN_LOG_GROUPCOMMITINTERVAL_KEY, TXN_LOG_GROUPCOMMITINTERVAL_DEFAULT,
+ PropertyInterpreters.getIntegerPropertyInterpreter());
+ }
+
+ public int getCheckpointLSNThreshold() {
+ return accessor.getProperty(TXN_LOG_CHECKPOINT_LSNTHRESHOLD_KEY, TXN_LOG_CHECKPOINT_LSNTHRESHOLD_DEFAULT,
+ PropertyInterpreters.getIntegerPropertyInterpreter());
+ }
+
+ public int getCheckpointPollFrequency() {
+ return accessor.getProperty(TXN_LOG_CHECKPOINT_POLLFREQUENCY_KEY, TXN_LOG_CHECKPOINT_POLLFREQUENCY_DEFAULT,
+ PropertyInterpreters.getIntegerPropertyInterpreter());
+ }
+
+ public int getEntityToDatasetLockEscalationThreshold() {
+ return accessor.getProperty(TXN_LOCK_ESCALATIONTHRESHOLD_KEY, TXN_LOCK_ESCALATIONTHRESHOLD_DEFAULT,
+ PropertyInterpreters.getIntegerPropertyInterpreter());
+ }
+
+ public int getLockManagerShrinkTimer() {
+ return accessor.getProperty(TXN_LOCK_SHRINKTIMER_KEY, TXN_LOCK_SHRINKTIMER_DEFAULT,
+ PropertyInterpreters.getIntegerPropertyInterpreter());
+ }
+
+}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/GlobalConfig.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/GlobalConfig.java
index ebce6fe..3948e69 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/GlobalConfig.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/GlobalConfig.java
@@ -3,61 +3,43 @@
import java.util.logging.Logger;
public class GlobalConfig {
- public static final boolean DEBUG = true;
+ public static final boolean DEBUG = true;
- public static final String ASTERIX_LOGGER_NAME = "edu.uci.ics.asterix";
+ public static final String ASTERIX_LOGGER_NAME = "edu.uci.ics.asterix";
- public static final Logger ASTERIX_LOGGER = Logger
- .getLogger(ASTERIX_LOGGER_NAME);
+ public static final Logger ASTERIX_LOGGER = Logger.getLogger(ASTERIX_LOGGER_NAME);
- public static final String ASTERIX_LOGFILE_PATTERN = "%t/asterix.log";
+ public static final String DEFAULT_CONFIG_FILE_NAME = "asterix-configuration.xml";
- public static final String DEFAULT_CONFIG_FILE_NAME = "asterix-configuration.xml";
+ public static final String CONFIG_FILE_PROPERTY = "AsterixConfigFileName";
- public static final String TEST_CONFIG_FILE_NAME = "src/main/resources/asterix-configuration.xml";
+ public static final String WEB_SERVER_PORT_PROPERTY = "AsterixWebServerPort";
- public static final String CONFIG_FILE_PROPERTY = "AsterixConfigFileName";
+ public static final String JSON_API_SERVER_PORT_PROPERTY = "AsterixJSONAPIServerPort";
- public static final String WEB_SERVER_PORT_PROPERTY = "AsterixWebServerPort";
+ public static final int DEFAULT_FRAME_SIZE = 32768;
- public static final String JSON_API_SERVER_PORT_PROPERTY = "AsterixJSONAPIServerPort";
+ public static final String FRAME_SIZE_PROPERTY = "FrameSize";
- public static final String BUFFER_CACHE_PAGE_SIZE_PROPERTY = "BufferCachePageSize";
+ public static final float DEFAULT_BTREE_FILL_FACTOR = 1.00f;
- public static final String BUFFER_CACHE_NUM_PAGES_PROPERTY = "BufferCacheNumPages";
+ public static int DEFAULT_INPUT_DATA_COLUMN = 0;
- public static final int DEFAULT_BUFFER_CACHE_NUM_PAGES = 4096;
-
- public static final int DEFAULT_FRAME_SIZE = 32768;
-
- public static final String FRAME_SIZE_PROPERTY = "FrameSize";
-
- public static final float DEFAULT_BTREE_FILL_FACTOR = 1.00f;
-
- public static int DEFAULT_INPUT_DATA_COLUMN = 0;
-
- public static int DEFAULT_INDEX_MEM_PAGE_SIZE = 32768;
-
- public static int DEFAULT_INDEX_MEM_NUM_PAGES = 1000;
-
- public static int getFrameSize() {
- int frameSize = GlobalConfig.DEFAULT_FRAME_SIZE;
- String frameSizeStr = System
- .getProperty(GlobalConfig.FRAME_SIZE_PROPERTY);
- if (frameSizeStr != null) {
- int fz = -1;
- try {
- fz = Integer.parseInt(frameSizeStr);
- } catch (NumberFormatException nfe) {
- GlobalConfig.ASTERIX_LOGGER
- .warning("Wrong frame size size argument. Picking default value ("
- + GlobalConfig.DEFAULT_FRAME_SIZE
- + ") instead.\n");
- }
- if (fz >= 0) {
- frameSize = fz;
- }
- }
- return frameSize;
- }
+ public static int getFrameSize() {
+ int frameSize = GlobalConfig.DEFAULT_FRAME_SIZE;
+ String frameSizeStr = System.getProperty(GlobalConfig.FRAME_SIZE_PROPERTY);
+ if (frameSizeStr != null) {
+ int fz = -1;
+ try {
+ fz = Integer.parseInt(frameSizeStr);
+ } catch (NumberFormatException nfe) {
+ GlobalConfig.ASTERIX_LOGGER.warning("Wrong frame size size argument. Picking default value ("
+ + GlobalConfig.DEFAULT_FRAME_SIZE + ") instead.\n");
+ }
+ if (fz >= 0) {
+ frameSize = fz;
+ }
+ }
+ return frameSize;
+ }
}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/IAsterixPropertiesProvider.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/IAsterixPropertiesProvider.java
new file mode 100644
index 0000000..e42c827
--- /dev/null
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/IAsterixPropertiesProvider.java
@@ -0,0 +1,14 @@
+package edu.uci.ics.asterix.common.config;
+
+
+public interface IAsterixPropertiesProvider {
+ public AsterixStorageProperties getStorageProperties();
+
+ public AsterixTransactionProperties getTransactionProperties();
+
+ public AsterixCompilerProperties getCompilerProperties();
+
+ public AsterixMetadataProperties getMetadataProperties();
+
+ public AsterixExternalProperties getExternalProperties();
+}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/IPropertyInterpreter.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/IPropertyInterpreter.java
new file mode 100644
index 0000000..2cabbe5
--- /dev/null
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/IPropertyInterpreter.java
@@ -0,0 +1,7 @@
+package edu.uci.ics.asterix.common.config;
+
+import edu.uci.ics.asterix.common.configuration.Property;
+
+public interface IPropertyInterpreter<T> {
+ public T interpret(Property p) throws IllegalArgumentException;
+}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/PropertyInterpreters.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/PropertyInterpreters.java
new file mode 100644
index 0000000..9ff5dee
--- /dev/null
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/PropertyInterpreters.java
@@ -0,0 +1,71 @@
+package edu.uci.ics.asterix.common.config;
+
+import java.util.logging.Level;
+
+import edu.uci.ics.asterix.common.configuration.Property;
+
+public class PropertyInterpreters {
+
+ public static IPropertyInterpreter<Integer> getIntegerPropertyInterpreter() {
+ return new IPropertyInterpreter<Integer>() {
+
+ @Override
+ public Integer interpret(Property p) throws IllegalArgumentException {
+ try {
+ return Integer.parseInt(p.getValue());
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+ };
+ }
+
+ public static IPropertyInterpreter<Long> getLongPropertyInterpreter() {
+ return new IPropertyInterpreter<Long>() {
+
+ @Override
+ public Long interpret(Property p) throws IllegalArgumentException {
+ try {
+ return Long.parseLong(p.getValue());
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+ };
+ }
+
+ public static IPropertyInterpreter<Level> getLevelPropertyInterpreter() {
+ return new IPropertyInterpreter<Level>() {
+
+ @Override
+ public Level interpret(Property p) throws IllegalArgumentException {
+ return Level.parse(p.getValue());
+ }
+ };
+ }
+
+ public static IPropertyInterpreter<String> getStringPropertyInterpreter() {
+ return new IPropertyInterpreter<String>() {
+
+ @Override
+ public String interpret(Property p) throws IllegalArgumentException {
+ return p.getValue();
+ }
+ };
+ }
+
+ public static IPropertyInterpreter<Double> getDoublePropertyInterpreter() {
+ return new IPropertyInterpreter<Double>() {
+
+ @Override
+ public Double interpret(Property p) throws IllegalArgumentException {
+ try {
+ return Double.parseDouble(p.getValue());
+ } catch (NumberFormatException e) {
+ throw new IllegalArgumentException(e);
+ }
+ }
+ };
+ }
+
+}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/AsterixAppRuntimeContext.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/AsterixAppRuntimeContext.java
index 5070938..03e84ef 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/AsterixAppRuntimeContext.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/AsterixAppRuntimeContext.java
@@ -1,9 +1,16 @@
package edu.uci.ics.asterix.common.context;
import java.io.IOException;
-import java.util.logging.Level;
+import java.util.logging.Logger;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.config.AsterixCompilerProperties;
+import edu.uci.ics.asterix.common.config.AsterixExternalProperties;
+import edu.uci.ics.asterix.common.config.AsterixMetadataProperties;
+import edu.uci.ics.asterix.common.config.AsterixPropertiesAccessor;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
+import edu.uci.ics.asterix.common.config.AsterixTransactionProperties;
+import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
+import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.transaction.management.exception.ACIDException;
import edu.uci.ics.asterix.transaction.management.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
import edu.uci.ics.asterix.transaction.management.ioopcallbacks.LSMInvertedIndexIOOperationCallbackFactory;
@@ -39,12 +46,15 @@
import edu.uci.ics.hyracks.storage.common.file.ResourceIdFactory;
import edu.uci.ics.hyracks.storage.common.file.ResourceIdFactoryProvider;
-public class AsterixAppRuntimeContext {
- private static final int DEFAULT_BUFFER_CACHE_PAGE_SIZE = 32768;
- private static final int DEFAULT_LIFECYCLEMANAGER_MEMORY_BUDGET = 1024 * 1024 * 1024; // 1GB
- private static final int DEFAULT_MAX_OPEN_FILES = Integer.MAX_VALUE;
+public class AsterixAppRuntimeContext implements IAsterixPropertiesProvider {
private final INCApplicationContext ncApplicationContext;
+ private AsterixCompilerProperties compilerProperties;
+ private AsterixExternalProperties externalProperties;
+ private AsterixMetadataProperties metadataProperties;
+ private AsterixStorageProperties storageProperties;
+ private AsterixTransactionProperties txnProperties;
+
private IIndexLifecycleManager indexLifecycleManager;
private IFileMapManager fileMapManager;
private IBufferCache bufferCache;
@@ -64,24 +74,29 @@
this.ncApplicationContext = ncApplicationContext;
}
- public void initialize() throws IOException, ACIDException {
- int pageSize = getBufferCachePageSize();
- int numPages = getBufferCacheNumPages();
+ public void initialize() throws IOException, ACIDException, AsterixException {
+ AsterixPropertiesAccessor propertiesAccessor = new AsterixPropertiesAccessor();
+ compilerProperties = new AsterixCompilerProperties(propertiesAccessor);
+ externalProperties = new AsterixExternalProperties(propertiesAccessor);
+ metadataProperties = new AsterixMetadataProperties(propertiesAccessor);
+ storageProperties = new AsterixStorageProperties(propertiesAccessor);
+ txnProperties = new AsterixTransactionProperties(propertiesAccessor);
+
+ Logger.getLogger("edu.uci.ics").setLevel(externalProperties.getLogLevel());
fileMapManager = new AsterixFileMapManager();
ICacheMemoryAllocator allocator = new HeapBufferAllocator();
IPageReplacementStrategy prs = new ClockPageReplacementStrategy();
- ioManager = ncApplicationContext.getRootContext().getIOManager();
- indexLifecycleManager = new IndexLifecycleManager(DEFAULT_LIFECYCLEMANAGER_MEMORY_BUDGET);
- IAsterixAppRuntimeContextProvider asterixAppRuntimeContextProvider = new AsterixAppRuntimeContextProviderForRecovery(
- this);
- txnSubsystem = new TransactionSubsystem(ncApplicationContext.getNodeId(), asterixAppRuntimeContextProvider);
IPageCleanerPolicy pcp = new DelayPageCleanerPolicy(600000);
- bufferCache = new BufferCache(ioManager, allocator, prs, pcp, fileMapManager, pageSize, numPages,
- DEFAULT_MAX_OPEN_FILES);
+ ioManager = ncApplicationContext.getRootContext().getIOManager();
+ bufferCache = new BufferCache(ioManager, allocator, prs, pcp, fileMapManager,
+ storageProperties.getBufferCachePageSize(), storageProperties.getBufferCacheNumPages(),
+ storageProperties.getBufferCacheMaxOpenFiles());
+
+ indexLifecycleManager = new IndexLifecycleManager(storageProperties.getMemoryComponentGlobalBudget());
lsmIOScheduler = SynchronousScheduler.INSTANCE;
- mergePolicy = new ConstantMergePolicy(3, this);
+ mergePolicy = new ConstantMergePolicy(storageProperties.getLSMIndexMergeThreshold(), this);
lsmBTreeOpTrackerFactory = new IndexOperationTrackerFactory(LSMBTreeIOOperationCallbackFactory.INSTANCE);
lsmRTreeOpTrackerFactory = new IndexOperationTrackerFactory(LSMRTreeIOOperationCallbackFactory.INSTANCE);
lsmInvertedIndexOpTrackerFactory = new IndexOperationTrackerFactory(
@@ -92,6 +107,10 @@
localResourceRepository = (PersistentLocalResourceRepository) persistentLocalResourceRepositoryFactory
.createRepository();
resourceIdFactory = (new ResourceIdFactoryProvider(localResourceRepository)).createResourceIdFactory();
+
+ IAsterixAppRuntimeContextProvider asterixAppRuntimeContextProvider = new AsterixAppRuntimeContextProviderForRecovery(
+ this);
+ txnSubsystem = new TransactionSubsystem(ncApplicationContext.getNodeId(), asterixAppRuntimeContextProvider);
isShuttingdown = false;
}
@@ -103,49 +122,6 @@
this.isShuttingdown = isShuttingdown;
}
- private int getBufferCachePageSize() {
- int pageSize = DEFAULT_BUFFER_CACHE_PAGE_SIZE;
- String pageSizeStr = System.getProperty(GlobalConfig.BUFFER_CACHE_PAGE_SIZE_PROPERTY, null);
- if (pageSizeStr != null) {
- try {
- pageSize = Integer.parseInt(pageSizeStr);
- } catch (NumberFormatException nfe) {
- if (GlobalConfig.ASTERIX_LOGGER.isLoggable(Level.WARNING)) {
- GlobalConfig.ASTERIX_LOGGER.warning("Wrong buffer cache page size argument. "
- + "Using default value: " + pageSize);
- }
- }
- }
-
- if (GlobalConfig.ASTERIX_LOGGER.isLoggable(Level.INFO)) {
- GlobalConfig.ASTERIX_LOGGER.info("Buffer cache page size: " + pageSize);
- }
-
- return pageSize;
- }
-
- private int getBufferCacheNumPages() {
- int numPages = GlobalConfig.DEFAULT_BUFFER_CACHE_NUM_PAGES;
- String numPagesStr = System.getProperty(GlobalConfig.BUFFER_CACHE_NUM_PAGES_PROPERTY, null);
- if (numPagesStr != null) {
- try {
- numPages = Integer.parseInt(numPagesStr);
- } catch (NumberFormatException nfe) {
- if (GlobalConfig.ASTERIX_LOGGER.isLoggable(Level.WARNING)) {
- GlobalConfig.ASTERIX_LOGGER.warning("Wrong buffer cache size argument. " + "Using default value: "
- + numPages);
- }
- return numPages;
- }
- }
-
- if (GlobalConfig.ASTERIX_LOGGER.isLoggable(Level.INFO)) {
- GlobalConfig.ASTERIX_LOGGER.info("Buffer cache size (number of pages): " + numPages);
- }
-
- return numPages;
- }
-
public void deinitialize() throws HyracksDataException {
bufferCache.close();
for (IIndex index : indexLifecycleManager.getOpenIndexes()) {
@@ -216,4 +192,29 @@
public IIOManager getIOManager() {
return ioManager;
}
+
+ @Override
+ public AsterixStorageProperties getStorageProperties() {
+ return storageProperties;
+ }
+
+ @Override
+ public AsterixTransactionProperties getTransactionProperties() {
+ return txnProperties;
+ }
+
+ @Override
+ public AsterixCompilerProperties getCompilerProperties() {
+ return compilerProperties;
+ }
+
+ @Override
+ public AsterixMetadataProperties getMetadataProperties() {
+ return metadataProperties;
+ }
+
+ @Override
+ public AsterixExternalProperties getExternalProperties() {
+ return externalProperties;
+ }
}
\ No newline at end of file
diff --git a/asterix-doc/pom.xml b/asterix-doc/pom.xml
new file mode 100644
index 0000000..f44c06d
--- /dev/null
+++ b/asterix-doc/pom.xml
@@ -0,0 +1,18 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <artifactId>asterix</artifactId>
+ <groupId>edu.uci.ics.asterix</groupId>
+ <version>0.0.6-SNAPSHOT</version>
+ </parent>
+ <artifactId>asterix-doc</artifactId>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-site-plugin</artifactId>
+ <version>3.3</version>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git a/asterix-doc/src/site/markdown/aql.md b/asterix-doc/src/site/markdown/aql.md
new file mode 100644
index 0000000..c03d7e5
--- /dev/null
+++ b/asterix-doc/src/site/markdown/aql.md
@@ -0,0 +1,191 @@
+# The Asterix Query Language, Version 1.0
+## 1. Introduction
+
+This document provides an overview of the Asterix Query language.
+
+## 2. Expressions
+
+ Expression ::= ( OperatorExpr | IfThenElse | FLWOGR | QuantifiedExpression )
+
+### Primary Expressions
+
+ PrimaryExpr ::= Literal | VariableRef | ParenthesizedExpression | FunctionCallExpr
+ | DatasetAccessExpression | ListConstructor | RecordConstructor
+
+#### Literals
+
+ Literal ::= StringLiteral | <INTEGER_LITERAL> | <FLOAT_LITERAL> | <DOUBLE_LITERAL> | <NULL> | <TRUE> | <FALSE>
+ StringLiteral ::= <STRING_LITERAL>
+
+#### Variable References
+
+ VariableRef ::= <VARIABLE>
+
+#### Parenthesized Expressions
+
+ ParenthesizedExpression ::= <LEFTPAREN> Expression <RIGHTPAREN>
+
+#### Function Calls
+
+ FunctionCallExpr ::= FunctionOrTypeName <LEFTPAREN> ( Expression ( "," Expression )* )? <RIGHTPAREN>
+
+#### Dataset Access
+
+ DatasetAccessExpression ::= <DATASET> ( ( Identifier ( "." Identifier )? )
+ | ( <LEFTPAREN> Expression ( "," Expression )* <RIGHTPAREN> ) )
+ Identifier ::= <IDENTIFIER> | StringLiteral
+
+#### Constructors
+
+ ListConstructor ::= ( OrderedListConstructor | UnorderedListConstructor )
+ OrderedListConstructor ::= "[" ( Expression ( "," Expression )* )? "]"
+ UnorderedListConstructor ::= "{{" ( Expression ( "," Expression )* )? "}}"
+ RecordConstructor ::= "{" ( FieldBinding ( "," FieldBinding )* )? "}"
+ FieldBinding ::= Expression ":" Expression
+
+### Path Expressions
+
+ ValueExpr ::= PrimaryExpr ( Field | Index )*
+ Field ::= "." Identifier
+ Index ::= "[" ( Expression | "?" ) "]"
+
+### Logical Expressions
+
+ OperatorExpr ::= AndExpr ( "or" AndExpr )*
+ AndExpr ::= RelExpr ( "and" RelExpr )*
+
+### Comparison Expressions
+
+ RelExpr ::= AddExpr ( ( "<" | ">" | "<=" | ">=" | "=" | "!=" | "~=" ) AddExpr )?
+
+### Arithmetic Expressions
+
+ AddExpr ::= MultExpr ( ( "+" | "-" ) MultExpr )*
+ MultExpr ::= UnaryExpr ( ( "*" | "/" | "%" | <CARET> | "idiv" ) UnaryExpr )*
+ UnaryExpr ::= ( ( "+" | "-" ) )? ValueExpr
+
+### FLWOGR Expression
+
+ FLWOGR ::= ( ForClause | LetClause ) ( Clause )* "return" Expression
+ Clause ::= ForClause | LetClause | WhereClause | OrderbyClause
+ | GroupClause | LimitClause | DistinctClause
+ ForClause ::= "for" Variable ( "at" Variable )? "in" ( Expression )
+ LetClause ::= "let" Variable ":=" Expression
+ WhereClause ::= "where" Expression
+ OrderbyClause ::= "order" "by" Expression ( ( "asc" ) | ( "desc" ) )?
+ ( "," Expression ( ( "asc" ) | ( "desc" ) )? )*
+ GroupClause ::= "group" "by" ( Variable ":=" )? Expression ( "," ( Variable ":=" )? Expression )*
+ "with" VariableRef ( "," VariableRef )*
+ LimitClause ::= "limit" Expression ( "offset" Expression )?
+ DistinctClause ::= "distinct" "by" Expression ( "," Expression )*
+ Variable ::= <VARIABLE>
+
+
+### Conditional Expression
+
+ IfThenElse ::= "if" <LEFTPAREN> Expression <RIGHTPAREN> "then" Expression "else" Expression
+
+
+### Quantified Expressions
+
+ QuantifiedExpression ::= ( ( "some" ) | ( "every" ) ) Variable "in" Expression
+ ( "," Variable "in" Expression )* "satisfies" Expression
+
+
+## 3. Statements
+
+ Statement ::= ( SingleStatement ( ";" )? )* <EOF>
+ SingleStatement ::= DataverseDeclaration
+ | FunctionDeclaration
+ | CreateStatement
+ | DropStatement
+ | LoadStatement
+ | SetStatement
+ | InsertStatement
+ | DeleteStatement
+ | FeedStatement
+ | Query
+
+### Declarations
+
+ DataverseDeclaration ::= "use" "dataverse" Identifier
+ SetStatement ::= "set" Identifier StringLiteral
+ FunctionDeclaration ::= "declare" "function" Identifier <LEFTPAREN> ( <VARIABLE> ( "," <VARIABLE> )* )? <RIGHTPAREN> "{" Expression "}"
+
+### Lifecycle Management Statements
+
+ CreateStatement ::= "create" ( TypeSpecification | DatasetSpecification | IndexSpecification | DataverseSpecification | FunctionSpecification )
+
+ DropStatement ::= "drop" ( <DATASET> QualifiedName IfExists
+ | "index" DoubleQualifiedName IfExists
+ | "type" FunctionOrTypeName IfExists
+ | "dataverse" Identifier IfExists
+ | "function" FunctionSignature IfExists )
+ IfExists ::= ( "if" "exists" )?
+ QualifiedName ::= Identifier ( "." Identifier )?
+ DoubleQualifiedName ::= Identifier "." Identifier ( "." Identifier )?
+
+#### Types
+
+ TypeSpecification ::= "type" FunctionOrTypeName IfNotExists "as" TypeExpr
+ FunctionOrTypeName ::= QualifiedName
+ IfNotExists ::= ( "if not exists" )?
+ TypeExpr ::= RecordTypeDef | TypeReference | OrderedListTypeDef | UnorderedListTypeDef
+ RecordTypeDef ::= ( "closed" | "open" )? "{" ( RecordField ( "," RecordField )* )? "}"
+ RecordField ::= Identifier ":" ( TypeExpr ) ( "?" )?
+ TypeReference ::= Identifier
+ OrderedListTypeDef ::= "[" ( TypeExpr ) "]"
+ UnorderedListTypeDef ::= "{{" ( TypeExpr ) "}}"
+
+#### Datasets
+
+ DatasetSpecification ::= "external" <DATASET> QualifiedName <LEFTPAREN> Identifier <RIGHTPAREN> IfNotExists
+ "using" AdapterName Configuration ( "hints" Properties )?
+ | "feed" <DATASET> QualifiedName <LEFTPAREN> Identifier <RIGHTPAREN> IfNotExists
+ "using" AdapterName Configuration ( ApplyFunction )? PrimaryKey ( "on" Identifier )? ( "hints" Properties )?
+ | <DATASET> QualifiedName <LEFTPAREN> Identifier <RIGHTPAREN> IfNotExists
+ PrimaryKey ( "on" Identifier )? ( "hints" Properties )?
+ AdapterName ::= Identifier
+ Configuration ::= <LEFTPAREN> ( KeyValuePair ( "," KeyValuePair )* )? <RIGHTPAREN>
+ KeyValuePair ::= <LEFTPAREN> StringLiteral "=" StringLiteral <RIGHTPAREN>
+ Properties ::= ( <LEFTPAREN> Property ( "," Property )* <RIGHTPAREN> )?
+ Property ::= Identifier "=" ( StringLiteral | <INTEGER_LITERAL> )
+ ApplyFunction ::= "apply" "function" FunctionSignature
+ FunctionSignature ::= FunctionOrTypeName "@" <INTEGER_LITERAL>
+ PrimaryKey ::= "primary" "key" Identifier ( "," Identifier )*
+
+#### Indices
+
+ IndexSpecification ::= "index" Identifier IfNotExists "on" QualifiedName <LEFTPAREN> ( Identifier ) ( "," Identifier )* <RIGHTPAREN> ( "type" IndexType )?
+ IndexType ::= "btree" | "rtree" | "keyword" | "fuzzy keyword" | "ngram" <LEFTPAREN> <INTEGER_LITERAL> <RIGHTPAREN> | "fuzzy ngram" <LEFTPAREN> <INTEGER_LITERAL> <RIGHTPAREN>
+
+#### Dataverses
+
+ DataverseSpecification ::= "dataverse" Identifier IfNotExists ( "with format" StringLiteral )?
+
+#### Functions
+
+ FunctionSpecification ::= "function" FunctionOrTypeName IfNotExists <LEFTPAREN> ( <VARIABLE> ( "," <VARIABLE> )* )? <RIGHTPAREN> "{" Expression "}"
+
+
+### Import/Export Statements
+
+ LoadStatement ::= "load" <DATASET> QualifiedName "using" AdapterName Configuration ( "pre-sorted" )?
+
+### Modification Statements
+
+ InsertStatement ::= "insert" "into" <DATASET> QualifiedName Query
+ DeleteStatement ::= "delete" Variable "from" <DATASET> QualifiedName ( "where" Expression )?
+
+### Feed Management Statements
+
+ FeedStatement ::= "begin" "feed" QualifiedName
+ | "suspend" "feed" QualifiedName
+ | "resume" "feed" QualifiedName
+ | "end" "feed" QualifiedName
+ | "alter" "feed" QualifiedName "set" Configuration
+
+### Queries
+
+ Query ::= Expression
+
diff --git a/asterix-events/pom.xml b/asterix-events/pom.xml
index 8440b6d..94042e1 100644
--- a/asterix-events/pom.xml
+++ b/asterix-events/pom.xml
@@ -166,12 +166,12 @@
<artifactId>commons-io</artifactId>
<version>1.4</version>
</dependency>
- <dependency>
+ <dependency>
<groupId>edu.uci.ics.asterix</groupId>
<artifactId>asterix-common</artifactId>
<version>0.0.6-SNAPSHOT</version>
<type>jar</type>
<scope>compile</scope>
- </dependency>
+ </dependency>
</dependencies>
</project>
diff --git a/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/EventExecutor.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/EventExecutor.java
index fa3923b..d0c7ae0 100644
--- a/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/EventExecutor.java
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/EventExecutor.java
@@ -54,7 +54,7 @@
if (p.getKey().equals("JAVA_HOME")) {
String val = node.getJavaHome() == null ? p.getValue() : node.getJavaHome();
envBuffer.append(p.getKey() + "=" + val + " ");
- } else if (p.getKey().equals(AsterixProperties.AsterixConfigurationKeys.NC_JAVA_OPTS)) {
+ } else if (p.getKey().equals(EventUtil.NC_JAVA_OPTS)) {
if (!isMasterNode) {
StringBuilder builder = new StringBuilder();
builder.append("\"");
@@ -65,7 +65,7 @@
builder.append("\"");
envBuffer.append("JAVA_OPTS" + "=" + builder + " ");
}
- } else if (p.getKey().equals(AsterixProperties.AsterixConfigurationKeys.CC_JAVA_OPTS)) {
+ } else if (p.getKey().equals(EventUtil.CC_JAVA_OPTS)) {
if (isMasterNode) {
StringBuilder builder = new StringBuilder();
builder.append("\"");
diff --git a/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/EventUtil.java b/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/EventUtil.java
index 7cbb515..533b2a4 100644
--- a/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/EventUtil.java
+++ b/asterix-events/src/main/java/edu/uci/ics/asterix/event/management/EventUtil.java
@@ -36,230 +36,252 @@
public class EventUtil {
- public static final String EVENTS_DIR = "events";
- public static final String CLUSTER_CONF = "config/cluster.xml";
- public static final String PATTERN_CONF = "config/pattern.xml";
- public static final DateFormat dateFormat = new SimpleDateFormat("yyyy/MM/dd HH:mm:ss");
+ public static final String EVENTS_DIR = "events";
+ public static final String CLUSTER_CONF = "config/cluster.xml";
+ public static final String PATTERN_CONF = "config/pattern.xml";
+ public static final DateFormat dateFormat = new SimpleDateFormat(
+ "yyyy/MM/dd HH:mm:ss");
+ public static final String NC_JAVA_OPTS = "nc.java.opts";
+ public static final String CC_JAVA_OPTS = "cc.java.opts";
- private static final String IP_LOCATION = "IP_LOCATION";
- private static final String CLUSTER_ENV = "ENV";
- private static final String SCRIPT = "SCRIPT";
- private static final String ARGS = "ARGS";
- private static final String EXECUTE_SCRIPT = "events/execute.sh";
- private static final String LOCALHOST = "localhost";
- private static final String LOCALHOST_IP = "127.0.0.1";
+ private static final String IP_LOCATION = "IP_LOCATION";
+ private static final String CLUSTER_ENV = "ENV";
+ private static final String SCRIPT = "SCRIPT";
+ private static final String ARGS = "ARGS";
+ private static final String EXECUTE_SCRIPT = "events/execute.sh";
+ private static final String LOCALHOST = "localhost";
+ private static final String LOCALHOST_IP = "127.0.0.1";
- public static Cluster getCluster(String clusterConfigurationPath) throws JAXBException {
- File file = new File(clusterConfigurationPath);
- JAXBContext ctx = JAXBContext.newInstance(Cluster.class);
- Unmarshaller unmarshaller = ctx.createUnmarshaller();
- Cluster cluster = (Cluster) unmarshaller.unmarshal(file);
- if (cluster.getMasterNode().getClusterIp().equals(LOCALHOST)) {
- cluster.getMasterNode().setClusterIp(LOCALHOST_IP);
- }
- for (Node node : cluster.getNode()) {
- if (node.getClusterIp().equals(LOCALHOST)) {
- node.setClusterIp(LOCALHOST_IP);
- }
- }
- return cluster;
- }
+ public static Cluster getCluster(String clusterConfigurationPath)
+ throws JAXBException {
+ File file = new File(clusterConfigurationPath);
+ JAXBContext ctx = JAXBContext.newInstance(Cluster.class);
+ Unmarshaller unmarshaller = ctx.createUnmarshaller();
+ Cluster cluster = (Cluster) unmarshaller.unmarshal(file);
+ if (cluster.getMasterNode().getClusterIp().equals(LOCALHOST)) {
+ cluster.getMasterNode().setClusterIp(LOCALHOST_IP);
+ }
+ for (Node node : cluster.getNode()) {
+ if (node.getClusterIp().equals(LOCALHOST)) {
+ node.setClusterIp(LOCALHOST_IP);
+ }
+ }
+ return cluster;
+ }
- public static long parseTimeInterval(ValueType v, String unit) throws IllegalArgumentException {
- int val = 0;
- switch (v.getType()) {
- case ABS:
- val = Integer.parseInt(v.getAbsoluteValue());
- break;
- case RANDOM_MIN_MAX:
- val = Randomizer.getInstance().getRandomInt(v.getMin(), v.getMax());
- break;
- case RANDOM_RANGE:
- String[] values = v.getRangeSet();
- val = Integer.parseInt(values[Randomizer.getInstance().getRandomInt(0, values.length - 1)]);
- break;
- }
- return computeInterval(val, unit);
- }
+ public static long parseTimeInterval(ValueType v, String unit)
+ throws IllegalArgumentException {
+ int val = 0;
+ switch (v.getType()) {
+ case ABS:
+ val = Integer.parseInt(v.getAbsoluteValue());
+ break;
+ case RANDOM_MIN_MAX:
+ val = Randomizer.getInstance().getRandomInt(v.getMin(), v.getMax());
+ break;
+ case RANDOM_RANGE:
+ String[] values = v.getRangeSet();
+ val = Integer.parseInt(values[Randomizer.getInstance()
+ .getRandomInt(0, values.length - 1)]);
+ break;
+ }
+ return computeInterval(val, unit);
+ }
- public static long parseTimeInterval(String v, String unit) throws IllegalArgumentException {
- int value = Integer.parseInt(v);
- return computeInterval(value, unit);
- }
+ public static long parseTimeInterval(String v, String unit)
+ throws IllegalArgumentException {
+ int value = Integer.parseInt(v);
+ return computeInterval(value, unit);
+ }
- private static long computeInterval(int val, String unit) {
- int vmult = 1;
- if ("hr".equalsIgnoreCase(unit)) {
- vmult = 3600 * 1000;
- } else if ("min".equalsIgnoreCase(unit)) {
- vmult = 60 * 1000;
- } else if ("sec".equalsIgnoreCase(unit)) {
- vmult = 1000;
- } else
- throw new IllegalArgumentException(" invalid unit value specified for frequency (hr,min,sec)");
- return val * vmult;
+ private static long computeInterval(int val, String unit) {
+ int vmult = 1;
+ if ("hr".equalsIgnoreCase(unit)) {
+ vmult = 3600 * 1000;
+ } else if ("min".equalsIgnoreCase(unit)) {
+ vmult = 60 * 1000;
+ } else if ("sec".equalsIgnoreCase(unit)) {
+ vmult = 1000;
+ } else
+ throw new IllegalArgumentException(
+ " invalid unit value specified for frequency (hr,min,sec)");
+ return val * vmult;
- }
+ }
- public static Event getEvent(Pattern pattern, Events events) {
- for (Event event : events.getEvent()) {
- if (event.getType().equals(pattern.getEvent().getType())) {
- return event;
- }
- }
- throw new IllegalArgumentException(" Unknown event type" + pattern.getEvent().getType());
- }
+ public static Event getEvent(Pattern pattern, Events events) {
+ for (Event event : events.getEvent()) {
+ if (event.getType().equals(pattern.getEvent().getType())) {
+ return event;
+ }
+ }
+ throw new IllegalArgumentException(" Unknown event type"
+ + pattern.getEvent().getType());
+ }
- public static Node getEventLocation(Pattern pattern, List<Node> candidateLocations, Cluster cluster) {
- ValueType value = new ValueType(pattern.getEvent().getNodeid().getValue());
- Node location = null;
- Type vtype = value.getType();
+ public static Node getEventLocation(Pattern pattern,
+ List<Node> candidateLocations, Cluster cluster) {
+ ValueType value = new ValueType(pattern.getEvent().getNodeid()
+ .getValue());
+ Node location = null;
+ Type vtype = value.getType();
- switch (vtype) {
- case ABS:
- location = getNodeFromId(value.getAbsoluteValue(), cluster);
- break;
- case RANDOM_RANGE:
- int nodeIndex = Randomizer.getInstance().getRandomInt(0, candidateLocations.size() - 1);
- location = candidateLocations.get(nodeIndex);
- break;
- case RANDOM_MIN_MAX:
- throw new IllegalStateException(" Canont configure a min max value range for location");
- }
- return location;
+ switch (vtype) {
+ case ABS:
+ location = getNodeFromId(value.getAbsoluteValue(), cluster);
+ break;
+ case RANDOM_RANGE:
+ int nodeIndex = Randomizer.getInstance().getRandomInt(0,
+ candidateLocations.size() - 1);
+ location = candidateLocations.get(nodeIndex);
+ break;
+ case RANDOM_MIN_MAX:
+ throw new IllegalStateException(
+ " Canont configure a min max value range for location");
+ }
+ return location;
- }
+ }
- public static List<Node> getCandidateLocations(Pattern pattern, Cluster cluster) {
- ValueType value = new ValueType(pattern.getEvent().getNodeid().getValue());
- List<Node> candidateList = new ArrayList<Node>();
- switch (value.getType()) {
- case ABS:
- candidateList.add(getNodeFromId(value.getAbsoluteValue(), cluster));
- break;
- case RANDOM_RANGE:
- boolean anyOption = false;
- String[] values = value.getRangeSet();
- for (String v : values) {
- if (v.equalsIgnoreCase("ANY")) {
- anyOption = true;
- }
- }
- if (anyOption) {
- for (Node node : cluster.getNode()) {
- candidateList.add(node);
- }
- } else {
- boolean found = false;
- for (String v : values) {
- for (Node node : cluster.getNode()) {
- if (node.getId().equals(v)) {
- candidateList.add(node);
- found = true;
- break;
- }
- }
- if (!found) {
- throw new IllegalStateException("Unknonw nodeId : " + v);
- }
- found = false;
- }
+ public static List<Node> getCandidateLocations(Pattern pattern,
+ Cluster cluster) {
+ ValueType value = new ValueType(pattern.getEvent().getNodeid()
+ .getValue());
+ List<Node> candidateList = new ArrayList<Node>();
+ switch (value.getType()) {
+ case ABS:
+ candidateList.add(getNodeFromId(value.getAbsoluteValue(), cluster));
+ break;
+ case RANDOM_RANGE:
+ boolean anyOption = false;
+ String[] values = value.getRangeSet();
+ for (String v : values) {
+ if (v.equalsIgnoreCase("ANY")) {
+ anyOption = true;
+ }
+ }
+ if (anyOption) {
+ for (Node node : cluster.getNode()) {
+ candidateList.add(node);
+ }
+ } else {
+ boolean found = false;
+ for (String v : values) {
+ for (Node node : cluster.getNode()) {
+ if (node.getId().equals(v)) {
+ candidateList.add(node);
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ throw new IllegalStateException("Unknonw nodeId : " + v);
+ }
+ found = false;
+ }
- }
- String[] excluded = value.getRangeExcluded();
- if (excluded != null && excluded.length > 0) {
- List<Node> markedForRemoval = new ArrayList<Node>();
- for (String exclusion : excluded) {
- for (Node node : candidateList) {
- if (node.getId().equals(exclusion)) {
- markedForRemoval.add(node);
- }
- }
- }
- candidateList.removeAll(markedForRemoval);
- }
- break;
- case RANDOM_MIN_MAX:
- throw new IllegalStateException(" Invalid value configured for location");
- }
- return candidateList;
- }
+ }
+ String[] excluded = value.getRangeExcluded();
+ if (excluded != null && excluded.length > 0) {
+ List<Node> markedForRemoval = new ArrayList<Node>();
+ for (String exclusion : excluded) {
+ for (Node node : candidateList) {
+ if (node.getId().equals(exclusion)) {
+ markedForRemoval.add(node);
+ }
+ }
+ }
+ candidateList.removeAll(markedForRemoval);
+ }
+ break;
+ case RANDOM_MIN_MAX:
+ throw new IllegalStateException(
+ " Invalid value configured for location");
+ }
+ return candidateList;
+ }
- private static Node getNodeFromId(String nodeid, Cluster cluster) {
- if (nodeid.equals(EventDriver.CLIENT_NODE.getId())) {
- return EventDriver.CLIENT_NODE;
- }
+ private static Node getNodeFromId(String nodeid, Cluster cluster) {
+ if (nodeid.equals(EventDriver.CLIENT_NODE.getId())) {
+ return EventDriver.CLIENT_NODE;
+ }
- if (nodeid.equals(cluster.getMasterNode().getId())) {
- String logDir = cluster.getMasterNode().getLogDir() == null ? cluster.getLogDir() : cluster.getMasterNode()
- .getLogDir();
- String javaHome = cluster.getMasterNode().getJavaHome() == null ? cluster.getJavaHome() : cluster
- .getMasterNode().getJavaHome();
- return new Node(cluster.getMasterNode().getId(), cluster.getMasterNode().getClusterIp(), javaHome, logDir,
- null, null, null);
- }
+ if (nodeid.equals(cluster.getMasterNode().getId())) {
+ String logDir = cluster.getMasterNode().getLogDir() == null ? cluster
+ .getLogDir()
+ : cluster.getMasterNode().getLogDir();
+ String javaHome = cluster.getMasterNode().getJavaHome() == null ? cluster
+ .getJavaHome()
+ : cluster.getMasterNode().getJavaHome();
+ return new Node(cluster.getMasterNode().getId(), cluster
+ .getMasterNode().getClusterIp(), javaHome, logDir, null,
+ null, null);
+ }
- List<Node> nodeList = cluster.getNode();
- for (Node node : nodeList) {
- if (node.getId().equals(nodeid)) {
- return node;
- }
- }
- StringBuffer buffer = new StringBuffer();
- buffer.append(EventDriver.CLIENT_NODE.getId() + ",");
- buffer.append(cluster.getMasterNode().getId() + ",");
- for (Node v : cluster.getNode()) {
- buffer.append(v.getId() + ",");
- }
- buffer.deleteCharAt(buffer.length() - 1);
- throw new IllegalArgumentException("Unknown node id :" + nodeid + " valid ids:" + buffer);
- }
+ List<Node> nodeList = cluster.getNode();
+ for (Node node : nodeList) {
+ if (node.getId().equals(nodeid)) {
+ return node;
+ }
+ }
+ StringBuffer buffer = new StringBuffer();
+ buffer.append(EventDriver.CLIENT_NODE.getId() + ",");
+ buffer.append(cluster.getMasterNode().getId() + ",");
+ for (Node v : cluster.getNode()) {
+ buffer.append(v.getId() + ",");
+ }
+ buffer.deleteCharAt(buffer.length() - 1);
+ throw new IllegalArgumentException("Unknown node id :" + nodeid
+ + " valid ids:" + buffer);
+ }
- public static void executeEventScript(Node node, String script, List<String> args, Cluster cluster)
- throws IOException, InterruptedException {
- List<String> pargs = new ArrayList<String>();
- pargs.add("/bin/bash");
- pargs.add(EventDriver.getEventsDir() + "/" + EXECUTE_SCRIPT);
- StringBuffer argBuffer = new StringBuffer();
- String env = EventDriver.getStringifiedEnv(cluster) + " " + IP_LOCATION + "=" + node.getClusterIp();
- if (args != null) {
- for (String arg : args) {
- argBuffer.append(arg + " ");
- }
- }
- ProcessBuilder pb = new ProcessBuilder(pargs);
- pb.environment().putAll(EventDriver.getEnvironment());
- pb.environment().put(IP_LOCATION, node.getClusterIp());
- pb.environment().put(CLUSTER_ENV, env);
- pb.environment().put(SCRIPT, script);
- pb.environment().put(ARGS, argBuffer.toString());
- pb.start();
- }
+ public static void executeEventScript(Node node, String script,
+ List<String> args, Cluster cluster) throws IOException,
+ InterruptedException {
+ List<String> pargs = new ArrayList<String>();
+ pargs.add("/bin/bash");
+ pargs.add(EventDriver.getEventsDir() + "/" + EXECUTE_SCRIPT);
+ StringBuffer argBuffer = new StringBuffer();
+ String env = EventDriver.getStringifiedEnv(cluster) + " " + IP_LOCATION
+ + "=" + node.getClusterIp();
+ if (args != null) {
+ for (String arg : args) {
+ argBuffer.append(arg + " ");
+ }
+ }
+ ProcessBuilder pb = new ProcessBuilder(pargs);
+ pb.environment().putAll(EventDriver.getEnvironment());
+ pb.environment().put(IP_LOCATION, node.getClusterIp());
+ pb.environment().put(CLUSTER_ENV, env);
+ pb.environment().put(SCRIPT, script);
+ pb.environment().put(ARGS, argBuffer.toString());
+ pb.start();
+ }
- public static void executeLocalScript(Node node, String script, List<String> args) throws IOException,
- InterruptedException {
- List<String> pargs = new ArrayList<String>();
- pargs.add("/bin/bash");
- pargs.add(script);
- if (args != null) {
- pargs.addAll(args);
- }
- ProcessBuilder pb = new ProcessBuilder(pargs);
- pb.environment().putAll(EventDriver.getEnvironment());
- pb.environment().put(IP_LOCATION, node.getClusterIp());
- pb.start();
- }
+ public static void executeLocalScript(Node node, String script,
+ List<String> args) throws IOException, InterruptedException {
+ List<String> pargs = new ArrayList<String>();
+ pargs.add("/bin/bash");
+ pargs.add(script);
+ if (args != null) {
+ pargs.addAll(args);
+ }
+ ProcessBuilder pb = new ProcessBuilder(pargs);
+ pb.environment().putAll(EventDriver.getEnvironment());
+ pb.environment().put(IP_LOCATION, node.getClusterIp());
+ pb.start();
+ }
- public static List<String> getEventArgs(Pattern pattern) {
- List<String> pargs = new ArrayList<String>();
- if (pattern.getEvent().getPargs() == null) {
- return pargs;
- }
- String[] args = pattern.getEvent().getPargs().split(" ");
- for (String arg : args) {
- pargs.add(arg.trim());
- }
- return pargs;
- }
+ public static List<String> getEventArgs(Pattern pattern) {
+ List<String> pargs = new ArrayList<String>();
+ if (pattern.getEvent().getPargs() == null) {
+ return pargs;
+ }
+ String[] args = pattern.getEvent().getPargs().split(" ");
+ for (String arg : args) {
+ pargs.add(arg.trim());
+ }
+ return pargs;
+ }
}
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/driver/InstallerUtil.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/driver/InstallerUtil.java
index ebd23d7..2b884ef 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/driver/InstallerUtil.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/driver/InstallerUtil.java
@@ -51,6 +51,7 @@
import edu.uci.ics.asterix.common.configuration.Store;
import edu.uci.ics.asterix.event.driver.EventDriver;
import edu.uci.ics.asterix.event.management.EventrixClient;
+import edu.uci.ics.asterix.event.management.EventUtil;
import edu.uci.ics.asterix.event.schema.cluster.Cluster;
import edu.uci.ics.asterix.event.schema.cluster.Env;
import edu.uci.ics.asterix.event.schema.cluster.Node;
@@ -96,10 +97,10 @@
clusterProperties = new ArrayList<Property>();
}
for (edu.uci.ics.asterix.common.configuration.Property property : asterixConfiguration.getProperty()) {
- if (property.getName().equalsIgnoreCase(AsterixInstance.CC_JAVA_OPTS)) {
- clusterProperties.add(new Property(AsterixInstance.CC_JAVA_OPTS, property.getValue()));
- } else if (property.getName().equalsIgnoreCase(AsterixInstance.NC_JAVA_OPTS)) {
- clusterProperties.add(new Property(AsterixInstance.NC_JAVA_OPTS, property.getValue()));
+ if (property.getName().equalsIgnoreCase(EventUtil.CC_JAVA_OPTS)) {
+ clusterProperties.add(new Property(EventUtil.CC_JAVA_OPTS, property.getValue()));
+ } else if (property.getName().equalsIgnoreCase(EventUtil.NC_JAVA_OPTS)) {
+ clusterProperties.add(new Property(EventUtil.NC_JAVA_OPTS, property.getValue()));
}
}
clusterProperties.add(new Property("ASTERIX_HOME", cluster.getWorkingDir().getDir() + File.separator
diff --git a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/model/AsterixInstance.java b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/model/AsterixInstance.java
index 76182f6..440e1fd 100644
--- a/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/model/AsterixInstance.java
+++ b/asterix-installer/src/main/java/edu/uci/ics/asterix/installer/model/AsterixInstance.java
@@ -29,9 +29,7 @@
private static final long serialVersionUID = 2874439550187520449L;
- public static final String CC_JAVA_OPTS = AsterixProperties.AsterixConfigurationKeys.CC_JAVA_OPTS;
- public static final String NC_JAVA_OPTS = AsterixProperties.AsterixConfigurationKeys.NC_JAVA_OPTS;
-
+
public enum State {
ACTIVE,
INACTIVE,
diff --git a/asterix-installer/src/main/resources/conf/asterix-configuration.xml b/asterix-installer/src/main/resources/conf/asterix-configuration.xml
index 56a88ee..b6958c5 100644
--- a/asterix-installer/src/main/resources/conf/asterix-configuration.xml
+++ b/asterix-installer/src/main/resources/conf/asterix-configuration.xml
@@ -3,114 +3,188 @@
<property>
<name>nc.java.opts</name>
<value>-Xmx1024m</value>
- <description>JVM parameters for each Node Contoller (NC)</description>
+ <description>JVM parameters for each Node Contoller (NC)</description>
</property>
<property>
<name>cc.java.opts</name>
<value>-Xmx1024m</value>
- <description>JVM parameters for each Cluster Contoller (CC)</description>
+ <description>JVM parameters for each Cluster Contoller (CC)
+ </description>
</property>
<property>
- <name>size.memory.component</name>
- <value>512m</value>
- <description></description>
+ <name>storage.buffercache.pagesize</name>
+ <value>32768</value>
+ <description>The page size in bytes for pages in the buffer cache.
+ (Default = "32768" // 32KB)
+ </description>
</property>
<property>
- <name>total.size.memory.component</name>
- <value>512m</value>
- <description></description>
+ <name>storage.buffercache.numpages</name>
+ <value>1024</value>
+ <description>The number of pages allocated to the disk buffer cache.
+ (Default = "1024")
+ </description>
</property>
<property>
- <name>log.buffer.num.pages</name>
+ <name>storage.buffercache.maxopenfiles</name>
+ <value>214748364</value>
+ <description>The maximum number of open files in the buffer cache.
+ (Default = "214748364")
+ </description>
+ </property>
+
+ <property>
+ <name>storage.memorycomponent.pagesize</name>
+ <value>32768</value>
+ <description>The page size in bytes for pages allocated to memory
+ components. (Default = "32768" // 32KB)
+ </description>
+ </property>
+
+ <property>
+ <name>storage.memorycomponent.numpages</name>
+ <value>4096</value>
+ <description>The number of pages to allocate for a memory component.
+ (Default = 4096)
+ </description>
+ </property>
+
+ <property>
+ <name>storage.memorycomponent.globalbudget</name>
+ <value>1073741824</value>
+ <description>The total size of memory in bytes that the sum of all
+ open memory
+ components cannot exceed. (Default = "1073741824" // 1GB)
+ </description>
+ </property>
+
+ <property>
+ <name>storage.lsm.mergethreshold</name>
+ <value>3</value>
+ <description>The number of on-disk components an LSM index can have
+ before a merge is triggered. (Default = 3)
+ </description>
+ </property>
+
+ <property>
+ <name>storage.lsm.bloomfilter.falsepositiverate</name>
+ <value>0.01</value>
+ <description>The maximum acceptable false positive rate for bloom
+ filters associated with LSM indexes. (Default = "0.01" // 1%)
+ </description>
+ </property>
+
+ <property>
+ <name>txn.log.buffer.numpages</name>
<value>8</value>
- <description></description>
+ <description>The number of in-memory log buffer pages. (Default = "8")
+ </description>
</property>
<property>
- <name>group.commit.interval</name>
- <value>200ms</value>
- <description></description>
+ <name>txn.log.buffer.pagesize</name>
+ <value>131072</value>
+ <description>The size of pages in the in-memory log buffer. (Default =
+ "131072" // 128KB)
+ </description>
</property>
<property>
- <name>sort.op.memory</name>
- <value>200m</value>
- <description></description>
+ <name>txn.log.partitionsize</name>
+ <value>2147483648</value>
+ <description>The maximum size of a log file partition allowed before
+ rotating the log to the next partition. (Default = "2147483648" //
+ 2GB)
+ </description>
</property>
<property>
- <name>join.op.memory</name>
- <value>200m</value>
- <description></description>
+ <name>txn.log.groupcommitinterval</name>
+ <value>200</value>
+ <description>The group commit wait time in milliseconds. (Default =
+ "200" // 2ms)
+ </description>
</property>
<property>
- <name>web.interface.port</name>
+ <name>txn.log.checkpoint.lsnthreshold</name>
+ <value>67108864</value>
+ <description>The size of the window that the maximum LSN is allowed to
+ be ahead of the checkpoint LSN by. (Default = ""67108864" // 64M)
+ </description>
+ </property>
+
+ <property>
+ <name>txn.log.checkpoint.pollfrequency</name>
+ <value>120</value>
+ <description>The time in seconds between that the checkpoint thread
+ waits between polls. (Default = "120" // 120s)
+ </description>
+ </property>
+
+ <property>
+ <name>txn.lock.escalationthreshold</name>
+ <value>1000</value>
+ <description>The number of entity level locks that need to be acquired
+ before the locks are coalesced and escalated into a dataset level
+ lock. (Default = "1000")
+ </description>
+ </property>
+
+ <property>
+ <name>txn.lock.shrinktimer</name>
+ <value>120000</value>
+ <description>The time in milliseconds to wait before deallocating
+ unused lock manager memory. (Default = "120000" // 120s)
+ </description>
+ </property>
+
+ <property>
+ <name>compiler.sortmemory</name>
+ <value>536870912</value>
+ <description>The amount of memory in bytes given to sort operations.
+ (Default = "536870912" // 512mb)
+ </description>
+ </property>
+
+ <property>
+ <name>compiler.joinmemory</name>
+ <value>536870912</value>
+ <description>The amount of memory in bytes given to join operations.
+ (Default = "536870912" // 512mb)
+ </description>
+ </property>
+
+ <property>
+ <name>compiler.framesize</name>
+ <value>32768</value>
+ <description>The Hyracks frame size that the compiler configures per
+ job. (Default = "32768" // 32KB)
+ </description>
+ </property>
+
+ <property>
+ <name>web.port</name>
<value>19001</value>
- <description></description>
+ <description>The port for the ASTERIX web interface. (Default = 19001)
+ </description>
</property>
<property>
- <name>num.pages.buffer.cache</name>
- <value>8</value>
- <description></description>
+ <name>api.port</name>
+ <value>19101</value>
+ <description>The port for the ASTERIX API server. (Default = 19101)
+ </description>
</property>
<property>
<name>log.level</name>
<value>INFO</value>
- <description></description>
- </property>
-
- <property>
- <name>lsn.threshold</name>
- <value>64m</value>
- <description></description>
- </property>
-
- <property>
- <name>checkpoint.terms.in.secs</name>
- <value>120</value>
- <description></description>
- </property>
-
- <property>
- <name>escalate.threshold.entity.to.dataset</name>
- <value>8</value>
- <description></description>
- </property>
-
- <property>
- <name>num.pages.buffer.cache</name>
- <value>1000</value>
- <description></description>
- </property>
-
- <property>
- <name>log.buffer.page.size</name>
- <value>128k</value>
- <description></description>
- </property>
-
- <property>
- <name>log.partition.size</name>
- <value>2147483648</value>
- <description></description>
- </property>
-
- <property>
- <name>shrink.timer.threshold</name>
- <value>120000</value>
- <description></description>
- </property>
-
- <property>
- <name>log.buffer.num.pages</name>
- <value>8</value>
- <description></description>
+ <description>The minimum log level to be displayed. (Default = INFO)
+ </description>
</property>
</asterixConfiguration>
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java
index 994262d..09ec2f7 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java
@@ -20,11 +20,11 @@
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import edu.uci.ics.asterix.common.config.AsterixMetadataProperties;
import edu.uci.ics.asterix.common.functions.FunctionSignature;
import edu.uci.ics.asterix.metadata.api.IAsterixStateProxy;
import edu.uci.ics.asterix.metadata.api.IMetadataManager;
import edu.uci.ics.asterix.metadata.api.IMetadataNode;
-import edu.uci.ics.asterix.metadata.bootstrap.MetadataConstants;
import edu.uci.ics.asterix.metadata.entities.Dataset;
import edu.uci.ics.asterix.metadata.entities.DatasourceAdapter;
import edu.uci.ics.asterix.metadata.entities.Datatype;
@@ -85,12 +85,14 @@
private IAsterixStateProxy proxy;
private IMetadataNode metadataNode;
private final ReadWriteLock metadataLatch;
+ private final AsterixMetadataProperties metadataProperties;
- public MetadataManager(IAsterixStateProxy proxy) {
+ public MetadataManager(IAsterixStateProxy proxy, AsterixMetadataProperties metadataProperties) {
if (proxy == null) {
throw new Error("Null proxy given to MetadataManager.");
}
this.proxy = proxy;
+ this.metadataProperties = metadataProperties;
this.metadataNode = null;
this.metadataLatch = new ReentrantReadWriteLock(true);
}
@@ -105,7 +107,7 @@
metadataNode = proxy.getMetadataNode();
if (metadataNode == null) {
throw new Error("Failed to get the MetadataNode.\n" + "The MetadataNode was configured to run on NC: "
- + proxy.getAsterixProperties().getMetadataNodeName());
+ + metadataProperties.getMetadataNodeName());
}
}
}
@@ -544,10 +546,10 @@
throw new MetadataException(e);
}
}
-
+
@Override
public int getMostRecentDatasetId() throws MetadataException {
- try {
+ try {
return metadataNode.getMostRecentDatasetId();
} catch (RemoteException e) {
throw new MetadataException(e);
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IAsterixStateProxy.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IAsterixStateProxy.java
index d67cd27..7f06c30 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IAsterixStateProxy.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IAsterixStateProxy.java
@@ -19,17 +19,11 @@
import java.rmi.Remote;
import java.rmi.RemoteException;
-import edu.uci.ics.asterix.common.config.AsterixProperties;
-
/**
* Interface for setting/getting distributed state of Asterix.
*/
public interface IAsterixStateProxy extends Remote, Serializable {
public void setMetadataNode(IMetadataNode metadataNode) throws RemoteException;
- public void setAsterixProperties(AsterixProperties asterixProperties) throws RemoteException;
-
public IMetadataNode getMetadataNode() throws RemoteException;
-
- public AsterixProperties getAsterixProperties() throws RemoteException;
}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/AsterixStateProxy.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/AsterixStateProxy.java
index 55aaf33..33f6994 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/AsterixStateProxy.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/AsterixStateProxy.java
@@ -19,7 +19,6 @@
import java.rmi.server.UnicastRemoteObject;
import java.util.logging.Logger;
-import edu.uci.ics.asterix.common.config.AsterixProperties;
import edu.uci.ics.asterix.metadata.api.IAsterixStateProxy;
import edu.uci.ics.asterix.metadata.api.IMetadataNode;
@@ -31,7 +30,6 @@
private static final Logger LOGGER = Logger.getLogger(AsterixStateProxy.class.getName());
private IMetadataNode metadataNode;
- private AsterixProperties asterixProperties;
private static final IAsterixStateProxy cc = new AsterixStateProxy();
public static IAsterixStateProxy registerRemoteObject() throws RemoteException {
@@ -54,14 +52,4 @@
public IMetadataNode getMetadataNode() throws RemoteException {
return this.metadataNode;
}
-
- @Override
- public void setAsterixProperties(AsterixProperties asterixProperity) throws RemoteException {
- this.asterixProperties = asterixProperity;
- }
-
- @Override
- public AsterixProperties getAsterixProperties() throws RemoteException {
- return this.asterixProperties;
- }
}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
index d978f8a..8d5d13b 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -20,16 +20,17 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
-import edu.uci.ics.asterix.common.config.AsterixProperties;
+import edu.uci.ics.asterix.common.config.AsterixMetadataProperties;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
import edu.uci.ics.asterix.common.context.AsterixAppRuntimeContext;
import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.external.adapter.factory.IAdapterFactory;
@@ -110,12 +111,14 @@
private static String metadataNodeName;
private static String metadataStore;
- private static HashSet<String> nodeNames;
+ private static Set<String> nodeNames;
private static String outputDir;
private static IMetadataIndex[] primaryIndexes;
private static IMetadataIndex[] secondaryIndexes;
+ private static IAsterixPropertiesProvider propertiesProvider;
+
private static void initLocalIndexArrays() {
primaryIndexes = new IMetadataIndex[] { MetadataPrimaryIndexes.DATAVERSE_DATASET,
MetadataPrimaryIndexes.DATASET_DATASET, MetadataPrimaryIndexes.DATATYPE_DATASET,
@@ -127,9 +130,10 @@
MetadataSecondaryIndexes.DATATYPENAME_ON_DATATYPE_INDEX };
}
- public static void startUniverse(AsterixProperties asterixProperties, INCApplicationContext ncApplicationContext,
- boolean isNewUniverse) throws Exception {
+ public static void startUniverse(IAsterixPropertiesProvider asterixPropertiesProvider,
+ INCApplicationContext ncApplicationContext, boolean isNewUniverse) throws Exception {
runtimeContext = (AsterixAppRuntimeContext) ncApplicationContext.getApplicationObject();
+ propertiesProvider = asterixPropertiesProvider;
// Initialize static metadata objects, such as record types and metadata
// index descriptors.
@@ -149,9 +153,10 @@
resourceRepository.registerTransactionalResourceManager(ResourceType.LSM_INVERTED_INDEX,
new IndexResourceManager(ResourceType.LSM_INVERTED_INDEX, runtimeContext.getTransactionSubsystem()));
- metadataNodeName = asterixProperties.getMetadataNodeName();
- metadataStore = asterixProperties.getMetadataStore();
- nodeNames = asterixProperties.getNodeNames();
+ AsterixMetadataProperties metadataProperties = propertiesProvider.getMetadataProperties();
+ metadataNodeName = metadataProperties.getMetadataNodeName();
+ metadataStore = metadataProperties.getMetadataStore();
+ nodeNames = metadataProperties.getNodeNames();
// nodeStores = asterixProperity.getStores();
indexLifecycleManager = runtimeContext.getIndexLifecycleManager();
@@ -165,7 +170,7 @@
// Begin a transaction against the metadata.
// Lock the metadata in X mode.
MetadataManager.INSTANCE.lock(mdTxnCtx, LockMode.X);
-
+
if (isNewUniverse) {
for (int i = 0; i < primaryIndexes.length; i++) {
enlistMetadataDataset(primaryIndexes[i], true);
@@ -196,7 +201,7 @@
LOGGER.info("Finished enlistment of metadata B-trees.");
}
}
-
+
//#. initialize datasetIdFactory
MetadataManager.INSTANCE.initializeDatasetIdFactory(mdTxnCtx);
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
@@ -327,7 +332,7 @@
adapterFactoryClassName, DatasourceAdapter.AdapterType.INTERNAL);
}
- public static void enlistMetadataDataset(IMetadataIndex index, boolean create) throws Exception {
+ private static void enlistMetadataDataset(IMetadataIndex index, boolean create) throws Exception {
String filePath = metadataStore + File.separator + index.getFileNameRelativePath();
FileReference file = new FileReference(new File(filePath));
IInMemoryBufferCache memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), DEFAULT_MEM_PAGE_SIZE,
@@ -349,9 +354,10 @@
resourceID = runtimeContext.getResourceIdFactory().createId();
indexLifecycleManager.register(resourceID, lsmBtree);
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(typeTraits,
comparatorFactories, bloomFilterKeyFields, index.isPrimaryIndex(),
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages());
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMBTreeResource);
ILocalResourceFactory localResourceFactory = localResourceFactoryProvider.getLocalResourceFactory();
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlCompiledMetadataDeclarations.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlCompiledMetadataDeclarations.java
index 70383f9..f79b9b7 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlCompiledMetadataDeclarations.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlCompiledMetadataDeclarations.java
@@ -22,7 +22,8 @@
import java.util.logging.Logger;
import edu.uci.ics.asterix.common.annotations.TypeDataGen;
-import edu.uci.ics.asterix.common.config.AsterixProperties;
+import edu.uci.ics.asterix.common.api.AsterixAppContextInfo;
+import edu.uci.ics.asterix.common.config.AsterixMetadataProperties;
import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.formats.base.IDataFormat;
@@ -73,8 +74,9 @@
this.dataverseName = dataverseName;
this.outputFile = outputFile;
this.config = config;
+ AsterixMetadataProperties metadataProperties = AsterixAppContextInfo.getInstance().getMetadataProperties();
if (stores == null && online) {
- this.stores = AsterixProperties.INSTANCE.getStores();
+ this.stores = metadataProperties.getStores();
} else {
this.stores = stores;
}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
index 8bbddec..4773ed0 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
@@ -23,7 +23,8 @@
import java.util.Map;
import java.util.logging.Logger;
-import edu.uci.ics.asterix.common.config.AsterixProperties;
+import edu.uci.ics.asterix.common.api.AsterixAppContextInfo;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
import edu.uci.ics.asterix.common.config.GlobalConfig;
@@ -151,6 +152,8 @@
private final Dataverse defaultDataverse;
private JobId jobId;
+ private final AsterixStorageProperties storageProperties;
+
private static final Map<String, String> adapterFactoryMapping = initializeAdapterFactoryMapping();
public String getPropertyValue(String propertyName) {
@@ -171,7 +174,8 @@
public AqlMetadataProvider(Dataverse defaultDataverse) {
this.defaultDataverse = defaultDataverse;
- this.stores = AsterixProperties.INSTANCE.getStores();
+ this.stores = AsterixAppContextInfo.getInstance().getMetadataProperties().getStores();
+ this.storageProperties = AsterixAppContextInfo.getInstance().getStorageProperties();
}
public void setJobId(JobId jobId) {
@@ -543,8 +547,8 @@
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
- retainInput, searchCallbackFactory);
+ storageProperties.getMemoryComponentPageSize(),
+ storageProperties.getMemoryComponentNumPages()), retainInput, searchCallbackFactory);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(btreeSearchOp, spPc.second);
} catch (MetadataException me) {
@@ -612,8 +616,8 @@
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER, proposeLinearizer(
nestedKeyType.getTypeTag(), comparatorFactories.length),
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
- retainInput, searchCallbackFactory);
+ storageProperties.getMemoryComponentPageSize(),
+ storageProperties.getMemoryComponentNumPages()), retainInput, searchCallbackFactory);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(rtreeSearchOp, spPc.second);
} catch (MetadataException me) {
@@ -772,8 +776,8 @@
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
- NoOpOperationCallbackFactory.INSTANCE);
+ storageProperties.getMemoryComponentPageSize(),
+ storageProperties.getMemoryComponentNumPages()), NoOpOperationCallbackFactory.INSTANCE);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(btreeBulkLoad,
splitsAndConstraint.second);
} catch (MetadataException me) {
@@ -839,9 +843,9 @@
new LSMBTreeDataflowHelperFactory(AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES), null,
- modificationCallbackFactory);
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
+ null, modificationCallbackFactory);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(btreeBulkLoad,
splitsAndConstraint.second);
@@ -1035,8 +1039,8 @@
new LSMBTreeDataflowHelperFactory(AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
filterFactory, modificationCallbackFactory);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(btreeBulkLoad,
splitsAndConstraint.second);
@@ -1161,8 +1165,8 @@
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
+ AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER, storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
filterFactory, modificationCallbackFactory);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(insertDeleteOp,
splitsAndConstraint.second);
@@ -1257,8 +1261,8 @@
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER, proposeLinearizer(
nestedKeyType.getTypeTag(), comparatorFactories.length),
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
- filterFactory, modificationCallbackFactory);
+ storageProperties.getMemoryComponentPageSize(),
+ storageProperties.getMemoryComponentNumPages()), filterFactory, modificationCallbackFactory);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(rtreeUpdate, splitsAndConstraint.second);
} catch (MetadataException | IOException e) {
throw new AlgebricksException(e);
@@ -1295,6 +1299,17 @@
public Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitProviderAndPartitionConstraintsForInternalOrFeedDataset(
String dataverseName, String datasetName, String targetIdxName) throws AlgebricksException {
FileSplit[] splits = splitsForInternalOrFeedDataset(mdTxnCtx, dataverseName, datasetName, targetIdxName);
+ return splitProviderAndPartitionConstraints(splits);
+ }
+
+ public Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitProviderAndPartitionConstraintsForDataverse(
+ String dataverse) {
+ FileSplit[] splits = splitsForDataverse(mdTxnCtx, dataverse);
+ return splitProviderAndPartitionConstraints(splits);
+ }
+
+ private Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitProviderAndPartitionConstraints(
+ FileSplit[] splits) {
IFileSplitProvider splitProvider = new ConstantFileSplitProvider(splits);
String[] loc = new String[splits.length];
for (int p = 0; p < splits.length; p++) {
@@ -1304,6 +1319,23 @@
return new Pair<IFileSplitProvider, AlgebricksPartitionConstraint>(splitProvider, pc);
}
+ private FileSplit[] splitsForDataverse(MetadataTransactionContext mdTxnCtx, String dataverseName) {
+ File relPathFile = new File(dataverseName);
+ List<FileSplit> splits = new ArrayList<FileSplit>();
+ for (Map.Entry<String, String[]> entry : stores.entrySet()) {
+ String node = entry.getKey();
+ String[] nodeStores = entry.getValue();
+ if (nodeStores == null) {
+ continue;
+ }
+ for (int i = 0; i < nodeStores.length; i++) {
+ File f = new File(nodeStores[i] + File.separator + relPathFile);
+ splits.add(new FileSplit(node, new FileReference(f)));
+ }
+ }
+ return splits.toArray(new FileSplit[] {});
+ }
+
private FileSplit[] splitsForInternalOrFeedDataset(MetadataTransactionContext mdTxnCtx, String dataverseName,
String datasetName, String targetIdxName) throws AlgebricksException {
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/comparators/AObjectAscBinaryComparatorFactory.java b/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/comparators/AObjectAscBinaryComparatorFactory.java
index 0e486e7..ca926e4 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/comparators/AObjectAscBinaryComparatorFactory.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/comparators/AObjectAscBinaryComparatorFactory.java
@@ -2,7 +2,6 @@
import edu.uci.ics.asterix.om.types.ATypeTag;
import edu.uci.ics.asterix.om.types.EnumDeserializer;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
@@ -39,6 +38,7 @@
.createBinaryComparator();
final IBinaryComparator ascDateOrTimeComp = ADateOrTimeAscBinaryComparatorFactory.INSTANCE
.createBinaryComparator();
+ final IBinaryComparator rawComp = RawBinaryComparatorFactory.INSTANCE.createBinaryComparator();
@Override
public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
@@ -84,7 +84,7 @@
return ascDateOrTimeComp.compare(b1, s1 + 1, l1 - 1, b2, s2 + 1, l2 - 1);
}
default: {
- throw new NotImplementedException("Comparison for type " + tag + " is not implemented");
+ return rawComp.compare(b1, s1 + 1, l1 - 1, b2, s2 + 1, l2 - 1);
}
}
}
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/comparators/RawBinaryComparatorFactory.java b/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/comparators/RawBinaryComparatorFactory.java
new file mode 100644
index 0000000..2c4f951
--- /dev/null
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/comparators/RawBinaryComparatorFactory.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.dataflow.data.nontagged.comparators;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
+
+public class RawBinaryComparatorFactory implements IBinaryComparatorFactory {
+
+ private static final long serialVersionUID = 1L;
+ public static IBinaryComparatorFactory INSTANCE = new RawBinaryComparatorFactory();
+
+ private RawBinaryComparatorFactory() {
+ }
+
+ @Override
+ public IBinaryComparator createBinaryComparator() {
+ return new IBinaryComparator() {
+
+ @Override
+ public int compare(byte[] b1, int s1, int l1, byte[] b2, int s2, int l2) {
+ int commonLength = Math.min(l1, l2);
+ for (int i = 0; i < commonLength; i++) {
+ if (b1[s1 + i] != b2[s2 + i]) {
+ return b1[s1 + i] - b2[s2 + i];
+ }
+ }
+ int difference = l1 - l2;
+ return difference == 0 ? 0 : (difference > 0 ? 1 : -1);
+ }
+
+ };
+ }
+}
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/hash/AObjectBinaryHashFunctionFactory.java b/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/hash/AObjectBinaryHashFunctionFactory.java
index 15fd8c8..2d19320 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/hash/AObjectBinaryHashFunctionFactory.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/hash/AObjectBinaryHashFunctionFactory.java
@@ -2,7 +2,6 @@
import edu.uci.ics.asterix.om.types.ATypeTag;
import edu.uci.ics.asterix.om.types.EnumDeserializer;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunction;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryHashFunctionFactory;
@@ -36,6 +35,7 @@
.createBinaryHashFunction();
private IBinaryHashFunction rectangleHash = RectangleBinaryHashFunctionFactory.INSTANCE
.createBinaryHashFunction();
+ private IBinaryHashFunction rawHash = RawBinaryHashFunctionFactory.INSTANCE.createBinaryHashFunction();
@Override
public int hash(byte[] bytes, int offset, int length) {
@@ -66,7 +66,7 @@
return 0;
}
default: {
- throw new NotImplementedException("Binary hashing for the " + tag + " type is not implemented.");
+ return rawHash.hash(bytes, offset + 1, length - 1);
}
}
}
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/hash/RawBinaryHashFunctionFactory.java b/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/hash/RawBinaryHashFunctionFactory.java
new file mode 100644
index 0000000..4aeb00e
--- /dev/null
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/dataflow/data/nontagged/hash/RawBinaryHashFunctionFactory.java
@@ -0,0 +1,44 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.dataflow.data.nontagged.hash;
+
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunction;
+import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
+
+public class RawBinaryHashFunctionFactory implements IBinaryHashFunctionFactory {
+ private static final long serialVersionUID = 1L;
+
+ public static IBinaryHashFunctionFactory INSTANCE = new RawBinaryHashFunctionFactory();
+
+ private RawBinaryHashFunctionFactory() {
+ }
+
+ @Override
+ public IBinaryHashFunction createBinaryHashFunction() {
+
+ return new IBinaryHashFunction() {
+ @Override
+ public int hash(byte[] bytes, int offset, int length) {
+ int value = 1;
+ int end = offset + length;
+ for (int i = offset; i < end; i++)
+ value = value * 31 + (int) bytes[i];
+ return value;
+ }
+ };
+ }
+
+}
\ No newline at end of file
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/formats/nontagged/AqlBinaryComparatorFactoryProvider.java b/asterix-om/src/main/java/edu/uci/ics/asterix/formats/nontagged/AqlBinaryComparatorFactoryProvider.java
index 02f0e47..38fddb7 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/formats/nontagged/AqlBinaryComparatorFactoryProvider.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/formats/nontagged/AqlBinaryComparatorFactoryProvider.java
@@ -7,10 +7,10 @@
import edu.uci.ics.asterix.dataflow.data.nontagged.comparators.AObjectAscBinaryComparatorFactory;
import edu.uci.ics.asterix.dataflow.data.nontagged.comparators.AObjectDescBinaryComparatorFactory;
import edu.uci.ics.asterix.dataflow.data.nontagged.comparators.BooleanBinaryComparatorFactory;
+import edu.uci.ics.asterix.dataflow.data.nontagged.comparators.RawBinaryComparatorFactory;
import edu.uci.ics.asterix.dataflow.data.nontagged.comparators.RectangleBinaryComparatorFactory;
import edu.uci.ics.asterix.om.types.ATypeTag;
import edu.uci.ics.asterix.om.types.IAType;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
import edu.uci.ics.hyracks.algebricks.data.IBinaryComparatorFactoryProvider;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparatorFactory;
@@ -128,7 +128,7 @@
return addOffset(ADateTimeAscBinaryComparatorFactory.INSTANCE, ascending);
}
default: {
- throw new NotImplementedException("No binary comparator factory implemented for type " + type + " .");
+ return addOffset(RawBinaryComparatorFactory.INSTANCE, ascending);
}
}
}
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/formats/nontagged/AqlBinaryHashFunctionFactoryProvider.java b/asterix-om/src/main/java/edu/uci/ics/asterix/formats/nontagged/AqlBinaryHashFunctionFactoryProvider.java
index 1afdbf8..5498374 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/formats/nontagged/AqlBinaryHashFunctionFactoryProvider.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/formats/nontagged/AqlBinaryHashFunctionFactoryProvider.java
@@ -6,9 +6,9 @@
import edu.uci.ics.asterix.dataflow.data.nontagged.hash.BooleanBinaryHashFunctionFactory;
import edu.uci.ics.asterix.dataflow.data.nontagged.hash.DoubleBinaryHashFunctionFactory;
import edu.uci.ics.asterix.dataflow.data.nontagged.hash.LongBinaryHashFunctionFactory;
+import edu.uci.ics.asterix.dataflow.data.nontagged.hash.RawBinaryHashFunctionFactory;
import edu.uci.ics.asterix.dataflow.data.nontagged.hash.RectangleBinaryHashFunctionFactory;
import edu.uci.ics.asterix.om.types.IAType;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.NotImplementedException;
import edu.uci.ics.hyracks.algebricks.data.IBinaryHashFunctionFactoryProvider;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunction;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryHashFunctionFactory;
@@ -88,8 +88,7 @@
return addOffset(RectangleBinaryHashFunctionFactory.INSTANCE);
}
default: {
- throw new NotImplementedException("No binary hash function factory implemented for type "
- + aqlType.getTypeTag() + " .");
+ return addOffset(RawBinaryHashFunctionFactory.INSTANCE);
}
}
}
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/om/functions/AsterixBuiltinFunctions.java b/asterix-om/src/main/java/edu/uci/ics/asterix/om/functions/AsterixBuiltinFunctions.java
index 714bf3b..c47b123 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/om/functions/AsterixBuiltinFunctions.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/om/functions/AsterixBuiltinFunctions.java
@@ -730,7 +730,7 @@
add(RECTANGLE_CONSTRUCTOR, OptionalARectangleTypeComputer.INSTANCE);
// add(RECORD_TYPE_CONSTRUCTOR, null);
add(SCALAR_AVG, ScalarVersionOfAggregateResultType.INSTANCE);
- add(SCALAR_COUNT, ScalarVersionOfAggregateResultType.INSTANCE);
+ add(SCALAR_COUNT, AInt32TypeComputer.INSTANCE);
add(SCALAR_GLOBAL_AVG, ScalarVersionOfAggregateResultType.INSTANCE);
add(SCALAR_LOCAL_AVG, ScalarVersionOfAggregateResultType.INSTANCE);
add(SCALAR_MAX, ScalarVersionOfAggregateResultType.INSTANCE);
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/om/typecomputer/impl/NonTaggedFieldAccessByNameResultType.java b/asterix-om/src/main/java/edu/uci/ics/asterix/om/typecomputer/impl/NonTaggedFieldAccessByNameResultType.java
index db9d932..6d5880e 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/om/typecomputer/impl/NonTaggedFieldAccessByNameResultType.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/om/typecomputer/impl/NonTaggedFieldAccessByNameResultType.java
@@ -70,6 +70,9 @@
if (t1.getTypeTag() == ATypeTag.RECORD) {
return (ARecordType) t1;
}
+ if (t1.getTypeTag() == ATypeTag.ANY) {
+ return DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE;
+ }
}
}
default: {
diff --git a/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixRuntimeUtil.java b/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixRuntimeUtil.java
index 76f3301..e280b2e 100644
--- a/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixRuntimeUtil.java
+++ b/asterix-om/src/main/java/edu/uci/ics/asterix/om/util/AsterixRuntimeUtil.java
@@ -23,7 +23,7 @@
import java.util.Map;
import java.util.Set;
-import edu.uci.ics.asterix.common.api.AsterixAppContextInfoImpl;
+import edu.uci.ics.asterix.common.api.AsterixAppContextInfo;
/**
* Utility class for obtaining information on the set of Hyracks NodeController
@@ -52,7 +52,7 @@
public static Map<String, Set<String>> getNodeControllerMap()
throws Exception {
Map<String, Set<String>> map = new HashMap<String, Set<String>>();
- AsterixAppContextInfoImpl.getInstance().getCCApplicationContext()
+ AsterixAppContextInfo.getInstance().getCCApplicationContext()
.getCCContext().getIPAddressNodeMap(map);
return map;
}
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/formats/NonTaggedDataFormat.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/formats/NonTaggedDataFormat.java
index 20d70d6..8bc792c 100644
--- a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/formats/NonTaggedDataFormat.java
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/formats/NonTaggedDataFormat.java
@@ -35,6 +35,7 @@
import edu.uci.ics.asterix.om.functions.IFunctionDescriptor;
import edu.uci.ics.asterix.om.functions.IFunctionDescriptorFactory;
import edu.uci.ics.asterix.om.functions.IFunctionManager;
+import edu.uci.ics.asterix.om.pointables.base.DefaultOpenFieldType;
import edu.uci.ics.asterix.om.typecomputer.base.TypeComputerUtilities;
import edu.uci.ics.asterix.om.types.AOrderedListType;
import edu.uci.ics.asterix.om.types.ARecordType;
@@ -694,14 +695,20 @@
if (fd.getIdentifier().equals(AsterixBuiltinFunctions.CAST_RECORD)) {
AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
ARecordType rt = (ARecordType) TypeComputerUtilities.getRequiredType(funcExpr);
- ARecordType it = (ARecordType) context.getType(funcExpr.getArguments().get(0).getValue());
- ((CastRecordDescriptor) fd).reset(rt, it);
+ IAType it = (IAType) context.getType(funcExpr.getArguments().get(0).getValue());
+ if (it.getTypeTag().equals(ATypeTag.ANY)) {
+ it = DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE;
+ }
+ ((CastRecordDescriptor) fd).reset(rt, (ARecordType) it);
}
if (fd.getIdentifier().equals(AsterixBuiltinFunctions.CAST_LIST)) {
AbstractFunctionCallExpression funcExpr = (AbstractFunctionCallExpression) expr;
AbstractCollectionType rt = (AbstractCollectionType) TypeComputerUtilities.getRequiredType(funcExpr);
- AbstractCollectionType it = (AbstractCollectionType) context.getType(funcExpr.getArguments().get(0).getValue());
- ((CastListDescriptor) fd).reset(rt, it);
+ IAType it = (IAType) context.getType(funcExpr.getArguments().get(0).getValue());
+ if (it.getTypeTag().equals(ATypeTag.ANY)) {
+ it = DefaultOpenFieldType.NESTED_OPEN_AORDERED_LIST_TYPE;
+ }
+ ((CastListDescriptor) fd).reset(rt, (AbstractCollectionType) it);
}
if (fd.getIdentifier().equals(AsterixBuiltinFunctions.OPEN_RECORD_CONSTRUCTOR)) {
ARecordType rt = (ARecordType) context.getType(expr);
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/IndexOperationTracker.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/IndexOperationTracker.java
index b3a6533..2016d08 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/IndexOperationTracker.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/opcallbacks/IndexOperationTracker.java
@@ -15,6 +15,8 @@
package edu.uci.ics.asterix.transaction.management.opcallbacks;
+import java.util.concurrent.atomic.AtomicInteger;
+
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
import edu.uci.ics.hyracks.storage.am.common.api.IModificationOperationCallback;
import edu.uci.ics.hyracks.storage.am.common.api.ISearchOperationCallback;
@@ -30,7 +32,7 @@
public class IndexOperationTracker implements ILSMOperationTracker {
// Number of active operations on a ILSMIndex instance.
- private int numActiveOperations = 0;
+ private AtomicInteger numActiveOperations;
private long lastLSN;
private long firstLSN;
private final ILSMIndex index;
@@ -38,6 +40,7 @@
private ILSMIndexAccessor accessor;
public IndexOperationTracker(ILSMIndex index, ILSMIOOperationCallbackFactory ioOpCallbackFactory) {
+ this.numActiveOperations = new AtomicInteger(0);
this.index = index;
//TODO
//This code is added to avoid NullPointException when the index's comparatorFactory is null.
@@ -54,7 +57,7 @@
public void beforeOperation(LSMOperationType opType, ISearchOperationCallback searchCallback,
IModificationOperationCallback modificationCallback) throws HyracksDataException {
if (opType != LSMOperationType.FORCE_MODIFICATION) {
- numActiveOperations++;
+ numActiveOperations.incrementAndGet();
// Increment transactor-local active operations count.
AbstractOperationCallback opCallback = getOperationCallback(searchCallback, modificationCallback);
@@ -76,7 +79,6 @@
@Override
public void completeOperation(LSMOperationType opType, ISearchOperationCallback searchCallback,
IModificationOperationCallback modificationCallback) throws HyracksDataException {
- numActiveOperations--;
// Decrement transactor-local active operations count.
AbstractOperationCallback opCallback = getOperationCallback(searchCallback, modificationCallback);
@@ -85,7 +87,7 @@
}
// If we need a flush, and this is the last completing operation, then schedule the flush.
// Once the flush has completed notify all waiting operations.
- if (index.getFlushStatus() && numActiveOperations == 0 && opType != LSMOperationType.FLUSH) {
+ if (index.getFlushStatus() && numActiveOperations.decrementAndGet() == 0 && opType != LSMOperationType.FLUSH) {
if (accessor == null) {
accessor = (ILSMIndexAccessor) index.createAccessor(NoOpOperationCallback.INSTANCE,
NoOpOperationCallback.INSTANCE);
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/FileBasedBuffer.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/FileBasedBuffer.java
index 4319b8b..e88d74e 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/FileBasedBuffer.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/FileBasedBuffer.java
@@ -19,6 +19,9 @@
import java.io.RandomAccessFile;
import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
* Represent a buffer that is backed by a physical file. Provider custom APIs
@@ -27,22 +30,32 @@
public class FileBasedBuffer extends Buffer implements IFileBasedBuffer {
private String filePath;
- private long nextWritePosition;
private FileChannel fileChannel;
private RandomAccessFile raf;
- private int size;
+ private int bufferSize;
- public FileBasedBuffer(String filePath, long offset, int size) throws IOException {
+ private int bufferLastFlushOffset;
+ private int bufferNextWriteOffset;
+ private final int diskSectorSize;
+
+ private final ReadWriteLock latch;
+ private final AtomicInteger referenceCount;
+
+ public FileBasedBuffer(String filePath, long offset, int bufferSize, int diskSectorSize) throws IOException {
this.filePath = filePath;
- this.nextWritePosition = offset;
- buffer = ByteBuffer.allocate(size);
+ buffer = ByteBuffer.allocate(bufferSize);
raf = new RandomAccessFile(new File(filePath), "rw");
- raf.seek(offset);
fileChannel = raf.getChannel();
+ fileChannel.position(offset);
fileChannel.read(buffer);
buffer.position(0);
- this.size = size;
- buffer.limit(size);
+ this.bufferSize = bufferSize;
+ buffer.limit(bufferSize);
+ bufferLastFlushOffset = 0;
+ bufferNextWriteOffset = 0;
+ this.diskSectorSize = diskSectorSize;
+ latch = new ReentrantReadWriteLock(true);
+ referenceCount = new AtomicInteger(0);
}
public String getFilePath() {
@@ -53,17 +66,9 @@
this.filePath = filePath;
}
- public long getOffset() {
- return nextWritePosition;
- }
-
- public void setOffset(long offset) {
- this.nextWritePosition = offset;
- }
-
@Override
public int getSize() {
- return buffer.limit();
+ return bufferSize;
}
public void clear() {
@@ -72,11 +77,18 @@
@Override
public void flush() throws IOException {
- buffer.position(0);
- buffer.limit(size);
+ //flush
+ int pos = bufferLastFlushOffset;
+ int limit = (((bufferNextWriteOffset - 1) / diskSectorSize) + 1) * diskSectorSize;
+ buffer.position(pos);
+ buffer.limit(limit);
fileChannel.write(buffer);
fileChannel.force(true);
- erase();
+
+ //update variables
+ bufferLastFlushOffset = limit;
+ bufferNextWriteOffset = limit;
+ buffer.limit(bufferSize);
}
@Override
@@ -124,45 +136,110 @@
* starting at offset.
*/
@Override
- public void reset(String filePath, long nextWritePosition, int size) throws IOException {
+ public void reset(String filePath, long diskNextWriteOffset, int bufferSize) throws IOException {
if (!filePath.equals(this.filePath)) {
raf.close();//required?
fileChannel.close();
raf = new RandomAccessFile(filePath, "rw");
this.filePath = filePath;
}
- this.nextWritePosition = nextWritePosition;
- raf.seek(nextWritePosition);
fileChannel = raf.getChannel();
+ fileChannel.position(diskNextWriteOffset);
erase();
buffer.position(0);
- buffer.limit(size);
- this.size = size;
+ buffer.limit(bufferSize);
+ this.bufferSize = bufferSize;
+
+ bufferLastFlushOffset = 0;
+ bufferNextWriteOffset = 0;
}
-
+
@Override
public void close() throws IOException {
fileChannel.close();
}
-
+
@Override
- public void open(String filePath, long offset, int size) throws IOException {
+ public void open(String filePath, long offset, int bufferSize) throws IOException {
raf = new RandomAccessFile(filePath, "rw");
- this.nextWritePosition = offset;
fileChannel = raf.getChannel();
fileChannel.position(offset);
erase();
buffer.position(0);
- buffer.limit(size);
- this.size = size;
+ buffer.limit(bufferSize);
+ this.bufferSize = bufferSize;
+ bufferLastFlushOffset = 0;
+ bufferNextWriteOffset = 0;
}
- public long getNextWritePosition() {
- return nextWritePosition;
+ @Override
+ public long getDiskNextWriteOffset() throws IOException {
+ return fileChannel.position();
}
- public void setNextWritePosition(long nextWritePosition) {
- this.nextWritePosition = nextWritePosition;
+ @Override
+ public void setDiskNextWriteOffset(long offset) throws IOException {
+ fileChannel.position(offset);
}
+ @Override
+ public int getBufferLastFlushOffset() {
+ return bufferLastFlushOffset;
+ }
+
+ @Override
+ public void setBufferLastFlushOffset(int offset) {
+ this.bufferLastFlushOffset = offset;
+ }
+
+ @Override
+ public int getBufferNextWriteOffset() {
+ synchronized (fileChannel) {
+ return bufferNextWriteOffset;
+ }
+ }
+
+ @Override
+ public void setBufferNextWriteOffset(int offset) {
+ synchronized (fileChannel) {
+ if (bufferNextWriteOffset < offset) {
+ bufferNextWriteOffset = offset;
+ }
+ }
+ }
+
+ @Override
+ public void acquireWriteLatch() {
+ latch.writeLock().lock();
+ }
+
+ @Override
+ public void releaseWriteLatch() {
+ latch.writeLock().unlock();
+ }
+
+ @Override
+ public void acquireReadLatch() {
+ latch.readLock().lock();
+ }
+
+ @Override
+ public void releaseReadLatch() {
+ latch.readLock().unlock();
+ }
+
+ @Override
+ public void incRefCnt() {
+ referenceCount.incrementAndGet();
+ }
+
+ @Override
+ public void decRefCnt() {
+ referenceCount.decrementAndGet();
+ }
+
+ @Override
+ public int getRefCnt() {
+ return referenceCount.get();
+ }
}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/FileUtil.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/FileUtil.java
index f2ffa0e..46e03f1 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/FileUtil.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/FileUtil.java
@@ -38,8 +38,8 @@
return (new File(path)).mkdir();
}
- public static IFileBasedBuffer getFileBasedBuffer(String filePath, long offset, int size) throws IOException {
- IFileBasedBuffer fileBasedBuffer = new FileBasedBuffer(filePath, offset, size);
+ public static IFileBasedBuffer getFileBasedBuffer(String filePath, long offset, int bufferSize, int diskSectorSize) throws IOException {
+ IFileBasedBuffer fileBasedBuffer = new FileBasedBuffer(filePath, offset, bufferSize, diskSectorSize);
return fileBasedBuffer;
}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/IFileBasedBuffer.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/IFileBasedBuffer.java
index e1f9f95..a4ea3cb 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/IFileBasedBuffer.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/IFileBasedBuffer.java
@@ -31,12 +31,34 @@
*/
public void reset(String filePath, long offset, int size) throws IOException;
- public long getNextWritePosition();
+ public long getDiskNextWriteOffset() throws IOException;
- public void setNextWritePosition(long writePosition);
+ public void setDiskNextWriteOffset(long writePosition) throws IOException;
public void close() throws IOException;
public void open(String filePath, long offset, int size) throws IOException;
+ public int getBufferLastFlushOffset();
+
+ public void setBufferLastFlushOffset(int offset);
+
+ public int getBufferNextWriteOffset();
+
+ public void setBufferNextWriteOffset(int offset);
+
+ public void acquireWriteLatch();
+
+ public void releaseWriteLatch();
+
+ public void acquireReadLatch();
+
+ public void releaseReadLatch();
+
+ public void incRefCnt();
+
+ public void decRefCnt();
+
+ public int getRefCnt();
+
}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/ILogManager.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/ILogManager.java
index c629d03..26229a7 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/ILogManager.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/ILogManager.java
@@ -53,17 +53,6 @@
ACIDException;
/**
- * Provides a cursor for retrieving logs that satisfy a given ILogFilter
- * instance. Log records are retrieved in increasing order of lsn
- *
- * @param logFilter
- * specifies the filtering criteria for the retrieved logs
- * @return LogCursor an iterator for the retrieved logs
- * @throws ACIDException
- */
- public ILogCursor readLog(ILogFilter logFilter) throws ACIDException;
-
- /**
* @param logicalLogLocator TODO
* @param PhysicalLogLocator
* specifies the location of the log record to be read
@@ -72,15 +61,6 @@
public void readLog(long lsnValue, LogicalLogLocator logicalLogLocator) throws ACIDException;
/**
- * Flushes the log records up to the lsn represented by the
- * logicalLogLocator
- *
- * @param logicalLogLocator
- * @throws ACIDException
- */
- public void flushLog(LogicalLogLocator logicalLogLocator) throws ACIDException;
-
- /**
* Retrieves the configuration parameters of the ILogManager
*
* @return LogManagerProperties: the configuration parameters for the
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/ILogRecordHelper.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/ILogRecordHelper.java
index 80f74cb..0e24f9d 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/ILogRecordHelper.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/ILogRecordHelper.java
@@ -25,41 +25,43 @@
public interface ILogRecordHelper {
- byte getLogType(LogicalLogLocator logicalLogLocator);
+ public byte getLogType(LogicalLogLocator logicalLogLocator);
- int getJobId(LogicalLogLocator logicalLogLocator);
+ public int getJobId(LogicalLogLocator logicalLogLocator);
- int getDatasetId(LogicalLogLocator logicalLogLocator);
+ public int getDatasetId(LogicalLogLocator logicalLogLocator);
- int getPKHashValue(LogicalLogLocator logicalLogLocator);
+ public int getPKHashValue(LogicalLogLocator logicalLogLocator);
- PhysicalLogLocator getPrevLSN(LogicalLogLocator logicalLogLocator);
+ public PhysicalLogLocator getPrevLSN(LogicalLogLocator logicalLogLocator);
- boolean getPrevLSN(PhysicalLogLocator physicalLogLocator, LogicalLogLocator logicalLogLocator);
+ public boolean getPrevLSN(PhysicalLogLocator physicalLogLocator, LogicalLogLocator logicalLogLocator);
- long getResourceId(LogicalLogLocator logicalLogLocator);
+ public long getResourceId(LogicalLogLocator logicalLogLocator);
- byte getResourceMgrId(LogicalLogLocator logicalLogLocater);
+ public byte getResourceMgrId(LogicalLogLocator logicalLogLocater);
- int getLogContentSize(LogicalLogLocator logicalLogLocater);
+ public int getLogContentSize(LogicalLogLocator logicalLogLocater);
- long getLogChecksum(LogicalLogLocator logicalLogLocator);
+ public long getLogChecksum(LogicalLogLocator logicalLogLocator);
- int getLogContentBeginPos(LogicalLogLocator logicalLogLocator);
+ public int getLogContentBeginPos(LogicalLogLocator logicalLogLocator);
- int getLogContentEndPos(LogicalLogLocator logicalLogLocator);
+ public int getLogContentEndPos(LogicalLogLocator logicalLogLocator);
- String getLogRecordForDisplay(LogicalLogLocator logicalLogLocator);
+ public String getLogRecordForDisplay(LogicalLogLocator logicalLogLocator);
- void writeLogHeader(LogicalLogLocator logicalLogLocator, byte logType, TransactionContext context, int datasetId,
+ public void writeLogHeader(LogicalLogLocator logicalLogLocator, byte logType, TransactionContext context, int datasetId,
int PKHashValue, long prevLogicalLogLocator, long resourceId, byte resourceMgrId, int logRecordSize);
- boolean validateLogRecord(LogicalLogLocator logicalLogLocator);
+ public boolean validateLogRecord(LogicalLogLocator logicalLogLocator);
- int getLogRecordSize(byte logType, int logBodySize);
+ public int getLogRecordSize(byte logType, int logBodySize);
- int getLogHeaderSize(byte logType);
+ public int getLogHeaderSize(byte logType);
- int getLogChecksumSize();
+ public int getLogChecksumSize();
+
+ public int getCommitLogSize();
}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogCursor.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogCursor.java
index 8a2b188..7e954d8 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogCursor.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogCursor.java
@@ -23,27 +23,16 @@
private final LogManager logManager;
private final ILogFilter logFilter;
+ private final int logPageSize;
private IBuffer readOnlyBuffer;
private LogicalLogLocator logicalLogLocator = null;
- private long bufferIndex = 0;
- private boolean firstNext = true;
- private boolean readMemory = false;
- private long readLSN = 0;
private boolean needReloadBuffer = true;
- /**
- * @param logFilter
- */
- public LogCursor(final LogManager logManager, ILogFilter logFilter) throws ACIDException {
+ public LogCursor(final LogManager logManager, PhysicalLogLocator startingPhysicalLogLocator, ILogFilter logFilter,
+ int logPageSize) throws IOException, ACIDException {
this.logFilter = logFilter;
this.logManager = logManager;
-
- }
-
- public LogCursor(final LogManager logManager, PhysicalLogLocator startingPhysicalLogLocator, ILogFilter logFilter)
- throws IOException, ACIDException {
- this.logFilter = logFilter;
- this.logManager = logManager;
+ this.logPageSize = logPageSize;
initialize(startingPhysicalLogLocator);
}
@@ -57,7 +46,8 @@
File file = new File(filePath);
if (file.exists()) {
return FileUtil.getFileBasedBuffer(filePath, lsn
- % logManager.getLogManagerProperties().getLogPartitionSize(), size);
+ % logManager.getLogManagerProperties().getLogPartitionSize(), size, logManager
+ .getLogManagerProperties().getDiskSectorSize());
} else {
return null;
}
@@ -87,8 +77,7 @@
return false;
}
- //if the lsn to read is greater than the last flushed lsn, then read from memory
- if (logicalLogLocator.getLsn() > logManager.getLastFlushedLsn().get()) {
+ if (logManager.isMemoryRead(logicalLogLocator.getLsn())) {
return readFromMemory(currentLogLocator);
}
@@ -96,10 +85,9 @@
//needReloadBuffer is set to true if the log record is read from the memory log page.
if (needReloadBuffer) {
//log page size doesn't exceed integer boundary
- int offset = (int)(logicalLogLocator.getLsn() % logManager.getLogManagerProperties().getLogPageSize());
+ int offset = (int) (logicalLogLocator.getLsn() % logPageSize);
long adjustedLSN = logicalLogLocator.getLsn() - offset;
- readOnlyBuffer = getReadOnlyBuffer(adjustedLSN, logManager.getLogManagerProperties()
- .getLogPageSize());
+ readOnlyBuffer = getReadOnlyBuffer(adjustedLSN, logPageSize);
logicalLogLocator.setBuffer(readOnlyBuffer);
logicalLogLocator.setMemoryOffset(offset);
needReloadBuffer = false;
@@ -110,14 +98,14 @@
while (logicalLogLocator.getMemoryOffset() <= readOnlyBuffer.getSize()
- logManager.getLogRecordHelper().getLogHeaderSize(LogType.COMMIT)) {
integerRead = readOnlyBuffer.readInt(logicalLogLocator.getMemoryOffset());
- if (integerRead == logManager.getLogManagerProperties().LOG_MAGIC_NUMBER) {
+ if (integerRead == LogManagerProperties.LOG_MAGIC_NUMBER) {
logRecordBeginPosFound = true;
break;
}
logicalLogLocator.increaseMemoryOffset(1);
logicalLogLocator.incrementLsn();
bytesSkipped++;
- if (bytesSkipped > logManager.getLogManagerProperties().getLogPageSize()) {
+ if (bytesSkipped > logPageSize) {
return false; // the maximum size of a log record is limited to
// a log page size. If we have skipped as many
// bytes without finding a log record, it
@@ -133,10 +121,9 @@
// need to reload the buffer
// TODO
// reduce IO by reading more pages(equal to logBufferSize) at a time.
- long lsnpos = ((logicalLogLocator.getLsn() / logManager.getLogManagerProperties().getLogPageSize()) + 1)
- * logManager.getLogManagerProperties().getLogPageSize();
+ long lsnpos = ((logicalLogLocator.getLsn() / logPageSize) + 1) * logPageSize;
- readOnlyBuffer = getReadOnlyBuffer(lsnpos, logManager.getLogManagerProperties().getLogPageSize());
+ readOnlyBuffer = getReadOnlyBuffer(lsnpos, logPageSize);
if (readOnlyBuffer != null) {
logicalLogLocator.setBuffer(readOnlyBuffer);
logicalLogLocator.setLsn(lsnpos);
@@ -190,13 +177,12 @@
IFileBasedBuffer logPage = logManager.getLogPage(pageIndex);
synchronized (logPage) {
// need to check again if the log record in the log buffer or has reached the disk
- if (lsn > logManager.getLastFlushedLsn().get()) {
+ if (logManager.isMemoryRead(lsn)) {
//find the magic number to identify the start of the log record
//----------------------------------------------------------------
int readNumber = -1;
- int logPageSize = logManager.getLogManagerProperties().getLogPageSize();
- int logMagicNumber = logManager.getLogManagerProperties().LOG_MAGIC_NUMBER;
+ int logMagicNumber = LogManagerProperties.LOG_MAGIC_NUMBER;
int bytesSkipped = 0;
boolean logRecordBeginPosFound = false;
//check whether the currentOffset has enough space to have new log record by comparing
@@ -223,7 +209,8 @@
// need to read the next log page
readOnlyBuffer = null;
logicalLogLocator.setBuffer(null);
- logicalLogLocator.setLsn(lsn / logPageSize + 1);
+ lsn = ((logicalLogLocator.getLsn() / logPageSize) + 1) * logPageSize;
+ logicalLogLocator.setLsn(lsn);
logicalLogLocator.setMemoryOffset(0);
return next(currentLogLocator);
}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogManager.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogManager.java
index 9b8f09c..199fd0f 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogManager.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogManager.java
@@ -27,15 +27,12 @@
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import java.util.logging.Level;
import java.util.logging.Logger;
import edu.uci.ics.asterix.transaction.management.exception.ACIDException;
import edu.uci.ics.asterix.transaction.management.service.logging.IndexLogger.ReusableLogContentObject;
-import edu.uci.ics.asterix.transaction.management.service.logging.LogManager.PageOwnershipStatus;
-import edu.uci.ics.asterix.transaction.management.service.logging.LogManager.PageState;
import edu.uci.ics.asterix.transaction.management.service.transaction.TransactionContext;
import edu.uci.ics.asterix.transaction.management.service.transaction.TransactionManagementConstants;
import edu.uci.ics.asterix.transaction.management.service.transaction.TransactionSubsystem;
@@ -48,6 +45,9 @@
private final TransactionSubsystem provider;
private LogManagerProperties logManagerProperties;
private LogPageFlushThread logPageFlusher;
+ private final int logPageSize;
+ private long statLogSize;
+ private long statLogCount;
/*
* the array of log pages. The number of log pages is configurable. Pages
@@ -62,47 +62,6 @@
*/
private int numLogPages;
- /*
- * Initially all pages have an owner count of 1 that is the LogManager. When
- * a transaction requests to write in a log page, the owner count is
- * incremented. The log manager reserves space in the log page and puts in
- * the log header but leaves the space for the content and the checksum
- * (covering the whole log record). When the content has been put, the log
- * manager computes the checksum and puts it after the content. At this
- * point, the ownership count is decremented as the transaction is done with
- * using the page. When a page is requested to be flushed, logPageFlusher
- * set the count to 0(LOG_FLUSHER: meaning that the page is being flushed)
- * only if the count is 1(LOG_WRITER: meaning that there is no other
- * transactions who own the page to write logs.) After flushing the page,
- * logPageFlusher set this count to 1.
- */
- private AtomicInteger[] logPageOwnerCount;
-
- static class PageOwnershipStatus {
- public static final int LOG_WRITER = 1;
- public static final int LOG_FLUSHER = 0;
- }
-
- /*
- * LogPageStatus: A page is either ACTIVE or INACTIVE. The status for each
- * page is maintained in logPageStatus. A page is ACTIVE when the LogManager
- * can allocate space in the page for writing a log record. Initially all
- * pages are ACTIVE. As transactions fill up space by writing log records, a
- * page may not have sufficient space left for serving a request by a
- * transaction. When this happens, the page is flushed to disk by calling
- * logPageFlusher.requestFlush(). In the requestFlush(), after
- * groupCommitWaitTime, the page status is set to INACTIVE. Then, there is
- * no more writer on the page(meaning the corresponding logPageOwnerCount is
- * 1), the page is flushed by the logPageFlusher and the status is reset to
- * ACTIVE by the logPageFlusher.
- */
- private AtomicInteger[] logPageStatus;
-
- static class PageState {
- public static final int INACTIVE = 0;
- public static final int ACTIVE = 1;
- }
-
private AtomicLong lastFlushedLSN = new AtomicLong(-1);
/*
@@ -129,10 +88,6 @@
return lastFlushedLSN;
}
- public AtomicInteger getLogPageStatus(int pageIndex) {
- return logPageStatus[pageIndex];
- }
-
public AtomicLong getCurrentLsn() {
return lsn;
}
@@ -144,13 +99,19 @@
public LogManager(TransactionSubsystem provider) throws ACIDException {
this.provider = provider;
initLogManagerProperties(this.provider.getId());
+ logPageSize = logManagerProperties.getLogPageSize();
initLogManager();
+ statLogSize = 0;
+ statLogCount = 0;
}
public LogManager(TransactionSubsystem provider, String nodeId) throws ACIDException {
this.provider = provider;
initLogManagerProperties(nodeId);
+ logPageSize = logManagerProperties.getLogPageSize();
initLogManager();
+ statLogSize = 0;
+ statLogCount = 0;
}
/*
@@ -186,9 +147,6 @@
private void initLogManager() throws ACIDException {
logRecordHelper = new LogRecordHelper(this);
numLogPages = logManagerProperties.getNumLogPages();
- logPageOwnerCount = new AtomicInteger[numLogPages];
- logPageStatus = new AtomicInteger[numLogPages];
-
activeTxnCountMaps = new ArrayList<HashMap<TransactionContext, Integer>>(numLogPages);
for (int i = 0; i < numLogPages; i++) {
activeTxnCountMaps.add(new HashMap<TransactionContext, Integer>());
@@ -199,20 +157,12 @@
/*
* place the log anchor at the end of the last log record written.
*/
- PhysicalLogLocator nextPhysicalLsn = initLSN();
-
- /*
- * initialize meta data for each log page.
- */
- for (int i = 0; i < numLogPages; i++) {
- logPageOwnerCount[i] = new AtomicInteger(PageOwnershipStatus.LOG_WRITER);
- logPageStatus[i] = new AtomicInteger(PageState.ACTIVE);
- }
+ initLSN();
/*
* initialize the log pages.
*/
- initializeLogPages(nextPhysicalLsn);
+ initializeLogPages(startingLSN);
/*
* Instantiate and begin the LogFlusher thread. The Log Flusher thread
@@ -226,7 +176,7 @@
}
public int getLogPageIndex(long lsnValue) {
- return (int) (((lsnValue - startingLSN) / logManagerProperties.getLogPageSize()) % numLogPages);
+ return (int) (((lsnValue - startingLSN) / logPageSize) % numLogPages);
}
/*
@@ -242,28 +192,7 @@
* record is (to be) placed.
*/
public int getLogPageOffset(long lsnValue) {
- return (int) ((lsnValue - startingLSN) % logManagerProperties.getLogPageSize());
- }
-
- /*
- * a transaction thread under certain scenarios is required to wait until
- * the page where it has to write a log record becomes available for writing
- * a log record.
- */
- private void waitUntillPageIsAvailableForWritingLog(int pageIndex) throws ACIDException {
- if (logPageStatus[pageIndex].get() == PageState.ACTIVE
- && logPageOwnerCount[pageIndex].get() >= PageOwnershipStatus.LOG_WRITER) {
- return;
- }
- try {
- synchronized (logPages[pageIndex]) {
- while (!(logPageStatus[pageIndex].get() == PageState.ACTIVE && logPageOwnerCount[pageIndex].get() >= PageOwnershipStatus.LOG_WRITER)) {
- logPages[pageIndex].wait();
- }
- }
- } catch (InterruptedException e) {
- throw new ACIDException(" thread interrupted while waiting for page " + pageIndex + " to be available ", e);
- }
+ return (int) (lsnValue % logPageSize);
}
/*
@@ -277,7 +206,6 @@
* @param logType: the type of log record.
*/
private long getLsn(int entrySize, byte logType) throws ACIDException {
- long pageSize = logManagerProperties.getLogPageSize();
while (true) {
boolean forwardPage = false;
@@ -294,9 +222,9 @@
// check if the log record will cross page boundaries, a case that
// is not allowed.
- if ((next - 1) / pageSize != old / pageSize || (next % pageSize == 0)) {
+ if ((next - 1) / logPageSize != old / logPageSize || (next % logPageSize == 0)) {
- if ((old != 0 && old % pageSize == 0)) {
+ if ((old != 0 && old % logPageSize == 0)) {
// On second thought, this shall never be the case as it
// means that the lsn is
// currently at the beginning of a page and we still need to
@@ -309,7 +237,7 @@
} else {
// set the lsn to point to the beginning of the next page.
- retVal = ((old / pageSize) + 1) * pageSize;
+ retVal = ((old / logPageSize) + 1) * logPageSize;
}
next = retVal;
@@ -323,20 +251,6 @@
pageIndex = getNextPageInSequence(pageIndex);
}
- /*
- * we do not want to keep allocating LSNs if the corresponding page
- * is unavailable. Consider a scenario when the log flusher thread
- * is incredibly slow in flushing pages. Transaction threads will
- * acquire an lsn each for writing their next log record. When a
- * page has been made available, mulltiple transaction threads that
- * were waiting can continue to write their log record at the
- * assigned LSNs. Two transaction threads may get LSNs that are on
- * the same log page but actually differ by the size of the log
- * buffer. This would be erroneous. Transaction threads are made to
- * wait upfront for avoiding this situation.
- */
- waitUntillPageIsAvailableForWritingLog(pageIndex);
-
if (!lsn.compareAndSet(old, next)) {
// Atomic call -> returns true only when the value represented
// by lsn is same as
@@ -345,6 +259,10 @@
}
if (forwardPage) {
+
+ // forward the nextWriteOffset in the log page
+ logPages[pageIndex].setBufferNextWriteOffset(logPageSize);
+
addFlushRequest(prevPage, old, false);
// The transaction thread that discovers the need to forward a
@@ -352,21 +270,18 @@
continue;
} else {
- // the transaction thread has been given a space in a log page,
- // but is made to wait until the page is available.
- // (Is this needed? when does this wait happen?)
- waitUntillPageIsAvailableForWritingLog(pageIndex);
-
+ logPages[pageIndex].acquireReadLatch();
// increment the counter as the transaction thread now holds a
// space in the log page and hence is an owner.
- logPageOwnerCount[pageIndex].incrementAndGet();
+ logPages[pageIndex].incRefCnt();
+ logPages[pageIndex].releaseReadLatch();
// Before the count is incremented, if the flusher flushed the
// allocated page,
// then retry to get new LSN. Otherwise, the log with allocated
// lsn will be lost.
if (lastFlushedLSN.get() >= retVal) {
- logPageOwnerCount[pageIndex].decrementAndGet();
+ logPages[pageIndex].decRefCnt();
continue;
}
}
@@ -396,10 +311,10 @@
int totalLogSize = logRecordHelper.getLogRecordSize(logType, logContentSize);
// check for the total space requirement to be less than a log page.
- if (totalLogSize > logManagerProperties.getLogPageSize()) {
+ if (totalLogSize > logPageSize) {
throw new ACIDException(
" Maximum Log Content Size is "
- + (logManagerProperties.getLogPageSize() - logRecordHelper.getLogHeaderSize(LogType.UPDATE) - logRecordHelper
+ + (logPageSize - logRecordHelper.getLogHeaderSize(LogType.UPDATE) - logRecordHelper
.getLogChecksumSize()));
}
@@ -482,16 +397,21 @@
logPages[pageIndex].writeLong(pageOffset + logRecordHelper.getLogHeaderSize(logType) + logContentSize,
checksum);
- if (IS_DEBUG_MODE) {
- System.out.println("--------------> LSN(" + currentLSN + ") is written");
+ // forward the nextWriteOffset in the log page
+ int bufferNextWriteOffset = (int) ((currentLSN + totalLogSize) % logPageSize);
+ if (bufferNextWriteOffset == 0) {
+ bufferNextWriteOffset = logPageSize;
}
+ logPages[pageIndex].setBufferNextWriteOffset(bufferNextWriteOffset);
- // release the ownership as the log record has been placed in
- // created space.
- logPageOwnerCount[pageIndex].decrementAndGet();
+ if (logType != LogType.ENTITY_COMMIT) {
+ // release the ownership as the log record has been placed in
+ // created space.
+ logPages[pageIndex].decRefCnt();
- // indicating that the transaction thread has released ownership
- decremented = true;
+ // indicating that the transaction thread has released ownership
+ decremented = true;
+ }
if (logType == LogType.ENTITY_COMMIT) {
map = activeTxnCountMaps.get(pageIndex);
@@ -502,18 +422,42 @@
} else {
map.put(txnCtx, 1);
}
+ //------------------------------------------------------------------------------
+ // [Notice]
+ // reference count should be decremented
+ // after activeTxnCount is incremented, but before addFlushRequest() is called.
+ //------------------------------------------------------------------------------
+ // release the ownership as the log record has been placed in
+ // created space.
+ logPages[pageIndex].decRefCnt();
+
+ // indicating that the transaction thread has released ownership
+ decremented = true;
+
addFlushRequest(pageIndex, currentLSN, false);
} else if (logType == LogType.COMMIT) {
+
addFlushRequest(pageIndex, currentLSN, true);
+ if (IS_DEBUG_MODE) {
+ System.out.println("Running sum of log size: " + statLogSize + ", log count: " + statLogCount);
+ }
}
+ if (IS_DEBUG_MODE) {
+ System.out.println("--------------> LSN(" + currentLSN + ") is written");
+ }
+
+ //collect statistics
+ statLogSize += totalLogSize;
+ statLogCount++;
+
} catch (Exception e) {
e.printStackTrace();
throw new ACIDException(txnCtx, "Thread: " + Thread.currentThread().getName()
+ " logger encountered exception", e);
} finally {
if (!decremented) {
- logPageOwnerCount[pageIndex].decrementAndGet();
+ logPages[pageIndex].decRefCnt();
}
}
}
@@ -526,20 +470,13 @@
String filePath = LogUtil.getLogFilePath(logManagerProperties, getLogFileId(lsn));
- logPages[pageIndex].reset(filePath, LogUtil.getFileOffset(this, nextWritePosition),
- logManagerProperties.getLogPageSize());
- }
-
- @Override
- public ILogCursor readLog(ILogFilter logFilter) throws ACIDException {
- LogCursor cursor = new LogCursor(this, logFilter);
- return cursor;
+ logPages[pageIndex].reset(filePath, LogUtil.getFileOffset(this, nextWritePosition), logPageSize);
}
@Override
public ILogCursor readLog(PhysicalLogLocator physicalLogLocator, ILogFilter logFilter) throws IOException,
ACIDException {
- LogCursor cursor = new LogCursor(this, physicalLogLocator, logFilter);
+ LogCursor cursor = new LogCursor(this, physicalLogLocator, logFilter, logPageSize);
return cursor;
}
@@ -550,7 +487,7 @@
String filePath = LogUtil.getLogFilePath(logManagerProperties, LogUtil.getFileId(this, lsnValue));
long fileOffset = LogUtil.getFileOffset(this, lsnValue);
- ByteBuffer buffer = ByteBuffer.allocate(logManagerProperties.getLogPageSize());
+ ByteBuffer buffer = ByteBuffer.allocate(logPageSize);
RandomAccessFile raf = null;
FileChannel fileChannel = null;
try {
@@ -603,7 +540,7 @@
}
/* check if the log record in the log buffer or has reached the disk. */
- if (lsnValue > getLastFlushedLsn().get()) {
+ if (isMemoryRead(lsnValue)) {
int pageIndex = getLogPageIndex(lsnValue);
int pageOffset = getLogPageOffset(lsnValue);
@@ -611,7 +548,7 @@
// minimize memory allocation overhead. current code allocates the
// log page size per reading a log record.
- byte[] pageContent = new byte[logManagerProperties.getLogPageSize()];
+ byte[] pageContent = new byte[logPageSize];
// take a lock on the log page so that the page is not flushed to
// disk interim
@@ -619,8 +556,8 @@
// need to check again (this thread may have got de-scheduled
// and must refresh!)
- if (lsnValue > getLastFlushedLsn().get()) {
+ if (isMemoryRead(lsnValue)) {
// get the log record length
logPages[pageIndex].getBytes(pageContent, 0, pageContent.length);
byte logType = pageContent[pageOffset + 4];
@@ -656,6 +593,20 @@
readDiskLog(lsnValue, logicalLogLocator);
}
+ public boolean isMemoryRead(long currentLSN) {
+ long flushLSN = lastFlushedLSN.get();
+ if ((flushLSN + 1) % logPageSize == 0) {
+ return false;
+ }
+ long logPageBeginOffset = flushLSN - (flushLSN % logPageSize);
+ long logPageEndOffset = logPageBeginOffset + logPageSize;
+ if (currentLSN > flushLSN || (currentLSN >= logPageBeginOffset && currentLSN < logPageEndOffset)) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
public void renewLogFiles() throws ACIDException {
List<String> logFileNames = LogUtil.getLogFiles(logManagerProperties);
for (String name : logFileNames) {
@@ -670,7 +621,7 @@
logPageFlusher.renew();
}
- private PhysicalLogLocator initLSN() throws ACIDException {
+ private void initLSN() throws ACIDException {
PhysicalLogLocator nextPhysicalLsn = LogUtil.initializeLogAnchor(this);
startingLSN = nextPhysicalLsn.getLsn();
lastFlushedLSN.set(startingLSN - 1);
@@ -678,7 +629,6 @@
LOGGER.info(" Starting lsn is : " + startingLSN);
}
lsn.set(startingLSN);
- return nextPhysicalLsn;
}
private void closeLogPages() throws ACIDException {
@@ -695,9 +645,7 @@
try {
String filePath = LogUtil.getLogFilePath(logManagerProperties, LogUtil.getFileId(this, startingLSN));
for (int i = 0; i < numLogPages; i++) {
- logPages[i].open(filePath,
- LogUtil.getFileOffset(this, startingLSN) + i * logManagerProperties.getLogPageSize(),
- logManagerProperties.getLogPageSize());
+ logPages[i].open(filePath, LogUtil.getFileOffset(this, startingLSN) + i * logPageSize, logPageSize);
}
} catch (Exception e) {
throw new ACIDException(Thread.currentThread().getName() + " unable to create log buffer", e);
@@ -710,33 +658,25 @@
}
/*
- * This method shall be called by the Buffer manager when it needs to evict
- * a page from the cache. TODO: Change the implementation from a looping
- * logic to event based when log manager support is integrated with the
- * Buffer Manager.
- */
- @Override
- public synchronized void flushLog(LogicalLogLocator logicalLogLocator) throws ACIDException {
- if (logicalLogLocator.getLsn() > lsn.get()) {
- throw new ACIDException(" invalid lsn " + logicalLogLocator.getLsn());
- }
- while (lastFlushedLSN.get() < logicalLogLocator.getLsn());
- }
-
- /*
* Map each log page to cover a physical byte range over a log file. When a
* page is flushed, the page contents are put to disk in the corresponding
* byte range.
*/
- private void initializeLogPages(PhysicalLogLocator physicalLogLocator) throws ACIDException {
+ private void initializeLogPages(long beginLsn) throws ACIDException {
try {
- String filePath = LogUtil.getLogFilePath(logManagerProperties,
- LogUtil.getFileId(this, physicalLogLocator.getLsn()));
+ String filePath = LogUtil.getLogFilePath(logManagerProperties, LogUtil.getFileId(this, beginLsn));
+ long nextDiskWriteOffset = LogUtil.getFileOffset(this, beginLsn);
+ long nextBufferWriteOffset = nextDiskWriteOffset % logPageSize;
+ long bufferBeginOffset = nextDiskWriteOffset - nextBufferWriteOffset;
+
for (int i = 0; i < numLogPages; i++) {
- logPages[i] = FileUtil.getFileBasedBuffer(
- filePath,
- LogUtil.getFileOffset(this, physicalLogLocator.getLsn()) + i
- * logManagerProperties.getLogPageSize(), logManagerProperties.getLogPageSize());
+ logPages[i] = FileUtil.getFileBasedBuffer(filePath, bufferBeginOffset + i * logPageSize, logPageSize,
+ logManagerProperties.getDiskSectorSize());
+ if (i == 0) {
+ logPages[i].setBufferLastFlushOffset((int) nextBufferWriteOffset);
+ logPages[i].setBufferNextWriteOffset((int) nextBufferWriteOffset);
+ logPages[i].setDiskNextWriteOffset(nextDiskWriteOffset);
+ }
}
} catch (Exception e) {
e.printStackTrace();
@@ -764,10 +704,6 @@
return logPages[pageIndex];
}
- public AtomicInteger getLogPageOwnershipCount(int pageIndex) {
- return logPageOwnerCount[pageIndex];
- }
-
public IFileBasedBuffer[] getLogPages() {
return logPages;
}
@@ -829,7 +765,7 @@
*/
private final LinkedBlockingQueue<Object>[] flushRequestQueue;
private final Object[] flushRequests;
- private int pageToFlush;
+ private int flushPageIndex;
private final long groupCommitWaitPeriod;
private boolean isRenewRequest;
@@ -843,14 +779,14 @@
flushRequestQueue[i] = new LinkedBlockingQueue<Object>(1);
flushRequests[i] = new Object();
}
- this.pageToFlush = -1;
+ this.flushPageIndex = 0;
groupCommitWaitPeriod = logManager.getLogManagerProperties().getGroupCommitWaitPeriod();
isRenewRequest = false;
}
public void renew() {
isRenewRequest = true;
- pageToFlush = -1;
+ flushPageIndex = 0;
this.interrupt();
isRenewRequest = false;
}
@@ -886,15 +822,19 @@
@Override
public void run() {
+ int logPageSize = logManager.getLogManagerProperties().getLogPageSize();
+ int logBufferSize = logManager.getLogManagerProperties().getLogBufferSize();
+ int beforeFlushOffset = 0;
+ int afterFlushOffset = 0;
+ boolean resetFlushPageIndex = false;
+
while (true) {
try {
- pageToFlush = logManager.getNextPageInSequence(pageToFlush);
-
// A wait call on the linkedBLockingQueue. The flusher thread is
// notified when an object is added to the queue. Please note
// that each page has an associated blocking queue.
try {
- flushRequestQueue[pageToFlush].take();
+ flushRequestQueue[flushPageIndex].take();
} catch (InterruptedException ie) {
while (isRenewRequest) {
sleep(1);
@@ -902,58 +842,67 @@
continue;
}
- synchronized (logManager.getLogPage(pageToFlush)) {
-
- // #. sleep during the groupCommitWaitTime
+ //if the log page is already full, don't wait.
+ if (logManager.getLogPage(flushPageIndex).getBufferNextWriteOffset() < logPageSize
+ - logManager.getLogRecordHelper().getCommitLogSize()) {
+ // #. sleep for the groupCommitWaitTime
sleep(groupCommitWaitPeriod);
+ }
- // #. set the logPageStatus to INACTIVE in order to prevent
- // other txns from writing on this page.
- logManager.getLogPageStatus(pageToFlush).set(PageState.INACTIVE);
+ synchronized (logManager.getLogPage(flushPageIndex)) {
+ logManager.getLogPage(flushPageIndex).acquireWriteLatch();
+ try {
- // #. need to wait until the logPageOwnerCount reaches 1
- // (LOG_WRITER)
- // meaning every one has finished writing logs on this page.
- while (logManager.getLogPageOwnershipCount(pageToFlush).get() != PageOwnershipStatus.LOG_WRITER) {
- sleep(0);
+ // #. need to wait until the reference count reaches 0
+ while (logManager.getLogPage(flushPageIndex).getRefCnt() != 0) {
+ sleep(0);
+ }
+
+ beforeFlushOffset = logManager.getLogPage(flushPageIndex).getBufferLastFlushOffset();
+
+ // put the content to disk (the thread still has a lock on the log page)
+ logManager.getLogPage(flushPageIndex).flush();
+
+ afterFlushOffset = logManager.getLogPage(flushPageIndex).getBufferLastFlushOffset();
+
+ // increment the last flushed lsn
+ logManager.incrementLastFlushedLsn(afterFlushOffset - beforeFlushOffset);
+
+ // increment currentLSN if currentLSN is less than flushLSN.
+ if (logManager.getLastFlushedLsn().get() + 1 > logManager.getCurrentLsn().get()) {
+ logManager.getCurrentLsn().set(logManager.getLastFlushedLsn().get() + 1);
+ }
+
+ // Map the log page to a new region in the log file if the flushOffset reached the logPageSize
+ if (afterFlushOffset == logPageSize) {
+ long diskNextWriteOffset = logManager.getLogPages()[flushPageIndex].getDiskNextWriteOffset()
+ + logBufferSize;
+ logManager.resetLogPage(logManager.getLastFlushedLsn().get() + 1 + logBufferSize,
+ diskNextWriteOffset, flushPageIndex);
+ resetFlushPageIndex = true;
+ }
+
+ // decrement activeTxnCountOnIndexes
+ logManager.decrementActiveTxnCountOnIndexes(flushPageIndex);
+
+ } finally {
+ logManager.getLogPage(flushPageIndex).releaseWriteLatch();
}
- // #. set the logPageOwnerCount to 0 (LOG_FLUSHER)
- // meaning it is flushing.
- logManager.getLogPageOwnershipCount(pageToFlush).set(PageOwnershipStatus.LOG_FLUSHER);
-
- // put the content to disk (the thread still has a lock on
- // the log page)
- logManager.getLogPage(pageToFlush).flush();
-
- // Map the log page to a new region in the log file.
- long nextWritePosition = logManager.getLogPages()[pageToFlush].getNextWritePosition()
- + logManager.getLogManagerProperties().getLogBufferSize();
-
- logManager.resetLogPage(logManager.getLastFlushedLsn().get() + 1
- + logManager.getLogManagerProperties().getLogBufferSize(), nextWritePosition, pageToFlush);
-
- // increment the last flushed lsn and lastFlushedPage
- logManager.incrementLastFlushedLsn(logManager.getLogManagerProperties().getLogPageSize());
-
- // decrement activeTxnCountOnIndexes
- logManager.decrementActiveTxnCountOnIndexes(pageToFlush);
-
- // reset the count to 1
- logManager.getLogPageOwnershipCount(pageToFlush).set(PageOwnershipStatus.LOG_WRITER);
-
- // mark the page as ACTIVE
- logManager.getLogPageStatus(pageToFlush).set(LogManager.PageState.ACTIVE);
-
// #. checks the queue whether there is another flush
// request on the same log buffer
// If there is another request, then simply remove it.
- if (flushRequestQueue[pageToFlush].peek() != null) {
- flushRequestQueue[pageToFlush].take();
+ if (flushRequestQueue[flushPageIndex].peek() != null) {
+ flushRequestQueue[flushPageIndex].take();
}
// notify all waiting (transaction) threads.
- logManager.getLogPage(pageToFlush).notifyAll();
+ logManager.getLogPage(flushPageIndex).notifyAll();
+
+ if (resetFlushPageIndex) {
+ flushPageIndex = logManager.getNextPageInSequence(flushPageIndex);
+ resetFlushPageIndex = false;
+ }
}
} catch (IOException ioe) {
ioe.printStackTrace();
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogManagerProperties.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogManagerProperties.java
index 5040fa9..581ce4c 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogManagerProperties.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogManagerProperties.java
@@ -28,6 +28,7 @@
public static final String NUM_LOG_PAGES_KEY = "num_log_pages";
public static final String LOG_FILE_PREFIX_KEY = "log_file_prefix";
public static final String GROUP_COMMIT_WAIT_PERIOD_KEY = "group_commit_wait_period";
+ public static final String DISK_SECTOR_SIZE_KEY = "disk_sector_size";
private static final int DEFAULT_LOG_PAGE_SIZE = 128 * 1024; //128KB
private static final int DEFAULT_NUM_LOG_PAGES = 8;
@@ -35,6 +36,7 @@
private static final long DEFAULT_GROUP_COMMIT_WAIT_PERIOD = 200; // time in millisec.
private static final String DEFAULT_LOG_FILE_PREFIX = "asterix_transaction_log";
private static final String DEFAULT_LOG_DIRECTORY = "asterix_logs/";
+ private static final int DEFAULT_DISK_SECTOR_SIZE = 4096;
// follow the naming convention <logFilePrefix>_<number> where number starts from 0
private final String logFilePrefix;
@@ -51,6 +53,8 @@
private final int logBufferSize;
// maximum size of each log file
private final long logPartitionSize;
+ // default disk sector size
+ private final int diskSectorSize;
public LogManagerProperties(Properties properties, String nodeId) {
this.logDirKey = new String(nodeId + LOG_DIR_SUFFIX_KEY);
@@ -66,6 +70,8 @@
this.logBufferSize = logPageSize * numLogPages;
//make sure that the log partition size is the multiple of log buffer size.
this.logPartitionSize = (logPartitionSize / logBufferSize) * logBufferSize;
+ this.diskSectorSize = Integer.parseInt(properties.getProperty(DISK_SECTOR_SIZE_KEY, ""
+ + DEFAULT_DISK_SECTOR_SIZE));
}
public long getLogPartitionSize() {
@@ -99,6 +105,10 @@
public String getLogDirKey() {
return logDirKey;
}
+
+ public int getDiskSectorSize() {
+ return diskSectorSize;
+ }
public String toString() {
StringBuilder builder = new StringBuilder();
@@ -108,6 +118,7 @@
builder.append("num_log_pages : " + numLogPages + FileUtil.lineSeparator);
builder.append("log_partition_size : " + logPartitionSize + FileUtil.lineSeparator);
builder.append("group_commit_wait_period : " + groupCommitWaitPeriod + FileUtil.lineSeparator);
+ builder.append("disk_sector_size : " + diskSectorSize + FileUtil.lineSeparator);
return builder.toString();
}
}
diff --git a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogRecordHelper.java b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogRecordHelper.java
index 6b882ef..1b65d8f 100644
--- a/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogRecordHelper.java
+++ b/asterix-transactions/src/main/java/edu/uci/ics/asterix/transaction/management/service/logging/LogRecordHelper.java
@@ -50,6 +50,9 @@
public class LogRecordHelper implements ILogRecordHelper {
private final int LOG_CHECKSUM_SIZE = 8;
+ private final int LOG_HEADER_PART1_SIZE = 17;
+ private final int LOG_HEADER_PART2_SIZE = 21;
+ private final int COMMIT_LOG_SIZE = LOG_HEADER_PART1_SIZE + LOG_CHECKSUM_SIZE;
private final int MAGIC_NO_POS = 0;
private final int LOG_TYPE_POS = 4;
@@ -60,7 +63,9 @@
private final int RESOURCE_ID_POS = 25;
private final int RESOURCE_MGR_ID_POS = 33;
private final int LOG_RECORD_SIZE_POS = 34;
+
+
private ILogManager logManager;
public LogRecordHelper(ILogManager logManager) {
@@ -118,7 +123,11 @@
@Override
public int getLogContentSize(LogicalLogLocator logicalLogLocater) {
- return logicalLogLocater.getBuffer().readInt(logicalLogLocater.getMemoryOffset() + LOG_RECORD_SIZE_POS);
+ if (getLogType(logicalLogLocater) == LogType.COMMIT || getLogType(logicalLogLocater) == LogType.ENTITY_COMMIT) {
+ return 0;
+ } else {
+ return logicalLogLocater.getBuffer().readInt(logicalLogLocater.getMemoryOffset() + LOG_RECORD_SIZE_POS);
+ }
}
@Override
@@ -178,7 +187,7 @@
/* magic no */
(logicalLogLocator.getBuffer()).writeInt(logicalLogLocator.getMemoryOffset() + MAGIC_NO_POS,
- logManager.getLogManagerProperties().LOG_MAGIC_NUMBER);
+ LogManagerProperties.LOG_MAGIC_NUMBER);
/* log type */
(logicalLogLocator.getBuffer()).put(logicalLogLocator.getMemoryOffset() + LOG_TYPE_POS, logType);
@@ -230,18 +239,18 @@
@Override
public int getLogRecordSize(byte logType, int logBodySize) {
if (logType == LogType.UPDATE) {
- return 46 + logBodySize;
+ return LOG_HEADER_PART1_SIZE + LOG_HEADER_PART2_SIZE + LOG_CHECKSUM_SIZE + logBodySize;
} else {
- return 25;
+ return COMMIT_LOG_SIZE;
}
}
@Override
public int getLogHeaderSize(byte logType) {
if (logType == LogType.UPDATE) {
- return 38;
+ return LOG_HEADER_PART1_SIZE + LOG_HEADER_PART2_SIZE;
} else {
- return 17;
+ return LOG_HEADER_PART1_SIZE;
}
}
@@ -249,4 +258,8 @@
public int getLogChecksumSize() {
return LOG_CHECKSUM_SIZE;
}
+
+ public int getCommitLogSize() {
+ return COMMIT_LOG_SIZE;
+ }
}
diff --git a/pom.xml b/pom.xml
index 20efdb3..f4c46c6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -84,6 +84,7 @@
<module>asterix-server</module>
<module>asterix-installer</module>
<module>asterix-events</module>
+ <module>asterix-doc</module>
</modules>
<repositories>