Merge branch 'master' into zheilbron/asterix_lsm_stabilization_config_changes
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
index 49ca5ba..792375e 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
@@ -3,7 +3,8 @@
import java.util.ArrayList;
import java.util.List;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.api.AsterixAppContextInfoImpl;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.common.dataflow.IAsterixApplicationContextInfo;
import edu.uci.ics.asterix.metadata.MetadataException;
@@ -196,20 +197,23 @@
IBinaryTokenizerFactory queryTokenizerFactory = InvertedIndexAccessMethod.getBinaryTokenizerFactory(
searchModifierType, searchKeyType, secondaryIndex);
IIndexDataflowHelperFactory dataflowHelperFactory;
+
+ AsterixStorageProperties storageProperties = ((AsterixAppContextInfoImpl) AsterixAppContextInfoImpl
+ .getInstance()).getStorageProperties();
if (!isPartitioned) {
dataflowHelperFactory = new LSMInvertedIndexDataflowHelperFactory(
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages());
} else {
dataflowHelperFactory = new PartitionedLSMInvertedIndexDataflowHelperFactory(
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages());
}
LSMInvertedIndexSearchOperatorDescriptor invIndexSearchOp = new LSMInvertedIndexSearchOperatorDescriptor(
jobSpec, queryField, appContext.getStorageManagerInterface(), secondarySplitsAndConstraint.first,
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java
index 09fc2b9..ff854d1 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/translator/AqlExpressionToPlanTranslator.java
@@ -71,7 +71,8 @@
import edu.uci.ics.asterix.aql.expression.WriteStatement;
import edu.uci.ics.asterix.aql.expression.visitor.IAqlExpressionVisitor;
import edu.uci.ics.asterix.aql.util.FunctionUtils;
-import edu.uci.ics.asterix.common.config.AsterixProperties;
+import edu.uci.ics.asterix.common.api.AsterixAppContextInfoImpl;
+import edu.uci.ics.asterix.common.config.AsterixMetadataProperties;
import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.common.functions.FunctionConstants;
@@ -288,7 +289,9 @@
String outputDir = System.getProperty("java.io.tmpDir");
String filePath = outputDir + System.getProperty("file.separator") + OUTPUT_FILE_PREFIX
+ outputFileID.incrementAndGet();
- return new FileSplit(AsterixProperties.INSTANCE.getMetadataNodeName(), new FileReference(new File(filePath)));
+ AsterixMetadataProperties metadataProperties = ((AsterixAppContextInfoImpl) AsterixAppContextInfoImpl
+ .getInstance()).getMetadataProperties();
+ return new FileSplit(metadataProperties.getMetadataNodeName(), new FileReference(new File(filePath)));
}
@Override
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java
index 862c701..c840ad2 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/api/common/APIFramework.java
@@ -27,7 +27,7 @@
import edu.uci.ics.asterix.aql.expression.visitor.AQLPrintVisitor;
import edu.uci.ics.asterix.aql.rewrites.AqlRewriter;
import edu.uci.ics.asterix.common.api.AsterixAppContextInfoImpl;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.config.AsterixCompilerProperties;
import edu.uci.ics.asterix.common.config.OptimizationConfUtil;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.dataflow.data.common.AqlExpressionTypeComputer;
@@ -258,7 +258,9 @@
}
}
- int frameSize = GlobalConfig.getFrameSize();
+ AsterixCompilerProperties compilerProperties = ((AsterixAppContextInfoImpl) AsterixAppContextInfoImpl
+ .getInstance()).getCompilerProperties();
+ int frameSize = compilerProperties.getFrameSize();
HeuristicCompilerFactoryBuilder builder = new HeuristicCompilerFactoryBuilder(
AqlOptimizationContextFactory.INSTANCE);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
index b9b9524..2173aea 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
@@ -22,6 +22,8 @@
import java.util.logging.Logger;
import edu.uci.ics.asterix.api.common.Job;
+import edu.uci.ics.asterix.common.api.AsterixAppContextInfoImpl;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
import edu.uci.ics.asterix.common.config.GlobalConfig;
import edu.uci.ics.asterix.common.config.OptimizationConfUtil;
@@ -118,14 +120,16 @@
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider
.splitProviderAndPartitionConstraintsForInternalOrFeedDataset(dataset.getDataverseName(), datasetName,
datasetName);
+ AsterixStorageProperties storageProperties = ((AsterixAppContextInfoImpl) AsterixAppContextInfoImpl
+ .getInstance()).getStorageProperties();
IndexDropOperatorDescriptor primaryBtreeDrop = new IndexDropOperatorDescriptor(specPrimary,
AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER, AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
splitsAndConstraint.first, new LSMBTreeDataflowHelperFactory(
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES));
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()));
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(specPrimary, primaryBtreeDrop,
splitsAndConstraint.second);
@@ -163,10 +167,12 @@
}
LOGGER.info("CREATING File Splits: " + sb.toString());
+ AsterixStorageProperties storageProperties = ((AsterixAppContextInfoImpl) AsterixAppContextInfoImpl
+ .getInstance()).getStorageProperties();
//prepare a LocalResourceMetadata which will be stored in NC's local resource repository
ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(typeTraits,
- comparatorFactories, blooFilterKeyFields, true, GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
+ comparatorFactories, blooFilterKeyFields, true, storageProperties.getMemoryComponentPageSize(),
+ storageProperties.getMemoryComponentNumPages());
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMBTreeResource);
@@ -176,9 +182,9 @@
new LSMBTreeDataflowHelperFactory(AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES), localResourceFactoryProvider,
- NoOpOperationCallbackFactory.INSTANCE);
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
+ localResourceFactoryProvider, NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, indexCreateOp,
splitsAndConstraint.second);
spec.addRoot(indexCreateOp);
@@ -258,15 +264,18 @@
numElementsHint = Long.parseLong(dataset.getHints().get("CARDINALITY"));
}
+ AsterixStorageProperties storageProperties = ((AsterixAppContextInfoImpl) AsterixAppContextInfoImpl
+ .getInstance()).getStorageProperties();
TreeIndexBulkLoadOperatorDescriptor btreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER, AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
splitsAndConstraint.first, typeTraits, comparatorFactories, blooFilterKeyFields, fieldPermutation,
- GlobalConfig.DEFAULT_BTREE_FILL_FACTOR, false, numElementsHint, new LSMBTreeDataflowHelperFactory(
+ GlobalConfig.DEFAULT_BTREE_FILL_FACTOR, false, numElementsHint,
+ new LSMBTreeDataflowHelperFactory(AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES), NoOpOperationCallbackFactory.INSTANCE);
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
+ NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, btreeBulkLoad,
splitsAndConstraint.second);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
index dbbbe44..d1ba0f4 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
@@ -1,6 +1,7 @@
package edu.uci.ics.asterix.file;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.api.AsterixAppContextInfoImpl;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.config.OptimizationConfUtil;
import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
@@ -48,14 +49,16 @@
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider
.splitProviderAndPartitionConstraintsForInternalOrFeedDataset(dataverseName, datasetName, indexName);
+ AsterixStorageProperties storageProperties = ((AsterixAppContextInfoImpl) AsterixAppContextInfoImpl
+ .getInstance()).getStorageProperties();
IndexDropOperatorDescriptor btreeDrop = new IndexDropOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER, AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
splitsAndConstraint.first, new LSMBTreeDataflowHelperFactory(
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES));
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()));
AlgebricksPartitionConstraintHelper
.setPartitionConstraintInJobSpec(spec, btreeDrop, splitsAndConstraint.second);
spec.addRoot(btreeDrop);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeCreator.java
index 2a1349b..958f8d1 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeCreator.java
@@ -1,6 +1,7 @@
package edu.uci.ics.asterix.file;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
+import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.transaction.management.resource.ILocalResourceMetadata;
@@ -26,31 +27,33 @@
public class SecondaryBTreeCreator extends SecondaryIndexCreator {
- protected SecondaryBTreeCreator(PhysicalOptimizationConfig physOptConf) {
- super(physOptConf);
+ protected SecondaryBTreeCreator(PhysicalOptimizationConfig physOptConf,
+ IAsterixPropertiesProvider propertiesProvider) {
+ super(physOptConf, propertiesProvider);
}
@Override
public JobSpecification buildCreationJobSpec() throws AsterixException, AlgebricksException {
JobSpecification spec = new JobSpecification();
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
//prepare a LocalResourceMetadata which will be stored in NC's local resource repository
ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(
secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, secondaryBloomFilterKeyFields, false,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages());
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMBTreeResource);
TreeIndexCreateOperatorDescriptor secondaryIndexCreateOp = new TreeIndexCreateOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER, AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
secondaryFileSplitProvider, secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories,
- secondaryBloomFilterKeyFields, new LSMBTreeDataflowHelperFactory(
+ secondaryBloomFilterKeyFields,
+ new LSMBTreeDataflowHelperFactory(AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES), localResourceFactoryProvider,
- NoOpOperationCallbackFactory.INSTANCE);
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
+ localResourceFactoryProvider, NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, secondaryIndexCreateOp,
secondaryPartitionConstraint);
spec.addRoot(secondaryIndexCreateOp);
@@ -80,13 +83,17 @@
// Sort by secondary keys.
ExternalSortOperatorDescriptor sortOp = createSortOp(spec, secondaryComparatorFactories, secondaryRecDesc);
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
// Create secondary BTree bulk load op.
- TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec, numSecondaryKeys,
+ TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(
+ spec,
+ numSecondaryKeys,
new LSMBTreeDataflowHelperFactory(AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES), BTree.DEFAULT_FILL_FACTOR);
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
+ BTree.DEFAULT_FILL_FACTOR);
// Connect the operators.
spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexCreator.java
index e7c2e3e..357c55e 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexCreator.java
@@ -19,8 +19,10 @@
import java.io.IOException;
import java.util.List;
+import edu.uci.ics.asterix.common.api.AsterixAppContextInfoImpl;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.formats.nontagged.AqlBinaryBooleanInspectorImpl;
@@ -103,29 +105,35 @@
protected RecordDescriptor secondaryRecDesc;
protected ICopyEvaluatorFactory[] secondaryFieldAccessEvalFactories;
+ protected IAsterixPropertiesProvider propertiesProvider;
+
// Prevent public construction. Should be created via createIndexCreator().
- protected SecondaryIndexCreator(PhysicalOptimizationConfig physOptConf) {
+ protected SecondaryIndexCreator(PhysicalOptimizationConfig physOptConf,
+ IAsterixPropertiesProvider propertiesProvider) {
this.physOptConf = physOptConf;
+ this.propertiesProvider = propertiesProvider;
}
public static SecondaryIndexCreator createIndexCreator(CompiledCreateIndexStatement createIndexStmt,
AqlMetadataProvider metadataProvider, PhysicalOptimizationConfig physOptConf) throws AsterixException,
AlgebricksException {
+ IAsterixPropertiesProvider asterixPropertiesProvider = ((AsterixAppContextInfoImpl) AsterixAppContextInfoImpl
+ .getInstance());
SecondaryIndexCreator indexCreator = null;
switch (createIndexStmt.getIndexType()) {
case BTREE: {
- indexCreator = new SecondaryBTreeCreator(physOptConf);
+ indexCreator = new SecondaryBTreeCreator(physOptConf, asterixPropertiesProvider);
break;
}
case RTREE: {
- indexCreator = new SecondaryRTreeCreator(physOptConf);
+ indexCreator = new SecondaryRTreeCreator(physOptConf, asterixPropertiesProvider);
break;
}
case WORD_INVIX:
case NGRAM_INVIX:
case FUZZY_WORD_INVIX:
case FUZZY_NGRAM_INVIX: {
- indexCreator = new SecondaryInvertedIndexCreator(physOptConf);
+ indexCreator = new SecondaryInvertedIndexCreator(physOptConf, asterixPropertiesProvider);
break;
}
default: {
@@ -266,6 +274,7 @@
int[] lowKeyFields = null;
// +Infinity
int[] highKeyFields = null;
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
BTreeSearchOperatorDescriptor primarySearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER, AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
primaryFileSplitProvider, primaryRecDesc.getTypeTraits(), primaryComparatorFactories,
@@ -273,8 +282,9 @@
new LSMBTreeDataflowHelperFactory(AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES), false, NoOpOperationCallbackFactory.INSTANCE);
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()), false,
+ NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, primarySearchOp,
primaryPartitionConstraint);
return primarySearchOp;
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
index 0bf379d..7b53a28 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
@@ -2,8 +2,9 @@
import java.util.List;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
@@ -60,8 +61,9 @@
private RecordDescriptor tokenKeyPairRecDesc;
private boolean isPartitioned;
- protected SecondaryInvertedIndexCreator(PhysicalOptimizationConfig physOptConf) {
- super(physOptConf);
+ protected SecondaryInvertedIndexCreator(PhysicalOptimizationConfig physOptConf,
+ IAsterixPropertiesProvider propertiesProvider) {
+ super(physOptConf, propertiesProvider);
}
@Override
@@ -150,10 +152,12 @@
public JobSpecification buildCreationJobSpec() throws AsterixException, AlgebricksException {
JobSpecification spec = new JobSpecification();
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
//prepare a LocalResourceMetadata which will be stored in NC's local resource repository
ILocalResourceMetadata localResourceMetadata = new LSMInvertedIndexLocalResourceMetadata(invListsTypeTraits,
primaryComparatorFactories, tokenTypeTraits, tokenComparatorFactories, tokenizerFactory,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES, isPartitioned);
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages(),
+ isPartitioned);
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMInvertedIndexResource);
@@ -259,20 +263,21 @@
}
private IIndexDataflowHelperFactory createDataflowHelperFactory() {
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
if (!isPartitioned) {
return new LSMInvertedIndexDataflowHelperFactory(
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages());
} else {
return new PartitionedLSMInvertedIndexDataflowHelperFactory(
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages());
}
}
}
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java
index 6f400f9..1a5e0da 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java
@@ -2,7 +2,8 @@
import java.util.List;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
+import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.dataflow.data.nontagged.valueproviders.AqlPrimitiveValueProviderFactory;
@@ -49,20 +50,22 @@
protected int numNestedSecondaryKeyFields;
protected ATypeTag keyType;
- protected SecondaryRTreeCreator(PhysicalOptimizationConfig physOptConf) {
- super(physOptConf);
+ protected SecondaryRTreeCreator(PhysicalOptimizationConfig physOptConf,
+ IAsterixPropertiesProvider propertiesProvider) {
+ super(physOptConf, propertiesProvider);
}
@Override
public JobSpecification buildCreationJobSpec() throws AsterixException, AlgebricksException {
JobSpecification spec = new JobSpecification();
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
//prepare a LocalResourceMetadata which will be stored in NC's local resource repository
ILocalResourceMetadata localResourceMetadata = new LSMRTreeLocalResourceMetadata(
secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, primaryComparatorFactories,
valueProviderFactories, RTreePolicyType.RTREE, AqlMetadataProvider.proposeLinearizer(keyType,
- secondaryComparatorFactories.length), GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
+ secondaryComparatorFactories.length), storageProperties.getMemoryComponentPageSize(),
+ storageProperties.getMemoryComponentNumPages());
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMRTreeResource);
@@ -74,8 +77,8 @@
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER, AqlMetadataProvider.proposeLinearizer(
- keyType, secondaryComparatorFactories.length),
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
+ keyType, secondaryComparatorFactories.length), storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
localResourceFactoryProvider, NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, secondaryIndexCreateOp,
secondaryPartitionConstraint);
@@ -149,6 +152,7 @@
selectOp = createFilterNullsSelectOp(spec, numNestedSecondaryKeyFields);
}
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
// Create secondary RTree bulk load op.
TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(
spec,
@@ -158,8 +162,8 @@
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER, AqlMetadataProvider.proposeLinearizer(
- keyType, secondaryComparatorFactories.length),
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
+ keyType, secondaryComparatorFactories.length), storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
BTree.DEFAULT_FILL_FACTOR);
// Connect the operators.
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
index 1b8990b..d92e884 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/CCApplicationEntryPoint.java
@@ -14,7 +14,8 @@
import edu.uci.ics.asterix.api.http.servlet.QueryStatusAPIServlet;
import edu.uci.ics.asterix.api.http.servlet.UpdateAPIServlet;
import edu.uci.ics.asterix.common.api.AsterixAppContextInfoImpl;
-import edu.uci.ics.asterix.common.config.AsterixProperties;
+import edu.uci.ics.asterix.common.config.AsterixExternalProperties;
+import edu.uci.ics.asterix.common.config.AsterixMetadataProperties;
import edu.uci.ics.asterix.common.config.GlobalConfig;
import edu.uci.ics.asterix.metadata.MetadataManager;
import edu.uci.ics.asterix.metadata.api.IAsterixStateProxy;
@@ -29,8 +30,6 @@
private static final String HYRACKS_CONNECTION_ATTR = "edu.uci.ics.asterix.HYRACKS_CONNECTION";
- private static final int DEFAULT_WEB_SERVER_PORT = 19001;
-
private static final int DEFAULT_JSON_API_SERVER_PORT = 19101;
private Server webServer;
@@ -45,11 +44,14 @@
LOGGER.info("Starting Asterix cluster controller");
}
+ AsterixAppContextInfoImpl.initialize(appCtx);
+
proxy = AsterixStateProxy.registerRemoteObject();
- proxy.setAsterixProperties(AsterixProperties.INSTANCE);
appCtx.setDistributedState(proxy);
- MetadataManager.INSTANCE = new MetadataManager(proxy);
+ AsterixMetadataProperties metadataProperties = ((AsterixAppContextInfoImpl) AsterixAppContextInfoImpl
+ .getInstance()).getMetadataProperties();
+ MetadataManager.INSTANCE = new MetadataManager(proxy, metadataProperties);
setupWebServer();
webServer.start();
@@ -57,8 +59,6 @@
// Setup and start the web interface
setupJSONAPIServer();
jsonAPIServer.start();
-
- AsterixAppContextInfoImpl.initialize(appCtx);
}
@Override
@@ -79,9 +79,9 @@
}
private void setupWebServer() throws Exception {
- int port = Integer.parseInt((String) AsterixProperties.INSTANCE
- .getProperty(AsterixProperties.AsterixConfigurationKeys.WEB_INTERFACE_PORT));
- webServer = new Server(port);
+ AsterixExternalProperties externalProperties = ((AsterixAppContextInfoImpl) AsterixAppContextInfoImpl
+ .getInstance()).getExternalProperties();
+ webServer = new Server(externalProperties.getWebInterfacePort());
ServletContextHandler context = new ServletContextHandler(ServletContextHandler.SESSIONS);
context.setContextPath("/");
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
index 8ad8c67..d29d92b 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
@@ -5,6 +5,7 @@
import java.util.logging.Level;
import java.util.logging.Logger;
+import edu.uci.ics.asterix.common.config.AsterixMetadataProperties;
import edu.uci.ics.asterix.common.context.AsterixAppRuntimeContext;
import edu.uci.ics.asterix.metadata.MetadataManager;
import edu.uci.ics.asterix.metadata.MetadataNode;
@@ -86,6 +87,7 @@
@Override
public void notifyStartupComplete() throws Exception {
IAsterixStateProxy proxy = (IAsterixStateProxy) ncApplicationContext.getDistributedState();
+ AsterixMetadataProperties metadataProperties = runtimeContext.getMetadataProperties();
if (systemState == SystemState.NEW_UNIVERSE) {
PersistentLocalResourceRepository localResourceRepository = (PersistentLocalResourceRepository) runtimeContext
@@ -94,24 +96,23 @@
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("nodeid" + nodeId);
LOGGER.info("proxy" + proxy);
- LOGGER.info("stores" + proxy.getAsterixProperties().getStores());
- LOGGER.info("store" + proxy.getAsterixProperties().getStores().get(nodeId)[0]);
+ LOGGER.info("stores" + metadataProperties.getStores());
+ LOGGER.info("store" + metadataProperties.getStores().get(nodeId)[0]);
}
- localResourceRepository.initialize(nodeId, proxy.getAsterixProperties().getStores().get(nodeId)[0], true,
- null);
+ localResourceRepository.initialize(nodeId, metadataProperties.getStores().get(nodeId)[0], true, null);
}
- isMetadataNode = nodeId.equals(proxy.getAsterixProperties().getMetadataNodeName());
+ isMetadataNode = nodeId.equals(metadataProperties.getMetadataNodeName());
if (isMetadataNode) {
registerRemoteMetadataNode(proxy);
if (LOGGER.isLoggable(Level.INFO)) {
LOGGER.info("Bootstrapping metadata");
}
- MetadataManager.INSTANCE = new MetadataManager(proxy);
+ MetadataManager.INSTANCE = new MetadataManager(proxy, metadataProperties);
MetadataManager.INSTANCE.init();
- MetadataBootstrap.startUniverse(proxy.getAsterixProperties(), ncApplicationContext,
+ MetadataBootstrap.startUniverse(runtimeContext, ncApplicationContext,
systemState == SystemState.NEW_UNIVERSE);
MetadataBootstrap.startDDLRecovery();
}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/api/AsterixAppContextInfoImpl.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/api/AsterixAppContextInfoImpl.java
index 0f1de56..c7393b9 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/api/AsterixAppContextInfoImpl.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/api/AsterixAppContextInfoImpl.java
@@ -14,8 +14,18 @@
*/
package edu.uci.ics.asterix.common.api;
+import java.util.logging.Logger;
+
+import edu.uci.ics.asterix.common.config.AsterixCompilerProperties;
+import edu.uci.ics.asterix.common.config.AsterixExternalProperties;
+import edu.uci.ics.asterix.common.config.AsterixMetadataProperties;
+import edu.uci.ics.asterix.common.config.AsterixPropertiesAccessor;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
+import edu.uci.ics.asterix.common.config.AsterixTransactionProperties;
+import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.common.dataflow.IAsterixApplicationContextInfo;
+import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.hyracks.api.application.ICCApplicationContext;
import edu.uci.ics.hyracks.storage.am.common.api.IIndexLifecycleManagerProvider;
import edu.uci.ics.hyracks.storage.common.IStorageManagerInterface;
@@ -25,16 +35,30 @@
* instances that are accessed from the NCs. In addition an instance of ICCApplicationContext
* is stored for access by the CC.
*/
-public class AsterixAppContextInfoImpl implements IAsterixApplicationContextInfo {
+public class AsterixAppContextInfoImpl implements IAsterixApplicationContextInfo, IAsterixPropertiesProvider {
private static AsterixAppContextInfoImpl INSTANCE;
private final ICCApplicationContext appCtx;
- public static void initialize(ICCApplicationContext ccAppCtx) {
+ private static AsterixCompilerProperties compilerProperties;
+ private static AsterixExternalProperties externalProperties;
+ private static AsterixMetadataProperties metadataProperties;
+ private static AsterixStorageProperties storageProperties;
+ private static AsterixTransactionProperties txnProperties;
+
+ public static void initialize(ICCApplicationContext ccAppCtx) throws AsterixException {
if (INSTANCE == null) {
INSTANCE = new AsterixAppContextInfoImpl(ccAppCtx);
}
+ AsterixPropertiesAccessor propertiesAccessor = new AsterixPropertiesAccessor();
+ compilerProperties = new AsterixCompilerProperties(propertiesAccessor);
+ externalProperties = new AsterixExternalProperties(propertiesAccessor);
+ metadataProperties = new AsterixMetadataProperties(propertiesAccessor);
+ storageProperties = new AsterixStorageProperties(propertiesAccessor);
+ txnProperties = new AsterixTransactionProperties(propertiesAccessor);
+
+ Logger.getLogger(".").setLevel(externalProperties.getLogLevel());
}
private AsterixAppContextInfoImpl(ICCApplicationContext ccAppCtx) {
@@ -59,4 +83,29 @@
public IIndexLifecycleManagerProvider getIndexLifecycleManagerProvider() {
return AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER;
}
+
+ @Override
+ public AsterixStorageProperties getStorageProperties() {
+ return storageProperties;
+ }
+
+ @Override
+ public AsterixTransactionProperties getTransactionProperties() {
+ return txnProperties;
+ }
+
+ @Override
+ public AsterixCompilerProperties getCompilerProperties() {
+ return compilerProperties;
+ }
+
+ @Override
+ public AsterixMetadataProperties getMetadataProperties() {
+ return metadataProperties;
+ }
+
+ @Override
+ public AsterixExternalProperties getExternalProperties() {
+ return externalProperties;
+ }
}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AbstractAsterixProperties.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AbstractAsterixProperties.java
new file mode 100644
index 0000000..4968f40
--- /dev/null
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AbstractAsterixProperties.java
@@ -0,0 +1,9 @@
+package edu.uci.ics.asterix.common.config;
+
+public abstract class AbstractAsterixProperties {
+ protected final AsterixPropertiesAccessor accessor;
+
+ public AbstractAsterixProperties(AsterixPropertiesAccessor accessor) {
+ this.accessor = accessor;
+ }
+}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixCompilerProperties.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixCompilerProperties.java
new file mode 100644
index 0000000..e1c80bd
--- /dev/null
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixCompilerProperties.java
@@ -0,0 +1,28 @@
+package edu.uci.ics.asterix.common.config;
+
+public class AsterixCompilerProperties extends AbstractAsterixProperties {
+ private static final String COMPILER_SORTMEMORY_KEY = "compiler.sortmemory";
+ private static final int COMPILER_SORTMEMORY_DEFAULT = (512 << 20); // 512MB
+
+ private static final String COMPILER_JOINMEMORY_KEY = "compiler.joinmemory";
+ private static final int COMPILER_JOINMEMORY_DEFAULT = (512 << 20); // 512MB
+
+ private static final String COMPILER_FRAMESIZE_KEY = "compiler.framesize";
+ private static int COMPILER_FRAMESIZE_DEFAULT = (32 << 10); // 32KB
+
+ public AsterixCompilerProperties(AsterixPropertiesAccessor accessor) {
+ super(accessor);
+ }
+
+ public int getSortMemorySize() {
+ return accessor.getInt(COMPILER_SORTMEMORY_KEY, COMPILER_SORTMEMORY_DEFAULT);
+ }
+
+ public int getJoinMemorySize() {
+ return accessor.getInt(COMPILER_JOINMEMORY_KEY, COMPILER_JOINMEMORY_DEFAULT);
+ }
+
+ public int getFrameSize() {
+ return accessor.getInt(COMPILER_FRAMESIZE_KEY, COMPILER_FRAMESIZE_DEFAULT);
+ }
+}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixExternalProperties.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixExternalProperties.java
new file mode 100644
index 0000000..958aa7f
--- /dev/null
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixExternalProperties.java
@@ -0,0 +1,26 @@
+package edu.uci.ics.asterix.common.config;
+
+import java.util.logging.Level;
+
+public class AsterixExternalProperties extends AbstractAsterixProperties {
+
+ private static final String EXTERNAL_WEBPORT_KEY = "external.webport";
+ private static int EXTERNAL_WEBPORT_DEFAULT = 19001;
+
+ private static final String EXTERNAL_LOGLEVEL_KEY = "external.loglevel";
+ private static String EXTERNAL_LOGLEVEL_DEFAULT = "INFO";
+
+ public AsterixExternalProperties(AsterixPropertiesAccessor accessor) {
+ super(accessor);
+ }
+
+ public int getWebInterfacePort() {
+ return accessor.getInt(EXTERNAL_WEBPORT_KEY, EXTERNAL_WEBPORT_DEFAULT);
+ }
+
+ public Level getLogLevel() {
+ String level = accessor.getString(EXTERNAL_LOGLEVEL_KEY, EXTERNAL_LOGLEVEL_DEFAULT);
+ return Level.parse(level);
+ }
+
+}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixMetadataProperties.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixMetadataProperties.java
new file mode 100644
index 0000000..6d47e78
--- /dev/null
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixMetadataProperties.java
@@ -0,0 +1,28 @@
+package edu.uci.ics.asterix.common.config;
+
+import java.util.Map;
+import java.util.Set;
+
+public class AsterixMetadataProperties extends AbstractAsterixProperties {
+
+ public AsterixMetadataProperties(AsterixPropertiesAccessor accessor) {
+ super(accessor);
+ }
+
+ public String getMetadataNodeName() {
+ return accessor.getMetadataNodeName();
+ }
+
+ public String getMetadataStore() {
+ return accessor.getMetadataStore();
+ }
+
+ public Map<String, String[]> getStores() {
+ return accessor.getStores();
+ }
+
+ public Set<String> getNodeNames() {
+ return accessor.getNodeNames();
+ }
+
+}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixProperties.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixProperties.java
deleted file mode 100644
index 04e974d..0000000
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixProperties.java
+++ /dev/null
@@ -1,252 +0,0 @@
-/*
- * Copyright 2009-2010 by The Regents of the University of California
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * you may obtain a copy of the License from
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package edu.uci.ics.asterix.common.config;
-
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.InputStream;
-import java.io.Serializable;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import javax.xml.bind.JAXBContext;
-import javax.xml.bind.Unmarshaller;
-
-import edu.uci.ics.asterix.common.configuration.AsterixConfiguration;
-import edu.uci.ics.asterix.common.configuration.Property;
-import edu.uci.ics.asterix.common.configuration.Store;
-import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-
-/**
- * Holder for Asterix properties values typically set as Java Properties.
- * Intended to live in the AsterixStateProxy so it can be accessed remotely.
- */
-public class AsterixProperties implements Serializable {
-
- private static final long serialVersionUID = 1L;
- private static String metadataNodeName;
- private static HashSet<String> nodeNames;
- private static Map<String, String[]> stores;
- private static Map<String, String> asterixConfigurationParams;
-
- public static AsterixProperties INSTANCE = new AsterixProperties();
-
- public static class AsterixConfigurationKeys {
-
- // JVM parameters for each Node Contoller (NC)
- public static final String NC_JAVA_OPTS = "nc_java_opts";
- public static final String NC_JAVA_OPTS_DEFAULT = "-Xmx1024m";
-
- // JVM parameters for the Cluster Contoller (CC)
- public static final String CC_JAVA_OPTS = "cc_java_opts";
- public static final String CC_JAVA_OPTS_DEFAULT = "-Xmx1024m";
-
- public static final String SIZE_MEMORY_COMPONENT = "size_memory_component";
- public static final String SIZE_MEMORY_COMPONENT_DEFAULT = "512m";
-
- public static final String TOTAL_SIZE_MEMORY_COMPONENT = "total_size_memory_component";
- public static final String TOTAL_SIZE_MEMORY_COMPONENT_DEFAULT = "512m";
-
- public static final String LOG_BUFFER_NUM_PAGES = "log_buffer_num_pages";
- public static final String LOG_BUFFER_NUM_PAGES_DEFAULT = "8";
-
- public static final String LOG_BUFFER_PAGE_SIZE = "log_buffer_page_size";
- public static final String LOG_BUFFER_PAGE_SIZE_DEFAULT = "131072";
-
- public static final String LOG_PARTITION_SIZE = "log_partition_size";
- public static final String LOG_PARTITION_SIZE_DEFAULT = "2147483648";
-
- public static final String GROUP_COMMIT_INTERVAL = "group_commit_interval";
- public static final String GROUP_COMMIT_INTERVAL_DEFAULT = "200ms";
-
- public static final String SORT_OP_MEMORY = "sort_op_memory";
- public static final String SORT_OP_MEMORY_DEFAULT = "512m";
-
- public static final String JOIN_OP_MEMORY = "join_op_memory";
- public static final String JOIN_OP_MEMORY_DEFAULT = "512m";
-
- public static final String WEB_INTERFACE_PORT = "web_interface_port";
- public static final String WEB_INTERFACE_PORT_DEFAULT = "19001";
-
- public static final String NUM_PAGES_BUFFER_CACHE = "num_pages_buffer_cache";
- public static final String NUM_PAGES_BUFFER_CACHE_DEFAULT = "1000";
-
- public static final String LOG_LEVEL = "log_level";
- public static final String LOG_LEVEL_DEFAULT = "INFO";
-
- public static final String LSN_THRESHOLD = "lsn_threshold";
- public static final String LSN_THRESHOLD_DEFAULT = "64m";
-
- public static final String CHECKPOINT_TERMS_IN_SECS = "checkpoint_terms_in_secs";
- public static final String CHECKPOINT_TERMS_IN_SECS_DEFAULT = "120";
-
- public static final String ESCALATE_THRSHOLD_ENTITY_TO_DATASET = "escalate_threshold_entity_to_dataset";
- public static final String ESCALATE_THRSHOLD_ENTITY_TO_DATASET_DEFAULT = "8";
-
- public static final String SHRINK_TIMER_THRESHOLD = "shrink_timer_threshold";
- public static final String SHRINK_TIMER_THRESHOLD_DEFAULT = "120000";
-
- }
-
- private AsterixProperties() {
- try {
- String fileName = System.getProperty(GlobalConfig.CONFIG_FILE_PROPERTY);
- if (fileName == null) {
- fileName = GlobalConfig.DEFAULT_CONFIG_FILE_NAME;
- }
- InputStream is = this.getClass().getClassLoader().getResourceAsStream(fileName);
- if (is == null) {
- try {
- fileName = GlobalConfig.DEFAULT_CONFIG_FILE_NAME;
- is = new FileInputStream(fileName);
- } catch (FileNotFoundException fnf) {
- throw new AlgebricksException("Could not find the configuration file " + fileName);
- }
- }
-
- JAXBContext ctx = JAXBContext.newInstance(AsterixConfiguration.class);
- Unmarshaller unmarshaller = ctx.createUnmarshaller();
- AsterixConfiguration asterixConfiguration = (AsterixConfiguration) unmarshaller.unmarshal(is);
- metadataNodeName = asterixConfiguration.getMetadataNode();
- stores = new HashMap<String, String[]>();
- List<Store> configuredStores = asterixConfiguration.getStore();
- nodeNames = new HashSet<String>();
- for (Store store : configuredStores) {
- String trimmedStoreDirs = store.getStoreDirs().trim();
- stores.put(store.getNcId(), trimmedStoreDirs.split(","));
- nodeNames.add(store.getNcId());
- }
- asterixConfigurationParams = new HashMap<String, String>();
- for (Property p : asterixConfiguration.getProperty()) {
- asterixConfigurationParams.put(p.getName(), p.getValue());
- }
-
- initializeLogLevel(getProperty(AsterixConfigurationKeys.LOG_LEVEL));
- } catch (Exception e) {
- throw new IllegalStateException(e);
- }
- }
-
- public String getMetadataNodeName() {
- return metadataNodeName;
- }
-
- public String getMetadataStore() {
- return stores.get(metadataNodeName)[0];
- }
-
- public Map<String, String[]> getStores() {
- return stores;
- }
-
- public HashSet<String> getNodeNames() {
- return nodeNames;
- }
-
- public String getProperty(String property) {
- String propValue = asterixConfigurationParams.get(property);
- if (propValue == null) {
- switch (property) {
- case AsterixConfigurationKeys.NC_JAVA_OPTS:
- propValue = AsterixConfigurationKeys.NC_JAVA_OPTS_DEFAULT;
- break;
- case AsterixConfigurationKeys.CC_JAVA_OPTS:
- propValue = AsterixConfigurationKeys.CC_JAVA_OPTS_DEFAULT;
- break;
- case AsterixConfigurationKeys.SIZE_MEMORY_COMPONENT:
- propValue = AsterixConfigurationKeys.SIZE_MEMORY_COMPONENT_DEFAULT;
- break;
- case AsterixConfigurationKeys.TOTAL_SIZE_MEMORY_COMPONENT:
- propValue = AsterixConfigurationKeys.TOTAL_SIZE_MEMORY_COMPONENT_DEFAULT;
- break;
- case AsterixConfigurationKeys.LOG_BUFFER_NUM_PAGES:
- propValue = AsterixConfigurationKeys.LOG_BUFFER_NUM_PAGES_DEFAULT;
- break;
- case AsterixConfigurationKeys.LOG_BUFFER_PAGE_SIZE:
- propValue = AsterixConfigurationKeys.LOG_BUFFER_PAGE_SIZE_DEFAULT;
- break;
- case AsterixConfigurationKeys.LOG_PARTITION_SIZE:
- propValue = AsterixConfigurationKeys.LOG_PARTITION_SIZE_DEFAULT;
- break;
- case AsterixConfigurationKeys.GROUP_COMMIT_INTERVAL:
- propValue = AsterixConfigurationKeys.GROUP_COMMIT_INTERVAL_DEFAULT;
- break;
- case AsterixConfigurationKeys.SORT_OP_MEMORY:
- propValue = AsterixConfigurationKeys.SORT_OP_MEMORY_DEFAULT;
- break;
- case AsterixConfigurationKeys.JOIN_OP_MEMORY:
- propValue = AsterixConfigurationKeys.JOIN_OP_MEMORY_DEFAULT;
- break;
- case AsterixConfigurationKeys.WEB_INTERFACE_PORT:
- propValue = AsterixConfigurationKeys.WEB_INTERFACE_PORT_DEFAULT;
- break;
- case AsterixConfigurationKeys.NUM_PAGES_BUFFER_CACHE:
- propValue = AsterixConfigurationKeys.NUM_PAGES_BUFFER_CACHE_DEFAULT;
- break;
- case AsterixConfigurationKeys.LOG_LEVEL:
- propValue = AsterixConfigurationKeys.LOG_LEVEL_DEFAULT;
- break;
- case AsterixConfigurationKeys.LSN_THRESHOLD:
- propValue = AsterixConfigurationKeys.LSN_THRESHOLD_DEFAULT;
- break;
- case AsterixConfigurationKeys.CHECKPOINT_TERMS_IN_SECS:
- propValue = AsterixConfigurationKeys.CHECKPOINT_TERMS_IN_SECS_DEFAULT;
- break;
- case AsterixConfigurationKeys.ESCALATE_THRSHOLD_ENTITY_TO_DATASET:
- propValue = AsterixConfigurationKeys.ESCALATE_THRSHOLD_ENTITY_TO_DATASET_DEFAULT;
- break;
- case AsterixConfigurationKeys.SHRINK_TIMER_THRESHOLD:
- propValue = AsterixConfigurationKeys.SHRINK_TIMER_THRESHOLD_DEFAULT;
- break;
- }
- }
- return propValue;
- }
-
- private void initializeLogLevel(String configuredLogLevel) {
- Level level = null;
- switch (configuredLogLevel.toLowerCase()) {
- case "info":
- level = Level.INFO;
- break;
- case "fine":
- level = Level.FINE;
- break;
- case "finer":
- level = Level.FINER;
- break;
- case "finest":
- level = Level.FINEST;
- break;
- case "severe":
- level = Level.SEVERE;
- break;
- case "off":
- level = Level.OFF;
- break;
- case "warning":
- level = Level.WARNING;
- break;
- default:
- level = Level.ALL;
- }
- Logger.getLogger(".").setLevel(level);
- }
-}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixPropertiesAccessor.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixPropertiesAccessor.java
new file mode 100644
index 0000000..eeec537
--- /dev/null
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixPropertiesAccessor.java
@@ -0,0 +1,91 @@
+package edu.uci.ics.asterix.common.config;
+
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.InputStream;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import javax.xml.bind.JAXBContext;
+import javax.xml.bind.JAXBException;
+import javax.xml.bind.Unmarshaller;
+
+import edu.uci.ics.asterix.common.configuration.AsterixConfiguration;
+import edu.uci.ics.asterix.common.configuration.Property;
+import edu.uci.ics.asterix.common.configuration.Store;
+import edu.uci.ics.asterix.common.exceptions.AsterixException;
+
+public class AsterixPropertiesAccessor {
+
+ private final String metadataNodeName;
+ private final Set<String> nodeNames;
+ private final Map<String, String[]> stores;
+ private final Map<String, String> asterixConfigurationParams;
+
+ public AsterixPropertiesAccessor() throws AsterixException {
+ String fileName = System.getProperty(GlobalConfig.CONFIG_FILE_PROPERTY);
+ if (fileName == null) {
+ fileName = GlobalConfig.DEFAULT_CONFIG_FILE_NAME;
+ }
+ InputStream is = this.getClass().getClassLoader().getResourceAsStream(fileName);
+ if (is == null) {
+ try {
+ fileName = GlobalConfig.DEFAULT_CONFIG_FILE_NAME;
+ is = new FileInputStream(fileName);
+ } catch (FileNotFoundException fnf) {
+ throw new AsterixException("Could not find configuration file " + fileName);
+ }
+ }
+
+ AsterixConfiguration asterixConfiguration = null;
+ try {
+ JAXBContext ctx = JAXBContext.newInstance(AsterixConfiguration.class);
+ Unmarshaller unmarshaller = ctx.createUnmarshaller();
+ asterixConfiguration = (AsterixConfiguration) unmarshaller.unmarshal(is);
+ } catch (JAXBException e) {
+ throw new AsterixException("Failed to read configuration file " + fileName);
+ }
+ metadataNodeName = asterixConfiguration.getMetadataNode();
+ stores = new HashMap<String, String[]>();
+ List<Store> configuredStores = asterixConfiguration.getStore();
+ nodeNames = new HashSet<String>();
+ for (Store store : configuredStores) {
+ String trimmedStoreDirs = store.getStoreDirs().trim();
+ stores.put(store.getNcId(), trimmedStoreDirs.split(","));
+ nodeNames.add(store.getNcId());
+ }
+ asterixConfigurationParams = new HashMap<String, String>();
+ for (Property p : asterixConfiguration.getProperty()) {
+ asterixConfigurationParams.put(p.getName(), p.getValue());
+ }
+ }
+
+ public String getMetadataNodeName() {
+ return metadataNodeName;
+ }
+
+ public String getMetadataStore() {
+ return stores.get(metadataNodeName)[0];
+ }
+
+ public Map<String, String[]> getStores() {
+ return stores;
+ }
+
+ public Set<String> getNodeNames() {
+ return nodeNames;
+ }
+
+ public int getInt(String property, int defaultValue) {
+ String propertyValue = asterixConfigurationParams.get(property);
+ return propertyValue == null ? defaultValue : Integer.parseInt(propertyValue);
+ }
+
+ public String getString(String property, String defaultValue) {
+ String propertyValue = asterixConfigurationParams.get(property);
+ return propertyValue == null ? defaultValue : propertyValue;
+ }
+}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixStorageProperties.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixStorageProperties.java
new file mode 100644
index 0000000..1e41ce8
--- /dev/null
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixStorageProperties.java
@@ -0,0 +1,57 @@
+package edu.uci.ics.asterix.common.config;
+
+public class AsterixStorageProperties extends AbstractAsterixProperties {
+
+ private static final String STORAGE_BUFFERCACHE_PAGESIZE_KEY = "storage.buffercache.pagesize";
+ private static int STORAGE_BUFFERCACHE_PAGESIZE_DEFAULT = (32 << 10); // 32KB
+
+ private static final String STORAGE_BUFFERCACHE_NUMPAGES_KEY = "storage.buffercache.numpages";
+ private static final int STORAGE_BUFFERCACHE_NUMPAGES_DEFAULT = 1024;
+
+ private static final String STORAGE_BUFFERCACHE_MAXOPENFILES_KEY = "storage.buffercache.maxopenfiles";
+ private static int STORAGE_BUFFERCACHE_MAXOPENFILES_DEFAULT = Integer.MAX_VALUE;
+
+ private static final String STORAGE_MEMORYCOMPONENT_PAGESIZE_KEY = "storage.memorycomponent.pagesize";
+ private static final int STORAGE_MEMORYCOMPONENT_PAGESIZE_DEFAULT = (32 << 10); // 32KB
+
+ private static final String STORAGE_MEMORYCOMPONENT_NUMPAGES_KEY = "storage.memorycomponent.numpages";
+ private static final int STORAGE_MEMORYCOMPONENT_NUMPAGES_DEFAULT = 4096; // ... so 128MB components
+
+ private static final String STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_KEY = "storage.memorycomponent.globalbudget";
+ private static final int STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_DEFAULT = (1 << 30); // 1GB
+
+ private static final String STORAGE_LSM_MERGETHRESHOLD_KEY = "storage.lsm.mergethreshold";
+ private static int STORAGE_LSM_MERGETHRESHOLD_DEFAULT = 3;
+
+ public AsterixStorageProperties(AsterixPropertiesAccessor accessor) {
+ super(accessor);
+ }
+
+ public int getBufferCachePageSize() {
+ return accessor.getInt(STORAGE_BUFFERCACHE_PAGESIZE_KEY, STORAGE_BUFFERCACHE_PAGESIZE_DEFAULT);
+ }
+
+ public int getBufferCacheNumPages() {
+ return accessor.getInt(STORAGE_BUFFERCACHE_NUMPAGES_KEY, STORAGE_BUFFERCACHE_NUMPAGES_DEFAULT);
+ }
+
+ public int getBufferCacheMaxOpenFiles() {
+ return accessor.getInt(STORAGE_BUFFERCACHE_MAXOPENFILES_KEY, STORAGE_BUFFERCACHE_MAXOPENFILES_DEFAULT);
+ }
+
+ public int getMemoryComponentPageSize() {
+ return accessor.getInt(STORAGE_MEMORYCOMPONENT_PAGESIZE_KEY, STORAGE_MEMORYCOMPONENT_PAGESIZE_DEFAULT);
+ }
+
+ public int getMemoryComponentNumPages() {
+ return accessor.getInt(STORAGE_MEMORYCOMPONENT_NUMPAGES_KEY, STORAGE_MEMORYCOMPONENT_NUMPAGES_DEFAULT);
+ }
+
+ public int getMemoryComponentGlobalBudget() {
+ return accessor.getInt(STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_KEY, STORAGE_MEMORYCOMPONENT_GLOBALBUDGET_DEFAULT);
+ }
+
+ public int getLSMIndexMergeThreshold() {
+ return accessor.getInt(STORAGE_LSM_MERGETHRESHOLD_KEY, STORAGE_LSM_MERGETHRESHOLD_DEFAULT);
+ }
+}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixTransactionProperties.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixTransactionProperties.java
new file mode 100644
index 0000000..f001e37
--- /dev/null
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/AsterixTransactionProperties.java
@@ -0,0 +1,64 @@
+package edu.uci.ics.asterix.common.config;
+
+public class AsterixTransactionProperties extends AbstractAsterixProperties {
+ private static final String TXN_LOG_BUFFER_NUMPAGES_KEY = "txn.log.buffer.numpages";
+ private static int TXN_LOG_BUFFER_NUMPAGES_DEFAULT = 8;
+
+ private static final String TXN_LOG_BUFFER_PAGESIZE_KEY = "txn.log.buffer.pagesize";
+ private static final int TXN_LOG_BUFFER_PAGESIZE_DEFAULT = (128 << 10); // 128KB
+
+ private static final String TXN_LOG_PARTITIONSIZE_KEY = "txn.log.partitionsize";
+ private static final int TXN_LOG_PARTITIONSIZE_DEFAULT = (2 << 30); // 2GB
+
+ private static final String TXN_LOG_GROUPCOMMITINTERVAL_KEY = "txn.log.groupcommitinterval";
+ private static int TXN_LOG_GROUPCOMMITINTERVAL_DEFAULT = 200; // 200ms
+
+ private static final String TXN_LOG_CHECKPOINT_LSNTHRESHOLD_KEY = "txn.log.checkpoint.lsnthreshold";
+ private static final int TXN_LOG_CHECKPOINT_LSNTHRESHOLD_DEFAULT = (64 << 20); // 64M
+
+ private static final String TXN_LOG_CHECKPOINT_POLLFREQUENCY_KEY = "txn.log.checkpoint.pollfrequency";
+ private static int TXN_LOG_CHECKPOINT_POLLFREQUENCY_DEFAULT = 120; // 120s
+
+ private static final String TXN_LOCK_ESCALATIONTHRESHOLD_KEY = "txn.lock.escalationthreshold";
+ private static int TXN_LOCK_ESCALATIONTHRESHOLD_DEFAULT = 1000;
+
+ private static final String TXN_LOCK_SHRINKTIMER_KEY = "txn.lock.shrinktimer";
+ private static int TXN_LOCK_SHRINKTIMER_DEFAULT = 120000; // 2m
+
+ public AsterixTransactionProperties(AsterixPropertiesAccessor accessor) {
+ super(accessor);
+ }
+
+ public int getLogBufferNumPages() {
+ return accessor.getInt(TXN_LOG_BUFFER_NUMPAGES_KEY, TXN_LOG_BUFFER_NUMPAGES_DEFAULT);
+ }
+
+ public int getLogBufferPageSize() {
+ return accessor.getInt(TXN_LOG_BUFFER_PAGESIZE_KEY, TXN_LOG_BUFFER_PAGESIZE_DEFAULT);
+ }
+
+ public int getLogPartitionSize() {
+ return accessor.getInt(TXN_LOG_PARTITIONSIZE_KEY, TXN_LOG_PARTITIONSIZE_DEFAULT);
+ }
+
+ public int getGroupCommitInterval() {
+ return accessor.getInt(TXN_LOG_GROUPCOMMITINTERVAL_KEY, TXN_LOG_GROUPCOMMITINTERVAL_DEFAULT);
+ }
+
+ public int getCheckpointLSNThreshold() {
+ return accessor.getInt(TXN_LOG_CHECKPOINT_LSNTHRESHOLD_KEY, TXN_LOG_CHECKPOINT_LSNTHRESHOLD_DEFAULT);
+ }
+
+ public int getCheckpointPollFrequency() {
+ return accessor.getInt(TXN_LOG_CHECKPOINT_POLLFREQUENCY_KEY, TXN_LOG_CHECKPOINT_POLLFREQUENCY_DEFAULT);
+ }
+
+ public int getEntityToDatasetLockEscalationThreshold() {
+ return accessor.getInt(TXN_LOCK_ESCALATIONTHRESHOLD_KEY, TXN_LOCK_ESCALATIONTHRESHOLD_DEFAULT);
+ }
+
+ public int getLockManagerShrinkTimer() {
+ return accessor.getInt(TXN_LOCK_SHRINKTIMER_KEY, TXN_LOCK_SHRINKTIMER_DEFAULT);
+ }
+
+}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/GlobalConfig.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/GlobalConfig.java
index ebce6fe..3948e69 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/GlobalConfig.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/GlobalConfig.java
@@ -3,61 +3,43 @@
import java.util.logging.Logger;
public class GlobalConfig {
- public static final boolean DEBUG = true;
+ public static final boolean DEBUG = true;
- public static final String ASTERIX_LOGGER_NAME = "edu.uci.ics.asterix";
+ public static final String ASTERIX_LOGGER_NAME = "edu.uci.ics.asterix";
- public static final Logger ASTERIX_LOGGER = Logger
- .getLogger(ASTERIX_LOGGER_NAME);
+ public static final Logger ASTERIX_LOGGER = Logger.getLogger(ASTERIX_LOGGER_NAME);
- public static final String ASTERIX_LOGFILE_PATTERN = "%t/asterix.log";
+ public static final String DEFAULT_CONFIG_FILE_NAME = "asterix-configuration.xml";
- public static final String DEFAULT_CONFIG_FILE_NAME = "asterix-configuration.xml";
+ public static final String CONFIG_FILE_PROPERTY = "AsterixConfigFileName";
- public static final String TEST_CONFIG_FILE_NAME = "src/main/resources/asterix-configuration.xml";
+ public static final String WEB_SERVER_PORT_PROPERTY = "AsterixWebServerPort";
- public static final String CONFIG_FILE_PROPERTY = "AsterixConfigFileName";
+ public static final String JSON_API_SERVER_PORT_PROPERTY = "AsterixJSONAPIServerPort";
- public static final String WEB_SERVER_PORT_PROPERTY = "AsterixWebServerPort";
+ public static final int DEFAULT_FRAME_SIZE = 32768;
- public static final String JSON_API_SERVER_PORT_PROPERTY = "AsterixJSONAPIServerPort";
+ public static final String FRAME_SIZE_PROPERTY = "FrameSize";
- public static final String BUFFER_CACHE_PAGE_SIZE_PROPERTY = "BufferCachePageSize";
+ public static final float DEFAULT_BTREE_FILL_FACTOR = 1.00f;
- public static final String BUFFER_CACHE_NUM_PAGES_PROPERTY = "BufferCacheNumPages";
+ public static int DEFAULT_INPUT_DATA_COLUMN = 0;
- public static final int DEFAULT_BUFFER_CACHE_NUM_PAGES = 4096;
-
- public static final int DEFAULT_FRAME_SIZE = 32768;
-
- public static final String FRAME_SIZE_PROPERTY = "FrameSize";
-
- public static final float DEFAULT_BTREE_FILL_FACTOR = 1.00f;
-
- public static int DEFAULT_INPUT_DATA_COLUMN = 0;
-
- public static int DEFAULT_INDEX_MEM_PAGE_SIZE = 32768;
-
- public static int DEFAULT_INDEX_MEM_NUM_PAGES = 1000;
-
- public static int getFrameSize() {
- int frameSize = GlobalConfig.DEFAULT_FRAME_SIZE;
- String frameSizeStr = System
- .getProperty(GlobalConfig.FRAME_SIZE_PROPERTY);
- if (frameSizeStr != null) {
- int fz = -1;
- try {
- fz = Integer.parseInt(frameSizeStr);
- } catch (NumberFormatException nfe) {
- GlobalConfig.ASTERIX_LOGGER
- .warning("Wrong frame size size argument. Picking default value ("
- + GlobalConfig.DEFAULT_FRAME_SIZE
- + ") instead.\n");
- }
- if (fz >= 0) {
- frameSize = fz;
- }
- }
- return frameSize;
- }
+ public static int getFrameSize() {
+ int frameSize = GlobalConfig.DEFAULT_FRAME_SIZE;
+ String frameSizeStr = System.getProperty(GlobalConfig.FRAME_SIZE_PROPERTY);
+ if (frameSizeStr != null) {
+ int fz = -1;
+ try {
+ fz = Integer.parseInt(frameSizeStr);
+ } catch (NumberFormatException nfe) {
+ GlobalConfig.ASTERIX_LOGGER.warning("Wrong frame size size argument. Picking default value ("
+ + GlobalConfig.DEFAULT_FRAME_SIZE + ") instead.\n");
+ }
+ if (fz >= 0) {
+ frameSize = fz;
+ }
+ }
+ return frameSize;
+ }
}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/IAsterixPropertiesProvider.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/IAsterixPropertiesProvider.java
new file mode 100644
index 0000000..e42c827
--- /dev/null
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/IAsterixPropertiesProvider.java
@@ -0,0 +1,14 @@
+package edu.uci.ics.asterix.common.config;
+
+
+public interface IAsterixPropertiesProvider {
+ public AsterixStorageProperties getStorageProperties();
+
+ public AsterixTransactionProperties getTransactionProperties();
+
+ public AsterixCompilerProperties getCompilerProperties();
+
+ public AsterixMetadataProperties getMetadataProperties();
+
+ public AsterixExternalProperties getExternalProperties();
+}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/AsterixAppRuntimeContext.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/AsterixAppRuntimeContext.java
index 5070938..f7c2a59 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/AsterixAppRuntimeContext.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/context/AsterixAppRuntimeContext.java
@@ -1,9 +1,16 @@
package edu.uci.ics.asterix.common.context;
import java.io.IOException;
-import java.util.logging.Level;
+import java.util.logging.Logger;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.config.AsterixCompilerProperties;
+import edu.uci.ics.asterix.common.config.AsterixExternalProperties;
+import edu.uci.ics.asterix.common.config.AsterixMetadataProperties;
+import edu.uci.ics.asterix.common.config.AsterixPropertiesAccessor;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
+import edu.uci.ics.asterix.common.config.AsterixTransactionProperties;
+import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
+import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.transaction.management.exception.ACIDException;
import edu.uci.ics.asterix.transaction.management.ioopcallbacks.LSMBTreeIOOperationCallbackFactory;
import edu.uci.ics.asterix.transaction.management.ioopcallbacks.LSMInvertedIndexIOOperationCallbackFactory;
@@ -39,12 +46,15 @@
import edu.uci.ics.hyracks.storage.common.file.ResourceIdFactory;
import edu.uci.ics.hyracks.storage.common.file.ResourceIdFactoryProvider;
-public class AsterixAppRuntimeContext {
- private static final int DEFAULT_BUFFER_CACHE_PAGE_SIZE = 32768;
- private static final int DEFAULT_LIFECYCLEMANAGER_MEMORY_BUDGET = 1024 * 1024 * 1024; // 1GB
- private static final int DEFAULT_MAX_OPEN_FILES = Integer.MAX_VALUE;
+public class AsterixAppRuntimeContext implements IAsterixPropertiesProvider {
private final INCApplicationContext ncApplicationContext;
+ private AsterixCompilerProperties compilerProperties;
+ private AsterixExternalProperties externalProperties;
+ private AsterixMetadataProperties metadataProperties;
+ private AsterixStorageProperties storageProperties;
+ private AsterixTransactionProperties txnProperties;
+
private IIndexLifecycleManager indexLifecycleManager;
private IFileMapManager fileMapManager;
private IBufferCache bufferCache;
@@ -64,24 +74,29 @@
this.ncApplicationContext = ncApplicationContext;
}
- public void initialize() throws IOException, ACIDException {
- int pageSize = getBufferCachePageSize();
- int numPages = getBufferCacheNumPages();
+ public void initialize() throws IOException, ACIDException, AsterixException {
+ AsterixPropertiesAccessor propertiesAccessor = new AsterixPropertiesAccessor();
+ compilerProperties = new AsterixCompilerProperties(propertiesAccessor);
+ externalProperties = new AsterixExternalProperties(propertiesAccessor);
+ metadataProperties = new AsterixMetadataProperties(propertiesAccessor);
+ storageProperties = new AsterixStorageProperties(propertiesAccessor);
+ txnProperties = new AsterixTransactionProperties(propertiesAccessor);
+
+ Logger.getLogger(".").setLevel(externalProperties.getLogLevel());
fileMapManager = new AsterixFileMapManager();
ICacheMemoryAllocator allocator = new HeapBufferAllocator();
IPageReplacementStrategy prs = new ClockPageReplacementStrategy();
- ioManager = ncApplicationContext.getRootContext().getIOManager();
- indexLifecycleManager = new IndexLifecycleManager(DEFAULT_LIFECYCLEMANAGER_MEMORY_BUDGET);
- IAsterixAppRuntimeContextProvider asterixAppRuntimeContextProvider = new AsterixAppRuntimeContextProviderForRecovery(
- this);
- txnSubsystem = new TransactionSubsystem(ncApplicationContext.getNodeId(), asterixAppRuntimeContextProvider);
IPageCleanerPolicy pcp = new DelayPageCleanerPolicy(600000);
- bufferCache = new BufferCache(ioManager, allocator, prs, pcp, fileMapManager, pageSize, numPages,
- DEFAULT_MAX_OPEN_FILES);
+ ioManager = ncApplicationContext.getRootContext().getIOManager();
+ bufferCache = new BufferCache(ioManager, allocator, prs, pcp, fileMapManager,
+ storageProperties.getBufferCachePageSize(), storageProperties.getBufferCacheNumPages(),
+ storageProperties.getBufferCacheMaxOpenFiles());
+
+ indexLifecycleManager = new IndexLifecycleManager(storageProperties.getMemoryComponentGlobalBudget());
lsmIOScheduler = SynchronousScheduler.INSTANCE;
- mergePolicy = new ConstantMergePolicy(3, this);
+ mergePolicy = new ConstantMergePolicy(storageProperties.getLSMIndexMergeThreshold(), this);
lsmBTreeOpTrackerFactory = new IndexOperationTrackerFactory(LSMBTreeIOOperationCallbackFactory.INSTANCE);
lsmRTreeOpTrackerFactory = new IndexOperationTrackerFactory(LSMRTreeIOOperationCallbackFactory.INSTANCE);
lsmInvertedIndexOpTrackerFactory = new IndexOperationTrackerFactory(
@@ -92,6 +107,10 @@
localResourceRepository = (PersistentLocalResourceRepository) persistentLocalResourceRepositoryFactory
.createRepository();
resourceIdFactory = (new ResourceIdFactoryProvider(localResourceRepository)).createResourceIdFactory();
+
+ IAsterixAppRuntimeContextProvider asterixAppRuntimeContextProvider = new AsterixAppRuntimeContextProviderForRecovery(
+ this);
+ txnSubsystem = new TransactionSubsystem(ncApplicationContext.getNodeId(), asterixAppRuntimeContextProvider);
isShuttingdown = false;
}
@@ -103,49 +122,6 @@
this.isShuttingdown = isShuttingdown;
}
- private int getBufferCachePageSize() {
- int pageSize = DEFAULT_BUFFER_CACHE_PAGE_SIZE;
- String pageSizeStr = System.getProperty(GlobalConfig.BUFFER_CACHE_PAGE_SIZE_PROPERTY, null);
- if (pageSizeStr != null) {
- try {
- pageSize = Integer.parseInt(pageSizeStr);
- } catch (NumberFormatException nfe) {
- if (GlobalConfig.ASTERIX_LOGGER.isLoggable(Level.WARNING)) {
- GlobalConfig.ASTERIX_LOGGER.warning("Wrong buffer cache page size argument. "
- + "Using default value: " + pageSize);
- }
- }
- }
-
- if (GlobalConfig.ASTERIX_LOGGER.isLoggable(Level.INFO)) {
- GlobalConfig.ASTERIX_LOGGER.info("Buffer cache page size: " + pageSize);
- }
-
- return pageSize;
- }
-
- private int getBufferCacheNumPages() {
- int numPages = GlobalConfig.DEFAULT_BUFFER_CACHE_NUM_PAGES;
- String numPagesStr = System.getProperty(GlobalConfig.BUFFER_CACHE_NUM_PAGES_PROPERTY, null);
- if (numPagesStr != null) {
- try {
- numPages = Integer.parseInt(numPagesStr);
- } catch (NumberFormatException nfe) {
- if (GlobalConfig.ASTERIX_LOGGER.isLoggable(Level.WARNING)) {
- GlobalConfig.ASTERIX_LOGGER.warning("Wrong buffer cache size argument. " + "Using default value: "
- + numPages);
- }
- return numPages;
- }
- }
-
- if (GlobalConfig.ASTERIX_LOGGER.isLoggable(Level.INFO)) {
- GlobalConfig.ASTERIX_LOGGER.info("Buffer cache size (number of pages): " + numPages);
- }
-
- return numPages;
- }
-
public void deinitialize() throws HyracksDataException {
bufferCache.close();
for (IIndex index : indexLifecycleManager.getOpenIndexes()) {
@@ -216,4 +192,29 @@
public IIOManager getIOManager() {
return ioManager;
}
+
+ @Override
+ public AsterixStorageProperties getStorageProperties() {
+ return storageProperties;
+ }
+
+ @Override
+ public AsterixTransactionProperties getTransactionProperties() {
+ return txnProperties;
+ }
+
+ @Override
+ public AsterixCompilerProperties getCompilerProperties() {
+ return compilerProperties;
+ }
+
+ @Override
+ public AsterixMetadataProperties getMetadataProperties() {
+ return metadataProperties;
+ }
+
+ @Override
+ public AsterixExternalProperties getExternalProperties() {
+ return externalProperties;
+ }
}
\ No newline at end of file
diff --git a/asterix-installer/src/main/resources/conf/asterix-configuration.xml b/asterix-installer/src/main/resources/conf/asterix-configuration.xml
index 37ddf88..9920ecf 100644
--- a/asterix-installer/src/main/resources/conf/asterix-configuration.xml
+++ b/asterix-installer/src/main/resources/conf/asterix-configuration.xml
@@ -1,116 +1,150 @@
<asterixConfiguration xmlns="asterixconf">
<property>
- <name>nc_java_opts</name>
+ <name>nc.java.opts</name>
<value>-Xmx1024m</value>
- <description>JVM parameters for each Node Contoller (NC)</description>
+ <description>JVM parameters for each Node Contoller (NC)</description>
</property>
<property>
- <name>cc_java_opts</name>
+ <name>cc.java.opts</name>
<value>-Xmx1024m</value>
- <description>JVM parameters for each Cluster Contoller (CC)</description>
+ <description>JVM parameters for each Cluster Contoller (CC)
+ </description>
</property>
<property>
- <name>size_memory_component</name>
- <value>512m</value>
- <description></description>
+ <name>storage.buffercache.pagesize</name>
+ <value>32768</value>
+ <description>The page size in bytes for pages in the buffer cache.
+ (Default = 32KB)
+ </description>
</property>
<property>
- <name>total_size_memory_component</name>
- <value>512m</value>
- <description></description>
+ <name>storage.buffercache.numpages</name>
+ <value>1024</value>
+ <description>The number of pages allocated to the disk buffer cache.
+ (Default = 1024)
+ </description>
</property>
<property>
- <name>log_buffer_num_pages</name>
+ <name>storage.buffercache.maxopenfiles</name>
+ <value>214748364</value>
+ <description>The maximum number of open files in the buffer cache.
+ (Default = 214748364)
+ </description>
+ </property>
+
+ <property>
+ <name>storage.memorycomponent.pagesize</name>
+ <value>32768</value>
+ <description>The page size in bytes for pages allocated to memory
+ components. (Default = 32KB)
+ </description>
+ </property>
+
+ <property>
+ <name>storage.memorycomponent.numpages</name>
+ <value>4096</value>
+ <description>The number of pages to allocate for a memory component.
+ (Default = 4096)
+ </description>
+ </property>
+
+ <property>
+ <name>storage.memorycomponent.globalbudget</name>
+ <value>1073741824</value>
+ <description>The total size of memory in bytes that the sum of all
+ open memory
+ components cannot exceed. (Default = 1073741824 // 1GB)
+ </description>
+ </property>
+
+ <property>
+ <name>storage.lsm.mergethreshold</name>
+ <value>3</value>
+ <description>The number of on-disk components an LSM index can have
+ before a merge is triggered. (Default = 3)
+ </description>
+ </property>
+
+ <property>
+ <name>txn.log.buffer.numpages</name>
<value>8</value>
- <description></description>
+ <description></description>
</property>
<property>
- <name>group_commit_interval</name>
- <value>200ms</value>
- <description></description>
+ <name>txn.log.buffer.pagesize</name>
+ <value>131072</value>
+ <description></description>
</property>
<property>
- <name>sort_op_memory</name>
- <value>200m</value>
- <description></description>
- </property>
-
- <property>
- <name>join_op_memory</name>
- <value>200m</value>
- <description></description>
- </property>
-
- <property>
- <name>web_interface_port</name>
- <value>19001</value>
- <description></description>
- </property>
-
- <property>
- <name>num_pages_buffer_cache</name>
- <value>8</value>
- <description></description>
- </property>
-
- <property>
- <name>log_level</name>
- <value>INFO</value>
- <description></description>
- </property>
-
- <property>
- <name>lsn_threshold</name>
- <value>64m</value>
- <description></description>
- </property>
-
- <property>
- <name>checkpointTermsInSecs</name>
- <value>120</value>
- <description></description>
- </property>
-
- <property>
- <name>escalate_threshold_entity_to_dataset</name>
- <value>8</value>
- <description></description>
- </property>
-
- <property>
- <name>num_pages_buffer_cache</name>
- <value>1000</value>
- <description></description>
- </property>
-
- <property>
- <name>log_buffer_page_size</name>
- <value>128k</value>
- <description></description>
- </property>
-
- <property>
- <name>log_partition_size</name>
+ <name>txn.log.partitionsize</name>
<value>2147483648</value>
- <description></description>
+ <description></description>
</property>
<property>
- <name>shrink_timer_threshold</name>
+ <name>txn.log.groupcommitinterval</name>
+ <value>200</value>
+ <description></description>
+ </property>
+
+ <property>
+ <name>txn.log.checkpoint.lsnthreshold</name>
+ <value>67108864</value>
+ <description></description>
+ </property>
+
+ <property>
+ <name>txn.log.checkpoint.pollfrequency</name>
+ <value>120</value>
+ <description></description>
+ </property>
+
+ <property>
+ <name>txn.lock.escalationthreshold</name>
+ <value>1000</value>
+ <description></description>
+ </property>
+
+ <property>
+ <name>txn.lock.shrinktimer</name>
<value>120000</value>
- <description></description>
+ <description></description>
</property>
<property>
- <name>log_buffer_num_pages</name>
- <value>8</value>
- <description></description>
+ <name>compiler.sortmemory</name>
+ <value>536870912</value>
+ <description></description>
+ </property>
+
+ <property>
+ <name>compiler.joinmemory</name>
+ <value>536870912</value>
+ <description></description>
+ </property>
+
+ <property>
+ <name>compiler.framesize</name>
+ <value>32768</value>
+ <description></description>
+ </property>
+
+ <property>
+ <name>external.webport</name>
+ <value>19001</value>
+ <description></description>
+ </property>
+
+ <property>
+ <name>external.loglevel</name>
+ <value>INFO</value>
+ <description></description>
</property>
</asterixConfiguration>
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java
index 994262d..09ec2f7 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataManager.java
@@ -20,11 +20,11 @@
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import edu.uci.ics.asterix.common.config.AsterixMetadataProperties;
import edu.uci.ics.asterix.common.functions.FunctionSignature;
import edu.uci.ics.asterix.metadata.api.IAsterixStateProxy;
import edu.uci.ics.asterix.metadata.api.IMetadataManager;
import edu.uci.ics.asterix.metadata.api.IMetadataNode;
-import edu.uci.ics.asterix.metadata.bootstrap.MetadataConstants;
import edu.uci.ics.asterix.metadata.entities.Dataset;
import edu.uci.ics.asterix.metadata.entities.DatasourceAdapter;
import edu.uci.ics.asterix.metadata.entities.Datatype;
@@ -85,12 +85,14 @@
private IAsterixStateProxy proxy;
private IMetadataNode metadataNode;
private final ReadWriteLock metadataLatch;
+ private final AsterixMetadataProperties metadataProperties;
- public MetadataManager(IAsterixStateProxy proxy) {
+ public MetadataManager(IAsterixStateProxy proxy, AsterixMetadataProperties metadataProperties) {
if (proxy == null) {
throw new Error("Null proxy given to MetadataManager.");
}
this.proxy = proxy;
+ this.metadataProperties = metadataProperties;
this.metadataNode = null;
this.metadataLatch = new ReentrantReadWriteLock(true);
}
@@ -105,7 +107,7 @@
metadataNode = proxy.getMetadataNode();
if (metadataNode == null) {
throw new Error("Failed to get the MetadataNode.\n" + "The MetadataNode was configured to run on NC: "
- + proxy.getAsterixProperties().getMetadataNodeName());
+ + metadataProperties.getMetadataNodeName());
}
}
}
@@ -544,10 +546,10 @@
throw new MetadataException(e);
}
}
-
+
@Override
public int getMostRecentDatasetId() throws MetadataException {
- try {
+ try {
return metadataNode.getMostRecentDatasetId();
} catch (RemoteException e) {
throw new MetadataException(e);
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IAsterixStateProxy.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IAsterixStateProxy.java
index d67cd27..7f06c30 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IAsterixStateProxy.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/api/IAsterixStateProxy.java
@@ -19,17 +19,11 @@
import java.rmi.Remote;
import java.rmi.RemoteException;
-import edu.uci.ics.asterix.common.config.AsterixProperties;
-
/**
* Interface for setting/getting distributed state of Asterix.
*/
public interface IAsterixStateProxy extends Remote, Serializable {
public void setMetadataNode(IMetadataNode metadataNode) throws RemoteException;
- public void setAsterixProperties(AsterixProperties asterixProperties) throws RemoteException;
-
public IMetadataNode getMetadataNode() throws RemoteException;
-
- public AsterixProperties getAsterixProperties() throws RemoteException;
}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/AsterixStateProxy.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/AsterixStateProxy.java
index 55aaf33..33f6994 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/AsterixStateProxy.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/AsterixStateProxy.java
@@ -19,7 +19,6 @@
import java.rmi.server.UnicastRemoteObject;
import java.util.logging.Logger;
-import edu.uci.ics.asterix.common.config.AsterixProperties;
import edu.uci.ics.asterix.metadata.api.IAsterixStateProxy;
import edu.uci.ics.asterix.metadata.api.IMetadataNode;
@@ -31,7 +30,6 @@
private static final Logger LOGGER = Logger.getLogger(AsterixStateProxy.class.getName());
private IMetadataNode metadataNode;
- private AsterixProperties asterixProperties;
private static final IAsterixStateProxy cc = new AsterixStateProxy();
public static IAsterixStateProxy registerRemoteObject() throws RemoteException {
@@ -54,14 +52,4 @@
public IMetadataNode getMetadataNode() throws RemoteException {
return this.metadataNode;
}
-
- @Override
- public void setAsterixProperties(AsterixProperties asterixProperity) throws RemoteException {
- this.asterixProperties = asterixProperity;
- }
-
- @Override
- public AsterixProperties getAsterixProperties() throws RemoteException {
- return this.asterixProperties;
- }
}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
index d978f8a..8d5d13b 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -20,16 +20,17 @@
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
+import java.util.Set;
import java.util.logging.Level;
import java.util.logging.Logger;
-import edu.uci.ics.asterix.common.config.AsterixProperties;
+import edu.uci.ics.asterix.common.config.AsterixMetadataProperties;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
import edu.uci.ics.asterix.common.context.AsterixAppRuntimeContext;
import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.external.adapter.factory.IAdapterFactory;
@@ -110,12 +111,14 @@
private static String metadataNodeName;
private static String metadataStore;
- private static HashSet<String> nodeNames;
+ private static Set<String> nodeNames;
private static String outputDir;
private static IMetadataIndex[] primaryIndexes;
private static IMetadataIndex[] secondaryIndexes;
+ private static IAsterixPropertiesProvider propertiesProvider;
+
private static void initLocalIndexArrays() {
primaryIndexes = new IMetadataIndex[] { MetadataPrimaryIndexes.DATAVERSE_DATASET,
MetadataPrimaryIndexes.DATASET_DATASET, MetadataPrimaryIndexes.DATATYPE_DATASET,
@@ -127,9 +130,10 @@
MetadataSecondaryIndexes.DATATYPENAME_ON_DATATYPE_INDEX };
}
- public static void startUniverse(AsterixProperties asterixProperties, INCApplicationContext ncApplicationContext,
- boolean isNewUniverse) throws Exception {
+ public static void startUniverse(IAsterixPropertiesProvider asterixPropertiesProvider,
+ INCApplicationContext ncApplicationContext, boolean isNewUniverse) throws Exception {
runtimeContext = (AsterixAppRuntimeContext) ncApplicationContext.getApplicationObject();
+ propertiesProvider = asterixPropertiesProvider;
// Initialize static metadata objects, such as record types and metadata
// index descriptors.
@@ -149,9 +153,10 @@
resourceRepository.registerTransactionalResourceManager(ResourceType.LSM_INVERTED_INDEX,
new IndexResourceManager(ResourceType.LSM_INVERTED_INDEX, runtimeContext.getTransactionSubsystem()));
- metadataNodeName = asterixProperties.getMetadataNodeName();
- metadataStore = asterixProperties.getMetadataStore();
- nodeNames = asterixProperties.getNodeNames();
+ AsterixMetadataProperties metadataProperties = propertiesProvider.getMetadataProperties();
+ metadataNodeName = metadataProperties.getMetadataNodeName();
+ metadataStore = metadataProperties.getMetadataStore();
+ nodeNames = metadataProperties.getNodeNames();
// nodeStores = asterixProperity.getStores();
indexLifecycleManager = runtimeContext.getIndexLifecycleManager();
@@ -165,7 +170,7 @@
// Begin a transaction against the metadata.
// Lock the metadata in X mode.
MetadataManager.INSTANCE.lock(mdTxnCtx, LockMode.X);
-
+
if (isNewUniverse) {
for (int i = 0; i < primaryIndexes.length; i++) {
enlistMetadataDataset(primaryIndexes[i], true);
@@ -196,7 +201,7 @@
LOGGER.info("Finished enlistment of metadata B-trees.");
}
}
-
+
//#. initialize datasetIdFactory
MetadataManager.INSTANCE.initializeDatasetIdFactory(mdTxnCtx);
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
@@ -327,7 +332,7 @@
adapterFactoryClassName, DatasourceAdapter.AdapterType.INTERNAL);
}
- public static void enlistMetadataDataset(IMetadataIndex index, boolean create) throws Exception {
+ private static void enlistMetadataDataset(IMetadataIndex index, boolean create) throws Exception {
String filePath = metadataStore + File.separator + index.getFileNameRelativePath();
FileReference file = new FileReference(new File(filePath));
IInMemoryBufferCache memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), DEFAULT_MEM_PAGE_SIZE,
@@ -349,9 +354,10 @@
resourceID = runtimeContext.getResourceIdFactory().createId();
indexLifecycleManager.register(resourceID, lsmBtree);
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(typeTraits,
comparatorFactories, bloomFilterKeyFields, index.isPrimaryIndex(),
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages());
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMBTreeResource);
ILocalResourceFactory localResourceFactory = localResourceFactoryProvider.getLocalResourceFactory();
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlCompiledMetadataDeclarations.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlCompiledMetadataDeclarations.java
index 70383f9..51702a7 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlCompiledMetadataDeclarations.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlCompiledMetadataDeclarations.java
@@ -22,7 +22,8 @@
import java.util.logging.Logger;
import edu.uci.ics.asterix.common.annotations.TypeDataGen;
-import edu.uci.ics.asterix.common.config.AsterixProperties;
+import edu.uci.ics.asterix.common.api.AsterixAppContextInfoImpl;
+import edu.uci.ics.asterix.common.config.AsterixMetadataProperties;
import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.formats.base.IDataFormat;
@@ -73,8 +74,10 @@
this.dataverseName = dataverseName;
this.outputFile = outputFile;
this.config = config;
+ AsterixMetadataProperties metadataProperties = ((AsterixAppContextInfoImpl) AsterixAppContextInfoImpl
+ .getInstance()).getMetadataProperties();
if (stores == null && online) {
- this.stores = AsterixProperties.INSTANCE.getStores();
+ this.stores = metadataProperties.getStores();
} else {
this.stores = stores;
}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
index 8bbddec..87e2e58 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
@@ -23,7 +23,9 @@
import java.util.Map;
import java.util.logging.Logger;
-import edu.uci.ics.asterix.common.config.AsterixProperties;
+import edu.uci.ics.asterix.common.api.AsterixAppContextInfoImpl;
+import edu.uci.ics.asterix.common.config.AsterixMetadataProperties;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
import edu.uci.ics.asterix.common.config.GlobalConfig;
@@ -151,6 +153,8 @@
private final Dataverse defaultDataverse;
private JobId jobId;
+ private final AsterixStorageProperties storageProperties;
+
private static final Map<String, String> adapterFactoryMapping = initializeAdapterFactoryMapping();
public String getPropertyValue(String propertyName) {
@@ -171,7 +175,11 @@
public AqlMetadataProvider(Dataverse defaultDataverse) {
this.defaultDataverse = defaultDataverse;
- this.stores = AsterixProperties.INSTANCE.getStores();
+ AsterixMetadataProperties metadataProperties = ((AsterixAppContextInfoImpl) AsterixAppContextInfoImpl
+ .getInstance()).getMetadataProperties();
+ this.stores = metadataProperties.getStores();
+ this.storageProperties = ((AsterixAppContextInfoImpl) AsterixAppContextInfoImpl.getInstance())
+ .getStorageProperties();
}
public void setJobId(JobId jobId) {
@@ -543,8 +551,8 @@
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
- retainInput, searchCallbackFactory);
+ storageProperties.getMemoryComponentPageSize(),
+ storageProperties.getMemoryComponentNumPages()), retainInput, searchCallbackFactory);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(btreeSearchOp, spPc.second);
} catch (MetadataException me) {
@@ -612,8 +620,8 @@
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER, proposeLinearizer(
nestedKeyType.getTypeTag(), comparatorFactories.length),
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
- retainInput, searchCallbackFactory);
+ storageProperties.getMemoryComponentPageSize(),
+ storageProperties.getMemoryComponentNumPages()), retainInput, searchCallbackFactory);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(rtreeSearchOp, spPc.second);
} catch (MetadataException me) {
@@ -772,8 +780,8 @@
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
- NoOpOperationCallbackFactory.INSTANCE);
+ storageProperties.getMemoryComponentPageSize(),
+ storageProperties.getMemoryComponentNumPages()), NoOpOperationCallbackFactory.INSTANCE);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(btreeBulkLoad,
splitsAndConstraint.second);
} catch (MetadataException me) {
@@ -839,9 +847,9 @@
new LSMBTreeDataflowHelperFactory(AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES), null,
- modificationCallbackFactory);
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
+ null, modificationCallbackFactory);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(btreeBulkLoad,
splitsAndConstraint.second);
@@ -1035,8 +1043,8 @@
new LSMBTreeDataflowHelperFactory(AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
filterFactory, modificationCallbackFactory);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(btreeBulkLoad,
splitsAndConstraint.second);
@@ -1161,8 +1169,8 @@
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
+ AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER, storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
filterFactory, modificationCallbackFactory);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(insertDeleteOp,
splitsAndConstraint.second);
@@ -1257,8 +1265,8 @@
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER, proposeLinearizer(
nestedKeyType.getTypeTag(), comparatorFactories.length),
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
- filterFactory, modificationCallbackFactory);
+ storageProperties.getMemoryComponentPageSize(),
+ storageProperties.getMemoryComponentNumPages()), filterFactory, modificationCallbackFactory);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(rtreeUpdate, splitsAndConstraint.second);
} catch (MetadataException | IOException e) {
throw new AlgebricksException(e);