storage configurations are now accessed from config file
diff --git a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
index 49ca5ba..792375e 100644
--- a/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
+++ b/asterix-algebra/src/main/java/edu/uci/ics/asterix/algebra/operators/physical/InvertedIndexPOperator.java
@@ -3,7 +3,8 @@
import java.util.ArrayList;
import java.util.List;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.api.AsterixAppContextInfoImpl;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.common.dataflow.IAsterixApplicationContextInfo;
import edu.uci.ics.asterix.metadata.MetadataException;
@@ -196,20 +197,23 @@
IBinaryTokenizerFactory queryTokenizerFactory = InvertedIndexAccessMethod.getBinaryTokenizerFactory(
searchModifierType, searchKeyType, secondaryIndex);
IIndexDataflowHelperFactory dataflowHelperFactory;
+
+ AsterixStorageProperties storageProperties = ((AsterixAppContextInfoImpl) AsterixAppContextInfoImpl
+ .getInstance()).getStorageProperties();
if (!isPartitioned) {
dataflowHelperFactory = new LSMInvertedIndexDataflowHelperFactory(
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages());
} else {
dataflowHelperFactory = new PartitionedLSMInvertedIndexDataflowHelperFactory(
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages());
}
LSMInvertedIndexSearchOperatorDescriptor invIndexSearchOp = new LSMInvertedIndexSearchOperatorDescriptor(
jobSpec, queryField, appContext.getStorageManagerInterface(), secondarySplitsAndConstraint.first,
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
index b9b9524..2173aea 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/DatasetOperations.java
@@ -22,6 +22,8 @@
import java.util.logging.Logger;
import edu.uci.ics.asterix.api.common.Job;
+import edu.uci.ics.asterix.common.api.AsterixAppContextInfoImpl;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
import edu.uci.ics.asterix.common.config.GlobalConfig;
import edu.uci.ics.asterix.common.config.OptimizationConfUtil;
@@ -118,14 +120,16 @@
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider
.splitProviderAndPartitionConstraintsForInternalOrFeedDataset(dataset.getDataverseName(), datasetName,
datasetName);
+ AsterixStorageProperties storageProperties = ((AsterixAppContextInfoImpl) AsterixAppContextInfoImpl
+ .getInstance()).getStorageProperties();
IndexDropOperatorDescriptor primaryBtreeDrop = new IndexDropOperatorDescriptor(specPrimary,
AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER, AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
splitsAndConstraint.first, new LSMBTreeDataflowHelperFactory(
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES));
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()));
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(specPrimary, primaryBtreeDrop,
splitsAndConstraint.second);
@@ -163,10 +167,12 @@
}
LOGGER.info("CREATING File Splits: " + sb.toString());
+ AsterixStorageProperties storageProperties = ((AsterixAppContextInfoImpl) AsterixAppContextInfoImpl
+ .getInstance()).getStorageProperties();
//prepare a LocalResourceMetadata which will be stored in NC's local resource repository
ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(typeTraits,
- comparatorFactories, blooFilterKeyFields, true, GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
+ comparatorFactories, blooFilterKeyFields, true, storageProperties.getMemoryComponentPageSize(),
+ storageProperties.getMemoryComponentNumPages());
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMBTreeResource);
@@ -176,9 +182,9 @@
new LSMBTreeDataflowHelperFactory(AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES), localResourceFactoryProvider,
- NoOpOperationCallbackFactory.INSTANCE);
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
+ localResourceFactoryProvider, NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, indexCreateOp,
splitsAndConstraint.second);
spec.addRoot(indexCreateOp);
@@ -258,15 +264,18 @@
numElementsHint = Long.parseLong(dataset.getHints().get("CARDINALITY"));
}
+ AsterixStorageProperties storageProperties = ((AsterixAppContextInfoImpl) AsterixAppContextInfoImpl
+ .getInstance()).getStorageProperties();
TreeIndexBulkLoadOperatorDescriptor btreeBulkLoad = new TreeIndexBulkLoadOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER, AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
splitsAndConstraint.first, typeTraits, comparatorFactories, blooFilterKeyFields, fieldPermutation,
- GlobalConfig.DEFAULT_BTREE_FILL_FACTOR, false, numElementsHint, new LSMBTreeDataflowHelperFactory(
+ GlobalConfig.DEFAULT_BTREE_FILL_FACTOR, false, numElementsHint,
+ new LSMBTreeDataflowHelperFactory(AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES), NoOpOperationCallbackFactory.INSTANCE);
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
+ NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, btreeBulkLoad,
splitsAndConstraint.second);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
index dbbbe44..d1ba0f4 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/IndexOperations.java
@@ -1,6 +1,7 @@
package edu.uci.ics.asterix.file;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.api.AsterixAppContextInfoImpl;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.config.OptimizationConfUtil;
import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
@@ -48,14 +49,16 @@
Pair<IFileSplitProvider, AlgebricksPartitionConstraint> splitsAndConstraint = metadataProvider
.splitProviderAndPartitionConstraintsForInternalOrFeedDataset(dataverseName, datasetName, indexName);
+ AsterixStorageProperties storageProperties = ((AsterixAppContextInfoImpl) AsterixAppContextInfoImpl
+ .getInstance()).getStorageProperties();
IndexDropOperatorDescriptor btreeDrop = new IndexDropOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER, AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
splitsAndConstraint.first, new LSMBTreeDataflowHelperFactory(
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES));
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()));
AlgebricksPartitionConstraintHelper
.setPartitionConstraintInJobSpec(spec, btreeDrop, splitsAndConstraint.second);
spec.addRoot(btreeDrop);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeCreator.java
index 2a1349b..958f8d1 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryBTreeCreator.java
@@ -1,6 +1,7 @@
package edu.uci.ics.asterix.file;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
+import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.transaction.management.resource.ILocalResourceMetadata;
@@ -26,31 +27,33 @@
public class SecondaryBTreeCreator extends SecondaryIndexCreator {
- protected SecondaryBTreeCreator(PhysicalOptimizationConfig physOptConf) {
- super(physOptConf);
+ protected SecondaryBTreeCreator(PhysicalOptimizationConfig physOptConf,
+ IAsterixPropertiesProvider propertiesProvider) {
+ super(physOptConf, propertiesProvider);
}
@Override
public JobSpecification buildCreationJobSpec() throws AsterixException, AlgebricksException {
JobSpecification spec = new JobSpecification();
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
//prepare a LocalResourceMetadata which will be stored in NC's local resource repository
ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(
secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, secondaryBloomFilterKeyFields, false,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages());
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMBTreeResource);
TreeIndexCreateOperatorDescriptor secondaryIndexCreateOp = new TreeIndexCreateOperatorDescriptor(spec,
AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER, AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
secondaryFileSplitProvider, secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories,
- secondaryBloomFilterKeyFields, new LSMBTreeDataflowHelperFactory(
+ secondaryBloomFilterKeyFields,
+ new LSMBTreeDataflowHelperFactory(AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES), localResourceFactoryProvider,
- NoOpOperationCallbackFactory.INSTANCE);
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
+ localResourceFactoryProvider, NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, secondaryIndexCreateOp,
secondaryPartitionConstraint);
spec.addRoot(secondaryIndexCreateOp);
@@ -80,13 +83,17 @@
// Sort by secondary keys.
ExternalSortOperatorDescriptor sortOp = createSortOp(spec, secondaryComparatorFactories, secondaryRecDesc);
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
// Create secondary BTree bulk load op.
- TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(spec, numSecondaryKeys,
+ TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(
+ spec,
+ numSecondaryKeys,
new LSMBTreeDataflowHelperFactory(AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES), BTree.DEFAULT_FILL_FACTOR);
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
+ BTree.DEFAULT_FILL_FACTOR);
// Connect the operators.
spec.connect(new OneToOneConnectorDescriptor(spec), keyProviderOp, 0, primaryScanOp, 0);
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexCreator.java
index e7c2e3e..357c55e 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryIndexCreator.java
@@ -19,8 +19,10 @@
import java.io.IOException;
import java.util.List;
+import edu.uci.ics.asterix.common.api.AsterixAppContextInfoImpl;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.formats.nontagged.AqlBinaryBooleanInspectorImpl;
@@ -103,29 +105,35 @@
protected RecordDescriptor secondaryRecDesc;
protected ICopyEvaluatorFactory[] secondaryFieldAccessEvalFactories;
+ protected IAsterixPropertiesProvider propertiesProvider;
+
// Prevent public construction. Should be created via createIndexCreator().
- protected SecondaryIndexCreator(PhysicalOptimizationConfig physOptConf) {
+ protected SecondaryIndexCreator(PhysicalOptimizationConfig physOptConf,
+ IAsterixPropertiesProvider propertiesProvider) {
this.physOptConf = physOptConf;
+ this.propertiesProvider = propertiesProvider;
}
public static SecondaryIndexCreator createIndexCreator(CompiledCreateIndexStatement createIndexStmt,
AqlMetadataProvider metadataProvider, PhysicalOptimizationConfig physOptConf) throws AsterixException,
AlgebricksException {
+ IAsterixPropertiesProvider asterixPropertiesProvider = ((AsterixAppContextInfoImpl) AsterixAppContextInfoImpl
+ .getInstance());
SecondaryIndexCreator indexCreator = null;
switch (createIndexStmt.getIndexType()) {
case BTREE: {
- indexCreator = new SecondaryBTreeCreator(physOptConf);
+ indexCreator = new SecondaryBTreeCreator(physOptConf, asterixPropertiesProvider);
break;
}
case RTREE: {
- indexCreator = new SecondaryRTreeCreator(physOptConf);
+ indexCreator = new SecondaryRTreeCreator(physOptConf, asterixPropertiesProvider);
break;
}
case WORD_INVIX:
case NGRAM_INVIX:
case FUZZY_WORD_INVIX:
case FUZZY_NGRAM_INVIX: {
- indexCreator = new SecondaryInvertedIndexCreator(physOptConf);
+ indexCreator = new SecondaryInvertedIndexCreator(physOptConf, asterixPropertiesProvider);
break;
}
default: {
@@ -266,6 +274,7 @@
int[] lowKeyFields = null;
// +Infinity
int[] highKeyFields = null;
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
BTreeSearchOperatorDescriptor primarySearchOp = new BTreeSearchOperatorDescriptor(spec, primaryRecDesc,
AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER, AsterixRuntimeComponentsProvider.NOINDEX_PROVIDER,
primaryFileSplitProvider, primaryRecDesc.getTypeTraits(), primaryComparatorFactories,
@@ -273,8 +282,9 @@
new LSMBTreeDataflowHelperFactory(AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES), false, NoOpOperationCallbackFactory.INSTANCE);
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()), false,
+ NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, primarySearchOp,
primaryPartitionConstraint);
return primarySearchOp;
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
index 0bf379d..7b53a28 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryInvertedIndexCreator.java
@@ -2,8 +2,9 @@
import java.util.List;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
@@ -60,8 +61,9 @@
private RecordDescriptor tokenKeyPairRecDesc;
private boolean isPartitioned;
- protected SecondaryInvertedIndexCreator(PhysicalOptimizationConfig physOptConf) {
- super(physOptConf);
+ protected SecondaryInvertedIndexCreator(PhysicalOptimizationConfig physOptConf,
+ IAsterixPropertiesProvider propertiesProvider) {
+ super(physOptConf, propertiesProvider);
}
@Override
@@ -150,10 +152,12 @@
public JobSpecification buildCreationJobSpec() throws AsterixException, AlgebricksException {
JobSpecification spec = new JobSpecification();
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
//prepare a LocalResourceMetadata which will be stored in NC's local resource repository
ILocalResourceMetadata localResourceMetadata = new LSMInvertedIndexLocalResourceMetadata(invListsTypeTraits,
primaryComparatorFactories, tokenTypeTraits, tokenComparatorFactories, tokenizerFactory,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES, isPartitioned);
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages(),
+ isPartitioned);
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMInvertedIndexResource);
@@ -259,20 +263,21 @@
}
private IIndexDataflowHelperFactory createDataflowHelperFactory() {
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
if (!isPartitioned) {
return new LSMInvertedIndexDataflowHelperFactory(
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages());
} else {
return new PartitionedLSMInvertedIndexDataflowHelperFactory(
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages());
}
}
}
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java
index 6f400f9..1a5e0da 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/file/SecondaryRTreeCreator.java
@@ -2,7 +2,8 @@
import java.util.List;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
+import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.common.exceptions.AsterixException;
import edu.uci.ics.asterix.dataflow.data.nontagged.valueproviders.AqlPrimitiveValueProviderFactory;
@@ -49,20 +50,22 @@
protected int numNestedSecondaryKeyFields;
protected ATypeTag keyType;
- protected SecondaryRTreeCreator(PhysicalOptimizationConfig physOptConf) {
- super(physOptConf);
+ protected SecondaryRTreeCreator(PhysicalOptimizationConfig physOptConf,
+ IAsterixPropertiesProvider propertiesProvider) {
+ super(physOptConf, propertiesProvider);
}
@Override
public JobSpecification buildCreationJobSpec() throws AsterixException, AlgebricksException {
JobSpecification spec = new JobSpecification();
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
//prepare a LocalResourceMetadata which will be stored in NC's local resource repository
ILocalResourceMetadata localResourceMetadata = new LSMRTreeLocalResourceMetadata(
secondaryRecDesc.getTypeTraits(), secondaryComparatorFactories, primaryComparatorFactories,
valueProviderFactories, RTreePolicyType.RTREE, AqlMetadataProvider.proposeLinearizer(keyType,
- secondaryComparatorFactories.length), GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE,
- GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
+ secondaryComparatorFactories.length), storageProperties.getMemoryComponentPageSize(),
+ storageProperties.getMemoryComponentNumPages());
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMRTreeResource);
@@ -74,8 +77,8 @@
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER, AqlMetadataProvider.proposeLinearizer(
- keyType, secondaryComparatorFactories.length),
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
+ keyType, secondaryComparatorFactories.length), storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
localResourceFactoryProvider, NoOpOperationCallbackFactory.INSTANCE);
AlgebricksPartitionConstraintHelper.setPartitionConstraintInJobSpec(spec, secondaryIndexCreateOp,
secondaryPartitionConstraint);
@@ -149,6 +152,7 @@
selectOp = createFilterNullsSelectOp(spec, numNestedSecondaryKeyFields);
}
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
// Create secondary RTree bulk load op.
TreeIndexBulkLoadOperatorDescriptor secondaryBulkLoadOp = createTreeIndexBulkLoadOp(
spec,
@@ -158,8 +162,8 @@
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER, AqlMetadataProvider.proposeLinearizer(
- keyType, secondaryComparatorFactories.length),
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
+ keyType, secondaryComparatorFactories.length), storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
BTree.DEFAULT_FILL_FACTOR);
// Connect the operators.
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
index 04d5452..d29d92b 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/hyracks/bootstrap/NCApplicationEntryPoint.java
@@ -112,7 +112,7 @@
}
MetadataManager.INSTANCE = new MetadataManager(proxy, metadataProperties);
MetadataManager.INSTANCE.init();
- MetadataBootstrap.startUniverse(metadataProperties, ncApplicationContext,
+ MetadataBootstrap.startUniverse(runtimeContext, ncApplicationContext,
systemState == SystemState.NEW_UNIVERSE);
MetadataBootstrap.startDDLRecovery();
}
diff --git a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/GlobalConfig.java b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/GlobalConfig.java
index c41856d..3948e69 100644
--- a/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/GlobalConfig.java
+++ b/asterix-common/src/main/java/edu/uci/ics/asterix/common/config/GlobalConfig.java
@@ -25,10 +25,6 @@
public static int DEFAULT_INPUT_DATA_COLUMN = 0;
- public static int DEFAULT_INDEX_MEM_PAGE_SIZE = 32768;
-
- public static int DEFAULT_INDEX_MEM_NUM_PAGES = 1000;
-
public static int getFrameSize() {
int frameSize = GlobalConfig.DEFAULT_FRAME_SIZE;
String frameSizeStr = System.getProperty(GlobalConfig.FRAME_SIZE_PROPERTY);
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
index 8322ae1..8d5d13b 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -27,9 +27,10 @@
import java.util.logging.Logger;
import edu.uci.ics.asterix.common.config.AsterixMetadataProperties;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
-import edu.uci.ics.asterix.common.config.GlobalConfig;
+import edu.uci.ics.asterix.common.config.IAsterixPropertiesProvider;
import edu.uci.ics.asterix.common.context.AsterixAppRuntimeContext;
import edu.uci.ics.asterix.common.context.AsterixRuntimeComponentsProvider;
import edu.uci.ics.asterix.external.adapter.factory.IAdapterFactory;
@@ -116,6 +117,8 @@
private static IMetadataIndex[] primaryIndexes;
private static IMetadataIndex[] secondaryIndexes;
+ private static IAsterixPropertiesProvider propertiesProvider;
+
private static void initLocalIndexArrays() {
primaryIndexes = new IMetadataIndex[] { MetadataPrimaryIndexes.DATAVERSE_DATASET,
MetadataPrimaryIndexes.DATASET_DATASET, MetadataPrimaryIndexes.DATATYPE_DATASET,
@@ -127,9 +130,10 @@
MetadataSecondaryIndexes.DATATYPENAME_ON_DATATYPE_INDEX };
}
- public static void startUniverse(AsterixMetadataProperties metadataProperties,
+ public static void startUniverse(IAsterixPropertiesProvider asterixPropertiesProvider,
INCApplicationContext ncApplicationContext, boolean isNewUniverse) throws Exception {
runtimeContext = (AsterixAppRuntimeContext) ncApplicationContext.getApplicationObject();
+ propertiesProvider = asterixPropertiesProvider;
// Initialize static metadata objects, such as record types and metadata
// index descriptors.
@@ -149,6 +153,7 @@
resourceRepository.registerTransactionalResourceManager(ResourceType.LSM_INVERTED_INDEX,
new IndexResourceManager(ResourceType.LSM_INVERTED_INDEX, runtimeContext.getTransactionSubsystem()));
+ AsterixMetadataProperties metadataProperties = propertiesProvider.getMetadataProperties();
metadataNodeName = metadataProperties.getMetadataNodeName();
metadataStore = metadataProperties.getMetadataStore();
nodeNames = metadataProperties.getNodeNames();
@@ -327,7 +332,7 @@
adapterFactoryClassName, DatasourceAdapter.AdapterType.INTERNAL);
}
- public static void enlistMetadataDataset(IMetadataIndex index, boolean create) throws Exception {
+ private static void enlistMetadataDataset(IMetadataIndex index, boolean create) throws Exception {
String filePath = metadataStore + File.separator + index.getFileNameRelativePath();
FileReference file = new FileReference(new File(filePath));
IInMemoryBufferCache memBufferCache = new InMemoryBufferCache(new HeapBufferAllocator(), DEFAULT_MEM_PAGE_SIZE,
@@ -349,9 +354,10 @@
resourceID = runtimeContext.getResourceIdFactory().createId();
indexLifecycleManager.register(resourceID, lsmBtree);
+ AsterixStorageProperties storageProperties = propertiesProvider.getStorageProperties();
ILocalResourceMetadata localResourceMetadata = new LSMBTreeLocalResourceMetadata(typeTraits,
comparatorFactories, bloomFilterKeyFields, index.isPrimaryIndex(),
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES);
+ storageProperties.getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages());
ILocalResourceFactoryProvider localResourceFactoryProvider = new PersistentLocalResourceFactoryProvider(
localResourceMetadata, LocalResource.LSMBTreeResource);
ILocalResourceFactory localResourceFactory = localResourceFactoryProvider.getLocalResourceFactory();
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
index 22c9013..87e2e58 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
@@ -25,6 +25,7 @@
import edu.uci.ics.asterix.common.api.AsterixAppContextInfoImpl;
import edu.uci.ics.asterix.common.config.AsterixMetadataProperties;
+import edu.uci.ics.asterix.common.config.AsterixStorageProperties;
import edu.uci.ics.asterix.common.config.DatasetConfig.DatasetType;
import edu.uci.ics.asterix.common.config.DatasetConfig.IndexType;
import edu.uci.ics.asterix.common.config.GlobalConfig;
@@ -152,6 +153,8 @@
private final Dataverse defaultDataverse;
private JobId jobId;
+ private final AsterixStorageProperties storageProperties;
+
private static final Map<String, String> adapterFactoryMapping = initializeAdapterFactoryMapping();
public String getPropertyValue(String propertyName) {
@@ -175,6 +178,8 @@
AsterixMetadataProperties metadataProperties = ((AsterixAppContextInfoImpl) AsterixAppContextInfoImpl
.getInstance()).getMetadataProperties();
this.stores = metadataProperties.getStores();
+ this.storageProperties = ((AsterixAppContextInfoImpl) AsterixAppContextInfoImpl.getInstance())
+ .getStorageProperties();
}
public void setJobId(JobId jobId) {
@@ -546,8 +551,8 @@
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
- retainInput, searchCallbackFactory);
+ storageProperties.getMemoryComponentPageSize(),
+ storageProperties.getMemoryComponentNumPages()), retainInput, searchCallbackFactory);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(btreeSearchOp, spPc.second);
} catch (MetadataException me) {
@@ -615,8 +620,8 @@
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER, proposeLinearizer(
nestedKeyType.getTypeTag(), comparatorFactories.length),
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
- retainInput, searchCallbackFactory);
+ storageProperties.getMemoryComponentPageSize(),
+ storageProperties.getMemoryComponentNumPages()), retainInput, searchCallbackFactory);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(rtreeSearchOp, spPc.second);
} catch (MetadataException me) {
@@ -775,8 +780,8 @@
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
- NoOpOperationCallbackFactory.INSTANCE);
+ storageProperties.getMemoryComponentPageSize(),
+ storageProperties.getMemoryComponentNumPages()), NoOpOperationCallbackFactory.INSTANCE);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(btreeBulkLoad,
splitsAndConstraint.second);
} catch (MetadataException me) {
@@ -842,9 +847,9 @@
new LSMBTreeDataflowHelperFactory(AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES), null,
- modificationCallbackFactory);
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
+ null, modificationCallbackFactory);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(btreeBulkLoad,
splitsAndConstraint.second);
@@ -1038,8 +1043,8 @@
new LSMBTreeDataflowHelperFactory(AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
+ AsterixRuntimeComponentsProvider.LSMBTREE_PROVIDER, storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
filterFactory, modificationCallbackFactory);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(btreeBulkLoad,
splitsAndConstraint.second);
@@ -1164,8 +1169,8 @@
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
- AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER,
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
+ AsterixRuntimeComponentsProvider.LSMINVERTEDINDEX_PROVIDER, storageProperties
+ .getMemoryComponentPageSize(), storageProperties.getMemoryComponentNumPages()),
filterFactory, modificationCallbackFactory);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(insertDeleteOp,
splitsAndConstraint.second);
@@ -1260,8 +1265,8 @@
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER,
AsterixRuntimeComponentsProvider.LSMRTREE_PROVIDER, proposeLinearizer(
nestedKeyType.getTypeTag(), comparatorFactories.length),
- GlobalConfig.DEFAULT_INDEX_MEM_PAGE_SIZE, GlobalConfig.DEFAULT_INDEX_MEM_NUM_PAGES),
- filterFactory, modificationCallbackFactory);
+ storageProperties.getMemoryComponentPageSize(),
+ storageProperties.getMemoryComponentNumPages()), filterFactory, modificationCallbackFactory);
return new Pair<IOperatorDescriptor, AlgebricksPartitionConstraint>(rtreeUpdate, splitsAndConstraint.second);
} catch (MetadataException | IOException e) {
throw new AlgebricksException(e);