[NO ISSUE][CONF] Make NC Properties Extensible

- user model changes: no
- storage format changes: no
- interface changes: yes

Details:
- Add extensible IPropertiesFactory to allow
  extensions to override default properties.
- Make StorageProperties extensible.
- Remove metadata datasets config option.

Change-Id: I2b3cb2f47a286ecca90227d7756a62a102205cfc
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2442
Reviewed-by: Michael Blow <mblow@apache.org>
Integration-Tests: Michael Blow <mblow@apache.org>
Tested-by: Michael Blow <mblow@apache.org>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index 3780078..c724952 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -36,6 +36,7 @@
 import org.apache.asterix.common.api.IDatasetLifecycleManager;
 import org.apache.asterix.common.api.IDatasetMemoryManager;
 import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.common.api.IPropertiesFactory;
 import org.apache.asterix.common.cluster.ClusterPartition;
 import org.apache.asterix.common.config.ActiveProperties;
 import org.apache.asterix.common.config.AsterixExtension;
@@ -87,7 +88,6 @@
 import org.apache.hyracks.control.nc.NodeControllerService;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
 import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
-import org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTracker;
 import org.apache.hyracks.storage.am.lsm.common.impls.AsynchronousScheduler;
 import org.apache.hyracks.storage.am.lsm.common.impls.PrefixMergePolicyFactory;
 import org.apache.hyracks.storage.common.ILocalResourceRepository;
@@ -142,26 +142,26 @@
     private IIndexCheckpointManagerProvider indexCheckpointManagerProvider;
     private IReplicaManager replicaManager;
 
-    public NCAppRuntimeContext(INCServiceContext ncServiceContext, List<AsterixExtension> extensions)
-            throws AsterixException, InstantiationException, IllegalAccessException, ClassNotFoundException,
-            IOException {
+    public NCAppRuntimeContext(INCServiceContext ncServiceContext, List<AsterixExtension> extensions,
+            IPropertiesFactory propertiesFactory) throws AsterixException, InstantiationException,
+            IllegalAccessException, ClassNotFoundException, IOException {
         List<AsterixExtension> allExtensions = new ArrayList<>();
         this.ncServiceContext = ncServiceContext;
-        PropertiesAccessor propertiesAccessor = PropertiesAccessor.getInstance(ncServiceContext.getAppConfig());
-        compilerProperties = new CompilerProperties(propertiesAccessor);
-        externalProperties = new ExternalProperties(propertiesAccessor);
-        metadataProperties = new MetadataProperties(propertiesAccessor);
-        storageProperties = new StorageProperties(propertiesAccessor);
-        txnProperties = new TransactionProperties(propertiesAccessor);
-        activeProperties = new ActiveProperties(propertiesAccessor);
-        buildProperties = new BuildProperties(propertiesAccessor);
-        replicationProperties = new ReplicationProperties(propertiesAccessor);
-        messagingProperties = new MessagingProperties(propertiesAccessor);
-        nodeProperties = new NodeProperties(propertiesAccessor);
+        compilerProperties = propertiesFactory.newCompilerProperties();
+        externalProperties = propertiesFactory.newExternalProperties();
+        metadataProperties = propertiesFactory.newMetadataProperties();
+        storageProperties = propertiesFactory.newStorageProperties();
+        txnProperties = propertiesFactory.newTransactionProperties();
+        activeProperties = propertiesFactory.newActiveProperties();
+        buildProperties = propertiesFactory.newBuildProperties();
+        replicationProperties = propertiesFactory.newReplicationProperties();
+        messagingProperties = propertiesFactory.newMessagingProperties();
+        nodeProperties = propertiesFactory.newNodeProperties();
         libraryManager = new ExternalLibraryManager();
         if (extensions != null) {
             allExtensions.addAll(extensions);
         }
+        PropertiesAccessor propertiesAccessor = PropertiesAccessor.getInstance(ncServiceContext.getAppConfig());
         allExtensions.addAll(propertiesAccessor.getExtensions());
         ncExtensionManager = new NCExtensionManager(allExtensions);
         componentProvider = new StorageComponentProvider();
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
index 3a8a92f..0cd01ae 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
@@ -18,6 +18,7 @@
  */
 package org.apache.asterix.hyracks.bootstrap;
 
+import java.io.IOException;
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.List;
@@ -29,13 +30,17 @@
 import org.apache.asterix.app.replication.message.RegistrationTasksRequestMessage;
 import org.apache.asterix.common.api.AsterixThreadFactory;
 import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.common.api.IPropertiesFactory;
 import org.apache.asterix.common.config.AsterixExtension;
 import org.apache.asterix.common.config.ExternalProperties;
 import org.apache.asterix.common.config.GlobalConfig;
 import org.apache.asterix.common.config.MessagingProperties;
 import org.apache.asterix.common.config.MetadataProperties;
 import org.apache.asterix.common.config.NodeProperties;
+import org.apache.asterix.common.config.PropertiesAccessor;
+import org.apache.asterix.common.config.PropertiesFactory;
 import org.apache.asterix.common.config.StorageProperties;
+import org.apache.asterix.common.exceptions.AsterixException;
 import org.apache.asterix.common.transactions.Checkpoint;
 import org.apache.asterix.common.transactions.IRecoveryManager;
 import org.apache.asterix.common.transactions.IRecoveryManager.SystemState;
@@ -106,7 +111,7 @@
                     (controllerService).getConfiguration().getClusterPublicAddress());
         }
         MetadataBuiltinFunctions.init();
-        runtimeContext = new NCAppRuntimeContext(ncServiceCtx, getExtensions());
+        runtimeContext = new NCAppRuntimeContext(ncServiceCtx, getExtensions(), getPropertiesFactory());
         MetadataProperties metadataProperties = runtimeContext.getMetadataProperties();
         if (!metadataProperties.getNodeNames().contains(this.ncServiceCtx.getNodeId())) {
             if (LOGGER.isInfoEnabled()) {
@@ -155,6 +160,11 @@
         return Collections.emptyList();
     }
 
+    protected IPropertiesFactory getPropertiesFactory() throws IOException, AsterixException {
+        PropertiesAccessor propertiesAccessor = PropertiesAccessor.getInstance(ncServiceCtx.getAppConfig());
+        return new PropertiesFactory(propertiesAccessor);
+    }
+
     @Override
     public void stop() throws Exception {
         if (!stopInitiated) {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IPropertiesFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IPropertiesFactory.java
new file mode 100644
index 0000000..7857f18
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/IPropertiesFactory.java
@@ -0,0 +1,103 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.api;
+
+import org.apache.asterix.common.config.ActiveProperties;
+import org.apache.asterix.common.config.BuildProperties;
+import org.apache.asterix.common.config.CompilerProperties;
+import org.apache.asterix.common.config.ExternalProperties;
+import org.apache.asterix.common.config.MessagingProperties;
+import org.apache.asterix.common.config.MetadataProperties;
+import org.apache.asterix.common.config.NodeProperties;
+import org.apache.asterix.common.config.ReplicationProperties;
+import org.apache.asterix.common.config.StorageProperties;
+import org.apache.asterix.common.config.TransactionProperties;
+
+public interface IPropertiesFactory {
+
+    /**
+     * Creates new {@link StorageProperties}
+     *
+     * @return new storage properties
+     */
+    StorageProperties newStorageProperties();
+
+    /**
+     * Creates new {@link TransactionProperties}
+     *
+     * @return new transaction properties
+     */
+    TransactionProperties newTransactionProperties();
+
+    /**
+     * Creates new {@link CompilerProperties}
+     *
+     * @return new compiler properties
+     */
+    CompilerProperties newCompilerProperties();
+
+    /**
+     * Creates new {@link MetadataProperties}
+     *
+     * @return new metadata properties
+     */
+    MetadataProperties newMetadataProperties();
+
+    /**
+     * Creates new {@link ExternalProperties}
+     *
+     * @return new external properties
+     */
+    ExternalProperties newExternalProperties();
+
+    /**
+     * Creates new {@link ActiveProperties}
+     *
+     * @return new active properties
+     */
+    ActiveProperties newActiveProperties();
+
+    /**
+     * Creates new {@link BuildProperties}
+     *
+     * @return new build properties
+     */
+    BuildProperties newBuildProperties();
+
+    /**
+     * Creates new {@link ReplicationProperties}
+     *
+     * @return new replication properties
+     */
+    ReplicationProperties newReplicationProperties();
+
+    /**
+     * Creates new {@link MessagingProperties}
+     *
+     * @return new messaging properties
+     */
+    MessagingProperties newMessagingProperties();
+
+    /**
+     * Creates new {@link NodeProperties}
+     *
+     * @return new node properties
+     */
+    NodeProperties newNodeProperties();
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesFactory.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesFactory.java
new file mode 100644
index 0000000..8f75397
--- /dev/null
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/PropertiesFactory.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.config;
+
+import org.apache.asterix.common.api.IPropertiesFactory;
+
+public class PropertiesFactory implements IPropertiesFactory {
+
+    protected final PropertiesAccessor propertiesAccessor;
+
+    public PropertiesFactory(PropertiesAccessor propertiesAccessor) {
+        this.propertiesAccessor = propertiesAccessor;
+    }
+
+    @Override
+    public StorageProperties newStorageProperties() {
+        return new StorageProperties(propertiesAccessor);
+    }
+
+    @Override
+    public TransactionProperties newTransactionProperties() {
+        return new TransactionProperties(propertiesAccessor);
+    }
+
+    @Override
+    public CompilerProperties newCompilerProperties() {
+        return new CompilerProperties(propertiesAccessor);
+    }
+
+    @Override
+    public MetadataProperties newMetadataProperties() {
+        return new MetadataProperties(propertiesAccessor);
+    }
+
+    @Override
+    public ExternalProperties newExternalProperties() {
+        return new ExternalProperties(propertiesAccessor);
+    }
+
+    @Override
+    public ActiveProperties newActiveProperties() {
+        return new ActiveProperties(propertiesAccessor);
+    }
+
+    @Override
+    public BuildProperties newBuildProperties() {
+        return new BuildProperties(propertiesAccessor);
+    }
+
+    @Override
+    public ReplicationProperties newReplicationProperties() {
+        return new ReplicationProperties(propertiesAccessor);
+    }
+
+    @Override
+    public MessagingProperties newMessagingProperties() {
+        return new MessagingProperties(propertiesAccessor);
+    }
+
+    @Override
+    public NodeProperties newNodeProperties() {
+        return new NodeProperties(propertiesAccessor);
+    }
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
index e01198f..e813968 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
@@ -26,6 +26,7 @@
 
 import java.util.function.Function;
 
+import org.apache.asterix.common.metadata.MetadataIndexImmutableProperties;
 import org.apache.hyracks.api.config.IApplicationConfig;
 import org.apache.hyracks.api.config.IOption;
 import org.apache.hyracks.api.config.IOptionType;
@@ -45,7 +46,6 @@
         STORAGE_MEMORYCOMPONENT_NUMCOMPONENTS(INTEGER, 2),
         STORAGE_METADATA_MEMORYCOMPONENT_NUMPAGES(INTEGER, 32),
         STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE(DOUBLE, 0.01d),
-        STORAGE_METADATA_DATASETS(INTEGER, 14),
         STORAGE_MAX_ACTIVE_WRITABLE_DATASETS(INTEGER, 8);
 
         private final IOptionType interpreter;
@@ -85,8 +85,6 @@
                     return "The number of pages to allocate for a metadata memory component";
                 case STORAGE_LSM_BLOOMFILTER_FALSEPOSITIVERATE:
                     return "The maximum acceptable false positive rate for bloom filters associated with LSM indexes";
-                case STORAGE_METADATA_DATASETS:
-                    return "The number of metadata datasets";
                 case STORAGE_MAX_ACTIVE_WRITABLE_DATASETS:
                     return "The maximum number of datasets that can be concurrently modified";
                 default:
@@ -113,6 +111,8 @@
         }
     }
 
+    private static final int SYSTEM_RESERVED_DATASETS = 0;
+
     public StorageProperties(PropertiesAccessor accessor) {
         super(accessor);
     }
@@ -136,7 +136,8 @@
     public int getMemoryComponentNumPages() {
         final long metadataReservedMem = getMetadataReservedMemory();
         final long globalUserDatasetMem = getMemoryComponentGlobalBudget() - metadataReservedMem;
-        final long userDatasetMem = globalUserDatasetMem / getMaxActiveWritableDatasets();
+        final long userDatasetMem =
+                globalUserDatasetMem / (getMaxActiveWritableDatasets() + geSystemReservedDatasets());
         return (int) (userDatasetMem / getMemoryComponentPageSize());
     }
 
@@ -177,8 +178,12 @@
         return accessor.getInt(Option.STORAGE_MAX_ACTIVE_WRITABLE_DATASETS);
     }
 
-    private int getMetadataDatasets() {
-        return accessor.getInt(Option.STORAGE_METADATA_DATASETS);
+    protected int getMetadataDatasets() {
+        return MetadataIndexImmutableProperties.METADATA_DATASETS_COUNT;
+    }
+
+    protected int geSystemReservedDatasets() {
+        return SYSTEM_RESERVED_DATASETS;
     }
 
     private long getMetadataReservedMemory() {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/MetadataIndexImmutableProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/MetadataIndexImmutableProperties.java
index 2876e11..b131d01 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/MetadataIndexImmutableProperties.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/metadata/MetadataIndexImmutableProperties.java
@@ -23,6 +23,7 @@
     public static final int FIRST_AVAILABLE_EXTENSION_METADATA_DATASET_ID = 52;
     public static final int FIRST_AVAILABLE_USER_DATASET_ID = 100;
     public static final int METADATA_DATASETS_PARTITIONS = 1;
+    public static final int METADATA_DATASETS_COUNT = 14;
 
     private final String indexName;
     private final int datasetId;