[NO ISSUE][OTH] Only order nodes in node group for new datasets
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Revert the change that sorts the cluster partitions.
- Order the nodes in the node group for newly created datasets.
- Make dump index function use the same locations as the index
split files in the index dataflow helper.
Change-Id: Id90a9c88ed44655631e44155c737459e4501c081
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11343
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mblow@apache.org>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
index 6318f8e..3351075 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.app.function;
+import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.metadata.api.IDatasourceFunction;
import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.declared.FunctionDataSource;
@@ -36,19 +37,26 @@
private final IndexDataflowHelperFactory indexDataflowHelperFactory;
private final RecordDescriptor recDesc;
private final IBinaryComparatorFactory[] comparatorFactories;
+ private final AlgebricksAbsolutePartitionConstraint storageLocations;
public DumpIndexDatasource(INodeDomain domain, IndexDataflowHelperFactory indexDataflowHelperFactory,
- RecordDescriptor recDesc, IBinaryComparatorFactory[] comparatorFactories) throws AlgebricksException {
+ RecordDescriptor recDesc, IBinaryComparatorFactory[] comparatorFactories,
+ AlgebricksAbsolutePartitionConstraint storageLocations) throws AlgebricksException {
super(DUMP_INDEX_DATASOURCE_ID, DumpIndexRewriter.DUMP_INDEX, domain);
this.indexDataflowHelperFactory = indexDataflowHelperFactory;
this.recDesc = recDesc;
this.comparatorFactories = comparatorFactories;
+ this.storageLocations = storageLocations;
+ }
+
+ @Override
+ protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm) {
+ return storageLocations;
}
@Override
protected IDatasourceFunction createFunction(MetadataProvider metadataProvider,
AlgebricksAbsolutePartitionConstraint locations) {
- return new DumpIndexFunction(metadataProvider.getClusterLocations(), indexDataflowHelperFactory, recDesc,
- comparatorFactories);
+ return new DumpIndexFunction(locations, indexDataflowHelperFactory, recDesc, comparatorFactories);
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
index 000e910..30eaf93 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
@@ -26,6 +26,7 @@
import org.apache.asterix.metadata.entities.Dataset;
import org.apache.asterix.metadata.entities.Index;
import org.apache.asterix.metadata.utils.SecondaryIndexOperationsHelper;
+import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext;
import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression;
@@ -67,7 +68,10 @@
IndexDataflowHelperFactory indexDataflowHelperFactory =
new IndexDataflowHelperFactory(metadataProvider.getStorageComponentProvider().getStorageManager(),
secondaryIndexHelper.getSecondaryFileSplitProvider());
+ AlgebricksAbsolutePartitionConstraint secondaryPartitionConstraint =
+ (AlgebricksAbsolutePartitionConstraint) secondaryIndexHelper.getSecondaryPartitionConstraint();
return new DumpIndexDatasource(context.getComputationNodeDomain(), indexDataflowHelperFactory,
- secondaryIndexHelper.getSecondaryRecDesc(), secondaryIndexHelper.getSecondaryComparatorFactories());
+ secondaryIndexHelper.getSecondaryRecDesc(), secondaryIndexHelper.getSecondaryComparatorFactories(),
+ secondaryPartitionConstraint);
}
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
index 1c4ec64..a8d027f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/NodeGroup.java
@@ -22,8 +22,6 @@
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import java.util.Set;
-import java.util.TreeSet;
import org.apache.asterix.metadata.MetadataCache;
import org.apache.asterix.metadata.api.IMetadataEntity;
@@ -33,19 +31,21 @@
*/
public class NodeGroup implements IMetadataEntity<NodeGroup> {
- private static final long serialVersionUID = 2L;
+ private static final long serialVersionUID = 1L;
// Enforced to be unique within an Asterix cluster.
private final String groupName;
- private final Set<String> nodeNames;
+ private final List<String> nodeNames;
public NodeGroup(String groupName, List<String> nodeNames) {
this.groupName = groupName;
- if (nodeNames != null) {
- this.nodeNames = new TreeSet<>(nodeNames);
- } else {
- this.nodeNames = Collections.emptySet();
- }
+ this.nodeNames = nodeNames;
+ }
+
+ public static NodeGroup createOrdered(String groupName, List<String> nodeNames) {
+ List<String> sortedNodeNames = new ArrayList<>(nodeNames);
+ Collections.sort(sortedNodeNames);
+ return new NodeGroup(groupName, sortedNodeNames);
}
public String getNodeGroupName() {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
index 4ef349b..2e10d77 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java
@@ -617,7 +617,7 @@
nodeGroup = nodeGroup + "_" + UUID.randomUUID().toString();
appCtx.getMetadataLockManager().acquireNodeGroupWriteLock(metadataProvider.getLocks(), nodeGroup);
}
- MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, new NodeGroup(nodeGroup, new ArrayList<>(ncNames)));
+ MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, NodeGroup.createOrdered(nodeGroup, new ArrayList<>(ncNames)));
return nodeGroup;
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
index fd45ff4..3586c21 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java
@@ -517,4 +517,8 @@
public IFileSplitProvider getSecondaryFileSplitProvider() {
return secondaryFileSplitProvider;
}
+
+ public AlgebricksPartitionConstraint getSecondaryPartitionConstraint() {
+ return secondaryPartitionConstraint;
+ }
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
index bf7ca34..98a97b0 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
@@ -301,8 +301,6 @@
}
}
clusterActiveLocations.removeAll(pendingRemoval);
- // for operators attempting to access storage, order the nodes list similar to a nodegroup
- Collections.sort(clusterActiveLocations);
clusterPartitionConstraint =
new AlgebricksAbsolutePartitionConstraint(clusterActiveLocations.toArray(new String[] {}));
}