[ASTERIXDB-3621][ASTERIXDB-3580][COMP] Use sameAs() for comparing nodes domain when getting partitions map
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
When getting the partitions map to be included in the required
partitioning property, use sameAs() nodes domain comparison
similar to PropertiesUtil.matchPartitioningProps() instead of
enforcing nodes order as well.
Ext-ref: MB-67128
Change-Id: Ie53228de01e7980cd96b39523d8c20da3d188860
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19903
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hussainht@gmail.com>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
Tested-by: Hussain Towaileb <hussainht@gmail.com>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java
index ac19b00..89d4120 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java
@@ -70,7 +70,7 @@
@Override
protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm, MetadataProvider md) {
- return md.getDataPartitioningProvider().getClusterLocations();
+ return md.getClusterLocations();
}
@Override
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java
index 6551346..46a9303 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java
@@ -78,7 +78,7 @@
@Override
protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm, MetadataProvider md) {
- return md.getDataPartitioningProvider().getClusterLocations();
+ return md.getClusterLocations();
}
@Override
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
index 2322e53..02ffc25 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
@@ -127,9 +127,8 @@
AlgebricksAbsolutePartitionConstraint locations);
protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm, MetadataProvider md) {
- String[] sortedLocations = md.getDataPartitioningProvider().getClusterLocations().getLocations();
- return new AlgebricksAbsolutePartitionConstraint(
- Arrays.stream(sortedLocations).distinct().toArray(String[]::new));
+ String[] locations = md.getClusterLocations().getLocations();
+ return new AlgebricksAbsolutePartitionConstraint(Arrays.stream(locations).distinct().toArray(String[]::new));
}
protected IDataParserFactory createDataParserFactory() {
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
index 71f479f..941c8f7 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java
@@ -25,7 +25,6 @@
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -1059,10 +1058,6 @@
return dataPartitioningProvider.getClusterLocations();
}
- public DataPartitioningProvider getDataPartitioningProvider() {
- return dataPartitioningProvider;
- }
-
public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildExternalDataLookupRuntime(
JobSpecification jobSpec, Dataset dataset, int[] ridIndexes, boolean retainInput,
IVariableTypeEnvironment typeEnv, IOperatorSchema opSchema, JobGenContext context,
@@ -1880,10 +1875,8 @@
if (!(nodeDomain instanceof DefaultNodeGroupDomain inputDomain)) {
return null;
}
- String[] inputLocations = inputDomain.getSortedNodes();
AlgebricksAbsolutePartitionConstraint locations = dataPartitioningProvider.getClusterLocations();
- String[] clusterLocations = locations.getLocations();
- if (!Arrays.equals(inputLocations, clusterLocations)) {
+ if (!inputDomain.sameAs(new DefaultNodeGroupDomain(locations))) {
return null;
}
return dataPartitioningProvider.getPartitionsMap();
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
index 20fc9c5..a74f100 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IMetadataProvider.java
@@ -201,6 +201,11 @@
boolean isBlockingOperatorDisabled();
+ /**
+ * Returns the cluster partitions map when the input node domain is the same as the cluster. Otherwise, null.
+ * @param nodeDomain input node domain
+ * @return the cluster partitions map.
+ */
int[][] getPartitionsMap(INodeDomain nodeDomain);
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/DefaultNodeGroupDomain.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/DefaultNodeGroupDomain.java
index 5921eea..f785fec 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/DefaultNodeGroupDomain.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/properties/DefaultNodeGroupDomain.java
@@ -18,7 +18,6 @@
*/
package org.apache.hyracks.algebricks.core.algebra.properties;
-import java.util.Arrays;
import java.util.List;
import org.apache.commons.collections4.MultiSet;
@@ -73,10 +72,4 @@
public String[] getNodes() {
return nodes.toArray(new String[0]);
}
-
- public String[] getSortedNodes() {
- String[] sortedNodes = nodes.toArray(new String[0]);
- Arrays.sort(sortedNodes);
- return sortedNodes;
- }
}