checkpoint
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
index d57d424..9fbb24a 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
@@ -22,6 +22,8 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Random;
+import java.util.Set;
import org.json.JSONArray;
import org.json.JSONException;
@@ -75,6 +77,9 @@
import edu.uci.ics.asterix.metadata.MetadataManager;
import edu.uci.ics.asterix.metadata.MetadataTransactionContext;
import edu.uci.ics.asterix.metadata.api.IMetadataEntity;
+import edu.uci.ics.asterix.metadata.bootstrap.MetadataConstants;
+import edu.uci.ics.asterix.metadata.dataset.hints.DatasetHints;
+import edu.uci.ics.asterix.metadata.dataset.hints.DatasetHints.DatasetNodegroupCardinalityHint;
import edu.uci.ics.asterix.metadata.declared.AqlMetadataProvider;
import edu.uci.ics.asterix.metadata.entities.Dataset;
import edu.uci.ics.asterix.metadata.entities.Datatype;
@@ -93,6 +98,7 @@
import edu.uci.ics.asterix.om.types.ATypeTag;
import edu.uci.ics.asterix.om.types.IAType;
import edu.uci.ics.asterix.om.types.TypeSignature;
+import edu.uci.ics.asterix.om.util.AsterixAppContextInfo;
import edu.uci.ics.asterix.result.ResultReader;
import edu.uci.ics.asterix.result.ResultUtils;
import edu.uci.ics.asterix.transaction.management.service.transaction.DatasetIdFactory;
@@ -416,10 +422,11 @@
.getPartitioningExprs();
ARecordType aRecordType = (ARecordType) itemType;
aRecordType.validatePartitioningExpressions(partitioningExprs);
- String ngName = ((InternalDetailsDecl) dd.getDatasetDetailsDecl()).getNodegroupName().getValue();
+ Identifier ngName = ((InternalDetailsDecl) dd.getDatasetDetailsDecl()).getNodegroupName();
+ String nodegroupName = configureNodegroupForDataset(dd, dataverseName, ngName, mdTxnCtx);
datasetDetails = new InternalDatasetDetails(InternalDatasetDetails.FileStructure.BTREE,
InternalDatasetDetails.PartitioningStrategy.HASH, partitioningExprs, partitioningExprs,
- ngName);
+ nodegroupName);
break;
}
case EXTERNAL: {
@@ -437,7 +444,8 @@
.getPartitioningExprs();
ARecordType aRecordType = (ARecordType) itemType;
aRecordType.validatePartitioningExpressions(partitioningExprs);
- String ngName = ((FeedDetailsDecl) dd.getDatasetDetailsDecl()).getNodegroupName().getValue();
+ Identifier ngName = ((FeedDetailsDecl) dd.getDatasetDetailsDecl()).getNodegroupName();
+ String nodegroupName = configureNodegroupForDataset(dd, dataverseName, ngName, mdTxnCtx);
String adapter = ((FeedDetailsDecl) dd.getDatasetDetailsDecl()).getAdapterFactoryClassname();
Map<String, String> configuration = ((FeedDetailsDecl) dd.getDatasetDetailsDecl())
.getConfiguration();
@@ -445,7 +453,7 @@
datasetDetails = new FeedDatasetDetails(InternalDatasetDetails.FileStructure.BTREE,
InternalDatasetDetails.PartitioningStrategy.HASH, partitioningExprs, partitioningExprs,
- ngName, adapter, configuration, signature);
+ nodegroupName, adapter, configuration, signature);
break;
}
}
@@ -535,6 +543,53 @@
}
}
+ private String configureNodegroupForDataset(DatasetDecl dd, String dataverse, Identifier nodegroup,
+ MetadataTransactionContext mdTxnCtx) throws AsterixException {
+ boolean allNodesNodegroup = false;
+ int nodegroupCardinality = -1;
+ String nodegroupName = null;
+ if (nodegroup == null) {
+ String hintValue = dd.getHints().get(DatasetNodegroupCardinalityHint.NAME);
+ if (hintValue == null) {
+ allNodesNodegroup = true;
+ } else {
+ boolean valid = DatasetHints.validate(DatasetNodegroupCardinalityHint.NAME, hintValue).first;
+ if (!valid) {
+ throw new AsterixException("Incorrect use of hint:" + DatasetNodegroupCardinalityHint.NAME);
+ } else {
+ nodegroupCardinality = Integer.parseInt(hintValue);
+ }
+ }
+ } else {
+ allNodesNodegroup = nodegroup.getValue()
+ .equalsIgnoreCase(MetadataConstants.METADATA_DEFAULT_NODEGROUP_NAME);
+ }
+
+ if (allNodesNodegroup) {
+ nodegroupName = MetadataConstants.METADATA_DEFAULT_NODEGROUP_NAME;
+ } else {
+ Random random = new Random();
+ Set<String> nodeNames = AsterixAppContextInfo.getInstance().getMetadataProperties().getNodeNames();
+ String[] nodes = nodeNames.toArray(new String[] {});
+ int[] b = new int[nodeNames.size()];
+ for (int i = 0; i < b.length; i++) {
+ b[i] = i;
+ }
+ List<String> selectedNodes = new ArrayList<String>();
+ for (int i = 0; i < nodegroupCardinality; i++) {
+ int selected = i + random.nextInt(nodeNames.size() - i);
+ int selNodeIndex = b[selected];
+ selectedNodes.add(nodes[selNodeIndex]);
+ int temp = b[0];
+ b[0] = b[selected];
+ b[selected] = temp;
+ }
+ nodegroupName = dataverse + ":" + dd.getName().getValue();
+ MetadataManager.INSTANCE.addNodegroup(mdTxnCtx, new NodeGroup(nodegroupName, selectedNodes));
+ }
+ return nodegroupName;
+ }
+
private void handleCreateIndexStatement(AqlMetadataProvider metadataProvider, Statement stmt,
IHyracksClientConnection hcc) throws Exception {
@@ -892,6 +947,13 @@
//#. finally, delete the dataset.
MetadataManager.INSTANCE.dropDataset(mdTxnCtx, dataverseName, datasetName);
+ // Drop the associated nodegroup
+ if (ds.getDatasetType() == DatasetType.INTERNAL || ds.getDatasetType() == DatasetType.FEED) {
+ String nodegroup = ((InternalDatasetDetails) ds.getDatasetDetails()).getNodeGroupName();
+ if (!nodegroup.equalsIgnoreCase(MetadataConstants.METADATA_NODEGROUP_NAME)) {
+ MetadataManager.INSTANCE.dropNodegroup(mdTxnCtx, dataverseName + ":" + datasetName);
+ }
+ }
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
} catch (Exception e) {
diff --git a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/InternalDetailsDecl.java b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/InternalDetailsDecl.java
index a50625b..b574ee6 100644
--- a/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/InternalDetailsDecl.java
+++ b/asterix-aql/src/main/java/edu/uci/ics/asterix/aql/expression/InternalDetailsDecl.java
@@ -23,8 +23,9 @@
private final List<String> partitioningExprs;
public InternalDetailsDecl(Identifier nodeGroupName, List<String> partitioningExpr) {
- this.nodegroupName = nodeGroupName == null ? new Identifier(MetadataConstants.METADATA_DEFAULT_NODEGROUP_NAME)
- : nodeGroupName;
+ // this.nodegroupName = nodeGroupName == null ? new Identifier(MetadataConstants.METADATA_DEFAULT_NODEGROUP_NAME)
+ // : nodeGroupName;
+ this.nodegroupName = nodeGroupName;
this.partitioningExprs = partitioningExpr;
}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataNode.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataNode.java
index 7014500..0398777 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataNode.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/MetadataNode.java
@@ -37,6 +37,7 @@
import edu.uci.ics.asterix.metadata.api.IMetadataIndex;
import edu.uci.ics.asterix.metadata.api.IMetadataNode;
import edu.uci.ics.asterix.metadata.api.IValueExtractor;
+import edu.uci.ics.asterix.metadata.bootstrap.MetadataConstants;
import edu.uci.ics.asterix.metadata.bootstrap.MetadataPrimaryIndexes;
import edu.uci.ics.asterix.metadata.bootstrap.MetadataSecondaryIndexes;
import edu.uci.ics.asterix.metadata.entities.Dataset;
@@ -314,6 +315,7 @@
// Drop all datasets in this dataverse.
for (int i = 0; i < dataverseDatasets.size(); i++) {
dropDataset(jobId, dataverseName, dataverseDatasets.get(i).getDatasetName());
+ dropNodegroup(jobId, dataverseName + ":" + dataverseDatasets.get(i).getDatasetName());
}
}
List<Datatype> dataverseDatatypes;
@@ -426,6 +428,7 @@
}
}
}
+
} catch (Exception e) {
throw new MetadataException(e);
}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/dataset/hints/DatasetHints.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/dataset/hints/DatasetHints.java
index 0b7a7d9..2a80963 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/dataset/hints/DatasetHints.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/dataset/hints/DatasetHints.java
@@ -17,6 +17,8 @@
import java.util.HashSet;
import java.util.Set;
+import edu.uci.ics.asterix.common.config.AsterixMetadataProperties;
+import edu.uci.ics.asterix.om.util.AsterixAppContextInfo;
import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
/**
@@ -51,6 +53,7 @@
private static Set<IHint> initHints() {
Set<IHint> hints = new HashSet<IHint>();
hints.add(new DatasetCardinalityHint());
+ hints.add(new DatasetNodegroupCardinalityHint());
return hints;
}
@@ -84,4 +87,43 @@
}
}
+
+ /**
+ * Hint representing the cardinality of nodes forming the nodegroup for the dataset.
+ */
+ public static class DatasetNodegroupCardinalityHint implements IHint {
+ public static final String NAME = "NODEGROUP_CARDINALITY";
+
+ public static final int DEFAULT = 1;
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+
+ @Override
+ public Pair<Boolean, String> validateValue(String value) {
+ boolean valid = true;
+ int intValue;
+ try {
+ intValue = Integer.parseInt(value);
+ if (intValue < 0) {
+ return new Pair<Boolean, String>(false, "Value must be >= 0");
+ }
+ int numNodesInCluster = AsterixAppContextInfo.getInstance().getMetadataProperties().getNodeNames()
+ .size();
+ if (numNodesInCluster < intValue) {
+ return new Pair<Boolean, String>(false,
+ "Value must be greater or equal to the existing number of nodes in cluster ("
+ + numNodesInCluster + ")");
+ }
+ } catch (NumberFormatException nfe) {
+ valid = false;
+ return new Pair<Boolean, String>(valid, "Inappropriate value");
+ }
+ return new Pair<Boolean, String>(true, null);
+ }
+
+ }
+
}