Correct behaviour of drop dataverse and drop dataset to drop external data indexes
diff --git a/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java b/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
index 855ee19..6acf6c1 100644
--- a/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
+++ b/asterix-app/src/main/java/edu/uci/ics/asterix/aql/translator/AqlTranslator.java
@@ -744,7 +744,7 @@
for (int j = 0; j < datasets.size(); j++) {
String datasetName = datasets.get(j).getDatasetName();
DatasetType dsType = datasets.get(j).getDatasetType();
- if (dsType == DatasetType.INTERNAL || dsType == DatasetType.FEED) {
+ if (dsType == DatasetType.INTERNAL || dsType == DatasetType.FEED || dsType == DatasetType.EXTERNAL) {
List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName,
datasetName);
@@ -857,7 +857,7 @@
}
}
- if (ds.getDatasetType() == DatasetType.INTERNAL || ds.getDatasetType() == DatasetType.FEED) {
+ if (ds.getDatasetType() == DatasetType.INTERNAL || ds.getDatasetType() == DatasetType.FEED || ds.getDatasetType() == DatasetType.EXTERNAL) {
//#. prepare jobs to drop the datatset and the indexes in NC
List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes(mdTxnCtx, dataverseName, datasetName);
@@ -958,7 +958,7 @@
+ dataverseName);
}
- if (ds.getDatasetType() == DatasetType.INTERNAL || ds.getDatasetType() == DatasetType.FEED) {
+ if (ds.getDatasetType() == DatasetType.INTERNAL || ds.getDatasetType() == DatasetType.FEED || ds.getDatasetType() == DatasetType.EXTERNAL) {
indexName = stmtIndexDrop.getIndexName().getValue();
Index index = MetadataManager.INSTANCE.getIndex(mdTxnCtx, dataverseName, datasetName, indexName);
if (index == null) {
@@ -996,8 +996,8 @@
//#. finally, delete the existing index
MetadataManager.INSTANCE.dropIndex(mdTxnCtx, dataverseName, datasetName, indexName);
} else {
- throw new AlgebricksException(datasetName
- + " is an external dataset. Indexes are not maintained for external datasets.");
+ //throw new AlgebricksException(datasetName
+ // + " is an external dataset. Indexes are not maintained for external datasets.");
}
MetadataManager.INSTANCE.commitTransaction(mdTxnCtx);
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/HDFSAdapterFactory.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/HDFSAdapterFactory.java
index 7e63e84..e9d2175 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/HDFSAdapterFactory.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/adapter/factory/HDFSAdapterFactory.java
@@ -210,20 +210,20 @@
ITypeTraits[] tt = new ITypeTraits[numOfPrimaryKeys];
if(optimize)
{
- serde[0] = AqlSerializerDeserializerProvider.INSTANCE.getNonTaggedSerializerDeserializer(BuiltinType.AINT32);
+ serde[0] = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT32);
tt[0] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(BuiltinType.AINT32);
}
else
{
- serde[0] = AqlSerializerDeserializerProvider.INSTANCE.getNonTaggedSerializerDeserializer(BuiltinType.ASTRING);
+ serde[0] = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ASTRING);
tt[0] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(BuiltinType.ASTRING);
}
- serde[1] = AqlSerializerDeserializerProvider.INSTANCE.getNonTaggedSerializerDeserializer(BuiltinType.AINT64);
+ serde[1] = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT64);
tt[1] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(BuiltinType.AINT64);
if(isRCFile)
{
//we add the row number for rc-files
- serde[2] = AqlSerializerDeserializerProvider.INSTANCE.getNonTaggedSerializerDeserializer(BuiltinType.AINT32);
+ serde[2] = AqlSerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT32);
tt[2] = AqlTypeTraitProvider.INSTANCE.getTypeTrait(BuiltinType.AINT32);
}
return new RecordDescriptor(serde, tt);
diff --git a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/util/ExternalIndexHashPartitionComputerFactory.java b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/util/ExternalIndexHashPartitionComputerFactory.java
index 85814f5..29fcfb1 100644
--- a/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/util/ExternalIndexHashPartitionComputerFactory.java
+++ b/asterix-external-data/src/main/java/edu/uci/ics/asterix/external/util/ExternalIndexHashPartitionComputerFactory.java
@@ -24,7 +24,6 @@
public class ExternalIndexHashPartitionComputerFactory implements ITuplePartitionComputerFactory{
private static final long serialVersionUID = 1L;
private final int[] hashFields;
- private transient ByteBuffer serializedLong;
private final int bytesInHDFSBlock = 67108864;
private final IBinaryHashFunctionFactory[] hashFunctionFactories;
@SuppressWarnings("unchecked")
@@ -41,8 +40,8 @@
for (int i = 0; i < hashFunctionFactories.length; ++i) {
hashFunctions[i] = hashFunctionFactories[i].createBinaryHashFunction();
}
- serializedLong = ByteBuffer.allocate(8);
return new ITuplePartitionComputer() {
+ private ByteBuffer serializedLong = ByteBuffer.allocate(8);;
private AInt64 byteLocation;
private ByteBufferInputStream bbis = new ByteBufferInputStream();
private DataInputStream dis = new DataInputStream(bbis);
@@ -61,7 +60,7 @@
int fEnd = accessor.getFieldEndOffset(tIndex, fIdx);
if(j == 1)
{
- //clear the buffer
+ //reset the buffer
serializedLong.clear();
//read byte location
bbis.setByteBuffer(accessor.getBuffer() , accessor.getTupleStartOffset(tIndex) + accessor.getFieldSlotsLength() + accessor.getFieldStartOffset(tIndex, hashFields[1]));
diff --git a/asterix-maven-plugins/lexer-generator-maven-plugin/src/main/resources/Lexer.java b/asterix-maven-plugins/lexer-generator-maven-plugin/src/main/resources/Lexer.java
index ac97e0b..92d1c8e 100644
--- a/asterix-maven-plugins/lexer-generator-maven-plugin/src/main/resources/Lexer.java
+++ b/asterix-maven-plugins/lexer-generator-maven-plugin/src/main/resources/Lexer.java
@@ -54,6 +54,8 @@
// ================================================================================
public int next() throws [LEXER_NAME]Exception, IOException{
+ if(bufpos < 0)
+ readNextChar();
char currentChar = buffer[bufpos];
while (currentChar == ' ' || currentChar=='\t' || currentChar == '\n' || currentChar=='\r')
currentChar = readNextChar();
@@ -71,7 +73,14 @@
//used before processing a new patch in the inputStream
public void reset(){
- bufpos = endOf_USED_Buffer = 0; // -- -- -- reuse the buffer
+ line = 1;
+ column = 0;
+ bufpos = -1;
+ endOf_UNUSED_Buffer = bufsize;
+ endOf_USED_Buffer = 0;
+ prevCharIsCR = false;
+ prevCharIsLF = false;
+ tokenBegin = -1;
}
// ================================================================================
@@ -96,7 +105,6 @@
buffer = new char[bufsize];
tokenBegin = -1;
maxUnusedBufferSize = 4096/2;
- readNextChar();
}
public String getLastTokenImage() {
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
index d37bf8f..6dd0f16 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/declared/AqlMetadataProvider.java
@@ -1611,9 +1611,20 @@
}
int numPartitions = 0;
- InternalDatasetDetails datasetDetails = (InternalDatasetDetails) dataset.getDatasetDetails();
- List<String> nodeGroup = MetadataManager.INSTANCE.getNodegroup(mdTxnCtx, datasetDetails.getNodeGroupName())
- .getNodeNames();
+ List<String> nodeGroup = null;
+ if(dataset.getDatasetType() == DatasetType.EXTERNAL)
+ {
+ ExternalDatasetDetails datasetDetails = (ExternalDatasetDetails) dataset.getDatasetDetails();
+ nodeGroup = MetadataManager.INSTANCE.getNodegroup(mdTxnCtx, datasetDetails.getNodeGroupName())
+ .getNodeNames();
+ }
+ else
+ {
+ InternalDatasetDetails datasetDetails = (InternalDatasetDetails) dataset.getDatasetDetails();
+ nodeGroup = MetadataManager.INSTANCE.getNodegroup(mdTxnCtx, datasetDetails.getNodeGroupName())
+ .getNodeNames();
+ }
+
for (String nd : nodeGroup) {
numPartitions += AsterixClusterProperties.INSTANCE.getNumberOfIODevices(nd);
}
diff --git a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/ExternalDatasetDetails.java b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/ExternalDatasetDetails.java
index 87fc5a5..849c6bd 100644
--- a/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/ExternalDatasetDetails.java
+++ b/asterix-metadata/src/main/java/edu/uci/ics/asterix/metadata/entities/ExternalDatasetDetails.java
@@ -40,13 +40,13 @@
private static final long serialVersionUID = 1L;
private final String adapter;
- private final Map<String, String> properties;
+ private final Map<String,String> properties;
private final String nodeGroupName;
- private final static ARecordType externalRecordType = MetadataRecordTypes.EXTERNAL_DETAILS_RECORDTYPE;
+ private final static ARecordType externalRecordType = MetadataRecordTypes.EXTERNAL_DETAILS_RECORDTYPE;
private final static ARecordType propertyRecordType = MetadataRecordTypes.DATASOURCE_ADAPTER_PROPERTIES_RECORDTYPE;
- public ExternalDatasetDetails(String adapter, Map<String, String> properties, String nodeGroupName) {
+ public ExternalDatasetDetails(String adapter, Map<String,String> properties, String nodeGroupName) {
this.properties = properties;
this.adapter = adapter;
this.nodeGroupName = nodeGroupName;
@@ -58,7 +58,7 @@
this.nodeGroupName = null;
}
- public String getAdapter() {
+ public String getAdapter() {
return adapter;
}
@@ -66,15 +66,15 @@
return properties;
}
+ public String getNodeGroupName() {
+ return nodeGroupName;
+ }
+
@Override
public DatasetType getDatasetType() {
return DatasetType.EXTERNAL;
}
- public String getNodeGroupName() {
- return nodeGroupName;
- }
-
@Override
public void writeDatasetDetailsRecordType(DataOutput out) throws HyracksDataException {
IARecordBuilder externalRecordBuilder = new RecordBuilder();
@@ -102,16 +102,15 @@
writePropertyTypeRecord(name, value, itemValue.getDataOutput());
listBuilder.addItem(itemValue);
}
+ fieldValue.reset();
+ listBuilder.write(fieldValue.getDataOutput(), true);
+ externalRecordBuilder.addField(MetadataRecordTypes.EXTERNAL_DETAILS_ARECORD_PROPERTIES_FIELD_INDEX, fieldValue);
//write field 2
fieldValue.reset();
aString.setValue(getNodeGroupName());
stringSerde.serialize(aString, fieldValue.getDataOutput());
externalRecordBuilder.addField(MetadataRecordTypes.EXTERNAL_DETAILS_ARECORD_GROUPNAME_FIELD_INDEX, fieldValue);
-
- fieldValue.reset();
- listBuilder.write(fieldValue.getDataOutput(), true);
- externalRecordBuilder.addField(MetadataRecordTypes.EXTERNAL_DETAILS_ARECORD_PROPERTIES_FIELD_INDEX, fieldValue);
try {
externalRecordBuilder.write(out, true);