[NO ISSUE][MTD] Add metadata API to get metadata entities as JSON
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
Change-Id: I4aeb520d49d1b2548e83d9afc16ff02eb0d8246d
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18028
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
index 0ab5c7b..254c372 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataManager.java
@@ -39,6 +39,7 @@
import org.apache.asterix.metadata.api.IAsterixStateProxy;
import org.apache.asterix.metadata.api.IExtensionMetadataEntity;
import org.apache.asterix.metadata.api.IExtensionMetadataSearchKey;
+import org.apache.asterix.metadata.api.IMetadataIndex;
import org.apache.asterix.metadata.api.IMetadataManager;
import org.apache.asterix.metadata.api.IMetadataNode;
import org.apache.asterix.metadata.entities.CompactionPolicy;
@@ -64,6 +65,7 @@
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import com.fasterxml.jackson.databind.JsonNode;
import com.google.common.base.Strings;
/**
@@ -1192,6 +1194,16 @@
}
@Override
+ public JsonNode getEntitiesAsJson(MetadataTransactionContext mdTxnCtx, IMetadataIndex metadataIndex,
+ int payloadPosition) throws AlgebricksException {
+ try {
+ return metadataNode.getEntitiesAsJson(mdTxnCtx.getTxnId(), metadataIndex, payloadPosition);
+ } catch (RemoteException e) {
+ throw new MetadataException(ErrorCode.REMOTE_EXCEPTION_WHEN_CALLING_METADATA_NODE, e);
+ }
+ }
+
+ @Override
public void rebindMetadataNode() {
rebindMetadataNode = true;
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
index 87f5129..7a6de19 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/MetadataNode.java
@@ -22,6 +22,7 @@
import static org.apache.asterix.common.api.IIdentifierMapper.Modifier.PLURAL;
import static org.apache.asterix.common.utils.IdentifierUtil.dataset;
+import java.io.PrintStream;
import java.rmi.RemoteException;
import java.util.ArrayList;
import java.util.Arrays;
@@ -48,6 +49,7 @@
import org.apache.asterix.common.transactions.TxnId;
import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.asterix.external.indexing.ExternalFile;
+import org.apache.asterix.formats.nontagged.CleanJSONPrinterFactoryProvider;
import org.apache.asterix.formats.nontagged.NullIntrospector;
import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
import org.apache.asterix.formats.nontagged.TypeTraitProvider;
@@ -119,11 +121,14 @@
import org.apache.asterix.transaction.management.service.transaction.DatasetIdFactory;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.common.utils.Triple;
+import org.apache.hyracks.algebricks.data.IPrinter;
+import org.apache.hyracks.algebricks.data.IPrinterFactory;
import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
import org.apache.hyracks.api.dataflow.value.IBinaryComparatorFactory;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.util.ByteArrayAccessibleOutputStream;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleBuilder;
import org.apache.hyracks.dataflow.common.comm.io.ArrayTupleReference;
import org.apache.hyracks.dataflow.common.data.accessors.ITupleReference;
@@ -142,9 +147,12 @@
import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.util.ExitUtil;
+import org.apache.hyracks.util.JSONUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
import com.google.common.base.Strings;
public class MetadataNode implements IMetadataNode {
@@ -327,6 +335,16 @@
return getEntities(txnId, searchKey.getSearchKey(), tupleTranslator, index);
}
+ @Override
+ public JsonNode getEntitiesAsJson(TxnId txnId, IMetadataIndex metadataIndex, int payloadPosition)
+ throws AlgebricksException, RemoteException {
+ try {
+ return getJsonNodes(txnId, metadataIndex, payloadPosition);
+ } catch (HyracksDataException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+
private <T extends IExtensionMetadataEntity> ExtensionMetadataDataset<T> getExtensionMetadataDataset(
ExtensionMetadataDatasetId datasetId) throws AlgebricksException {
ExtensionMetadataDataset<T> index = (ExtensionMetadataDataset<T>) extensionDatasets.get(datasetId);
@@ -2485,4 +2503,44 @@
public ITxnIdFactory getTxnIdFactory() {
return txnIdFactory;
}
+
+ private ArrayNode getJsonNodes(TxnId txnId, IMetadataIndex mdIndex, int payloadPosition)
+ throws AlgebricksException, HyracksDataException {
+ IValueExtractor<JsonNode> valueExtractor = createValueExtractor(mdIndex, payloadPosition);
+ List<JsonNode> results = new ArrayList<>();
+ searchIndex(txnId, mdIndex, null, valueExtractor, results);
+ ArrayNode array = JSONUtil.createArray();
+ results.forEach(array::add);
+ return array;
+ }
+
+ private static IValueExtractor<JsonNode> createValueExtractor(IMetadataIndex mdIndex, int payloadFieldIndex) {
+ return new IValueExtractor<>() {
+
+ final ARecordType payloadRecordType = mdIndex.getPayloadRecordType();
+ final IPrinterFactory printerFactory =
+ CleanJSONPrinterFactoryProvider.INSTANCE.getPrinterFactory(payloadRecordType);
+ final IPrinter printer = printerFactory.createPrinter();
+ final ByteArrayAccessibleOutputStream outputStream = new ByteArrayAccessibleOutputStream();
+ final PrintStream printStream = new PrintStream(outputStream);
+
+ @Override
+ public JsonNode getValue(TxnId txnId, ITupleReference tuple) {
+ try {
+ byte[] serRecord = tuple.getFieldData(payloadFieldIndex);
+ int recordStartOffset = tuple.getFieldStart(payloadFieldIndex);
+ int recordLength = tuple.getFieldLength(payloadFieldIndex);
+
+ printer.init();
+ outputStream.reset();
+
+ printer.print(serRecord, recordStartOffset, recordLength, printStream);
+ printStream.flush();
+ return JSONUtil.readTree(outputStream.getByteArray(), 0, outputStream.getLength());
+ } catch (Throwable th) {
+ return JSONUtil.createObject();
+ }
+ }
+ };
+ }
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
index 5ff8a03..d08578a 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataManager.java
@@ -46,6 +46,8 @@
import org.apache.asterix.metadata.entities.Synonym;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import com.fasterxml.jackson.databind.JsonNode;
+
/**
* A metadata manager provides user access to Asterix metadata (e.g., types,
* datasets, indexes, etc.). A metadata manager satisfies requests by contacting
@@ -865,6 +867,20 @@
IExtensionMetadataSearchKey searchKey) throws AlgebricksException;
/**
+ * Gets all the records of a metadata dataset as JSON.
+ *
+ * @param mdTxnCtx metadata transaction context
+ * @param metadataIndex the metadata dataset
+ * @param payloadPosition the position of the record in the tuple
+ *
+ * @return the metadata records as JSON
+ *
+ * @throws AlgebricksException AlgebricksException
+ */
+ JsonNode getEntitiesAsJson(MetadataTransactionContext mdTxnCtx, IMetadataIndex metadataIndex, int payloadPosition)
+ throws AlgebricksException;
+
+ /**
* Indicate when the metadata node has left or rejoined the cluster, and the
* MetadataManager should rebind it
*/
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
index 561a4fa..378cf47 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/api/IMetadataNode.java
@@ -47,6 +47,8 @@
import org.apache.asterix.transaction.management.opcallbacks.AbstractIndexModificationOperationCallback;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+import com.fasterxml.jackson.databind.JsonNode;
+
/**
* A metadata node stores metadata in its local storage structures (currently
* BTrees). A metadata node services requests on behalf of the (possibly remote)
@@ -969,6 +971,21 @@
<T extends IExtensionMetadataEntity> List<T> getEntities(TxnId txnId, IExtensionMetadataSearchKey searchKey)
throws AlgebricksException, RemoteException;
+ /**
+ * Gets all the records of a metadata dataset as JSON.
+ *
+ * @param txnId transaction id
+ * @param metadataIndex the metadata dataset
+ * @param payloadPosition the position of the record in the tuple
+ *
+ * @return the metadata records as JSON
+ *
+ * @throws AlgebricksException AlgebricksException
+ * @throws RemoteException RemoteException
+ */
+ JsonNode getEntitiesAsJson(TxnId txnId, IMetadataIndex metadataIndex, int payloadPosition)
+ throws AlgebricksException, RemoteException;
+
void addFeedConnection(TxnId txnId, FeedConnection feedConnection) throws AlgebricksException, RemoteException;
FeedConnection getFeedConnection(TxnId txnId, DataverseName dataverseName, String feedName, String datasetName)
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
index 0d05d46..d502322 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DatasetTupleTranslator.java
@@ -84,7 +84,7 @@
public class DatasetTupleTranslator extends AbstractTupleTranslator<Dataset> {
// Payload field containing serialized Dataset.
- private static final int DATASET_PAYLOAD_TUPLE_FIELD_INDEX = 2;
+ public static final int DATASET_PAYLOAD_TUPLE_FIELD_INDEX = 2;
protected AMutableInt32 aInt32;
protected AMutableInt64 aInt64;
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DataverseTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DataverseTupleTranslator.java
index 77c838a..700beb9 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DataverseTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/DataverseTupleTranslator.java
@@ -39,7 +39,7 @@
public class DataverseTupleTranslator extends AbstractTupleTranslator<Dataverse> {
// Payload field containing serialized Dataverse.
- private static final int DATAVERSE_PAYLOAD_TUPLE_FIELD_INDEX = 1;
+ public static final int DATAVERSE_PAYLOAD_TUPLE_FIELD_INDEX = 1;
protected AMutableInt32 aInt32;
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java
index ef9c143..9ad83c1 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java
@@ -81,7 +81,7 @@
public class FunctionTupleTranslator extends AbstractDatatypeTupleTranslator<Function> {
// Payload field containing serialized Function.
- private static final int FUNCTION_PAYLOAD_TUPLE_FIELD_INDEX = 3;
+ public static final int FUNCTION_PAYLOAD_TUPLE_FIELD_INDEX = 3;
protected OrderedListBuilder dependenciesListBuilder;
protected OrderedListBuilder dependencyListBuilder;
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
index b00a706..08f947f 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/IndexTupleTranslator.java
@@ -83,7 +83,7 @@
public class IndexTupleTranslator extends AbstractTupleTranslator<Index> {
// Payload field containing serialized Index.
- private static final int INDEX_PAYLOAD_TUPLE_FIELD_INDEX = 3;
+ public static final int INDEX_PAYLOAD_TUPLE_FIELD_INDEX = 3;
// Field name of open field.
public static final String GRAM_LENGTH_FIELD_NAME = "GramLength";
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java
index e3e879b..f255fff 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/LibraryTupleTranslator.java
@@ -45,7 +45,7 @@
public class LibraryTupleTranslator extends AbstractTupleTranslator<Library> {
// Payload field containing serialized Library.
- private static final int LIBRARY_PAYLOAD_TUPLE_FIELD_INDEX = 2;
+ public static final int LIBRARY_PAYLOAD_TUPLE_FIELD_INDEX = 2;
protected LibraryTupleTranslator(boolean getTuple) {
super(getTuple, MetadataPrimaryIndexes.LIBRARY_DATASET, LIBRARY_PAYLOAD_TUPLE_FIELD_INDEX);
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/SynonymTupleTranslator.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/SynonymTupleTranslator.java
index 6bad2d4..12b410c 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/SynonymTupleTranslator.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/SynonymTupleTranslator.java
@@ -36,7 +36,7 @@
// Payload field containing serialized Synonym.
- private static final int SYNONYM_PAYLOAD_TUPLE_FIELD_INDEX = 2;
+ public static final int SYNONYM_PAYLOAD_TUPLE_FIELD_INDEX = 2;
protected SynonymTupleTranslator(boolean getTuple) {
super(getTuple, MetadataPrimaryIndexes.SYNONYM_DATASET, SYNONYM_PAYLOAD_TUPLE_FIELD_INDEX);
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java
index f937e97..191df1f 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/JSONUtil.java
@@ -50,6 +50,10 @@
SORTED_MAPPER.configure(MapperFeature.SORT_PROPERTIES_ALPHABETICALLY, true);
}
+ public static JsonNode readTree(byte[] bytes, int start, int size) throws IOException {
+ return OBJECT_MAPPER.readTree(bytes, start, size);
+ }
+
public static String convertNode(final JsonNode node) throws JsonProcessingException {
return SORTED_MAPPER.writeValueAsString(SORTED_MAPPER.treeToValue(node, Object.class));
}