[ASTERIXDB-1130][STO]: JSON serialization for persisted objects
- user model changes: no
- storage format changes:
This change replaces the use of Java serialization for persisted objects
such as dataset/index metadata, checkpoints, ect...
This will break backward compatibly with any existing AsterixDB instance.
However, the change is needed to enable future backward compatibility support
for persisted objects.
- interface changes:
IJsonSerializable: contains API to serialize a class as a JsonNode.
IPersistedResourceRegistry: contains a mapping between an IJsonSerializable
class and a unique type id. An IPersistedResourceRegistry is responsible
for generating the class identifier in the JSON output.
The class identifier will always contain the following attributes:
@type: a unique type id that identifies the object type.
@version: the version of the serialized class.
@class: the serialized class full name.
Any registered class with PersistedResourceRegistry must provide
a static fromJson(IPersistedResourceRegistry, JsonNode) method for
deserialization. This is ensured during the class registration process.
Change-Id: I5b103e06eab6627dbfe9d531caae1a3ac4b296da
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2752
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
Integration-Tests: Murtadha Hubail <mhubail@apache.org>
Tested-by: Murtadha Hubail <mhubail@apache.org>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/pom.xml
index 1c118de..7ed450c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/pom.xml
@@ -91,5 +91,9 @@
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeLocalResource.java
index f72e17c..f74a23b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/ExternalRTreeLocalResource.java
@@ -27,6 +27,8 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.io.IJsonSerializable;
+import org.apache.hyracks.api.io.IPersistedResourceRegistry;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
@@ -38,6 +40,9 @@
import org.apache.hyracks.storage.common.IIndex;
import org.apache.hyracks.storage.common.IStorageManager;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
/**
* The local resource class for disk only lsm r-tree
*/
@@ -62,6 +67,15 @@
bloomFilterFalsePositiveRate);
}
+ private ExternalRTreeLocalResource(IPersistedResourceRegistry registry, JsonNode json,
+ IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
+ RTreePolicyType rtreePolicyType, ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields,
+ int[] buddyBTreeFields, boolean isPointMBR, double bloomFilterFalsePositiveRate)
+ throws HyracksDataException {
+ super(registry, json, btreeCmpFactories, valueProviderFactories, rtreePolicyType, linearizeCmpFactory,
+ rtreeFields, buddyBTreeFields, isPointMBR, bloomFilterFalsePositiveRate);
+ }
+
@Override
public IIndex createInstance(INCServiceContext ncServiceCtx) throws HyracksDataException {
IIOManager ioManager = ncServiceCtx.getIoManager();
@@ -75,4 +89,20 @@
buddyBTreeFields, durable, isPointMBR, metadataPageManagerFactory, ncServiceCtx.getTracer());
}
+
+ @Override
+ public JsonNode toJson(IPersistedResourceRegistry registry) throws HyracksDataException {
+ final ObjectNode jsonObject = registry.getClassIdentifier(getClass(), serialVersionUID);
+ super.appendToJson(jsonObject, registry);
+ return jsonObject;
+ }
+
+ public static IJsonSerializable fromJson(IPersistedResourceRegistry registry, JsonNode json)
+ throws HyracksDataException {
+ LSMRTreeLocalResource lsmRTree = (LSMRTreeLocalResource) LSMRTreeLocalResource.fromJson(registry, json);
+ return new ExternalRTreeLocalResource(registry, json, lsmRTree.btreeCmpFactories,
+ lsmRTree.valueProviderFactories, lsmRTree.rtreePolicyType, lsmRTree.linearizeCmpFactory,
+ lsmRTree.rtreeFields, lsmRTree.buddyBTreeFields, lsmRTree.isPointMBR,
+ lsmRTree.bloomFilterFalsePositiveRate);
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeLocalResource.java
index 634504b..d744ab3 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeLocalResource.java
@@ -18,6 +18,7 @@
*/
package org.apache.hyracks.storage.am.lsm.rtree.dataflow;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -28,6 +29,8 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.io.IJsonSerializable;
+import org.apache.hyracks.api.io.IPersistedResourceRegistry;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
@@ -42,6 +45,10 @@
import org.apache.hyracks.storage.common.IIndex;
import org.apache.hyracks.storage.common.IStorageManager;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
public class LSMRTreeLocalResource extends LsmResource {
private static final long serialVersionUID = 1L;
@@ -78,6 +85,22 @@
this.bloomFilterFalsePositiveRate = bloomFilterFalsePositiveRate;
}
+ protected LSMRTreeLocalResource(IPersistedResourceRegistry registry, JsonNode json,
+ IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
+ RTreePolicyType rtreePolicyType, ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields,
+ int[] buddyBTreeFields, boolean isPointMBR, double bloomFilterFalsePositiveRate)
+ throws HyracksDataException {
+ super(registry, json);
+ this.btreeCmpFactories = btreeCmpFactories;
+ this.valueProviderFactories = valueProviderFactories;
+ this.rtreePolicyType = rtreePolicyType;
+ this.linearizeCmpFactory = linearizeCmpFactory;
+ this.rtreeFields = rtreeFields;
+ this.buddyBTreeFields = buddyBTreeFields;
+ this.isPointMBR = isPointMBR;
+ this.bloomFilterFalsePositiveRate = bloomFilterFalsePositiveRate;
+ }
+
@Override
public IIndex createInstance(INCServiceContext ncServiceCtx) throws HyracksDataException {
IIOManager ioManager = ncServiceCtx.getIoManager();
@@ -93,4 +116,60 @@
buddyBTreeFields, filterTypeTraits, filterCmpFactories, filterFields, durable, isPointMBR,
metadataPageManagerFactory);
}
+
+ @Override
+ public JsonNode toJson(IPersistedResourceRegistry registry) throws HyracksDataException {
+ ObjectNode jsonObject = registry.getClassIdentifier(getClass(), serialVersionUID);
+ super.appendToJson(jsonObject, registry);
+ return jsonObject;
+ }
+
+ @Override
+ protected void appendToJson(final ObjectNode json, IPersistedResourceRegistry registry)
+ throws HyracksDataException {
+ super.appendToJson(json, registry);
+ final ArrayNode btreeCmpFactoriesArray = OBJECT_MAPPER.createArrayNode();
+ for (IBinaryComparatorFactory factory : btreeCmpFactories) {
+ btreeCmpFactoriesArray.add(factory.toJson(registry));
+ }
+ json.set("btreeCmpFactories", btreeCmpFactoriesArray);
+ json.set("linearizeCmpFactory", linearizeCmpFactory.toJson(registry));
+ final ArrayNode valueProviderFactoriesArray = OBJECT_MAPPER.createArrayNode();
+ for (IPrimitiveValueProviderFactory factory : valueProviderFactories) {
+ valueProviderFactoriesArray.add(factory.toJson(registry));
+ }
+ json.set("valueProviderFactories", valueProviderFactoriesArray);
+ json.set("rtreePolicyType", rtreePolicyType.toJson(registry));
+ json.putPOJO("rtreeFields", rtreeFields);
+ json.putPOJO("buddyBTreeFields", buddyBTreeFields);
+ json.put("isPointMBR", isPointMBR);
+ json.put("bloomFilterFalsePositiveRate", bloomFilterFalsePositiveRate);
+ }
+
+ public static IJsonSerializable fromJson(IPersistedResourceRegistry registry, JsonNode json)
+ throws HyracksDataException {
+ final int[] buddyBTreeFields = OBJECT_MAPPER.convertValue(json.get("buddyBTreeFields"), int[].class);
+ final int[] rtreeFields = OBJECT_MAPPER.convertValue(json.get("rtreeFields"), int[].class);
+ final double bloomFilterFalsePositiveRate = json.get("bloomFilterFalsePositiveRate").asDouble();
+ final boolean isPointMBR = json.get("isPointMBR").asBoolean();
+ final RTreePolicyType rtreePolicyType = (RTreePolicyType) registry.deserialize(json.get("rtreePolicyType"));
+ final ILinearizeComparatorFactory linearizeCmpFactory =
+ (ILinearizeComparatorFactory) registry.deserialize(json.get("linearizeCmpFactory"));
+ final List<IBinaryComparatorFactory> btreeCmpFactoriesList = new ArrayList<>();
+ final ArrayNode jsonBtreeCmpFactories = (ArrayNode) json.get("btreeCmpFactories");
+ for (JsonNode cf : jsonBtreeCmpFactories) {
+ btreeCmpFactoriesList.add((IBinaryComparatorFactory) registry.deserialize(cf));
+ }
+ final IBinaryComparatorFactory[] btreeCmpFactories =
+ btreeCmpFactoriesList.toArray(new IBinaryComparatorFactory[0]);
+ final List<IPrimitiveValueProviderFactory> valueProviderFactoriesList = new ArrayList<>();
+ final ArrayNode jsonValueProviderFactories = (ArrayNode) json.get("valueProviderFactories");
+ for (JsonNode cf : jsonValueProviderFactories) {
+ valueProviderFactoriesList.add((IPrimitiveValueProviderFactory) registry.deserialize(cf));
+ }
+ final IPrimitiveValueProviderFactory[] valueProviderFactories =
+ valueProviderFactoriesList.toArray(new IPrimitiveValueProviderFactory[0]);
+ return new LSMRTreeLocalResource(registry, json, btreeCmpFactories, valueProviderFactories, rtreePolicyType,
+ linearizeCmpFactory, rtreeFields, buddyBTreeFields, isPointMBR, bloomFilterFalsePositiveRate);
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterLocalResource.java
index f91a5f7..4b6f1de 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/dataflow/LSMRTreeWithAntiMatterLocalResource.java
@@ -18,6 +18,7 @@
*/
package org.apache.hyracks.storage.am.lsm.rtree.dataflow;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -28,6 +29,8 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.io.IJsonSerializable;
+import org.apache.hyracks.api.io.IPersistedResourceRegistry;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.common.api.IPrimitiveValueProviderFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
@@ -42,11 +45,15 @@
import org.apache.hyracks.storage.am.rtree.frames.RTreePolicyType;
import org.apache.hyracks.storage.common.IStorageManager;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
public class LSMRTreeWithAntiMatterLocalResource extends LsmResource {
private static final long serialVersionUID = 1L;
- protected final IBinaryComparatorFactory[] btreeComparatorFactories;
+ protected final IBinaryComparatorFactory[] btreeCmpFactories;
protected final IPrimitiveValueProviderFactory[] valueProviderFactories;
protected final RTreePolicyType rtreePolicyType;
protected final ILinearizeComparatorFactory linearizeCmpFactory;
@@ -59,13 +66,26 @@
ILSMOperationTrackerFactory opTrackerProvider, ILSMIOOperationCallbackFactory ioOpCallbackFactory,
IMetadataPageManagerFactory metadataPageManagerFactory, IVirtualBufferCacheProvider vbcProvider,
ILSMIOOperationSchedulerProvider ioSchedulerProvider, ILSMMergePolicyFactory mergePolicyFactory,
- Map<String, String> mergePolicyProperties, IBinaryComparatorFactory[] btreeComparatorFactories,
+ Map<String, String> mergePolicyProperties, IBinaryComparatorFactory[] btreeCmpFactories,
IPrimitiveValueProviderFactory[] valueProviderFactories, RTreePolicyType rtreePolicyType,
ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields, boolean isPointMBR, boolean durable) {
super(path, storageManager, typeTraits, rtreeCmpFactories, filterTypeTraits, filterCmpFactories, filterFields,
opTrackerProvider, ioOpCallbackFactory, metadataPageManagerFactory, vbcProvider, ioSchedulerProvider,
mergePolicyFactory, mergePolicyProperties, durable);
- this.btreeComparatorFactories = btreeComparatorFactories;
+ this.btreeCmpFactories = btreeCmpFactories;
+ this.valueProviderFactories = valueProviderFactories;
+ this.rtreePolicyType = rtreePolicyType;
+ this.linearizeCmpFactory = linearizeCmpFactory;
+ this.rtreeFields = rtreeFields;
+ this.isPointMBR = isPointMBR;
+ }
+
+ private LSMRTreeWithAntiMatterLocalResource(IPersistedResourceRegistry registry, JsonNode json,
+ IBinaryComparatorFactory[] btreeCmpFactories, IPrimitiveValueProviderFactory[] valueProviderFactories,
+ RTreePolicyType rtreePolicyType, ILinearizeComparatorFactory linearizeCmpFactory, int[] rtreeFields,
+ boolean isPointMBR) throws HyracksDataException {
+ super(registry, json);
+ this.btreeCmpFactories = btreeCmpFactories;
this.valueProviderFactories = valueProviderFactories;
this.rtreePolicyType = rtreePolicyType;
this.linearizeCmpFactory = linearizeCmpFactory;
@@ -80,11 +100,59 @@
List<IVirtualBufferCache> virtualBufferCaches = vbcProvider.getVirtualBufferCaches(serviceCtx, file);
ioOpCallbackFactory.initialize(serviceCtx, this);
return LSMRTreeUtils.createLSMTreeWithAntiMatterTuples(ioManager, virtualBufferCaches, file,
- storageManager.getBufferCache(serviceCtx), typeTraits, cmpFactories, btreeComparatorFactories,
+ storageManager.getBufferCache(serviceCtx), typeTraits, cmpFactories, btreeCmpFactories,
valueProviderFactories, rtreePolicyType,
mergePolicyFactory.createMergePolicy(mergePolicyProperties, serviceCtx),
opTrackerProvider.getOperationTracker(serviceCtx, this), ioSchedulerProvider.getIoScheduler(serviceCtx),
ioOpCallbackFactory, linearizeCmpFactory, rtreeFields, filterTypeTraits, filterCmpFactories,
filterFields, durable, isPointMBR, metadataPageManagerFactory);
}
+
+ @Override
+ public JsonNode toJson(IPersistedResourceRegistry registry) throws HyracksDataException {
+ ObjectNode jsonObject = registry.getClassIdentifier(getClass(), serialVersionUID);
+ super.appendToJson(jsonObject, registry);
+ ArrayNode btreeCmpFactoriesArray = OBJECT_MAPPER.createArrayNode();
+ for (IBinaryComparatorFactory factory : btreeCmpFactories) {
+ btreeCmpFactoriesArray.add(factory.toJson(registry));
+ }
+ jsonObject.set("btreeCmpFactories", btreeCmpFactoriesArray);
+ jsonObject.set("linearizeCmpFactory", linearizeCmpFactory.toJson(registry));
+
+ final ArrayNode valueProviderFactoriesArray = OBJECT_MAPPER.createArrayNode();
+ for (IPrimitiveValueProviderFactory factory : valueProviderFactories) {
+ valueProviderFactoriesArray.add(factory.toJson(registry));
+ }
+ jsonObject.set("valueProviderFactories", valueProviderFactoriesArray);
+ jsonObject.set("rtreePolicyType", rtreePolicyType.toJson(registry));
+ jsonObject.putPOJO("rtreeFields", rtreeFields);
+ jsonObject.put("isPointMBR", isPointMBR);
+ return jsonObject;
+ }
+
+ public static IJsonSerializable fromJson(IPersistedResourceRegistry registry, JsonNode json)
+ throws HyracksDataException {
+ final int[] rtreeFields = OBJECT_MAPPER.convertValue(json.get("rtreeFields"), int[].class);
+ final boolean isPointMBR = json.get("isPointMBR").asBoolean();
+ final RTreePolicyType rtreePolicyType = (RTreePolicyType) registry.deserialize(json.get("rtreePolicyType"));
+ final ILinearizeComparatorFactory linearizeCmpFactory =
+ (ILinearizeComparatorFactory) registry.deserialize(json.get("linearizeCmpFactory"));
+
+ final List<IBinaryComparatorFactory> btreeCmpFactoriesList = new ArrayList<>();
+ final ArrayNode jsonBtreeCmpFactories = (ArrayNode) json.get("btreeCmpFactories");
+ for (JsonNode cf : jsonBtreeCmpFactories) {
+ btreeCmpFactoriesList.add((IBinaryComparatorFactory) registry.deserialize(cf));
+ }
+ final IBinaryComparatorFactory[] btreeCmpFactories =
+ btreeCmpFactoriesList.toArray(new IBinaryComparatorFactory[0]);
+ final List<IPrimitiveValueProviderFactory> valueProviderFactoriesList = new ArrayList<>();
+ final ArrayNode jsonValueProviderFactories = (ArrayNode) json.get("valueProviderFactories");
+ for (JsonNode cf : jsonValueProviderFactories) {
+ valueProviderFactoriesList.add((IPrimitiveValueProviderFactory) registry.deserialize(cf));
+ }
+ final IPrimitiveValueProviderFactory[] valueProviderFactories =
+ valueProviderFactoriesList.toArray(new IPrimitiveValueProviderFactory[0]);
+ return new LSMRTreeWithAntiMatterLocalResource(registry, json, btreeCmpFactories, valueProviderFactories,
+ rtreePolicyType, linearizeCmpFactory, rtreeFields, isPointMBR);
+ }
}