[ASTERIXDB-1130][STO]: JSON serialization for persisted objects
- user model changes: no
- storage format changes:
This change replaces the use of Java serialization for persisted objects
such as dataset/index metadata, checkpoints, ect...
This will break backward compatibly with any existing AsterixDB instance.
However, the change is needed to enable future backward compatibility support
for persisted objects.
- interface changes:
IJsonSerializable: contains API to serialize a class as a JsonNode.
IPersistedResourceRegistry: contains a mapping between an IJsonSerializable
class and a unique type id. An IPersistedResourceRegistry is responsible
for generating the class identifier in the JSON output.
The class identifier will always contain the following attributes:
@type: a unique type id that identifies the object type.
@version: the version of the serialized class.
@class: the serialized class full name.
Any registered class with PersistedResourceRegistry must provide
a static fromJson(IPersistedResourceRegistry, JsonNode) method for
deserialization. This is ensured during the class registration process.
Change-Id: I5b103e06eab6627dbfe9d531caae1a3ac4b296da
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2752
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
Integration-Tests: Murtadha Hubail <mhubail@apache.org>
Tested-by: Murtadha Hubail <mhubail@apache.org>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamousaa@gmail.com>
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml
index 5259a7c..f880e66 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/pom.xml
@@ -92,5 +92,9 @@
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
</dependencies>
</project>
\ No newline at end of file
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java
index f2fb9af..5614ca5 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/dataflow/LSMInvertedIndexLocalResource.java
@@ -18,6 +18,7 @@
*/
package org.apache.hyracks.storage.am.lsm.invertedindex.dataflow;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -27,6 +28,8 @@
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.io.IJsonSerializable;
+import org.apache.hyracks.api.io.IPersistedResourceRegistry;
import org.apache.hyracks.storage.am.common.api.IMetadataPageManagerFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
@@ -43,6 +46,10 @@
import org.apache.hyracks.storage.common.IStorageManager;
import org.apache.hyracks.storage.common.buffercache.IBufferCache;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
public class LSMInvertedIndexLocalResource extends LsmResource {
private static final long serialVersionUID = 1L;
@@ -79,6 +86,22 @@
this.bloomFilterFalsePositiveRate = bloomFilterFalsePositiveRate;
}
+ private LSMInvertedIndexLocalResource(IPersistedResourceRegistry registry, JsonNode json,
+ ITypeTraits[] tokenTypeTraits, IBinaryComparatorFactory[] tokenCmpFactories,
+ IBinaryTokenizerFactory tokenizerFactory, boolean isPartitioned, int[] invertedIndexFields,
+ int[] filterFieldsForNonBulkLoadOps, int[] invertedIndexFieldsForNonBulkLoadOps,
+ double bloomFilterFalsePositiveRate) throws HyracksDataException {
+ super(registry, json);
+ this.tokenTypeTraits = tokenTypeTraits;
+ this.tokenCmpFactories = tokenCmpFactories;
+ this.tokenizerFactory = tokenizerFactory;
+ this.isPartitioned = isPartitioned;
+ this.invertedIndexFields = invertedIndexFields;
+ this.filterFieldsForNonBulkLoadOps = filterFieldsForNonBulkLoadOps;
+ this.invertedIndexFieldsForNonBulkLoadOps = invertedIndexFieldsForNonBulkLoadOps;
+ this.bloomFilterFalsePositiveRate = bloomFilterFalsePositiveRate;
+ }
+
@Override
public ILSMIndex createInstance(INCServiceContext serviceCtx) throws HyracksDataException {
IIOManager ioManager = serviceCtx.getIoManager();
@@ -105,4 +128,57 @@
metadataPageManagerFactory, serviceCtx.getTracer());
}
}
+
+ @Override
+ public JsonNode toJson(IPersistedResourceRegistry registry) throws HyracksDataException {
+ ObjectNode jsonObject = registry.getClassIdentifier(getClass(), serialVersionUID);
+ super.appendToJson(jsonObject, registry);
+ final ArrayNode tokenTypeTraitsArray = OBJECT_MAPPER.createArrayNode();
+ for (ITypeTraits tt : tokenTypeTraits) {
+ tokenTypeTraitsArray.add(tt.toJson(registry));
+ }
+ jsonObject.set("tokenTypeTraits", tokenTypeTraitsArray);
+ final ArrayNode tokenCmpFactoriesArray = OBJECT_MAPPER.createArrayNode();
+ for (IBinaryComparatorFactory factory : tokenCmpFactories) {
+ tokenCmpFactoriesArray.add(factory.toJson(registry));
+ }
+ jsonObject.set("tokenCmpFactories", tokenCmpFactoriesArray);
+ jsonObject.set("tokenizerFactory", tokenizerFactory.toJson(registry));
+ jsonObject.put("isPartitioned", isPartitioned);
+ jsonObject.putPOJO("invertedIndexFields", invertedIndexFields);
+ jsonObject.putPOJO("filterFieldsForNonBulkLoadOps", filterFieldsForNonBulkLoadOps);
+ jsonObject.putPOJO("invertedIndexFieldsForNonBulkLoadOps", invertedIndexFieldsForNonBulkLoadOps);
+ jsonObject.putPOJO("bloomFilterFalsePositiveRate", bloomFilterFalsePositiveRate);
+ return jsonObject;
+ }
+
+ public static IJsonSerializable fromJson(IPersistedResourceRegistry registry, JsonNode json)
+ throws HyracksDataException {
+ final List<ITypeTraits> tokenTypeTraitsList = new ArrayList<>();
+ final ArrayNode jsonTokenTypeTraits = (ArrayNode) json.get("tokenTypeTraits");
+ for (JsonNode tt : jsonTokenTypeTraits) {
+ tokenTypeTraitsList.add((ITypeTraits) registry.deserialize(tt));
+ }
+ final ITypeTraits[] tokenTypeTraits = tokenTypeTraitsList.toArray(new ITypeTraits[0]);
+
+ final List<IBinaryComparatorFactory> tokenCmpFactoriesList = new ArrayList<>();
+ final ArrayNode jsontokenCmpFactories = (ArrayNode) json.get("tokenCmpFactories");
+ for (JsonNode cf : jsontokenCmpFactories) {
+ tokenCmpFactoriesList.add((IBinaryComparatorFactory) registry.deserialize(cf));
+ }
+ final IBinaryComparatorFactory[] tokenCmpFactories =
+ tokenCmpFactoriesList.toArray(new IBinaryComparatorFactory[0]);
+ final IBinaryTokenizerFactory tokenizerFactory =
+ (IBinaryTokenizerFactory) registry.deserialize(json.get("tokenizerFactory"));
+ final boolean isPartitioned = json.get("isPartitioned").asBoolean();
+ final int[] invertedIndexFields = OBJECT_MAPPER.convertValue(json.get("invertedIndexFields"), int[].class);
+ final int[] filterFieldsForNonBulkLoadOps =
+ OBJECT_MAPPER.convertValue(json.get("filterFieldsForNonBulkLoadOps"), int[].class);
+ final int[] invertedIndexFieldsForNonBulkLoadOps =
+ OBJECT_MAPPER.convertValue(json.get("invertedIndexFieldsForNonBulkLoadOps"), int[].class);
+ final double bloomFilterFalsePositiveRate = json.get("bloomFilterFalsePositiveRate").asDouble();
+ return new LSMInvertedIndexLocalResource(registry, json, tokenTypeTraits, tokenCmpFactories, tokenizerFactory,
+ isPartitioned, invertedIndexFields, filterFieldsForNonBulkLoadOps, invertedIndexFieldsForNonBulkLoadOps,
+ bloomFilterFalsePositiveRate);
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/DelimitedUTF8StringBinaryTokenizerFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/DelimitedUTF8StringBinaryTokenizerFactory.java
index e583c7d..1a0f387 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/DelimitedUTF8StringBinaryTokenizerFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/DelimitedUTF8StringBinaryTokenizerFactory.java
@@ -19,6 +19,13 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.IJsonSerializable;
+import org.apache.hyracks.api.io.IPersistedResourceRegistry;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
public class DelimitedUTF8StringBinaryTokenizerFactory implements IBinaryTokenizerFactory {
private static final long serialVersionUID = 1L;
@@ -37,4 +44,21 @@
public IBinaryTokenizer createTokenizer() {
return new DelimitedUTF8StringBinaryTokenizer(ignoreTokenCount, sourceHasTypeTag, tokenFactory);
}
+
+ @Override
+ public JsonNode toJson(IPersistedResourceRegistry registry) throws HyracksDataException {
+ final ObjectNode json = registry.getClassIdentifier(getClass(), serialVersionUID);
+ json.set("tokenFactory", tokenFactory.toJson(registry));
+ json.put("ignoreTokenCount", ignoreTokenCount);
+ json.put("sourceHasTypeTag", sourceHasTypeTag);
+ return json;
+ }
+
+ public static IJsonSerializable fromJson(IPersistedResourceRegistry registry, JsonNode json)
+ throws HyracksDataException {
+ final ITokenFactory tokenFactory = (ITokenFactory) registry.deserialize(json.get("tokenFactory"));
+ final boolean ignoreTokenCount = json.get("ignoreTokenCount").asBoolean();
+ final boolean sourceHasTypeTag = json.get("sourceHasTypeTag").asBoolean();
+ return new DelimitedUTF8StringBinaryTokenizerFactory(ignoreTokenCount, sourceHasTypeTag, tokenFactory);
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/HashedUTF8NGramTokenFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/HashedUTF8NGramTokenFactory.java
index 0bc01b6..2daa09d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/HashedUTF8NGramTokenFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/HashedUTF8NGramTokenFactory.java
@@ -19,6 +19,13 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.IJsonSerializable;
+import org.apache.hyracks.api.io.IPersistedResourceRegistry;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
public class HashedUTF8NGramTokenFactory extends AbstractUTF8TokenFactory {
private static final long serialVersionUID = 1L;
@@ -35,4 +42,19 @@
public IToken createToken() {
return new HashedUTF8NGramToken(tokenTypeTag, countTypeTag);
}
+
+ @Override
+ public JsonNode toJson(IPersistedResourceRegistry registry) throws HyracksDataException {
+ final ObjectNode json = registry.getClassIdentifier(getClass(), serialVersionUID);
+ json.put("tokenTypeTag", tokenTypeTag);
+ json.put("countTypeTag", countTypeTag);
+ return json;
+ }
+
+ @SuppressWarnings("squid:S1172") // unused parameter
+ public static IJsonSerializable fromJson(IPersistedResourceRegistry registry, JsonNode json) {
+ final byte tokenTypeTag = (byte) json.get("tokenTypeTag").asInt();
+ final byte countTypeTag = (byte) json.get("countTypeTag").asInt();
+ return new HashedUTF8NGramTokenFactory(tokenTypeTag, countTypeTag);
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/HashedUTF8WordTokenFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/HashedUTF8WordTokenFactory.java
index 7819e16..e8a83d3 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/HashedUTF8WordTokenFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/HashedUTF8WordTokenFactory.java
@@ -19,6 +19,13 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.IJsonSerializable;
+import org.apache.hyracks.api.io.IPersistedResourceRegistry;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
public class HashedUTF8WordTokenFactory extends AbstractUTF8TokenFactory {
private static final long serialVersionUID = 1L;
@@ -35,4 +42,19 @@
public IToken createToken() {
return new HashedUTF8WordToken(tokenTypeTag, countTypeTag);
}
+
+ @Override
+ public JsonNode toJson(IPersistedResourceRegistry registry) throws HyracksDataException {
+ final ObjectNode json = registry.getClassIdentifier(getClass(), serialVersionUID);
+ json.put("tokenTypeTag", tokenTypeTag);
+ json.put("countTypeTag", countTypeTag);
+ return json;
+ }
+
+ @SuppressWarnings("squid:S1172") // unused parameter
+ public static IJsonSerializable fromJson(IPersistedResourceRegistry registry, JsonNode json) {
+ final byte tokenTypeTag = (byte) json.get("tokenTypeTag").asInt();
+ final byte countTypeTag = (byte) json.get("countTypeTag").asInt();
+ return new HashedUTF8WordTokenFactory(tokenTypeTag, countTypeTag);
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/IBinaryTokenizerFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/IBinaryTokenizerFactory.java
index 01e83a3..77c604c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/IBinaryTokenizerFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/IBinaryTokenizerFactory.java
@@ -21,6 +21,8 @@
import java.io.Serializable;
-public interface IBinaryTokenizerFactory extends Serializable {
+import org.apache.hyracks.api.io.IJsonSerializable;
+
+public interface IBinaryTokenizerFactory extends Serializable, IJsonSerializable {
public IBinaryTokenizer createTokenizer();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/ITokenFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/ITokenFactory.java
index 0439be4..21cadbc 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/ITokenFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/ITokenFactory.java
@@ -21,6 +21,8 @@
import java.io.Serializable;
-public interface ITokenFactory extends Serializable {
+import org.apache.hyracks.api.io.IJsonSerializable;
+
+public interface ITokenFactory extends Serializable, IJsonSerializable {
public IToken createToken();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/NGramUTF8StringBinaryTokenizerFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/NGramUTF8StringBinaryTokenizerFactory.java
index a95072d..fbd9f8c 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/NGramUTF8StringBinaryTokenizerFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/NGramUTF8StringBinaryTokenizerFactory.java
@@ -19,6 +19,13 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.IJsonSerializable;
+import org.apache.hyracks.api.io.IPersistedResourceRegistry;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
public class NGramUTF8StringBinaryTokenizerFactory implements IBinaryTokenizerFactory {
private static final long serialVersionUID = 1L;
@@ -43,4 +50,26 @@
tokenFactory);
}
+ @Override
+ public JsonNode toJson(IPersistedResourceRegistry registry) throws HyracksDataException {
+ final ObjectNode json = registry.getClassIdentifier(getClass(), serialVersionUID);
+ json.set("tokenFactory", tokenFactory.toJson(registry));
+ json.put("gramLength", gramLength);
+ json.put("usePrePost", usePrePost);
+ json.put("ignoreTokenCount", ignoreTokenCount);
+ json.put("sourceHasTypeTag", sourceHasTypeTag);
+ return json;
+ }
+
+ public static IJsonSerializable fromJson(IPersistedResourceRegistry registry, JsonNode json)
+ throws HyracksDataException {
+ final ITokenFactory tokenFactory = (ITokenFactory) registry.deserialize(json.get("tokenFactory"));
+ final int gramLength = json.get("gramLength").asInt();
+ final boolean usePrePost = json.get("usePrePost").asBoolean();
+ final boolean ignoreTokenCount = json.get("ignoreTokenCount").asBoolean();
+ final boolean sourceHasTypeTag = json.get("sourceHasTypeTag").asBoolean();
+ return new NGramUTF8StringBinaryTokenizerFactory(gramLength, usePrePost, ignoreTokenCount, sourceHasTypeTag,
+ tokenFactory);
+ }
+
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/UTF8NGramTokenFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/UTF8NGramTokenFactory.java
index 64f1e99..aa4d30f 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/UTF8NGramTokenFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/UTF8NGramTokenFactory.java
@@ -19,6 +19,13 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.IJsonSerializable;
+import org.apache.hyracks.api.io.IPersistedResourceRegistry;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
public class UTF8NGramTokenFactory extends AbstractUTF8TokenFactory {
private static final long serialVersionUID = 1L;
@@ -36,4 +43,18 @@
return new UTF8NGramToken(tokenTypeTag, countTypeTag);
}
+ @Override
+ public JsonNode toJson(IPersistedResourceRegistry registry) throws HyracksDataException {
+ final ObjectNode json = registry.getClassIdentifier(getClass(), serialVersionUID);
+ json.put("tokenTypeTag", tokenTypeTag);
+ json.put("countTypeTag", countTypeTag);
+ return json;
+ }
+
+ @SuppressWarnings("squid:S1172") // unused parameter
+ public static IJsonSerializable fromJson(IPersistedResourceRegistry registry, JsonNode json) {
+ final byte tokenTypeTag = (byte) json.get("tokenTypeTag").asInt();
+ final byte countTypeTag = (byte) json.get("countTypeTag").asInt();
+ return new UTF8NGramTokenFactory(tokenTypeTag, countTypeTag);
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/UTF8WordTokenFactory.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/UTF8WordTokenFactory.java
index 3a8a0483..d50e83b 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/UTF8WordTokenFactory.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/tokenizers/UTF8WordTokenFactory.java
@@ -19,6 +19,13 @@
package org.apache.hyracks.storage.am.lsm.invertedindex.tokenizers;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.IJsonSerializable;
+import org.apache.hyracks.api.io.IPersistedResourceRegistry;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
public class UTF8WordTokenFactory extends AbstractUTF8TokenFactory {
private static final long serialVersionUID = 1L;
@@ -36,4 +43,19 @@
return new UTF8WordToken(tokenTypeTag, countTypeTag);
}
+ @Override
+ public JsonNode toJson(IPersistedResourceRegistry registry) throws HyracksDataException {
+ final ObjectNode json = registry.getClassIdentifier(getClass(), serialVersionUID);
+ json.put("tokenTypeTag", tokenTypeTag);
+ json.put("countTypeTag", countTypeTag);
+ return json;
+ }
+
+ @SuppressWarnings("squid:S1172") // unused parameter
+ public static IJsonSerializable fromJson(IPersistedResourceRegistry registry, JsonNode json) {
+ final byte tokenTypeTag = (byte) json.get("tokenTypeTag").asInt();
+ final byte countTypeTag = (byte) json.get("countTypeTag").asInt();
+ return new UTF8NGramTokenFactory(tokenTypeTag, countTypeTag);
+ }
+
}