Add the Binary data type and corresponding helper functions to Asterix.
The binary data type is implemented as a bytearray. Its storage format
follows the String type which has 2 bytes for length and then store the
bytes contents.
Binary data will take hex("") or base64("") as the constructor method to
passing a hex string or base64 string into Asterix. For output we use
hex("") format.
The parse-[hex|base64](string) function will parse the corresponding hex
or base64 string to binary type. The print-[hex|base64](binary)
functions will print the binary to hex or base64 STRING format.
The sub-binary(binary, offset, [length]) function works the same as
substring(string, offset, [length])
The find-binary(srcbinary, targetbinary, [start-offset]) will find the
position of the targetbinary in the srcbinary.
Change-Id: I5ecf0cc115c44070fb5c1fc5b0ec12a95d4243a4
Reviewed-on: http://fulliautomatix.ics.uci.edu:8443/175
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Yingyi Bu <buyingyi@gmail.com>
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/AbstractAsterixListIterator.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/AbstractAsterixListIterator.java
index 56e5139..e95e9cb 100644
--- a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/AbstractAsterixListIterator.java
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/common/AbstractAsterixListIterator.java
@@ -124,6 +124,10 @@
}
break;
}
+ case BINARY: {
+ cmp = AqlBinaryComparatorFactoryProvider.BINARY_POINTABLE_INSTANCE.createBinaryComparator();
+ break;
+ }
default: {
cmp = null;
break;
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/comparisons/AbstractComparisonEvaluator.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/comparisons/AbstractComparisonEvaluator.java
index e5a26fc..ee1d98a 100644
--- a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/comparisons/AbstractComparisonEvaluator.java
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/comparisons/AbstractComparisonEvaluator.java
@@ -42,6 +42,8 @@
import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
import edu.uci.ics.hyracks.api.dataflow.value.IBinaryComparator;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.data.std.accessors.PointableBinaryComparatorFactory;
+import edu.uci.ics.hyracks.data.std.primitive.ByteArrayPointable;
import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
import edu.uci.ics.hyracks.dataflow.common.data.marshalling.FloatSerializerDeserializer;
@@ -54,7 +56,9 @@
EQUAL,
GREATER_THAN,
UNKNOWN
- };
+ }
+
+ ;
protected DataOutput out;
protected ArrayBackedValueStorage outLeft = new ArrayBackedValueStorage();
@@ -85,6 +89,9 @@
.createBinaryComparator();
protected IBinaryComparator rectangleBinaryComparator = ARectanglePartialBinaryComparatorFactory.INSTANCE
.createBinaryComparator();
+ protected final IBinaryComparator byteArrayComparator = new PointableBinaryComparatorFactory(
+ ByteArrayPointable.FACTORY)
+ .createBinaryComparator();
public AbstractComparisonEvaluator(DataOutput out, ICopyEvaluatorFactory evalLeftFactory,
ICopyEvaluatorFactory evalRightFactory) throws AlgebricksException {
@@ -142,8 +149,9 @@
}
}
- if (isLeftNull || isRightNull)
+ if (isLeftNull || isRightNull) {
return ComparisonResult.UNKNOWN;
+ }
switch (typeTag1) {
case INT8: {
@@ -228,15 +236,20 @@
result = rectangleBinaryComparator.compare(outLeft.getByteArray(), 1, outLeft.getLength() - 1,
outRight.getByteArray(), 1, outRight.getLength() - 1);
break;
+ case BINARY:
+ result = byteArrayComparator.compare(outLeft.getByteArray(), 1, outLeft.getLength() - 1,
+ outRight.getByteArray(), 1, outRight.getLength() - 1);
+ break;
default:
throw new AlgebricksException("Comparison for " + actualTypeTag + " is not supported.");
}
- if (result == 0)
+ if (result == 0) {
return ComparisonResult.EQUAL;
- else if (result < 0)
+ } else if (result < 0) {
return ComparisonResult.LESS_THAN;
- else
+ } else {
return ComparisonResult.GREATER_THAN;
+ }
}
private ComparisonResult compareBooleanWithArg(ATypeTag typeTag2) throws AlgebricksException {
@@ -252,12 +265,13 @@
if (typeTag2 == ATypeTag.STRING) {
int result = strBinaryComp.compare(outLeft.getByteArray(), 1, outLeft.getLength() - 1,
outRight.getByteArray(), 1, outRight.getLength() - 1);
- if (result == 0)
+ if (result == 0) {
return ComparisonResult.EQUAL;
- else if (result < 0)
+ } else if (result < 0) {
return ComparisonResult.LESS_THAN;
- else
+ } else {
return ComparisonResult.GREATER_THAN;
+ }
}
throw new AlgebricksException("Comparison is undefined between types AString and " + typeTag2 + " .");
}
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/constructors/ABinaryBase64StringConstructorDescriptor.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/constructors/ABinaryBase64StringConstructorDescriptor.java
new file mode 100644
index 0000000..e492c82
--- /dev/null
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/constructors/ABinaryBase64StringConstructorDescriptor.java
@@ -0,0 +1,35 @@
+package edu.uci.ics.asterix.runtime.evaluators.constructors;
+
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.asterix.om.functions.IFunctionDescriptor;
+import edu.uci.ics.asterix.om.functions.IFunctionDescriptorFactory;
+import edu.uci.ics.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.ByteArrayBase64ParserFactory;
+
+public class ABinaryBase64StringConstructorDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override public IFunctionDescriptor createFunctionDescriptor() {
+ return new ABinaryBase64StringConstructorDescriptor();
+ }
+ };
+
+ @Override public ICopyEvaluatorFactory createEvaluatorFactory(final ICopyEvaluatorFactory[] args)
+ throws AlgebricksException {
+ return new ICopyEvaluatorFactory() {
+ @Override public ICopyEvaluator createEvaluator(IDataOutputProvider output) throws AlgebricksException {
+ return new ABinaryHexStringConstructorDescriptor.ABinaryConstructorEvaluator(output, args[0],
+ ByteArrayBase64ParserFactory.INSTANCE);
+ }
+ };
+ }
+
+ @Override public FunctionIdentifier getIdentifier() {
+ return AsterixBuiltinFunctions.BINARY_BASE64_CONSTRUCTOR;
+ }
+}
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/constructors/ABinaryHexStringConstructorDescriptor.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/constructors/ABinaryHexStringConstructorDescriptor.java
new file mode 100644
index 0000000..bcf2715
--- /dev/null
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/constructors/ABinaryHexStringConstructorDescriptor.java
@@ -0,0 +1,110 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.runtime.evaluators.constructors;
+
+import edu.uci.ics.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
+import edu.uci.ics.asterix.om.base.ANull;
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.asterix.om.functions.IFunctionDescriptor;
+import edu.uci.ics.asterix.om.functions.IFunctionDescriptorFactory;
+import edu.uci.ics.asterix.om.types.ATypeTag;
+import edu.uci.ics.asterix.om.types.BuiltinType;
+import edu.uci.ics.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
+import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.ByteArrayHexParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParser;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.IValueParserFactory;
+
+import java.io.DataOutput;
+import java.io.IOException;
+
+public class ABinaryHexStringConstructorDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+ private static final long serialVersionUID = 1L;
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new ABinaryHexStringConstructorDescriptor();
+ }
+ };
+
+ @Override public ICopyEvaluatorFactory createEvaluatorFactory(final ICopyEvaluatorFactory[] args)
+ throws AlgebricksException {
+ return new ICopyEvaluatorFactory() {
+ @Override public ICopyEvaluator createEvaluator(final IDataOutputProvider output)
+ throws AlgebricksException {
+ return new ABinaryConstructorEvaluator(output, args[0], ByteArrayHexParserFactory.INSTANCE);
+ }
+ };
+ }
+
+ @Override public FunctionIdentifier getIdentifier() {
+ return AsterixBuiltinFunctions.BINARY_HEX_CONSTRUCTOR;
+ }
+
+ static class ABinaryConstructorEvaluator implements ICopyEvaluator {
+ private DataOutput out;
+ private ArrayBackedValueStorage outInput;
+ private ICopyEvaluator eval;
+ private IValueParser byteArrayParser;
+
+ @SuppressWarnings("unchecked")
+ private ISerializerDeserializer<ANull> nullSerde = AqlSerializerDeserializerProvider.INSTANCE
+ .getSerializerDeserializer(BuiltinType.ANULL);
+
+ public ABinaryConstructorEvaluator(final IDataOutputProvider output, ICopyEvaluatorFactory copyEvaluatorFactory,
+ IValueParserFactory valueParserFactory)
+ throws AlgebricksException {
+ out = output.getDataOutput();
+ outInput = new ArrayBackedValueStorage();
+ eval = copyEvaluatorFactory.createEvaluator(outInput);
+ byteArrayParser = valueParserFactory.createValueParser();
+ }
+
+ @Override public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
+
+ try {
+ outInput.reset();
+ eval.evaluate(tuple);
+ byte[] binary = outInput.getByteArray();
+
+ ATypeTag tt = ATypeTag.VALUE_TYPE_MAPPING[binary[0]];
+ if (tt == ATypeTag.NULL) {
+ nullSerde.serialize(ANull.NULL, out);
+ } else if (tt == ATypeTag.BINARY) {
+ out.write(outInput.getByteArray(), outInput.getStartOffset(), outInput.getLength());
+ } else if (tt == ATypeTag.STRING) {
+ String string = new String(outInput.getByteArray(), 3, outInput.getLength() - 3,
+ "UTF-8");
+ char[] buffer = string.toCharArray();
+ out.write(ATypeTag.BINARY.serialize());
+ byteArrayParser.parse(buffer, 0, buffer.length, out);
+ } else {
+ throw new AlgebricksException("binary type of " + tt + "haven't implemented yet.");
+ }
+ } catch (IOException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+ }
+
+ ;
+}
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/AbstractCopyEvaluator.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/AbstractCopyEvaluator.java
new file mode 100644
index 0000000..09dc82d
--- /dev/null
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/AbstractCopyEvaluator.java
@@ -0,0 +1,101 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.runtime.evaluators.functions.binary;
+
+import edu.uci.ics.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
+import edu.uci.ics.asterix.om.base.ANull;
+import edu.uci.ics.asterix.om.types.ATypeTag;
+import edu.uci.ics.asterix.om.types.BuiltinType;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
+import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+import java.io.DataOutput;
+
+public abstract class AbstractCopyEvaluator implements ICopyEvaluator {
+ @SuppressWarnings("unchecked")
+ protected ISerializerDeserializer<ANull> nullSerde = AqlSerializerDeserializerProvider.INSTANCE
+ .getSerializerDeserializer(BuiltinType.ANULL);
+
+ protected DataOutput dataOutput;
+ protected ArrayBackedValueStorage[] storages;
+ protected ICopyEvaluator[] evaluators;
+
+ public AbstractCopyEvaluator(final IDataOutputProvider output, final ICopyEvaluatorFactory[] copyEvaluatorFactories)
+ throws AlgebricksException {
+ dataOutput = output.getDataOutput();
+ storages = new ArrayBackedValueStorage[copyEvaluatorFactories.length];
+ evaluators = new ICopyEvaluator[copyEvaluatorFactories.length];
+ for (int i = 0; i < evaluators.length; ++i) {
+ storages[i] = new ArrayBackedValueStorage();
+ evaluators[i] = copyEvaluatorFactories[i].createEvaluator(storages[i]);
+ }
+ }
+
+ public ATypeTag evaluateTuple(IFrameTupleReference tuple, int id) throws AlgebricksException {
+ storages[id].reset();
+ evaluators[id].evaluate(tuple);
+ return ATypeTag.VALUE_TYPE_MAPPING[storages[id].getByteArray()[0]];
+ }
+
+ public boolean serializeNullIfAnyNull(ATypeTag... tags) throws HyracksDataException {
+ for (ATypeTag typeTag : tags) {
+ if (typeTag == ATypeTag.NULL) {
+ nullSerde.serialize(ANull.NULL, dataOutput);
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static final String FIRST = "1st";
+ private static final String SECOND = "2nd";
+ private static final String THIRD = "3rd";
+ private static final String TH = "th";
+
+ public static String idToString(int i) {
+ String prefix = "";
+ if (i >= 10) {
+ prefix = String.valueOf(i / 10);
+ }
+ switch (i % 10) {
+ case 1:
+ return prefix + FIRST;
+ case 2:
+ return prefix + SECOND;
+ case 3:
+ return prefix + THIRD;
+ default:
+ return String.valueOf(i) + TH;
+ }
+ }
+
+ public static void checkTypeMachingThrowsIfNot(String title, ATypeTag[] expected, ATypeTag... actual)
+ throws AlgebricksException {
+ for (int i = 0; i < expected.length; i++) {
+ if (expected[i] != actual[i]) {
+ throw new AlgebricksException(title +
+ ": expects " + expected[i] + " at " + idToString(i + 1) + " argument, but got " + actual[i]);
+ }
+ }
+ }
+
+}
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/BinaryConcatDescriptor.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/BinaryConcatDescriptor.java
new file mode 100644
index 0000000..ab1d6e1
--- /dev/null
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/BinaryConcatDescriptor.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.runtime.evaluators.functions.binary;
+
+import edu.uci.ics.asterix.common.exceptions.AsterixException;
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.asterix.om.functions.IFunctionDescriptor;
+import edu.uci.ics.asterix.om.functions.IFunctionDescriptorFactory;
+import edu.uci.ics.asterix.om.types.ATypeTag;
+import edu.uci.ics.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import edu.uci.ics.asterix.runtime.evaluators.common.AsterixListAccessor;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
+import edu.uci.ics.hyracks.data.std.primitive.ByteArrayPointable;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+import java.io.IOException;
+
+public class BinaryConcatDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override public IFunctionDescriptor createFunctionDescriptor() {
+ return new BinaryConcatDescriptor();
+ }
+ };
+
+ @Override public FunctionIdentifier getIdentifier() {
+ return AsterixBuiltinFunctions.BINARY_CONCAT;
+ }
+
+ @Override public ICopyEvaluatorFactory createEvaluatorFactory(final ICopyEvaluatorFactory[] args)
+ throws AlgebricksException {
+ return new ICopyEvaluatorFactory() {
+ @Override public ICopyEvaluator createEvaluator(final IDataOutputProvider output)
+ throws AlgebricksException {
+ return new AbstractCopyEvaluator(output, args) {
+
+ private final AsterixListAccessor listAccessor = new AsterixListAccessor();
+ private final byte SER_BINARY_TYPE = ATypeTag.BINARY.serialize();
+
+ @Override public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
+ ATypeTag typeTag = evaluateTuple(tuple, 0);
+ if (typeTag != ATypeTag.UNORDEREDLIST && typeTag != ATypeTag.ORDEREDLIST) {
+ throw new AlgebricksException(getIdentifier().getName()
+ + ": expects input type ORDEREDLIST/UNORDEREDLIST, but got "
+ + typeTag);
+ }
+ try {
+ listAccessor.reset(storages[0].getByteArray(), 0);
+
+ int concatLength = 0;
+ for (int i = 0; i < listAccessor.size(); i++) {
+ int itemOffset = listAccessor.getItemOffset(i);
+ ATypeTag itemType = listAccessor.getItemType(itemOffset);
+ if (itemType != ATypeTag.BINARY) {
+ if (serializeNullIfAnyNull(itemType)) {
+ return;
+ }
+ throw new AlgebricksException(getIdentifier().getName()
+ + ": expects type STRING/NULL for the list item but got " + itemType);
+ }
+ concatLength += ByteArrayPointable.getLength(storages[0].getByteArray(), itemOffset);
+ }
+ if (concatLength > ByteArrayPointable.MAX_LENGTH) {
+ throw new AlgebricksException("the concatenated binary is too long.");
+ }
+ dataOutput.writeByte(SER_BINARY_TYPE);
+ dataOutput.writeShort(concatLength);
+
+ for (int i = 0; i < listAccessor.size(); i++) {
+ int itemOffset = listAccessor.getItemOffset(i);
+ int length = ByteArrayPointable.getLength(storages[0].getByteArray(), itemOffset);
+ dataOutput.write(storages[0].getByteArray(),
+ itemOffset + ByteArrayPointable.SIZE_OF_LENGTH, length);
+ }
+ } catch (HyracksDataException e) {
+ throw new AlgebricksException(e);
+ } catch (IOException e) {
+ throw new AlgebricksException(e);
+ } catch (AsterixException e) {
+ throw new AlgebricksException(e);
+ }
+
+ }
+ };
+
+ }
+ };
+
+ }
+}
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/BinaryLengthDescriptor.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/BinaryLengthDescriptor.java
new file mode 100644
index 0000000..1d4717e
--- /dev/null
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/BinaryLengthDescriptor.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.runtime.evaluators.functions.binary;
+
+import edu.uci.ics.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
+import edu.uci.ics.asterix.om.base.AInt32;
+import edu.uci.ics.asterix.om.base.AMutableInt32;
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.asterix.om.functions.IFunctionDescriptor;
+import edu.uci.ics.asterix.om.functions.IFunctionDescriptorFactory;
+import edu.uci.ics.asterix.om.types.ATypeTag;
+import edu.uci.ics.asterix.om.types.BuiltinType;
+import edu.uci.ics.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
+import edu.uci.ics.hyracks.data.std.primitive.ByteArrayPointable;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class BinaryLengthDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+ private static final long serialVersionUID = 1L;
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new BinaryLengthDescriptor();
+ }
+ };
+
+ private static final ATypeTag[] EXPECTED_TAGS = { ATypeTag.BINARY };
+
+ @Override public ICopyEvaluatorFactory createEvaluatorFactory(final ICopyEvaluatorFactory[] args)
+ throws AlgebricksException {
+ return new ICopyEvaluatorFactory() {
+ @Override public ICopyEvaluator createEvaluator(final IDataOutputProvider output)
+ throws AlgebricksException {
+ return new AbstractCopyEvaluator(output, args) {
+
+ private AMutableInt32 result = new AMutableInt32(0);
+ @SuppressWarnings("unchecked")
+ private ISerializerDeserializer<AInt32> intSerde = AqlSerializerDeserializerProvider.INSTANCE
+ .getSerializerDeserializer(BuiltinType.AINT32);
+
+ @Override public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
+ ATypeTag tag = evaluateTuple(tuple, 0);
+ try {
+ if (serializeNullIfAnyNull(tag)) {
+ return;
+ }
+ checkTypeMachingThrowsIfNot(getIdentifier().getName(), EXPECTED_TAGS, tag);
+ int len = ByteArrayPointable.getLength(storages[0].getByteArray(), 1);
+ result.setValue(len);
+ intSerde.serialize(result, dataOutput);
+ } catch (HyracksDataException e) {
+ throw new AlgebricksException(e);
+ }
+
+ }
+ };
+ }
+ };
+ }
+
+ @Override public FunctionIdentifier getIdentifier() {
+ return AsterixBuiltinFunctions.BINARY_LENGTH;
+ }
+}
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/FindBinaryDescriptor.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/FindBinaryDescriptor.java
new file mode 100644
index 0000000..4ba91d9
--- /dev/null
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/FindBinaryDescriptor.java
@@ -0,0 +1,147 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.runtime.evaluators.functions.binary;
+
+import edu.uci.ics.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
+import edu.uci.ics.asterix.om.base.AInt32;
+import edu.uci.ics.asterix.om.base.AMutableInt32;
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.asterix.om.functions.IFunctionDescriptor;
+import edu.uci.ics.asterix.om.functions.IFunctionDescriptorFactory;
+import edu.uci.ics.asterix.om.types.ATypeTag;
+import edu.uci.ics.asterix.om.types.BuiltinType;
+import edu.uci.ics.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
+import edu.uci.ics.hyracks.data.std.primitive.ByteArrayPointable;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class FindBinaryDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override public IFunctionDescriptor createFunctionDescriptor() {
+ return new FindBinaryDescriptor();
+ }
+ };
+
+ @Override public FunctionIdentifier getIdentifier() {
+ return AsterixBuiltinFunctions.FIND_BINARY;
+ }
+
+ private static final ATypeTag[] EXPECTED_INPUT_TAG = { ATypeTag.BINARY, ATypeTag.BINARY };
+
+ @Override public ICopyEvaluatorFactory createEvaluatorFactory(final ICopyEvaluatorFactory[] args)
+ throws AlgebricksException {
+ return new ICopyEvaluatorFactory() {
+ @Override public ICopyEvaluator createEvaluator(final IDataOutputProvider output)
+ throws AlgebricksException {
+ return new AbstractFindBinaryCopyEvaluator(output, args, getIdentifier().getName()) {
+ @Override protected int getFromOffset(IFrameTupleReference tuple) throws AlgebricksException {
+ return 0;
+ }
+ };
+ }
+ };
+ }
+
+ static abstract class AbstractFindBinaryCopyEvaluator extends AbstractCopyEvaluator {
+
+ public AbstractFindBinaryCopyEvaluator(IDataOutputProvider output,
+ ICopyEvaluatorFactory[] copyEvaluatorFactories, String functionName) throws AlgebricksException {
+ super(output, copyEvaluatorFactories);
+ this.functionName = functionName;
+ }
+
+ protected String functionName;
+ protected AMutableInt32 result = new AMutableInt32(-1);
+
+ @SuppressWarnings("unchecked")
+ protected ISerializerDeserializer<AInt32> intSerde = AqlSerializerDeserializerProvider.INSTANCE
+ .getSerializerDeserializer(BuiltinType.AINT32);
+
+ @Override public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
+ ATypeTag textTag = evaluateTuple(tuple, 0);
+ ATypeTag wordTag = evaluateTuple(tuple, 1);
+ int fromOffset = getFromOffset(tuple);
+
+ try {
+ if (serializeNullIfAnyNull(textTag, wordTag)) {
+ return;
+ }
+ checkTypeMachingThrowsIfNot(functionName, EXPECTED_INPUT_TAG, textTag,
+ wordTag);
+
+ byte[] textBytes = storages[0].getByteArray();
+ byte[] wordBytes = storages[1].getByteArray();
+ int textLength = ByteArrayPointable.getLength(textBytes, 1);
+ int wordLength = ByteArrayPointable.getLength(wordBytes, 1);
+ result.setValue(1 + indexOf(textBytes, 3, textLength, wordBytes, 3, wordLength, fromOffset));
+ intSerde.serialize(result, dataOutput);
+ } catch (HyracksDataException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+
+ protected abstract int getFromOffset(IFrameTupleReference tuple) throws AlgebricksException;
+ }
+
+ // copy from String.indexOf(String)
+ static int indexOf(byte[] source, int sourceOffset, int sourceCount,
+ byte[] target, int targetOffset, int targetCount,
+ int fromIndex) {
+ if (fromIndex >= sourceCount) {
+ return (targetCount == 0 ? sourceCount : -1);
+ }
+ if (fromIndex < 0) {
+ fromIndex = 0;
+ }
+ if (targetCount == 0) {
+ return fromIndex;
+ }
+
+ byte first = target[targetOffset];
+ int max = sourceOffset + (sourceCount - targetCount);
+
+ for (int i = sourceOffset + fromIndex; i <= max; i++) {
+ /* Look for first character. */
+ if (source[i] != first) {
+ while (++i <= max && source[i] != first)
+ ;
+ }
+
+ /* Found first character, now look at the rest of v2 */
+ if (i <= max) {
+ int j = i + 1;
+ int end = j + targetCount - 1;
+ for (int k = targetOffset + 1; j < end && source[j]
+ == target[k]; j++, k++)
+ ;
+
+ if (j == end) {
+ /* Found whole string. */
+ return i - sourceOffset;
+ }
+ }
+ }
+ return -1;
+ }
+
+}
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/FindBinaryFromDescriptor.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/FindBinaryFromDescriptor.java
new file mode 100644
index 0000000..4d0598b
--- /dev/null
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/FindBinaryFromDescriptor.java
@@ -0,0 +1,60 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.runtime.evaluators.functions.binary;
+
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.asterix.om.functions.IFunctionDescriptor;
+import edu.uci.ics.asterix.om.functions.IFunctionDescriptorFactory;
+import edu.uci.ics.asterix.om.types.ATypeTag;
+import edu.uci.ics.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+
+public class FindBinaryFromDescriptor extends AbstractScalarFunctionDynamicDescriptor{
+
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override public IFunctionDescriptor createFunctionDescriptor() {
+ return new FindBinaryFromDescriptor();
+ }
+ };
+
+ @Override public FunctionIdentifier getIdentifier() {
+ return AsterixBuiltinFunctions.FIND_BINARY_FROM;
+ }
+
+ @Override public ICopyEvaluatorFactory createEvaluatorFactory(final ICopyEvaluatorFactory[] args)
+ throws AlgebricksException {
+ return new ICopyEvaluatorFactory() {
+ @Override public ICopyEvaluator createEvaluator(final IDataOutputProvider output) throws AlgebricksException {
+ return new FindBinaryDescriptor.AbstractFindBinaryCopyEvaluator(output, args, getIdentifier().getName()) {
+ @Override protected int getFromOffset(IFrameTupleReference tuple) throws AlgebricksException {
+ ATypeTag offsetTag = evaluateTuple(tuple, 2);
+ if (offsetTag != ATypeTag.INT32){
+ throw new AlgebricksException(functionName + ":expects INT32 at 3rd arguments, but got " + offsetTag);
+ }
+ return IntegerSerializerDeserializer.getInt(storages[2].getByteArray(), 1);
+ }
+ };
+ }
+ };
+ }
+
+}
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/ParseBinaryDescriptor.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/ParseBinaryDescriptor.java
new file mode 100644
index 0000000..35a7fec
--- /dev/null
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/ParseBinaryDescriptor.java
@@ -0,0 +1,124 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.runtime.evaluators.functions.binary;
+
+import edu.uci.ics.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
+import edu.uci.ics.asterix.om.base.ABinary;
+import edu.uci.ics.asterix.om.base.AMutableBinary;
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.asterix.om.functions.IFunctionDescriptor;
+import edu.uci.ics.asterix.om.functions.IFunctionDescriptorFactory;
+import edu.uci.ics.asterix.om.types.ATypeTag;
+import edu.uci.ics.asterix.om.types.BuiltinType;
+import edu.uci.ics.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.ByteArrayBase64ParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.ByteArrayHexParserFactory;
+
+public class ParseBinaryDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ static final String HEX_FORMAT = "hex";
+ static final String BASE64_FORMAT = "base64";
+
+ static boolean isCaseIgnoreEqual(String expectedLowerCaseString, byte[] bytes, int start,
+ int length) {
+ int pos = start;
+ int index = 0;
+ while (pos < start + length) {
+ char c1 = UTF8StringPointable.charAt(bytes, pos);
+ c1 = Character.toLowerCase(c1);
+ if (expectedLowerCaseString.charAt(index++) != c1) {
+ return false;
+ }
+ pos += UTF8StringPointable.charSize(bytes, pos);
+ }
+ return index == expectedLowerCaseString.length();
+ }
+
+ public final static IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new ParseBinaryDescriptor();
+ }
+ };
+
+ public static final ATypeTag[] EXPECTED_INPUT_TAGS = { ATypeTag.STRING, ATypeTag.STRING };
+
+ @Override public FunctionIdentifier getIdentifier() {
+ return AsterixBuiltinFunctions.PARSE_BINARY;
+ }
+
+ @Override public ICopyEvaluatorFactory createEvaluatorFactory(final ICopyEvaluatorFactory[] args)
+ throws AlgebricksException {
+ return new ICopyEvaluatorFactory() {
+ @Override public ICopyEvaluator createEvaluator(final IDataOutputProvider output)
+ throws AlgebricksException {
+ return new AbstractCopyEvaluator(output, args) {
+
+ @SuppressWarnings("unchecked")
+ private ISerializerDeserializer<ABinary> binarySerde = AqlSerializerDeserializerProvider.INSTANCE
+ .getSerializerDeserializer(BuiltinType.ABINARY);
+
+ private AMutableBinary aBinary = new AMutableBinary(new byte[0], 0, 0);
+ private final byte[] quadruplet = new byte[4];
+
+ @Override public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
+ ATypeTag binaryTag = evaluateTuple(tuple, 0);
+ ATypeTag formatTag = evaluateTuple(tuple, 1);
+
+ try {
+ if (serializeNullIfAnyNull(binaryTag, formatTag)) {
+ return;
+ }
+ checkTypeMachingThrowsIfNot(getIdentifier().getName(), EXPECTED_INPUT_TAGS, binaryTag,
+ formatTag);
+ int lengthString = UTF8StringPointable.getUTFLength(storages[0].getByteArray(), 1);
+ int lengthFormat = UTF8StringPointable.getUTFLength(storages[1].getByteArray(), 1);
+ byte[] buffer;
+ if (isCaseIgnoreEqual(HEX_FORMAT, storages[1].getByteArray(), 3, lengthFormat)) {
+ buffer = ByteArrayHexParserFactory
+ .extractPointableArrayFromHexString(storages[0].getByteArray(), 3,
+ lengthString, aBinary.getBytes());
+ } else if (isCaseIgnoreEqual(BASE64_FORMAT, storages[1].getByteArray(), 3, lengthFormat)) {
+ buffer = ByteArrayBase64ParserFactory
+ .extractPointableArrayFromBase64String(storages[0].getByteArray(), 3
+ , lengthString, aBinary.getBytes(), quadruplet);
+ } else {
+ throw new AlgebricksException(getIdentifier().getName()
+ + ": expects format indicator of \"hex\" or \"base64\" in the 2nd argument");
+ }
+ aBinary.setValue(buffer, 0, buffer.length);
+ binarySerde.serialize(aBinary, dataOutput);
+ } catch (HyracksDataException e) {
+ e.printStackTrace();
+ }
+ }
+ };
+
+ }
+
+ };
+ }
+
+}
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/PrintBinaryDescriptor.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/PrintBinaryDescriptor.java
new file mode 100644
index 0000000..dd29782
--- /dev/null
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/PrintBinaryDescriptor.java
@@ -0,0 +1,106 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.runtime.evaluators.functions.binary;
+
+import edu.uci.ics.asterix.dataflow.data.nontagged.printers.ABinaryBase64Printer;
+import edu.uci.ics.asterix.dataflow.data.nontagged.printers.ABinaryHexPrinter;
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.asterix.om.functions.IFunctionDescriptor;
+import edu.uci.ics.asterix.om.functions.IFunctionDescriptorFactory;
+import edu.uci.ics.asterix.om.types.ATypeTag;
+import edu.uci.ics.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
+import edu.uci.ics.hyracks.data.std.primitive.ByteArrayPointable;
+import edu.uci.ics.hyracks.data.std.primitive.UTF8StringPointable;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+import java.io.IOException;
+
+import static edu.uci.ics.asterix.runtime.evaluators.functions.binary.ParseBinaryDescriptor.*;
+
+public class PrintBinaryDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ private static final byte SER_STRING_BYTE = ATypeTag.STRING.serialize();
+
+ @Override public FunctionIdentifier getIdentifier() {
+ return AsterixBuiltinFunctions.PRINT_BINARY;
+ }
+
+ public final static IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override
+ public IFunctionDescriptor createFunctionDescriptor() {
+ return new PrintBinaryDescriptor();
+ }
+ };
+
+ public final static ATypeTag[] EXPECTED_INPUT_TAGS = { ATypeTag.BINARY, ATypeTag.STRING };
+
+ @Override public ICopyEvaluatorFactory createEvaluatorFactory(final ICopyEvaluatorFactory[] args)
+ throws AlgebricksException {
+ return new ICopyEvaluatorFactory() {
+ @Override public ICopyEvaluator createEvaluator(final IDataOutputProvider output)
+ throws AlgebricksException {
+ return new AbstractCopyEvaluator(output, args) {
+
+ private StringBuilder stringBuilder = new StringBuilder();
+
+ @Override public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
+ ATypeTag arg0Tag = evaluateTuple(tuple, 0);
+ ATypeTag arg1Tag = evaluateTuple(tuple, 1);
+
+ try {
+ if (serializeNullIfAnyNull(arg0Tag, arg1Tag)) {
+ return;
+ }
+ checkTypeMachingThrowsIfNot(getIdentifier().getName(), EXPECTED_INPUT_TAGS, arg0Tag,
+ arg1Tag);
+
+ byte[] binaryBytes = storages[0].getByteArray();
+ byte[] formatBytes = storages[1].getByteArray();
+
+ int lengthBinary = ByteArrayPointable.getLength(binaryBytes, 1);
+ int lengthFormat = UTF8StringPointable.getUTFLength(formatBytes, 1);
+ stringBuilder.setLength(0);
+ if (isCaseIgnoreEqual(HEX_FORMAT, formatBytes, 3, lengthFormat)) {
+ ABinaryHexPrinter
+ .printHexString(binaryBytes, 3, lengthBinary, stringBuilder);
+
+ } else if (isCaseIgnoreEqual(BASE64_FORMAT, formatBytes, 3, lengthFormat)) {
+ ABinaryBase64Printer
+ .printBase64Binary(binaryBytes, 3, lengthBinary, stringBuilder);
+ } else {
+ throw new AlgebricksException(getIdentifier().getName()
+ + ": expects format indicator of \"hex\" or \"base64\" in the 2nd argument");
+ }
+ dataOutput.writeByte(SER_STRING_BYTE);
+ dataOutput.writeUTF(stringBuilder.toString());
+ } catch (HyracksDataException e) {
+ throw new AlgebricksException(e);
+ } catch (IOException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+ };
+ }
+ };
+
+ }
+}
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/SubBinaryFromDescriptor.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/SubBinaryFromDescriptor.java
new file mode 100644
index 0000000..2247f2e
--- /dev/null
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/SubBinaryFromDescriptor.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.runtime.evaluators.functions.binary;
+
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.asterix.om.functions.IFunctionDescriptor;
+import edu.uci.ics.asterix.om.functions.IFunctionDescriptorFactory;
+import edu.uci.ics.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
+import edu.uci.ics.hyracks.data.std.primitive.ByteArrayPointable;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+
+public class SubBinaryFromDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override public IFunctionDescriptor createFunctionDescriptor() {
+ return new SubBinaryFromDescriptor();
+ }
+ };
+
+ @Override public FunctionIdentifier getIdentifier() {
+ return AsterixBuiltinFunctions.SUBBINARY_FROM;
+ }
+
+ @Override public ICopyEvaluatorFactory createEvaluatorFactory(final ICopyEvaluatorFactory[] args)
+ throws AlgebricksException {
+ return new ICopyEvaluatorFactory() {
+ @Override public ICopyEvaluator createEvaluator(final IDataOutputProvider output)
+ throws AlgebricksException {
+ return new SubBinaryFromToDescriptor.AbstractSubBinaryCopyEvaluator(output, args,
+ getIdentifier().getName()) {
+ @Override protected int getSubLength(IFrameTupleReference tuple) throws AlgebricksException {
+ return ByteArrayPointable.MAX_LENGTH;
+ }
+ };
+ }
+ };
+ }
+
+}
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/SubBinaryFromToDescriptor.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/SubBinaryFromToDescriptor.java
new file mode 100644
index 0000000..9b3a48f
--- /dev/null
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/evaluators/functions/binary/SubBinaryFromToDescriptor.java
@@ -0,0 +1,120 @@
+/*
+ * Copyright 2009-2013 by The Regents of the University of California
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * you may obtain a copy of the License from
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package edu.uci.ics.asterix.runtime.evaluators.functions.binary;
+
+import edu.uci.ics.asterix.om.functions.AsterixBuiltinFunctions;
+import edu.uci.ics.asterix.om.functions.IFunctionDescriptor;
+import edu.uci.ics.asterix.om.functions.IFunctionDescriptorFactory;
+import edu.uci.ics.asterix.om.types.ATypeTag;
+import edu.uci.ics.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
+import edu.uci.ics.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluator;
+import edu.uci.ics.hyracks.algebricks.runtime.base.ICopyEvaluatorFactory;
+import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
+import edu.uci.ics.hyracks.data.std.api.IDataOutputProvider;
+import edu.uci.ics.hyracks.data.std.primitive.ByteArrayPointable;
+import edu.uci.ics.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+import edu.uci.ics.hyracks.dataflow.common.data.marshalling.IntegerSerializerDeserializer;
+
+import java.io.IOException;
+
+public class SubBinaryFromToDescriptor extends AbstractScalarFunctionDynamicDescriptor {
+
+ public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() {
+ @Override public IFunctionDescriptor createFunctionDescriptor() {
+ return new SubBinaryFromToDescriptor();
+ }
+ };
+
+ @Override public FunctionIdentifier getIdentifier() {
+ return AsterixBuiltinFunctions.SUBBINARY_FROM_TO;
+ }
+
+ @Override public ICopyEvaluatorFactory createEvaluatorFactory(final ICopyEvaluatorFactory[] args)
+ throws AlgebricksException {
+ return new ICopyEvaluatorFactory() {
+ @Override public ICopyEvaluator createEvaluator(final IDataOutputProvider output)
+ throws AlgebricksException {
+ return new AbstractSubBinaryCopyEvaluator(output, args, getIdentifier().getName()) {
+ @Override protected int getSubLength(IFrameTupleReference tuple) throws AlgebricksException {
+ ATypeTag tagSubLength = evaluateTuple(tuple, 2);
+ if (tagSubLength != ATypeTag.INT32) {
+ throw new AlgebricksException(
+ functionName + ":expects INT32 at 3rd arguments, but got " + tagSubLength);
+ }
+ return IntegerSerializerDeserializer.getInt(storages[2].getByteArray(), 1);
+ }
+ };
+ }
+ };
+ }
+
+ static abstract class AbstractSubBinaryCopyEvaluator extends AbstractCopyEvaluator {
+ public AbstractSubBinaryCopyEvaluator(IDataOutputProvider output,
+ ICopyEvaluatorFactory[] copyEvaluatorFactories, String functionName)
+ throws AlgebricksException {
+ super(output, copyEvaluatorFactories);
+ this.functionName = functionName;
+ }
+
+ protected final String functionName;
+
+ static final ATypeTag[] EXPECTED_INPUT_TAGS = { ATypeTag.BINARY, ATypeTag.INT32 };
+
+ @Override public void evaluate(IFrameTupleReference tuple) throws AlgebricksException {
+
+ ATypeTag argTag0 = evaluateTuple(tuple, 0);
+ ATypeTag argTag1 = evaluateTuple(tuple, 1);
+
+ try {
+ if (serializeNullIfAnyNull(argTag0, argTag1)) {
+ return;
+ }
+ checkTypeMachingThrowsIfNot(functionName, EXPECTED_INPUT_TAGS, argTag0,
+ argTag1);
+
+ byte[] binaryBytes = storages[0].getByteArray();
+ byte[] startBytes = storages[1].getByteArray();
+
+ int start = IntegerSerializerDeserializer.getInt(startBytes, 1) - 1; // strange SQL index convention
+ int totalLength = ByteArrayPointable.getLength(binaryBytes, 1);
+ int subLength = getSubLength(tuple);
+
+ if (start < 0) {
+ start = 0;
+ }
+
+ if (start >= totalLength || subLength < 0) {
+ subLength = 0;
+ } else if (start + subLength > totalLength) {
+ subLength = totalLength - start;
+ }
+
+ dataOutput.write(ATypeTag.BINARY.serialize());
+ dataOutput.writeShort(subLength);
+ dataOutput
+ .write(binaryBytes, 1 + ByteArrayPointable.SIZE_OF_LENGTH + start, subLength);
+ } catch (HyracksDataException e) {
+ throw new AlgebricksException(e);
+ } catch (IOException e) {
+ throw new AlgebricksException(e);
+ }
+ }
+
+ protected abstract int getSubLength(IFrameTupleReference tuple) throws AlgebricksException;
+ }
+}
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/formats/NonTaggedDataFormat.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/formats/NonTaggedDataFormat.java
index 28ad34b..d9bf623 100644
--- a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/formats/NonTaggedDataFormat.java
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/formats/NonTaggedDataFormat.java
@@ -127,6 +127,8 @@
import edu.uci.ics.asterix.runtime.evaluators.common.CreateMBREvalFactory;
import edu.uci.ics.asterix.runtime.evaluators.common.FieldAccessByIndexEvalFactory;
import edu.uci.ics.asterix.runtime.evaluators.common.FunctionManagerImpl;
+import edu.uci.ics.asterix.runtime.evaluators.constructors.ABinaryBase64StringConstructorDescriptor;
+import edu.uci.ics.asterix.runtime.evaluators.constructors.ABinaryHexStringConstructorDescriptor;
import edu.uci.ics.asterix.runtime.evaluators.constructors.ABooleanConstructorDescriptor;
import edu.uci.ics.asterix.runtime.evaluators.constructors.ACircleConstructorDescriptor;
import edu.uci.ics.asterix.runtime.evaluators.constructors.ADateConstructorDescriptor;
@@ -242,6 +244,14 @@
import edu.uci.ics.asterix.runtime.evaluators.functions.SwitchCaseDescriptor;
import edu.uci.ics.asterix.runtime.evaluators.functions.UnorderedListConstructorDescriptor;
import edu.uci.ics.asterix.runtime.evaluators.functions.WordTokensDescriptor;
+import edu.uci.ics.asterix.runtime.evaluators.functions.binary.BinaryConcatDescriptor;
+import edu.uci.ics.asterix.runtime.evaluators.functions.binary.BinaryLengthDescriptor;
+import edu.uci.ics.asterix.runtime.evaluators.functions.binary.FindBinaryDescriptor;
+import edu.uci.ics.asterix.runtime.evaluators.functions.binary.FindBinaryFromDescriptor;
+import edu.uci.ics.asterix.runtime.evaluators.functions.binary.ParseBinaryDescriptor;
+import edu.uci.ics.asterix.runtime.evaluators.functions.binary.PrintBinaryDescriptor;
+import edu.uci.ics.asterix.runtime.evaluators.functions.binary.SubBinaryFromDescriptor;
+import edu.uci.ics.asterix.runtime.evaluators.functions.binary.SubBinaryFromToDescriptor;
import edu.uci.ics.asterix.runtime.evaluators.functions.temporal.AdjustDateTimeForTimeZoneDescriptor;
import edu.uci.ics.asterix.runtime.evaluators.functions.temporal.AdjustTimeForTimeZoneDescriptor;
import edu.uci.ics.asterix.runtime.evaluators.functions.temporal.CalendarDuartionFromDateDescriptor;
@@ -410,6 +420,17 @@
temp.add(NumericRoundDescriptor.FACTORY);
temp.add(NumericRoundHalfToEvenDescriptor.FACTORY);
temp.add(NumericRoundHalfToEven2Descriptor.FACTORY);
+
+ // Binary functions
+ temp.add(BinaryLengthDescriptor.FACTORY);
+ temp.add(ParseBinaryDescriptor.FACTORY);
+ temp.add(PrintBinaryDescriptor.FACTORY);
+ temp.add(BinaryConcatDescriptor.FACTORY);
+ temp.add(SubBinaryFromDescriptor.FACTORY);
+ temp.add(SubBinaryFromToDescriptor.FACTORY);
+ temp.add(FindBinaryDescriptor.FACTORY);
+ temp.add(FindBinaryFromDescriptor.FACTORY);
+
// String functions
temp.add(StringEqualDescriptor.FACTORY);
temp.add(StringStartWithDescrtiptor.FACTORY);
@@ -491,6 +512,8 @@
// new functions - constructors
temp.add(ABooleanConstructorDescriptor.FACTORY);
temp.add(ANullConstructorDescriptor.FACTORY);
+ temp.add(ABinaryHexStringConstructorDescriptor.FACTORY);
+ temp.add(ABinaryBase64StringConstructorDescriptor.FACTORY);
temp.add(AStringConstructorDescriptor.FACTORY);
temp.add(AInt8ConstructorDescriptor.FACTORY);
temp.add(AInt16ConstructorDescriptor.FACTORY);
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/ADMDataParser.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/ADMDataParser.java
index 0cd3098..a6419fe 100644
--- a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/ADMDataParser.java
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/ADMDataParser.java
@@ -18,7 +18,6 @@
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayDeque;
-import java.util.ArrayList;
import java.util.BitSet;
import java.util.List;
import java.util.Queue;
@@ -32,9 +31,7 @@
import edu.uci.ics.asterix.dataflow.data.nontagged.serde.AIntervalSerializerDeserializer;
import edu.uci.ics.asterix.dataflow.data.nontagged.serde.APolygonSerializerDeserializer;
import edu.uci.ics.asterix.om.base.ABoolean;
-import edu.uci.ics.asterix.om.base.AMutableUUID;
import edu.uci.ics.asterix.om.base.ANull;
-import edu.uci.ics.asterix.om.base.AUUID;
import edu.uci.ics.asterix.om.types.AOrderedListType;
import edu.uci.ics.asterix.om.types.ARecordType;
import edu.uci.ics.asterix.om.types.ATypeTag;
@@ -151,10 +148,11 @@
public void initialize(InputStream in, ARecordType recordType, boolean datasetRec) throws AsterixException {
// Use original record type variable - origRecTypeForAutoGeneratedPK if we have auto-generated PK.
// The recordType variable does not contain field information about auto-generated PK.
- if (!isPKAutoGenerated)
+ if (!isPKAutoGenerated) {
this.recordType = recordType;
- else
+ } else {
this.recordType = origRecTypeForAutoGeneratedPK;
+ }
this.datasetRec = datasetRec;
try {
admLexer = new AdmLexer(new java.io.InputStreamReader(in));
@@ -181,15 +179,17 @@
case AdmLexer.TOKEN_NULL_LITERAL: {
if (checkType(ATypeTag.NULL, objectType)) {
nullSerde.serialize(ANull.NULL, out);
- } else
+ } else {
throw new ParseException("This field can not be null");
+ }
break;
}
case AdmLexer.TOKEN_TRUE_LITERAL: {
if (checkType(ATypeTag.BOOLEAN, objectType)) {
booleanSerde.serialize(ABoolean.TRUE, out);
- } else
+ } else {
throw new ParseException(mismatchErrorMessage + objectType.getTypeName());
+ }
break;
}
case AdmLexer.TOKEN_BOOLEAN_CONS: {
@@ -199,8 +199,9 @@
case AdmLexer.TOKEN_FALSE_LITERAL: {
if (checkType(ATypeTag.BOOLEAN, objectType)) {
booleanSerde.serialize(ABoolean.FALSE, out);
- } else
+ } else {
throw new ParseException(mismatchErrorMessage + objectType.getTypeName());
+ }
break;
}
case AdmLexer.TOKEN_DOUBLE_LITERAL: {
@@ -267,14 +268,29 @@
admLexer.getLastTokenImage().length() - 1);
aUUID.fromStringToAMuatbleUUID(tokenImage);
uuidSerde.serialize(aUUID, out);
- } else
+ } else {
throw new ParseException(mismatchErrorMessage + objectType.getTypeName());
+ }
break;
}
case AdmLexer.TOKEN_STRING_CONS: {
parseConstructor(ATypeTag.STRING, objectType, out);
break;
}
+ case AdmLexer.TOKEN_HEX_CONS:
+ case AdmLexer.TOKEN_BASE64_CONS: {
+ if (checkType(ATypeTag.BINARY, objectType)) {
+ if (admLexer.next() == AdmLexer.TOKEN_CONSTRUCTOR_OPEN) {
+ if (admLexer.next() == AdmLexer.TOKEN_STRING_LITERAL) {
+ parseToBinaryTarget(token, admLexer.getLastTokenImage(), out);
+ if (admLexer.next() == AdmLexer.TOKEN_CONSTRUCTOR_CLOSE) {
+ break;
+ }
+ }
+ }
+ }
+ throw new ParseException(mismatchErrorMessage + objectType.getTypeName());
+ }
case AdmLexer.TOKEN_DATE_CONS: {
parseConstructor(ATypeTag.DATE, objectType, out);
break;
@@ -369,8 +385,9 @@
if (checkType(ATypeTag.UNORDEREDLIST, objectType)) {
objectType = getComplexType(objectType, ATypeTag.UNORDEREDLIST);
parseUnorderedList((AUnorderedListType) objectType, out);
- } else
+ } else {
throw new ParseException(mismatchErrorMessage + objectType.getTypeTag());
+ }
break;
}
@@ -378,16 +395,18 @@
if (checkType(ATypeTag.ORDEREDLIST, objectType)) {
objectType = getComplexType(objectType, ATypeTag.ORDEREDLIST);
parseOrderedList((AOrderedListType) objectType, out);
- } else
+ } else {
throw new ParseException(mismatchErrorMessage + objectType.getTypeTag());
+ }
break;
}
case AdmLexer.TOKEN_START_RECORD: {
if (checkType(ATypeTag.RECORD, objectType)) {
objectType = getComplexType(objectType, ATypeTag.RECORD);
parseRecord((ARecordType) objectType, out, datasetRec);
- } else
+ } else {
throw new ParseException(mismatchErrorMessage + objectType.getTypeTag());
+ }
break;
}
case AdmLexer.TOKEN_UUID_CONS: {
@@ -401,6 +420,7 @@
throw new ParseException("Unexpected ADM token kind: " + AdmLexer.tokenKindToString(token) + ".");
}
}
+
}
private String replaceEscapes(String tokenImage) throws ParseException {
@@ -464,15 +484,17 @@
return null;
}
- if (aObjectType.getTypeTag() == tag)
+ if (aObjectType.getTypeTag() == tag) {
return aObjectType;
+ }
if (aObjectType.getTypeTag() == ATypeTag.UNION) {
List<IAType> unionList = ((AUnionType) aObjectType).getUnionList();
- for (int i = 0; i < unionList.size(); i++)
+ for (int i = 0; i < unionList.size(); i++) {
if (unionList.get(i).getTypeTag() == tag) {
return unionList.get(i);
}
+ }
}
return null; // wont get here
}
@@ -512,13 +534,15 @@
if (recType != null) {
nulls = new BitSet(recType.getFieldNames().length);
recBuilder.reset(recType);
- } else
+ } else {
recBuilder.reset(null);
+ }
} else if (recType != null) {
nulls = new BitSet(recType.getFieldNames().length);
recBuilder.reset(recType);
- } else
+ } else {
recBuilder.reset(null);
+ }
recBuilder.init();
int token;
@@ -581,8 +605,9 @@
token = admLexer.next();
this.admFromLexerStream(token, fieldType, fieldValueBuffer.getDataOutput(), false);
if (openRecordField) {
- if (fieldValueBuffer.getByteArray()[0] != ATypeTag.NULL.serialize())
+ if (fieldValueBuffer.getByteArray()[0] != ATypeTag.NULL.serialize()) {
recBuilder.addField(fieldNameBuffer, fieldValueBuffer);
+ }
} else if (recType.getFieldTypes()[fieldId].getTypeTag() == ATypeTag.UNION) {
if (NonTaggedFormatUtil.isOptionalField((AUnionType) recType.getFieldTypes()[fieldId])) {
if (fieldValueBuffer.getByteArray()[0] != ATypeTag.NULL.serialize()) {
@@ -621,21 +646,26 @@
// where nullableFieldId equals primaryKey field.
// In this case, we create a random ID and assign it as a PK
if (isPKAutoGenerated && nullableFieldId == primaryKeyPosition) {
- fieldValueBuffer.reset();
- aUUID.nextUUID();
+ fieldValueBuffer.reset();
+ aUUID.nextUUID();
fieldValueBuffer.getDataOutput().writeByte(AUUIDTag);
- Integer64SerializerDeserializer.INSTANCE.serialize(aUUID.getMostSignificantBits(), fieldValueBuffer.getDataOutput());
- Integer64SerializerDeserializer.INSTANCE.serialize(aUUID.getLeastSignificantBits(), fieldValueBuffer.getDataOutput());
+ Integer64SerializerDeserializer.INSTANCE
+ .serialize(aUUID.getMostSignificantBits(), fieldValueBuffer.getDataOutput());
+ Integer64SerializerDeserializer.INSTANCE
+ .serialize(aUUID.getLeastSignificantBits(), fieldValueBuffer.getDataOutput());
recBuilder.addField(primaryKeyPosition, fieldValueBuffer);
insertedAutoGeneratedPK = true;
nulls.set(nullableFieldId);
// Check from the primary key field position to make sure no other field is missing
nullableFieldId = checkNullConstraints(recType, nulls, nullableFieldId);
}
- if (nullableFieldId != -1)
+ if (nullableFieldId != -1) {
throw new ParseException("Field: " + recType.getFieldNames()[nullableFieldId] + " can not be null");
- if (isPKAutoGenerated && !insertedAutoGeneratedPK)
- throw new ParseException("Auto-generated PK Field: " + recType.getFieldNames()[primaryKeyPosition] + " should not exist in the ADM file.");
+ }
+ if (isPKAutoGenerated && !insertedAutoGeneratedPK) {
+ throw new ParseException("Auto-generated PK Field: " + recType.getFieldNames()[primaryKeyPosition]
+ + " should not exist in the ADM file.");
+ }
}
recBuilder.write(out, true);
returnRecordBuilder(recBuilder);
@@ -645,23 +675,27 @@
private int checkNullConstraints(ARecordType recType, BitSet nulls, int startingPosition) {
boolean isNull = false;
- for (int i = startingPosition; i < recType.getFieldTypes().length; i++)
+ for (int i = startingPosition; i < recType.getFieldTypes().length; i++) {
if (nulls.get(i) == false) {
IAType type = recType.getFieldTypes()[i];
- if (type.getTypeTag() != ATypeTag.NULL && type.getTypeTag() != ATypeTag.UNION)
+ if (type.getTypeTag() != ATypeTag.NULL && type.getTypeTag() != ATypeTag.UNION) {
return i;
+ }
if (type.getTypeTag() == ATypeTag.UNION) { // union
List<IAType> unionList = ((AUnionType) type).getUnionList();
- for (int j = 0; j < unionList.size(); j++)
+ for (int j = 0; j < unionList.size(); j++) {
if (unionList.get(j).getTypeTag() == ATypeTag.NULL) {
isNull = true;
break;
}
- if (!isNull)
+ }
+ if (!isNull) {
return i;
+ }
}
}
+ }
return -1;
}
@@ -671,8 +705,9 @@
OrderedListBuilder orderedListBuilder = (OrderedListBuilder) getOrderedListBuilder();
IAType itemType = null;
- if (oltype != null)
+ if (oltype != null) {
itemType = oltype.getItemType();
+ }
orderedListBuilder.reset(oltype);
int token;
@@ -715,8 +750,9 @@
IAType itemType = null;
- if (uoltype != null)
+ if (uoltype != null) {
itemType = uoltype.getItemType();
+ }
unorderedListBuilder.reset(uoltype);
int token;
@@ -758,10 +794,11 @@
private IARecordBuilder getRecordBuilder() {
RecordBuilder recBuilder = (RecordBuilder) recordBuilderPool.poll();
- if (recBuilder != null)
+ if (recBuilder != null) {
return recBuilder;
- else
+ } else {
return new RecordBuilder();
+ }
}
private void returnRecordBuilder(IARecordBuilder recBuilder) {
@@ -770,10 +807,11 @@
private IAsterixListBuilder getOrderedListBuilder() {
OrderedListBuilder orderedListBuilder = (OrderedListBuilder) orderedListBuilderPool.poll();
- if (orderedListBuilder != null)
+ if (orderedListBuilder != null) {
return orderedListBuilder;
- else
+ } else {
return new OrderedListBuilder();
+ }
}
private void returnOrderedListBuilder(IAsterixListBuilder orderedListBuilder) {
@@ -782,10 +820,11 @@
private IAsterixListBuilder getUnorderedListBuilder() {
UnorderedListBuilder unorderedListBuilder = (UnorderedListBuilder) unorderedListBuilderPool.poll();
- if (unorderedListBuilder != null)
+ if (unorderedListBuilder != null) {
return unorderedListBuilder;
- else
+ } else {
return new UnorderedListBuilder();
+ }
}
private void returnUnorderedListBuilder(IAsterixListBuilder unorderedListBuilder) {
@@ -805,6 +844,20 @@
baaosPool.add(tempBaaos);
}
+ private void parseToBinaryTarget(int lexerToken, String tokenImage, DataOutput out)
+ throws ParseException, HyracksDataException {
+ switch (lexerToken) {
+ case AdmLexer.TOKEN_HEX_CONS: {
+ parseHexBinaryString(tokenImage.toCharArray(), 1, tokenImage.length() - 2, out);
+ break;
+ }
+ case AdmLexer.TOKEN_BASE64_CONS: {
+ parseBase64BinaryString(tokenImage.toCharArray(), 1, tokenImage.length() - 2, out);
+ break;
+ }
+ }
+ }
+
private void parseToNumericTarget(ATypeTag typeTag, IAType objectType, DataOutput out) throws AsterixException,
IOException {
final ATypeTag targetTypeTag = getTargetTypeTag(typeTag, objectType);
@@ -951,12 +1004,13 @@
private void parseBoolean(String bool, DataOutput out) throws AsterixException, HyracksDataException {
String errorMessage = "This can not be an instance of boolean";
- if (bool.equals("true"))
+ if (bool.equals("true")) {
booleanSerde.serialize(ABoolean.TRUE, out);
- else if (bool.equals("false"))
+ } else if (bool.equals("false")) {
booleanSerde.serialize(ABoolean.FALSE, out);
- else
+ } else {
throw new ParseException(errorMessage);
+ }
}
private void parseInt8(String int8, DataOutput out) throws AsterixException, HyracksDataException {
@@ -965,24 +1019,27 @@
byte value = 0;
int offset = 0;
- if (int8.charAt(offset) == '+')
+ if (int8.charAt(offset) == '+') {
offset++;
- else if (int8.charAt(offset) == '-') {
+ } else if (int8.charAt(offset) == '-') {
offset++;
positive = false;
}
for (; offset < int8.length(); offset++) {
- if (int8.charAt(offset) >= '0' && int8.charAt(offset) <= '9')
+ if (int8.charAt(offset) >= '0' && int8.charAt(offset) <= '9') {
value = (byte) (value * 10 + int8.charAt(offset) - '0');
- else if (int8.charAt(offset) == 'i' && int8.charAt(offset + 1) == '8' && offset + 2 == int8.length())
+ } else if (int8.charAt(offset) == 'i' && int8.charAt(offset + 1) == '8' && offset + 2 == int8.length()) {
break;
- else
+ } else {
throw new ParseException(errorMessage);
+ }
}
- if (value < 0)
+ if (value < 0) {
throw new ParseException(errorMessage);
- if (value > 0 && !positive)
+ }
+ if (value > 0 && !positive) {
value *= -1;
+ }
aInt8.setValue(value);
int8Serde.serialize(aInt8, out);
}
@@ -993,25 +1050,28 @@
short value = 0;
int offset = 0;
- if (int16.charAt(offset) == '+')
+ if (int16.charAt(offset) == '+') {
offset++;
- else if (int16.charAt(offset) == '-') {
+ } else if (int16.charAt(offset) == '-') {
offset++;
positive = false;
}
for (; offset < int16.length(); offset++) {
- if (int16.charAt(offset) >= '0' && int16.charAt(offset) <= '9')
+ if (int16.charAt(offset) >= '0' && int16.charAt(offset) <= '9') {
value = (short) (value * 10 + int16.charAt(offset) - '0');
- else if (int16.charAt(offset) == 'i' && int16.charAt(offset + 1) == '1' && int16.charAt(offset + 2) == '6'
- && offset + 3 == int16.length())
+ } else if (int16.charAt(offset) == 'i' && int16.charAt(offset + 1) == '1' && int16.charAt(offset + 2) == '6'
+ && offset + 3 == int16.length()) {
break;
- else
+ } else {
throw new ParseException(errorMessage);
+ }
}
- if (value < 0)
+ if (value < 0) {
throw new ParseException(errorMessage);
- if (value > 0 && !positive)
+ }
+ if (value > 0 && !positive) {
value *= -1;
+ }
aInt16.setValue(value);
int16Serde.serialize(aInt16, out);
}
@@ -1022,25 +1082,28 @@
int value = 0;
int offset = 0;
- if (int32.charAt(offset) == '+')
+ if (int32.charAt(offset) == '+') {
offset++;
- else if (int32.charAt(offset) == '-') {
+ } else if (int32.charAt(offset) == '-') {
offset++;
positive = false;
}
for (; offset < int32.length(); offset++) {
- if (int32.charAt(offset) >= '0' && int32.charAt(offset) <= '9')
+ if (int32.charAt(offset) >= '0' && int32.charAt(offset) <= '9') {
value = (value * 10 + int32.charAt(offset) - '0');
- else if (int32.charAt(offset) == 'i' && int32.charAt(offset + 1) == '3' && int32.charAt(offset + 2) == '2'
- && offset + 3 == int32.length())
+ } else if (int32.charAt(offset) == 'i' && int32.charAt(offset + 1) == '3' && int32.charAt(offset + 2) == '2'
+ && offset + 3 == int32.length()) {
break;
- else
+ } else {
throw new ParseException(errorMessage);
+ }
}
- if (value < 0)
+ if (value < 0) {
throw new ParseException(errorMessage);
- if (value > 0 && !positive)
+ }
+ if (value > 0 && !positive) {
value *= -1;
+ }
aInt32.setValue(value);
int32Serde.serialize(aInt32, out);
@@ -1052,25 +1115,28 @@
long value = 0;
int offset = 0;
- if (int64.charAt(offset) == '+')
+ if (int64.charAt(offset) == '+') {
offset++;
- else if (int64.charAt(offset) == '-') {
+ } else if (int64.charAt(offset) == '-') {
offset++;
positive = false;
}
for (; offset < int64.length(); offset++) {
- if (int64.charAt(offset) >= '0' && int64.charAt(offset) <= '9')
+ if (int64.charAt(offset) >= '0' && int64.charAt(offset) <= '9') {
value = (value * 10 + int64.charAt(offset) - '0');
- else if (int64.charAt(offset) == 'i' && int64.charAt(offset + 1) == '6' && int64.charAt(offset + 2) == '4'
- && offset + 3 == int64.length())
+ } else if (int64.charAt(offset) == 'i' && int64.charAt(offset + 1) == '6' && int64.charAt(offset + 2) == '4'
+ && offset + 3 == int64.length()) {
break;
- else
+ } else {
throw new ParseException(errorMessage);
+ }
}
- if (value < 0)
+ if (value < 0) {
throw new ParseException(errorMessage);
- if (value > 0 && !positive)
+ }
+ if (value > 0 && !positive) {
value *= -1;
+ }
aInt64.setValue(value);
int64Serde.serialize(aInt64, out);
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/AbstractDataParser.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/AbstractDataParser.java
index e7ae93d..402b639 100644
--- a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/AbstractDataParser.java
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/AbstractDataParser.java
@@ -17,6 +17,7 @@
import java.io.DataOutput;
import edu.uci.ics.asterix.formats.nontagged.AqlSerializerDeserializerProvider;
+import edu.uci.ics.asterix.om.base.ABinary;
import edu.uci.ics.asterix.om.base.ABoolean;
import edu.uci.ics.asterix.om.base.ACircle;
import edu.uci.ics.asterix.om.base.ADate;
@@ -30,6 +31,7 @@
import edu.uci.ics.asterix.om.base.AInt64;
import edu.uci.ics.asterix.om.base.AInt8;
import edu.uci.ics.asterix.om.base.ALine;
+import edu.uci.ics.asterix.om.base.AMutableBinary;
import edu.uci.ics.asterix.om.base.AMutableCircle;
import edu.uci.ics.asterix.om.base.AMutableDate;
import edu.uci.ics.asterix.om.base.AMutableDateTime;
@@ -64,10 +66,10 @@
import edu.uci.ics.asterix.om.base.temporal.GregorianCalendarSystem;
import edu.uci.ics.asterix.om.types.BuiltinType;
import edu.uci.ics.hyracks.algebricks.common.exceptions.AlgebricksException;
-import edu.uci.ics.hyracks.algebricks.common.utils.Pair;
import edu.uci.ics.hyracks.api.dataflow.value.ISerializerDeserializer;
import edu.uci.ics.hyracks.api.exceptions.HyracksDataException;
-import edu.uci.ics.hyracks.data.std.util.ArrayBackedValueStorage;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.ByteArrayBase64ParserFactory;
+import edu.uci.ics.hyracks.dataflow.common.data.parsers.ByteArrayHexParserFactory;
/**
* Base class for data parsers. Includes the common set of definitions for
@@ -82,6 +84,7 @@
protected AMutableDouble aDouble = new AMutableDouble(0);
protected AMutableFloat aFloat = new AMutableFloat(0);
protected AMutableString aString = new AMutableString("");
+ protected AMutableBinary aBinary = new AMutableBinary(new byte[] { }, 0, 0);
protected AMutableString aStringFieldName = new AMutableString("");
protected AMutableUUID aUUID = new AMutableUUID(0, 0);
// For temporal and spatial data types
@@ -98,6 +101,8 @@
protected AMutableLine aLine = new AMutableLine(null, null);
protected AMutableDate aDate = new AMutableDate(0);
+ private final byte[] base64ParserQuadruplet = new byte[4];
+
// Serializers
@SuppressWarnings("unchecked")
protected ISerializerDeserializer<ADouble> doubleSerde = AqlSerializerDeserializerProvider.INSTANCE
@@ -166,6 +171,9 @@
@SuppressWarnings("unchecked")
protected final static ISerializerDeserializer<ALine> lineSerde = AqlSerializerDeserializerProvider.INSTANCE
.getSerializerDeserializer(BuiltinType.ALINE);
+ @SuppressWarnings("unchecked")
+ protected final static ISerializerDeserializer<ABinary> binarySerde = AqlSerializerDeserializerProvider.INSTANCE
+ .getSerializerDeserializer(BuiltinType.ABINARY);
protected String filename;
@@ -289,8 +297,9 @@
protected void parseRectangle(String rectangle, DataOutput out) throws HyracksDataException {
try {
String[] points = rectangle.split(" ");
- if (points.length != 2)
+ if (points.length != 2) {
throw new HyracksDataException("rectangle consists of only 2 points.");
+ }
aPoint.setValue(Double.parseDouble(points[0].split(",")[0]), Double.parseDouble(points[0].split(",")[1]));
aPoint2.setValue(Double.parseDouble(points[1].split(",")[0]), Double.parseDouble(points[1].split(",")[1]));
if (aPoint.getX() > aPoint2.getX() && aPoint.getY() > aPoint2.getY()) {
@@ -310,8 +319,9 @@
protected void parseLine(String line, DataOutput out) throws HyracksDataException {
try {
String[] points = line.split(" ");
- if (points.length != 2)
+ if (points.length != 2) {
throw new HyracksDataException("line consists of only 2 points.");
+ }
aPoint.setValue(Double.parseDouble(points[0].split(",")[0]), Double.parseDouble(points[0].split(",")[1]));
aPoint2.setValue(Double.parseDouble(points[1].split(",")[0]), Double.parseDouble(points[1].split(",")[1]));
aLine.setValue(aPoint, aPoint2);
@@ -321,4 +331,20 @@
}
}
+ protected void parseHexBinaryString(char[] input, int start, int length, DataOutput out)
+ throws HyracksDataException {
+ byte[] newBytes = ByteArrayHexParserFactory
+ .extractPointableArrayFromHexString(input, start, length, aBinary.getBytes());
+ aBinary.setValue(newBytes, 0, newBytes.length);
+ binarySerde.serialize(aBinary, out);
+ }
+
+ protected void parseBase64BinaryString(char[] input, int start, int length, DataOutput out)
+ throws HyracksDataException {
+ byte[] newBytes = ByteArrayBase64ParserFactory
+ .extractPointableArrayFromBase64String(input, start, length,
+ aBinary.getBytes(), base64ParserQuadruplet);
+ aBinary.setValue(newBytes, 0, newBytes.length);
+ binarySerde.serialize(aBinary, out);
+ }
}
diff --git a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/DelimitedDataParser.java b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/DelimitedDataParser.java
index 18f9553..7a0ae4d 100644
--- a/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/DelimitedDataParser.java
+++ b/asterix-runtime/src/main/java/edu/uci/ics/asterix/runtime/operators/file/DelimitedDataParser.java
@@ -503,6 +503,7 @@
lastDelimiterPosition = p;
start = p + 1;
state = State.CR;
+ startedQuote = false;
return true;
}
}
diff --git a/asterix-runtime/src/main/resources/adm.grammar b/asterix-runtime/src/main/resources/adm.grammar
index b9db2e7..b0ca26f 100644
--- a/asterix-runtime/src/main/resources/adm.grammar
+++ b/asterix-runtime/src/main/resources/adm.grammar
@@ -19,6 +19,8 @@
DATETIME_CONS = string(datetime)
DURATION_CONS = string(duration)
STRING_CONS = string(string)
+HEX_CONS = string(hex)
+BASE64_CONS = string(base64)
POINT_CONS = string(point)
POINT3D_CONS = string(point3d)
LINE_CONS = string(line)