[ASTERIXDB-3236][EXT]: Part 2: Extract computed field values from object key
Change-Id: Iacded6137bf701dc5ac0b12e0627f5e571e4bea9
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17711
Reviewed-by: Wail Alkowaileet <wael.y.k@gmail.com>
Reviewed-by: Hussain Towaileb <hussainht@gmail.com>
Tested-by: Hussain Towaileb <hussainht@gmail.com>
Integration-Tests: Hussain Towaileb <hussainht@gmail.com>
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/PrefixComputedFieldsTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/PrefixComputedFieldsTest.java
index 2736969..efa4c79 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/PrefixComputedFieldsTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/PrefixComputedFieldsTest.java
@@ -19,11 +19,15 @@
package org.apache.asterix.test.external_dataset;
-import static org.apache.asterix.om.types.BuiltinType.AINT32;
+import static org.apache.asterix.om.types.BuiltinType.AINT64;
import static org.apache.asterix.om.types.BuiltinType.ASTRING;
+import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.asterix.external.util.ExternalDataPrefix;
import org.junit.Test;
@@ -42,6 +46,7 @@
assertEquals(Collections.emptyList(), prefix.getComputedFieldNames());
assertEquals(Collections.emptyList(), prefix.getComputedFieldTypes());
assertEquals(Collections.emptyList(), prefix.getComputedFieldSegmentIndexes());
+ assertTrue(prefix.getIndexToComputedFieldsMap().isEmpty());
String prefix1 = "";
prefix = new ExternalDataPrefix(prefix1);
@@ -52,6 +57,7 @@
assertEquals(Collections.emptyList(), prefix.getComputedFieldNames());
assertEquals(Collections.emptyList(), prefix.getComputedFieldTypes());
assertEquals(Collections.emptyList(), prefix.getComputedFieldSegmentIndexes());
+ assertTrue(prefix.getIndexToComputedFieldsMap().isEmpty());
String prefix2 = "hotel";
prefix = new ExternalDataPrefix(prefix2);
@@ -62,6 +68,7 @@
assertEquals(Collections.emptyList(), prefix.getComputedFieldNames());
assertEquals(Collections.emptyList(), prefix.getComputedFieldTypes());
assertEquals(Collections.emptyList(), prefix.getComputedFieldSegmentIndexes());
+ assertTrue(prefix.getIndexToComputedFieldsMap().isEmpty());
String prefix3 = "hotel/{hotel-id:inT}/";
prefix = new ExternalDataPrefix(prefix3);
@@ -70,8 +77,9 @@
assertTrue(prefix.isEndsWithSlash());
assertEquals(List.of("hotel", "{hotel-id:inT}"), prefix.getSegments());
assertEquals(List.of("hotel-id"), prefix.getComputedFieldNames());
- assertEquals(List.of(AINT32), prefix.getComputedFieldTypes());
+ assertEquals(List.of(AINT64), prefix.getComputedFieldTypes());
assertEquals(List.of(1), prefix.getComputedFieldSegmentIndexes());
+ assertEquals("(.+)", prefix.getIndexToComputedFieldsMap().get(1).getExpression());
String prefix4 = "hotel/{hotel-id:int}-{hotel-name:sTRing}";
prefix = new ExternalDataPrefix(prefix4);
@@ -80,8 +88,9 @@
assertFalse(prefix.isEndsWithSlash());
assertEquals(List.of("hotel", "{hotel-id:int}-{hotel-name:sTRing}"), prefix.getSegments());
assertEquals(List.of("hotel-id", "hotel-name"), prefix.getComputedFieldNames());
- assertEquals(List.of(AINT32, ASTRING), prefix.getComputedFieldTypes());
+ assertEquals(List.of(AINT64, ASTRING), prefix.getComputedFieldTypes());
assertEquals(List.of(1, 1), prefix.getComputedFieldSegmentIndexes());
+ assertEquals("(.+)-(.+)", prefix.getIndexToComputedFieldsMap().get(1).getExpression());
String prefix5 = "hotel/something/{hotel-id:int}-{hotel-name:sTRing}/review/{year:int}-{month:int}-{day:int}/";
prefix = new ExternalDataPrefix(prefix5);
@@ -92,8 +101,10 @@
assertEquals(List.of("hotel", "something", "{hotel-id:int}-{hotel-name:sTRing}", "review",
"{year:int}-{month:int}-{day:int}"), prefix.getSegments());
assertEquals(List.of("hotel-id", "hotel-name", "year", "month", "day"), prefix.getComputedFieldNames());
- assertEquals(List.of(AINT32, ASTRING, AINT32, AINT32, AINT32), prefix.getComputedFieldTypes());
+ assertEquals(List.of(AINT64, ASTRING, AINT64, AINT64, AINT64), prefix.getComputedFieldTypes());
assertEquals(List.of(2, 2, 4, 4, 4), prefix.getComputedFieldSegmentIndexes());
+ assertEquals("(.+)-(.+)", prefix.getIndexToComputedFieldsMap().get(2).getExpression());
+ assertEquals("(.+)-(.+)-(.+)", prefix.getIndexToComputedFieldsMap().get(4).getExpression());
String prefix6 = "hotel/something/{hotel-id:int}-{hotel-name:sTRing}/review/{year:int}/{month:int}/{day:int}";
prefix = new ExternalDataPrefix(prefix6);
@@ -104,8 +115,12 @@
assertEquals(List.of("hotel", "something", "{hotel-id:int}-{hotel-name:sTRing}", "review", "{year:int}",
"{month:int}", "{day:int}"), prefix.getSegments());
assertEquals(List.of("hotel-id", "hotel-name", "year", "month", "day"), prefix.getComputedFieldNames());
- assertEquals(List.of(AINT32, ASTRING, AINT32, AINT32, AINT32), prefix.getComputedFieldTypes());
+ assertEquals(List.of(AINT64, ASTRING, AINT64, AINT64, AINT64), prefix.getComputedFieldTypes());
assertEquals(List.of(2, 2, 4, 5, 6), prefix.getComputedFieldSegmentIndexes());
+ assertEquals("(.+)-(.+)", prefix.getIndexToComputedFieldsMap().get(2).getExpression());
+ assertEquals("(.+)", prefix.getIndexToComputedFieldsMap().get(4).getExpression());
+ assertEquals("(.+)", prefix.getIndexToComputedFieldsMap().get(5).getExpression());
+ assertEquals("(.+)", prefix.getIndexToComputedFieldsMap().get(6).getExpression());
String prefix7 = "hotel/{hotel.details.id:int}-{hotel-name:sTRing}";
prefix = new ExternalDataPrefix(prefix7);
@@ -113,7 +128,51 @@
assertEquals("hotel", prefix.getRoot());
assertFalse(prefix.isEndsWithSlash());
assertEquals(List.of("hotel.details.id", "hotel-name"), prefix.getComputedFieldNames());
- assertEquals(List.of(AINT32, ASTRING), prefix.getComputedFieldTypes());
+ assertEquals(List.of(AINT64, ASTRING), prefix.getComputedFieldTypes());
assertEquals(List.of(1, 1), prefix.getComputedFieldSegmentIndexes());
+ assertEquals("(.+)-(.+)", prefix.getIndexToComputedFieldsMap().get(1).getExpression());
+
+ String prefix8 =
+ "hotel/hotel-{hotel-id:int}-hotel-{hotel-name:sTRing}/review/year-{year:int}/{month:int}-month/day-{day:int}-day";
+ prefix = new ExternalDataPrefix(prefix8);
+ assertEquals(
+ "hotel/hotel-{hotel-id:int}-hotel-{hotel-name:sTRing}/review/year-{year:int}/{month:int}-month/day-{day:int}-day",
+ prefix.getOriginal());
+ assertEquals("hotel", prefix.getRoot());
+ assertFalse(prefix.isEndsWithSlash());
+ assertEquals(List.of("hotel", "hotel-{hotel-id:int}-hotel-{hotel-name:sTRing}", "review", "year-{year:int}",
+ "{month:int}-month", "day-{day:int}-day"), prefix.getSegments());
+ assertEquals(List.of("hotel-id", "hotel-name", "year", "month", "day"), prefix.getComputedFieldNames());
+ assertEquals(List.of(AINT64, ASTRING, AINT64, AINT64, AINT64), prefix.getComputedFieldTypes());
+ assertEquals(List.of(1, 1, 3, 4, 5), prefix.getComputedFieldSegmentIndexes());
+ assertEquals("hotel-(.+)-hotel-(.+)", prefix.getIndexToComputedFieldsMap().get(1).getExpression());
+ assertEquals("year-(.+)", prefix.getIndexToComputedFieldsMap().get(3).getExpression());
+ assertEquals("(.+)-month", prefix.getIndexToComputedFieldsMap().get(4).getExpression());
+ assertEquals("day-(.+)-day", prefix.getIndexToComputedFieldsMap().get(5).getExpression());
+
+ List<String> keys = new ArrayList<>();
+ keys.add("hotel/hotel-1-hotel-name1/review/year-2000/January-month/day-1-day");
+ keys.add("hotel/hotel-2-hotel-name2/review/year-2001/February-month/day-2-day");
+ keys.add("hotel/hotel-3-hotel-name3/review/year-2002/March-month/day-3-day");
+
+ for (String key : keys) {
+ List<String> keySegments = ExternalDataPrefix.extractPrefixSegments(key);
+ for (Map.Entry<Integer, ExternalDataPrefix.PrefixSegment> entry : prefix.getIndexToComputedFieldsMap()
+ .entrySet()) {
+ int index = entry.getKey();
+ ExternalDataPrefix.PrefixSegment segment = entry.getValue();
+
+ String expression = segment.getExpression();
+
+ String keySegment = keySegments.get(index);
+ Matcher matcher = Pattern.compile(expression).matcher(keySegment);
+
+ matcher.find();
+ for (int i = 1; i <= matcher.groupCount(); i++) {
+ System.out.println(matcher.group(i));
+ }
+ }
+ System.out.println("\n");
+ }
}
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 26e1bba..9101aab 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -274,6 +274,7 @@
UNSUPPORTED_ICEBERG_FORMAT_VERSION(1179),
ERROR_READING_ICEBERG_METADATA(1180),
UNSUPPORTED_COMPUTED_FIELD_TYPE(1181),
+ FAILED_TO_CALCULATE_COMPUTED_FIELDS(1182),
// Feed errors
DATAFLOW_ILLEGAL_STATE(3001),
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 96c8843..980fa0c 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -276,6 +276,7 @@
1179 = Unsupported iceberg format version
1180 = Error reading iceberg data
1181 = Unsupported computed field type: %1$s
+1182 = Failed to calculate computed fields: %1$s
# Feed Errors
3001 = Illegal state.
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
index 69393a1..9fa2b5a 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
@@ -65,7 +65,7 @@
// very expensive since at the root of the prefix we might load millions of files, we should consider (when
// possible) to get the value and add it
List<S3Object> filesOnly = S3Utils.listS3Objects(configuration, includeExcludeMatcher, warningCollector);
- filterPrefixes(externalDataPrefix, filesOnly, filterEvaluatorFactory.create(ctx, warningCollector));
+ filesOnly = filterPrefixes(externalDataPrefix, filesOnly, filterEvaluatorFactory.create(ctx, warningCollector));
// Distribute work load amongst the partitions
distributeWorkLoad(filesOnly, getPartitionsCount());
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
index 18dbf43..2a707fb 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
@@ -33,6 +33,7 @@
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
@@ -43,7 +44,6 @@
import org.apache.asterix.om.types.BuiltinTypeMap;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.utils.ProjectionFiltrationTypeUtil;
-import org.apache.commons.lang3.tuple.Pair;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -58,14 +58,14 @@
private final List<IAType> computedFieldTypes = new ArrayList<>();
private final List<Integer> computedFieldSegmentIndexes = new ArrayList<>();
private final List<ARecordType> paths = new ArrayList<>();
- private final Map<Integer, Pair<List<String>, List<IAType>>> computedFields = new HashMap<>();
+ private final Map<Integer, PrefixSegment> indexToComputedFieldsMap = new HashMap<>();
public static final String PREFIX_ROOT_FIELD_NAME = "prefix-root";
public static final Set<ATypeTag> supportedTypes = new HashSet<>();
static {
- supportedTypes.add(BuiltinType.ASTRING.getTypeTag());
- supportedTypes.add(BuiltinType.AINT32.getTypeTag());
+ supportedTypes.add(ATypeTag.STRING);
+ supportedTypes.add(ATypeTag.BIGINT);
}
public ExternalDataPrefix(Map<String, String> configuration) throws AlgebricksException {
@@ -79,23 +79,6 @@
segments = extractPrefixSegments(original);
extractComputedFields();
extractRoot();
-
- for (int i = 0; i < computedFieldSegmentIndexes.size(); i++) {
- int segmentIndex = computedFieldSegmentIndexes.get(i);
-
- if (computedFields.containsKey(segmentIndex)) {
- Pair<List<String>, List<IAType>> pair = computedFields.get(segmentIndex);
- pair.getLeft().add(computedFieldNames.get(i));
- pair.getRight().add(computedFieldTypes.get(i));
- } else {
- List<String> names = new ArrayList<>();
- List<IAType> types = new ArrayList<>();
-
- names.add(computedFieldNames.get(i));
- types.add(computedFieldTypes.get(i));
- computedFields.put(segmentIndex, Pair.of(names, types));
- }
- }
}
public String getOriginal() {
@@ -134,10 +117,14 @@
return paths;
}
+ public Map<Integer, PrefixSegment> getIndexToComputedFieldsMap() {
+ return indexToComputedFieldsMap;
+ }
+
/**
* extracts the segments of a prefix, separated by the delimiter
*/
- private List<String> extractPrefixSegments(String prefix) {
+ public static List<String> extractPrefixSegments(String prefix) {
return prefix.isEmpty() ? Collections.emptyList() : Arrays.asList(prefix.split(PREFIX_DEFAULT_DELIMITER));
}
@@ -149,29 +136,61 @@
if (!segments.isEmpty()) {
// search for computed fields in each segment
Matcher matcher = COMPUTED_FIELD_PATTERN.matcher("");
+
+ // we need to keep track of the end position
+ StringBuilder expression = new StringBuilder();
+ int end = 0;
+
for (int i = 0; i < segments.size(); i++) {
matcher.reset(segments.get(i));
+ expression.setLength(0);
+ end = 0;
while (matcher.find()) {
+ expression.append(segments.get(i), end, matcher.start());
+
String computedField = matcher.group();
String[] splits = computedField.split(":");
String namePart = splits[0].substring(1);
String typePart = splits[1].substring(0, splits[1].length() - 1);
IAType type = BuiltinTypeMap.getBuiltinType(typePart);
+ type = getUpdatedType(type);
validateSupported(type.getTypeTag());
computedFieldNames.add(namePart);
computedFieldTypes.add(type);
computedFieldSegmentIndexes.add(i);
+ updateIndexToComputedFieldMap(i, namePart, type);
List<String> nameParts = List.of(namePart.split("\\."));
paths.add(ProjectionFiltrationTypeUtil.getPathRecordType(nameParts));
+
+ expression.append("(.+)");
+ end = matcher.end();
+ }
+
+ if (expression.length() > 0) {
+ expression.append(segments.get(i).substring(end));
+ indexToComputedFieldsMap.get(i).setExpression(expression.toString());
}
}
}
}
+ private void updateIndexToComputedFieldMap(int segmentIndex, String computedFieldName, IAType computedFieldType) {
+ if (indexToComputedFieldsMap.containsKey(segmentIndex)) {
+ PrefixSegment prefixSegment = indexToComputedFieldsMap.get(segmentIndex);
+ prefixSegment.getComputedFieldNames().add(computedFieldName);
+ prefixSegment.getComputedFieldTypes().add(computedFieldType);
+ } else {
+ PrefixSegment prefixSegment = new PrefixSegment();
+ prefixSegment.getComputedFieldNames().add(computedFieldName);
+ prefixSegment.getComputedFieldTypes().add(computedFieldType);
+ indexToComputedFieldsMap.put(segmentIndex, prefixSegment);
+ }
+ }
+
/**
* Returns the longest static path (root) before encountering the first computed field
*/
@@ -224,11 +243,18 @@
return false;
}
+ // no computed fields used in WHERE clause, accept object
+ if (evaluator.isEmpty()) {
+ return true;
+ }
+
// extract values for all compute fields and set them in the evaluator
// TODO provide the List to avoid array creation
List<String> values = extractValues(keySegments);
for (int i = 0; i < computedFieldNames.size(); i++) {
- evaluator.setValue(i, values.get(i));
+ if (evaluator.isComputedFieldUsed(i)) {
+ evaluator.setValue(i, values.get(i));
+ }
}
return evaluator.evaluate();
@@ -243,14 +269,60 @@
private List<String> extractValues(List<String> keySegments) {
List<String> values = new ArrayList<>();
- for (Integer computedFieldSegmentIndex : computedFieldSegmentIndexes) {
- values.add(keySegments.get(computedFieldSegmentIndex));
+ for (Map.Entry<Integer, PrefixSegment> entry : indexToComputedFieldsMap.entrySet()) {
+ int index = entry.getKey();
+ String expression = entry.getValue().getExpression();
+
+ String keySegment = keySegments.get(index);
+ Matcher matcher = Pattern.compile(expression).matcher(keySegment);
+
+ if (matcher.find()) {
+ for (int i = 1; i <= matcher.groupCount(); i++) {
+ values.add(matcher.group(i));
+ }
+ }
}
return values;
}
+ private IAType getUpdatedType(IAType type) {
+ switch (type.getTypeTag()) {
+ case TINYINT:
+ case SMALLINT:
+ case INTEGER:
+ return BuiltinType.AINT64;
+ default:
+ return type;
+ }
+ }
+
private static String getDefinitionOrPath(Map<String, String> configuration) {
return configuration.getOrDefault(DEFINITION_FIELD_NAME, configuration.get(KEY_PATH));
}
+
+ public static class PrefixSegment {
+ private String expression;
+ private final List<String> computedFieldNames = new ArrayList<>();
+ private final List<IAType> computedFieldTypes = new ArrayList<>();
+
+ public PrefixSegment() {
+ }
+
+ public String getExpression() {
+ return expression;
+ }
+
+ public List<String> getComputedFieldNames() {
+ return computedFieldNames;
+ }
+
+ public List<IAType> getComputedFieldTypes() {
+ return computedFieldTypes;
+ }
+
+ public void setExpression(String expression) {
+ this.expression = expression;
+ }
+ }
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
index 6775bf12b..a4f3a1e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/aws/s3/S3Utils.java
@@ -24,7 +24,25 @@
import static org.apache.asterix.common.exceptions.ErrorCode.S3_REGION_NOT_SUPPORTED;
import static org.apache.asterix.external.util.ExternalDataUtils.getPrefix;
import static org.apache.asterix.external.util.ExternalDataUtils.validateIncludeExclude;
-import static org.apache.asterix.external.util.aws.s3.S3Constants.*;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.ACCESS_KEY_ID_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_INTERNAL_ERROR;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_METHOD_NOT_IMPLEMENTED;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.ERROR_SLOW_DOWN;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_ACCESS_KEY_ID;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_ANONYMOUS_ACCESS;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_CREDENTIAL_PROVIDER_KEY;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_PATH_STYLE_ACCESS;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_S3_CONNECTION_POOL_SIZE;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_S3_PROTOCOL;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_SECRET_ACCESS_KEY;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_SERVICE_END_POINT;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_SESSION_TOKEN;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.HADOOP_TEMP_ACCESS;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.INSTANCE_PROFILE_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.REGION_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.SECRET_ACCESS_KEY_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.SERVICE_END_POINT_FIELD_NAME;
+import static org.apache.asterix.external.util.aws.s3.S3Constants.SESSION_TOKEN_FIELD_NAME;
import static org.apache.hyracks.api.util.ExceptionUtils.getMessageOrToString;
import java.net.URI;
@@ -40,9 +58,11 @@
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStreamFactory;
import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.external.util.ExternalDataPrefix;
import org.apache.asterix.external.util.HDFSUtils;
import org.apache.hadoop.fs.s3a.Constants;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.api.exceptions.Warning;
@@ -258,6 +278,12 @@
}
validateIncludeExclude(configuration);
+ try {
+ // TODO(htowaileb): maybe something better, this will check to ensure type is supported before creation
+ new ExternalDataPrefix(configuration);
+ } catch (AlgebricksException ex) {
+ throw new CompilationException(ErrorCode.FAILED_TO_CALCULATE_COMPUTED_FIELDS, ex);
+ }
// Check if the bucket is present
S3Client s3Client = buildAwsS3Client(configuration);