[ASTERIXDB-3228][EXT]: Add utility to extract computed fields from external prefix
Change-Id: Id968a04f693a1a41bba61a3693bc2729bc885874
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17656
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hussainht@gmail.com>
Reviewed-by: Wail Alkowaileet <wael.y.k@gmail.com>
Tested-by: Hussain Towaileb <hussainht@gmail.com>
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/PrefixComputedFieldsTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/PrefixComputedFieldsTest.java
new file mode 100644
index 0000000..b2c405a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/PrefixComputedFieldsTest.java
@@ -0,0 +1,127 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.test.external_dataset;
+
+import static org.apache.asterix.om.types.BuiltinType.AINT32;
+import static org.apache.asterix.om.types.BuiltinType.ASTRING;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.asterix.external.util.ExternalDataPrefix;
+import org.junit.Test;
+
+import junit.framework.TestCase;
+
+public class PrefixComputedFieldsTest extends TestCase {
+
+ @Test
+ public void test() throws Exception {
+ ExternalDataPrefix prefix = new ExternalDataPrefix(null);
+ assertEquals("", prefix.getOriginal());
+ assertEquals("", prefix.getRoot());
+ assertFalse(prefix.isEndsWithSlash());
+ assertEquals(Collections.emptyList(), prefix.getSegments());
+ assertEquals(Collections.emptyList(), prefix.getComputedFieldDetails().getComputedFieldNames());
+ assertEquals(Collections.emptyList(), prefix.getComputedFieldDetails().getComputedFieldTypes());
+ assertEquals(Collections.emptyList(), prefix.getComputedFieldDetails().getComputedFieldIndexes());
+
+ String prefix1 = "";
+ prefix = new ExternalDataPrefix(prefix1);
+ assertEquals("", prefix.getOriginal());
+ assertEquals("", prefix.getRoot());
+ assertFalse(prefix.isEndsWithSlash());
+ assertEquals(Collections.emptyList(), prefix.getSegments());
+ assertEquals(Collections.emptyList(), prefix.getComputedFieldDetails().getComputedFieldNames());
+ assertEquals(Collections.emptyList(), prefix.getComputedFieldDetails().getComputedFieldTypes());
+ assertEquals(Collections.emptyList(), prefix.getComputedFieldDetails().getComputedFieldIndexes());
+
+ String prefix2 = "hotel";
+ prefix = new ExternalDataPrefix(prefix2);
+ assertEquals("hotel", prefix.getOriginal());
+ assertEquals("hotel", prefix.getRoot());
+ assertFalse(prefix.isEndsWithSlash());
+ assertEquals(List.of("hotel"), prefix.getSegments());
+ assertEquals(Collections.emptyList(), prefix.getComputedFieldDetails().getComputedFieldNames());
+ assertEquals(Collections.emptyList(), prefix.getComputedFieldDetails().getComputedFieldTypes());
+ assertEquals(Collections.emptyList(), prefix.getComputedFieldDetails().getComputedFieldIndexes());
+
+ String prefix3 = "hotel/{hotel-id:inT}/";
+ prefix = new ExternalDataPrefix(prefix3);
+ assertEquals("hotel/{hotel-id:inT}/", prefix.getOriginal());
+ assertEquals("hotel/", prefix.getRoot());
+ assertTrue(prefix.isEndsWithSlash());
+ assertEquals(List.of("hotel", "{hotel-id:inT}"), prefix.getSegments());
+ assertEquals(List.of(List.of("hotel-id")), prefix.getComputedFieldDetails().getComputedFieldNames());
+ assertEquals(List.of(AINT32), prefix.getComputedFieldDetails().getComputedFieldTypes());
+ assertEquals(List.of(1), prefix.getComputedFieldDetails().getComputedFieldIndexes());
+
+ String prefix4 = "hotel/{hotel-id:int}-{hotel-name:sTRing}";
+ prefix = new ExternalDataPrefix(prefix4);
+ assertEquals("hotel/{hotel-id:int}-{hotel-name:sTRing}", prefix.getOriginal());
+ assertEquals("hotel", prefix.getRoot());
+ assertFalse(prefix.isEndsWithSlash());
+ assertEquals(List.of("hotel", "{hotel-id:int}-{hotel-name:sTRing}"), prefix.getSegments());
+ assertEquals(List.of(List.of("hotel-id"), List.of("hotel-name")),
+ prefix.getComputedFieldDetails().getComputedFieldNames());
+ assertEquals(List.of(AINT32, ASTRING), prefix.getComputedFieldDetails().getComputedFieldTypes());
+ assertEquals(List.of(1, 1), prefix.getComputedFieldDetails().getComputedFieldIndexes());
+
+ String prefix5 = "hotel/something/{hotel-id:int}-{hotel-name:sTRing}/review/{year:int}-{month:int}-{day:int}/";
+ prefix = new ExternalDataPrefix(prefix5);
+ assertEquals("hotel/something/{hotel-id:int}-{hotel-name:sTRing}/review/{year:int}-{month:int}-{day:int}/",
+ prefix.getOriginal());
+ assertEquals("hotel/something/", prefix.getRoot());
+ assertTrue(prefix.isEndsWithSlash());
+ assertEquals(List.of("hotel", "something", "{hotel-id:int}-{hotel-name:sTRing}", "review",
+ "{year:int}-{month:int}-{day:int}"), prefix.getSegments());
+ assertEquals(
+ List.of(List.of("hotel-id"), List.of("hotel-name"), List.of("year"), List.of("month"), List.of("day")),
+ prefix.getComputedFieldDetails().getComputedFieldNames());
+ assertEquals(List.of(AINT32, ASTRING, AINT32, AINT32, AINT32),
+ prefix.getComputedFieldDetails().getComputedFieldTypes());
+ assertEquals(List.of(2, 2, 4, 4, 4), prefix.getComputedFieldDetails().getComputedFieldIndexes());
+
+ String prefix6 = "hotel/something/{hotel-id:int}-{hotel-name:sTRing}/review/{year:int}/{month:int}/{day:int}";
+ prefix = new ExternalDataPrefix(prefix6);
+ assertEquals("hotel/something/{hotel-id:int}-{hotel-name:sTRing}/review/{year:int}/{month:int}/{day:int}",
+ prefix.getOriginal());
+ assertEquals("hotel/something", prefix.getRoot());
+ assertFalse(prefix.isEndsWithSlash());
+ assertEquals(List.of("hotel", "something", "{hotel-id:int}-{hotel-name:sTRing}", "review", "{year:int}",
+ "{month:int}", "{day:int}"), prefix.getSegments());
+ assertEquals(
+ List.of(List.of("hotel-id"), List.of("hotel-name"), List.of("year"), List.of("month"), List.of("day")),
+ prefix.getComputedFieldDetails().getComputedFieldNames());
+ assertEquals(List.of(AINT32, ASTRING, AINT32, AINT32, AINT32),
+ prefix.getComputedFieldDetails().getComputedFieldTypes());
+ assertEquals(List.of(2, 2, 4, 5, 6), prefix.getComputedFieldDetails().getComputedFieldIndexes());
+
+ String prefix7 = "hotel/{hotel.details.id:int}-{hotel-name:sTRing}";
+ prefix = new ExternalDataPrefix(prefix7);
+ assertEquals("hotel/{hotel.details.id:int}-{hotel-name:sTRing}", prefix.getOriginal());
+ assertEquals("hotel", prefix.getRoot());
+ assertFalse(prefix.isEndsWithSlash());
+ assertEquals(List.of(List.of("hotel", "details", "id"), List.of("hotel-name")),
+ prefix.getComputedFieldDetails().getComputedFieldNames());
+ assertEquals(List.of(AINT32, ASTRING), prefix.getComputedFieldDetails().getComputedFieldTypes());
+ assertEquals(List.of(1, 1), prefix.getComputedFieldDetails().getComputedFieldIndexes());
+ }
+}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 2135c67..26e1bba 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -273,6 +273,7 @@
UNSUPPORTED_ICEBERG_TABLE(1178),
UNSUPPORTED_ICEBERG_FORMAT_VERSION(1179),
ERROR_READING_ICEBERG_METADATA(1180),
+ UNSUPPORTED_COMPUTED_FIELD_TYPE(1181),
// Feed errors
DATAFLOW_ILLEGAL_STATE(3001),
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index 3f54340..96c8843 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -275,6 +275,8 @@
1178 = Unsupported iceberg table
1179 = Unsupported iceberg format version
1180 = Error reading iceberg data
+1181 = Unsupported computed field type: %1$s
+
# Feed Errors
3001 = Illegal state.
3002 = Tuple is too large for a frame
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
index a241354..3043f7a 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
@@ -18,15 +18,21 @@
*/
package org.apache.asterix.external.input.record.reader.aws;
+import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
import java.util.PriorityQueue;
+import java.util.function.Supplier;
import org.apache.asterix.external.api.AsterixInputStream;
import org.apache.asterix.external.input.record.reader.abstracts.AbstractExternalInputStreamFactory;
+import org.apache.asterix.external.util.ExternalDataConstants;
+import org.apache.asterix.external.util.ExternalDataPrefix;
import org.apache.asterix.external.util.ExternalDataUtils;
import org.apache.asterix.external.util.aws.s3.S3Utils;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
@@ -54,11 +60,62 @@
IncludeExcludeMatcher includeExcludeMatcher = ExternalDataUtils.getIncludeExcludeMatchers(configuration);
//Get a list of S3 objects
+ String prefix = configuration.get(ExternalDataConstants.DEFINITION_FIELD_NAME);
+ ExternalDataPrefix externalDataPrefix = new ExternalDataPrefix(prefix);
+ configuration.put(ExternalDataPrefix.PREFIX_ROOT_FIELD_NAME, externalDataPrefix.getRoot());
+
+ // TODO(htowaileb): Since we're using the root to load the files then start filtering, it might end up being
+ // very expensive since at the root of the prefix we might load millions of files, we should consider (when
+ // possible) to get the value and add it
List<S3Object> filesOnly = S3Utils.listS3Objects(configuration, includeExcludeMatcher, warningCollector);
+
+ filesOnly = filterPrefixes(externalDataPrefix, filesOnly, () -> true);
+
// Distribute work load amongst the partitions
distributeWorkLoad(filesOnly, getPartitionsCount());
}
+ private List<S3Object> filterPrefixes(ExternalDataPrefix prefix, List<S3Object> filesOnly,
+ Supplier<Boolean> evaluator) {
+
+ // if no computed fields, return the original list
+ if (prefix.getComputedFieldDetails().isEmpty()) {
+ return filesOnly;
+ }
+
+ List<S3Object> filteredList = new ArrayList<>();
+ for (S3Object file : filesOnly) {
+ List<String> segments = ExternalDataPrefix.getPrefixSegments(file.key());
+ boolean match = false;
+
+ // if the object key has fewer segments than the expected prefix, then filter it out
+ // TODO(htowaileb): potentially also exclude if the size matches, key should be longer than prefix
+ if (segments.size() < prefix.getComputedFieldDetails().getComputedFieldNames().size()) {
+ continue;
+ }
+
+ for (int i = 0; i < prefix.getComputedFieldDetails().getComputedFieldNames().size(); i++) {
+ int index = prefix.getComputedFieldDetails().getComputedFieldIndexes().get(i);
+
+ // TODO(htowaileb): evaluator will container an expression that evaluates whether to include an object or not
+ match = evaluator.get();
+ if (!match) {
+ break;
+ }
+ }
+
+ if (match) {
+ filteredList.add(file);
+ }
+ }
+
+ return filteredList;
+ }
+
+ private ARecordType createRecord(String[] fieldNames, IAType[] fieldTypes) {
+ return new ARecordType("root", fieldNames, fieldTypes, false);
+ }
+
/**
* To efficiently utilize the parallelism, work load will be distributed amongst the partitions based on the file
* size.
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index 0080e9b..bc2ce63 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -24,6 +24,7 @@
import java.util.TimeZone;
import java.util.function.LongSupplier;
import java.util.function.Supplier;
+import java.util.regex.Pattern;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.hyracks.util.StorageUtil;
@@ -303,6 +304,8 @@
public static final String DEFINITION_FIELD_NAME = "definition";
public static final String CONTAINER_NAME_FIELD_NAME = "container";
public static final String SUBPATH = "subpath";
+ public static final String PREFIX_DEFAULT_DELIMITER = "/";
+ public static final Pattern COMPUTED_FIELD_PATTERN = Pattern.compile("\\{[^{}:]+:[^{}:]+}");
public static class ParquetOptions {
private ParquetOptions() {
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
new file mode 100644
index 0000000..c2a047b
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
@@ -0,0 +1,272 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.external.util;
+
+import static org.apache.asterix.external.util.ExternalDataConstants.COMPUTED_FIELD_PATTERN;
+import static org.apache.asterix.external.util.ExternalDataConstants.PREFIX_DEFAULT_DELIMITER;
+import static org.apache.asterix.om.utils.ProjectionFiltrationTypeUtil.getRecordTypeWithFieldTypes;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.regex.Matcher;
+
+import org.apache.asterix.common.exceptions.CompilationException;
+import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.BuiltinTypeMap;
+import org.apache.asterix.om.types.IAType;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
+
+public class ExternalDataPrefix {
+
+ private final String original;
+ private final String root;
+ private final boolean endsWithSlash;
+
+ private final List<String> segments;
+ private final ComputedFieldDetails computedFieldDetails;
+
+ public static final String PREFIX_ROOT_FIELD_NAME = "prefix-root";
+ public static final Set<IAType> supportedTypes = new HashSet<>();
+
+ static {
+ supportedTypes.add(BuiltinType.ASTRING);
+ supportedTypes.add(BuiltinType.AINT32);
+ }
+
+ public ExternalDataPrefix(String prefix) throws AlgebricksException {
+ this.original = prefix != null ? prefix : "";
+ this.endsWithSlash = this.original.endsWith("/");
+
+ this.segments = getPrefixSegments(this.original);
+
+ computedFieldDetails = getComputedFields(segments);
+ this.root = getPrefixRoot(segments, computedFieldDetails.getComputedFieldIndexes());
+ }
+
+ public String getOriginal() {
+ return original;
+ }
+
+ public String getRoot() {
+ return root;
+ }
+
+ public boolean isEndsWithSlash() {
+ return endsWithSlash;
+ }
+
+ public List<String> getSegments() {
+ return segments;
+ }
+
+ public ComputedFieldDetails getComputedFieldDetails() {
+ return computedFieldDetails;
+ }
+
+ /**
+ * returns the segments of a prefix, separated by the delimiter
+ *
+ * @param prefix prefix
+ * @return an array of prefix segments
+ */
+ public static List<String> getPrefixSegments(String prefix) {
+ return prefix.isEmpty() ? Collections.emptyList() : Arrays.asList(prefix.split(PREFIX_DEFAULT_DELIMITER));
+ }
+
+ /**
+ * Extracts and returns the computed fields and their indexes from the provided prefix
+ * @param prefix prefix
+ *
+ * @return Pair of computed field names and their segment index in the prefix
+ */
+ public static ComputedFieldDetails getComputedFields(String prefix) throws AlgebricksException {
+ List<String> segments = getPrefixSegments(prefix);
+ return getComputedFields(segments);
+ }
+
+ public static ComputedFieldDetails getComputedFields(List<String> segments) throws AlgebricksException {
+ List<List<String>> computedFieldsNames = new ArrayList<>();
+ List<IAType> computedFieldTypes = new ArrayList<>();
+ List<Integer> computedFieldIndexes = new ArrayList<>();
+
+ // check if there are any segments before doing any testing
+ if (segments.size() != 0) {
+ // search for computed fields in each segment
+ Matcher matcher = COMPUTED_FIELD_PATTERN.matcher(segments.get(0));
+ if (matcher.find()) {
+ String computedField = matcher.group();
+ String[] splits = computedField.split(":");
+ String namePart = splits[0].substring(1);
+ String typePart = splits[1].substring(0, splits[1].length() - 1);
+
+ IAType type = BuiltinTypeMap.getBuiltinType(typePart.substring(0, typePart.length() - 1));
+ validateSupported(type);
+
+ List<String> nameParts = List.of(namePart.substring(1, segments.indexOf(":") - 1).split("\\."));
+ computedFieldsNames.add(nameParts);
+ computedFieldTypes.add(type);
+ computedFieldIndexes.add(0);
+ }
+
+ if (segments.size() > 1) {
+ for (int i = 1; i < segments.size(); i++) {
+ matcher.reset(segments.get(i));
+
+ while (matcher.find()) {
+ String computedField = matcher.group();
+ String[] splits = computedField.split(":");
+ String namePart = splits[0].substring(1);
+ String typePart = splits[1].substring(0, splits[1].length() - 1);
+
+ IAType type = BuiltinTypeMap.getBuiltinType(typePart);
+ validateSupported(type);
+
+ List<String> nameParts = List.of(namePart.split("\\."));
+ computedFieldsNames.add(nameParts);
+ computedFieldTypes.add(type);
+ computedFieldIndexes.add(i);
+ }
+ }
+ }
+ }
+
+ return new ComputedFieldDetails(computedFieldsNames, computedFieldTypes, computedFieldIndexes);
+ }
+
+ private static void validateSupported(IAType type) throws CompilationException {
+ if (!isSupportedType(type)) {
+ throw new CompilationException(ErrorCode.UNSUPPORTED_COMPUTED_FIELD_TYPE, type);
+ }
+ }
+
+ /**
+ * Returns the longest static path (root) before encountering the first computed field
+ *
+ * @param prefix prefix
+ * @return prefix root
+ */
+ public String getPrefixRoot(String prefix) throws AlgebricksException {
+ List<String> prefixSegments = getPrefixSegments(prefix);
+ List<Integer> computedFieldIndexes = getComputedFields(prefix).getComputedFieldIndexes();
+ return getPrefixRoot(prefixSegments, computedFieldIndexes);
+ }
+
+ public String getPrefixRoot(List<String> prefixSegments, List<Integer> computedFieldIndexes) {
+ StringBuilder root = new StringBuilder();
+
+ // check if there are any computed fields before doing any testing
+ if (computedFieldIndexes.size() == 0) {
+ return this.original;
+ }
+
+ // construct all static parts before encountering the first computed field
+ for (int i = 0; i < computedFieldIndexes.get(0); i++) {
+ root.append(prefixSegments.get(i)).append("/");
+ }
+
+ // remove last "/" and append it only if needed
+ String finalRoot = root.toString();
+ finalRoot = finalRoot.substring(0, finalRoot.length() - 1);
+ return ExternalDataUtils.appendSlash(finalRoot, this.endsWithSlash);
+ }
+
+ /**
+ * Checks whether the provided type is in the supported types for dynamic prefixes
+ *
+ * @param type type to check
+ *
+ * @return true if type is supported, false otherwise
+ */
+ public static boolean isSupportedType(IAType type) {
+ return supportedTypes.contains(type);
+ }
+
+ public static class ComputedFieldDetails {
+ private final List<List<String>> computedFieldNames;
+ private final List<IAType> computedFieldTypes;
+ private final List<Integer> computedFieldIndexes;
+ private final Map<Integer, Pair<List<List<String>>, List<IAType>>> computedFields = new HashMap<>();
+ private final ARecordType recordType;
+
+ public ComputedFieldDetails(List<List<String>> computedFieldNames, List<IAType> computedFieldTypes,
+ List<Integer> computedFieldIndexes) throws AlgebricksException {
+ this.computedFieldNames = computedFieldNames;
+ this.computedFieldTypes = computedFieldTypes;
+ this.computedFieldIndexes = computedFieldIndexes;
+
+ this.recordType = getRecordTypeWithFieldTypes(computedFieldNames, computedFieldTypes);
+
+ for (int i = 0; i < computedFieldIndexes.size(); i++) {
+ int index = computedFieldIndexes.get(i);
+
+ if (computedFields.containsKey(index)) {
+ Pair<List<List<String>>, List<IAType>> pair = computedFields.get(index);
+ pair.getLeft().add(computedFieldNames.get(i));
+ pair.getRight().add(computedFieldTypes.get(i));
+ } else {
+ List<List<String>> names = new ArrayList<>();
+ List<IAType> types = new ArrayList<>();
+
+ names.add(computedFieldNames.get(i));
+ types.add(computedFieldTypes.get(i));
+ computedFields.put(index, Pair.of(names, types));
+ }
+ }
+ }
+
+ public boolean isEmpty() {
+ return computedFieldNames.isEmpty();
+ }
+
+ public List<List<String>> getComputedFieldNames() {
+ return computedFieldNames;
+ }
+
+ public List<IAType> getComputedFieldTypes() {
+ return computedFieldTypes;
+ }
+
+ public List<Integer> getComputedFieldIndexes() {
+ return computedFieldIndexes;
+ }
+
+ public ARecordType getRecordType() {
+ return recordType;
+ }
+
+ public Map<Integer, Pair<List<List<String>>, List<IAType>>> getComputedFields() {
+ return computedFields;
+ }
+
+ @Override
+ public String toString() {
+ return computedFields.toString();
+ }
+ }
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 794c18c..084e1ae 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -754,10 +754,11 @@
public static String getPrefix(Map<String, String> configuration, boolean appendSlash) {
String definition = configuration.get(ExternalDataConstants.DEFINITION_FIELD_NAME);
String subPath = configuration.get(ExternalDataConstants.SUBPATH);
+
boolean hasDefinition = definition != null && !definition.isEmpty();
boolean hasSubPath = subPath != null && !subPath.isEmpty();
if (hasDefinition && !hasSubPath) {
- return appendSlash ? definition + (!definition.endsWith("/") ? "/" : "") : definition;
+ return appendSlash(definition, appendSlash);
}
String fullPath = "";
if (hasSubPath) {
@@ -772,11 +773,15 @@
}
fullPath = definition + subPath;
}
- fullPath = appendSlash ? fullPath + (!fullPath.endsWith("/") ? "/" : "") : fullPath;
+ fullPath = appendSlash(fullPath, appendSlash);
}
return fullPath;
}
+ public static String appendSlash(String string, boolean appendSlash) {
+ return appendSlash ? string + (!string.endsWith("/") ? "/" : "") : string;
+ }
+
/**
* @param configuration configuration map
* @throws CompilationException Compilation exception