[ASTERIXDB-3269][EXT]: Handle root properly when computed field is at first segment
Change-Id: Idc97e6eef1d13953f16a37b340f4ba13983ecd74
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17806
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Hussain Towaileb <hussainht@gmail.com>
Reviewed-by: Ian Maxon <imaxon@uci.edu>
diff --git a/asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/bar-2023-01-01/data.json b/asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/bar-2023-01-01/data.json
new file mode 100644
index 0000000..4b16428
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/bar-2023-01-01/data.json
@@ -0,0 +1 @@
+{ "id": 2 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/foo-2023-01-01/data.json b/asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/foo-2023-01-01/data.json
new file mode 100644
index 0000000..7052c42
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/external-filter/computed-field-at-start/foo-2023-01-01/data.json
@@ -0,0 +1 @@
+{ "id": 1 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
index 0db9a2a..90f46ad 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
@@ -19,6 +19,7 @@
package org.apache.asterix.test.external_dataset;
import static org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.BOM_FILE_CONTAINER;
+import static org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.DYNAMIC_PREFIX_AT_START_CONTAINER;
import static org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.FIXED_DATA_CONTAINER;
import static org.apache.asterix.test.external_dataset.parquet.BinaryFileConverterUtil.BINARY_GEN_BASEDIR;
@@ -69,6 +70,7 @@
public static final int OVER_1000_OBJECTS_COUNT = 2999;
private static Uploader playgroundDataLoader;
+ private static Uploader dynamicPrefixAtStartDataLoader;
private static Uploader fixedDataLoader;
private static Uploader mixedDataLoader;
private static Uploader bomFileLoader;
@@ -118,9 +120,10 @@
TSV_DATA_PATH = tsvDataPath;
}
- public static void setUploaders(Uploader playgroundDataLoader, Uploader fixedDataLoader, Uploader mixedDataLoader,
- Uploader bomFileLoader) {
+ public static void setUploaders(Uploader playgroundDataLoader, Uploader dynamicPrefixAtStartDataLoader,
+ Uploader fixedDataLoader, Uploader mixedDataLoader, Uploader bomFileLoader) {
ExternalDatasetTestUtils.playgroundDataLoader = playgroundDataLoader;
+ ExternalDatasetTestUtils.dynamicPrefixAtStartDataLoader = dynamicPrefixAtStartDataLoader;
ExternalDatasetTestUtils.fixedDataLoader = fixedDataLoader;
ExternalDatasetTestUtils.mixedDataLoader = mixedDataLoader;
ExternalDatasetTestUtils.bomFileLoader = bomFileLoader;
@@ -158,6 +161,23 @@
}
/**
+ * Special container where dynamic prefix is the first segment
+ */
+ public static void prepareDynamicPrefixAtStartContainer() {
+ LOGGER.info("Loading dynamic prefix data to " + DYNAMIC_PREFIX_AT_START_CONTAINER);
+
+ // Files data
+ String path =
+ Paths.get(JSON_DATA_PATH, "external-filter", "computed-field-at-start", "foo-2023-01-01", "data.json")
+ .toString();
+ dynamicPrefixAtStartDataLoader.upload("foo-2023-01-01/data.json", path, true, false);
+
+ path = Paths.get(JSON_DATA_PATH, "external-filter", "computed-field-at-start", "bar-2023-01-01", "data.json")
+ .toString();
+ dynamicPrefixAtStartDataLoader.upload("bar-2023-01-01/data.json", path, true, false);
+ }
+
+ /**
* This bucket is being filled by fixed data, a test is counting all records in this bucket. If this bucket is
* changed, the test case will fail and its result will need to be updated each time
*/
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java
index c3f22a4..86d03a1 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetOnePartitionTest.java
@@ -44,6 +44,8 @@
ONLY_TESTS = "only_external_dataset.xml";
TEST_CONFIG_FILE_NAME = "src/test/resources/cc-single.conf";
PREPARE_BUCKET = AwsS3ExternalDatasetOnePartitionTest::prepareS3Bucket;
+ PREPARE_DYNAMIC_PREFIX_AT_START_BUCKET =
+ AwsS3ExternalDatasetOnePartitionTest::prepareDynamicPrefixAtStartContainer;
PREPARE_FIXED_DATA_BUCKET = AwsS3ExternalDatasetOnePartitionTest::prepareFixedDataBucket;
PREPARE_MIXED_DATA_BUCKET = AwsS3ExternalDatasetOnePartitionTest::prepareMixedDataBucket;
PREPARE_BOM_FILE_BUCKET = AwsS3ExternalDatasetOnePartitionTest::prepareBomDataBucket;
@@ -54,6 +56,9 @@
private static void prepareS3Bucket() {
}
+ private static void prepareDynamicPrefixAtStartContainer() {
+ }
+
private static void prepareFixedDataBucket() {
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
index 246ea13..532da56 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
@@ -115,6 +115,7 @@
static String ONLY_TESTS;
static String TEST_CONFIG_FILE_NAME;
static Runnable PREPARE_BUCKET;
+ static Runnable PREPARE_DYNAMIC_PREFIX_AT_START_BUCKET;
static Runnable PREPARE_FIXED_DATA_BUCKET;
static Runnable PREPARE_MIXED_DATA_BUCKET;
static Runnable PREPARE_BOM_FILE_BUCKET;
@@ -144,6 +145,7 @@
protected TestCaseContext tcCtx;
public static final String PLAYGROUND_CONTAINER = "playground";
+ public static final String DYNAMIC_PREFIX_AT_START_CONTAINER = "dynamic-prefix-at-start-container";
public static final String FIXED_DATA_CONTAINER = "fixed-data"; // Do not use, has fixed data
public static final String INCLUDE_EXCLUDE_CONTAINER = "include-exclude";
public static final String BOM_FILE_CONTAINER = "bom-file-container";
@@ -151,6 +153,8 @@
public static final PutObjectRequest.Builder playgroundBuilder =
PutObjectRequest.builder().bucket(PLAYGROUND_CONTAINER);
+ public static final PutObjectRequest.Builder dynamicPrefixAtStartBuilder =
+ PutObjectRequest.builder().bucket(DYNAMIC_PREFIX_AT_START_CONTAINER);
public static final PutObjectRequest.Builder fixedDataBuilder =
PutObjectRequest.builder().bucket(FIXED_DATA_CONTAINER);
public static final PutObjectRequest.Builder includeExcludeBuilder =
@@ -166,7 +170,6 @@
}
// iceberg
-
private static final Schema SCHEMA =
new Schema(required(1, "id", Types.IntegerType.get()), required(2, "data", Types.StringType.get()));
private static final Configuration CONF = new Configuration();
@@ -348,6 +351,7 @@
ONLY_TESTS = "only_external_dataset.xml";
TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf";
PREPARE_BUCKET = ExternalDatasetTestUtils::preparePlaygroundContainer;
+ PREPARE_DYNAMIC_PREFIX_AT_START_BUCKET = ExternalDatasetTestUtils::prepareDynamicPrefixAtStartContainer;
PREPARE_FIXED_DATA_BUCKET = ExternalDatasetTestUtils::prepareFixedDataContainer;
PREPARE_MIXED_DATA_BUCKET = ExternalDatasetTestUtils::prepareMixedDataContainer;
PREPARE_BOM_FILE_BUCKET = ExternalDatasetTestUtils::prepareBomFileContainer;
@@ -397,6 +401,7 @@
.endpointOverride(endpoint);
client = builder.build();
client.createBucket(CreateBucketRequest.builder().bucket(PLAYGROUND_CONTAINER).build());
+ client.createBucket(CreateBucketRequest.builder().bucket(DYNAMIC_PREFIX_AT_START_CONTAINER).build());
client.createBucket(CreateBucketRequest.builder().bucket(FIXED_DATA_CONTAINER).build());
client.createBucket(CreateBucketRequest.builder().bucket(INCLUDE_EXCLUDE_CONTAINER).build());
client.createBucket(CreateBucketRequest.builder().bucket(BOM_FILE_CONTAINER).build());
@@ -405,9 +410,11 @@
// Create the bucket and upload some json files
setDataPaths(JSON_DATA_PATH, CSV_DATA_PATH, TSV_DATA_PATH);
- setUploaders(AwsS3ExternalDatasetTest::loadPlaygroundData, AwsS3ExternalDatasetTest::loadFixedData,
+ setUploaders(AwsS3ExternalDatasetTest::loadPlaygroundData,
+ AwsS3ExternalDatasetTest::loadDynamicPrefixAtStartData, AwsS3ExternalDatasetTest::loadFixedData,
AwsS3ExternalDatasetTest::loadMixedData, AwsS3ExternalDatasetTest::loadBomData);
PREPARE_BUCKET.run();
+ PREPARE_DYNAMIC_PREFIX_AT_START_BUCKET.run();
PREPARE_FIXED_DATA_BUCKET.run();
PREPARE_MIXED_DATA_BUCKET.run();
PREPARE_BOM_FILE_BUCKET.run();
@@ -418,6 +425,10 @@
client.putObject(playgroundBuilder.key(key).build(), getRequestBody(content, fromFile, gzipped));
}
+ private static void loadDynamicPrefixAtStartData(String key, String content, boolean fromFile, boolean gzipped) {
+ client.putObject(dynamicPrefixAtStartBuilder.key(key).build(), getRequestBody(content, fromFile, gzipped));
+ }
+
private static void loadFixedData(String key, String content, boolean fromFile, boolean gzipped) {
client.putObject(fixedDataBuilder.key(key).build(), getRequestBody(content, fromFile, gzipped));
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetOnePartitionTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetOnePartitionTest.java
index 59c375a..9f9e783 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetOnePartitionTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetOnePartitionTest.java
@@ -41,6 +41,8 @@
ONLY_TESTS = "only_external_dataset.xml";
TEST_CONFIG_FILE_NAME = "src/test/resources/cc-single.conf";
PREPARE_PLAYGROUND_CONTAINER = AzureBlobStorageExternalDatasetOnePartitionTest::preparePlaygroundContainer;
+ PREPARE_DYNAMIC_PREFIX_AT_START_CONTAINER =
+ AzureBlobStorageExternalDatasetOnePartitionTest::prepareDynamicPrefixAtStartContainer;
PREPARE_FIXED_DATA_CONTAINER = AzureBlobStorageExternalDatasetOnePartitionTest::prepareFixedDataContainer;
PREPARE_INCLUDE_EXCLUDE_CONTAINER = AzureBlobStorageExternalDatasetOnePartitionTest::prepareMixedDataContainer;
PREPARE_BOM_FILE_BUCKET = AzureBlobStorageExternalDatasetOnePartitionTest::prepareBomDataContainer;
@@ -50,6 +52,9 @@
private static void preparePlaygroundContainer() {
}
+ private static void prepareDynamicPrefixAtStartContainer() {
+ }
+
private static void prepareFixedDataContainer() {
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
index 08f3816..9858e56 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
@@ -85,6 +85,7 @@
static String ONLY_TESTS;
static String TEST_CONFIG_FILE_NAME;
static Runnable PREPARE_PLAYGROUND_CONTAINER;
+ static Runnable PREPARE_DYNAMIC_PREFIX_AT_START_CONTAINER;
static Runnable PREPARE_FIXED_DATA_CONTAINER;
static Runnable PREPARE_INCLUDE_EXCLUDE_CONTAINER;
static Runnable PREPARE_BOM_FILE_BUCKET;
@@ -98,6 +99,7 @@
// Region, container and definitions
private static final String PLAYGROUND_CONTAINER = "playground";
+ private static final String DYNAMIC_PREFIX_AT_START_CONTAINER = "dynamic-prefix-at-start-container";
private static final String FIXED_DATA_CONTAINER = "fixed-data"; // Do not use, has fixed data
private static final String INCLUDE_EXCLUDE_CONTAINER = "include-exclude";
private static final String BOM_FILE_CONTAINER = "bom-file-container";
@@ -108,6 +110,7 @@
// Create a BlobServiceClient object which will be used to create a container client
private static BlobServiceClient blobServiceClient;
private static BlobContainerClient playgroundContainer;
+ private static BlobContainerClient dynamicPrefixAtStartContainer;
private static BlobContainerClient publicAccessContainer;
private static BlobContainerClient fixedDataContainer;
private static BlobContainerClient mixedDataContainer;
@@ -140,6 +143,7 @@
ONLY_TESTS = "only_external_dataset.xml";
TEST_CONFIG_FILE_NAME = "src/main/resources/cc.conf";
PREPARE_PLAYGROUND_CONTAINER = ExternalDatasetTestUtils::preparePlaygroundContainer;
+ PREPARE_DYNAMIC_PREFIX_AT_START_CONTAINER = ExternalDatasetTestUtils::prepareDynamicPrefixAtStartContainer;
PREPARE_FIXED_DATA_CONTAINER = ExternalDatasetTestUtils::prepareFixedDataContainer;
PREPARE_INCLUDE_EXCLUDE_CONTAINER = ExternalDatasetTestUtils::prepareMixedDataContainer;
PREPARE_BOM_FILE_BUCKET = ExternalDatasetTestUtils::prepareBomFileContainer;
@@ -177,6 +181,7 @@
LOGGER.info("Creating containers");
playgroundContainer = blobServiceClient.createBlobContainer(PLAYGROUND_CONTAINER);
+ dynamicPrefixAtStartContainer = blobServiceClient.createBlobContainer(DYNAMIC_PREFIX_AT_START_CONTAINER);
fixedDataContainer = blobServiceClient.createBlobContainer(FIXED_DATA_CONTAINER);
mixedDataContainer = blobServiceClient.createBlobContainer(INCLUDE_EXCLUDE_CONTAINER);
bomContainer = blobServiceClient.createBlobContainer(BOM_FILE_CONTAINER);
@@ -195,9 +200,11 @@
// Create the bucket and upload some json files
setDataPaths(JSON_DATA_PATH, CSV_DATA_PATH, TSV_DATA_PATH);
setUploaders(AzureBlobStorageExternalDatasetTest::loadPlaygroundData,
+ AzureBlobStorageExternalDatasetTest::loadDynamicPrefixAtStartData,
AzureBlobStorageExternalDatasetTest::loadFixedData, AzureBlobStorageExternalDatasetTest::loadMixedData,
AzureBlobStorageExternalDatasetTest::loadBomData);
PREPARE_PLAYGROUND_CONTAINER.run();
+ PREPARE_DYNAMIC_PREFIX_AT_START_CONTAINER.run();
PREPARE_FIXED_DATA_CONTAINER.run();
PREPARE_INCLUDE_EXCLUDE_CONTAINER.run();
PREPARE_BOM_FILE_BUCKET.run();
@@ -240,6 +247,35 @@
}
}
+ private static void loadDynamicPrefixAtStartData(String key, String content, boolean fromFile, boolean gzipped) {
+ if (!fromFile) {
+ try (ByteArrayInputStream inputStream = new ByteArrayInputStream(content.getBytes())) {
+ dynamicPrefixAtStartContainer.getBlobClient(key).upload(inputStream, inputStream.available());
+ } catch (IOException ex) {
+ throw new IllegalArgumentException(ex.toString());
+ }
+ } else {
+ if (!gzipped) {
+ dynamicPrefixAtStartContainer.getBlobClient(key).uploadFromFile(content);
+ } else {
+ try (ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+ GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream)) {
+ gzipOutputStream.write(Files.readAllBytes(Paths.get(content)));
+ gzipOutputStream.close(); // Need to close or data will be invalid
+ byte[] gzipBytes = byteArrayOutputStream.toByteArray();
+
+ try (ByteArrayInputStream inputStream = new ByteArrayInputStream(gzipBytes)) {
+ dynamicPrefixAtStartContainer.getBlobClient(key).upload(inputStream, inputStream.available());
+ } catch (IOException ex) {
+ throw new IllegalArgumentException(ex.toString());
+ }
+ } catch (IOException ex) {
+ throw new IllegalArgumentException(ex.toString());
+ }
+ }
+ }
+ }
+
private static void loadFixedData(String key, String content, boolean fromFile, boolean gzipped) {
if (!fromFile) {
try (ByteArrayInputStream inputStream = new ByteArrayInputStream(content.getBytes())) {
@@ -417,6 +453,7 @@
private static void deleteContainersSilently() {
deleteContainerSilently(PLAYGROUND_CONTAINER);
+ deleteContainerSilently(DYNAMIC_PREFIX_AT_START_CONTAINER);
deleteContainerSilently(FIXED_DATA_CONTAINER);
deleteContainerSilently(PUBLIC_ACCESS_CONTAINER);
deleteContainerSilently(INCLUDE_EXCLUDE_CONTAINER);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.000.ddl.sqlpp
new file mode 100644
index 0000000..cb78e58
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.000.ddl.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE test AS {
+};
+
+CREATE EXTERNAL DATASET test1(test) USING %adapter% (
+ %template%,
+ ("container"="dynamic-prefix-at-start-container"),
+ ("definition"="foo-{year:int}-{month:int}-{day:int}/"),
+ ("embed-filter-values" = "true"),
+ ("format"="json")
+);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.010.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.010.query.sqlpp
new file mode 100644
index 0000000..7d47884
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.010.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// param max-warnings:json=10
+
+USE test;
+
+SELECT value t
+FROM test1 t
+order by t.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.999.ddl.sqlpp
new file mode 100644
index 0000000..36b2bab
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/computed-field-at-start/test.999.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.000.ddl.sqlpp
new file mode 100644
index 0000000..3f09568
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.000.ddl.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE test AS {
+};
+
+CREATE EXTERNAL DATASET test1(test) USING %adapter% (
+ %template%,
+ ("container"="playground"),
+ ("definition"="parquet-data/foo-{year:int}-{month:int}-{day:int}/"),
+ ("embed-filter-values" = "true"),
+ ("format"="parquet")
+);
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.010.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.010.query.sqlpp
new file mode 100644
index 0000000..7d47884
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.010.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// param max-warnings:json=10
+
+USE test;
+
+SELECT value t
+FROM test1 t
+order by t.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.999.ddl.sqlpp
new file mode 100644
index 0000000..36b2bab
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/test.999.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/computed-field-at-start/result.010.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/computed-field-at-start/result.010.adm
new file mode 100644
index 0000000..841ede8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/computed-field-at-start/result.010.adm
@@ -0,0 +1 @@
+{ "id": 1, "month": 1, "year": 2023, "day": 1 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/result.010.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/result.010.adm
new file mode 100644
index 0000000..8c86668
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/computed-field-at-start/result.010.adm
@@ -0,0 +1 @@
+{ "id": 2, "month": 1, "year": 2023, "day": 1 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
index 55764ed..5809912 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
@@ -302,6 +302,12 @@
<output-dir compare="Text">computed-field-segment-pattern-mismatch</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-dataset/common/dynamic-prefixes">
+ <compilation-unit name="computed-field-at-start">
+ <placeholder name="adapter" value="AZUREBLOB" />
+ <output-dir compare="Text">computed-field-at-start</output-dir>
+ </compilation-unit>
+ </test-case>
<!--
<test-case FilePath="external-dataset/common/dynamic-prefixes/parquet">
<compilation-unit name="computed-field-segment-pattern-mismatch">
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 623e7bc..54758e3 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -274,6 +274,12 @@
<output-dir compare="Text">computed-field-segment-pattern-mismatch</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="external-dataset/common/dynamic-prefixes">
+ <compilation-unit name="computed-field-at-start">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">computed-field-at-start</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="external-dataset/common/dynamic-prefixes/parquet">
<compilation-unit name="one-field">
<placeholder name="adapter" value="S3" />
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
index 299d0e4..2edf326 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
@@ -210,7 +210,7 @@
// remove last "/" and append it only if needed
root = builder.toString();
- root = root.substring(0, root.length() - 1);
+ root = root.isEmpty() ? root : root.substring(0, root.length() - 1);
root = ExternalDataUtils.appendSlash(root, endsWithSlash);
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 6902175..d282ab4 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -762,7 +762,7 @@
String definition = configuration.get(ExternalDataConstants.DEFINITION_FIELD_NAME);
String subPath = configuration.get(ExternalDataConstants.SUBPATH);
- boolean hasRoot = root != null && !root.isEmpty();
+ boolean hasRoot = root != null;
boolean hasDefinition = definition != null && !definition.isEmpty();
boolean hasSubPath = subPath != null && !subPath.isEmpty();
@@ -794,7 +794,7 @@
}
public static String appendSlash(String string, boolean appendSlash) {
- return appendSlash ? string + (!string.endsWith("/") ? "/" : "") : string;
+ return appendSlash && !string.isEmpty() ? string + (!string.endsWith("/") ? "/" : "") : string;
}
/**