Merge branch 'gerrit/cheshire-cat'
Change-Id: I94f48e4cf9322d85743b1eb0f9d750502ee84e70
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
index 3c129b4..c064281 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
@@ -18,7 +18,7 @@
*/
package org.apache.asterix.test.external_dataset;
-import static org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.*;
+import static org.apache.asterix.test.external_dataset.aws.AwsS3ExternalDatasetTest.FIXED_DATA_CONTAINER;
import java.io.BufferedWriter;
import java.io.FileWriter;
@@ -344,4 +344,4 @@
mixedDataLoader.upload(MIXED_DEFINITION + "json/NO-EXTENSION/" + "goodbye-world-2019", "{\"id\":" + 5 + "}");
mixedDataLoader.upload(MIXED_DEFINITION + "json/NO-EXTENSION/" + "goodbye-world-2020", "{\"id\":" + 6 + "}");
}
-}
\ No newline at end of file
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
index 8372d87..2d41686 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
@@ -66,11 +66,13 @@
import com.azure.storage.blob.BlobContainerClient;
import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.blob.BlobServiceClientBuilder;
+import com.azure.storage.blob.models.PublicAccessType;
import com.azure.storage.common.sas.AccountSasPermission;
import com.azure.storage.common.sas.AccountSasResourceType;
import com.azure.storage.common.sas.AccountSasService;
import com.azure.storage.common.sas.AccountSasSignatureValues;
+// TODO(Hussain): Need to run the test manually to ensure new tests (anonymous access) are working fine
@Ignore
@RunWith(Parameterized.class)
@FixMethodOrder(MethodSorters.NAME_ASCENDING)
@@ -100,6 +102,7 @@
private static final String PLAYGROUND_CONTAINER = "playground";
private static final String FIXED_DATA_CONTAINER = "fixed-data"; // Do not use, has fixed data
private static final String INCLUDE_EXCLUDE_CONTAINER = "include-exclude";
+ private static final String PUBLIC_ACCESS_CONTAINER = "public-access-container"; // requires no authentication
private static final String JSON_DEFINITION = "json-data/reviews/";
private static final String CSV_DEFINITION = "csv-data/reviews/";
private static final String TSV_DEFINITION = "tsv-data/reviews/";
@@ -116,6 +119,7 @@
+ "BlobEndpoint=" + BLOB_SERVICE_ENDPOINT + "/devstoreaccount1;";
private static BlobServiceClient blobServiceClient;
private static BlobContainerClient playgroundContainer;
+ private static BlobContainerClient publicAccessContainer;
protected TestCaseContext tcCtx;
@@ -195,6 +199,8 @@
LOGGER.info("creating container " + PLAYGROUND_CONTAINER);
playgroundContainer = blobServiceClient.createBlobContainer(PLAYGROUND_CONTAINER);
+ publicAccessContainer = blobServiceClient.createBlobContainer(PUBLIC_ACCESS_CONTAINER);
+ publicAccessContainer.setAccessPolicy(PublicAccessType.CONTAINER, null);
LOGGER.info("container " + PLAYGROUND_CONTAINER + " created successfully");
LOGGER.info("Adding JSON files");
@@ -342,6 +348,7 @@
// Load the data
playgroundContainer.getBlobClient(basePath + finalFileName).uploadFromFile(filePath.toString());
+ publicAccessContainer.getBlobClient(basePath + finalFileName).uploadFromFile(filePath.toString());
if (copyToSubLevels) {
playgroundContainer.getBlobClient(basePath + "level1a/" + finalFileName)
.uploadFromFile(filePath.toString());
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-no-endpoint/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-no-endpoint/test.000.ddl.sqlpp
new file mode 100644
index 0000000..d6bc862
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-no-endpoint/test.000.ddl.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+// bad case: no auth method and no endpoint provided
+drop dataset test if exists;
+CREATE EXTERNAL DATASET test(test) USING AZUREBLOB (
+("container"="playground"),
+("definition"="json-data/reviews/single-line/json"),
+("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-no-endpoint/test.099.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-no-endpoint/test.099.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.000.ddl.sqlpp
new file mode 100644
index 0000000..598831e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.000.ddl.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+drop dataset test if exists;
+CREATE EXTERNAL DATASET test(test) USING AZUREBLOB (
+("blobEndpoint"="%azureblob-endpoint%"),
+("container"="public-access"),
+("definition"="json-data/reviews/single-line/json"),
+("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.001.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.001.query.sqlpp
new file mode 100644
index 0000000..8ec9cc0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.001.query.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+select count(*) `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.099.ddl.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/test.099.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.000.ddl.sqlpp
similarity index 100%
rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp
rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.000.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.001.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.001.query.sqlpp
new file mode 100644
index 0000000..8ec9cc0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.001.query.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+select count(*) `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.099.ddl.sqlpp
similarity index 100%
copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp
copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-not-allowed/test.099.ddl.sqlpp
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/result.001.adm
new file mode 100644
index 0000000..187a8cb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/azure_blob_storage/auth-methods/anonymous-no-auth-public-access-allowed/result.001.adm
@@ -0,0 +1 @@
+{ "count": 100 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
index fca1590..b732c55 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
@@ -68,9 +68,20 @@
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset/azure_blob_storage/auth-methods">
- <compilation-unit name="invalid-no-auth">
- <output-dir compare="Text">invalid-no-auth</output-dir>
- <expected-error>ASX1139: No authentication parameters provided</expected-error>
+ <compilation-unit name="anonymous-no-auth-public-access-allowed">
+ <output-dir compare="Text">anonymous-no-auth-public-access-allowed</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/azure_blob_storage/auth-methods">
+ <compilation-unit name="anonymous-no-auth-public-access-not-allowed">
+ <output-dir compare="Text">anonymous-no-auth-public-access-not-allowed</output-dir>
+ <expected-error>Server failed to authenticate the request. Make sure the value of the Authorization header is formed correctly including the signature.</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/azure_blob_storage/auth-methods">
+ <compilation-unit name="anonymous-no-auth-no-endpoint">
+ <output-dir compare="Text">anonymous-no-auth-no-endpoint</output-dir>
+ <expected-error>ASX1151: No authentication credentials provided, 'blobEndpoint' field is required for anonymous access</expected-error>
</compilation-unit>
</test-case>
</test-group>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 72ad331..02b5721 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -8000,14 +8000,14 @@
<test-case FilePath="array-index/error-handling">
<compilation-unit name="index-with-enforced-type">
<output-dir compare="Text">index-with-enforced-type</output-dir>
- <expected-error>ASX1154: Incompatible index type ARRAY</expected-error>
+ <expected-error>ASX1155: Incompatible index type ARRAY</expected-error>
<source-location>false</source-location>
</compilation-unit>
</test-case>
<test-case FilePath="array-index/error-handling">
<compilation-unit name="index-mixed-composite">
<output-dir compare="Text">index-mixed-composite</output-dir>
- <expected-error>ASX1154: Incompatible index type ARRAY</expected-error>
+ <expected-error>ASX1155: Incompatible index type ARRAY</expected-error>
<source-location>false</source-location>
</compilation-unit>
</test-case>
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index c87cb7f..1f25cc1 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -235,12 +235,13 @@
CANNOT_DROP_OBJECT_DEPENDENT_EXISTS(1148),
ILLEGAL_FUNCTION_RECURSION(1149),
ILLEGAL_FUNCTION_USE(1150),
- FULL_TEXT_CONFIG_NOT_FOUND(1151),
- FULL_TEXT_FILTER_NOT_FOUND(1152),
- FULL_TEXT_DEFAULT_CONFIG_CANNOT_BE_DELETED_OR_CREATED(1153),
- COMPILATION_INCOMPATIBLE_INDEX_TYPE(1154),
- FULL_TEXT_CONFIG_ALREADY_EXISTS(1155),
- FULL_TEXT_FILTER_ALREADY_EXISTS(1156),
+ NO_AUTH_PROVIDED_ENDPOINT_REQUIRED_FOR_ANONYMOUS_ACCESS(1151),
+ FULL_TEXT_CONFIG_NOT_FOUND(1152),
+ FULL_TEXT_FILTER_NOT_FOUND(1153),
+ FULL_TEXT_DEFAULT_CONFIG_CANNOT_BE_DELETED_OR_CREATED(1154),
+ COMPILATION_INCOMPATIBLE_INDEX_TYPE(1155),
+ FULL_TEXT_CONFIG_ALREADY_EXISTS(1156),
+ FULL_TEXT_FILTER_ALREADY_EXISTS(1157),
// Feed errors
DATAFLOW_ILLEGAL_STATE(3001),
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index f7c6a77..e68db6a 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -237,12 +237,13 @@
1148 = Cannot drop %1$s %2$s being used by %3$s %4$s
1149 = Illegal function recursion
1150 = Illegal use of function %1$s
-1151 = Full-text config %1$s not found
-1152 = Full-text filter %1$s not found
-1153 = Default full-text config with a name of null cannot be deleted or created
-1154 = Incompatible index type %1$s
-1155 = Full-text config %1$s already exists
-1156 = Full-text filter %1$s already exists
+1151 = No authentication credentials provided, '%1$s' field is required for anonymous access
+1152 = Full-text config %1$s not found
+1153 = Full-text filter %1$s not found
+1154 = Default full-text config with a name of null cannot be deleted or created
+1155 = Incompatible index type %1$s
+1156 = Full-text config %1$s already exists
+1157 = Full-text filter %1$s already exists
# Feed Errors
3001 = Illegal state.
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 8e94263..7872cef 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -899,7 +899,7 @@
String blobEndpoint = configuration.get(BLOB_ENDPOINT_FIELD_NAME);
String endpointSuffix = configuration.get(ENDPOINT_SUFFIX_FIELD_NAME);
- // Constructor the connection string
+ // Construct the connection string
// Connection string format: name1=value1;name2=value2;....
StringBuilder connectionStringBuilder = new StringBuilder();
BlobServiceClientBuilder builder = new BlobServiceClientBuilder();
@@ -926,17 +926,12 @@
if (authMethodFound) {
throw new CompilationException(ErrorCode.ONLY_SINGLE_AUTHENTICATION_IS_ALLOWED);
}
- authMethodFound = true;
// account name + shared access token
connectionStringBuilder.append(CONNECTION_STRING_ACCOUNT_NAME).append("=").append(accountName)
.append(";").append(CONNECTION_STRING_SHARED_ACCESS_SIGNATURE).append("=")
.append(sharedAccessSignature).append(";");
}
- if (!authMethodFound) {
- throw new CompilationException(ErrorCode.NO_AUTH_METHOD_PROVIDED);
- }
-
// Add blobEndpoint and endpointSuffix if present, adjust any '/' as needed
if (blobEndpoint != null) {
connectionStringBuilder.append(CONNECTION_STRING_BLOB_ENDPOINT).append("=").append(blobEndpoint)
@@ -954,6 +949,12 @@
}
}
+ // No credentials or endpoint provided
+ if (connectionStringBuilder.length() == 0) {
+ throw new CompilationException(ErrorCode.NO_AUTH_PROVIDED_ENDPOINT_REQUIRED_FOR_ANONYMOUS_ACCESS,
+ BLOB_ENDPOINT_FIELD_NAME);
+ }
+
try {
return builder.connectionString(connectionStringBuilder.toString()).buildClient();
} catch (Exception ex) {