Merge branch 'gerrit/cheshire-cat'
Change-Id: Ida0aef82a8fc4553bed96ceed230ce97af2c6f3f
diff --git a/asterixdb/NOTICE b/asterixdb/NOTICE
index b4729a8..4aabe27 100644
--- a/asterixdb/NOTICE
+++ b/asterixdb/NOTICE
@@ -1,5 +1,5 @@
Apache AsterixDB
-Copyright 2015-2020 The Apache Software Foundation
+Copyright 2015-2021 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
index f7cbb9d..02dc6f6 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/subplan/InlineAllNtsInSubplanVisitor.java
@@ -655,7 +655,17 @@
@Override
public ILogicalOperator visitWindowOperator(WindowOperator op, Void arg) throws AlgebricksException {
- return visitSingleInputOperator(op);
+ visitSingleInputOperator(op);
+ List<LogicalVariable> partitionByVars = op.getPartitionVarList();
+ for (LogicalVariable keyVar : correlatedKeyVars) {
+ if (!partitionByVars.contains(keyVar)) {
+ VariableReferenceExpression keyVarRef = new VariableReferenceExpression(keyVar);
+ keyVarRef.setSourceLocation(op.getSourceLocation());
+ op.getPartitionExpressions().add(new MutableObject<>(keyVarRef));
+ }
+ }
+ context.computeAndSetTypeEnvironmentForOperator(op);
+ return op;
}
/**
diff --git a/asterixdb/asterix-app/pom.xml b/asterixdb/asterix-app/pom.xml
index 8b5fed1..da9d881 100644
--- a/asterixdb/asterix-app/pom.xml
+++ b/asterixdb/asterix-app/pom.xml
@@ -854,6 +854,10 @@
<artifactId>azure-storage-blob</artifactId>
</dependency>
<dependency>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-storage-common</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-1.2-api</artifactId>
</dependency>
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java
index ef2ec6c..86bffd2 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestConstants.java
@@ -36,14 +36,37 @@
+ "\"),\n" + "(\"serviceEndpoint\"=\"" + S3_SERVICE_ENDPOINT_DEFAULT + "\")";
// Azure blob storage constants and place holders
- public static final String AZURE_ACCOUNT_NAME_PLACEHOLDER = "%accountName%";
+ // account name
+ public static final String AZURE_ACCOUNT_NAME_PLACEHOLDER = "%azureblob-accountname%";
public static final String AZURE_AZURITE_ACCOUNT_NAME_DEFAULT = "devstoreaccount1";
- public static final String AZURE_ACCOUNT_KEY_PLACEHOLDER = "%accountKey%";
+
+ // account key
+ public static final String AZURE_ACCOUNT_KEY_PLACEHOLDER = "%azureblob-accountkey%";
public static final String AZURE_AZURITE_ACCOUNT_KEY_DEFAULT =
- "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsu" + "Fq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";
- public static final String AZURE_BLOB_ENDPOINT_PLACEHOLDER = "%blobEndpoint%";
+ "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";
+
+ // SAS token: this is generated and assigned at runtime at the start of the test
+ public static final String AZURE_SAS_TOKEN_PLACEHOLDER = "%azureblob-sas%";
+ public static String sasToken = "";
+
+ // blob endpoint
+ public static final String AZURE_BLOB_ENDPOINT_PLACEHOLDER = "%azureblob-endpoint%";
public static final String AZURE_BLOB_ENDPOINT_DEFAULT =
"http://localhost:20000/" + AZURE_AZURITE_ACCOUNT_NAME_DEFAULT;
+
+ // connection string with account name & account key
+ public static final String AZURE_CONNECTION_STRING_ACCOUNT_KEY_PLACEHOLDER =
+ "%azureblob-connectionstringaccountkey%";
+ public static final String AZURE_CONNECTION_STRING_ACCOUNT_KEY = "AccountName=" + AZURE_ACCOUNT_NAME_PLACEHOLDER
+ + ";AccountKey=" + AZURE_ACCOUNT_KEY_PLACEHOLDER + ";BlobEndpoint=" + AZURE_BLOB_ENDPOINT_PLACEHOLDER;
+
+ // connection string with account name & sas token
+ public static final String AZURE_CONNECTION_STRING_SAS_TOKEN_PLACEHOLDER = "%azureblob-connectionstringsas%";
+ public static final String AZURE_CONNECTION_STRING_SAS_TOKEN =
+ "AccountName=" + AZURE_ACCOUNT_NAME_PLACEHOLDER + ";SharedAccessSignature=" + AZURE_SAS_TOKEN_PLACEHOLDER
+ + ";BlobEndpoint=" + AZURE_BLOB_ENDPOINT_PLACEHOLDER;
+
+ // azure template and default template
public static final String AZURE_TEMPLATE = "(\"accountName\"=\"" + AZURE_AZURITE_ACCOUNT_NAME_DEFAULT + "\"),\n"
+ "(\"accountKey\"=\"" + AZURE_AZURITE_ACCOUNT_KEY_DEFAULT + "\"),\n" + "(\"blobEndpoint\"=\""
+ AZURE_BLOB_ENDPOINT_PLACEHOLDER + "\")";
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 00d1034..e202ddf 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -2068,28 +2068,46 @@
}
protected String applyExternalDatasetSubstitution(String str, List<Placeholder> placeholders) {
+ // This replaces the full template of parameters depending on the adapter type
for (Placeholder placeholder : placeholders) {
+ // For adapter placeholder, it means we have a template to replace
if (placeholder.getName().equals("adapter")) {
str = str.replace("%adapter%", placeholder.getValue());
// Early terminate if there are no template place holders to replace
if (noTemplateRequired(str)) {
- return str;
+ continue;
}
if (placeholder.getValue().equalsIgnoreCase("S3")) {
- return applyS3Substitution(str, placeholders);
+ str = applyS3Substitution(str, placeholders);
} else if (placeholder.getValue().equalsIgnoreCase("AzureBlob")) {
- return applyAzureSubstitution(str, placeholders);
- } else {
- return str;
+ str = applyAzureSubstitution(str, placeholders);
}
+ } else {
+ // Any other place holders, just replace with the value
+ str = str.replace("%" + placeholder.getName() + "%", placeholder.getValue());
}
}
+ // This replaces specific external dataset placeholders
+ str = str.replace(TestConstants.AZURE_CONNECTION_STRING_ACCOUNT_KEY_PLACEHOLDER,
+ TestConstants.AZURE_CONNECTION_STRING_ACCOUNT_KEY);
+ str = str.replace(TestConstants.AZURE_CONNECTION_STRING_SAS_TOKEN_PLACEHOLDER,
+ TestConstants.AZURE_CONNECTION_STRING_SAS_TOKEN);
+ str = str.replace(TestConstants.AZURE_ACCOUNT_NAME_PLACEHOLDER,
+ TestConstants.AZURE_AZURITE_ACCOUNT_NAME_DEFAULT);
+ str = str.replace(TestConstants.AZURE_ACCOUNT_KEY_PLACEHOLDER, TestConstants.AZURE_AZURITE_ACCOUNT_KEY_DEFAULT);
+ str = str.replace(TestConstants.AZURE_SAS_TOKEN_PLACEHOLDER, TestConstants.sasToken);
+ str = replaceExternalEndpoint(str);
+
return str;
}
+ protected String replaceExternalEndpoint(String str) {
+ return str.replace(TestConstants.AZURE_BLOB_ENDPOINT_PLACEHOLDER, TestConstants.AZURE_BLOB_ENDPOINT_DEFAULT);
+ }
+
protected boolean noTemplateRequired(String str) {
return !str.contains("%template%");
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
index da9f912..27a46c1 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/microsoft/AzureBlobStorageExternalDatasetTest.java
@@ -29,6 +29,8 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.time.OffsetDateTime;
+import java.time.temporal.ChronoUnit;
import java.util.BitSet;
import java.util.Collection;
import java.util.HashMap;
@@ -39,6 +41,7 @@
import java.util.zip.GZIPOutputStream;
import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.test.common.TestConstants;
import org.apache.asterix.test.common.TestExecutor;
import org.apache.asterix.test.runtime.ExecutionTestUtil;
import org.apache.asterix.test.runtime.LangExecutionUtil;
@@ -64,6 +67,10 @@
import com.azure.storage.blob.BlobContainerClient;
import com.azure.storage.blob.BlobServiceClient;
import com.azure.storage.blob.BlobServiceClientBuilder;
+import com.azure.storage.common.sas.AccountSasPermission;
+import com.azure.storage.common.sas.AccountSasResourceType;
+import com.azure.storage.common.sas.AccountSasService;
+import com.azure.storage.common.sas.AccountSasSignatureValues;
@Ignore
@RunWith(Parameterized.class)
@@ -164,12 +171,23 @@
blobServiceClient = new BlobServiceClientBuilder().connectionString(connectionString).buildClient();
LOGGER.info("Azurite Blob Service client created successfully");
+ // Generate the SAS token for the SAS test cases
+ TestConstants.sasToken = generateSasToken();
+
// Create the container and upload some json files
PREPARE_PLAYGROUND_CONTAINER.run();
PREPARE_FIXED_DATA_CONTAINER.run();
PREPARE_MIXED_DATA_CONTAINER.run();
}
+ private static String generateSasToken() {
+ OffsetDateTime expiry = OffsetDateTime.now().plus(1, ChronoUnit.YEARS);
+ AccountSasService service = AccountSasService.parse("b");
+ AccountSasPermission permission = AccountSasPermission.parse("acdlpruw");
+ AccountSasResourceType type = AccountSasResourceType.parse("cos");
+ return blobServiceClient.generateAccountSas(new AccountSasSignatureValues(expiry, permission, service, type));
+ }
+
/**
* Creates a container and fills it with some files for testing purpose.
*/
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/subquery/query-ASTERIXDB-2815-2.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/subquery/query-ASTERIXDB-2815-2.sqlpp
new file mode 100644
index 0000000..74034d6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/subquery/query-ASTERIXDB-2815-2.sqlpp
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type t1 as {
+ _id: uuid
+};
+
+create dataset RawTweet(t1) primary key _id autogenerated;
+
+create dataset Evidence(t1) primary key _id autogenerated;
+
+create dataset Verification(t1) primary key _id autogenerated;
+
+select t.id, array_sort(ranks) ranks
+from RawTweet t
+let ranks = (
+ select value rank() over(order by e.url)
+ from Verification v, v.evidence ve, Evidence e
+ where t.id = v.tweet_id and ve = e.ev_id
+)
+order by t.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/queries/subquery/query-ASTERIXDB-2815-3.sqlpp b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/subquery/query-ASTERIXDB-2815-3.sqlpp
new file mode 100644
index 0000000..6b0501e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/queries/subquery/query-ASTERIXDB-2815-3.sqlpp
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type t1 as {
+ _id: uuid
+};
+
+create dataset RawTweet(t1) primary key _id autogenerated;
+
+create dataset Evidence(t1) primary key _id autogenerated;
+
+create dataset Verification(t1) primary key _id autogenerated;
+
+select t.id, array_sort(ranks) ranks
+from RawTweet t
+let ranks = (
+ select value rank() over(
+ partition by (tobigint(substring(e.url, -4)) % 2)
+ order by e.url
+ )
+ from Verification v, v.evidence ve, Evidence e
+ where t.id = v.tweet_id and ve = e.ev_id
+)
+order by t.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/query-ASTERIXDB-2815-2.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/query-ASTERIXDB-2815-2.plan
new file mode 100644
index 0000000..897163e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/query-ASTERIXDB-2815-2.plan
@@ -0,0 +1,72 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$82(ASC) ] |PARTITIONED|
+ -- STABLE_SORT [$$82(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$78] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$78][$$89] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.RawTweet) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- WINDOW_STREAM |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$89(ASC), $$e.url(ASC)] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$89] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$ve][$$81] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$ve] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- UNNEST |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$87][$$83] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$87] |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.RawTweet) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$83] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.Verification) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$81] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.Evidence) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/query-ASTERIXDB-2815-3.plan b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/query-ASTERIXDB-2815-3.plan
new file mode 100644
index 0000000..6ca6f1c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/optimizerts/results/subquery/query-ASTERIXDB-2815-3.plan
@@ -0,0 +1,73 @@
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- SORT_MERGE_EXCHANGE [$$88(ASC) ] |PARTITIONED|
+ -- STABLE_SORT [$$88(ASC)] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- PRE_CLUSTERED_GROUP_BY[$$84] |PARTITIONED|
+ {
+ -- AGGREGATE |LOCAL|
+ -- STREAM_SELECT |LOCAL|
+ -- NESTED_TUPLE_SOURCE |LOCAL|
+ }
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$84][$$95] |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.RawTweet) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$95] |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- WINDOW_STREAM |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STABLE_SORT [$$72(ASC), $$95(ASC), $$83(ASC)] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$72, $$95] |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$ve][$$87] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$ve] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- UNNEST |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- HYBRID_HASH_JOIN [$$93][$$89] |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$93] |PARTITIONED|
+ -- REPLICATE |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.RawTweet) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$89] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.Verification) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ -- HASH_PARTITION_EXCHANGE [$$87] |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ASSIGN |PARTITIONED|
+ -- STREAM_PROJECT |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- DATASOURCE_SCAN (test.Evidence) |PARTITIONED|
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-auth-methods/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-auth-methods/test.000.ddl.sqlpp
new file mode 100644
index 0000000..b3c8bc4
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-auth-methods/test.000.ddl.sqlpp
@@ -0,0 +1,38 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+// bad case: more than one authentication method is provided at once
+drop dataset test if exists;
+CREATE EXTERNAL DATASET test(test) USING AZUREBLOB (
+("accountName"="%azureblob-accountname%"),
+("%azureblob-credentialsname-1%"="%azureblob-credentialsvalue-1%"),
+("%azureblob-credentialsname-2%"="%azureblob-credentialsvalue-2%"),
+("blobEndpoint"="%azureblob-endpoint%"),
+("container"="playground"),
+("definition"="json-data/reviews/single-line/json"),
+("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-auth-methods/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-auth-methods/test.099.ddl.sqlpp
new file mode 100644
index 0000000..548e632
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-auth-methods/test.099.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp
new file mode 100644
index 0000000..43db107
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.000.ddl.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+// bad case: no auth method is provided
+drop dataset test if exists;
+CREATE EXTERNAL DATASET test(test) USING AZUREBLOB (
+("blobEndpoint"="%azureblob-endpoint%"),
+("container"="playground"),
+("definition"="json-data/reviews/single-line/json"),
+("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp
new file mode 100644
index 0000000..548e632
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/invalid-no-auth/test.099.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.000.ddl.sqlpp
new file mode 100644
index 0000000..afeaeae
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.000.ddl.sqlpp
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+};
+
+drop dataset test if exists;
+CREATE EXTERNAL DATASET test(test) USING AZUREBLOB (
+("accountName"="%azureblob-accountname%"),
+("%azureblob-credentialsname%"="%azureblob-credentialsvalue%"),
+("blobEndpoint"="%azureblob-endpoint%"),
+("container"="playground"),
+("definition"="json-data/reviews/single-line/json"),
+("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.001.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.001.query.sqlpp
new file mode 100644
index 0000000..8ec9cc0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.001.query.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+select count(*) `count` from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.099.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.099.ddl.sqlpp
new file mode 100644
index 0000000..548e632
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/test.099.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/subquery/query-ASTERIXDB-2815/query-ASTERIXDB-2815.4.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/subquery/query-ASTERIXDB-2815/query-ASTERIXDB-2815.4.query.sqlpp
new file mode 100644
index 0000000..d94a669
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/subquery/query-ASTERIXDB-2815/query-ASTERIXDB-2815.4.query.sqlpp
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test decorrelation of window functions
+ */
+
+use test;
+
+select t.id, array_sort(ranks) ranks
+from RawTweet t
+let ranks = (
+ select value rank() over(order by e.url)
+ from Verification v, v.evidence ve, Evidence e
+ where t.id = v.tweet_id and ve = e.ev_id
+)
+order by t.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/subquery/query-ASTERIXDB-2815/query-ASTERIXDB-2815.5.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/subquery/query-ASTERIXDB-2815/query-ASTERIXDB-2815.5.query.sqlpp
new file mode 100644
index 0000000..3bec7f9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/subquery/query-ASTERIXDB-2815/query-ASTERIXDB-2815.5.query.sqlpp
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test decorrelation of window functions
+ */
+
+use test;
+
+select t.id, array_sort(ranks) ranks
+from RawTweet t
+let ranks = (
+ select value rank() over(
+ partition by (tobigint(substring(e.url, -4)) % 2)
+ order by e.url
+ )
+ from Verification v, v.evidence ve, Evidence e
+ where t.id = v.tweet_id and ve = e.ev_id
+)
+order by t.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/result.001.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/result.001.adm
new file mode 100644
index 0000000..187a8cb
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/azure_blob_storage/auth-methods/valid-auth-methods/result.001.adm
@@ -0,0 +1 @@
+{ "count": 100 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/subquery/query-ASTERIXDB-2815/query-ASTERIXDB-2815.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/subquery/query-ASTERIXDB-2815/query-ASTERIXDB-2815.4.adm
new file mode 100644
index 0000000..c35bf4e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/subquery/query-ASTERIXDB-2815/query-ASTERIXDB-2815.4.adm
@@ -0,0 +1,5 @@
+{ "ranks": [ 1, 2 ], "id": 1 }
+{ "ranks": [ 1, 2, 3, 4 ], "id": 2 }
+{ "ranks": [ ], "id": 3 }
+{ "ranks": [ 1, 2, 3, 4, 5, 6 ], "id": 4 }
+{ "ranks": [ 1 ], "id": 5 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/subquery/query-ASTERIXDB-2815/query-ASTERIXDB-2815.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/subquery/query-ASTERIXDB-2815/query-ASTERIXDB-2815.5.adm
new file mode 100644
index 0000000..9ddd8aa
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/subquery/query-ASTERIXDB-2815/query-ASTERIXDB-2815.5.adm
@@ -0,0 +1,5 @@
+{ "ranks": [ 1, 1 ], "id": 1 }
+{ "ranks": [ 1, 1, 2, 2 ], "id": 2 }
+{ "ranks": [ ], "id": 3 }
+{ "ranks": [ 1, 1, 2, 2, 3, 3 ], "id": 4 }
+{ "ranks": [ 1 ], "id": 5 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
index a715e7e..c6bad70 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_azure_blob_storage.xml
@@ -18,6 +18,62 @@
! under the License.
!-->
<test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" QueryOffsetPath="queries_sqlpp" QueryFileExtension=".sqlpp">
+ <test-group name="authentication">
+ <test-case FilePath="external-dataset/azure_blob_storage/auth-methods">
+ <compilation-unit name="valid-auth-methods">
+ <placeholder name="azureblob-credentialsname" value="accountKey" />
+ <placeholder name="azureblob-credentialsvalue" value="%azureblob-accountkey%" />
+ <output-dir compare="Text">valid-auth-methods</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/azure_blob_storage/auth-methods">
+ <compilation-unit name="valid-auth-methods">
+ <placeholder name="azureblob-credentialsname" value="sharedAccessSignature" />
+ <placeholder name="azureblob-credentialsvalue" value="%azureblob-sas%" />
+ <output-dir compare="Text">valid-auth-methods</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/azure_blob_storage/auth-methods">
+ <compilation-unit name="valid-auth-methods">
+ <placeholder name="azureblob-credentialsname" value="connectionString" />
+ <placeholder name="azureblob-credentialsvalue" value="%azureblob-connectionstringaccountkey%" />
+ <output-dir compare="Text">valid-auth-methods</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/azure_blob_storage/auth-methods">
+ <compilation-unit name="valid-auth-methods">
+ <placeholder name="azureblob-credentialsname" value="connectionString" />
+ <placeholder name="azureblob-credentialsvalue" value="%azureblob-connectionstringsas%" />
+ <output-dir compare="Text">valid-auth-methods</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/azure_blob_storage/auth-methods">
+ <compilation-unit name="invalid-auth-methods">
+ <placeholder name="azureblob-credentialsname-1" value="accountKey" />
+ <placeholder name="azureblob-credentialsvalue-1" value="%azureblob-accountkey%" />
+ <placeholder name="azureblob-credentialsname-2" value="connectionString" />
+ <placeholder name="azureblob-credentialsvalue-2" value="%azureblob-connectionstringaccountkey%" />
+ <output-dir compare="Text">invalid-auth-methods</output-dir>
+ <expected-error>ASX1138: Only a single authentication method is allowed: connectionString, accountName & accountKey, or accountName & sharedAccessSignature</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/azure_blob_storage/auth-methods">
+ <compilation-unit name="invalid-auth-methods">
+ <placeholder name="azureblob-credentialsname-1" value="sharedAccessSignature" />
+ <placeholder name="azureblob-credentialsvalue-1" value="%azureblob-sas%" />
+ <placeholder name="azureblob-credentialsname-2" value="connectionString" />
+ <placeholder name="azureblob-credentialsvalue-2" value="%azureblob-connectionstringaccountkey%" />
+ <output-dir compare="Text">invalid-auth-methods</output-dir>
+ <expected-error>ASX1138: Only a single authentication method is allowed: connectionString, accountName & accountKey, or accountName & sharedAccessSignature</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/azure_blob_storage/auth-methods">
+ <compilation-unit name="invalid-no-auth">
+ <output-dir compare="Text">invalid-no-auth</output-dir>
+ <expected-error>ASX1139: No authentication parameters provided</expected-error>
+ </compilation-unit>
+ </test-case>
+ </test-group>
<test-group name="external-dataset">
<test-case FilePath="external-dataset">
<compilation-unit name="common/json/json">
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java
index a51e13c..0bb9bd5 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java
@@ -73,6 +73,7 @@
private final List<ILSMIndex> primaryIndexes = new ArrayList<>();
private final Set<ILSMIndex> flushingIndexes = Collections.synchronizedSet(new HashSet<>());
+ private final Set<ILSMMemoryComponent> flushingComponents = Collections.synchronizedSet(new HashSet<>());
private volatile int flushPtr;
private final int filteredMemoryComponentMaxNumPages;
@@ -164,6 +165,7 @@
@Override
public void flushed(ILSMMemoryComponent memoryComponent) throws HyracksDataException {
+ flushingComponents.remove(memoryComponent);
if (flushingIndexes.remove(memoryComponent.getLsmIndex())) {
LOGGER.info("Completed flushing {}.", memoryComponent.getIndex());
// After the flush operation is completed, we may have 2 cases:
@@ -208,8 +210,7 @@
@Override
public boolean isFull(ILSMMemoryComponent memoryComponent) {
- return flushingIndexes.contains(memoryComponent.getLsmIndex())
- || isFilteredMemoryComponentFull(memoryComponent);
+ return flushingComponents.contains(memoryComponent) || isFilteredMemoryComponentFull(memoryComponent);
}
private boolean isFilteredMemoryComponentFull(ILSMMemoryComponent memoryComponent) {
@@ -508,7 +509,6 @@
// future writers
memoryComponent.setUnwritable();
}
-
opTracker.setFlushOnExit(true);
opTracker.flushIfNeeded();
// If the flush cannot be scheduled at this time, then there must be active writers.
@@ -522,6 +522,7 @@
if ((flushable || opTracker.isFlushLogCreated()) && !isMetadataIndex(primaryIndex)) {
// global vbc cannot wait on metadata indexes because metadata indexes support full
// ACID transactions. Waiting on metadata indexes can introduce deadlocks.
+ flushingComponents.add(primaryIndex.getCurrentMemoryComponent());
return primaryIndex;
}
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
index 93d0ee9..77c6b8d 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/ErrorCode.java
@@ -227,6 +227,8 @@
public static final int FULL_TEXT_CONFIG_ALREADY_EXISTS = 1135;
public static final int FULL_TEXT_FILTER_ALREADY_EXISTS = 1136;
public static final int FULL_TEXT_CONFIG_NOT_FOUND = 1137;
+ public static final int ONLY_SINGLE_AUTHENTICATION_IS_ALLOWED = 1138;
+ public static final int NO_AUTH_METHOD_PROVIDED = 1139;
// Feed errors
public static final int DATAFLOW_ILLEGAL_STATE = 3001;
diff --git a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
index c5e4ce9..5b993a7 100644
--- a/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
+++ b/asterixdb/asterix-common/src/main/resources/asx_errormsg/en.properties
@@ -224,6 +224,8 @@
1135 = Full-text config %1$s already exists
1136 = Full-text filter %1$s already exists
1137 = Full-text config %1$s not found
+1138 = Only a single authentication method is allowed: connectionString, accountName & accountKey, or accountName & sharedAccessSignature
+1139 = No authentication parameters provided
# Feed Errors
3001 = Illegal state.
diff --git a/asterixdb/asterix-doc/src/main/grammar/sqlpp.ebnf b/asterixdb/asterix-doc/src/main/grammar/sqlpp.ebnf
index 886efc9..aae81ec 100644
--- a/asterixdb/asterix-doc/src/main/grammar/sqlpp.ebnf
+++ b/asterixdb/asterix-doc/src/main/grammar/sqlpp.ebnf
@@ -5,8 +5,7 @@
| OperatorExpr Operator OperatorExpr?
| OperatorExpr "BETWEEN" OperatorExpr "AND" OperatorExpr
-QuantifiedExpr::= ( "SOME" | "EVERY" ) Variable "IN" Expr ( "," Variable "in" Expr )*
- "SATISFIES" Expr ("END")?
+QuantifiedExpr::= ( "SOME" | "EVERY" ) Variable "IN" Expr ( "," Variable "IN" Expr )* "SATISFIES" Expr ("END")?
PathExpr ::= PrimaryExpr ("." Identifier | "[" Expr (":" (Expr)? )? "]")*
@@ -33,64 +32,71 @@
FunctionCall ::= OrdinaryFunctionCall | AggregateFunctionCall | WindowFunctionCall
-OrdinaryFunctionCall ::= Identifier "(" Expr ("," Expr)* ")"
+OrdinaryFunctionCall ::= (DataverseName ".")? Identifier "(" Expr ("," Expr)* ")"
-AggregateFunctionCall ::= Identifier "(" ("DISTINCT")? Expr ")"
+AggregateFunctionCall ::= Identifier "(" ("DISTINCT")? Expr ")" ("FILTER" "(" "WHERE" Expr ")")?
CaseExpr ::= SimpleCaseExpr | SearchedCaseExpr
SimpleCaseExpr ::= "CASE" Expr ("WHEN" Expr "THEN" Expr)+ ("ELSE" Expr)? "END"
-SearchedCaseExpr ::= "CASE"("WHEN" Expr "THEN" Expr)+ ("ELSE" Expr)? "END"
+SearchedCaseExpr ::= "CASE" ("WHEN" Expr "THEN" Expr)+ ("ELSE" Expr)? "END"
Constructor ::= ObjectConstructor | ArrayConstructor | MultisetConstructor
-ObjectConstructor ::= "{" ( Expr ( ":" Expr )?( ","Expr ( ":" Expr )? )* )? "}"
+ObjectConstructor ::= "{" ( Expr ( ":" Expr )? ( "," Expr ( ":" Expr )? )* )? "}"
ArrayConstructor ::= "[" Expr ("," Expr)* "]"
MultisetConstructor ::= "{{" Expr ("," Expr)* "}}"
-Query ::= (Expr | Selection) ";"
+Query ::= (Expr | Selection)
-Selection ::= WithClause? QueryBlock UnionOption* OrderByClause? LimitClause?
+Selection ::= WithClause? QueryBlock UnionOption* OrderByClause? ( LimitClause | OffsetClause )?
-QueryBlock ::= SelectClause StreamGenerator?
- |StreamGenerator SelectClause
+QueryBlock ::= SelectClause StreamGenerator?
+ | StreamGenerator SelectClause
StreamGenerator::= FromClause LetClause? WhereClause? (GroupByClause LetClause? HavingClause?)?
-SelectClause ::= "SELECT" ("DISTINCT" | "ALL")? "VALUE" Expr
- | "SELECT" ("DISTINCT" | "ALL")? ((Expr ("AS"? Identifier)?) | "*" | Identifier "." "*") ("," ((Expr ("AS"? Identifier)?) | "*" | Identifier "." "*"))*
+SelectClause ::= "SELECT" ("DISTINCT" | "ALL")? ( "VALUE" Expr | Projection ("," Projection)*)
+
+Projection ::= (Expr ("AS"? Identifier)?) | (VariableRef "." "*") | "*"
FromClause ::= "FROM" FromTerm ("," FromTerm)*
-FromTerm ::= NamedExpr JoinStep*
+FromTerm ::= NamedExpr (JoinStep | UnnestStep)*
-NamedExpr ::= Expr
- |Expr "AS"? Variable
+NamedExpr ::= Expr ("AS"? Variable)?
-JoinStep ::= ("INNER" | ("LEFT" "OUTER"?))? ("JOIN" NamedExpr "ON" Expr | "UNNEST" NamedExpr)
+JoinStep ::= ("INNER" | ( ( "LEFT" | "RIGHT" ) "OUTER"?))? "JOIN" NamedExpr "ON" Expr
+
+UnnestStep ::= ("INNER" | ( "LEFT" "OUTER"?))? "UNNEST" NamedExpr
LetClause ::= "LET" Variable "=" Expr ("," Variable "=" Expr)*
WhereClause ::= "WHERE" Expr
-GroupByClause ::= "GROUP BY" Expr ("AS"? Identifier)? ( "," Expr ("AS"? Identifier)?)* GroupAsClause?
+GroupByClause ::= "GROUP BY" GroupingElement ("," GroupingElement)* GroupAsClause?
+
+GroupingElement ::= OrdinaryGroupingSet
+ | ( "GROUPING" "SETS" "(" GroupingElement ("," GroupingElement)* ")" )
+ | ( ( "ROLLUP" | "CUBE" ) "(" OrdinaryGroupingSet ( "," OrdinaryGroupingSet )* ")" )
+ | ( "(" ")" )
+
+OrdinaryGroupingSet ::= NamedExpr | ( "(" NamedExpr ( "," NamedExpr )* ")")
+
+GroupAsClause ::= "GROUP AS" Variable
HavingClause ::= "HAVING" Expr
-GroupAsClause ::= "GROUP AS" Identifier
-
Selection ::= WithClause? QueryBlock UnionOption* OrderByClause? ( LimitClause | OffsetClause )?
UnionOption ::= "UNION ALL" (QueryBlock | Subquery)
-WithClause ::= "WITH" Identifier "AS" Expr
- ("," Identifier "AS" Expr)*
+WithClause ::= "WITH" Variable "AS" Expr ("," Variable "AS" Expr)*
-OrderbyClause ::= "ORDER BY" Expr ( "ASC" | "DESC" )?
- ( "," Expr ( "ASC" | "DESC" )? )*
+OrderbyClause ::= "ORDER BY" Expr ( "ASC" | "DESC" )? ( "," Expr ( "ASC" | "DESC" )? )*
LimitClause ::= "LIMIT" Expr OffsetClause?
@@ -100,8 +106,7 @@
WindowFunctionCall ::= WindowFunctionType "(" WindowFunctionArguments ")" WindowFunctionOptions? "OVER" (Variable "AS")? "(" WindowDefinition")"
-WindowFunctionType ::= AggregateFunction
- | WindowFunction
+WindowFunctionType ::= AggregateFunction | WindowFunction
WindowFunctionArguments ::= ( ("DISTINCT")? Expr | (Expr ("," Expr ("," Expr)? )? )? )
@@ -116,11 +121,10 @@
WindowFrameClause ::= ("ROWS" | "RANGE" | "GROUPS") WindowFrameExtent
WindowFrameExtent ::= ( ( "UNBOUNDED" | Expr ) "PRECEDING" | "CURRENT" "ROW" ) |
-"BETWEEN"( "UNBOUNDED" "PRECEDING" | "CURRENT" "ROW" | Expr ( "PRECEDING" | "FOLLOWING" ) )
-"AND" ( "UNBOUNDED" "FOLLOWING" | "CURRENT" "ROW" | Expr ( "PRECEDING" | "FOLLOWING" ) )
+ "BETWEEN" ( "UNBOUNDED" "PRECEDING" | "CURRENT" "ROW" | Expr ( "PRECEDING" | "FOLLOWING" ) )
+ "AND" ( "UNBOUNDED" "FOLLOWING" | "CURRENT" "ROW" | Expr ( "PRECEDING" | "FOLLOWING" ) )
-WindowFrameExclusion ::= "EXCLUDE" ( "CURRENT" "ROW" | "GROUP" | "TIES" |
-"NO" "OTHERS" )
+WindowFrameExclusion ::= "EXCLUDE" ( "CURRENT" "ROW" | "GROUP" | "TIES" | "NO" "OTHERS" )
Stmnt::= (SingleStmnt ";")+ "EOF"
@@ -135,11 +139,9 @@
|UpsertStmnt
|DeleteStmnt
-UseStmnt ::= "USE" Identifier
+UseStmnt ::= "USE" DataverseName
-FunctionDeclaration ::= "DELCARE" "FUNCTION" Identifier ParameterList "{" Expr "}"
-
-ParameterList ::= "(" ("VARIABLE" ("," "VARIABLE")*)?")"
+SetStmnt ::= "SET" Identifier StringLiteral
CreateStmnt ::= CreateDataverse
| CreateType
@@ -148,11 +150,13 @@
| CreateSynonym
| CreateFunction
-QualifiedName ::= Identifier ("." Identifier)?
+DataverseName ::= Identifier ("." Identifier)*
-DoubleQualifiedName ::= Identifier "." Identifier ("." Identifier)?
+QualifiedName ::= (DataverseName ".")? Identifier
-CreateDataverse ::= "CREATE" "DATAVERSE" Identifier ("IF" "NOT" "EXISTS")?
+DoubleQualifiedName ::= (DataverseName ".")? Identifier "." Identifier
+
+CreateDataverse ::= "CREATE" "DATAVERSE" DataverseName ("IF" "NOT" "EXISTS")?
CreateType ::= "CREATE" "TYPE" QualifiedName ("IF" "NOT" "EXISTS")? "AS" ObjectTypeDef
@@ -161,26 +165,39 @@
ObjectField ::= Identifier ":" Identifier "?"?
TypeExpr ::= ObjectTypeDef
- |ArrayTypeDef
- |MultisetTypeDef
- |TypeRef
+ | ArrayTypeDef
+ | MultisetTypeDef
+ | TypeReference
ArrayTypeDef ::= "[" TypeExpr "]"
MultisetTypeDef ::= "{{" TypeExpr "}}"
-TypeRef ::= Identifier
+TypeReference ::= QualifiedName
CreateDataset ::= CreateInternalDataset | CreateExternalDataset
-CreateInternalDataset ::= ( "INTERNAL" )? "DATASET" QualifiedName "(" QualifiedName ")" ("IF" "NOT" "EXISTS")?
- PrimaryKey ( "ON" Identifier )? ( "HINTS" Properties )?
- ( "USING" "COMPACTION" "POLICY" CompactionPolicy ( Configuration )? )?
- ( "WITH" "FILTER" "ON" Identifier )?
+CreateInternalDataset ::= ( "INTERNAL" )? "DATASET" QualifiedName DatasetTypeDef ( "WITH" "META" DatasetTypeDef )?
+ ("IF" "NOT" "EXISTS")?
+ PrimaryKey
+ ( "HINTS" Properties )?
+ ( "WITH" "FILTER" "ON" NestedField )?
+ ( "WITH" ObjectConstructor )?
-CreateExternalDataset ::= "EXTERNAL" "DATASET" QualifiedName "(" QualifiedName ")" ("IF" "NOT" "EXISTS")? "USING" AdapterName
- Configuration ( "HINTS" Properties )?
- ( "USING" "COMPACTION" "POLICY" CompactionPolicy ( Configuration )? )?
+CreateExternalDataset ::= "EXTERNAL" "DATASET" QualifiedName DatasetTypeDef
+ ("IF" "NOT" "EXISTS")?
+ "USING" AdapterName Configuration
+ ( "HINTS" Properties )?
+ ( "WITH" ObjectConstructor )?
+
+DatasetTypeDef ::= ( "(" TypeReference ")" )
+ | ( "(" DatasetFieldDef ("," DatasetFieldDef )* ")" ( ("CLOSED" | "OPEN") "TYPE" )? )
+
+DatasetFieldDef ::= Identifier TypeReference ("NOT" "UNKNOWN")?
+
+PrimaryKey ::= "PRIMARY" "KEY" NestedField ( "," NestedField )* ("AUTOGENERATED")?
+
+NestedField ::= Identifier ( "." Identifier )*
AdapterName ::= Identifier
@@ -190,56 +207,43 @@
Properties ::= ( "(" Identifier "=" ( StringLiteral | IntegerLiteral ) ( "," Identifier "=" ( StringLiteral | IntegerLiteral ) )* ")" )?
-PrimaryKey ::= "PRIMARY" "KEY" NestedField ( "," NestedField )* ("AUTOGENERATED")?
-
-NestedField ::= Identifier ( "." Identifier )*
-
-CompactionPolicy ::= Identifier
-
CreateIndex ::= CreateSecondaryIndex | CreatePrimaryKeyIndex
-CreateSecondaryIndex ::= "CREATE" ("INDEX" Identifier ("IF" "NOT" "EXISTS")? "ON" QualifiedName
- "(" ( IndexField ) ( "," IndexField )* ")" ("TYPE" IndexType)? ("ENFORCED")?)
+CreateSecondaryIndex ::= "CREATE" "INDEX" Identifier ("IF" "NOT" "EXISTS")? "ON" QualifiedName
+ "(" IndexField ( "," IndexField )* ")" ("TYPE" IndexType)? ("ENFORCED")?
-CreatePrimaryKeyIndex ::= "CREATE" "PRIMARY" "INDEX" Identifier? ("IF" "NOT" "EXISTS")? "ON" QualifiedName ("TYPE" "BTREE")?
+CreatePrimaryKeyIndex ::= "CREATE" "PRIMARY" "INDEX" Identifier? ("IF" "NOT" "EXISTS")? "ON" QualifiedName ("TYPE" "BTREE")?
-IndexField ::= NestedField (":" TypeRef)?
+IndexField ::= NestedField ( ":" TypeReference "?"? )?
IndexType ::= "BTREE"
|"RTREE"
|"KEYWORD"
+ |"FULLTEXT"
|"NGRAM" "(" IntegerLiteral ")"
CreateSynonym ::= "CREATE" "SYNONYM" QualifiedName "FOR" QualifiedName ("IF" "NOT" "EXISTS")?
-FunctionParameters ::= "(" ("VARIABLE" (":" TypeExpr)? ("," "VARIABLE" (":" TypeExpr)? )* )? ")"
+FunctionDeclaration ::= "DECLARE" "FUNCTION" Identifier "(" ( (Identifier ("," Identifier)*) | "..." )? ")" "{" Expr "}"
+CreateFunction ::= "CREATE" ("OR" "REPLACE")? "FUNCTION" QualifiedName ("IF" "NOT" "EXISTS")? "(" FunctionParameters? ")"
+ ( ("{" Expr "}") | ExternalFunctionDef )
-CreateFunction ::= "CREATE" ("OR" "REPLACE")? "FUNCTION" FunctionOrTypeName ("IF" "NOT" "EXISTS")? FunctionParameters ( "RETURNS" TypeExpr)?
- ( ("{" Expr "}") | ("AS" FunctionExternalIdentifier "AT" QualifiedName ("WITH" ObjectConstructor)?))
+FunctionParameters ::= ( Identifier ((":")? TypeExpr)? ("," Identifier ((":")? TypeExpr)? )* ) | "..."
-DropStmnt ::= "DROP" ("DATAVERSE" Identifier
- | "TYPE" FunctionOrTypeName
- | ("DATSET" | "SYNONYM") QualifiedName
+ExternalFunctionDef ::= ("RETURNS" TypeExpr)? "AS" StringLiteral ("," StringLiteral )* "AT" QualifiedName ("WITH" ObjectConstructor)?
+
+DropStmnt ::= "DROP" ("DATAVERSE" DataverseName
+ | ("TYPE" |"DATASET" | "SYNONYM") QualifiedName
| "INDEX" DoubleQualifiedName
| "FUNCTION" FunctionSignature ) ("IF" "EXISTS")?
-FunctionSignature ::= FunctionOrTypeName "@" IntegerLiteral
+FunctionSignature ::= QualifiedName ( ( "(" ( FunctionParameters? | IntegerLiteral ) ")" ) | ("@" IntegerLiteral) )
LoadStmnt ::= "LOAD" "DATASET" QualifiedName "USING" AdapterName Configuration ("PRE-SORTED")?
-InsertStmnt ::= "INSERT" "INTO" QualifiedName Query
+InsertStmnt ::= "INSERT" "INTO" QualifiedName ("AS" Variable)? Query ("RETURNING" Expr)?
-UpsertStmnt ::= "UPSERT" "INTO" QualifiedName Query
+UpsertStmnt ::= "UPSERT" "INTO" QualifiedName ("AS" Variable)? Query ("RETURNING" Expr)?
DeleteStmnt ::= "DELETE" "FROM" QualifiedName (("AS")? Variable)? ("WHERE" Expr)?
-
-SetStmnt ::= "SET" Identifier StringLiteral
-
-
-
-
-
-
-
-
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/0_toc.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/0_toc.md
index 5d084c5..47ed4e7 100644
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/0_toc.md
+++ b/asterixdb/asterix-doc/src/main/markdown/sqlpp/0_toc.md
@@ -33,7 +33,7 @@
* [Identifiers and Variable References](#Variable_references)
* [Parameter References](#Parameter_references)
* [Parenthesized Expressions](#Parenthesized_expressions)
- * [Function calls](#Function_call_expressions)
+ * [Function Calls](#Function_call_expressions)
* [Case Expressions](#Case_expressions)
* [Constructors](#Constructors)
* [3. Queries](#Queries)
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/2_expr.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/2_expr.md
index 9979570..4453099 100644
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/2_expr.md
+++ b/asterixdb/asterix-doc/src/main/markdown/sqlpp/2_expr.md
@@ -190,8 +190,8 @@
### QuantifiedExpr
****
-##### Synonym for `SOME`: `ANY`
-
+##### Synonym for `SOME`: `ANY`
+
---
Quantified expressions are used for expressing existential or universal predicates involving the elements of a collection.
@@ -207,7 +207,7 @@
##### Examples
EVERY x IN [ 1, 2, 3 ] SATISFIES x < 3 Returns FALSE
- SOME x IN [ 1, 2, 3 ] SATISFIES x < 3 Returns TRUE
+ SOME x IN [ 1, 2, 3 ] SATISFIES x < 3 Returns TRUE
## <a id="Path_expressions">Path Expressions</a>
@@ -242,7 +242,7 @@
({"name": "MyABCs", "array": [ "a", "b", "c"]}).array Returns [["a", "b", "c"]]
(["a", "b", "c"])[2] Returns ["c"]
-
+
(["a", "b", "c"])[-1] Returns ["c"]
({"name": "MyABCs", "array": [ "a", "b", "c"]}).array[2] Returns ["c"]
@@ -250,7 +250,7 @@
(["a", "b", "c"])[0:2] Returns [["a", "b"]]
(["a", "b", "c"])[0:] Returns [["a", "b", "c"]]
-
+
(["a", "b", "c"])[-2:-1] Returns [["b"]]
@@ -279,7 +279,7 @@
The simplest kind of expression is a literal that directly represents a value in JSON format. Here are some examples:
-
+
-42
"Hello"
@@ -287,18 +287,18 @@
false
null
-
+
Numeric literals may include a sign and an optional decimal point. They may also be written in exponential notation, like this:
-
+
5e2
-4.73E-2
-
+
String literals may be enclosed in either single quotes or double quotes. Inside a string literal, the delimiter character for that string must be "escaped" by a backward slash, as in these examples:
-
+
"I read \"War and Peace\" today."
'I don\'t believe everything I read.'
@@ -321,7 +321,7 @@
### <a id="Variable_references">Identifiers and Variable References</a>
-
+
Like SQL, SQL++ makes use of a language construct called an *identifier*. An identifier starts with an alphabetic character or the underscore character _ , and contains only case-sensitive alphabetic characters, numeric digits, or the special characters _ and $. It is also possible for an identifier to include other special characters, or to be the same as a reserved word, by enclosing the identifier in back-ticks (it's then called a *delimited identifier*). Identifiers are used in variable names and in certain other places in SQL++ syntax, such as in path expressions, which we'll discuss soon. Here are some examples of identifiers:
X
@@ -330,7 +330,7 @@
`spaces in here`
`@&#`
-
+
A very simple kind of SQL++ expression is a variable, which is simply an identifier. As in SQL, a variable can be bound to a value, which may be an input dataset, some intermediate result during processing of a query, or the final result of a query. We'll learn more about variables when we discuss queries.
Note that the SQL++ rules for delimiting strings and identifiers are different from the SQL rules. In SQL, strings are always enclosed in single quotes, and double quotes are used for delimited identifiers.
@@ -390,6 +390,9 @@
### AggregateFunctionCall
****
+### DataverseName
+****
+
---
Functions are included in SQL++, like most languages, as a way to package useful functionality or to
@@ -400,6 +403,8 @@
Note that Window functions, and aggregate functions used as window functions, have a more complex syntax.
Window function calls are described in the section on [Window Queries](#Over_clauses).
+Also note that FILTER expressions can only be specified when calling [Aggregation Pseudo-Functions](#Aggregation_PseudoFunctions).
+
The following example is a function call expression whose value is 8.
##### Example
@@ -453,16 +458,16 @@
Structured JSON values can be represented by constructors, as in these examples:
An object: { "name": "Bill", "age": 42 }
- An array: [ 1, 2, "Hello", null ]
-
+ An array: [ 1, 2, "Hello", null ]
+
In a constructed object, the names of the fields must be strings (either literal strings or computed strings), and an object may not contain any duplicate names. Of course, structured literals can be nested, as in this example:
-
+
[ {"name": "Bill",
"address":
{"street": "25 Main St.",
- "city": "Cincinnati, OH"
+ "city": "Cincinnati, OH"
}
},
{"name": "Mary",
@@ -473,33 +478,33 @@
}
]
-
+
The array items in an array constructor, and the field-names and field-values in an object constructor, may be represented by expressions. For example, suppose that the variables firstname, lastname, salary, and bonus are bound to appropriate values. Then structured values might be constructed by the following expressions:
-
+
An object:
- {
- "name": firstname || " " || lastname,
- "income": salary + bonus
+ {
+ "name": firstname || " " || lastname,
+ "income": salary + bonus
}
-
+
An array:
["1984", lastname, salary + bonus, null]
If only one expression is specified instead of the field-name/field-value pair in an object constructor then this
-expression is supposed to provide the field value. The field name is then automatically generated based on the
+expression is supposed to provide the field value. The field name is then automatically generated based on the
kind of the value expression as in Q2.1:
* If it is a variable reference expression then the generated field name is the name of that variable.
* If it is a field access expression then the generated field name is the last identifier in that expression.
* For all other cases, a compilation error will be raised.
-
+
##### Example
(Q2.1)
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_declare_dataverse.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_declare_dataverse.md
deleted file mode 100644
index d33d680..0000000
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_declare_dataverse.md
+++ /dev/null
@@ -1,33 +0,0 @@
-<!--
- ! Licensed to the Apache Software Foundation (ASF) under one
- ! or more contributor license agreements. See the NOTICE file
- ! distributed with this work for additional information
- ! regarding copyright ownership. The ASF licenses this file
- ! to you under the Apache License, Version 2.0 (the
- ! "License"); you may not use this file except in compliance
- ! with the License. You may obtain a copy of the License at
- !
- ! http://www.apache.org/licenses/LICENSE-2.0
- !
- ! Unless required by applicable law or agreed to in writing,
- ! software distributed under the License is distributed on an
- ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ! KIND, either express or implied. See the License for the
- ! specific language governing permissions and limitations
- ! under the License.
- !-->
-
-## <a id="Declarations">Declarations</a>
-
- DatabaseDeclaration ::= "USE" Identifier
-
-At the uppermost level, the world of data is organized into data namespaces called **dataverses**.
-To set the default dataverse for statements, the USE statement is provided.
-
-As an example, the following statement sets the default dataverse to be "TinySocial".
-
-##### Example
-
- USE TinySocial;
-
-
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_declare_function.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_declare_function.md
deleted file mode 100644
index 8e77de9..0000000
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_declare_function.md
+++ /dev/null
@@ -1,45 +0,0 @@
-<!--
- ! Licensed to the Apache Software Foundation (ASF) under one
- ! or more contributor license agreements. See the NOTICE file
- ! distributed with this work for additional information
- ! regarding copyright ownership. The ASF licenses this file
- ! to you under the Apache License, Version 2.0 (the
- ! "License"); you may not use this file except in compliance
- ! with the License. You may obtain a copy of the License at
- !
- ! http://www.apache.org/licenses/LICENSE-2.0
- !
- ! Unless required by applicable law or agreed to in writing,
- ! software distributed under the License is distributed on an
- ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ! KIND, either express or implied. See the License for the
- ! specific language governing permissions and limitations
- ! under the License.
- !-->
-
-When writing a complex query, it can sometimes be helpful to define one or more auxilliary functions
-that each address a sub-piece of the overall query.
-The declare function statement supports the creation of such helper functions.
-In general, the function body (expression) can be any legal query expression.
-
- FunctionDeclaration ::= "DECLARE" "FUNCTION" Identifier ParameterList "{" Expression "}"
- ParameterList ::= "(" ( <VARIABLE> ( "," <VARIABLE> )* )? ")"
-
-The following is a simple example of a temporary function definition and its use.
-
-##### Example
-
- DECLARE FUNCTION friendInfo(userId) {
- (SELECT u.id, u.name, len(u.friendIds) AS friendCount
- FROM GleambookUsers u
- WHERE u.id = userId)[0]
- };
-
- SELECT VALUE friendInfo(2);
-
-For our sample data set, this returns:
-
- [
- { "id": 2, "name": "IsbelDull", "friendCount": 2 }
- ]
-
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_query.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_query.md
index 17e6339..f4bbc4d 100644
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_query.md
+++ b/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_query.md
@@ -23,6 +23,8 @@
### SelectClause
****
+### Projection
+****
##### Synonyms for `VALUE`: `ELEMENT`, `RAW`
---
@@ -322,6 +324,8 @@
### JoinStep
****
+### UnnestStep
+****
##### Synonyms for `UNNEST`: `CORRELATE`, `FLATTEN`
---
@@ -652,6 +656,14 @@
### GroupByClause
****
+### GroupingElement
+****
+
+### OrdinaryGroupingSet
+****
+
+### NamedExpr
+****
---
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/4_error.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/4_error.md
deleted file mode 100644
index 18fce14..0000000
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/4_error.md
+++ /dev/null
@@ -1,113 +0,0 @@
-<!--
- ! Licensed to the Apache Software Foundation (ASF) under one
- ! or more contributor license agreements. See the NOTICE file
- ! distributed with this work for additional information
- ! regarding copyright ownership. The ASF licenses this file
- ! to you under the Apache License, Version 2.0 (the
- ! "License"); you may not use this file except in compliance
- ! with the License. You may obtain a copy of the License at
- !
- ! http://www.apache.org/licenses/LICENSE-2.0
- !
- ! Unless required by applicable law or agreed to in writing,
- ! software distributed under the License is distributed on an
- ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ! KIND, either express or implied. See the License for the
- ! specific language governing permissions and limitations
- ! under the License.
- !-->
-
-A query can potentially result in one of the following errors:
-
- * syntax error,
- * identifier resolution error,
- * type error,
- * resource error.
-
-If the query processor runs into any error, it will
-terminate the ongoing processing of the query and
-immediately return an error message to the client.
-
-## <a id="Syntax_errors">Syntax Errors</a>
-A valid query must satisfy the grammar rules of the query language.
-Otherwise, a syntax error will be raised.
-
-##### Example
-
- SELECT *
- GleambookUsers user
-
-Since the query misses a `FROM` keyword before the dataset `GleambookUsers`,
-we will get a syntax error as follows:
-
- Syntax error: In line 2 >>GleambookUsers user;<< Encountered <IDENTIFIER> \"GleambookUsers\" at column 1.
-
-##### Example
-
- SELECT *
- FROM GleambookUsers user
- WHERE type="advertiser";
-
-Since "type" is a reserved keyword in the query parser,
-we will get a syntax error as follows:
-
- Error: Syntax error: In line 3 >>WHERE type="advertiser";<< Encountered 'type' "type" at column 7.
- ==> WHERE type="advertiser";
-
-
-## <a id="Identifier_resolution_errors">Identifier Resolution Errors</a>
-Referring to an undefined identifier can cause an error if the identifier
-cannot be successfully resolved as a valid field access.
-
-##### Example
-
- SELECT *
- FROM GleambookUser user;
-
-If we have a typo as above in "GleambookUsers" that misses the dataset name's ending "s",
-we will get an identifier resolution error as follows:
-
- Error: Cannot find dataset GleambookUser in dataverse Default nor an alias with name GleambookUser!
-
-##### Example
-
- SELECT name, message
- FROM GleambookUsers u JOIN GleambookMessages m ON m.authorId = u.id;
-
-If the compiler cannot figure out how to resolve an unqualified field name, which will occur if there is more than one variable in scope (e.g., `GleambookUsers u` and `GleambookMessages m` as above),
-we will get an identifier resolution error as follows:
-
- Error: Cannot resolve ambiguous alias reference for undefined identifier name
-
-
-## <a id="Type_errors">Type Errors</a>
-
-The query compiler does type checks based on its available type information.
-In addition, the query runtime also reports type errors if a data model instance
-it processes does not satisfy the type requirement.
-
-##### Example
-
- abs("123");
-
-Since function `abs` can only process numeric input values,
-we will get a type error as follows:
-
- Error: Type mismatch: function abs expects its 1st input parameter to be of type tinyint, smallint, integer, bigint, float or double, but the actual input type is string
-
-
-## <a id="Resource_errors">Resource Errors</a>
-A query can potentially exhaust system resources, such
-as the number of open files and disk spaces.
-For instance, the following two resource errors could be potentially
-be seen when running the system:
-
- Error: no space left on device
- Error: too many open files
-
-The "no space left on device" issue usually can be fixed by
-cleaning up disk spaces and reserving more disk spaces for the system.
-The "too many open files" issue usually can be fixed by a system
-administrator, following the instructions
-[here](https://easyengine.io/tutorials/linux/increase-open-files-limit/).
-
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/4_error_title.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/4_error_title.md
deleted file mode 100644
index 6279d5e..0000000
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/4_error_title.md
+++ /dev/null
@@ -1,20 +0,0 @@
-<!--
- ! Licensed to the Apache Software Foundation (ASF) under one
- ! or more contributor license agreements. See the NOTICE file
- ! distributed with this work for additional information
- ! regarding copyright ownership. The ASF licenses this file
- ! to you under the Apache License, Version 2.0 (the
- ! "License"); you may not use this file except in compliance
- ! with the License. You may obtain a copy of the License at
- !
- ! http://www.apache.org/licenses/LICENSE-2.0
- !
- ! Unless required by applicable law or agreed to in writing,
- ! software distributed under the License is distributed on an
- ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ! KIND, either express or implied. See the License for the
- ! specific language governing permissions and limitations
- ! under the License.
- !-->
-
-# <a id="Errors">4. Errors</a>
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/5_ddl_dataset_index.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/5_ddl_dataset_index.md
deleted file mode 100644
index 9e10aed..0000000
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/5_ddl_dataset_index.md
+++ /dev/null
@@ -1,296 +0,0 @@
-<!--
- ! Licensed to the Apache Software Foundation (ASF) under one
- ! or more contributor license agreements. See the NOTICE file
- ! distributed with this work for additional information
- ! regarding copyright ownership. The ASF licenses this file
- ! to you under the Apache License, Version 2.0 (the
- ! "License"); you may not use this file except in compliance
- ! with the License. You may obtain a copy of the License at
- !
- ! http://www.apache.org/licenses/LICENSE-2.0
- !
- ! Unless required by applicable law or agreed to in writing,
- ! software distributed under the License is distributed on an
- ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ! KIND, either express or implied. See the License for the
- ! specific language governing permissions and limitations
- ! under the License.
- !-->
-
-## <a id="Lifecycle_management_statements">Lifecycle Management Statements</a>
-
- CreateStatement ::= "CREATE" ( DatabaseSpecification
- | TypeSpecification
- | DatasetSpecification
- | IndexSpecification
- | SynonymSpecification
- | FunctionSpecification )
-
- QualifiedName ::= Identifier ( "." Identifier )?
- DoubleQualifiedName ::= Identifier "." Identifier ( "." Identifier )?
-
-The CREATE statement is used for creating dataverses as well as other persistent artifacts in a dataverse.
-It can be used to create new dataverses, datatypes, datasets, indexes, and user-defined query functions.
-
-### <a id="Dataverses"> Dataverses</a>
-
- DatabaseSpecification ::= "DATAVERSE" Identifier IfNotExists
-
-The CREATE DATAVERSE statement is used to create new dataverses.
-To ease the authoring of reusable query scripts, an optional IF NOT EXISTS clause is included to allow
-creation to be requested either unconditionally or only if the dataverse does not already exist.
-If this clause is absent, an error is returned if a dataverse with the indicated name already exists.
-
-The following example creates a new dataverse named TinySocial if one does not already exist.
-
-##### Example
-
- CREATE DATAVERSE TinySocial IF NOT EXISTS;
-
-### <a id="Types"> Types</a>
-
- TypeSpecification ::= "TYPE" FunctionOrTypeName IfNotExists "AS" ObjectTypeDef
- FunctionOrTypeName ::= QualifiedName
- IfNotExists ::= ( <IF> <NOT> <EXISTS> )?
- TypeExpr ::= ObjectTypeDef | TypeReference | ArrayTypeDef | MultisetTypeDef
- ObjectTypeDef ::= ( <CLOSED> | <OPEN> )? "{" ( ObjectField ( "," ObjectField )* )? "}"
- ObjectField ::= Identifier ":" ( TypeExpr ) ( "?" )?
- NestedField ::= Identifier ( "." Identifier )*
- IndexField ::= NestedField ( ":" TypeReference )?
- TypeReference ::= Identifier
- ArrayTypeDef ::= "[" ( TypeExpr ) "]"
- MultisetTypeDef ::= "{{" ( TypeExpr ) "}}"
-
-The CREATE TYPE statement is used to create a new named datatype.
-This type can then be used to create stored collections or utilized when defining one or more other datatypes.
-Much more information about the data model is available in the [data model reference guide](../datamodel.html).
-A new type can be a object type, a renaming of another type, an array type, or a multiset type.
-A object type can be defined as being either open or closed.
-Instances of a closed object type are not permitted to contain fields other than those specified in the create type statement.
-Instances of an open object type may carry additional fields, and open is the default for new types if neither option is specified.
-
-The following example creates a new object type called GleambookUser type.
-Since it is defined as (defaulting to) being an open type,
-instances will be permitted to contain more than what is specified in the type definition.
-The first four fields are essentially traditional typed name/value pairs (much like SQL fields).
-The friendIds field is a multiset of integers.
-The employment field is an array of instances of another named object type, EmploymentType.
-
-##### Example
-
- CREATE TYPE GleambookUserType AS {
- id: int,
- alias: string,
- name: string,
- userSince: datetime,
- friendIds: {{ int }},
- employment: [ EmploymentType ]
- };
-
-The next example creates a new object type, closed this time, called MyUserTupleType.
-Instances of this closed type will not be permitted to have extra fields,
-although the alias field is marked as optional and may thus be NULL or MISSING in legal instances of the type.
-Note that the type of the id field in the example is UUID.
-This field type can be used if you want to have this field be an autogenerated-PK field.
-(Refer to the Datasets section later for more details on such fields.)
-
-##### Example
-
- CREATE TYPE MyUserTupleType AS CLOSED {
- id: uuid,
- alias: string?,
- name: string
- };
-
-### <a id="Datasets"> Datasets</a>
-
- DatasetSpecification ::= ( <INTERNAL> )? <DATASET> QualifiedName "(" QualifiedName ")" IfNotExists
- PrimaryKey ( <ON> Identifier )? ( <HINTS> Properties )?
- ( "USING" "COMPACTION" "POLICY" CompactionPolicy ( Configuration )? )?
- ( <WITH> <FILTER> <ON> Identifier )?
- |
- <EXTERNAL> <DATASET> QualifiedName "(" QualifiedName ")" IfNotExists <USING> AdapterName
- Configuration ( <HINTS> Properties )?
- ( <USING> <COMPACTION> <POLICY> CompactionPolicy ( Configuration )? )?
- AdapterName ::= Identifier
- Configuration ::= "(" ( KeyValuePair ( "," KeyValuePair )* )? ")"
- KeyValuePair ::= "(" StringLiteral "=" StringLiteral ")"
- Properties ::= ( "(" Property ( "," Property )* ")" )?
- Property ::= Identifier "=" ( StringLiteral | IntegerLiteral )
- FunctionSignature ::= FunctionOrTypeName "@" IntegerLiteral
- PrimaryKey ::= <PRIMARY> <KEY> NestedField ( "," NestedField )* ( <AUTOGENERATED> )?
- CompactionPolicy ::= Identifier
-
-The CREATE DATASET statement is used to create a new dataset.
-Datasets are named, multisets of object type instances;
-they are where data lives persistently and are the usual targets for queries.
-Datasets are typed, and the system ensures that their contents conform to their type definitions.
-An Internal dataset (the default kind) is a dataset whose content lives within and is managed by the system.
-It is required to have a specified unique primary key field which uniquely identifies the contained objects.
-(The primary key is also used in secondary indexes to identify the indexed primary data objects.)
-
-Internal datasets contain several advanced options that can be specified when appropriate.
-One such option is that random primary key (UUID) values can be auto-generated by declaring the field to be UUID and putting "AUTOGENERATED" after the "PRIMARY KEY" identifier.
-In this case, unlike other non-optional fields, a value for the auto-generated PK field should not be provided at insertion time by the user since each object's primary key field value will be auto-generated by the system.
-
-Another advanced option, when creating an Internal dataset, is to specify the merge policy to control which of the
-underlying LSM storage components to be merged.
-(The system supports Log-Structured Merge tree based physical storage for Internal datasets.)
-Currently the system supports four different component merging policies that can be chosen per dataset:
-no-merge, constant, prefix, and correlated-prefix.
-The no-merge policy simply never merges disk components.
-The constant policy merges disk components when the number of components reaches a constant number k that can be configured by the user.
-The prefix policy relies on both component sizes and the number of components to decide which components to merge.
-It works by first trying to identify the smallest ordered (oldest to newest) sequence of components such that the sequence does not contain a single component that exceeds some threshold size M and that either the sum of the component's sizes exceeds M or the number of components in the sequence exceeds another threshold C.
-If such a sequence exists, the components in the sequence are merged together to form a single component.
-Finally, the correlated-prefix policy is similar to the prefix policy, but it delegates the decision of merging the disk components of all the indexes in a dataset to the primary index.
-When the correlated-prefix policy decides that the primary index needs to be merged (using the same decision criteria as for the prefix policy), then it will issue successive merge requests on behalf of all other indexes associated with the same dataset.
-The system's default policy is the prefix policy except when there is a filter on a dataset, where the preferred policy for filters is the correlated-prefix.
-
-Another advanced option shown in the syntax above, related to performance and mentioned above, is that a **filter** can optionally be created on a field to further optimize range queries with predicates on the filter's field.
-Filters allow some range queries to avoid searching all LSM components when the query conditions match the filter.
-(Refer to [Filter-Based LSM Index Acceleration](../filters.html) for more information about filters.)
-
-An External dataset, in contrast to an Internal dataset, has data stored outside of the system's control.
-Files living in HDFS or in the local filesystem(s) of a cluster's nodes are currently supported.
-External dataset support allows queries to treat foreign data as though it were stored in the system,
-making it possible to query "legacy" file data (for example, Hive data) without having to physically import it.
-When defining an External dataset, an appropriate adapter type must be selected for the desired external data.
-(See the [Guide to External Data](../externaldata.html) for more information on the available adapters.)
-
-The following example creates an Internal dataset for storing FacefookUserType objects.
-It specifies that their id field is their primary key.
-
-#### Example
-
- CREATE INTERNAL DATASET GleambookUsers(GleambookUserType) PRIMARY KEY id;
-
-The next example creates another Internal dataset (the default kind when no dataset kind is specified) for storing MyUserTupleType objects.
-It specifies that the id field should be used as the primary key for the dataset.
-It also specifies that the id field is an auto-generated field,
-meaning that a randomly generated UUID value should be assigned to each incoming object by the system.
-(A user should therefore not attempt to provide a value for this field.)
-Note that the id field's declared type must be UUID in this case.
-
-#### Example
-
- CREATE DATASET MyUsers(MyUserTupleType) PRIMARY KEY id AUTOGENERATED;
-
-The next example creates an External dataset for querying LineItemType objects.
-The choice of the `hdfs` adapter means that this dataset's data actually resides in HDFS.
-The example CREATE statement also provides parameters used by the hdfs adapter:
-the URL and path needed to locate the data in HDFS and a description of the data format.
-
-#### Example
-
- CREATE EXTERNAL DATASET LineItem(LineItemType) USING hdfs (
- ("hdfs"="hdfs://HOST:PORT"),
- ("path"="HDFS_PATH"),
- ("input-format"="text-input-format"),
- ("format"="delimited-text"),
- ("delimiter"="|"));
-
-
-### <a id="Indices">Indices</a>
-
- IndexSpecification ::= (<INDEX> Identifier IfNotExists <ON> QualifiedName
- "(" ( IndexField ) ( "," IndexField )* ")" (<TYPE> IndexType)? (<ENFORCED>)?)
- |
- <PRIMARY> <INDEX> Identifier? IfNotExists <ON> QualifiedName (<TYPE> <BTREE>)?
- IndexType ::= <BTREE> | <RTREE> | <KEYWORD> | <NGRAM> "(" IntegerLiteral ")"
-
-The CREATE INDEX statement creates a secondary index on one or more fields of a specified dataset.
-Supported index types include `BTREE` for totally ordered datatypes, `RTREE` for spatial data,
-and `KEYWORD` and `NGRAM` for textual (string) data.
-An index can be created on a nested field (or fields) by providing a valid path expression as an index field identifier.
-
-An indexed field is not required to be part of the datatype associated with a dataset if the dataset's datatype
-is declared as open **and** if the field's type is provided along with its name and if the `ENFORCED` keyword is
-specified at the end of the index definition.
-`ENFORCING` an open field introduces a check that makes sure that the actual type of the indexed field
-(if the optional field exists in the object) always matches this specified (open) field type.
-
-The following example creates a btree index called gbAuthorIdx on the authorId field of the GleambookMessages dataset.
-This index can be useful for accelerating exact-match queries, range search queries, and joins involving the author-id
-field.
-
-#### Example
-
- CREATE INDEX gbAuthorIdx ON GleambookMessages(authorId) TYPE BTREE;
-
-The following example creates an open btree index called gbSendTimeIdx on the (non-declared) `sendTime` field of the GleambookMessages dataset having datetime type.
-This index can be useful for accelerating exact-match queries, range search queries, and joins involving the `sendTime` field.
-The index is enforced so that records that do not have the `sendTime` field or have a mismatched type on the field
-cannot be inserted into the dataset.
-
-#### Example
-
- CREATE INDEX gbSendTimeIdx ON GleambookMessages(sendTime: datetime?) TYPE BTREE ENFORCED;
-
-The following example creates an open btree index called gbReadTimeIdx on the (non-declared) `readTime`
-field of the GleambookMessages dataset having datetime type.
-This index can be useful for accelerating exact-match queries, range search queries,
-and joins involving the `readTime` field.
-The index is not enforced so that records that do not have the `readTime` field or have a mismatched type on the field
-can still be inserted into the dataset.
-
-#### Example
-
- CREATE INDEX gbReadTimeIdx ON GleambookMessages(readTime: datetime?);
-
-The following example creates a btree index called crpUserScrNameIdx on screenName,
-a nested field residing within a object-valued user field in the ChirpMessages dataset.
-This index can be useful for accelerating exact-match queries, range search queries,
-and joins involving the nested screenName field.
-Such nested fields must be singular, i.e., one cannot index through (or on) an array-valued field.
-
-#### Example
-
- CREATE INDEX crpUserScrNameIdx ON ChirpMessages(user.screenName) TYPE BTREE;
-
-The following example creates an rtree index called gbSenderLocIdx on the sender-location field of the GleambookMessages dataset. This index can be useful for accelerating queries that use the [`spatial-intersect` function](functions.html#spatial-intersect) in a predicate involving the sender-location field.
-
-#### Example
-
- CREATE INDEX gbSenderLocIndex ON GleambookMessages("sender-location") TYPE RTREE;
-
-The following example creates a 3-gram index called fbUserIdx on the name field of the GleambookUsers dataset. This index can be used to accelerate some similarity or substring maching queries on the name field. For details refer to the document on [similarity queries](similarity.html#NGram_Index).
-
-#### Example
-
- CREATE INDEX fbUserIdx ON GleambookUsers(name) TYPE NGRAM(3);
-
-The following example creates a keyword index called fbMessageIdx on the message field of the GleambookMessages dataset. This keyword index can be used to optimize queries with token-based similarity predicates on the message field. For details refer to the document on [similarity queries](similarity.html#Keyword_Index).
-
-#### Example
-
- CREATE INDEX fbMessageIdx ON GleambookMessages(message) TYPE KEYWORD;
-
-The following example creates a special secondary index which holds only the primary keys.
-This index is useful for speeding up aggregation queries which involve only primary keys.
-The name of the index is optional. If the name is not specified, the system will generate
-one. When the user would like to drop this index, the metadata can be queried to find the system-generated name.
-
-#### Example
-
- CREATE PRIMARY INDEX gb_pk_idx ON GleambookMessages;
-
-An example query that can be accelerated using the primary-key index:
-
- SELECT COUNT(*) FROM GleambookMessages;
-
-To look up the the above primary-key index, issue the following query:
-
- SELECT VALUE i
- FROM Metadata.`Index` i
- WHERE i.DataverseName = "TinySocial" AND i.DatasetName = "GleambookMessages";
-
-The query returns:
-
- [ { "DataverseName": "TinySocial", "DatasetName": "GleambookMessages", "IndexName": "GleambookMessages", "IndexStructure": "BTREE", "SearchKey": [ [ "messageId" ] ], "IsPrimary": true, "Timestamp": "Wed Nov 07 17:25:11 PST 2018", "PendingOp": 0 }
- , { "DataverseName": "TinySocial", "DatasetName": "GleambookMessages", "IndexName": "gb_pk_idx", "IndexStructure": "BTREE", "SearchKey": [ ], "IsPrimary": false, "Timestamp": "Wed Nov 07 17:25:11 PST 2018", "PendingOp": 0 }
- ]
-
-Remember that `CREATE PRIMARY INDEX` creates a secondary index.
-That is the reason the `IsPrimary` field is false.
-The primary-key index can be identified by the fact that the `SearchKey` field is empty since it only contains primary key fields.
\ No newline at end of file
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/5_ddl_dml.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/5_ddl_dml.md
deleted file mode 100644
index 7a2223e..0000000
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/5_ddl_dml.md
+++ /dev/null
@@ -1,92 +0,0 @@
-<!--
- ! Licensed to the Apache Software Foundation (ASF) under one
- ! or more contributor license agreements. See the NOTICE file
- ! distributed with this work for additional information
- ! regarding copyright ownership. The ASF licenses this file
- ! to you under the Apache License, Version 2.0 (the
- ! "License"); you may not use this file except in compliance
- ! with the License. You may obtain a copy of the License at
- !
- ! http://www.apache.org/licenses/LICENSE-2.0
- !
- ! Unless required by applicable law or agreed to in writing,
- ! software distributed under the License is distributed on an
- ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ! KIND, either express or implied. See the License for the
- ! specific language governing permissions and limitations
- ! under the License.
- !-->
-
-## <a id="Modification_statements">Modification statements</a>
-
-### <a id="Inserts">INSERTs</a>
-
- InsertStatement ::= <INSERT> <INTO> QualifiedName Query
-
-The INSERT statement is used to insert new data into a dataset.
-The data to be inserted comes from a query expression.
-This expression can be as simple as a constant expression, or in general it can be any legal query.
-In case the dataset has an auto-generated primary key, when performing an INSERT operation, the system allows the user to manually add the
-auto-generated key field in the INSERT statement, or skip that field and the system will automatically generate it and add it. However,
-it is important to note that if the a record already exists in the dataset with the auto-generated key provided by the user, then
-that operation is going to fail. As a general rule, insertion will fail if the dataset already has data with the primary key value(s)
-being inserted.
-
-Inserts are processed transactionally by the system.
-The transactional scope of each insert transaction is the insertion of a single object plus its affiliated secondary index entries (if any).
-If the query part of an insert returns a single object, then the INSERT statement will be a single, atomic transaction.
-If the query part returns multiple objects, each object being inserted will be treated as a separate tranaction.
-
-The target dataset name may be a synonym introduced by CREATE SYNONYM statement.
-
-The following example illustrates a query-based insertion.
-
-##### Example
-
- INSERT INTO UsersCopy (SELECT VALUE user FROM GleambookUsers user)
-
-### <a id="Upserts">UPSERTs</a>
-
- UpsertStatement ::= <UPSERT> <INTO> QualifiedName Query
-
-The UPSERT statement syntactically mirrors the INSERT statement discussed above.
-The difference lies in its semantics, which for UPSERT are "add or replace" instead of the INSERT "add if not present, else error" semantics.
-Whereas an INSERT can fail if another object already exists with the specified key, the analogous UPSERT will replace the previous object's value
-with that of the new object in such cases. Like the INSERT statement, the system allows the user to manually provide the auto-generated key
-for datasets with an auto-generated key as its primary key. This operation will insert the record if no record with that key already exists, but
-if a record with the key already exists, then the operation will be converted to a replace/update operation.
-
-The target dataset name may be a synonym introduced by CREATE SYNONYM statement.
-
-The following example illustrates a query-based upsert operation.
-
-##### Example
-
- UPSERT INTO UsersCopy (SELECT VALUE user FROM GleambookUsers user)
-
-*Editor's note: Upserts currently work in AQL but are not yet enabled (at the moment) in the current query language.
-
-### <a id="Deletes">DELETEs</a>
-
- DeleteStatement ::= <DELETE> <FROM> QualifiedName ( ( <AS> )? Variable )? ( <WHERE> Expression )?
-
-The DELETE statement is used to delete data from a target dataset.
-The data to be deleted is identified by a boolean expression involving the variable bound to the target dataset in the DELETE statement.
-
-Deletes are processed transactionally by the system.
-The transactional scope of each delete transaction is the deletion of a single object plus its affiliated secondary index entries (if any).
-If the boolean expression for a delete identifies a single object, then the DELETE statement itself will be a single, atomic transaction.
-If the expression identifies multiple objects, then each object deleted will be handled as a separate transaction.
-
-The target dataset name may be a synonym introduced by CREATE SYNONYM statement.
-
-The following examples illustrate single-object deletions.
-
-##### Example
-
- DELETE FROM GleambookUsers user WHERE user.id = 8;
-
-##### Example
-
- DELETE FROM GleambookUsers WHERE id = 5;
-
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/5_ddl_function_removal.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/5_ddl_function_removal.md
deleted file mode 100644
index a0742a3..0000000
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/5_ddl_function_removal.md
+++ /dev/null
@@ -1,120 +0,0 @@
-<!--
- ! Licensed to the Apache Software Foundation (ASF) under one
- ! or more contributor license agreements. See the NOTICE file
- ! distributed with this work for additional information
- ! regarding copyright ownership. The ASF licenses this file
- ! to you under the Apache License, Version 2.0 (the
- ! "License"); you may not use this file except in compliance
- ! with the License. You may obtain a copy of the License at
- !
- ! http://www.apache.org/licenses/LICENSE-2.0
- !
- ! Unless required by applicable law or agreed to in writing,
- ! software distributed under the License is distributed on an
- ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ! KIND, either express or implied. See the License for the
- ! specific language governing permissions and limitations
- ! under the License.
- !-->
-
-### <a id="Functions"> Functions</a>
-
-The CREATE FUNCTION statement creates a **named** function that can then be used and reused in queries.
-The body of a function can be any query expression involving the function's parameters.
-
- FunctionSpecification ::= (<OR> <REPLACE>)? "FUNCTION" FunctionOrTypeName IfNotExists ParameterList "{" Expression "}"
-
-The following is an example of a CREATE FUNCTION statement which is similar to our earlier DECLARE FUNCTION example.
-It differs from that example in that it results in a function that is persistently registered by name in the specified dataverse (the current dataverse being used, if not otherwise specified).
-
-##### Example
-
- CREATE FUNCTION friendInfo(userId) {
- (SELECT u.id, u.name, len(u.friendIds) AS friendCount
- FROM GleambookUsers u
- WHERE u.id = userId)[0]
- };
-
-The following is an example of CREATE FUNCTION statement that replaces an existing function.
-
-##### Example
-
- CREATE OR REPLACE FUNCTION friendInfo(userId) {
- (SELECT u.id, u.name
- FROM GleambookUsers u
- WHERE u.id = userId)[0]
- };
-
-### <a id="Synonyms"> Synonyms</a>
-
- SynonymSpecification ::= "SYNONYM" QualifiedName IfNotExists "FOR" QualifiedName
-
-The CREATE SYNONYM statement creates a synonym for a given dataset.
-This synonym may be used used instead of the dataset name in SELECT, INSERT, UPSERT, DELETE, and LOAD statements.
-The target dataset does not need to exist when the synonym is created.
-
-##### Example
-
- CREATE DATASET GleambookUsers(GleambookUserType) PRIMARY KEY id;
-
- CREATE SYNONYM GleambookUsersSynonym FOR GleambookUsers;
-
- SELECT * FROM GleambookUsersSynonym;
-
-More information on how synonyms are resolved can be found in the appendix section on Variable Resolution.
-
-### <a id="Removal"> Removal</a>
-
- DropStatement ::= "DROP" ( "DATAVERSE" Identifier IfExists
- | "TYPE" FunctionOrTypeName IfExists
- | "DATASET" QualifiedName IfExists
- | "INDEX" DoubleQualifiedName IfExists
- | "SYNONYM" QualifiedName IfExists
- | "FUNCTION" FunctionSignature IfExists )
- IfExists ::= ( "IF" "EXISTS" )?
-
-The DROP statement is the inverse of the CREATE statement. It can be used to drop dataverses, datatypes, datasets, indexes, functions, and synonyms.
-
-The following examples illustrate some uses of the DROP statement.
-
-##### Example
-
- DROP DATASET GleambookUsers IF EXISTS;
-
- DROP INDEX GleambookMessages.gbSenderLocIndex;
-
- DROP TYPE TinySocial2.GleambookUserType;
-
- DROP FUNCTION friendInfo@1;
-
- DROP SYNONYM GleambookUsersSynonym;
-
- DROP DATAVERSE TinySocial;
-
-When an artifact is dropped, it will be droppped from the current dataverse if none is specified
-(see the DROP DATASET example above) or from the specified dataverse (see the DROP TYPE example above)
-if one is specified by fully qualifying the artifact name in the DROP statement.
-When specifying an index to drop, the index name must be qualified by the dataset that it indexes.
-When specifying a function to drop, since the query language allows functions to be overloaded by their number of arguments,
-the identifying name of the function to be dropped must explicitly include that information.
-(`friendInfo@1` above denotes the 1-argument function named friendInfo in the current dataverse.)
-
-### <a id="Load_statement">Load Statement</a>
-
- LoadStatement ::= <LOAD> <DATASET> QualifiedName <USING> AdapterName Configuration ( <PRE-SORTED> )?
-
-The LOAD statement is used to initially populate a dataset via bulk loading of data from an external file.
-An appropriate adapter must be selected to handle the nature of the desired external data.
-The LOAD statement accepts the same adapters and the same parameters as discussed earlier for External datasets.
-(See the [guide to external data](externaldata.html) for more information on the available adapters.)
-If a dataset has an auto-generated primary key field, the file to be imported should not include that field in it.
-
-The target dataset name may be a synonym introduced by CREATE SYNONYM statement.
-
-The following example shows how to bulk load the GleambookUsers dataset from an external file containing data that has been prepared in ADM (Asterix Data Model) format.
-
-##### Example
-
- LOAD DATASET GleambookUsers USING localfs
- (("path"="127.0.0.1:///Users/bignosqlfan/tinysocialnew/gbu.adm"),("format"="adm"));
-
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/5_ddl_head.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/5_ddl_head.md
deleted file mode 100644
index 83fa4c9..0000000
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/5_ddl_head.md
+++ /dev/null
@@ -1,36 +0,0 @@
-<!--
- ! Licensed to the Apache Software Foundation (ASF) under one
- ! or more contributor license agreements. See the NOTICE file
- ! distributed with this work for additional information
- ! regarding copyright ownership. The ASF licenses this file
- ! to you under the Apache License, Version 2.0 (the
- ! "License"); you may not use this file except in compliance
- ! with the License. You may obtain a copy of the License at
- !
- ! http://www.apache.org/licenses/LICENSE-2.0
- !
- ! Unless required by applicable law or agreed to in writing,
- ! software distributed under the License is distributed on an
- ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- ! KIND, either express or implied. See the License for the
- ! specific language governing permissions and limitations
- ! under the License.
- !-->
-
-# <a id="DDL_and_DML_statements">5. DDL and DML statements</a>
-
- Statement ::= ( ( SingleStatement )? ( ";" )+ )* <EOF>
- SingleStatement ::= DatabaseDeclaration
- | FunctionDeclaration
- | CreateStatement
- | DropStatement
- | LoadStatement
- | SetStatement
- | InsertStatement
- | DeleteStatement
- | Query
-
-In addition to queries, an implementation of the query language needs to support statements for data definition
-and manipulation purposes as well as controlling the context to be used in evaluating query expressions.
-This section details the DDL and DML statements supported in the query language as realized today in Apache AsterixDB.
-
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_dml.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_dml.md
index 0afa49c..222c7a4 100644
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_dml.md
+++ b/asterixdb/asterix-doc/src/main/markdown/sqlpp/7_ddl_dml.md
@@ -25,6 +25,8 @@
### UseStmnt
****
+### DataverseName
+****
---
@@ -36,11 +38,12 @@
USE Commerce;
### <a id="Sets"> Set Statement</a>
+
The `SET` statement can be used to override certain configuration parameters. More information about `SET` can be found in [Appendix 2](#Performance_tuning).
### <a id="Functions"> Function Declaration</a>
-When writing a complex query, it can sometimes be helpful to define one or more auxiliary functions that each address a sub-piece of the overall query.
+When writing a complex query, it can sometimes be helpful to define one or more auxiliary functions that each address a sub-piece of the overall query.
The `DECLARE FUNCTION` statement supports the creation of such helper functions.
In general, the function body (expression) can be any legal query expression.
@@ -51,10 +54,8 @@
### FunctionDeclaration
****
-### ParameterList
-****
-
---
+
The following is a simple example of a temporary function definition and its use.
##### Example
@@ -79,6 +80,9 @@
### CreateStmnt
****
+### DataverseName
+****
+
### QualifiedName
****
@@ -97,6 +101,7 @@
****
---
+
The `CREATE DATAVERSE` statement is used to create new dataverses.
To ease the authoring of reusable query scripts, an optional `IF NOT EXISTS` clause is included to allow
creation to be requested either unconditionally or only if the dataverse does not already exist.
@@ -109,6 +114,7 @@
CREATE DATAVERSE Commerce IF NOT EXISTS;
#### <a id="Types"> Create Type </a>
+
---
### CreateType
****
@@ -128,8 +134,8 @@
### MultisetTypeDef
****
-### TypeRef
-****
+### TypeReference
+****
---
@@ -143,7 +149,7 @@
The following example creates three new object type called `addressType` , `customerType` and `itemType`.
Their fields are essentially traditional typed name/value pairs (much like SQL fields).
-Since it is defined as (defaulting to) being an open type, instances will be permitted to contain more than what is specified in the type definition. Indeed many of the customer objects contain a rating as well, however this is not necessary for the customer object to be created. As can be seen in the sample data, customers can exist without ratings or with part (or all) of the address missing.
+Since it is defined as (defaulting to) being an open type, instances will be permitted to contain more than what is specified in the type definition. Indeed many of the customer objects contain a rating as well, however this is not necessary for the customer object to be created. As can be seen in the sample data, customers can exist without ratings or with part (or all) of the address missing.
##### Example
@@ -178,7 +184,7 @@
The next example creates a new object type, closed this time, called `orderType`.
Instances of this closed type will not be permitted to have extra fields,
-although the `ship_date` field is marked as optional and may thus be `NULL` or `MISSING` in legal instances of the type. The items field is an array of instances of another object type, `itemType`.
+although the `ship_date` field is marked as optional and may thus be `NULL` or `MISSING` in legal instances of the type. The items field is an array of instances of another object type, `itemType`.
##### Example
@@ -202,6 +208,21 @@
### CreateExternalDataset
****
+### DatasetTypeDef
+****
+
+### DatasetFieldDef
+****
+
+### TypeReference
+****
+
+### PrimaryKey
+****
+
+### NestedField
+****
+
### AdapterName
****
@@ -214,15 +235,6 @@
### Properties
****
-### PrimaryKey
-****
-
-### NestedField
-****
-
-### CompactionPolicy
-****
-
---
The `CREATE DATASET` statement is used to create a new dataset.
@@ -262,8 +274,8 @@
When defining an External dataset, an appropriate adapter type must be selected for the desired external data.
(See the [Guide to External Data](../aql/externaldata.html) for more information on the available adapters.)
-The following example creates an Internal dataset for storing FacefookUserType objects.
-It specifies that their id field is their primary key.
+The following example creates an Internal dataset for storing `customerType` objects.
+It specifies that their `custid` field is their primary key.
#### Example
@@ -278,6 +290,12 @@
CREATE DATASET MyItems(itemType) PRIMARY KEY itemno AUTOGENERATED;
+Alternatively the dataset object type can be specified using inline type definition syntax.
+
+#### Example
+
+ CREATE DATASET MyItems(itemno INT NOT UNKNOWN, qty INT NOT UNKNOWN, price INT NOT UNKNOWN) PRIMARY KEY itemno AUTOGENERATED;
+
The next example creates an External dataset for querying LineItemType objects.
The choice of the `hdfs` adapter means that this dataset's data actually resides in HDFS.
The example `CREATE` statement also provides parameters used by the hdfs adapter:
@@ -313,7 +331,6 @@
### IndexType
****
-
---
The `CREATE INDEX` statement creates a secondary index on one or more fields of a specified dataset.
@@ -440,7 +457,6 @@
### CreateSynonym
****
-
---
The `CREATE SYNONYM` statement creates a synonym for a given dataset.
@@ -455,7 +471,7 @@
SELECT * FROM customersSynonym;
-More information on how synonyms are resolved can be found in the appendix section on Variable Resolution.
+More information on how synonyms are resolved can be found in the [Appendix 3. Variable Bindings and Name Resolution](#Variable_bindings_and_name_resolution).
#### <a id="Create_function">Create Function</a>
@@ -469,8 +485,11 @@
### FunctionParameters
****
+### ExternalFunctionDef
+****
---
+
The following is an example of a `CREATE FUNCTION` statement which is similar to our earlier `DECLARE FUNCTION` example.
It differs from that example in that it results in a function that is persistently registered by name in the specified dataverse (the current dataverse being used, if not otherwise specified).
@@ -493,14 +512,22 @@
WHERE u.id = userId)[0]
};
+The following is an example of CREATE FUNCTION statement that introduces a function with a variable number of arguments.
+The arguments are accessible in the function body via `args` array parameter.
+
+##### Example
+
+ CREATE FUNCTION strJoin(...) {
+ string_join(args, ",")
+ };
+
External functions can also be loaded into Libraries via the [UDF API](../udf.html). Given
an already loaded library `pylib`, a function `sentiment` mapping to a Python method `sent_model.sentiment` in `sentiment_mod`
would be as follows
##### Example
- CREATE FUNCTION sentiment(a)
- AS "sentiment_mod", "sent_model.sentiment" AT pylib;
+ CREATE FUNCTION sentiment(a) AS "sentiment_mod", "sent_model.sentiment" AT pylib;
### <a id="Removal">Drop Statement</a>
@@ -508,9 +535,21 @@
### DropStmnt
****
+### DataverseName
+****
+
+### QualifiedName
+****
+
+### DoubleQualifiedName
+****
+
### FunctionSignature
****
+### FunctionParameters
+****
+
---
The `DROP` statement is the inverse of the `CREATE` statement. It can be used to drop dataverses, datatypes, datasets, indexes, functions, and synonyms.
@@ -537,7 +576,7 @@
When specifying an index to drop, the index name must be qualified by the dataset that it indexes.
When specifying a function to drop, since the query language allows functions to be overloaded by their number of arguments,
the identifying name of the function to be dropped must explicitly include that information.
-(`nameSearch@1` above denotes the 1-argument function named nameSearch in the current dataverse.)
+(`nameSearch@1` above denotes the 1-argument function named `nameSearch` in the current dataverse.)
### <a id="Load_statement">Load Statement</a>
@@ -545,6 +584,9 @@
### LoadStmnt
****
+### AdapterName
+****
+
### Configuration
****
@@ -576,7 +618,6 @@
### InsertStmnt
****
-
---
The `INSERT` statement is used to insert new data into a dataset.
@@ -625,12 +666,13 @@
UPSERT INTO custCopy (SELECT VALUE c FROM customers c)
### <a id="Deletes">Delete Statement</a>
+
---
### DeleteStmnt
****
-
---
+
The `DELETE` statement is used to delete data from a target dataset.
The data to be deleted is identified by a boolean expression involving the variable bound to the target dataset in the `DELETE` statement.
@@ -650,5 +692,3 @@
##### Example
DELETE FROM customers WHERE custid = "C47";
-
-
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/appendix_1_keywords.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/appendix_1_keywords.md
index d7c02d8..8da032c 100644
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/appendix_1_keywords.md
+++ b/asterixdb/asterix-doc/src/main/markdown/sqlpp/appendix_1_keywords.md
@@ -21,22 +21,24 @@
| | | | | | |
| ----|-----|-------|----|-----|----|
-| AND | ANY | APPLY | AS | ASC | AT |
-| AUTOGENERATED | BETWEEN | BTREE | BY | CASE | CLOSED |
-| CREATE | COMPACTION | COMPACT | CONNECT | CORRELATE | DATASET |
-| COLLECTION | DATAVERSE | DECLARE | DEFINITION | DECLARE | DEFINITION |
-| DELETE | DESC | DISCONNECT | DISTINCT | DROP | ELEMENT |
+| ADAPTER | ALL | AND | ANY | APPLY | AS |
+| ASC | AT | AUTOGENERATED | BETWEEN | BTREE | BY |
+| CASE | CLOSED | COLLECTION | CREATE | COMPACTION | COMPACT |
+| CONNECT | CORRELATE | DATASET | DATAVERSE | DECLARE | DEFINITION |
+| DELETE | DESC | DISCONNECT | DISTINCT | DIV | DROP |
| ELEMENT | EXPLAIN | ELSE | ENFORCED | END | EVERY |
| EXCEPT | EXIST | EXTERNAL | FEED | FILTER | FLATTEN |
-| FOR | FROM | FULL | FUNCTION | GROUP | HAVING |
-| HINTS | IF | INTO | IN | INDEX | INGESTION |
-| INNER | INSERT | INTERNAL | INTERSECT | IS | JOIN |
-| KEYWORD | LEFT | LETTING | LET | LIKE | LIMIT |
-| LOAD | NODEGROUP | NGRAM | NOT | OFFSET | ON |
-| OPEN | OR | ORDER | OUTER | OUTPUT | OVER |
-| PATH | POLICY | PRE-SORTED | PRIMARY | RAW | REFRESH |
-| RETURN | RTREE | RUN | SATISFIES | SECONDARY | SELECT |
-| SET | SOME | TEMPORARY | THEN | TYPE | UNKNOWN |
-| UNNEST | UPDATE | USE | USING | VALUE | WHEN |
-| WHERE | WITH | WRITE | | | |
+| FOR | FROM | FULL | FULLTEXT | FUNCTION | GROUP |
+| HAVING | HINTS | IF | INTO | IN | INDEX |
+| INGESTION | INNER | INSERT | INTERNAL | INTERSECT | IS |
+| JOIN | KEYWORD | LEFT | LETTING | LET | LIKE |
+| LIMIT | LOAD | MISSING | NODEGROUP | NGRAM | NOT |
+| NULL | OFFSET | ON | OPEN | OR | ORDER |
+| OUTER | OUTPUT | OVER | PATH | POLICY | PRE-SORTED |
+| PRIMARY | RAW | REFRESH | RETURN | RETURNING | RIGHT |
+| RTREE | RUN | SATISFIES | SECONDARY | SELECT | SET |
+| SOME | START | STOP | SYNONYM | TEMPORARY | THEN |
+| TO | TRUE | TYPE | UNION | UNKNOWN | UNNEST |
+| UPDATE | UPSERT | USE | USING | VALUE | VALUED |
+| WHEN | WHERE | WITH | WRITE | | |
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
index 326f53d..6be694b 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataConstants.java
@@ -304,15 +304,17 @@
throw new AssertionError("do not instantiate");
}
+ public static final String CONTAINER_NAME_FIELD_NAME = "container";
+ public static final String DEFINITION_FIELD_NAME = "definition";
+ public static final String CONNECTION_STRING_FIELD_NAME = "connectionString";
public static final String ACCOUNT_NAME_FIELD_NAME = "accountName";
public static final String ACCOUNT_KEY_FIELD_NAME = "accountKey";
public static final String SHARED_ACCESS_SIGNATURE_FIELD_NAME = "sharedAccessSignature";
- public static final String CONTAINER_NAME_FIELD_NAME = "container";
- public static final String DEFINITION_FIELD_NAME = "definition";
public static final String BLOB_ENDPOINT_FIELD_NAME = "blobEndpoint";
public static final String ENDPOINT_SUFFIX_FIELD_NAME = "endpointSuffix";
// Connection string requires PascalCase (MyFieldFormat)
+ public static final String CONNECTION_STRING_CONNECTION_STRING = "ConnectionString";
public static final String CONNECTION_STRING_ACCOUNT_NAME = "AccountName";
public static final String CONNECTION_STRING_ACCOUNT_KEY = "AccountKey";
public static final String CONNECTION_STRING_SHARED_ACCESS_SIGNATURE = "SharedAccessSignature";
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
index 44ae25f..537933e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataUtils.java
@@ -18,10 +18,17 @@
*/
package org.apache.asterix.external.util;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ACCOUNT_KEY_FIELD_NAME;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ACCOUNT_NAME_FIELD_NAME;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.BLOB_ENDPOINT_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_ACCOUNT_KEY;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_ACCOUNT_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_BLOB_ENDPOINT;
import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_ENDPOINT_SUFFIX;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_FIELD_NAME;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.CONNECTION_STRING_SHARED_ACCESS_SIGNATURE;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.ENDPOINT_SUFFIX_FIELD_NAME;
+import static org.apache.asterix.external.util.ExternalDataConstants.AzureBlob.SHARED_ACCESS_SIGNATURE_FIELD_NAME;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_DELIMITER;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_ESCAPE;
import static org.apache.asterix.external.util.ExternalDataConstants.KEY_QUOTE;
@@ -819,25 +826,70 @@
public static BlobServiceClient buildAzureClient(Map<String, String> configuration)
throws CompilationException {
// TODO(Hussain): Need to ensure that all required parameters are present in a previous step
- String accountName = configuration.get(ExternalDataConstants.AzureBlob.ACCOUNT_NAME_FIELD_NAME);
- String accountKey = configuration.get(ExternalDataConstants.AzureBlob.ACCOUNT_KEY_FIELD_NAME);
- String blobEndpoint = configuration.get(ExternalDataConstants.AzureBlob.BLOB_ENDPOINT_FIELD_NAME);
- String endpointSuffix = configuration.get(ExternalDataConstants.AzureBlob.ENDPOINT_SUFFIX_FIELD_NAME);
+ String connectionString = configuration.get(CONNECTION_STRING_FIELD_NAME);
+ String accountName = configuration.get(ACCOUNT_NAME_FIELD_NAME);
+ String accountKey = configuration.get(ACCOUNT_KEY_FIELD_NAME);
+ String sharedAccessSignature = configuration.get(SHARED_ACCESS_SIGNATURE_FIELD_NAME);
+ String blobEndpoint = configuration.get(BLOB_ENDPOINT_FIELD_NAME);
+ String endpointSuffix = configuration.get(ENDPOINT_SUFFIX_FIELD_NAME);
- // format: name1=value1;name2=value2;....
- // TODO(Hussain): This will be different when SAS (Shared Access Signature) is introduced
- StringBuilder connectionString = new StringBuilder();
- connectionString.append(CONNECTION_STRING_ACCOUNT_NAME).append("=").append(accountName).append(";");
- connectionString.append(CONNECTION_STRING_ACCOUNT_KEY).append("=").append(accountKey).append(";");
- connectionString.append(CONNECTION_STRING_BLOB_ENDPOINT).append("=").append(blobEndpoint).append(";");
+ // Constructor the connection string
+ // Connection string format: name1=value1;name2=value2;....
+ StringBuilder connectionStringBuilder = new StringBuilder();
+ BlobServiceClientBuilder builder = new BlobServiceClientBuilder();
- if (endpointSuffix != null) {
- connectionString.append(CONNECTION_STRING_ENDPOINT_SUFFIX).append("=").append(endpointSuffix)
+ boolean authMethodFound = false;
+
+ if (connectionString != null) {
+ // connection string
+ authMethodFound = true;
+ connectionStringBuilder.append(connectionString).append(";");
+ }
+
+ if (accountName != null && accountKey != null) {
+ if (authMethodFound) {
+ throw new CompilationException(ErrorCode.ONLY_SINGLE_AUTHENTICATION_IS_ALLOWED);
+ }
+ authMethodFound = true;
+ // account name + account key
+ connectionStringBuilder.append(CONNECTION_STRING_ACCOUNT_NAME).append("=").append(accountName)
+ .append(";").append(CONNECTION_STRING_ACCOUNT_KEY).append("=").append(accountKey).append(";");
+ }
+
+ if (accountName != null && sharedAccessSignature != null) {
+ if (authMethodFound) {
+ throw new CompilationException(ErrorCode.ONLY_SINGLE_AUTHENTICATION_IS_ALLOWED);
+ }
+ authMethodFound = true;
+ // account name + shared access token
+ connectionStringBuilder.append(CONNECTION_STRING_ACCOUNT_NAME).append("=").append(accountName)
+ .append(";").append(CONNECTION_STRING_SHARED_ACCESS_SIGNATURE).append("=")
+ .append(sharedAccessSignature).append(";");
+ }
+
+ if (!authMethodFound) {
+ throw new CompilationException(ErrorCode.NO_AUTH_METHOD_PROVIDED);
+ }
+
+ // Add blobEndpoint and endpointSuffix if present, adjust any '/' as needed
+ if (blobEndpoint != null) {
+ connectionStringBuilder.append(CONNECTION_STRING_BLOB_ENDPOINT).append("=").append(blobEndpoint)
.append(";");
+ if (endpointSuffix != null) {
+ String endpointSuffixUpdated;
+ if (blobEndpoint.endsWith("/")) {
+ endpointSuffixUpdated =
+ endpointSuffix.startsWith("/") ? endpointSuffix.substring(1) : endpointSuffix;
+ } else {
+ endpointSuffixUpdated = endpointSuffix.startsWith("/") ? endpointSuffix : "/" + endpointSuffix;
+ }
+ connectionStringBuilder.append(CONNECTION_STRING_ENDPOINT_SUFFIX).append("=")
+ .append(endpointSuffixUpdated).append(";");
+ }
}
try {
- return new BlobServiceClientBuilder().connectionString(connectionString.toString()).buildClient();
+ return builder.connectionString(connectionStringBuilder.toString()).buildClient();
} catch (Exception ex) {
throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex.getMessage());
}
@@ -876,6 +928,8 @@
WarningUtil.forAsterix(srcLoc, ErrorCode.EXTERNAL_SOURCE_CONFIGURATION_RETURNED_NO_FILES);
collector.warn(warning);
}
+ } catch (CompilationException ex) {
+ throw ex;
} catch (Exception ex) {
throw new CompilationException(ErrorCode.EXTERNAL_SOURCE_ERROR, ex.getMessage());
}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index 5403272..56c7163 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -1133,17 +1133,6 @@
}
}
-String CompactionPolicy() throws ParseException :
-{
- String compactionPolicy = null;
-}
-{
- compactionPolicy = Identifier()
- {
- return compactionPolicy;
- }
-}
-
String FilterField() throws ParseException :
{
String filterField = null;
@@ -4729,7 +4718,7 @@
| <CASE : "case">
| <CLOSED : "closed">
| <CREATE : "create">
- | <COMPACTION : "compaction">
+ | <COMPACTION : "compaction"> // no longer used
| <COMPACT : "compact">
| <CONNECT : "connect">
| <CORRELATE : "correlate">
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index a6d4e7f..f8992d5 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -1542,6 +1542,53 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>com.azure</groupId>
+ <artifactId>azure-storage-common</artifactId>
+ <version>${azurejavasdk.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-handler</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-handler-proxy</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-http</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-codec-http2</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-buffer</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-native-epoll</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-transport-native-unix-common</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-tcnative-boringssl-static</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<!-- Azure Blob Storage end -->
<dependency>
<groupId>org.mindrot</groupId>
diff --git a/hyracks-fullstack/NOTICE b/hyracks-fullstack/NOTICE
index 95fe98a..57c58439 100644
--- a/hyracks-fullstack/NOTICE
+++ b/hyracks-fullstack/NOTICE
@@ -1,5 +1,5 @@
Apache Hyracks and Algebricks
-Copyright 2015-2020 The Apache Software Foundation
+Copyright 2015-2021 The Apache Software Foundation
This product includes software developed at
The Apache Software Foundation (http://www.apache.org/).
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java
index 1d8c774..79687ab 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/rewriter/runtime/SuperActivityOperatorNodePushable.java
@@ -55,7 +55,7 @@
* connected activities in a single thread.
*/
public class SuperActivityOperatorNodePushable implements IOperatorNodePushable {
- private static final String CLASS_SIMPLE_NAME = SuperActivityOperatorNodePushable.class.getSimpleName();
+ private static final String CLASS_ABBREVIATION = "SA";
private final Map<ActivityId, IOperatorNodePushable> operatorNodePushables = new HashMap<>();
private final List<IOperatorNodePushable> operatorNodePushablesBFSOrder = new ArrayList<>();
private final Map<ActivityId, IActivity> startActivities;
@@ -224,7 +224,7 @@
try {
Thread.currentThread()
.setName(Thread.currentThread().getName() + ":" + ctx.getJobletContext().getJobId()
- + ":" + ctx.getTaskAttemptId() + ":" + CLASS_SIMPLE_NAME);
+ + ":" + ctx.getTaskAttemptId() + ":" + CLASS_ABBREVIATION);
action.run(op);
} catch (Throwable th) { // NOSONAR: Must catch all causes of failure
failures.offer(th);
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
index 93762a6..97b8859 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServer.java
@@ -20,6 +20,9 @@
import java.io.IOException;
import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -39,6 +42,7 @@
import io.netty.bootstrap.ServerBootstrap;
import io.netty.buffer.PooledByteBufAllocator;
import io.netty.channel.Channel;
+import io.netty.channel.ChannelFuture;
import io.netty.channel.ChannelInitializer;
import io.netty.channel.ChannelOption;
import io.netty.channel.EventLoopGroup;
@@ -50,6 +54,8 @@
import io.netty.handler.codec.http.HttpScheme;
import io.netty.handler.logging.LogLevel;
import io.netty.handler.logging.LoggingHandler;
+import io.netty.util.concurrent.Future;
+import io.netty.util.concurrent.GenericFutureListener;
public class HttpServer {
// Constants
@@ -64,6 +70,7 @@
private static final int STARTING = 1;
private static final int STARTED = 2;
private static final int STOPPING = 3;
+ private static final int RECOVERING = 4;
// Final members
private final IChannelClosedHandler closedHandler;
private final Object lock = new Object();
@@ -73,38 +80,59 @@
private final ServletRegistry servlets;
private final EventLoopGroup bossGroup;
private final EventLoopGroup workerGroup;
- private final InetSocketAddress address;
+ private final InetSocketAddress defaultAddress;
+ private final List<InetSocketAddress> addresses;
private final ThreadPoolExecutor executor;
// Mutable members
private volatile int state = STOPPED;
private volatile Thread recoveryThread;
- private volatile Channel channel;
+ private final List<Channel> channels;
private Throwable cause;
private HttpServerConfig config;
- public HttpServer(EventLoopGroup bossGroup, EventLoopGroup workerGroup, int port, HttpServerConfig config) {
- this(bossGroup, workerGroup, new InetSocketAddress(port), config, null);
- }
+ private final GenericFutureListener<Future<Void>> channelCloseListener = f -> {
+ // This listener is invoked from within a netty IO thread. Hence, we can never block it
+ // For simplicity, we will submit the recovery task to a different thread. We will also
+ // close all channels on this server and attempt to rebind them.
+ synchronized (lock) {
+ if (state != STARTED) {
+ return;
+ }
+ LOGGER.log(Level.WARN, "{} has stopped unexpectedly. Starting server recovery", this);
+ MXHelper.logFileDescriptors();
+ state = RECOVERING;
+ triggerRecovery();
+ }
+ };
- public HttpServer(EventLoopGroup bossGroup, EventLoopGroup workerGroup, InetSocketAddress address,
- HttpServerConfig config) {
- this(bossGroup, workerGroup, address, config, null);
+ public HttpServer(EventLoopGroup bossGroup, EventLoopGroup workerGroup, int port, HttpServerConfig config) {
+ this(bossGroup, workerGroup, Collections.singletonList(new InetSocketAddress(port)), config, null);
}
public HttpServer(EventLoopGroup bossGroup, EventLoopGroup workerGroup, InetSocketAddress address,
HttpServerConfig config, IChannelClosedHandler closeHandler) {
+ this(bossGroup, workerGroup, Collections.singletonList(address), config, closeHandler);
+ }
+
+ public HttpServer(EventLoopGroup bossGroup, EventLoopGroup workerGroup, List<InetSocketAddress> addresses,
+ HttpServerConfig config, IChannelClosedHandler closeHandler) {
+ if (addresses.isEmpty()) {
+ throw new IllegalArgumentException("no addresses specified");
+ }
this.bossGroup = bossGroup;
this.workerGroup = workerGroup;
- this.address = address;
+ this.addresses = addresses;
+ defaultAddress = addresses.get(0);
this.closedHandler = closeHandler;
this.config = config;
+ channels = new ArrayList<>();
ctx = new ConcurrentHashMap<>();
servlets = new ServletRegistry();
workQueue = new LinkedBlockingQueue<>(config.getRequestQueueSize());
int numExecutorThreads = config.getThreadCount();
executor = new ThreadPoolExecutor(numExecutorThreads, numExecutorThreads, 0L, TimeUnit.MILLISECONDS, workQueue,
runnable -> new Thread(runnable,
- "HttpExecutor(port:" + address.getPort() + ")-" + threadId.getAndIncrement()));
+ "HttpExecutor(port:" + defaultAddress.getPort() + ")-" + threadId.getAndIncrement()));
long directMemoryBudget = numExecutorThreads * (long) HIGH_WRITE_BUFFER_WATER_MARK
+ numExecutorThreads * config.getMaxResponseChunkSize();
LOGGER.log(Level.DEBUG,
@@ -128,7 +156,7 @@
doStart();
setStarted();
} catch (Throwable e) { // NOSONAR
- LOGGER.error("Failure starting an Http Server at: {}", address, e);
+ LOGGER.error("Failure starting an Http Server at: {}", defaultAddress, e);
setFailed(e);
throw e;
}
@@ -175,6 +203,8 @@
return "STOPPING";
case STOPPED:
return "STOPPED";
+ case RECOVERING:
+ return "RECOVERING";
default:
return "UNKNOWN";
}
@@ -229,10 +259,10 @@
for (IServlet servlet : servlets.getServlets()) {
servlet.init();
}
- channel = bind();
+ bind();
}
- private Channel bind() throws InterruptedException {
+ private void bind() throws InterruptedException {
ServerBootstrap b = new ServerBootstrap();
b.group(bossGroup, workerGroup).channel(NioServerSocketChannel.class)
.childOption(ChannelOption.RCVBUF_ALLOCATOR, new FixedRecvByteBufAllocator(RECEIVE_BUFFER_SIZE))
@@ -240,23 +270,20 @@
.childOption(ChannelOption.ALLOCATOR, PooledByteBufAllocator.DEFAULT)
.childOption(ChannelOption.WRITE_BUFFER_WATER_MARK, WRITE_BUFFER_WATER_MARK)
.handler(new LoggingHandler(LogLevel.DEBUG)).childHandler(getChannelInitializer());
- Channel newChannel = b.bind(address).sync().channel();
- newChannel.closeFuture().addListener(f -> {
- // This listener is invoked from within a netty IO thread. Hence, we can never block it
- // For simplicity, we will submit the recovery task to a different thread
+ List<ChannelFuture> channelFutures = new ArrayList<>();
+ for (InetSocketAddress address : addresses) {
+ channelFutures.add(b.bind(address));
+ }
+ for (ChannelFuture future : channelFutures) {
+ Channel channel = future.sync().channel();
+ channel.closeFuture().addListener(channelCloseListener);
synchronized (lock) {
- if (state != STARTED) {
- return;
- }
- LOGGER.log(Level.WARN, "{} has stopped unexpectedly. Starting server recovery", this);
- MXHelper.logFileDescriptors();
- triggerRecovery();
+ channels.add(channel);
}
- });
- return newChannel;
+ }
}
- private void triggerRecovery() {
+ private void triggerRecovery() throws InterruptedException {
Thread rt = recoveryThread;
if (rt != null) {
try {
@@ -267,7 +294,7 @@
return;
}
}
- // try to revive the channel
+ // try to revive the channels
recoveryThread = new Thread(this::recover);
recoveryThread.start();
}
@@ -275,9 +302,11 @@
public void recover() {
try {
synchronized (lock) {
- while (state == STARTED) {
+ while (state == RECOVERING) {
try {
- channel = bind();
+ closeChannels();
+ bind();
+ setStarted();
break;
} catch (InterruptedException e) {
LOGGER.log(Level.WARN, this + " was interrupted while attempting to revive server channel", e);
@@ -329,10 +358,7 @@
} catch (Exception e) {
LOGGER.log(Level.ERROR, "Error while shutting down http server executor", e);
}
- if (channel != null) {
- channel.close();
- channel.closeFuture().sync();
- }
+ closeChannels();
}
public IServlet getServlet(FullHttpRequest request) {
@@ -369,8 +395,8 @@
@Override
public String toString() {
- return "{\"class\":\"" + getClass().getSimpleName() + "\",\"address\":" + address + ",\"state\":\"" + getState()
- + "\"}";
+ return "{\"class\":\"" + getClass().getSimpleName() + "\",\"address\":" + defaultAddress + ",\"state\":\""
+ + getState() + "\"}";
}
public HttpServerConfig getConfig() {
@@ -378,6 +404,17 @@
}
public InetSocketAddress getAddress() {
- return address;
+ return defaultAddress;
+ }
+
+ private void closeChannels() throws InterruptedException {
+ synchronized (lock) {
+ for (Channel channel : channels) {
+ channel.closeFuture().removeListener(channelCloseListener);
+ channel.close();
+ channel.closeFuture().sync();
+ }
+ channels.clear();
+ }
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpServerTest.java b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpServerTest.java
index 84c8c65..76438aa 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpServerTest.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/test/java/org/apache/hyracks/test/http/HttpServerTest.java
@@ -28,6 +28,7 @@
import java.net.URI;
import java.net.URISyntaxException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
@@ -263,7 +264,10 @@
WebManager webMgr = new WebManager();
final HttpServerConfig config = HttpServerConfigBuilder.custom().setThreadCount(numExecutors)
.setRequestQueueSize(serverQueueSize).build();
- HttpServer server = new HttpServer(webMgr.getBosses(), webMgr.getWorkers(), PORT, config);
+ List<InetSocketAddress> addresses = new ArrayList<>();
+ addresses.add(new InetSocketAddress(PORT));
+ addresses.add(new InetSocketAddress(PORT + 1));
+ HttpServer server = new HttpServer(webMgr.getBosses(), webMgr.getWorkers(), addresses, config, null);
ChattyServlet servlet = new ChattyServlet(server.ctx(), new String[] { PATH });
server.addServlet(servlet);
webMgr.add(server);
@@ -276,12 +280,12 @@
}
Assert.assertEquals(numRequests, SUCCESS_COUNT.get());
// close the channel
- Field channelField = server.getClass().getDeclaredField("channel");
+ Field channelField = server.getClass().getDeclaredField("channels");
channelField.setAccessible(true);
Field recoveryThreadField = server.getClass().getDeclaredField("recoveryThread");
recoveryThreadField.setAccessible(true);
- Channel channel = (Channel) channelField.get(server);
- channel.close();
+ List<Channel> channels = (ArrayList<Channel>) channelField.get(server);
+ channels.get(0).close();
Thread.sleep(1000);
final int sleeps = 10;
for (int i = 0; i < sleeps; i++) {
@@ -409,6 +413,43 @@
}
}
+ @Test
+ public void multiAddressServerTest() throws Exception {
+ final WebManager webMgr = new WebManager();
+ final HttpServerConfig config =
+ HttpServerConfigBuilder.custom().setThreadCount(16).setRequestQueueSize(16).build();
+ List<Integer> ports = Arrays.asList(PORT, PORT + 1);
+ List<InetSocketAddress> addresses = new ArrayList<>();
+ for (Integer port : ports) {
+ addresses.add(new InetSocketAddress(port));
+ }
+ HttpServer server = new HttpServer(webMgr.getBosses(), webMgr.getWorkers(), addresses, config, null);
+ EchoServlet servlet = new EchoServlet(server.ctx(), PATH);
+ server.addServlet(servlet);
+ webMgr.add(server);
+ webMgr.start();
+ try {
+ for (Integer port : ports) {
+ try (CloseableHttpClient httpClient = HttpClients.createDefault()) {
+ final URI uri = new URI(HttpServerTest.PROTOCOL, null, HttpServerTest.HOST, port,
+ HttpServerTest.PATH, null, null);
+ final HttpPost postRequest = new HttpPost(uri);
+ final String requestBody = "test";
+ final StringEntity chunkedEntity = new StringEntity(requestBody);
+ chunkedEntity.setChunked(true);
+ postRequest.setEntity(chunkedEntity);
+ try (CloseableHttpResponse response = httpClient.execute(postRequest)) {
+ final String responseBody = EntityUtils.toString(response.getEntity());
+ Assert.assertEquals(response.getStatusLine().getStatusCode(), HttpResponseStatus.OK.code());
+ Assert.assertEquals(responseBody, requestBody);
+ }
+ }
+ }
+ } finally {
+ webMgr.stop();
+ }
+ }
+
private void request(int count) throws URISyntaxException {
request(count, 0);
}