Merge branch 'gerrit/stabilization-40cfb8705b' into 'gerrit/neo'
Change-Id: If5ad779d54d15d70c0a05785271566d29da7f9e3
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CancelUnnestWithNestedListifyRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CancelUnnestWithNestedListifyRule.java
index 069ba49..bcb59b4 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CancelUnnestWithNestedListifyRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CancelUnnestWithNestedListifyRule.java
@@ -194,7 +194,7 @@
AggregateOperator agg = (AggregateOperator) nestedPlanRoot;
Mutable<ILogicalOperator> aggInputOpRef = agg.getInputs().get(0);
- if (agg.getVariables().size() > 1) {
+ if (agg.getVariables().size() != 1) {
return false;
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveRedundantListifyRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveRedundantListifyRule.java
index f968b35..0f490be 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveRedundantListifyRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/RemoveRedundantListifyRule.java
@@ -200,7 +200,7 @@
return false;
}
AggregateOperator agg = (AggregateOperator) r;
- if (agg.getVariables().size() > 1) {
+ if (agg.getVariables().size() != 1) {
return false;
}
LogicalVariable aggVar = agg.getVariables().get(0);
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/RecoveryTask.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/RecoveryTask.java
index 34a54d1..e4e7b36 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/RecoveryTask.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/RecoveryTask.java
@@ -140,6 +140,7 @@
if (!cancelRecovery && listener.getState() == ActivityState.TEMPORARILY_FAILED) {
listener.setState(ActivityState.RECOVERING);
listener.doRecover(metadataProvider);
+ listener.setRunning(metadataProvider, true);
}
LOGGER.log(level, "Recovery completed successfully");
return null;
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index 4d32967..61656b7 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -22,6 +22,7 @@
import java.rmi.RemoteException;
import java.rmi.server.UnicastRemoteObject;
import java.util.Collection;
+import java.util.OptionalInt;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -160,6 +161,7 @@
private ICacheManager cacheManager;
private IConfigValidator configValidator;
private IDiskWriteRateLimiterProvider diskWriteRateLimiterProvider;
+ private Integer metadataPartitionId;
public NCAppRuntimeContext(INCServiceContext ncServiceContext, NCExtensionManager extensionManager,
IPropertiesFactory propertiesFactory) {
@@ -443,6 +445,7 @@
@Override
public void initializeMetadata(boolean newUniverse, int partitionId) throws Exception {
LOGGER.info("Bootstrapping metadata");
+ metadataPartitionId = partitionId;
MetadataNode.INSTANCE.initialize(this, ncExtensionManager.getMetadataTupleTranslatorProvider(),
ncExtensionManager.getMetadataExtensions(), partitionId);
@@ -636,4 +639,9 @@
public IDiskWriteRateLimiterProvider getDiskWriteRateLimiterProvider() {
return diskWriteRateLimiterProvider;
}
+
+ @Override
+ public OptionalInt getMetadataPartitionId() {
+ return metadataPartitionId == null ? OptionalInt.empty() : OptionalInt.of(metadataPartitionId);
+ }
}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPrimaryIndexOperationTrackerFactory.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPrimaryIndexOperationTrackerFactory.java
index c4390fa..38fdf56 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPrimaryIndexOperationTrackerFactory.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dataflow/TestPrimaryIndexOperationTrackerFactory.java
@@ -19,7 +19,6 @@
package org.apache.asterix.test.dataflow;
import java.lang.reflect.Field;
-import java.lang.reflect.Modifier;
import java.util.Map;
import org.apache.asterix.common.api.INcApplicationContext;
@@ -70,20 +69,10 @@
}
}
- static void setFinal(Field field, Object obj, Object newValue) throws Exception {
- field.setAccessible(true);
- Field modifiersField = Field.class.getDeclaredField("modifiers");
- modifiersField.setAccessible(true);
- modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
- field.set(obj, newValue);
- }
-
@SuppressWarnings({ "rawtypes", "unchecked" })
static void replaceMapEntry(Field field, Object obj, Object key, Object value)
throws Exception, IllegalAccessException {
field.setAccessible(true);
- Field modifiersField = Field.class.getDeclaredField("modifiers");
- modifiersField.setAccessible(true);
Map map = (Map) field.get(obj);
map.put(key, value);
}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3403/query-ASTERIXDB-3403.1.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3403/query-ASTERIXDB-3403.1.ddl.sqlpp
new file mode 100644
index 0000000..0b94eb1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3403/query-ASTERIXDB-3403.1.ddl.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description: This test case is to verify the fix for ASTERIXDB-3403
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+
+use test;
+
+create type dt1 as {id:int};
+create dataset collection1(dt1) primary key id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3403/query-ASTERIXDB-3403.2.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3403/query-ASTERIXDB-3403.2.update.sqlpp
new file mode 100644
index 0000000..327d1d6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3403/query-ASTERIXDB-3403.2.update.sqlpp
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+insert into collection1
+([
+ {
+ "id": 1,
+ "f1": "f1",
+ "x1": [{"date":"01-02-2024", "item": '1234', "cnt":2}]
+
+ },
+ {
+ "id": 2,
+ "f1": "f1",
+ "x2": [{"date":"01-02-2024", "item": '5678', "cnt":2}]
+ },
+ {
+ "id": 3,
+ "f1": "f1",
+ "x3": [{"su": {"x4":2}, "item": "i1", "cnt":2}]
+ },
+ {
+ "id": 4,
+ "f1": "f1",
+ "x3": [{"su": {"x4":5}, "item": 1234, "cnt":2}]
+ }
+]);
+
+
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3403/query-ASTERIXDB-3403.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3403/query-ASTERIXDB-3403.3.query.sqlpp
new file mode 100644
index 0000000..ca70034
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/query-ASTERIXDB-3403/query-ASTERIXDB-3403.3.query.sqlpp
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Description: This test case is to verify the fix for ASTERIXDB-3403
+ */
+
+use test;
+
+ SELECT COUNT(id) AS matches
+ FROM collection1 AS d
+ WHERE d.`f1` = 'f1'
+ AND (ARRAY_SUM((
+ SELECT VALUE i.`cnt`
+ FROM d.`x1` AS i
+ WHERE i.`date` BETWEEN "01-01-2024" AND "02-02-2024"
+ AND i.`item` IN ['1234', '5678'] )) >= 1
+ OR ARRAY_SUM((
+ SELECT VALUE i.`cnt`
+ FROM d.`x2` AS i
+ WHERE i.`date` BETWEEN "01-01-2024" AND "02-02-2024"
+ AND i.`item` IN ['1234', '5678'] )) >= 1
+ OR (ANY e IN d.x3 SATISFIES e.item IN ['i1', 'i2', 'i3']
+ AND e.su.`x4` >= 1 END));
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-3403/query-ASTERIXDB-3403.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-3403/query-ASTERIXDB-3403.3.adm
new file mode 100644
index 0000000..433b8ee
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/query-ASTERIXDB-3403/query-ASTERIXDB-3403.3.adm
@@ -0,0 +1 @@
+{ "matches": 3 }
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index b6dbf7f..66e47ba 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -7300,6 +7300,11 @@
<output-dir compare="Text">query-ASTERIXDB-3418</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="misc">
+ <compilation-unit name="query-ASTERIXDB-3403">
+ <output-dir compare="Text">query-ASTERIXDB-3403</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="multipart-dataverse">
<test-case FilePath="multipart-dataverse">
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java
index 5475b97..a5ae099 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/api/INcApplicationContext.java
@@ -20,6 +20,7 @@
import java.io.IOException;
import java.rmi.RemoteException;
+import java.util.OptionalInt;
import java.util.concurrent.Executor;
import org.apache.asterix.common.context.IStorageComponentProvider;
@@ -147,4 +148,6 @@
* @return the disk write rate limiter provider
*/
IDiskWriteRateLimiterProvider getDiskWriteRateLimiterProvider();
+
+ OptionalInt getMetadataPartitionId();
}
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java
index 0bb9bd5..1565730 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java
@@ -173,13 +173,16 @@
// 2. there are still some active readers and memory cannot be reclaimed.
// But for both cases, we will notify all primary index op trackers to let their writers retry,
// if they have been blocked. Moreover, we will check whether more flushes are needed.
+ List<ILSMOperationTracker> opTrackers = new ArrayList<>();
synchronized (this) {
final int size = primaryIndexes.size();
for (int i = 0; i < size; i++) {
- ILSMOperationTracker opTracker = primaryIndexes.get(i).getOperationTracker();
- synchronized (opTracker) {
- opTracker.notifyAll();
- }
+ opTrackers.add(primaryIndexes.get(i).getOperationTracker());
+ }
+ }
+ for (ILSMOperationTracker opTracker : opTrackers) {
+ synchronized (opTracker) {
+ opTracker.notifyAll();
}
}
checkAndNotifyFlushThread();
diff --git a/asterixdb/asterix-doc/src/main/markdown/builtins/9_aggregate_sql.md b/asterixdb/asterix-doc/src/main/markdown/builtins/9_aggregate_sql.md
index 0657fb0..755fd39 100644
--- a/asterixdb/asterix-doc/src/main/markdown/builtins/9_aggregate_sql.md
+++ b/asterixdb/asterix-doc/src/main/markdown/builtins/9_aggregate_sql.md
@@ -48,8 +48,8 @@
* or, a `missing` value.
* Return Value:
* a `bigint` value representing the number of non-null and non-missing items in the given collection,
- * `null` is returned if the input is `null` or `missing`,
- * any other non-array and non-multiset input value will cause an error.
+ * `0` is returned if the input is `null` or `missing`,
+ * `0` is returned if the input is not an array or a multiset.
* Example:
@@ -77,8 +77,8 @@
* a `double` value representing the average of the non-null and non-missing numbers in the given collection,
* `null` is returned if the input is `null` or `missing`,
* `null` is returned if the given collection does not contain any non-null and non-missing items,
- * any other non-array and non-multiset input value will cause a type error,
- * any other non-numeric value in the input collection will cause a type error.
+ * `null` is returned if the input is not an array or a multiset,
+ * any other non-numeric value in the input collection will be ignored.
* Example:
@@ -107,8 +107,8 @@
items.
* `null` is returned if the input is `null` or `missing`,
* `null` is returned if the given collection does not contain any non-null and non-missing items,
- * any other non-array and non-multiset input value will cause a type error,
- * any other non-numeric value in the input collection will cause a type error.
+ * `null` is returned if the input is not an array or a multiset,
+ * any other non-numeric value in the input collection will be ignored.
* Example:
@@ -136,8 +136,8 @@
type promotion order (`tinyint`-> `smallint`->`integer`->`bigint`->`float`->`double`) among numeric items.
* `null` is returned if the input is `null` or `missing`,
* `null` is returned if the given collection does not contain any non-null and non-missing items,
- * multiple incomparable items in the input array or multiset will cause a type error,
- * any other non-array and non-multiset input value will cause a type error.
+ * `null` is returned if there are incomparable items in the input array or multiset,
+ * `null` is returned if the input is not an array or a multiset.
* Example:
@@ -165,8 +165,8 @@
type promotion order (`tinyint`-> `smallint`->`integer`->`bigint`->`float`->`double`) among numeric items.
* `null` is returned if the input is `null` or `missing`,
* `null` is returned if the given collection does not contain any non-null and non-missing items,
- * multiple incomparable items in the input array or multiset will cause a type error,
- * any other non-array and non-multiset input value will cause a type error.
+ * `null` is returned if there are incomparable items in the input array or multiset,
+ * `null` is returned if the input is not an array or a multiset.
* Example:
@@ -177,6 +177,44 @@
3.4
+### array_median ###
+ * Syntax:
+
+ array_median(num_collection)
+
+ * Gets the median value of the numeric items in the given collection, ignoring null, missing, and non-numeric items.
+
+ The function starts by sorting the numeric items.
+
+ - If there is an odd number of numeric items, the function returns the item that is exactly in the middle of the range: that is, it has the same number of items before and after.
+ - If there is an even number of numeric items, the function returns the mean of the two items that are exactly in the middle of the range.
+
+ * Note: You cannot use the `DISTINCT` keyword with this function, or with the `median` aggregation pseudo-function.
+ The `median` aggregation pseudo-function does support the `FILTER` clause.
+ There is no `strict_median` function corresponding to this function.
+ * Arguments:
+ * `num_collection` could be:
+ * an `array` or `multiset` of numbers,
+ * or, a `null` value,
+ * or, a `missing` value.
+ * Clauses: When used as a window function, this function supports the [Window Partition Clause](manual.html#Window_partition_clause), but not the [Window Order Clause](manual.html#Window_order_clause) or the [Window Frame Clause](manual.html#Window_frame_clause).
+ * Return Value:
+ * a `double` value representing the median of the numeric items in the given collection,
+ * `null` is returned if the input is `null` or `missing`,
+ * `null` is returned if the given collection does not contain any numeric items,
+ * `null` is returned if the input is not an array or a multiset,
+ * any other non-numeric value in the input collection will be ignored.
+ * Example:
+
+ { "v1": array_median( [1.2, 2.3, 3.4, 0, null, missing],
+ "v2": array_median( [1.2, 2.3, 3.4, 4.5, 0, null, missing] ) };
+
+ * The expected result is:
+
+ { "v1": 1.75,
+ "v2": 2.3 }
+
+
### array_stddev_samp ###
* Syntax:
@@ -193,7 +231,7 @@
* a `double` value representing the sample standard deviation of the non-null and non-missing numbers in the given collection,
* `null` is returned if the input is `null` or `missing`,
* `null` is returned if the given collection does not contain any non-null and non-missing items,
- * any other non-array and non-multiset input value will cause a type error,
+ * `null` is returned if the input is not an array or a multiset,
* any other non-numeric value in the input collection will cause a type error.
* Example:
@@ -220,7 +258,7 @@
* a `double` value representing the population standard deviation of the non-null and non-missing numbers in the given collection,
* `null` is returned if the input is `null` or `missing`,
* `null` is returned if the given collection does not contain any non-null and non-missing items,
- * any other non-array and non-multiset input value will cause a type error,
+ * `null` is returned if the input is not an array or a multiset,
* any other non-numeric value in the input collection will cause a type error.
* Example:
@@ -247,7 +285,7 @@
* a `double` value representing the sample variance of the non-null and non-missing numbers in the given collection,
* `null` is returned if the input is `null` or `missing`,
* `null` is returned if the given collection does not contain any non-null and non-missing items,
- * any other non-array and non-multiset input value will cause a type error,
+ * `null` is returned if the input is not an array or a multiset,
* any other non-numeric value in the input collection will cause a type error.
* Example:
@@ -274,7 +312,7 @@
* a `double` value representing the population variance of the non-null and non-missing numbers in the given collection,
* `null` is returned if the input is `null` or `missing`,
* `null` is returned if the given collection does not contain any non-null and non-missing items,
- * any other non-array and non-multiset input value will cause a type error,
+ * `null` is returned if the input is not an array or a multiset,
* any other non-numeric value in the input collection will cause a type error.
* Example:
@@ -301,7 +339,7 @@
* a `double` value representing the skewness of the non-null and non-missing numbers in the given collection,
* `null` is returned if the input is `null` or `missing`,
* `null` is returned if the given collection does not contain any non-null and non-missing items,
- * any other non-array and non-multiset input value will cause a type error,
+ * `null` is returned if the input is not an array or a multiset,
* any other non-numeric value in the input collection will cause a type error.
* Example:
@@ -328,7 +366,7 @@
* a `double` value representing the kurtosis from a normal distribution of the non-null and non-missing numbers in the given collection,
* `null` is returned if the input is `null` or `missing`,
* `null` is returned if the given collection does not contain any non-null and non-missing items,
- * any other non-array and non-multiset input value will cause a type error,
+ * `null` is returned if the input is not an array or a multiset,
* any other non-numeric value in the input collection will cause a type error.
* Example:
@@ -352,7 +390,8 @@
* or a `missing` value.
* Return Value:
* a `bigint` value representing the number of items in the given collection,
- * `null` is returned if the input is `null` or `missing`.
+ * `0` is returned if the input is `null` or `missing`,
+ * `0` is returned if the input is not an array or a multiset.
* Example:
@@ -377,7 +416,8 @@
* a `double` value representing the average of the numbers in the given collection,
* `null` is returned if the input is `null` or `missing`,
* `null` is returned if there is a `null` or `missing` in the input collection,
- * any other non-numeric value in the input collection will cause a type error.
+ * `null` is returned if the input is not an array or a multiset,
+ * `null` is returned if there are any other non-numeric values in the input collection.
* Example:
@@ -404,7 +444,8 @@
items.
* `null` is returned if the input is `null` or `missing`,
* `null` is returned if there is a `null` or `missing` in the input collection,
- * any other non-numeric value in the input collection will cause a type error.
+ * `null` is returned if the input is not an array or a multiset,
+ * `null` is returned if there are any other non-numeric values in the input collection.
* Example:
@@ -431,8 +472,8 @@
(`tinyint`-> `smallint`->`integer`->`bigint`->`float`->`double`) among numeric items.
* `null` is returned if the input is `null` or `missing`,
* `null` is returned if there is a `null` or `missing` in the input collection,
- * multiple incomparable items in the input array or multiset will cause a type error,
- * any other non-array and non-multiset input value will cause a type error.
+ * `null` is returned if there are incomparable items in the input array or multiset,
+ * `null` is returned if the input is not an array or a multiset.
* Example:
@@ -460,8 +501,8 @@
(`tinyint`-> `smallint`->`integer`->`bigint`->`float`->`double`) among numeric items.
* `null` is returned if the input is `null` or `missing`,
* `null` is returned if there is a `null` or `missing` in the input collection,
- * multiple incomparable items in the input array or multiset will cause a type error,
- * any other non-array and non-multiset input value will cause a type error.
+ * `null` is returned if there are incomparable items in the input array or multiset,
+ * `null` is returned if the input is not an array or a multiset.
* Example:
@@ -536,6 +577,7 @@
* a `double` value representing the sample variance of the numbers in the given collection,
* `null` is returned if the input is `null` or `missing`,
* `null` is returned if there is a `null` or `missing` in the input collection,
+ * `null` is returned if the input is not an array or a multiset,
* any other non-numeric value in the input collection will cause a type error.
* Example:
@@ -561,6 +603,7 @@
* a `double` value representing the population variance of the numbers in the given collection,
* `null` is returned if the input is `null` or `missing`,
* `null` is returned if there is a `null` or `missing` in the input collection,
+ * `null` is returned if the input is not an array or a multiset,
* any other non-numeric value in the input collection will cause a type error.
* Example:
@@ -586,6 +629,7 @@
* a `double` value representing the skewness of the numbers in the given collection,
* `null` is returned if the input is `null` or `missing`,
* `null` is returned if there is a `null` or `missing` in the input collection,
+ * `null` is returned if the input is not an array or a multiset,
* any other non-numeric value in the input collection will cause a type error.
* Example:
@@ -611,6 +655,7 @@
* a `double` value representing the kurtosis from a normal distribution of the numbers in the given collection,
* `null` is returned if the input is `null` or `missing`,
* `null` is returned if there is a `null` or `missing` in the input collection,
+ * `null` is returned if the input is not an array or a multiset,
* any other non-numeric value in the input collection will cause a type error.
* Example:
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_query.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_query.md
index 04a65d1..2bd9f49 100644
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_query.md
+++ b/asterixdb/asterix-doc/src/main/markdown/sqlpp/3_query.md
@@ -1215,20 +1215,21 @@
For example, `SELECT COUNT(*) FROM customers` simply returns the total number of customers, whereas `SELECT COUNT(rating) FROM customers` returns the number of customers who have known ratings (that is, their ratings are not `null` or `missing`).
-Because the aggregation pseudo-functions sometimes restructure their operands, they can be used only in query blocks where (explicit or implicit) grouping is being done. Therefore the pseudo-functions cannot operate directly on arrays or multisets. For operating directly on JSON collections, SQL++ provides a set of ordinary functions for computing aggregations. Each ordinary aggregation function (except the ones corresponding to `COUNT` and `ARRAY_AGG`) has two versions: one that ignores `null` and `missing` values and one that returns `null` if a `null` or `missing` value is encountered anywhere in the collection. The names of the aggregation functions are as follows:
+Because the aggregation pseudo-functions sometimes restructure their operands, they can be used only in query blocks where (explicit or implicit) grouping is being done. Therefore the pseudo-functions cannot operate directly on arrays or multisets. For operating directly on JSON collections, SQL++ provides a set of ordinary functions for computing aggregations. Each ordinary aggregation function (except as noted below) has two versions: one that ignores `null` and `missing` values, and one that returns `null` if a `null` or `missing` value is encountered anywhere in the collection. The names of the aggregation functions are as follows:
| Aggregation pseudo-function; operates on groups only | Ordinary function: Ignores NULL or MISSING values | Ordinary function: Returns NULL if NULL or MISSING are encountered|
|----------|----------|--------|
-|SUM| ARRAY_SUM| STRICT_SUM |
-| AVG |ARRAY_MAX| STRICT_MAX |
-| MAX | ARRAY_MIN| STRICT_MIN |
-| MIN | ARRAY_AVG| STRICT_AVG |
+| SUM | ARRAY_SUM| STRICT_SUM |
+| AVG | ARRAY_AVG| STRICT_AVG |
+| MAX | ARRAY_MAX| STRICT_MAX |
+| MIN | ARRAY_MIN| STRICT_MIN |
| COUNT |ARRAY_COUNT|STRICT_COUNT (see exception below) |
+| MEDIAN | ARRAY_MEDIAN | |
|STDDEV_SAMP|ARRAY_STDDEV_SAMP| STRICT_STDDEV_SAMP |
|STDDEV_POP|ARRAY_STDDEV_POP| STRICT_STDDEV_POP |
|VAR_SAMP|ARRAY_VAR_SAMP| STRICT_VAR_SAMP |
|VAR_POP|ARRAY_VAR_POP| STRICT_VAR_POP |
-|SKEWENESS|ARRAY_SKEWNESS| STRICT_SKEWNESS |
+|SKEWNESS|ARRAY_SKEWNESS| STRICT_SKEWNESS |
|KURTOSIS|ARRAY_KURTOSIS| STRICT_KURTOSIS |
| |ARRAY_AGG| |
diff --git a/asterixdb/asterix-doc/src/main/markdown/sqlpp/4_windowfunctions.md b/asterixdb/asterix-doc/src/main/markdown/sqlpp/4_windowfunctions.md
index d6e40e7..c41e8a3 100644
--- a/asterixdb/asterix-doc/src/main/markdown/sqlpp/4_windowfunctions.md
+++ b/asterixdb/asterix-doc/src/main/markdown/sqlpp/4_windowfunctions.md
@@ -108,7 +108,7 @@
The *window order clause* determines how tuples are ordered within each partition. The window function works on tuples in the order specified by this clause.
-This clause may be used with any [window function](builtins.html#WindowFunctions), or any [aggregate function](builtins.html#AggregateFunctions) used as a window function.
+This clause may be used with any [window function](builtins.html#WindowFunctions), and most [aggregate functions](builtins.html#AggregateFunctions) — refer to the descriptions of individual functions for more details.
This clause is optional. If omitted, all tuples are considered peers, i.e. their order is tied. When tuples in the window partition are tied, each window function behaves differently.
@@ -130,7 +130,7 @@
##### WindowFrameClause
-The *window frame clause* defines the window frame. It can be used with all [aggregate functions](builtins.html#AggregateFunctions) and some [window functions](builtins.html#WindowFunctions) — refer to the descriptions of individual functions for more details. It is optional and allowed only when the [window order clause](#Window_order_clause) is present.
+The *window frame clause* defines the window frame. It can be used with some [window functions](builtins.html#WindowFunctions) and most [aggregate functions](builtins.html#AggregateFunctions) — refer to the descriptions of individual functions for more details. It is optional and allowed only when the [window order clause](#Window_order_clause) is present.
* If this clause is omitted and there is no [window order clause](#Window_order_clause), the window frame is the entire partition.
diff --git a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/PartitionResourcesListTask.java b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/PartitionResourcesListTask.java
index 0f5949e..97d03e6 100644
--- a/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/PartitionResourcesListTask.java
+++ b/asterixdb/asterix-replication/src/main/java/org/apache/asterix/replication/messaging/PartitionResourcesListTask.java
@@ -51,6 +51,11 @@
@Override
public void perform(INcApplicationContext appCtx, IReplicationWorker worker) throws HyracksDataException {
LOGGER.debug("processing {}", this);
+ if (appCtx.getMetadataPartitionId().isPresent() && appCtx.getMetadataPartitionId().getAsInt() == partition
+ && appCtx.getReplicaManager().getPartitions().contains(partition)) {
+ LOGGER.warn("received request to get metadata files from non-master {}", worker.getRemoteAddress());
+ throw new IllegalStateException();
+ }
final PersistentLocalResourceRepository localResourceRepository =
(PersistentLocalResourceRepository) appCtx.getLocalResourceRepository();
localResourceRepository.cleanup(partition);
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/executor/JobExecutor.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/executor/JobExecutor.java
index 64f900e..da840af 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/executor/JobExecutor.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/executor/JobExecutor.java
@@ -577,8 +577,8 @@
inProgressTaskClusters.remove(tcAttempt.getTaskCluster());
TaskCluster tc = tcAttempt.getTaskCluster();
PartitionMatchMaker pmm = jobRun.getPartitionMatchMaker();
- pmm.removeUncommittedPartitions(tc.getProducedPartitions(), abortTaskIds);
- pmm.removePartitionRequests(tc.getRequiredPartitions(), abortTaskIds);
+ pmm.removeUncommittedPartitions(tc.getProducedPartitions(), abortTaskIds, jobId);
+ pmm.removePartitionRequests(tc.getRequiredPartitions(), abortTaskIds, jobId);
tcAttempt.setStatus(failedOrAbortedStatus);
tcAttempt.setEndTime(System.currentTimeMillis());
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/partitions/PartitionMatchMaker.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/partitions/PartitionMatchMaker.java
index b5df593..6278693 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/partitions/PartitionMatchMaker.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/partitions/PartitionMatchMaker.java
@@ -28,6 +28,7 @@
import org.apache.commons.lang3.tuple.Pair;
import org.apache.hyracks.api.dataflow.TaskAttemptId;
+import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.partitions.PartitionId;
import org.apache.hyracks.control.common.job.PartitionDescriptor;
import org.apache.hyracks.control.common.job.PartitionRequest;
@@ -43,14 +44,13 @@
private final Map<PartitionId, List<PartitionRequest>> partitionRequests;
public PartitionMatchMaker() {
- partitionDescriptors = new HashMap<PartitionId, List<PartitionDescriptor>>();
- partitionRequests = new HashMap<PartitionId, List<PartitionRequest>>();
+ partitionDescriptors = new HashMap<>();
+ partitionRequests = new HashMap<>();
}
public List<Pair<PartitionDescriptor, PartitionRequest>> registerPartitionDescriptor(
PartitionDescriptor partitionDescriptor) {
- List<Pair<PartitionDescriptor, PartitionRequest>> matches =
- new ArrayList<Pair<PartitionDescriptor, PartitionRequest>>();
+ List<Pair<PartitionDescriptor, PartitionRequest>> matches = new ArrayList<>();
PartitionId pid = partitionDescriptor.getPartitionId();
boolean matched = false;
List<PartitionRequest> requests = partitionRequests.get(pid);
@@ -73,11 +73,7 @@
}
if (!matched) {
- List<PartitionDescriptor> descriptors = partitionDescriptors.get(pid);
- if (descriptors == null) {
- descriptors = new ArrayList<PartitionDescriptor>();
- partitionDescriptors.put(pid, descriptors);
- }
+ List<PartitionDescriptor> descriptors = partitionDescriptors.computeIfAbsent(pid, k -> new ArrayList<>());
descriptors.add(partitionDescriptor);
}
@@ -108,11 +104,7 @@
}
if (match == null) {
- List<PartitionRequest> requests = partitionRequests.get(pid);
- if (requests == null) {
- requests = new ArrayList<PartitionRequest>();
- partitionRequests.put(pid, requests);
- }
+ List<PartitionRequest> requests = partitionRequests.computeIfAbsent(pid, k -> new ArrayList<>());
requests.add(partitionRequest);
}
@@ -133,17 +125,11 @@
}
private interface IEntryFilter<T> {
- public boolean matches(T o);
+ boolean matches(T o);
}
private static <T> void removeEntries(List<T> list, IEntryFilter<T> filter) {
- Iterator<T> j = list.iterator();
- while (j.hasNext()) {
- T o = j.next();
- if (filter.matches(o)) {
- j.remove();
- }
- }
+ list.removeIf(filter::matches);
}
private static <T> void removeEntries(Map<PartitionId, List<T>> map, IEntryFilter<T> filter) {
@@ -159,30 +145,16 @@
}
public void notifyNodeFailures(final Collection<String> deadNodes) {
- removeEntries(partitionDescriptors, new IEntryFilter<PartitionDescriptor>() {
- @Override
- public boolean matches(PartitionDescriptor o) {
- return deadNodes.contains(o.getNodeId());
- }
- });
- removeEntries(partitionRequests, new IEntryFilter<PartitionRequest>() {
- @Override
- public boolean matches(PartitionRequest o) {
- return deadNodes.contains(o.getNodeId());
- }
- });
+ removeEntries(partitionDescriptors, o -> deadNodes.contains(o.getNodeId()));
+ removeEntries(partitionRequests, o -> deadNodes.contains(o.getNodeId()));
}
- public void removeUncommittedPartitions(Set<PartitionId> partitionIds, final Set<TaskAttemptId> taIds) {
+ public void removeUncommittedPartitions(Set<PartitionId> partitionIds, Set<TaskAttemptId> taIds, JobId jobId) {
if (partitionIds != null && !partitionIds.isEmpty()) {
- LOGGER.debug("Removing uncommitted partitions {}", partitionIds);
+ LOGGER.debug("Removing uncommitted partitions {}: {}", jobId, partitionIds);
}
- IEntryFilter<PartitionDescriptor> filter = new IEntryFilter<PartitionDescriptor>() {
- @Override
- public boolean matches(PartitionDescriptor o) {
- return o.getState() != PartitionState.COMMITTED && taIds.contains(o.getProducingTaskAttemptId());
- }
- };
+ IEntryFilter<PartitionDescriptor> filter =
+ o -> o.getState() != PartitionState.COMMITTED && taIds.contains(o.getProducingTaskAttemptId());
for (PartitionId pid : partitionIds) {
List<PartitionDescriptor> descriptors = partitionDescriptors.get(pid);
if (descriptors != null) {
@@ -194,16 +166,11 @@
}
}
- public void removePartitionRequests(Set<PartitionId> partitionIds, final Set<TaskAttemptId> taIds) {
+ public void removePartitionRequests(Set<PartitionId> partitionIds, Set<TaskAttemptId> taIds, JobId jobId) {
if (partitionIds != null && !partitionIds.isEmpty()) {
- LOGGER.debug("Removing partition requests {}", partitionIds);
+ LOGGER.debug("Removing partition requests {}: {}", jobId, partitionIds);
}
- IEntryFilter<PartitionRequest> filter = new IEntryFilter<PartitionRequest>() {
- @Override
- public boolean matches(PartitionRequest o) {
- return taIds.contains(o.getRequestingTaskAttemptId());
- }
- };
+ IEntryFilter<PartitionRequest> filter = o -> taIds.contains(o.getRequestingTaskAttemptId());
for (PartitionId pid : partitionIds) {
List<PartitionRequest> requests = partitionRequests.get(pid);
if (requests != null) {
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java
index 85661fe..9c5a9fa 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-common/src/main/java/org/apache/hyracks/control/common/config/ConfigManager.java
@@ -148,7 +148,7 @@
if (configured) {
throw new IllegalStateException("configuration already processed");
}
- LOGGER.debug("registering option: " + option.toIniString());
+ LOGGER.trace("registering option: {}", option::toIniString);
Map<String, IOption> optionMap = sectionMap.computeIfAbsent(option.section(), section -> new HashMap<>());
IOption prev = optionMap.put(option.ini(), option);
if (prev != null) {
@@ -160,8 +160,13 @@
registeredOptions.add(option);
optionSetters.put(option, (node, value, isDefault) -> correctedMap(node, isDefault).put(option, value));
if (LOGGER.isDebugEnabled()) {
- optionSetters.put(option, (node, value, isDefault) -> LOGGER.debug("{} {} to {} for node {}",
- isDefault ? "defaulting" : "setting", option.toIniString(), value, node));
+ optionSetters.put(option, (node, value, isDefault) -> {
+ if (isDefault) {
+ LOGGER.trace("defaulting {} to {} for node {}", option.toIniString(), value, node);
+ } else {
+ LOGGER.debug("setting {} to {} for node {}", option.toIniString(), value, node);
+ }
+ });
}
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/CcConnection.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/CcConnection.java
index d1f7d5a..c99898d 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/CcConnection.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/CcConnection.java
@@ -70,6 +70,8 @@
InvokeUtil.runWithTimeout(() -> {
this.wait(REGISTRATION_RESPONSE_POLL_PERIOD); // NOSONAR while loop in timeout call
}, () -> !registrationPending, 1, TimeUnit.MINUTES);
+ } catch (InterruptedException e) {
+ throw e;
} catch (Exception e) {
registrationException = e;
}