Merge branch 'gerrit/goldfish' into 'master'
Change-Id: I5df0ec3c78700894fd3d8df6801cd19b4743255e
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveNotificationHandler.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveNotificationHandler.java
index 5fe082b..01e47b5 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveNotificationHandler.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/active/ActiveNotificationHandler.java
@@ -23,6 +23,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import org.apache.asterix.active.ActiveEvent;
import org.apache.asterix.active.ActiveEvent.Kind;
@@ -68,23 +69,27 @@
@Override
protected void handle(ActiveEvent event) {
+ resolveListenerForEvent(event).ifPresent(listener -> listener.notify(event));
+ }
+
+ private synchronized Optional<IActiveEntityEventsListener> resolveListenerForEvent(ActiveEvent event) {
JobId jobId = event.getJobId();
Kind eventKind = event.getEventKind();
EntityId entityId = jobId2EntityId.get(jobId);
+ IActiveEntityEventsListener listener = null;
if (entityId != null) {
- IActiveEntityEventsListener listener = entityEventListeners.get(entityId);
+ listener = entityEventListeners.get(entityId);
if (eventKind == Kind.JOB_FINISHED) {
LOGGER.debug("removing ingestion job {}", jobId);
jobId2EntityId.remove(jobId);
}
- if (listener != null) {
- listener.notify(event);
- } else {
- LOGGER.debug("listener not found for entity {} on event={}", entityId, event);
+ if (listener == null) {
+ LOGGER.debug("listener not found for entity {} on event {} for job {}", entityId, event, jobId);
}
} else {
- LOGGER.error("entity not found for event {}", event);
+ LOGGER.log(Level.ERROR, "entity not found for event {} for job {}", eventKind, jobId);
}
+ return Optional.ofNullable(listener);
}
// *** IJobLifecycleListener
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java
index 5239d01..acb2ca5 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/ActiveRequestsDatasource.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.app.function;
+import java.util.Objects;
+
import org.apache.asterix.app.message.ClientRequestsRequest;
import org.apache.asterix.metadata.api.IDatasourceFunction;
import org.apache.asterix.metadata.declared.DataSourceId;
@@ -48,4 +50,9 @@
public boolean skipJobCapacityAssignment() {
return true;
}
+
+ @Override
+ protected boolean sameFunctionDatasource(FunctionDataSource other) {
+ return Objects.equals(this.functionId, other.getFunctionId());
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java
index f02af21..4d48eef 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/CompletedRequestsDatasource.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.app.function;
+import java.util.Objects;
+
import org.apache.asterix.app.message.ClientRequestsRequest;
import org.apache.asterix.metadata.api.IDatasourceFunction;
import org.apache.asterix.metadata.declared.DataSourceId;
@@ -48,4 +50,9 @@
public boolean skipJobCapacityAssignment() {
return true;
}
+
+ @Override
+ protected boolean sameFunctionDatasource(FunctionDataSource other) {
+ return Objects.equals(this.functionId, other.getFunctionId());
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetResourcesDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetResourcesDatasource.java
index 3e5033b..d1c3c01 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetResourcesDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DatasetResourcesDatasource.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.app.function;
+import java.util.Objects;
+
import org.apache.asterix.metadata.api.IDatasourceFunction;
import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.declared.FunctionDataSource;
@@ -43,4 +45,17 @@
AlgebricksAbsolutePartitionConstraint locations) {
return new DatasetResourcesFunction(locations, datasetId);
}
+
+ public int getDatasetId() {
+ return datasetId;
+ }
+
+ @Override
+ public boolean sameFunctionDatasource(FunctionDataSource other) {
+ if (!Objects.equals(this.functionId, other.getFunctionId())) {
+ return false;
+ }
+ DatasetResourcesDatasource that = (DatasetResourcesDatasource) other;
+ return Objects.equals(this.datasetId, that.getDatasetId());
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
index 1eac011..23fcf8c 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexDatasource.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.app.function;
+import java.util.Objects;
+
import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.external.api.IDataParserFactory;
import org.apache.asterix.external.parser.factory.JSONDataParserFactory;
@@ -25,6 +27,7 @@
import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.declared.FunctionDataSource;
import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.metadata.entities.Index;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.properties.INodeDomain;
@@ -41,16 +44,23 @@
private final IBinaryComparatorFactory[] comparatorFactories;
private final AlgebricksAbsolutePartitionConstraint constraint;
private final int[][] partitionsMap;
+ private final Index index;
public DumpIndexDatasource(INodeDomain domain, IndexDataflowHelperFactory indexDataflowHelperFactory,
RecordDescriptor recDesc, IBinaryComparatorFactory[] comparatorFactories,
- AlgebricksAbsolutePartitionConstraint constraint, int[][] partitionsMap) throws AlgebricksException {
+ AlgebricksAbsolutePartitionConstraint constraint, int[][] partitionsMap, Index index)
+ throws AlgebricksException {
super(DUMP_INDEX_DATASOURCE_ID, DumpIndexRewriter.DUMP_INDEX, domain);
this.indexDataflowHelperFactory = indexDataflowHelperFactory;
this.recDesc = recDesc;
this.comparatorFactories = comparatorFactories;
this.constraint = constraint;
this.partitionsMap = partitionsMap;
+ this.index = index;
+ }
+
+ public Index getIndex() {
+ return index;
}
@Override
@@ -69,4 +79,13 @@
protected IDataParserFactory createDataParserFactory() {
return new JSONDataParserFactory();
}
+
+ @Override
+ public boolean sameFunctionDatasource(FunctionDataSource other) {
+ if (!Objects.equals(this.functionId, other.getFunctionId())) {
+ return false;
+ }
+ DumpIndexDatasource that = (DumpIndexDatasource) other;
+ return Objects.equals(this.index, that.getIndex());
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
index d50315b..825d571 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/DumpIndexRewriter.java
@@ -89,7 +89,7 @@
(AlgebricksAbsolutePartitionConstraint) partitioningProperties.getConstraints();
return new DumpIndexDatasource(context.getComputationNodeDomain(), indexDataflowHelperFactory,
secondaryIndexHelper.getSecondaryRecDesc(), secondaryIndexHelper.getSecondaryComparatorFactories(),
- secondaryPartitionConstraint, partitioningProperties.getComputeStorageMap());
+ secondaryPartitionConstraint, partitioningProperties.getComputeStorageMap(), index);
}
@Override
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/JobSummariesDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/JobSummariesDatasource.java
index 6bab0cd..1a23db3 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/JobSummariesDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/JobSummariesDatasource.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.app.function;
+import java.util.Objects;
+
import org.apache.asterix.metadata.api.IDatasourceFunction;
import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.declared.FunctionDataSource;
@@ -40,4 +42,9 @@
AlgebricksAbsolutePartitionConstraint locations) {
return new JobSummariesFunction(AlgebricksAbsolutePartitionConstraint.randomLocation(locations.getLocations()));
}
+
+ @Override
+ protected boolean sameFunctionDatasource(FunctionDataSource other) {
+ return Objects.equals(this.functionId, other.getFunctionId());
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java
index 1b6d807..6eabda9 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/PingDatasource.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.app.function;
+import java.util.Objects;
+
import org.apache.asterix.metadata.api.IDatasourceFunction;
import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.declared.FunctionDataSource;
@@ -40,4 +42,8 @@
return new PingFunction(locations);
}
+ @Override
+ protected boolean sameFunctionDatasource(FunctionDataSource other) {
+ return Objects.equals(this.functionId, other.getFunctionId());
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java
index 52054d6..d6a45af 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryIndexDatasource.java
@@ -20,6 +20,7 @@
import java.util.ArrayList;
import java.util.List;
+import java.util.Objects;
import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.metadata.api.IDatasourceFunction;
@@ -71,6 +72,14 @@
this.numSecKeys = numSecKeys;
}
+ public Dataset getDataset() {
+ return ds;
+ }
+
+ public String getIndexName() {
+ return indexName;
+ }
+
@Override
protected void initSchemaType(IAType iType) {
ARecordType type = (ARecordType) iType;
@@ -138,4 +147,13 @@
return new DataSourceId(dataset.getDatabaseName(), dataset.getDataverseName(), dataset.getDatasetName(),
new String[] { indexName, QueryIndexRewriter.QUERY_INDEX.getName() });
}
+
+ @Override
+ public boolean sameFunctionDatasource(FunctionDataSource other) {
+ if (!Objects.equals(this.functionId, other.getFunctionId())) {
+ return false;
+ }
+ QueryIndexDatasource that = (QueryIndexDatasource) other;
+ return Objects.equals(this.ds, that.getDataset()) && Objects.equals(this.indexName, that.getIndexName());
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryPartitionDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryPartitionDatasource.java
index 4d40ba6..4917b33 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryPartitionDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/QueryPartitionDatasource.java
@@ -20,6 +20,7 @@
import java.util.ArrayList;
import java.util.List;
+import java.util.Objects;
import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.metadata.api.IDatasourceFunction;
@@ -68,6 +69,14 @@
this.storageLocations = storageLocations;
}
+ public Dataset getDatasource() {
+ return ds;
+ }
+
+ public int getPartitionNumber() {
+ return partitionNum;
+ }
+
@Override
protected void initSchemaType(IAType iType) {
ARecordType type = (ARecordType) iType;
@@ -128,4 +137,14 @@
return new DataSourceId(dataset.getDatabaseName(), dataset.getDataverseName(), dataset.getDatasetName(),
new String[] { dataset.getDatasetName(), QueryPartitionRewriter.QUERY_PARTITION.getName() });
}
+
+ @Override
+ public boolean sameFunctionDatasource(FunctionDataSource other) {
+ if (!Objects.equals(this.functionId, other.getFunctionId())) {
+ return false;
+ }
+ QueryPartitionDatasource that = (QueryPartitionDatasource) other;
+ return Objects.equals(this.ds, that.getDatasource())
+ && Objects.equals(this.partitionNum, that.getPartitionNumber());
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsDatasource.java
index 6157412..c470a20 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/StorageComponentsDatasource.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.app.function;
+import java.util.Objects;
+
import org.apache.asterix.metadata.api.IDatasourceFunction;
import org.apache.asterix.metadata.declared.DataSourceId;
import org.apache.asterix.metadata.declared.FunctionDataSource;
@@ -38,9 +40,22 @@
this.datasetId = datasetId;
}
+ public int getDatasetId() {
+ return datasetId;
+ }
+
@Override
protected IDatasourceFunction createFunction(MetadataProvider metadataProvider,
AlgebricksAbsolutePartitionConstraint locations) {
return new StorageComponentsFunction(locations, datasetId);
}
+
+ @Override
+ public boolean sameFunctionDatasource(FunctionDataSource other) {
+ if (!Objects.equals(this.functionId, other.getFunctionId())) {
+ return false;
+ }
+ StorageComponentsDatasource that = (StorageComponentsDatasource) other;
+ return Objects.equals(this.datasetId, that.getDatasetId());
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java
index b4d1b03..ac19b00 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSAllTablesDataGeneratorDatasource.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.app.function;
+import java.util.Objects;
+
import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.metadata.api.IDatasourceFunction;
import org.apache.asterix.metadata.declared.DataSourceId;
@@ -42,6 +44,10 @@
this.scalingFactor = scalingFactor;
}
+ public double getScalingFactor() {
+ return scalingFactor;
+ }
+
/**
* This ensures that each function will have a unique DataSourceId by passing the table name as part of the
* DataSourceId. This eliminates the issue of creating a single function even though multiple functions calls
@@ -66,4 +72,13 @@
protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm, MetadataProvider md) {
return md.getDataPartitioningProvider().getClusterLocations();
}
+
+ @Override
+ public boolean sameFunctionDatasource(FunctionDataSource other) {
+ if (!Objects.equals(this.functionId, other.getFunctionId())) {
+ return false;
+ }
+ TPCDSAllTablesDataGeneratorDatasource that = (TPCDSAllTablesDataGeneratorDatasource) other;
+ return Objects.equals(this.scalingFactor, that.getScalingFactor());
+ }
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java
index 3bbaee2..6551346 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/function/TPCDSSingleTableDataGeneratorDatasource.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.app.function;
+import java.util.Objects;
+
import org.apache.asterix.common.cluster.IClusterStateManager;
import org.apache.asterix.metadata.api.IDatasourceFunction;
import org.apache.asterix.metadata.declared.DataSourceId;
@@ -44,6 +46,14 @@
this.scalingFactor = scalingFactor;
}
+ public String getTableName() {
+ return tableName;
+ }
+
+ public double getScalingFactor() {
+ return scalingFactor;
+ }
+
/**
* This ensures that each function will have a unique DataSourceId by passing the table name as part of the
* DataSourceId. This eliminates the issue of creating a single function even though multiple functions calls
@@ -70,4 +80,14 @@
protected AlgebricksAbsolutePartitionConstraint getLocations(IClusterStateManager csm, MetadataProvider md) {
return md.getDataPartitioningProvider().getClusterLocations();
}
+
+ @Override
+ public boolean sameFunctionDatasource(FunctionDataSource other) {
+ if (!Objects.equals(this.functionId, other.getFunctionId())) {
+ return false;
+ }
+ TPCDSSingleTableDataGeneratorDatasource that = (TPCDSSingleTableDataGeneratorDatasource) other;
+ return Objects.equals(this.tableName, that.getTableName())
+ && Objects.equals(this.scalingFactor, that.getScalingFactor());
+ }
}
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.000.ddl.sqlpp
new file mode 100644
index 0000000..737c344
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.000.ddl.sqlpp
@@ -0,0 +1,34 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+DROP TYPE test IF EXISTS;
+CREATE TYPE test AS OPEN {id: int};
+
+DROP DATASET test IF EXISTS;
+CREATE DATASET test(test) PRIMARY KEY id;
+
+DROP INDEX test.test_int_idx IF EXISTS;
+CREATE INDEX test_int_idx ON test(intField: int);
+
+DROP INDEX test.test_string_idx IF EXISTS;
+CREATE INDEX test_string_idx ON test(stringField: string);
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.010.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.010.update.sqlpp
new file mode 100644
index 0000000..aeadafc
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.010.update.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+INSERT INTO test({"id": 1, "intField": 15, "stringField": "foo"});
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.020.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.020.query.sqlpp
new file mode 100644
index 0000000..9752c75
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.020.query.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+SET `import-private-functions` `true`;
+
+SELECT dump_index("test", "test", "test_int_idx") as intIndex
+UNION ALL
+SELECT dump_index("test", "test", "test_string_idx") as stringIndex;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.030.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.030.query.sqlpp
new file mode 100644
index 0000000..b4209fc
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.030.query.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+SET `import-private-functions` `true`;
+
+explain
+SELECT dump_index("test", "test", "test_int_idx") as intIndex
+UNION ALL
+SELECT dump_index("test", "test", "test_string_idx") as stringIndex;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.999.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.999.ddl.sqlpp
new file mode 100644
index 0000000..20dc6fd
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/explain/explain_same_datasource_function_different_arguments/test.999.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/explain/explain_same_datasource_function_different_arguments/test.020.unorderedtxt b/asterixdb/asterix-app/src/test/resources/runtimets/results/explain/explain_same_datasource_function_different_arguments/test.020.unorderedtxt
new file mode 100644
index 0000000..24e0e67
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/explain/explain_same_datasource_function_different_arguments/test.020.unorderedtxt
@@ -0,0 +1,2 @@
+{ "intIndex": [ { "values": [ 15, 1 ] } ] }
+{ "stringIndex": [ { "values": [ "foo", 1 ] } ] }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/explain/explain_same_datasource_function_different_arguments/test.030.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/explain/explain_same_datasource_function_different_arguments/test.030.plan
new file mode 100644
index 0000000..825f43a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/explain/explain_same_datasource_function_different_arguments/test.030.plan
@@ -0,0 +1,38 @@
+distribute result [$$11] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT |UNPARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ union ($$16, $$17, $$11) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- UNION_ALL |UNPARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ project ([$$16]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |UNPARTITIONED|
+ assign [$$16] <- [cast({"intIndex": $$13})] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |UNPARTITIONED|
+ aggregate [$$13] <- [listify($$12)] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- AGGREGATE |UNPARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- RANDOM_MERGE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$12] <- asterix.dump-index. [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |UNPARTITIONED|
+ project ([$$17]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |UNPARTITIONED|
+ assign [$$17] <- [cast({"stringIndex": $$15})] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |UNPARTITIONED|
+ aggregate [$$15] <- [listify($$14)] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- AGGREGATE |UNPARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- RANDOM_MERGE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$14] <- asterix.dump-index. [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
index 2370d93..302efdb 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml
@@ -342,6 +342,11 @@
<expected-error>ASX1001: Syntax error: EXPLAIN is not supported for this kind of statement</expected-error>
</compilation-unit>
</test-case>
+ <test-case FilePath="explain">
+ <compilation-unit name="explain_same_datasource_function_different_arguments">
+ <output-dir compare="Text">explain_same_datasource_function_different_arguments</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
<test-group name="aggregate">
<test-case FilePath="aggregate">
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml
index c6a7cf5..5cf8bad 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_it_python.xml
@@ -50,12 +50,14 @@
<output-dir compare="Clean-JSON">python_open_type_validation</output-dir>
</compilation-unit>
</test-case>
+ <!-- TODO(Ian): disabling this test as it is intermittently failing as the output is dependent on the python version running on the test machine
<test-case FilePath="external-library" check-warnings="true">
<compilation-unit name="py_function_error">
<output-dir compare="Clean-JSON">py_function_error</output-dir>
<expected-warn>ArithmeticError: oof</expected-warn>
</compilation-unit>
</test-case>
+ -->
<test-case FilePath="external-library">
<compilation-unit name="mysentiment_twitter">
<output-dir compare="Text">mysentiment_twitter</output-dir>
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java
index 1fa84f6..7762041 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DataSource.java
@@ -24,6 +24,7 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint;
@@ -167,4 +168,27 @@
ITupleFilterFactory tupleFilterFactory, long outputLimit, IOperatorSchema opSchema,
IVariableTypeEnvironment typeEnv, JobGenContext context, JobSpecification jobSpec, Object implConfig,
IProjectionFiltrationInfo projectionFiltrationInfo) throws AlgebricksException;
+
+ @Override
+ public boolean sameAs(IDataSource<?> other) {
+ if (this == other) {
+ return true;
+ }
+
+ if (!(other instanceof DataSource)) {
+ return false;
+ }
+
+ DataSource that = (DataSource) other;
+ if (!Objects.equals(this.id, that.getId()) || !Objects.equals(this.datasourceType, that.getDatasourceType())) {
+ return false;
+ }
+
+ if (this.datasourceType == Type.EXTERNAL_DATASET && that.getDatasourceType() == Type.EXTERNAL_DATASET
+ && !Objects.equals(this.getProperties(), other.getProperties())) {
+ return false;
+ }
+
+ return true;
+ }
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
index 9d567fc..c404496 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java
@@ -224,9 +224,4 @@
public boolean isScanAccessPathALeaf() {
return dataset.getDatasetType() == DatasetType.EXTERNAL;
}
-
- @Override
- public boolean compareProperties() {
- return dataset.getDatasetType() == DatasetType.EXTERNAL;
- }
}
diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
index 91f7615..2322e53 100644
--- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
+++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FunctionDataSource.java
@@ -139,4 +139,20 @@
protected static DataSourceId createDataSourceId(FunctionIdentifier fid, String... parameters) {
return new DataSourceId(fid.getDatabase(), FunctionSignature.getDataverseName(fid), fid.getName(), parameters);
}
+
+ protected abstract boolean sameFunctionDatasource(FunctionDataSource other);
+
+ @Override
+ public boolean sameAs(IDataSource<?> other) {
+ if (!super.sameAs(other)) {
+ return false;
+ }
+
+ if (!(other instanceof FunctionDataSource)) {
+ return false;
+ }
+
+ FunctionDataSource that = (FunctionDataSource) other;
+ return sameFunctionDatasource(that);
+ }
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSource.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSource.java
index 5b75cd9..f7cd6da 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSource.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/metadata/IDataSource.java
@@ -48,7 +48,5 @@
public Map<String, Serializable> getProperties();
- default boolean compareProperties() {
- return false;
- }
+ boolean sameAs(IDataSource<?> other);
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
index 05f41a0..b985d09 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/IsomorphismOperatorVisitor.java
@@ -483,20 +483,21 @@
if (aop.getOperatorTag() != LogicalOperatorTag.DATASOURCESCAN) {
return Boolean.FALSE;
}
- DataSourceScanOperator argScan = (DataSourceScanOperator) arg;
- boolean isomorphic = op.getDataSource().getId().equals(argScan.getDataSource().getId())
- && op.getOutputLimit() == argScan.getOutputLimit()
- && Objects.equals(op.getProjectionFiltrationInfo(), argScan.getProjectionFiltrationInfo());
+ DataSourceScanOperator argScan = (DataSourceScanOperator) arg;
+ IDataSource<?> dataSource = op.getDataSource();
+ IDataSource<?> argDataSource = argScan.getDataSource();
+ boolean isomorphic = dataSource.sameAs(argDataSource);
if (!isomorphic) {
return Boolean.FALSE;
}
- IDataSource<?> dataSource = op.getDataSource();
- IDataSource<?> argDataSource = argScan.getDataSource();
- if (dataSource.compareProperties() && argDataSource.compareProperties()
- && !Objects.equals(dataSource.getProperties(), argDataSource.getProperties())) {
+
+ isomorphic = op.getOutputLimit() == argScan.getOutputLimit()
+ && Objects.equals(op.getProjectionFiltrationInfo(), argScan.getProjectionFiltrationInfo());
+ if (!isomorphic) {
return Boolean.FALSE;
}
+
DataSourceScanOperator scanOpArg = (DataSourceScanOperator) copyAndSubstituteVar(op, arg);
ILogicalExpression opCondition = op.getSelectCondition() != null ? op.getSelectCondition().getValue() : null;
ILogicalExpression argCondition =
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/SingleThreadEventProcessor.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/SingleThreadEventProcessor.java
index 85019c5..b07cb71 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/SingleThreadEventProcessor.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/util/SingleThreadEventProcessor.java
@@ -31,7 +31,7 @@
private static final Logger LOGGER = LogManager.getLogger();
protected final String name;
private final LinkedBlockingQueue<T> eventInbox;
- private volatile Thread executorThread;
+ private final Thread executorThread;
private volatile boolean stopped = false;
public SingleThreadEventProcessor(String threadName) {
@@ -43,18 +43,22 @@
@Override
public final void run() {
- LOGGER.log(Level.INFO, "Started " + Thread.currentThread().getName());
+ LOGGER.info("Started {}", name);
while (!stopped) {
try {
T event = eventInbox.take();
handle(event);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
+ if (!stopped) {
+ LOGGER.warn("Interrupt while waiting for an event and !stopped");
+ }
+ break;
} catch (Exception e) {
- LOGGER.log(Level.ERROR, "Error handling an event", e);
+ LOGGER.error("Error handling an event", e);
}
}
- LOGGER.log(Level.WARN, "Stopped " + Thread.currentThread().getName());
+ LOGGER.info("Stopped {}", name);
}
protected abstract void handle(T event) throws Exception; //NOSONAR