[NO ISSUE][COMP] Support runtime warnings during constant folding
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Add IWarningCollector accessor to IEvaluatorContext
- Provide compiler's warning collector to function evaluators
during constant folding, so they can emit warnings
Change-Id: I61dd53adfbcad2f8479333579185258876f074e9
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3484
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Dmitry Lychagin <dmitry.lychagin@couchbase.com>
Reviewed-by: Ali Alsuliman <ali.al.solaiman@gmail.com>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
index b095fe1..4c6f338 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
@@ -28,6 +28,7 @@
import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
+import org.apache.asterix.common.exceptions.WarningCollector;
import org.apache.asterix.dataflow.data.common.ExpressionTypeComputer;
import org.apache.asterix.dataflow.data.nontagged.MissingWriterFactory;
import org.apache.asterix.formats.nontagged.ADMPrinterFactoryProvider;
@@ -78,6 +79,7 @@
import org.apache.hyracks.algebricks.core.config.AlgebricksConfig;
import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext;
import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
import org.apache.hyracks.algebricks.runtime.evaluators.EvaluatorContext;
@@ -172,7 +174,7 @@
if (context.checkIfInDontApplySet(this, op)) {
return false;
}
-
+ cfv.reset(context);
return op.acceptExpressionTransform(cfv);
}
@@ -182,6 +184,13 @@
private final IPointable p = VoidPointable.FACTORY.createPointable();
private final ByteBufferInputStream bbis = new ByteBufferInputStream();
private final DataInputStream dis = new DataInputStream(bbis);
+ private final WarningCollector warningCollector = new WarningCollector();
+ private final IEvaluatorContext evalContext = new EvaluatorContext(warningCollector);
+ private IOptimizationContext optContext;
+
+ private void reset(IOptimizationContext context) {
+ optContext = context;
+ }
@Override
public boolean transform(Mutable<ILogicalExpression> exprRef) throws AlgebricksException {
@@ -248,7 +257,8 @@
IScalarEvaluatorFactory fact = jobGenCtx.getExpressionRuntimeProvider().createEvaluatorFactory(expr,
_emptyTypeEnv, _emptySchemas, jobGenCtx);
- IScalarEvaluator eval = fact.createScalarEvaluator(new EvaluatorContext(null));
+ warningCollector.clear();
+ IScalarEvaluator eval = fact.createScalarEvaluator(evalContext);
eval.evaluate(null, p);
IAType returnType = (IAType) _emptyTypeEnv.getType(expr);
ATypeTag runtimeType = PointableHelper.getTypeTag(p);
@@ -262,6 +272,7 @@
jobGenCtx.getSerializerDeserializerProvider().getSerializerDeserializer(returnType);
bbis.setByteBuffer(ByteBuffer.wrap(p.getByteArray(), p.getStartOffset(), p.getLength()), 0);
IAObject o = (IAObject) serde.deserialize(dis);
+ warningCollector.getWarnings(optContext.getWarningCollector());
return new Pair<>(true, new ConstantExpression(new AsterixConstantValue(o)));
} catch (HyracksDataException | AlgebricksException e) {
if (AlgebricksConfig.ALGEBRICKS_LOGGER.isTraceEnabled()) {
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/WarningCollector.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/WarningCollector.java
index 2f0af3b..a624e71 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/WarningCollector.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/exceptions/WarningCollector.java
@@ -46,4 +46,10 @@
public void getWarnings(Collection<? super Warning> outWarnings) {
outWarnings.addAll(warnings);
}
+
+ public void getWarnings(IWarningCollector outWarningCollector) {
+ for (Warning warning : warnings) {
+ outWarningCollector.warn(warning);
+ }
+ }
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/std/AbstractMinMaxAggregateFunction.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/std/AbstractMinMaxAggregateFunction.java
index b653135..aa45922 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/std/AbstractMinMaxAggregateFunction.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/std/AbstractMinMaxAggregateFunction.java
@@ -156,13 +156,13 @@
}
private void handleIncompatibleInput(ATypeTag typeTag) {
- context.getTaskContext()
+ context.getWarningCollector()
.warn(WarningUtil.forAsterix(sourceLoc, ErrorCode.TYPE_INCOMPATIBLE, "min/max", aggType, typeTag));
this.aggType = ATypeTag.NULL;
}
private void handleUnsupportedInput(ATypeTag typeTag) {
- context.getTaskContext()
+ context.getWarningCollector()
.warn(WarningUtil.forAsterix(sourceLoc, ErrorCode.TYPE_UNSUPPORTED, "min/max", typeTag));
this.aggType = ATypeTag.NULL;
}
diff --git a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/base/IEvaluatorContext.java b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/base/IEvaluatorContext.java
index efa613d..1c14c1d 100644
--- a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/base/IEvaluatorContext.java
+++ b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/base/IEvaluatorContext.java
@@ -20,6 +20,7 @@
package org.apache.hyracks.algebricks.runtime.base;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
/**
* Context for runtime function evaluators
@@ -30,4 +31,9 @@
* is being executed by the constant folding rule at compile time.
*/
IHyracksTaskContext getTaskContext();
+
+ /**
+ * Returns a warning collector, never {@code null}
+ */
+ IWarningCollector getWarningCollector();
}
diff --git a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/evaluators/EvaluatorContext.java b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/evaluators/EvaluatorContext.java
index 2113e57..beecee9 100644
--- a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/evaluators/EvaluatorContext.java
+++ b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/evaluators/EvaluatorContext.java
@@ -19,19 +19,36 @@
package org.apache.hyracks.algebricks.runtime.evaluators;
+import java.util.Objects;
+
import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
public final class EvaluatorContext implements IEvaluatorContext {
private final IHyracksTaskContext taskContext;
+ private final IWarningCollector warningCollector;
+
public EvaluatorContext(IHyracksTaskContext taskContext) {
this.taskContext = taskContext;
+ this.warningCollector = taskContext.getWarningCollector();
+ }
+
+ public EvaluatorContext(IWarningCollector warningCollector) {
+ Objects.requireNonNull(warningCollector);
+ this.taskContext = null;
+ this.warningCollector = warningCollector;
}
@Override
public IHyracksTaskContext getTaskContext() {
return taskContext;
}
+
+ @Override
+ public IWarningCollector getWarningCollector() {
+ return warningCollector;
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/context/IHyracksTaskContext.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/context/IHyracksTaskContext.java
index abe901a..1ffcf9f 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/context/IHyracksTaskContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/context/IHyracksTaskContext.java
@@ -25,7 +25,7 @@
import org.apache.hyracks.api.dataflow.TaskAttemptId;
import org.apache.hyracks.api.deployment.DeploymentId;
import org.apache.hyracks.api.exceptions.HyracksException;
-import org.apache.hyracks.api.exceptions.Warning;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.api.io.IWorkspaceFileFactory;
import org.apache.hyracks.api.job.IOperatorEnvironment;
import org.apache.hyracks.api.job.JobFlag;
@@ -60,10 +60,5 @@
IStatsCollector getStatsCollector();
- /**
- * Adds a warning to this {@link IHyracksTaskContext}
- *
- * @param warning
- */
- void warn(Warning warning);
+ IWarningCollector getWarningCollector();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java
index b8c1496..8dd94f3 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/Task.java
@@ -47,6 +47,7 @@
import org.apache.hyracks.api.deployment.DeploymentId;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.api.exceptions.Warning;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
@@ -119,6 +120,8 @@
private final Set<Warning> warnings;
+ private final IWarningCollector warningCollector;
+
public Task(Joblet joblet, Set<JobFlag> jobFlags, TaskAttemptId taskId, String displayName,
ExecutorService executor, NodeControllerService ncs,
List<List<PartitionChannel>> inputChannelsFromConnectors) {
@@ -138,6 +141,7 @@
this.inputChannelsFromConnectors = inputChannelsFromConnectors;
statsCollector = new StatsCollector();
warnings = ConcurrentHashMap.newKeySet();
+ warningCollector = warnings::add;
}
public void setTaskRuntime(IPartitionCollector[] collectors, IOperatorNodePushable operator) {
@@ -474,8 +478,8 @@
}
@Override
- public void warn(Warning warning) {
- warnings.add(warning);
+ public IWarningCollector getWarningCollector() {
+ return warningCollector;
}
public boolean isCompleted() {
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestTaskContext.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestTaskContext.java
index e328e3d..f264a4b 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestTaskContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestTaskContext.java
@@ -22,7 +22,6 @@
import java.nio.ByteBuffer;
import java.util.EnumSet;
import java.util.HashMap;
-import java.util.HashSet;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ExecutorService;
@@ -33,7 +32,7 @@
import org.apache.hyracks.api.dataflow.state.IStateObject;
import org.apache.hyracks.api.deployment.DeploymentId;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.exceptions.Warning;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.job.JobFlag;
@@ -51,7 +50,6 @@
private Map<Object, IStateObject> stateObjectMap = new HashMap<>();
private Object sharedObject;
private final IStatsCollector statsCollector = new StatsCollector();
- private final Set<Warning> warnings = new HashSet<>();
public TestTaskContext(TestJobletContext jobletContext, TaskAttemptId taskId) {
this.jobletContext = jobletContext;
@@ -180,7 +178,7 @@
}
@Override
- public void warn(Warning warning) {
- warnings.add(warning);
+ public IWarningCollector getWarningCollector() {
+ return TestUtils.NOOP_WARNING_COLLECTOR;
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestUtils.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestUtils.java
index e36b655..420f7e6 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/TestUtils.java
@@ -37,6 +37,7 @@
import org.apache.hyracks.api.dataflow.TaskAttemptId;
import org.apache.hyracks.api.dataflow.TaskId;
import org.apache.hyracks.api.exceptions.HyracksException;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.api.io.IODeviceHandle;
import org.apache.hyracks.api.job.JobId;
import org.apache.hyracks.api.util.CleanupUtils;
@@ -48,6 +49,10 @@
import org.apache.logging.log4j.core.config.Configuration;
public class TestUtils {
+
+ public static final IWarningCollector NOOP_WARNING_COLLECTOR = w -> {
+ };
+
public static IHyracksTaskContext create(int frameSize) {
IOManager ioManager = null;
try {
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java
index 9021fc5..aec9e35 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-invertedindex-test/src/test/java/org/apache/hyracks/storage/am/lsm/invertedindex/util/LSMInvertedIndexTestUtils.java
@@ -48,7 +48,7 @@
import org.apache.hyracks.api.exceptions.ErrorCode;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.HyracksException;
-import org.apache.hyracks.api.exceptions.Warning;
+import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.api.io.FileReference;
import org.apache.hyracks.api.io.IIOManager;
import org.apache.hyracks.api.job.JobFlag;
@@ -106,6 +106,7 @@
import org.apache.hyracks.storage.common.IIndexBulkLoader;
import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.MultiComparator;
+import org.apache.hyracks.test.support.TestUtils;
@SuppressWarnings("rawtypes")
public class LSMInvertedIndexTestUtils {
@@ -767,8 +768,8 @@
}
@Override
- public void warn(Warning warning) {
- // no-op
+ public IWarningCollector getWarningCollector() {
+ return TestUtils.NOOP_WARNING_COLLECTOR;
}
}