[NO ISSUE][OTH] Support log redaction
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Support log redaction.
Change-Id: I602c833ba2a055da8fbe8782ec62be683ff4581b
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3105
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Murtadha Hubail <mhubail@apache.org>
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFilterExpressionTypeRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFilterExpressionTypeRule.java
index 1eb3e75..4839ac3 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFilterExpressionTypeRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/CheckFilterExpressionTypeRule.java
@@ -35,6 +35,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.AbstractLogicalOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import org.apache.hyracks.util.LogRedactionUtil;
/**
* This rule is to check if all the filter expression are of the boolean type or a possible (determined
@@ -65,7 +66,8 @@
if (condType.getTypeTag() != ATypeTag.BOOLEAN && condType.getTypeTag() != ATypeTag.ANY
&& !isPossibleBoolean(condType)) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, condition.getSourceLocation(),
- "The select condition " + condition.toString() + " should be of the boolean type.");
+ "The select condition " + LogRedactionUtil.userData(condition.toString()) + " should be of the "
+ + "boolean type.");
}
return false;
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java
index 4d61307..6762e77 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/LoadRecordFieldsRule.java
@@ -223,7 +223,7 @@
}
}
throw new CompilationException(ErrorCode.COMPILATION_ERROR, a2.getSourceLocation(),
- "Field access " + getFirstExpr(a2) + " does not correspond to any input of operator " + topOp);
+ "Field access " + getFirstExpr(a2) + " does not correspond to any input");
}
}
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java
index e6f5d91..421b00d 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/PushFieldAccessRule.java
@@ -64,7 +64,6 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.GroupByOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.NestedTupleSourceOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.visitors.VariableUtilities;
-import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
import org.apache.hyracks.algebricks.core.algebra.util.OperatorPropertiesUtil;
import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
@@ -283,8 +282,7 @@
}
}
throw new CompilationException(ErrorCode.COMPILATION_ERROR, assignOp.getSourceLocation(),
- "Field access " + assignOp.getExpressions().get(0).getValue()
- + " does not correspond to any input of operator " + inputOp);
+ "Field access " + assignOp.getExpressions().get(0).getValue() + " doesn't correspond to any input");
} else {
// check if the accessed field is one of the partitioning key fields. If yes, we can equate the 2 variables
if (inputOp.getOperatorTag() == LogicalOperatorTag.DATASOURCESCAN) {
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java
index 2a42da1..cd75c1e 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixPhysicalOperatorsRule.java
@@ -332,10 +332,11 @@
.createMergeAggregation(aggProducedVars.get(i), aggFuncExpr, context);
if (mergeExpr == null) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, aggFuncExpr.getSourceLocation(),
- "The aggregation function " + aggFuncExpr
+ "The aggregation function "
+ + ((AbstractFunctionCallExpression) aggFuncExpr).getFunctionIdentifier().getName()
+ " does not have a registered intermediate aggregation function.");
}
- mergeExpressionRefs.add(new MutableObject<ILogicalExpression>(mergeExpr));
+ mergeExpressionRefs.add(new MutableObject<>(mergeExpr));
}
aggOp.setMergeExpressions(mergeExpressionRefs);
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetClosedRecordConstructorsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetClosedRecordConstructorsRule.java
index 52858e5..9ca80e5 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetClosedRecordConstructorsRule.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetClosedRecordConstructorsRule.java
@@ -18,6 +18,8 @@
*/
package org.apache.asterix.optimizer.rules;
+import static org.apache.asterix.common.config.GlobalConfig.ASTERIX_LOGGER;
+
import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.exceptions.ErrorCode;
@@ -42,6 +44,7 @@
import org.apache.hyracks.algebricks.core.algebra.visitors.AbstractConstVarFunVisitor;
import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalExpressionReferenceTransform;
import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
+import org.apache.hyracks.util.LogRedactionUtil;
/**
* open-record-constructor() becomes closed-record-constructor() if all the
@@ -113,7 +116,8 @@
int n = expr.getArguments().size();
if (n % 2 > 0) {
throw new CompilationException(ErrorCode.COMPILATION_ERROR, expr.getSourceLocation(),
- "Record constructor expected to have an even number of arguments: " + expr);
+ "Record constructor expected to have an even number of arguments: "
+ + LogRedactionUtil.userData(expr.toString()));
}
for (int i = 0; i < n / 2; i++) {
ILogicalExpression a0 = expr.getArguments().get(2 * i).getValue();
@@ -133,7 +137,8 @@
}
if (allClosed) {
expr.setFunctionInfo(FunctionUtil.getFunctionInfo(BuiltinFunctions.CLOSED_RECORD_CONSTRUCTOR));
- GlobalConfig.ASTERIX_LOGGER.trace("Switching to CLOSED record constructor in " + expr + ".\n");
+ ASTERIX_LOGGER.trace(() -> "Switching to CLOSED record constructor in "
+ + LogRedactionUtil.userData(expr.toString()) + ".\n");
changed = true;
}
}
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
index 405ff73..5d1d690 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SweepIllegalNonfunctionalFunctions.java
@@ -101,23 +101,23 @@
private class IllegalNonfunctionalFunctionSweeperOperatorVisitor implements ILogicalOperatorVisitor<Void, Void> {
- private void sweepExpression(ILogicalExpression expr, ILogicalOperator op) throws AlgebricksException {
+ private void sweepExpression(ILogicalExpression expr) throws AlgebricksException {
if (expr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL && !expr.isFunctional()) {
AbstractFunctionCallExpression fce = (AbstractFunctionCallExpression) expr;
throw new CompilationException(ErrorCode.COMPILATION_ERROR, fce.getSourceLocation(),
- "Found non-functional function " + fce.getFunctionIdentifier() + " in op " + op);
+ "Found non-functional function " + fce.getFunctionIdentifier());
}
}
@Override
public Void visitAggregateOperator(AggregateOperator op, Void arg) throws AlgebricksException {
for (Mutable<ILogicalExpression> me : op.getExpressions()) {
- sweepExpression(me.getValue(), op);
+ sweepExpression(me.getValue());
}
List<Mutable<ILogicalExpression>> mergeExprs = op.getMergeExpressions();
if (mergeExprs != null) {
for (Mutable<ILogicalExpression> me : mergeExprs) {
- sweepExpression(me.getValue(), op);
+ sweepExpression(me.getValue());
}
}
return null;
@@ -126,7 +126,7 @@
@Override
public Void visitRunningAggregateOperator(RunningAggregateOperator op, Void arg) throws AlgebricksException {
for (Mutable<ILogicalExpression> me : op.getExpressions()) {
- sweepExpression(me.getValue(), op);
+ sweepExpression(me.getValue());
}
return null;
}
@@ -139,10 +139,10 @@
@Override
public Void visitGroupByOperator(GroupByOperator op, Void arg) throws AlgebricksException {
for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getGroupByList()) {
- sweepExpression(p.second.getValue(), op);
+ sweepExpression(p.second.getValue());
}
for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : op.getDecorList()) {
- sweepExpression(p.second.getValue(), op);
+ sweepExpression(p.second.getValue());
}
return null;
}
@@ -154,13 +154,13 @@
@Override
public Void visitInnerJoinOperator(InnerJoinOperator op, Void arg) throws AlgebricksException {
- sweepExpression(op.getCondition().getValue(), op);
+ sweepExpression(op.getCondition().getValue());
return null;
}
@Override
public Void visitLeftOuterJoinOperator(LeftOuterJoinOperator op, Void arg) throws AlgebricksException {
- sweepExpression(op.getCondition().getValue(), op);
+ sweepExpression(op.getCondition().getValue());
return null;
}
@@ -172,7 +172,7 @@
@Override
public Void visitOrderOperator(OrderOperator op, Void arg) throws AlgebricksException {
for (Pair<IOrder, Mutable<ILogicalExpression>> p : op.getOrderExpressions()) {
- sweepExpression(p.second.getValue(), op);
+ sweepExpression(p.second.getValue());
}
return null;
}
@@ -266,7 +266,7 @@
@Override
public Void visitDistinctOperator(DistinctOperator op, Void arg) throws AlgebricksException {
for (Mutable<ILogicalExpression> expr : op.getExpressions()) {
- sweepExpression(expr.getValue(), op);
+ sweepExpression(expr.getValue());
}
return null;
}
@@ -310,40 +310,40 @@
@Override
public Void visitForwardOperator(ForwardOperator op, Void arg) throws AlgebricksException {
- sweepExpression(op.getRangeMapExpression().getValue(), op);
+ sweepExpression(op.getRangeMapExpression().getValue());
return null;
}
@Override
public Void visitWindowOperator(WindowOperator op, Void arg) throws AlgebricksException {
for (Mutable<ILogicalExpression> me : op.getPartitionExpressions()) {
- sweepExpression(me.getValue(), op);
+ sweepExpression(me.getValue());
}
for (Pair<IOrder, Mutable<ILogicalExpression>> p : op.getOrderExpressions()) {
- sweepExpression(p.second.getValue(), op);
+ sweepExpression(p.second.getValue());
}
for (Pair<IOrder, Mutable<ILogicalExpression>> p : op.getFrameValueExpressions()) {
- sweepExpression(p.second.getValue(), op);
+ sweepExpression(p.second.getValue());
}
for (Mutable<ILogicalExpression> me : op.getFrameStartExpressions()) {
- sweepExpression(me.getValue(), op);
+ sweepExpression(me.getValue());
}
for (Mutable<ILogicalExpression> me : op.getFrameEndExpressions()) {
- sweepExpression(me.getValue(), op);
+ sweepExpression(me.getValue());
}
for (Mutable<ILogicalExpression> me : op.getFrameExcludeExpressions()) {
- sweepExpression(me.getValue(), op);
+ sweepExpression(me.getValue());
}
ILogicalExpression frameOffset = op.getFrameOffset().getValue();
if (frameOffset != null) {
- sweepExpression(frameOffset, op);
+ sweepExpression(frameOffset);
}
for (Mutable<ILogicalExpression> me : op.getExpressions()) {
ILogicalExpression expr = me.getValue();
if (isStatefulFunctionCall(expr)) {
for (Mutable<ILogicalExpression> fcallArg : ((AbstractFunctionCallExpression) expr)
.getArguments()) {
- sweepExpression(fcallArg.getValue(), op);
+ sweepExpression(fcallArg.getValue());
}
} else {
throw new CompilationException(ErrorCode.COMPILATION_ILLEGAL_STATE, op.getSourceLocation());
diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
index 099f769..8d6f04d 100644
--- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
+++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/am/BTreeAccessMethod.java
@@ -74,6 +74,7 @@
import org.apache.hyracks.algebricks.core.algebra.operators.logical.SelectOperator;
import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestMapOperator;
import org.apache.hyracks.algebricks.core.algebra.util.OperatorManipulationUtil;
+import org.apache.hyracks.util.LogRedactionUtil;
/**
* Class for helping rewrite rules to choose and apply BTree indexes.
@@ -787,7 +788,7 @@
// The original select cond must be an AND. Check it just to be sure.
if (funcExpr.getFunctionIdentifier() != AlgebricksBuiltinFunctions.AND) {
throw new CompilationException(ErrorCode.COMPILATION_FUNC_EXPRESSION_CANNOT_UTILIZE_INDEX,
- funcExpr.getSourceLocation(), funcExpr.toString());
+ funcExpr.getSourceLocation(), LogRedactionUtil.userData(funcExpr.toString()));
}
// Clean the conjuncts.
for (Mutable<ILogicalExpression> arg : funcExpr.getArguments()) {
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index 7c42e5c..99df372 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -76,6 +76,7 @@
import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.http.api.IServletResponse;
import org.apache.hyracks.http.server.utils.HttpUtil;
+import org.apache.hyracks.util.LogRedactionUtil;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -520,7 +521,7 @@
optionalParams = optionalParamProvider.apply(request);
}
setRequestParam(request, param, optionalParams);
- LOGGER.info("handleRequest: {}", param);
+ LOGGER.info(() -> "handleRequest: " + LogRedactionUtil.userData(param.toString()));
ResultDelivery delivery = parseResultDelivery(param.getMode());
setSessionConfig(sessionOutput, param, delivery);
final ResultProperties resultProperties = param.getMaxResultReads() == null ? new ResultProperties(delivery)
@@ -624,33 +625,38 @@
if (t instanceof org.apache.asterix.aqlplus.parser.TokenMgrError || t instanceof TokenMgrError
|| t instanceof AlgebricksException) {
if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("handleException: {}: {}", t.getMessage(), param, t);
+ LOGGER.debug("handleException: {}: {}", t.getMessage(), LogRedactionUtil.userData(param.toString()), t);
} else {
- LOGGER.info("handleException: {}: {}", t.getMessage(), param);
+ LOGGER.info(() -> "handleException: " + t.getMessage() + ": "
+ + LogRedactionUtil.userData(param.toString()));
}
state.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST);
} else if (t instanceof HyracksException) {
HyracksException he = (HyracksException) t;
switch (he.getComponent() + he.getErrorCode()) {
case ASTERIX + REQUEST_TIMEOUT:
- LOGGER.info("handleException: request execution timed out: {}", param);
+ LOGGER.info(() -> "handleException: request execution timed out: "
+ + LogRedactionUtil.userData(param.toString()));
state.setStatus(ResultStatus.TIMEOUT, HttpResponseStatus.OK);
break;
case ASTERIX + REJECT_BAD_CLUSTER_STATE:
case ASTERIX + REJECT_NODE_UNREGISTERED:
- LOGGER.warn("handleException: {}: {}", he.getMessage(), param);
+ LOGGER.warn(() -> "handleException: " + he.getMessage() + ": "
+ + LogRedactionUtil.userData(param.toString()));
state.setStatus(ResultStatus.FATAL, HttpResponseStatus.SERVICE_UNAVAILABLE);
break;
case ASTERIX + NO_STATEMENT_PROVIDED:
state.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST);
break;
default:
- LOGGER.warn("handleException: unexpected exception {}: {}", he.getMessage(), param, he);
+ LOGGER.warn(() -> "handleException: unexpected exception " + he.getMessage() + ": "
+ + LogRedactionUtil.userData(param.toString()), he);
state.setStatus(ResultStatus.FATAL, HttpResponseStatus.INTERNAL_SERVER_ERROR);
break;
}
} else {
- LOGGER.warn("handleException: unexpected exception: {}", param, t);
+ LOGGER.warn(() -> "handleException: unexpected exception: " + LogRedactionUtil.userData(param.toString()),
+ t);
state.setStatus(ResultStatus.FATAL, HttpResponseStatus.INTERNAL_SERVER_ERROR);
}
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedLogManager.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedLogManager.java
index 57c9e2c..282b5366 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedLogManager.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/FeedLogManager.java
@@ -35,6 +35,7 @@
import org.apache.commons.io.FileUtils;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.util.LogRedactionUtil;
public class FeedLogManager implements Closeable {
@@ -45,13 +46,13 @@
SNAPSHOT // an identifier that partitions with identifiers before this one should be ignored
}
- public static final String PROGRESS_LOG_FILE_NAME = "progress.log";
- public static final String ERROR_LOG_FILE_NAME = "error.log";
- public static final String BAD_RECORDS_FILE_NAME = "failed_record.log";
- public static final String START_PREFIX = "s:";
- public static final String END_PREFIX = "e:";
+ private static final String PROGRESS_LOG_FILE_NAME = "progress.log";
+ private static final String ERROR_LOG_FILE_NAME = "error.log";
+ private static final String BAD_RECORDS_FILE_NAME = "failed_record.log";
+ private static final String START_PREFIX = "s:";
+ private static final String END_PREFIX = "e:";
private static final String DATE_FORMAT_STRING = "MM/dd/yyyy HH:mm:ss";
- public static final int PREFIX_SIZE = START_PREFIX.length() + DATE_FORMAT_STRING.length() + 1;
+ private static final int PREFIX_SIZE = START_PREFIX.length() + DATE_FORMAT_STRING.length() + 1;
private String currentPartition;
private final TreeSet<String> completed;
private final Path dir;
@@ -149,7 +150,7 @@
return true;
}
- public synchronized void logProgress(String log) throws IOException {
+ private synchronized void logProgress(String log) throws IOException {
stringBuilder.setLength(0);
stringBuilder.append(df.format((new Date())));
stringBuilder.append(' ');
@@ -173,7 +174,7 @@
public synchronized void logRecord(String record, String errorMessage) throws IOException {
stringBuilder.setLength(0);
- stringBuilder.append(record);
+ stringBuilder.append(LogRedactionUtil.userData(record));
stringBuilder.append(ExternalDataConstants.LF);
stringBuilder.append(df.format((new Date())));
stringBuilder.append(' ');
@@ -183,7 +184,7 @@
recordLogger.flush();
}
- public static String getSplitId(String log) {
+ private static String getSplitId(String log) {
return log.substring(PREFIX_SIZE);
}
diff --git a/asterixdb/asterix-lang-sqlpp/pom.xml b/asterixdb/asterix-lang-sqlpp/pom.xml
index dbfe80b..f13c219 100644
--- a/asterixdb/asterix-lang-sqlpp/pom.xml
+++ b/asterixdb/asterix-lang-sqlpp/pom.xml
@@ -137,6 +137,10 @@
<dependencies>
<dependency>
+ <groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-util</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.apache.asterix</groupId>
<artifactId>asterix-lang-common</artifactId>
<version>${project.version}</version>
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
index 35de051..1bed469 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
@@ -20,21 +20,18 @@
import java.util.ArrayList;
import java.util.Collection;
+import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import java.util.HashSet;
-
-import org.apache.asterix.lang.common.base.AbstractClause;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
import org.apache.asterix.common.exceptions.CompilationException;
import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.lang.common.base.AbstractClause;
import org.apache.asterix.lang.common.base.Expression;
import org.apache.asterix.lang.common.base.IQueryRewriter;
import org.apache.asterix.lang.common.base.IReturningStatement;
-import org.apache.asterix.lang.common.clause.LetClause;
import org.apache.asterix.lang.common.expression.CallExpr;
+import org.apache.asterix.lang.common.expression.ListSliceExpression;
import org.apache.asterix.lang.common.expression.VariableExpr;
import org.apache.asterix.lang.common.rewrites.LangRewritingContext;
import org.apache.asterix.lang.common.statement.FunctionDecl;
@@ -57,7 +54,6 @@
import org.apache.asterix.lang.sqlpp.clause.SelectSetOperation;
import org.apache.asterix.lang.sqlpp.clause.UnnestClause;
import org.apache.asterix.lang.sqlpp.expression.CaseExpression;
-import org.apache.asterix.lang.common.expression.ListSliceExpression;
import org.apache.asterix.lang.sqlpp.expression.SelectExpression;
import org.apache.asterix.lang.sqlpp.expression.WindowExpression;
import org.apache.asterix.lang.sqlpp.parser.FunctionParser;
@@ -73,19 +69,20 @@
import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppGroupByVisitor;
import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppInlineUdfsVisitor;
import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppListInputFunctionRewriteVisitor;
-import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppWindowRewriteVisitor;
import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppWindowAggregationSugarVisitor;
+import org.apache.asterix.lang.sqlpp.rewrites.visitor.SqlppWindowRewriteVisitor;
import org.apache.asterix.lang.sqlpp.rewrites.visitor.SubstituteGroupbyExpressionWithVariableVisitor;
import org.apache.asterix.lang.sqlpp.rewrites.visitor.VariableCheckAndRewriteVisitor;
import org.apache.asterix.lang.sqlpp.struct.SetOperationRight;
import org.apache.asterix.lang.sqlpp.util.FunctionMapUtil;
import org.apache.asterix.lang.sqlpp.util.SqlppAstPrintUtil;
import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil;
-import org.apache.asterix.lang.sqlpp.visitor.FreeVariableVisitor;
-import org.apache.asterix.lang.sqlpp.visitor.base.AbstractSqlppQueryExpressionVisitor;
import org.apache.asterix.lang.sqlpp.visitor.base.ISqlppVisitor;
import org.apache.asterix.metadata.declared.MetadataProvider;
import org.apache.hyracks.algebricks.common.utils.Pair;
+import org.apache.hyracks.util.LogRedactionUtil;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class SqlppQueryRewriter implements IQueryRewriter {
@@ -292,7 +289,7 @@
private void logExpression(String p0, String p1) throws CompilationException {
if (isLogEnabled) {
- LOGGER.trace("{} {}\n{}", p0, p1, SqlppAstPrintUtil.toString(topExpr));
+ LOGGER.trace("{} {}\n{}", p0, p1, LogRedactionUtil.userData(SqlppAstPrintUtil.toString(topExpr)));
}
}
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/ExpressionToVariableUtil.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/ExpressionToVariableUtil.java
index 4842026..d8534be 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/ExpressionToVariableUtil.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/util/ExpressionToVariableUtil.java
@@ -27,6 +27,7 @@
import org.apache.asterix.lang.common.struct.VarIdentifier;
import org.apache.asterix.lang.sqlpp.parser.ParseException;
import org.apache.asterix.lang.sqlpp.parser.SqlppParseException;
+import org.apache.hyracks.util.LogRedactionUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -54,7 +55,7 @@
throw new SqlppParseException(expr.getSourceLocation(), e.getLocalizedMessage());
}
throw new SqlppParseException(expr.getSourceLocation(),
- "Need an alias for the enclosed expression:\n" + exprText);
+ "Need an alias for the enclosed expression:\n" + LogRedactionUtil.userData(exprText));
}
}
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PointableHelper.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PointableHelper.java
index 1f086b8..068f239 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PointableHelper.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/PointableHelper.java
@@ -32,6 +32,7 @@
import org.apache.hyracks.data.std.api.IValueReference;
import org.apache.hyracks.data.std.primitive.UTF8StringPointable;
import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.util.LogRedactionUtil;
import org.apache.hyracks.util.string.UTF8StringWriter;
/**
@@ -134,7 +135,7 @@
}
utf8Writer.writeUTF8(str, output);
} catch (IOException e) {
- throw new HyracksDataException("Could not serialize " + str);
+ throw new HyracksDataException("Could not serialize " + LogRedactionUtil.userData(str));
}
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/pom.xml b/hyracks-fullstack/algebricks/algebricks-core/pom.xml
index 53ee28b..55aa465 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/pom.xml
+++ b/hyracks-fullstack/algebricks/algebricks-core/pom.xml
@@ -44,6 +44,11 @@
<dependencies>
<dependency>
<groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-util</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
<artifactId>hyracks-dataflow-std</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java
index d4a9d37..571a30a 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/AbstractLogicalOperator.java
@@ -18,6 +18,8 @@
*/
package org.apache.hyracks.algebricks.core.algebra.operators.logical;
+import static org.apache.hyracks.api.exceptions.ErrorCode.PHYS_OPERATOR_NOT_SET;
+
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -164,7 +166,7 @@
throws AlgebricksException {
if (bJobGenEnabled) {
if (physicalOperator == null) {
- throw new AlgebricksException("Physical operator not set for operator: " + this);
+ throw AlgebricksException.create(PHYS_OPERATOR_NOT_SET, getSourceLocation(), this.getOperatorTag());
}
physicalOperator.contributeRuntimeOperator(builder, context, this, propagatedSchema, inputSchemas,
outerPlanSchema);
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java
index 7042794..f5f5f96 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/logical/visitors/FDsAndEquivClassesVisitor.java
@@ -86,6 +86,7 @@
import org.apache.hyracks.algebricks.core.algebra.properties.LocalGroupingProperty;
import org.apache.hyracks.algebricks.core.algebra.visitors.ILogicalOperatorVisitor;
import org.apache.hyracks.algebricks.core.config.AlgebricksConfig;
+import org.apache.hyracks.util.LogRedactionUtil;
public class FDsAndEquivClassesVisitor implements ILogicalOperatorVisitor<Void, IOptimizationContext> {
@@ -316,9 +317,9 @@
}
}
if (changed && AlgebricksConfig.ALGEBRICKS_LOGGER.isTraceEnabled()) {
- AlgebricksConfig.ALGEBRICKS_LOGGER
- .trace(">>>> Group-by list changed from " + GroupByOperator.veListToString(gByList) + " to "
- + GroupByOperator.veListToString(newGbyList) + ".\n");
+ AlgebricksConfig.ALGEBRICKS_LOGGER.trace(">>>> Group-by list changed from {} to {}.\n",
+ LogRedactionUtil.userData(GroupByOperator.veListToString(gByList)),
+ LogRedactionUtil.userData(GroupByOperator.veListToString(newGbyList)));
}
gByList.clear();
gByList.addAll(newGbyList);
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractUnnestPOperator.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractUnnestPOperator.java
index 0602258..5d54e1d 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractUnnestPOperator.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/operators/physical/AbstractUnnestPOperator.java
@@ -36,6 +36,7 @@
import org.apache.hyracks.algebricks.runtime.base.IUnnestingEvaluatorFactory;
import org.apache.hyracks.algebricks.runtime.operators.std.UnnestRuntimeFactory;
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
+import org.apache.hyracks.util.LogRedactionUtil;
public abstract class AbstractUnnestPOperator extends AbstractScanPOperator {
private final boolean leftOuter;
@@ -73,7 +74,8 @@
}
}
if (exit) {
- throw new AlgebricksException("Unnest expression " + unnestExpr + " is not an unnesting function call.");
+ throw new AlgebricksException("Unnest expression " + LogRedactionUtil.userData(unnestExpr.toString())
+ + " is not an unnesting function call.");
}
UnnestingFunctionCallExpression agg = (UnnestingFunctionCallExpression) unnestExpr;
IUnnestingEvaluatorFactory unnestingFactory = expressionRuntimeProvider.createUnnestingFunctionFactory(agg,
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java
index 526add1..c8fff99 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/jobgen/impl/JobBuilder.java
@@ -18,6 +18,8 @@
*/
package org.apache.hyracks.algebricks.core.jobgen.impl;
+import static org.apache.hyracks.api.exceptions.ErrorCode.DESCRIPTOR_GENERATION_ERROR;
+
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -311,7 +313,7 @@
}
Integer metaOpKey = algebraicOpBelongingToMetaAsterixOp.get(op);
if (metaOpKey == null) {
- throw new AlgebricksException("Could not generate operator descriptor for operator " + op);
+ throw AlgebricksException.create(DESCRIPTOR_GENERATION_ERROR, op.getSourceLocation(), op.getOperatorTag());
}
return metaAsterixOps.get(metaOpKey);
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java
index 370ec6d..08fe460 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/AbstractRuleController.java
@@ -31,6 +31,7 @@
import org.apache.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
import org.apache.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter;
import org.apache.hyracks.algebricks.core.config.AlgebricksConfig;
+import org.apache.hyracks.util.LogRedactionUtil;
public abstract class AbstractRuleController {
@@ -47,7 +48,7 @@
* Each rewriting strategy may differ in the
*
* @param root
- * @param ruleClasses
+ * @param rules
* @return true iff one of the rules in the collection fired
*/
public abstract boolean rewriteWithRuleCollection(Mutable<ILogicalOperator> root,
@@ -74,12 +75,11 @@
return null;
}
- private void printRuleApplication(IAlgebraicRewriteRule rule, String beforePlan, String afterPlan)
- throws AlgebricksException {
+ private void printRuleApplication(IAlgebraicRewriteRule rule, String beforePlan, String afterPlan) {
if (AlgebricksConfig.ALGEBRICKS_LOGGER.isTraceEnabled()) {
- AlgebricksConfig.ALGEBRICKS_LOGGER.trace(">>>> Rule " + rule.getClass() + " fired.\n");
- AlgebricksConfig.ALGEBRICKS_LOGGER.trace(">>>> Before plan\n" + beforePlan + "\n");
- AlgebricksConfig.ALGEBRICKS_LOGGER.trace(">>>> After plan\n" + afterPlan + "\n");
+ AlgebricksConfig.ALGEBRICKS_LOGGER.trace(">> Rule " + rule.getClass() + " fired.\n");
+ AlgebricksConfig.ALGEBRICKS_LOGGER.trace(">> Before plan\n" + LogRedactionUtil.userData(beforePlan) + "\n");
+ AlgebricksConfig.ALGEBRICKS_LOGGER.trace(">> After plan\n" + LogRedactionUtil.userData(afterPlan) + "\n");
}
}
diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java
index 2aacc0e..66ee453 100644
--- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java
+++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/HeuristicOptimizer.java
@@ -33,6 +33,7 @@
import org.apache.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor;
import org.apache.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter;
import org.apache.hyracks.algebricks.core.config.AlgebricksConfig;
+import org.apache.hyracks.util.LogRedactionUtil;
import org.apache.logging.log4j.Level;
public class HeuristicOptimizer {
@@ -87,7 +88,8 @@
final LogicalOperatorPrettyPrintVisitor pvisitor = context.getPrettyPrintVisitor();
pvisitor.reset(new AlgebricksAppendable());
PlanPrettyPrinter.printPlan(plan, pvisitor, 0);
- AlgebricksConfig.ALGEBRICKS_LOGGER.log(lvl, name + ":\n" + pvisitor.get().toString());
+ AlgebricksConfig.ALGEBRICKS_LOGGER.log(lvl,
+ name + ":\n" + LogRedactionUtil.userData(pvisitor.get().toString()));
}
}
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/pom.xml b/hyracks-fullstack/algebricks/algebricks-rewriter/pom.xml
index f10ca9a..af48b78 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/pom.xml
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/pom.xml
@@ -39,6 +39,11 @@
<dependencies>
<dependency>
<groupId>org.apache.hyracks</groupId>
+ <artifactId>hyracks-util</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hyracks</groupId>
<artifactId>algebricks-core</artifactId>
<version>${project.version}</version>
</dependency>
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java
index 96e2e53..de9b4bc 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/EnforceStructuralPropertiesRule.java
@@ -106,6 +106,7 @@
import org.apache.hyracks.algebricks.rewriter.util.PhysicalOptimizationsUtil;
import org.apache.hyracks.api.exceptions.SourceLocation;
import org.apache.hyracks.dataflow.common.data.partition.range.RangeMap;
+import org.apache.hyracks.util.LogRedactionUtil;
public class EnforceStructuralPropertiesRule implements IAlgebraicRewriteRule {
@@ -901,7 +902,7 @@
LogicalOperatorPrettyPrintVisitor pvisitor = new LogicalOperatorPrettyPrintVisitor();
PlanPrettyPrinter.printOperator(op, pvisitor, 0);
if (AlgebricksConfig.ALGEBRICKS_LOGGER.isTraceEnabled()) {
- AlgebricksConfig.ALGEBRICKS_LOGGER.trace(pvisitor.get().toString());
+ AlgebricksConfig.ALGEBRICKS_LOGGER.trace(LogRedactionUtil.userData(pvisitor.get().toString()));
}
}
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
index 4273553..612f79e 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/rules/SetAlgebricksPhysicalOperatorsRule.java
@@ -18,6 +18,8 @@
*/
package org.apache.hyracks.algebricks.rewriter.rules;
+import static org.apache.hyracks.api.exceptions.ErrorCode.ORDER_EXPR_NOT_NORMALIZED;
+
import java.util.ArrayList;
import java.util.List;
@@ -222,7 +224,7 @@
for (Pair<IOrder, Mutable<ILogicalExpression>> p : oo.getOrderExpressions()) {
ILogicalExpression e = p.second.getValue();
if (e.getExpressionTag() != LogicalExpressionTag.VARIABLE) {
- throw new AlgebricksException("Order expression " + e + " has not been normalized.");
+ throw AlgebricksException.create(ORDER_EXPR_NOT_NORMALIZED, e.getSourceLocation());
}
}
if (topLevelOp) {
diff --git a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/JoinUtils.java b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/JoinUtils.java
index 6306338..3042548 100644
--- a/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/JoinUtils.java
+++ b/hyracks-fullstack/algebricks/algebricks-rewriter/src/main/java/org/apache/hyracks/algebricks/rewriter/util/JoinUtils.java
@@ -116,8 +116,8 @@
ILogicalPropertiesVector v = context.getLogicalPropertiesVector(opBuild);
boolean loggerTraceEnabled = AlgebricksConfig.ALGEBRICKS_LOGGER.isTraceEnabled();
if (loggerTraceEnabled) {
- AlgebricksConfig.ALGEBRICKS_LOGGER
- .trace("// HybridHashJoin inner branch -- Logical properties for " + opBuild + ": " + v + "\n");
+ AlgebricksConfig.ALGEBRICKS_LOGGER.trace("// HybridHashJoin inner branch -- Logical properties for "
+ + opBuild.getOperatorTag() + ": " + v + "\n");
}
if (v != null) {
int size2 = v.getMaxOutputFrames();
@@ -125,7 +125,7 @@
if (size2 > 0 && size2 * hhj.getFudgeFactor() <= hhj.getMemSizeInFrames()) {
if (loggerTraceEnabled) {
AlgebricksConfig.ALGEBRICKS_LOGGER
- .trace("// HybridHashJoin inner branch " + opBuild + " fits in memory\n");
+ .trace("// HybridHashJoin inner branch " + opBuild.getOperatorTag() + " fits in memory\n");
}
// maintains the local properties on the probe side
op.setPhysicalOperator(
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
index 690870c..d6860f7 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/java/org/apache/hyracks/api/exceptions/ErrorCode.java
@@ -157,6 +157,9 @@
// Compilation error codes.
public static final int RULECOLLECTION_NOT_INSTANCE_OF_LIST = 10000;
public static final int CANNOT_COMPOSE_PART_CONSTRAINTS = 10001;
+ public static final int PHYS_OPERATOR_NOT_SET = 10002;
+ public static final int DESCRIPTOR_GENERATION_ERROR = 10003;
+ public static final int ORDER_EXPR_NOT_NORMALIZED = 10004;
private static class Holder {
private static final Map<Integer, String> errorMessageMap;
diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
index 413bbee..e95bf76 100644
--- a/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
+++ b/hyracks-fullstack/hyracks/hyracks-api/src/main/resources/errormsg/en.properties
@@ -139,3 +139,6 @@
10000 = The given rule collection %1$s is not an instance of the List class.
10001 = Cannot compose partition constraint %1$s with %2$s
+10002 = Physical operator not set for operator: %1$s
+10003 = Could not generate operator descriptor for operator %1$s
+10004 = Order expression has not been normalized
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/FrameDeserializer.java b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/FrameDeserializer.java
index 5a39523..c4fd289 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/FrameDeserializer.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/FrameDeserializer.java
@@ -26,6 +26,7 @@
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.comm.util.ByteBufferInputStream;
+import org.apache.hyracks.util.LogRedactionUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -71,7 +72,7 @@
for (int i = 0; i < record.length; ++i) {
Object instance = recordDescriptor.getFields()[i].deserialize(di);
if (LOGGER.isTraceEnabled()) {
- LOGGER.trace(i + " " + instance);
+ LOGGER.trace(i + " " + LogRedactionUtil.userData(instance.toString()));
}
record[i] = instance;
if (FrameConstants.DEBUG_FRAME_IO) {
diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/SerializingDataWriter.java b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/SerializingDataWriter.java
index f7b5e3b..ce1d595 100644
--- a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/SerializingDataWriter.java
+++ b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/SerializingDataWriter.java
@@ -25,6 +25,7 @@
import org.apache.hyracks.api.dataflow.value.RecordDescriptor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.dataflow.common.comm.util.FrameUtils;
+import org.apache.hyracks.util.LogRedactionUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -82,7 +83,7 @@
for (int i = 0; i < data.length; ++i) {
Object instance = data[i];
if (LOGGER.isTraceEnabled()) {
- LOGGER.trace(i + " " + instance);
+ LOGGER.trace(i + " " + LogRedactionUtil.userData(instance.toString()));
}
tb.addField(recordDescriptor.getFields()[i], instance);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ILogRedactor.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ILogRedactor.java
new file mode 100644
index 0000000..18e9a34
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ILogRedactor.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.util;
+
+public interface ILogRedactor {
+
+ /**
+ * Redacts user data argument.
+ * @param text user data to redact.
+ * @return redacted user data.
+ */
+ String userData(String text);
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/LogRedactionUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/LogRedactionUtil.java
new file mode 100644
index 0000000..29242be
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/LogRedactionUtil.java
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hyracks.util;
+
+public class LogRedactionUtil {
+
+ private static final ILogRedactor defaultLogRedactor = text -> text;
+ private static ILogRedactor redactor = defaultLogRedactor;
+
+ private LogRedactionUtil() {
+ }
+
+ public static void setRedactor(ILogRedactor redactor) {
+ LogRedactionUtil.redactor = redactor;
+ }
+
+ public static String userData(String text) {
+ return redactor.userData(text);
+ }
+}