[NO ISSUE][API] Add query request param to indicate source
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
Add parameter 'source' to QueryServiceRequestParameters
to indicate the source of the request.
Change-Id: I04e2e2336d6bca82b4fa44a43cb188cfa9e807f7
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18006
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mblow@apache.org>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java
index b6913e4..20d79d5 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceRequestParameters.java
@@ -19,6 +19,8 @@
package org.apache.asterix.api.http.server;
+import static org.apache.asterix.utils.RedactionUtil.REDACTED_SENSITIVE_ENTRY_VALUE;
+
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
@@ -43,6 +45,7 @@
import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.http.server.utils.HttpUtil;
import org.apache.hyracks.util.JSONUtil;
+import org.apache.hyracks.util.LogRedactionUtil;
import com.fasterxml.jackson.core.JsonParseException;
import com.fasterxml.jackson.core.JsonProcessingException;
@@ -80,7 +83,8 @@
SIGNATURE("signature"),
MULTI_STATEMENT("multi-statement"),
MAX_WARNINGS("max-warnings"),
- SQL_COMPAT("sql-compat");
+ SQL_COMPAT("sql-compat"),
+ SOURCE("source");
private final String str;
@@ -124,6 +128,7 @@
private String statement;
private String clientContextID;
private String dataverse;
+ private String source;
private ClientType clientType = ClientType.ASTERIX;
private OutputFormat format = OutputFormat.CLEAN_JSON;
private ResultDelivery mode = ResultDelivery.IMMEDIATE;
@@ -172,6 +177,14 @@
this.statement = statement;
}
+ public String getSource() {
+ return source;
+ }
+
+ public void setSource(String source) {
+ this.source = source;
+ }
+
public OutputFormat getFormat() {
return format;
}
@@ -380,7 +393,8 @@
ObjectNode object = OBJECT_MAPPER.createObjectNode();
object.put("host", host);
object.put("path", path);
- object.put("statement", statement != null ? JSONUtil.escape(new StringBuilder(), statement).toString() : null);
+ object.put("statement", statement != null
+ ? LogRedactionUtil.statement(JSONUtil.escape(new StringBuilder(), statement).toString()) : null);
object.put("pretty", pretty);
object.put("mode", mode.getName());
object.put("clientContextID", clientContextID);
@@ -402,9 +416,10 @@
object.put("readOnly", readOnly);
object.put("maxWarnings", maxWarnings);
object.put("sqlCompat", sqlCompatMode);
+ object.put("source", source);
if (statementParams != null) {
for (Map.Entry<String, JsonNode> statementParam : statementParams.entrySet()) {
- object.set('$' + statementParam.getKey(), statementParam.getValue());
+ object.set('$' + statementParam.getKey(), REDACTED_SENSITIVE_ENTRY_VALUE);
}
}
return object;
@@ -486,6 +501,7 @@
setSignature(parseBoolean(req, Parameter.SIGNATURE.str(), valGetter, isSignature()));
setClientType(parseIfExists(req, Parameter.CLIENT_TYPE.str(), valGetter, getClientType(), clientTypes::get));
setSQLCompatMode(parseBoolean(req, Parameter.SQL_COMPAT.str(), valGetter, isSQLCompatMode()));
+ setSource(valGetter.apply(req, Parameter.SOURCE.str()));
}
protected void setExtraParams(JsonNode jsonRequest) throws HyracksDataException {
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
index 1966a88..9a8c0d5 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/QueryServiceServlet.java
@@ -90,7 +90,6 @@
import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.http.api.IServletResponse;
import org.apache.hyracks.http.server.utils.HttpUtil;
-import org.apache.hyracks.util.LogRedactionUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -278,7 +277,10 @@
if (forceReadOnly) {
param.setReadOnly(true);
}
- LOGGER.info(() -> "handleRequest: " + LogRedactionUtil.statement(param.toString()));
+ if (LOGGER.isInfoEnabled()) {
+ LOGGER.info("handleRequest: uuid={}, clientContextID={}, {}", requestRef.getUuid(),
+ param.getClientContextID(), param.toString());
+ }
delivery = param.getMode();
setSessionConfig(sessionOutput, param, delivery);
final ResultProperties resultProperties = new ResultProperties(delivery, param.getMaxResultReads());
@@ -429,14 +431,12 @@
executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST);
return true;
case REQUEST_TIMEOUT:
- LOGGER.info(() -> "handleException: request execution timed out: "
- + LogRedactionUtil.userData(param.toString()));
+ LOGGER.info(() -> "handleException: request execution timed out: " + param.toString());
executionState.setStatus(ResultStatus.TIMEOUT, HttpResponseStatus.OK);
return true;
case REJECT_NODE_UNREGISTERED:
case REJECT_BAD_CLUSTER_STATE:
- LOGGER.warn(() -> "handleException: " + ex.getMessage() + ": "
- + LogRedactionUtil.userData(param.toString()));
+ LOGGER.warn(() -> "handleException: " + ex.getMessage() + ": " + param.toString());
executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.SERVICE_UNAVAILABLE);
return true;
default:
@@ -456,11 +456,9 @@
QueryServiceRequestParameters param, IServletResponse response) {
if (t instanceof org.apache.asterix.lang.sqlpp.parser.TokenMgrError || t instanceof AlgebricksException) {
if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("handleException: {}: {}", t.getMessage(), LogRedactionUtil.statement(param.toString()),
- t);
+ LOGGER.debug("handleException: {}: {}", t.getMessage(), param.toString(), t);
} else {
- LOGGER.info(() -> "handleException: " + t.getMessage() + ": "
- + LogRedactionUtil.statement(param.toString()));
+ LOGGER.info(() -> "handleException: " + t.getMessage() + ": " + param.toString());
}
executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.BAD_REQUEST);
return;
@@ -472,7 +470,7 @@
return;
}
}
- LOGGER.warn(() -> "handleException: unexpected exception: " + LogRedactionUtil.userData(param.toString()), t);
+ LOGGER.warn(() -> "handleException: unexpected exception: " + param.toString(), t);
executionState.setStatus(ResultStatus.FATAL, HttpResponseStatus.INTERNAL_SERVER_ERROR);
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
index 05bc87b..e314177 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/message/ExecuteStatementRequestMessage.java
@@ -243,7 +243,8 @@
@Override
public String toString() {
- return String.format("%s(id=%s, from=%s, uuid=%s): %s", getClass().getSimpleName(), requestMessageId,
- requestNodeId, requestReference.getUuid(), LogRedactionUtil.statement(statementsText));
+ return String.format("%s(id=%s, from=%s, uuid=%s, clientContextID=%s): %s", getClass().getSimpleName(),
+ requestMessageId, requestNodeId, requestReference.getUuid(), clientContextID,
+ LogRedactionUtil.statement(statementsText));
}
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index afdd035..6b40cbf 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -4837,9 +4837,9 @@
// ensure request not cancelled before running job
ensureNotCancelled(clientRequest);
final JobId jobId = JobUtils.runJob(hcc, jobSpec, jobFlags, false);
- if (LOGGER.isDebugEnabled()) {
- LOGGER.debug("createAndRunJob jobId:{}, uuid:{}", jobId,
- requestParameters.getRequestReference().getUuid());
+ if (LOGGER.isInfoEnabled()) {
+ LOGGER.info("Created job {} for query uuid:{}, clientContextID:{}", jobId,
+ requestParameters.getRequestReference().getUuid(), requestParameters.getClientContextId());
}
clientRequest.setJobId(jobId);
if (jId != null) {
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java
index 156b78a..d835664 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/RedactionUtil.java
@@ -26,6 +26,8 @@
import org.apache.hyracks.util.ILogRedactor;
+import com.fasterxml.jackson.databind.node.TextNode;
+
public class RedactionUtil {
private RedactionUtil() {
throw new AssertionError("do not instantiate");
@@ -34,6 +36,7 @@
private static final Pattern STATEMENT_PATTERN =
Pattern.compile("(" + SECRET_ACCESS_KEY_FIELD_NAME + ").*", CASE_INSENSITIVE | DOTALL);
private static final String STATEMENT_REPLACEMENT = "$1...<redacted sensitive data>";
+ public static final TextNode REDACTED_SENSITIVE_ENTRY_VALUE = new TextNode("<redacted sensitive entry>");
public static final ILogRedactor LOG_REDACTOR = new ILogRedactor() {
@Override