Merge commit 'e20c7eea498263f92267f4cbc39ad9372006ff6c' from release-0.9.4-pre-rc
Change-Id: I32462904d0c876b627412a20bd65d2190544a016
diff --git a/asterixdb/asterix-external-data/pom.xml b/asterixdb/asterix-external-data/pom.xml
index d88b05e..cbaa889 100644
--- a/asterixdb/asterix-external-data/pom.xml
+++ b/asterixdb/asterix-external-data/pom.xml
@@ -131,20 +131,6 @@
</executions>
</plugin>
<plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-dependency-plugin</artifactId>
- <configuration>
- <ignoredUsedUndeclaredDependencies>
- <ignoredUsedUndeclaredDependency>org.json:json:*</ignoredUsedUndeclaredDependency>
- <ignoredUsedUndeclaredDependency>stax:stax-api:*</ignoredUsedUndeclaredDependency>
- <ignoredUsedUndeclaredDependency>javax.xml.bind:jaxb-api:*</ignoredUsedUndeclaredDependency>
- </ignoredUsedUndeclaredDependencies>
- <ignoredUnusedDeclaredDependencies>
- <ignoredUnusedDeclaredDependency>xml-apis:xml-apis:*</ignoredUnusedDeclaredDependency>
- </ignoredUnusedDeclaredDependencies>
- </configuration>
- </plugin>
- <plugin>
<groupId>org.apache.rat</groupId>
<artifactId>apache-rat-plugin</artifactId>
<configuration>
@@ -395,11 +381,6 @@
<artifactId>hyracks-api</artifactId>
</dependency>
<dependency>
- <groupId>xml-apis</groupId>
- <artifactId>xml-apis</artifactId>
- <version>1.4.01</version>
- </dependency>
- <dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</dependency>
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java
index b9cfac4..94ae75c 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/operators/FeedMetaStoreNodePushable.java
@@ -42,6 +42,7 @@
import org.apache.hyracks.dataflow.common.utils.TaskUtil;
import org.apache.hyracks.dataflow.std.base.AbstractUnaryInputUnaryOutputOperatorNodePushable;
import org.apache.hyracks.util.trace.ITracer;
+import org.apache.hyracks.util.trace.TraceUtils;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -108,7 +109,7 @@
this.recordDescProvider = recordDescProvider;
this.opDesc = feedMetaOperatorDescriptor;
tracer = ctx.getJobletContext().getServiceContext().getTracer();
- traceCategory = tracer.getRegistry().get("Process-Frame");
+ traceCategory = tracer.getRegistry().get(TraceUtils.STORAGE);
}
@Override
diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
index 68053d3..dba6760 100644
--- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
+++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/operators/LSMPrimaryUpsertOperatorNodePushable.java
@@ -21,6 +21,9 @@
import java.io.DataOutput;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Date;
import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.dataflow.LSMIndexUtil;
@@ -68,9 +71,17 @@
import org.apache.hyracks.storage.common.IIndexAccessParameters;
import org.apache.hyracks.storage.common.IIndexCursor;
import org.apache.hyracks.storage.common.MultiComparator;
+import org.apache.hyracks.util.trace.ITracer;
+import org.apache.hyracks.util.trace.ITracer.Scope;
+import org.apache.hyracks.util.trace.TraceUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
public class LSMPrimaryUpsertOperatorNodePushable extends LSMIndexInsertUpdateDeleteOperatorNodePushable {
+ private static final Logger LOGGER = LogManager.getLogger();
+ private static final ThreadLocal<DateFormat> DATE_FORMAT =
+ ThreadLocal.withInitial(() -> new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS"));
private final PermutingFrameTupleReference key;
private MultiComparator keySearchCmp;
private ArrayTupleBuilder missingTupleBuilder;
@@ -98,7 +109,9 @@
private final ISearchOperationCallbackFactory searchCallbackFactory;
private final IFrameTupleProcessor processor;
private LSMTreeIndexAccessor lsmAccessor;
- private IIndexAccessParameters iap;
+ private final ITracer tracer;
+ private final long traceCategory;
+ private long lastRecordInTimeStamp = 0L;
public LSMPrimaryUpsertOperatorNodePushable(IHyracksTaskContext ctx, int partition,
IIndexDataflowHelperFactory indexHelperFactory, int[] fieldPermutation, RecordDescriptor inputRecDesc,
@@ -190,6 +203,8 @@
lsmAccessor.getCtx().setOperation(IndexOperation.UPSERT);
}
};
+ tracer = ctx.getJobletContext().getServiceContext().getTracer();
+ traceCategory = tracer.getRegistry().get(TraceUtils.LATENCY);
}
// we have the permutation which has [pk locations, record location, optional:filter-location]
@@ -226,7 +241,7 @@
abstractModCallback = (AbstractIndexModificationOperationCallback) modCallback;
searchCallback = (LockThenSearchOperationCallback) searchCallbackFactory
.createSearchOperationCallback(indexHelper.getResource().getId(), ctx, this);
- iap = new IndexAccessParameters(abstractModCallback, searchCallback);
+ IIndexAccessParameters iap = new IndexAccessParameters(abstractModCallback, searchCallback);
indexAccessor = index.createAccessor(iap);
lsmAccessor = (LSMTreeIndexAccessor) indexAccessor;
cursor = indexAccessor.createSearchCursor(false);
@@ -289,7 +304,11 @@
@Override
public void nextFrame(ByteBuffer buffer) throws HyracksDataException {
accessor.reset(buffer);
+ int itemCount = accessor.getTupleCount();
lsmAccessor.batchOperate(accessor, tuple, processor, frameOpCallback);
+ if (itemCount > 0) {
+ lastRecordInTimeStamp = System.currentTimeMillis();
+ }
}
private void appendFilterToOutput() throws IOException {
@@ -366,6 +385,7 @@
@Override
public void close() throws HyracksDataException {
+ traceLastRecordIn();
Throwable failure = CleanupUtils.close(frameOpCallback, null);
failure = CleanupUtils.destroy(failure, cursor);
failure = CleanupUtils.close(writer, failure);
@@ -375,6 +395,24 @@
}
}
+ @SuppressWarnings({ "squid:S1181", "squid:S1166" })
+ private void traceLastRecordIn() {
+ try {
+ if (tracer.isEnabled(traceCategory) && lastRecordInTimeStamp > 0 && indexHelper != null
+ && indexHelper.getIndexInstance() != null) {
+ tracer.instant("UpsertClose", traceCategory, Scope.t,
+ "{\"last-record-in\":\"" + DATE_FORMAT.get().format(new Date(lastRecordInTimeStamp))
+ + "\", \"index\":" + indexHelper.getIndexInstance().toString() + "}");
+ }
+ } catch (Throwable traceFailure) {
+ try {
+ LOGGER.warn("Tracing last record in failed", traceFailure);
+ } catch (Throwable ignore) {
+ // Ignore logging failure
+ }
+ }
+ }
+
@Override
public void fail() throws HyracksDataException {
writer.fail();
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index 45d5297..6b2dfb4 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -863,6 +863,14 @@
<groupId>commons-logging</groupId>
<artifactId>commons-logging</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>stax</groupId>
+ <artifactId>stax-api</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.xml.bind</groupId>
+ <artifactId>jaxb-api</artifactId>
+ </exclusion>
</exclusions>
</dependency>
<dependency>
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/CcConnection.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/CcConnection.java
index dce7d35..627e972 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/CcConnection.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/CcConnection.java
@@ -33,6 +33,7 @@
public class CcConnection {
private static final Logger LOGGER = LogManager.getLogger();
+ private static final long REGISTRATION_RESPONSE_POLL_PERIOD = TimeUnit.SECONDS.toMillis(1);
private final IClusterController ccs;
private boolean registrationPending;
@@ -64,7 +65,9 @@
registrationPending = true;
ccs.registerNode(nodeRegistration, registrationId);
try {
- InvokeUtil.runWithTimeout(this::wait, () -> !registrationPending, 2, TimeUnit.MINUTES);
+ InvokeUtil.runWithTimeout(() -> {
+ this.wait(REGISTRATION_RESPONSE_POLL_PERIOD); // NOSONAR while loop in timeout call
+ }, () -> !registrationPending, 1, TimeUnit.MINUTES);
} catch (Exception e) {
registrationException = e;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
index a74a1ab..76b5c8c 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/NodeControllerService.java
@@ -96,6 +96,7 @@
import org.apache.hyracks.util.MaintainedThreadNameExecutorService;
import org.apache.hyracks.util.PidHelper;
import org.apache.hyracks.util.trace.ITracer;
+import org.apache.hyracks.util.trace.TraceUtils;
import org.apache.hyracks.util.trace.Tracer;
import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
@@ -713,7 +714,7 @@
public TraceCurrentTimeTask(ITracer tracer) {
this.tracer = tracer;
- this.traceCategory = tracer.getRegistry().get("Timestamp");
+ this.traceCategory = tracer.getRegistry().get(TraceUtils.TIMESTAMP);
}
@Override
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/api/IServletResponse.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/api/IServletResponse.java
index 1a7c65f..38f2d23 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/api/IServletResponse.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/api/IServletResponse.java
@@ -78,8 +78,12 @@
ChannelFuture lastContentFuture() throws IOException;
/**
- * Notifies the response that the channel has become writable
- * became writable or unwritable. Used for flow control
+ * Notifies the response that the channel has become writable. Used for flow control
*/
void notifyChannelWritable();
+
+ /**
+ * Notifies the response that the channel has become inactive.
+ */
+ void notifyChannelInactive();
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedNettyOutputStream.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedNettyOutputStream.java
index d5f81e5..d4f1b3d 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedNettyOutputStream.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedNettyOutputStream.java
@@ -43,12 +43,6 @@
this.response = response;
this.ctx = ctx;
buffer = ctx.alloc().buffer(chunkSize);
- // register listener for channel closed
- ctx.channel().closeFuture().addListener(futureListener -> {
- synchronized (ChunkedNettyOutputStream.this) {
- ChunkedNettyOutputStream.this.notifyAll();
- }
- });
}
@Override
@@ -128,8 +122,8 @@
private synchronized void ensureWritable() throws IOException {
while (!ctx.channel().isWritable()) {
try {
- if (!ctx.channel().isOpen()) {
- throw new IOException("Closed channel");
+ if (!ctx.channel().isActive()) {
+ throw new IOException("Inactive channel");
}
wait();
} catch (InterruptedException e) {
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedResponse.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedResponse.java
index 323a463..5a43d25 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedResponse.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/ChunkedResponse.java
@@ -187,4 +187,9 @@
public void notifyChannelWritable() {
outputStream.resume();
}
+
+ @Override
+ public void notifyChannelInactive() {
+ outputStream.resume();
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/FullResponse.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/FullResponse.java
index 598048e..90e33b6 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/FullResponse.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/FullResponse.java
@@ -105,4 +105,10 @@
// Do nothing.
// This response is sent as a single piece
}
+
+ @Override
+ public void notifyChannelInactive() {
+ // Do nothing.
+ // This response is sent as a single piece
+ }
}
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpRequestHandler.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpRequestHandler.java
index bf8e629..65a082c 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpRequestHandler.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpRequestHandler.java
@@ -82,6 +82,10 @@
response.notifyChannelWritable();
}
+ public void notifyChannelInactive() {
+ response.notifyChannelInactive();
+ }
+
public void reject() throws IOException {
try {
response.setStatus(HttpResponseStatus.SERVICE_UNAVAILABLE);
diff --git a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerHandler.java b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerHandler.java
index 2787b30..7b3d18a 100644
--- a/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerHandler.java
+++ b/hyracks-fullstack/hyracks/hyracks-http/src/main/java/org/apache/hyracks/http/server/HttpServerHandler.java
@@ -44,6 +44,9 @@
protected final T server;
protected final int chunkSize;
protected HttpRequestHandler handler;
+ protected IChannelClosedHandler closeHandler;
+ protected Future<Void> task;
+ protected IServlet servlet;
public HttpServerHandler(T server, int chunkSize) {
this.server = server;
@@ -64,10 +67,24 @@
}
@Override
+ public void channelInactive(ChannelHandlerContext ctx) throws Exception {
+ if (handler != null) {
+ handler.notifyChannelInactive();
+ }
+ if (closeHandler != null) {
+ closeHandler.channelClosed(server, servlet, task);
+ }
+ super.channelInactive(ctx);
+ }
+
+ @Override
protected void channelRead0(ChannelHandlerContext ctx, Object msg) {
FullHttpRequest request = (FullHttpRequest) msg;
+ handler = null;
+ task = null;
+ closeHandler = null;
try {
- IServlet servlet = server.getServlet(request);
+ servlet = server.getServlet(request);
if (servlet == null) {
handleServletNotFound(ctx, request);
} else {
@@ -94,16 +111,13 @@
return;
}
handler = new HttpRequestHandler(ctx, servlet, servletRequest, chunkSize);
- submit(ctx, servlet);
+ submit(servlet);
}
- private void submit(ChannelHandlerContext ctx, IServlet servlet) throws IOException {
+ private void submit(IServlet servlet) throws IOException {
try {
- Future<Void> task = server.getExecutor(handler).submit(handler);
- final IChannelClosedHandler closeHandler = servlet.getChannelClosedHandler(server);
- if (closeHandler != null) {
- ctx.channel().closeFuture().addListener(future -> closeHandler.channelClosed(server, servlet, task));
- }
+ task = server.getExecutor(handler).submit(handler);
+ closeHandler = servlet.getChannelClosedHandler(server);
} catch (RejectedExecutionException e) { // NOSONAR
LOGGER.log(Level.WARN, "Request rejected by server executor service. " + e.getMessage());
handler.reject();
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
index 0245eb3..e8625fc 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/GenerateFileMojo.java
@@ -182,8 +182,8 @@
throw new IOException("Could not load template " + generation.getTemplate());
}
- outputDir.mkdirs();
final File file = new File(outputDir, generation.getOutputFile());
+ file.getParentFile().mkdirs();
getLog().info("Writing " + file + "...");
try (final FileOutputStream fos = new FileOutputStream(file);
final Writer writer = new OutputStreamWriter(fos, StandardCharsets.UTF_8)) {
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseUtil.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseUtil.java
index a80dc1d..5ea768e 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/LicenseUtil.java
@@ -56,11 +56,11 @@
}
}
- public static String process(String input, boolean unpad, boolean wrap) throws IOException {
+ public static String process(String input, boolean unpad, boolean wrap, boolean strict) throws IOException {
try (BufferedReader reader = new BufferedReader(new StringReader(input))) {
reader.mark(input.length() + 1);
StringWriter sw = new StringWriter();
- trim(sw, reader, unpad, wrap);
+ trim(sw, reader, unpad, wrap, strict);
sw.append('\n');
return sw.toString();
}
@@ -75,20 +75,22 @@
}
private static void trim(Writer out, BufferedReader reader) throws IOException {
- trim(out, reader, true, true);
+ trim(out, reader, true, true, false);
}
- private static void trim(Writer out, BufferedReader reader, boolean unpad, boolean wrap) throws IOException {
+ private static void trim(Writer out, BufferedReader reader, boolean unpad, boolean wrap, boolean strict)
+ throws IOException {
Pair<Integer, Integer> result = null;
if (unpad || wrap) {
result = analyze(reader);
reader.reset();
}
doTrim(out, reader, unpad ? result.getLeft() : 0,
- wrap && (result.getRight() > wrapThreshold) ? wrapLength : Integer.MAX_VALUE);
+ wrap && (result.getRight() > wrapThreshold) ? wrapLength : Integer.MAX_VALUE, strict);
}
- private static void doTrim(Writer out, BufferedReader reader, int extraPadding, int wrapLength) throws IOException {
+ private static void doTrim(Writer out, BufferedReader reader, int extraPadding, int wrapLength, boolean strict)
+ throws IOException {
boolean head = true;
int empty = 0;
for (String line = reader.readLine(); line != null; line = reader.readLine()) {
@@ -110,6 +112,8 @@
out.append(trimmed.substring(0, cut));
out.append('\n');
trimmed = trimmed.substring(cut + 1);
+ } else if (!strict) {
+ break;
} else {
out.append(trimmed.substring(0, wrapLength));
out.append('\n');
diff --git a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/freemarker/IndentDirective.java b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/freemarker/IndentDirective.java
index f58b419..9237dde 100644
--- a/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/freemarker/IndentDirective.java
+++ b/hyracks-fullstack/hyracks/hyracks-maven-plugins/license-automation-plugin/src/main/java/org/apache/hyracks/maven/license/freemarker/IndentDirective.java
@@ -25,6 +25,9 @@
import java.util.Arrays;
import java.util.Map;
+import org.apache.commons.io.IOUtils;
+import org.apache.hyracks.maven.license.LicenseUtil;
+
import freemarker.core.Environment;
import freemarker.template.TemplateBooleanModel;
import freemarker.template.TemplateDirectiveBody;
@@ -33,14 +36,13 @@
import freemarker.template.TemplateModel;
import freemarker.template.TemplateModelException;
import freemarker.template.TemplateNumberModel;
-import org.apache.commons.io.IOUtils;
-import org.apache.hyracks.maven.license.LicenseUtil;
public class IndentDirective implements TemplateDirectiveModel {
private static final String PARAM_NAME_SPACES = "spaces";
private static final String PARAM_NAME_UNPAD = "unpad";
private static final String PARAM_NAME_WRAP = "wrap";
+ private static final String PARAM_NAME_STRICT = "strict";
@Override
public void execute(Environment env, Map params, TemplateModel[] loopVars, TemplateDirectiveBody body)
@@ -49,6 +51,7 @@
int numSpaces = -1;
boolean unpad = false;
boolean wrap = false;
+ boolean strict = false;
for (Object o : params.entrySet()) {
Map.Entry ent = (Map.Entry) o;
@@ -66,6 +69,9 @@
case PARAM_NAME_WRAP:
wrap = getBooleanParam(paramName, paramValue);
break;
+ case PARAM_NAME_STRICT:
+ strict = getBooleanParam(paramName, paramValue);
+ break;
default:
throw new TemplateModelException("Unsupported parameter: " + paramName);
}
@@ -81,7 +87,7 @@
// case we don't provide a special writer as the parameter:
StringWriter sw = new StringWriter();
body.render(sw);
- String fixedup = LicenseUtil.process(sw.toString(), unpad, wrap);
+ String fixedup = LicenseUtil.process(sw.toString(), unpad, wrap, strict);
IOUtils.copy(new StringReader(fixedup), new IndentingWriter(env.getOut(), numSpaces));
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexOperationContext.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexOperationContext.java
index 17aa394..c993874 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexOperationContext.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/AbstractLSMIndexOperationContext.java
@@ -31,12 +31,12 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperation.LSMIOOperationType;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndex;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexOperationContext;
-import org.apache.hyracks.storage.common.IModificationOperationCallback;
import org.apache.hyracks.storage.common.ISearchOperationCallback;
import org.apache.hyracks.storage.common.ISearchPredicate;
import org.apache.hyracks.storage.common.MultiComparator;
import org.apache.hyracks.util.trace.ITracer;
import org.apache.hyracks.util.trace.ITracer.Scope;
+import org.apache.hyracks.util.trace.TraceUtils;
public abstract class AbstractLSMIndexOperationContext implements ILSMIndexOperationContext {
@@ -79,7 +79,7 @@
filterTuple = null;
}
this.tracer = tracer;
- this.traceCategory = tracer.getRegistry().get("op-ctx");
+ this.traceCategory = tracer.getRegistry().get(TraceUtils.INDEX_OPERATIONS);
}
@Override
@@ -209,6 +209,7 @@
this.recovery = recovery;
}
+ @Override
public LSMIOOperationType getIoOperationType() {
return ioOpType;
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java
index 572e05c..7238f8e 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/TracedIOOperation.java
@@ -27,6 +27,7 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIndexAccessor;
import org.apache.hyracks.util.trace.ITracer;
import org.apache.hyracks.util.trace.ITracer.Scope;
+import org.apache.hyracks.util.trace.TraceUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -48,14 +49,14 @@
public static ILSMIOOperation wrap(final ILSMIOOperation ioOp, final ITracer tracer) {
final String ioOpName = ioOp.getIOOpertionType().name().toLowerCase();
- final long traceCategorySchedule = tracer.getRegistry().get("schedule-" + ioOpName);
- if (tracer.isEnabled(traceCategorySchedule)) {
- tracer.instant(ioOp.getTarget().getRelativePath(), traceCategorySchedule, Scope.p, null);
+ final long traceCategory = tracer.getRegistry().get(TraceUtils.INDEX_IO_OPERATIONS);
+ if (tracer.isEnabled(traceCategory)) {
+ tracer.instant("schedule-" + ioOpName, traceCategory, Scope.p,
+ "{\"path\": \"" + ioOp.getTarget().getRelativePath() + "\"}");
}
- final long traceCategoryExec = tracer.getRegistry().get(ioOpName);
- if (tracer.isEnabled(traceCategoryExec)) {
- return ioOp instanceof Comparable ? new ComparableTracedIOOperation(ioOp, tracer, traceCategoryExec)
- : new TracedIOOperation(ioOp, tracer, traceCategoryExec);
+ if (tracer.isEnabled(traceCategory)) {
+ return ioOp instanceof Comparable ? new ComparableTracedIOOperation(ioOp, tracer, traceCategory)
+ : new TracedIOOperation(ioOp, tracer, traceCategory);
}
return ioOp;
}
@@ -91,7 +92,8 @@
try {
return ioOp.call();
} finally {
- tracer.durationE(name, traceCategory, tid, "{\"size\":" + getTarget().getFile().length() + "}");
+ tracer.durationE(ioOp.getIOOpertionType().name().toLowerCase(), traceCategory, tid, "{\"size\":"
+ + getTarget().getFile().length() + ", \"path\": \"" + ioOp.getTarget().getRelativePath() + "\"}");
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/LicensingTestBase.java b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/LicensingTestBase.java
index 2fb8446..e2ab3a3 100644
--- a/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/LicensingTestBase.java
+++ b/hyracks-fullstack/hyracks/hyracks-test-support/src/main/java/org/apache/hyracks/test/support/LicensingTestBase.java
@@ -55,7 +55,7 @@
protected void verifyMissingLicenses() throws IOException {
for (String licenseArtifactName : getLicenseArtifactNames()) {
final File licenseFile =
- new File(FileUtil.joinPath(installerDir, pathToLicensingFiles(), licenseArtifactName));
+ new File(FileUtil.joinPath(getInstallerDir(), pathToLicensingFiles(), licenseArtifactName));
List<String> badLines = new ArrayList<>();
for (String line : FileUtils.readLines(licenseFile, StandardCharsets.UTF_8)) {
if (line.matches("^\\s*MISSING:.*")) {
@@ -66,9 +66,13 @@
}
}
+ protected String getInstallerDir() {
+ return installerDir;
+ }
+
protected void verifyAllRequiredArtifactsPresent() {
for (String name : getRequiredArtifactNames()) {
- final String fileName = FileUtil.joinPath(installerDir, pathToLicensingFiles(), name);
+ final String fileName = FileUtil.joinPath(getInstallerDir(), pathToLicensingFiles(), name);
Assert.assertTrue(fileName + " missing", new File(fileName).exists());
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThreadDumpUtil.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThreadDumpUtil.java
index 221d4b0..2de6700 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThreadDumpUtil.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/ThreadDumpUtil.java
@@ -19,7 +19,9 @@
package org.apache.hyracks.util;
import java.io.IOException;
+import java.lang.management.LockInfo;
import java.lang.management.ManagementFactory;
+import java.lang.management.MonitorInfo;
import java.lang.management.ThreadInfo;
import java.lang.management.ThreadMXBean;
import java.util.ArrayList;
@@ -29,6 +31,8 @@
import java.util.Map;
import java.util.stream.Stream;
+import org.apache.commons.lang3.mutable.MutableInt;
+
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.SerializationFeature;
import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -100,8 +104,62 @@
}
public static String takeDumpString() {
- StringBuilder buf = new StringBuilder(2048);
- Stream.of(threadMXBean.dumpAllThreads(true, true)).forEach(buf::append);
- return buf.toString();
+ ThreadDumpHelper helper = new ThreadDumpHelper();
+ Stream.of(threadMXBean.dumpAllThreads(true, true)).forEach(helper::addThread);
+ return helper.dumpAsString();
+ }
+
+ static class ThreadDumpHelper {
+
+ private final StringBuilder buf = new StringBuilder(32 * 1024);
+
+ private ThreadDumpHelper() {
+ }
+
+ private void addThread(ThreadInfo ti) {
+ buf.append('\n');
+ quote(ti.getThreadName()).append(" [tid=").append(ti.getThreadId()).append(" state=")
+ .append(ti.getThreadState());
+
+ if (ti.getLockName() != null) {
+ buf.append(" lock=").append(ti.getLockName());
+ if (ti.getLockOwnerName() != null) {
+ buf.append(" lockOwner=");
+ quote(ti.getLockOwnerName()).append(" (tid=").append(ti.getLockOwnerId());
+ }
+ }
+ if (ti.isSuspended()) {
+ buf.append(" suspended=true");
+ }
+ buf.append("]\n");
+ MutableInt depth = new MutableInt();
+ for (StackTraceElement frame : ti.getStackTrace()) {
+ int thisDepth = depth.getAndIncrement();
+ buf.append("\tat ").append(frame).append('\n');
+ Stream.of(ti.getLockedMonitors()).filter(m -> m.getLockedStackDepth() == thisDepth)
+ .forEach(this::output);
+ }
+ LockInfo[] lockedSynchronizers = ti.getLockedSynchronizers();
+ if (lockedSynchronizers.length > 0) {
+ buf.append("\n\tLocked synchronizers:\n");
+ Stream.of(lockedSynchronizers).forEachOrdered(this::output);
+ }
+ }
+
+ private StringBuilder quote(Object quotable) {
+ return buf.append('"').append(quotable).append('"');
+ }
+
+ private StringBuilder output(MonitorInfo info) {
+ return buf.append("\t- <").append("locked ").append(info).append(">\n");
+ }
+
+ private StringBuilder output(LockInfo info) {
+ return buf.append("\t- ").append("").append(info).append("\n");
+ }
+
+ public String dumpAsString() {
+ return buf.toString();
+ }
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/ITraceCategoryRegistry.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/ITraceCategoryRegistry.java
index efec0dc..4f17c46 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/ITraceCategoryRegistry.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/ITraceCategoryRegistry.java
@@ -25,6 +25,7 @@
long CATEGORIES_ALL = -1L;
long CATEGORIES_NONE = 0L;
+ String CATEGORIES_ALL_NAME = "*";
ITraceCategoryRegistry NONE = new TraceCategoryRegistry() {
@Override
@@ -33,19 +34,25 @@
}
@Override
- public long get(String... names) {
- return CATEGORIES_NONE;
- }
-
- @Override
public String getName(long categoryCode) {
return "";
}
};
+ /**
+ * Register the tracing category if not registered and return its code
+ *
+ * @param name
+ * the category name
+ * @return the long code of the category
+ */
long get(String name);
- long get(String... names);
-
+ /**
+ * Get the name of the category with the code categoryCode
+ *
+ * @param categoryCode
+ * @return the String name of the category
+ */
String getName(long categoryCode);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/TraceCategoryRegistry.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/TraceCategoryRegistry.java
index db11285..baf6082 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/TraceCategoryRegistry.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/TraceCategoryRegistry.java
@@ -22,15 +22,15 @@
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
-import java.util.Optional;
public class TraceCategoryRegistry implements ITraceCategoryRegistry {
- private Map<String, Long> categories = Collections.synchronizedMap(new HashMap<>());
+ private final Map<String, Long> categories = Collections.synchronizedMap(new HashMap<>());
+ private final String[] names = new String[NO_CATEGORIES];
private int bitPos = 0;
public TraceCategoryRegistry() {
- categories.put("*", ITraceCategoryRegistry.CATEGORIES_ALL);
+ categories.put(CATEGORIES_ALL_NAME, ITraceCategoryRegistry.CATEGORIES_ALL);
}
@Override
@@ -38,33 +38,40 @@
return categories.computeIfAbsent(name, this::nextCode);
}
- private long nextCode(String name) {
+ private synchronized long nextCode(String name) {
if (bitPos > NO_CATEGORIES - 1) {
throw new IllegalStateException("Cannot add category " + name);
}
+ names[bitPos] = name;
return 1L << bitPos++;
}
@Override
- public long get(String... names) {
- long result = 0;
- for (String name : names) {
- result |= get(name);
- }
- return result;
- }
-
- private Optional<Map.Entry<String, Long>> findEntry(long categoryCode) {
- return categories.entrySet().stream().filter(e -> e.getValue() == categoryCode).findFirst();
- }
-
- @Override
public String getName(long categoryCode) {
- Optional<Map.Entry<String, Long>> entry = findEntry(categoryCode);
- if (!entry.isPresent()) {
+ if (CATEGORIES_ALL == categoryCode) {
+ return CATEGORIES_ALL_NAME;
+ }
+ if (categoryCode == 0) {
+ throw new IllegalArgumentException("Illegal category code " + categoryCode);
+ }
+ int postition = mostSignificantBit(categoryCode);
+ if (postition >= bitPos) {
throw new IllegalArgumentException("No category for code " + categoryCode);
}
- return entry.get().getKey();
+ return nameAt(postition);
+ }
+
+ public String nameAt(int n) {
+ return names[n];
+ }
+
+ public static int mostSignificantBit(long n) {
+ int pos = -1;
+ while (n != 0) {
+ pos++;
+ n = n >>> 1;
+ }
+ return pos;
}
@Override
@@ -72,11 +79,10 @@
StringBuilder sb = new StringBuilder();
for (int pos = 0; pos < NO_CATEGORIES; ++pos) {
long categoryCode = 1L << pos;
- Optional<Map.Entry<String, Long>> entry = findEntry(categoryCode);
- if (!entry.isPresent()) {
+ String name = nameAt(pos);
+ if (name == null) {
continue;
}
- String name = entry.get().getKey();
String codeString = Long.toBinaryString(categoryCode);
sb.append(name).append(" -> ").append(codeString).append(' ');
}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/TraceUtils.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/TraceUtils.java
new file mode 100644
index 0000000..60ec419
--- /dev/null
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/TraceUtils.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.hyracks.util.trace;
+
+public class TraceUtils {
+
+ private TraceUtils() {
+ }
+
+ // CATEGORIES
+ public static final String TRACER = "Tracer";
+ public static final String TIMESTAMP = "Timestamp";
+ public static final String INDEX_OPERATIONS = "IndexOperations";
+ public static final String INDEX_IO_OPERATIONS = "IndexIoOperations";
+ public static final String INGESTION = "Ingestion";
+ public static final String STORAGE = "Storage";
+ public static final String LATENCY = "Latency";
+
+}
diff --git a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java
index ea3793d..1e8af75 100644
--- a/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java
+++ b/hyracks-fullstack/hyracks/hyracks-util/src/main/java/org/apache/hyracks/util/trace/Tracer.java
@@ -37,7 +37,6 @@
public static final Logger LOGGER = LogManager.getLogger();
protected static final Level TRACE_LOG_LEVEL = Level.INFO;
- protected static final String CAT = "Tracer";
protected static final ThreadLocal<DateFormat> DATE_FORMAT =
ThreadLocal.withInitial(() -> new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSSXXX"));
@@ -53,7 +52,7 @@
this.traceLog = LogManager.getLogger(traceLoggerName);
this.categories = categories;
this.registry = registry;
- final long traceCategory = getRegistry().get(CAT);
+ final long traceCategory = getRegistry().get(TraceUtils.TRACER);
instant("Trace-Start", traceCategory, Scope.p, dateTimeStamp());
}
@@ -62,9 +61,18 @@
setCategories(categories);
}
+ @Override
public void setCategories(String... categories) {
LOGGER.info("Set categories for Tracer " + this.traceLog.getName() + " to " + Arrays.toString(categories));
- this.categories = getRegistry().get(categories);
+ this.categories = set(categories);
+ }
+
+ private long set(String... names) {
+ long result = 0;
+ for (String name : names) {
+ result |= getRegistry().get(name);
+ }
+ return result;
}
public static String dateTimeStamp() {