Merge branch 'gerrit/trinity' into 'master'
Change-Id: If0be086e2af3499e965a67d520c781f441be7a68
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
index eaeb082..ed0a68a 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
@@ -21,7 +21,6 @@
import static org.apache.asterix.api.http.server.ServletConstants.ASTERIX_APP_CONTEXT_INFO_ATTR;
import java.io.IOException;
-import java.io.PrintWriter;
import java.util.concurrent.ConcurrentMap;
import java.util.function.Predicate;
@@ -29,6 +28,7 @@
import org.apache.hyracks.api.config.IOption;
import org.apache.hyracks.api.config.Section;
import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.exceptions.IFormattedException;
import org.apache.hyracks.control.common.config.ConfigUtils;
import org.apache.hyracks.control.common.controllers.ControllerConfig;
import org.apache.hyracks.http.api.IServletRequest;
@@ -36,7 +36,6 @@
import org.apache.hyracks.http.server.AbstractServlet;
import org.apache.hyracks.http.server.utils.HttpUtil;
import org.apache.hyracks.util.JSONUtil;
-import org.apache.logging.log4j.Level;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -67,7 +66,6 @@
@Override
protected void get(IServletRequest request, IServletResponse response) throws IOException {
HttpUtil.setContentType(response, HttpUtil.ContentType.APPLICATION_JSON, request);
- PrintWriter responseWriter = response.writer();
try {
ObjectNode json;
response.setStatus(HttpResponseStatus.OK);
@@ -81,15 +79,18 @@
default:
throw new IllegalArgumentException();
}
- JSONUtil.writeNode(responseWriter, json);
+ JSONUtil.writeNode(response.writer(), json);
} catch (IllegalArgumentException e) { // NOSONAR - exception not logged or rethrown
sendError(response, HttpResponseStatus.NOT_FOUND);
} catch (Exception e) {
- LOGGER.log(Level.INFO, "exception thrown for " + request, e);
- response.setStatus(HttpResponseStatus.INTERNAL_SERVER_ERROR);
- responseWriter.write(e.toString());
+ LOGGER.info("exception thrown for {}", request, e);
+ if (e instanceof IFormattedException) {
+ sendError(response, HttpResponseStatus.INTERNAL_SERVER_ERROR, (IFormattedException) e);
+ } else {
+ sendError(response, HttpResponseStatus.INTERNAL_SERVER_ERROR, e.toString());
+
+ }
}
- responseWriter.flush();
}
@Override
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/api.xml b/asterixdb/asterix-app/src/test/resources/runtimets/api.xml
index 056fa5d..c960475 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/api.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/api.xml
@@ -41,6 +41,12 @@
</compilation-unit>
</test-case>
<test-case FilePath="api">
+ <compilation-unit name="cluster_state_5">
+ <output-dir compare="Text">cluster_state_5</output-dir>
+ <expected-error>HTTP/1.1 404 Not Found</expected-error>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="api">
<compilation-unit name="cluster_state_cc_1">
<output-dir compare="Text">cluster_state_cc_1</output-dir>
</compilation-unit>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/cluster_state_5/cluster_state_5.1.get.http b/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/cluster_state_5/cluster_state_5.1.get.http
new file mode 100644
index 0000000..d6d5636
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries/api/cluster_state_5/cluster_state_5.1.get.http
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/*
+ * Test case Name : cluster_state_2
+ * Description : test cluster state api malformed uri (404)
+ * Expected Result : Negative
+ * Date : 7th September 2016
+ */
+/admin/cluster/
diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java
index 2012fe8..fb88c65 100644
--- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java
+++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/context/GlobalVirtualBufferCache.java
@@ -273,6 +273,11 @@
return page;
}
+ @Override
+ public ICachedPage pin(long dpid, boolean newPage, boolean incrementStats) throws HyracksDataException {
+ return pin(dpid, newPage);
+ }
+
private void incrementFilteredMemoryComponentUsage(long dpid, int pages) {
if (filteredMemoryComponentMaxNumPages > 0) {
// update memory usage of filtered index
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
index e77acea..959bb50 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/MultitenantVirtualBufferCache.java
@@ -70,6 +70,11 @@
}
@Override
+ public ICachedPage pin(long dpid, boolean newPage, boolean incrementStats) throws HyracksDataException {
+ return vbc.pin(dpid, newPage);
+ }
+
+ @Override
public void unpin(ICachedPage page) throws HyracksDataException {
vbc.unpin(page);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
index 5871b31..e222461 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/VirtualBufferCache.java
@@ -260,6 +260,11 @@
return page;
}
+ @Override
+ public ICachedPage pin(long dpid, boolean newPage, boolean incrementStats) throws HyracksDataException {
+ return pin(dpid, newPage);
+ }
+
private int hash(long dpid) {
int hashValue = (int) dpid ^ (Integer.reverse((int) (dpid >>> 32)) >>> 1);
return hashValue % buckets.length;
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
index 10284f2..fed916d 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/BufferCache.java
@@ -165,13 +165,18 @@
@Override
public ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException {
+ return pin(dpid, newPage, true);
+ }
+
+ @Override
+ public ICachedPage pin(long dpid, boolean newPage, boolean incrementStats) throws HyracksDataException {
// Calling the pinSanityCheck should be used only for debugging, since
// the synchronized block over the fileInfoMap is a hot spot.
if (DEBUG) {
pinSanityCheck(dpid);
}
final IThreadStats threadStats = statsSubscribers.get(Thread.currentThread());
- if (threadStats != null) {
+ if (threadStats != null && incrementStats) {
threadStats.pagePinned();
}
CachedPage cPage = findPage(dpid);
@@ -193,7 +198,7 @@
synchronized (cPage) {
if (!cPage.valid) {
try {
- tryRead(cPage);
+ tryRead(cPage, incrementStats);
cPage.valid = true;
} catch (Exception e) {
LOGGER.log(ExceptionUtils.causedByInterrupt(e) ? Level.DEBUG : Level.WARN,
@@ -520,10 +525,10 @@
return false;
}
- private void tryRead(CachedPage cPage) throws HyracksDataException {
+ private void tryRead(CachedPage cPage, boolean incrementStats) throws HyracksDataException {
for (int i = 1; i <= MAX_PAGE_READ_ATTEMPTS; i++) {
try {
- read(cPage);
+ read(cPage, incrementStats);
return;
} catch (HyracksDataException readException) {
if (readException.matches(ErrorCode.CANNOT_READ_CLOSED_FILE) && i != MAX_PAGE_READ_ATTEMPTS) {
@@ -547,12 +552,12 @@
}
}
- private void read(CachedPage cPage) throws HyracksDataException {
+ private void read(CachedPage cPage, boolean incrementStats) throws HyracksDataException {
BufferedFileHandle fInfo = getFileHandle(cPage);
cPage.buffer.clear();
fInfo.read(cPage);
final IThreadStats threadStats = statsSubscribers.get(Thread.currentThread());
- if (threadStats != null) {
+ if (threadStats != null && incrementStats) {
threadStats.coldRead();
}
}
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
index 8c3d492..c76c781 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/DebugBufferCache.java
@@ -77,13 +77,18 @@
}
@Override
- public ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException {
- ICachedPage page = bufferCache.pin(dpid, newPage);
+ public ICachedPage pin(long dpid, boolean newPage, boolean incrementStats) throws HyracksDataException {
+ ICachedPage page = bufferCache.pin(dpid, newPage, incrementStats);
pinCount.addAndGet(1);
return page;
}
@Override
+ public ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException {
+ return pin(dpid, newPage, true);
+ }
+
+ @Override
public void unpin(ICachedPage page) throws HyracksDataException {
bufferCache.unpin(page);
unpinCount.addAndGet(1);
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
index df0bea8..3e977bd 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/buffercache/IBufferCache.java
@@ -105,6 +105,28 @@
ICachedPage pin(long dpid, boolean newPage) throws HyracksDataException;
/**
+ * Pin the page so it can't be evicted from the buffer cache...
+ *
+ * @param dpid
+ * page id is a unique id that is a combination of file id and page id
+ * @param newPage
+ * whether this page is expected to be new.
+ * NOTE: undefined:
+ * -- what if the flag is true but the page exists?
+ * -- what if the flag is false but the page doesn't exist
+ *
+ * @param incrementStats
+ * whether to increment the coldRead and pinCount counters when
+ * the page is pinned. this is to not bias the count when using
+ * accessors that cause nested pins due to wrapping file handles,
+ * like compression
+ *
+ * @return the pinned page
+ * @throws HyracksDataException
+ */
+ ICachedPage pin(long dpid, boolean newPage, boolean incrementStats) throws HyracksDataException;
+
+ /**
* Unpin a pinned page so its buffer can be recycled
*
* @param page
diff --git a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/compression/file/CompressedFileManager.java b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/compression/file/CompressedFileManager.java
index d78ac24..8db171a 100644
--- a/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/compression/file/CompressedFileManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-storage-common/src/main/java/org/apache/hyracks/storage/common/compression/file/CompressedFileManager.java
@@ -314,7 +314,9 @@
final int numOfEntriesPerPage = bufferCache.getPageSize() / ENTRY_LENGTH;
//get the last page which may contain less entries than maxNumOfEntries
final long dpid = getDiskPageId(numOfPages - 1);
- final ICachedPage page = bufferCache.pin(dpid, false);
+ //exclude the LAF from the buffer cache pin/read stats, because it is hot in any
+ //sane scenario. otherwise it inflates the query's cache ratio greatly.
+ final ICachedPage page = bufferCache.pin(dpid, false, false);
try {
final ByteBuffer buf = page.getBuffer();
@@ -333,7 +335,10 @@
private ICachedPage pinAndGetPage(int compressedPageId) throws HyracksDataException {
final int pageId = compressedPageId * ENTRY_LENGTH / bufferCache.getPageSize();
- return bufferCache.pin(getDiskPageId(pageId), false);
+ //don't increment the stats with this pin. this is because we are pinning on behalf
+ //of a caller, so our pins will be an interfering access pattern that conflates itself
+ //with the caller's cache pattern.
+ return bufferCache.pin(getDiskPageId(pageId), false, false);
}
private long getDiskPageId(int pageId) {
diff --git a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestVirtualBufferCache.java b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestVirtualBufferCache.java
index 8cad497..e697b54 100644
--- a/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestVirtualBufferCache.java
+++ b/hyracks-fullstack/hyracks/hyracks-tests/hyracks-storage-am-lsm-btree-test/src/test/java/org/apache/hyracks/storage/am/lsm/btree/impl/TestVirtualBufferCache.java
@@ -101,6 +101,11 @@
}
@Override
+ public ICachedPage pin(long dpid, boolean newPage, boolean incrementStats) throws HyracksDataException {
+ return pin(dpid, newPage);
+ }
+
+ @Override
public void unpin(ICachedPage page) throws HyracksDataException {
vbc.unpin(page);
}