Fix Thread Dump Hang on Newly Dead NC
+ improved exception reporting from cluster state HTTP apis
Change-Id: I6ed674ec9e6c50c7da62b82694d1807b3338b289
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1207
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <tillw@apache.org>
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java
index 64cde54..0d9ff16 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java
@@ -30,7 +30,6 @@
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import org.apache.asterix.app.result.ResultUtil;
import org.apache.asterix.common.config.AbstractAsterixProperties;
import org.apache.asterix.runtime.util.ClusterStateManager;
import org.json.JSONArray;
@@ -65,12 +64,11 @@
json = getClusterStateJSON(request, "");
response.setStatus(HttpServletResponse.SC_OK);
responseWriter.write(json.toString(4));
- } catch (IllegalArgumentException e) {
- ResultUtil.apiErrorHandler(responseWriter, e);
- response.setStatus(HttpServletResponse.SC_BAD_REQUEST);
+ } catch (IllegalArgumentException e) { // NOSONAR - exception not logged or rethrown
+ response.sendError(HttpServletResponse.SC_BAD_REQUEST);
} catch (Exception e) {
- ResultUtil.apiErrorHandler(responseWriter, e);
- response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ LOGGER.log(Level.INFO, "exception thrown for " + request, e);
+ response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.toString());
}
responseWriter.flush();
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterCCDetailsAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterCCDetailsAPIServlet.java
index 64e65a5..75fba74 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterCCDetailsAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterCCDetailsAPIServlet.java
@@ -22,17 +22,19 @@
import java.io.IOException;
import java.io.PrintWriter;
+import java.util.logging.Level;
+import java.util.logging.Logger;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import org.apache.asterix.app.result.ResultUtil;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.json.JSONObject;
public class ClusterCCDetailsAPIServlet extends ClusterAPIServlet {
private static final long serialVersionUID = 1L;
+ private static final Logger LOGGER = Logger.getLogger(ClusterCCDetailsAPIServlet.class.getName());
@Override
protected void getUnsafe(HttpServletRequest request, HttpServletResponse response) throws IOException {
@@ -51,11 +53,11 @@
response.setContentType("application/json");
response.setCharacterEncoding("utf-8");
responseWriter.write(json.toString(4));
- } catch (IllegalArgumentException e) { //NOSONAR - exception not logged or rethrown
+ } catch (IllegalArgumentException e) { // NOSONAR - exception not logged or rethrown
response.sendError(HttpServletResponse.SC_NOT_FOUND);
} catch (Exception e) {
- ResultUtil.apiErrorHandler(responseWriter, e);
- response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ LOGGER.log(Level.INFO, "exception thrown for " + request, e);
+ response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.toString());
}
responseWriter.flush();
}
diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterNodeDetailsAPIServlet.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterNodeDetailsAPIServlet.java
index 5a91fee..f5509c2 100644
--- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterNodeDetailsAPIServlet.java
+++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterNodeDetailsAPIServlet.java
@@ -25,12 +25,14 @@
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import java.util.logging.Level;
+import java.util.logging.Logger;
import javax.servlet.ServletContext;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
-import org.apache.asterix.app.result.ResultUtil;
+import org.apache.asterix.runtime.util.ClusterStateManager;
import org.apache.hyracks.api.client.IHyracksClientConnection;
import org.json.JSONArray;
import org.json.JSONException;
@@ -38,6 +40,7 @@
public class ClusterNodeDetailsAPIServlet extends ClusterAPIServlet {
private static final long serialVersionUID = 1L;
+ private static final Logger LOGGER = Logger.getLogger(ClusterNodeDetailsAPIServlet.class.getName());
@Override
protected void getUnsafe(HttpServletRequest request, HttpServletResponse response) throws IOException {
@@ -57,11 +60,13 @@
response.setContentType("application/json");
response.setCharacterEncoding("utf-8");
responseWriter.write(json.toString(4));
- } catch (IllegalArgumentException e) { //NOSONAR - exception not logged or rethrown
+ } catch (IllegalStateException e) { // NOSONAR - exception not logged or rethrown
+ response.sendError(HttpServletResponse.SC_SERVICE_UNAVAILABLE);
+ } catch (IllegalArgumentException e) { // NOSONAR - exception not logged or rethrown
response.sendError(HttpServletResponse.SC_NOT_FOUND);
} catch (Exception e) {
- ResultUtil.apiErrorHandler(responseWriter, e);
- response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ LOGGER.log(Level.INFO, "exception thrown for " + request, e);
+ response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, e.toString());
}
responseWriter.flush();
}
@@ -187,7 +192,10 @@
}
String dump = hcc.getThreadDump(node);
if (dump == null) {
- throw new IllegalArgumentException();
+ // check to see if this is a node that is simply down
+ throw ClusterStateManager.INSTANCE.getNodePartitions(node) != null
+ ? new IllegalStateException()
+ : new IllegalArgumentException();
}
return new JSONObject(dump);
}
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java
index b18d388..be53232 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/GetThreadDumpWork.java
@@ -52,7 +52,11 @@
callback.setValue(null);
} else {
ccs.addThreadDumpRun(run.getRequestId(), run);
- ncState.getNodeController().takeThreadDump(run.getRequestId());
+ try {
+ ncState.getNodeController().takeThreadDump(run.getRequestId());
+ } catch (Exception e) {
+ callback.setException(e);
+ }
}
}
}