[NO ISSUE][CLUS] Attempt To Get Node IP From Existing Nodes

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- When we fail to resolve a node's host to an IP on node
  failure/removal, attempt to find its IP from the list
  of existing nodes. This is done to prevent failing the
  node removal operation when the node's host address
  cannot be resolved anymore.

Change-Id: I8f12a3dbb84a4cb731f1379efd65ca50606a8b07
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3544
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Till Westmann <tillw@apache.org>
diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/cluster/NodeManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/cluster/NodeManager.java
index 4f76ced..0e374a6 100644
--- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/cluster/NodeManager.java
+++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/cluster/NodeManager.java
@@ -205,8 +205,19 @@
         nodeRegistry.forEach(nodeFunction::apply);
     }
 
-    private void removeNodeFromIpAddressMap(String nodeId, NodeControllerState ncState) throws HyracksException {
-        InetAddress ipAddress = getIpAddress(ncState);
+    private void removeNodeFromIpAddressMap(String nodeId, NodeControllerState ncState) {
+        InetAddress ipAddress;
+        try {
+            ipAddress = getIpAddress(ncState);
+        } catch (Exception e) {
+            LOGGER.warn("failed to get ip address of node {}; attempting to find it on existing nodes lists", nodeId,
+                    e);
+            ipAddress = findNodeIpById(nodeId);
+        }
+        if (ipAddress == null) {
+            LOGGER.warn("failed to get ip address of node {}", nodeId);
+            return;
+        }
         Set<String> nodes = ipAddressNodeNameMap.get(ipAddress);
         if (nodes != null) {
             nodes.remove(nodeId);
@@ -242,4 +253,13 @@
             }
         });
     }
+
+    private InetAddress findNodeIpById(String nodeId) {
+        for (Map.Entry<InetAddress, Set<String>> ipToNodesEntry : ipAddressNodeNameMap.entrySet()) {
+            if (ipToNodesEntry.getValue().contains(nodeId)) {
+                return ipToNodesEntry.getKey();
+            }
+        }
+        return null;
+    }
 }