[NO ISSUE][CLUS] Synchronize restoration of IPC connection

- user model changes: no
- storage format changes: no
- interface changes: no

details:
- When an IPC connection is dead, then only a single sender
  will revive the connection.

Change-Id: I1ec58d7fbc96d7352882e6588ef2ce12dc66511e
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2397
Sonar-Qube: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenkins@fulliautomatix.ics.uci.edu>
Reviewed-by: Michael Blow <mblow@apache.org>
diff --git a/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/impl/ReconnectingIPCHandle.java b/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/impl/ReconnectingIPCHandle.java
index 9a2fc80..db0ed6b 100644
--- a/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/impl/ReconnectingIPCHandle.java
+++ b/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/impl/ReconnectingIPCHandle.java
@@ -72,12 +72,17 @@
         if (delegate.isConnected()) {
             return delegate;
         }
-        LOGGER.warn("ipcHandle {} disconnected; will attempt to reconnect {} times", delegate, reconnectAttempts);
-        listener.ipcHandleDisconnected(delegate);
-        delegate = ipc.getHandle(getRemoteAddress(), reconnectAttempts);
-        LOGGER.warn("ipcHandle {} restored", delegate);
-        listener.ipcHandleRestored(delegate);
-        return delegate;
+        synchronized (this) {
+            if (delegate.isConnected()) {
+                return delegate;
+            }
+            LOGGER.warn("ipcHandle {} disconnected; will attempt to reconnect {} times", delegate, reconnectAttempts);
+            listener.ipcHandleDisconnected(delegate);
+            delegate = ipc.getHandle(getRemoteAddress(), reconnectAttempts);
+            LOGGER.warn("ipcHandle {} restored", delegate);
+            listener.ipcHandleRestored(delegate);
+            return delegate;
+        }
     }
 
 }