QPID-6063: [Java Broker] Disambiguate the concept of a node being unreachable from a node awaiting election

git-svn-id: https://svn.apache.org/repos/asf/qpid/trunk@1621775 13f79535-47bb-0310-9956-ffa450edef68

6 files changed, 102 insertions, 45 deletions

diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHARemoteReplicationNode.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHARemoteReplicationNode.java
index 10b2b13bc9..bb9f564d64 100644
--- a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHARemoteReplicationNode.java
+++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHARemoteReplicationNode.java
@@ -42,7 +42,7 @@ public interface BDBHARemoteReplicationNode<X extends BDBHARemoteReplicationNode
     String getAddress();
 
     @ManagedAttribute(persist = false)
-    String getRole();
+    NodeRole getRole();
 
     @DerivedAttribute
     long getJoinTime();
diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHARemoteReplicationNodeImpl.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHARemoteReplicationNodeImpl.java
index 06671998ec..3bef144d2d 100644
--- a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHARemoteReplicationNodeImpl.java
+++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHARemoteReplicationNodeImpl.java
@@ -21,16 +21,12 @@
 
 package org.apache.qpid.server.virtualhostnode.berkeleydb;
 
-import static com.sleepycat.je.rep.ReplicatedEnvironment.State.MASTER;
-import static com.sleepycat.je.rep.ReplicatedEnvironment.State.REPLICA;
-
 import java.security.AccessControlException;
 import java.util.Map;
 import java.util.Set;
 import java.util.concurrent.atomic.AtomicReference;
 
 import com.sleepycat.je.rep.MasterStateException;
-import com.sleepycat.je.rep.ReplicatedEnvironment;
 
 import org.apache.log4j.Logger;
 import org.apache.qpid.server.configuration.IllegalConfigurationException;
@@ -60,16 +56,16 @@ public class BDBHARemoteReplicationNodeImpl extends AbstractConfiguredObject<BDB
 
     private volatile long _joinTime;
     private volatile long _lastTransactionId;
-    private volatile String _lastReplicatedEnvironmentState = ReplicatedEnvironment.State.UNKNOWN.name();
 
     @ManagedAttributeField(afterSet="afterSetRole")
-    private volatile String _role = ReplicatedEnvironment.State.UNKNOWN.name();
+    private volatile NodeRole _role;
 
     private final AtomicReference<State> _state;
     private final boolean _isMonitor;
     private boolean _detached;
     private BDBHAVirtualHostNodeLogSubject _virtualHostNodeLogSubject;
     private GroupLogSubject _groupLogSubject;
+    private volatile NodeRole _lastKnownRole;
 
     public BDBHARemoteReplicationNodeImpl(BDBHAVirtualHostNode<?> virtualHostNode, Map<String, Object> attributes, ReplicatedEnvironmentFacade replicatedEnvironmentFacade)
     {
@@ -77,7 +73,11 @@ public class BDBHARemoteReplicationNodeImpl extends AbstractConfiguredObject<BDB
         _broker = virtualHostNode.getParent(Broker.class);
         _address = (String)attributes.get(ADDRESS);
         _replicatedEnvironmentFacade = replicatedEnvironmentFacade;
-        _state = new AtomicReference<State>(State.ACTIVE);
+        _state = new AtomicReference<>(State.ACTIVE);
+
+        _role = NodeRole.UNREACHABLE;
+        _lastKnownRole = NodeRole.UNREACHABLE;
+
         _isMonitor = (Boolean)attributes.get(MONITOR);
     }
 
@@ -100,9 +100,9 @@ public class BDBHARemoteReplicationNodeImpl extends AbstractConfiguredObject<BDB
     }
 
     @Override
-    public String getRole()
+    public NodeRole getRole()
     {
-        return _lastReplicatedEnvironmentState;
+        return _lastKnownRole;
     }
 
     @Override
@@ -201,15 +201,16 @@ public class BDBHARemoteReplicationNodeImpl extends AbstractConfiguredObject<BDB
         super.validateChange(proxyForValidation, changedAttributes);
         if (changedAttributes.contains(ROLE))
         {
-            String currentRole = getRole();
-            if (!REPLICA.name().equals(currentRole))
+            NodeRole currentRole = getRole();
+            if (NodeRole.REPLICA != currentRole)
             {
                 throw new IllegalArgumentException("Cannot transfer mastership when not in replica role."
                                                  + " Current role " + currentRole);
             }
-            if (!MASTER.name().equals(((BDBHARemoteReplicationNode<?>)proxyForValidation).getRole()))
+            NodeRole newRole = (NodeRole) ((BDBHARemoteReplicationNode<?>) proxyForValidation).getAttribute(ROLE);
+            if (NodeRole.MASTER != newRole)
             {
-                throw new IllegalArgumentException("Changing role to other value then " + MASTER.name() + " is unsupported");
+                throw new IllegalArgumentException("Changing role to other value then " + NodeRole.MASTER + " is unsupported");
             }
         }
 
@@ -224,9 +225,9 @@ public class BDBHARemoteReplicationNodeImpl extends AbstractConfiguredObject<BDB
         }
     }
 
-    void setRole(String role)
+    void setRole(NodeRole role)
     {
-        _lastReplicatedEnvironmentState = role;
+        _lastKnownRole = role;
         _role = role;
         updateModelStateFromRole(role);
     }
@@ -241,7 +242,7 @@ public class BDBHARemoteReplicationNodeImpl extends AbstractConfiguredObject<BDB
         _lastTransactionId = lastTransactionId;
     }
 
-    private void updateModelStateFromRole(final String role)
+    private void updateModelStateFromRole(NodeRole role)
     {
         State currentState = _state.get();
         if (currentState == State.DELETED)
@@ -249,7 +250,7 @@ public class BDBHARemoteReplicationNodeImpl extends AbstractConfiguredObject<BDB
             return;
         }
 
-        boolean isActive = MASTER.name().equals(role) || REPLICA.name().equals(role);
+        boolean isActive = NodeRole.MASTER == role || NodeRole.REPLICA == role;
         _state.compareAndSet(currentState, isActive ? State.ACTIVE : State.UNAVAILABLE);
     }
 
@@ -260,7 +261,7 @@ public class BDBHARemoteReplicationNodeImpl extends AbstractConfiguredObject<BDB
 
     public void setDetached(boolean detached)
     {
-        this._detached = detached;
+        _detached = detached;
     }
 
     @Override
diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNode.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNode.java
index 252b00d44f..a23f4056fd 100644
--- a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNode.java
+++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNode.java
@@ -60,8 +60,8 @@ public interface BDBHAVirtualHostNode<X extends BDBHAVirtualHostNode<X>> extends
     @ManagedAttribute(defaultValue = "0")
     int getQuorumOverride();
 
-    @ManagedAttribute(persist = false, defaultValue = "UNKNOWN")
-    String getRole();
+    @ManagedAttribute(persist = false, defaultValue = "WAITING")
+    NodeRole getRole();
 
     @DerivedAttribute
     Long getLastKnownReplicationTransactionId();
diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeImpl.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeImpl.java
index fcb651181c..5b81e6eccf 100644
--- a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeImpl.java
+++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/BDBHAVirtualHostNodeImpl.java
@@ -95,7 +95,7 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
 
     private final AtomicReference<ReplicatedEnvironmentFacade> _environmentFacade = new AtomicReference<>();
 
-    private final AtomicReference<ReplicatedEnvironment.State> _lastReplicatedEnvironmentState = new AtomicReference<>(ReplicatedEnvironment.State.UNKNOWN);
+    private final AtomicReference<NodeRole> _lastReplicatedEnvironmentState = new AtomicReference<>(NodeRole.WAITING);
     private BDBHAVirtualHostNodeLogSubject _virtualHostNodeLogSubject;
     private GroupLogSubject _groupLogSubject;
     private String _virtualHostNodePrincipalName;
@@ -123,7 +123,7 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
     private int _quorumOverride;
 
     @ManagedAttributeField(afterSet="postSetRole")
-    private String _role;
+    private NodeRole _role;
 
     @ManagedAttributeField
     private String _helperNodeName;
@@ -145,15 +145,14 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
 
         if (changedAttributes.contains(ROLE))
         {
-            String currentRole = getRole();
-            if (!ReplicatedEnvironment.State.REPLICA.name().equals(currentRole))
+            NodeRole currentRole = getRole();
+            if (NodeRole.REPLICA != currentRole)
             {
-                throw new IllegalStateException("Cannot transfer mastership when not a replica, current role is " + currentRole);
+                throw new IllegalStateException("Cannot transfer mastership when not a " + NodeRole.REPLICA + ", current role is " + currentRole);
             }
-
-            if (!ReplicatedEnvironment.State.MASTER.name().equals(proposed.getRole()))
+            if (NodeRole.MASTER != proposed.getAttribute(ROLE))
             {
-                throw new IllegalArgumentException("Changing role to other value then " + ReplicatedEnvironment.State.MASTER.name() + " is unsupported");
+                throw new IllegalArgumentException("Changing role to other value then " + NodeRole.MASTER + " is unsupported");
             }
         }
 
@@ -206,9 +205,9 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
     }
 
     @Override
-    public String getRole()
+    public NodeRole getRole()
     {
-        return _lastReplicatedEnvironmentState.get().name();
+        return _lastReplicatedEnvironmentState.get();
     }
 
     @Override
@@ -350,9 +349,8 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
 
             // closing the environment does not cause a state change.  Adjust the role
             // so that our observers will see DETACHED rather than our previous role in the group.
-            ReplicatedEnvironment.State detached = ReplicatedEnvironment.State.DETACHED;
-            _lastReplicatedEnvironmentState.set(detached);
-            attributeSet(ROLE, _role, detached);
+            _lastReplicatedEnvironmentState.set(NodeRole.DETACHED);
+            attributeSet(ROLE, _role, NodeRole.DETACHED);
         }
     }
 
@@ -576,7 +574,7 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
             {
                 LOGGER.info("Received BDB event indicating transition to state " + state);
             }
-            String previousRole = getRole();
+            NodeRole previousRole = getRole();
             try
             {
                 switch (state)
@@ -598,10 +596,11 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
             }
             finally
             {
-                _lastReplicatedEnvironmentState.set(state);
+                NodeRole newRole = NodeRole.fromJeState(state);
+                _lastReplicatedEnvironmentState.set(newRole);
                 attributeSet(ROLE, _role, state.name());
                 getEventLogger().message(getGroupLogSubject(),
-                        HighAvailabilityMessages.ROLE_CHANGED(getName(), getAddress(), previousRole, state.name()));
+                        HighAvailabilityMessages.ROLE_CHANGED(getName(), getAddress(), previousRole.name(), newRole.name()));
             }
         }
     }
@@ -868,10 +867,10 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
             BDBHARemoteReplicationNodeImpl remoteNode = getChildByName(BDBHARemoteReplicationNodeImpl.class, node.getName());
             if (remoteNode != null)
             {
-                String currentRole = remoteNode.getRole();
+                NodeRole currentRole = remoteNode.getRole();
                 if (nodeState == null)
                 {
-                    remoteNode.setRole(ReplicatedEnvironment.State.UNKNOWN.name());
+                    remoteNode.setRole(NodeRole.UNREACHABLE);
                     remoteNode.setLastTransactionId(-1);
                     if (!remoteNode.isDetached())
                     {
@@ -883,7 +882,8 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
                 {
                     remoteNode.setJoinTime(nodeState.getJoinTime());
                     remoteNode.setLastTransactionId(nodeState.getCurrentTxnEndVLSN());
-                    remoteNode.setRole(nodeState.getNodeState().name());
+                    ReplicatedEnvironment.State state = nodeState.getNodeState();
+                    remoteNode.setRole(NodeRole.fromJeState(state));
                     if (remoteNode.isDetached())
                     {
                         getEventLogger().message(getGroupLogSubject(), HighAvailabilityMessages.JOINED(remoteNode.getName(), remoteNode.getAddress()));
@@ -908,10 +908,10 @@ public class BDBHAVirtualHostNodeImpl extends AbstractVirtualHostNode<BDBHAVirtu
                     }
                 }
 
-                String newRole = remoteNode.getRole();
-                if (!newRole.equals(currentRole))
+                NodeRole newRole = remoteNode.getRole();
+                if (newRole != currentRole)
                 {
-                    getEventLogger().message(getGroupLogSubject(), HighAvailabilityMessages.ROLE_CHANGED(remoteNode.getName(), remoteNode.getAddress(), currentRole, newRole));
+                    getEventLogger().message(getGroupLogSubject(), HighAvailabilityMessages.ROLE_CHANGED(remoteNode.getName(), remoteNode.getAddress(), currentRole.name(), newRole.name()));
                 }
             }
         }
diff --git a/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/NodeRole.java b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/NodeRole.java
new file mode 100644
index 0000000000..2ea0d666be
--- /dev/null
+++ b/qpid/java/bdbstore/src/main/java/org/apache/qpid/server/virtualhostnode/berkeleydb/NodeRole.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.qpid.server.virtualhostnode.berkeleydb;
+
+import com.sleepycat.je.rep.ReplicatedEnvironment.State;
+
+public enum NodeRole
+{
+    /** Node is master. */
+    MASTER,
+    /** Node is replica. */
+    REPLICA,
+    /** Node is awaiting an election result, or may be awaiting more nodes to join in order that an election may be held.  */
+    WAITING,
+    /**
+     * (Remote) node is unreachable.  Its virtual host node may be stopped, its Broker down,  or a network problem may
+     * be preventing it from being contacted.
+     */
+    UNREACHABLE,
+    /** (Local) node is not connected to the group */
+    DETACHED;
+
+    public static NodeRole fromJeState(final State state)
+    {
+        switch(state)
+        {
+            case DETACHED:
+                return DETACHED;
+            case UNKNOWN:
+                return WAITING;
+            case MASTER:
+                return MASTER;
+            case REPLICA:
+                return REPLICA;
+            default:
+                throw new IllegalArgumentException("Unrecognised JE node state " + state);
+        }
+    }
+}
diff --git a/qpid/java/bdbstore/src/main/java/resources/js/qpid/management/virtualhostnode/bdb_ha/show.js b/qpid/java/bdbstore/src/main/java/resources/js/qpid/management/virtualhostnode/bdb_ha/show.js
index 4997b16bc6..e27b640933 100644
--- a/qpid/java/bdbstore/src/main/java/resources/js/qpid/management/virtualhostnode/bdb_ha/show.js
+++ b/qpid/java/bdbstore/src/main/java/resources/js/qpid/management/virtualhostnode/bdb_ha/show.js
@@ -96,8 +96,8 @@ define(["dojo/_base/xhr",
           findNode("groupMembers", containerNode),
           [
            { name: 'Name', field: 'name', width: '10%' },
-           { name: 'Role', field: 'role', width: '10%' },
-           { name: 'Address', field: 'address', width: '35%' },
+           { name: 'Role', field: 'role', width: '15%' },
+           { name: 'Address', field: 'address', width: '30%' },
            { name: 'Join Time', field: 'joinTime', width: '25%', formatter: function(value){ return value ? UserPreferences.formatDateTime(value) : "";} },
            { name: 'Replication Transaction ID', field: 'lastKnownReplicationTransactionId', width: '20%', formatter: function(value){ return value > 0 ? value : "N/A";} }
           ],