QPID-4056: HAClusterManagementTest.testRemoveNodeFromGroup fails occasionally on Apache CI

Test should await all nodes knowning about one and other before commencing the remainder of the test.

git-svn-id: https://svn.apache.org/repos/asf/qpid/trunk@1384810 13f79535-47bb-0310-9956-ffa450edef68

1 files changed, 26 insertions, 6 deletions

diff --git a/qpid/java/bdbstore/jmx/src/test/java/org/apache/qpid/server/store/berkeleydb/HAClusterManagementTest.java b/qpid/java/bdbstore/jmx/src/test/java/org/apache/qpid/server/store/berkeleydb/HAClusterManagementTest.java
index 606983cdae..98e30bfaaf 100644
--- a/qpid/java/bdbstore/jmx/src/test/java/org/apache/qpid/server/store/berkeleydb/HAClusterManagementTest.java
+++ b/qpid/java/bdbstore/jmx/src/test/java/org/apache/qpid/server/store/berkeleydb/HAClusterManagementTest.java
@@ -37,7 +37,6 @@ import org.apache.log4j.Logger;
 import org.apache.qpid.jms.ConnectionURL;
 import org.apache.qpid.management.common.mbeans.ManagedBroker;
 import org.apache.qpid.server.store.berkeleydb.jmx.ManagedBDBHAMessageStore;
-import org.apache.qpid.server.virtualhost.ManagedVirtualHost;
 import org.apache.qpid.test.utils.JMXTestUtils;
 import org.apache.qpid.test.utils.QpidBrokerTestCase;
 
@@ -132,12 +131,11 @@ public class HAClusterManagementTest extends QpidBrokerTestCase
         final int brokerPortNumber = getBrokerPortNumbers().iterator().next();
 
         ManagedBDBHAMessageStore storeBean = getStoreBeanForNodeAtBrokerPort(brokerPortNumber);
+        awaitAllNodesJoiningGroup(storeBean, NUMBER_OF_NODES);
+
         final TabularData groupMembers = storeBean.getAllNodesInGroup();
         assertNotNull(groupMembers);
 
-        final int numberOfDataRows = groupMembers.size();
-        assertEquals("Unexpected number of data rows", NUMBER_OF_NODES ,numberOfDataRows);
-
         for(int bdbPortNumber : _clusterCreator.getBdbPortNumbers())
         {
             final String nodeName = _clusterCreator.getNodeNameForNodeAt(bdbPortNumber);
@@ -155,8 +153,7 @@ public class HAClusterManagementTest extends QpidBrokerTestCase
         final int brokerPortNumberToMakeObservation = brokerPortNumberIterator.next();
         final int brokerPortNumberToBeRemoved = brokerPortNumberIterator.next();
         final ManagedBDBHAMessageStore storeBean = getStoreBeanForNodeAtBrokerPort(brokerPortNumberToMakeObservation);
-        final int numberOfDataRows = storeBean.getAllNodesInGroup().size();
-        assertEquals("Unexpected number of data rows before test", NUMBER_OF_NODES ,numberOfDataRows);
+        awaitAllNodesJoiningGroup(storeBean, NUMBER_OF_NODES);
 
         final String removedNodeName = _clusterCreator.getNodeNameForNodeAt(_clusterCreator.getBdbPortForBrokerPort(brokerPortNumberToBeRemoved));
         _clusterCreator.stopNode(brokerPortNumberToBeRemoved);
@@ -266,4 +263,27 @@ public class HAClusterManagementTest extends QpidBrokerTestCase
 
         return _jmxUtils.getManagedBroker(VIRTUAL_HOST);
     }
+
+    private void awaitAllNodesJoiningGroup(ManagedBDBHAMessageStore storeBean, int expectedNumberOfNodes) throws Exception
+    {
+        long totalTimeWaited = 0l;
+        long waitInterval = 100l;
+        long maxWaitTime = 10000;
+
+        int currentNumberOfNodes = storeBean.getAllNodesInGroup().size();
+        while (expectedNumberOfNodes > currentNumberOfNodes || totalTimeWaited > maxWaitTime)
+        {
+            LOGGER.debug("Still awaiting nodes to join group; expecting "
+                + expectedNumberOfNodes + " node(s) but only have " + currentNumberOfNodes
+                + " after " + totalTimeWaited + " ms.");
+
+            totalTimeWaited += waitInterval;
+            Thread.sleep(waitInterval);
+
+            currentNumberOfNodes = storeBean.getAllNodesInGroup().size();
+        }
+
+        assertEquals("Unexpected number of nodes in group after " + totalTimeWaited + " ms",
+                expectedNumberOfNodes ,currentNumberOfNodes);
+    }
 }