fix(testing): cleanup ZK cluster between test cases

Now performing a ZK cluster nuking at the end of each test case since keeping the same ZK state from test cases to test cases actually led to some tests randomly failing.
Added `client_cluster_health` ZK client in the `setup_zookeeper()` step that is there to "guarantee" the cluster is up and running, without even considering the configuration that will be used by the test itself.

1 files changed, 30 insertions, 2 deletions

diff --git a/kazoo/testing/harness.py b/kazoo/testing/harness.py
index 2d28a5b..880b1d9 100644
--- a/kazoo/testing/harness.py
+++ b/kazoo/testing/harness.py
@@ -21,6 +21,7 @@ CLUSTER_DEFAULTS = {
     "ZOOKEEPER_OBSERVER_START_ID": -1,
     "ZOOKEEPER_LOCAL_SESSION_RO": "false",
 }
+MAX_INIT_TRIES = 5
 
 
 def get_global_cluster():
@@ -208,8 +209,29 @@ class KazooTestHarness(unittest.TestCase):
             self.cluster.start()
         namespace = "/kazootests" + uuid.uuid4().hex
         self.hosts = self.servers + namespace
-        if "timeout" not in client_options:
-            client_options["timeout"] = self.DEFAULT_CLIENT_TIMEOUT
+
+        tries = 0
+        while True:
+            try:
+                client_cluster_health = self._get_client()
+                client_cluster_health.start()
+                client_cluster_health.ensure_path("/")
+                client_cluster_health.stop()
+                self.log(logging.INFO, "cluster looks ready to go")
+                break
+            except Exception:
+                tries += 1
+                if tries >= MAX_INIT_TRIES:
+                    raise
+                if tries > 0 and tries % 2 == 0:
+                    self.log(
+                        logging.WARNING,
+                        "nuking current cluster and making another one",
+                    )
+                    self.cluster.terminate()
+                    self.cluster.start()
+                continue
+
         self.client = self._get_client(**client_options)
         self.client.start()
         self.client.ensure_path("/")
@@ -259,3 +281,9 @@ class KazooTestCase(KazooTestHarness):
 
     def tearDown(self):
         self.teardown_zookeeper()
+
+    @classmethod
+    def tearDownClass(cls):
+        cluster = get_global_cluster()
+        if cluster is not None:
+            cluster.terminate()