chore(ci): testing with Python3.10 and ZK 3.5.10, 3.6.3, 3.7.1 (#659)

Testing libs have been upgraded.
The used java classpath now considers more .jar possible locations.
test_connection.py tests is now done first, trying to make it less flaky.
conftest.py has been added to give a way to display ZK cluster logs.
Because of ZK 3.6 and 3.7 changes, configurations related to local session has been added.
Disable GA fail-fast because of the flakiness

9 files changed, 77 insertions, 13 deletions

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index bb47781..02d0314 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -13,10 +13,10 @@ jobs:
     - name: Handle the code
       uses: actions/checkout@v2
 
-    - name: Set up Python 3.9
+    - name: Set up Python 3.10
       uses: actions/setup-python@v2
       with:
-        python-version: 3.9
+        python-version: "3.10"
 
     - name: Install pypa/build
       run: >-
diff --git a/.github/workflows/testing.yml b/.github/workflows/testing.yml
index 0edfe49..7d90900 100644
--- a/.github/workflows/testing.yml
+++ b/.github/workflows/testing.yml
@@ -19,9 +19,10 @@ jobs:
     runs-on: ubuntu-latest
 
     strategy:
+      fail-fast: false
       matrix:
-        python-version: [3.7, 3.8, 3.9, pypy-3.7]
-        zk-version: [3.4.14, 3.5.8]
+        python-version: [3.7, 3.8, 3.9, "3.10", pypy-3.7]
+        zk-version: [3.4.14, 3.5.10, 3.6.3, 3.7.1]
         include:
           - python-version: 3.7
             tox-env: py37
@@ -29,6 +30,8 @@ jobs:
             tox-env: py38
           - python-version: 3.9
             tox-env: py39
+          - python-version: "3.10"
+            tox-env: py310
           - python-version: pypy-3.7
             tox-env: pypy3
     steps:
diff --git a/constraints.txt b/constraints.txt
index f661f97..92d70e5 100644
--- a/constraints.txt
+++ b/constraints.txt
@@ -1,10 +1,10 @@
 # Consistent testing environment.
 coverage==6.3.2
-flake8==3.7.9
+flake8==3.9.2
 mock==3.0.5
-objgraph==3.4.1
-pytest-cov~=2.12
-pytest~=4.6
+objgraph==3.5.0
+pytest==6.2.5
+pytest-cov==3.0.0
 
 # Documentation building.
 Jinja2==2.7.3
diff --git a/kazoo/testing/common.py b/kazoo/testing/common.py
index 9cf0ff0..810b55f 100644
--- a/kazoo/testing/common.py
+++ b/kazoo/testing/common.py
@@ -27,6 +27,7 @@ from itertools import chain
 import logging
 import os
 import os.path
+import pathlib
 import shutil
 import signal
 import subprocess
@@ -207,6 +208,9 @@ log4j.appender.ROLLINGFILE.File=""" + to_java_compatible_path(  # NOQA
             jars.extend(glob(os.path.join(
                 self.install_path,
                 "lib/*.jar")))
+            jars.extend(glob(os.path.join(
+                self.install_path,
+                "*.jar")))
             # support for different file locations on Debian/Ubuntu
             jars.extend(glob(os.path.join(
                 self.install_path,
@@ -268,6 +272,16 @@ log4j.appender.ROLLINGFILE.File=""" + to_java_compatible_path(  # NOQA
 
         shutil.rmtree(self.working_path, True)
 
+    def get_logs(self):
+        log_path = pathlib.Path(
+            self.working_path,
+            'zookeeper.log'
+        )
+        if log_path.exists():
+            log_file = log_path.open('r')
+            lines = log_file.readlines()
+            return lines[-100:]
+        return []
 
 class ZookeeperCluster(object):
 
@@ -339,3 +353,9 @@ class ZookeeperCluster(object):
     def reset(self):
         for server in self:
             server.reset()
+
+    def get_logs(self):
+        logs = []
+        for server in self:
+            logs += server.get_logs()
+        return logs
diff --git a/kazoo/testing/harness.py b/kazoo/testing/harness.py
index 4cd419d..d7dbad6 100644
--- a/kazoo/testing/harness.py
+++ b/kazoo/testing/harness.py
@@ -21,6 +21,7 @@ CLUSTER_DEFAULTS = {
     "ZOOKEEPER_PORT_OFFSET": 20000,
     "ZOOKEEPER_CLUSTER_SIZE": 3,
     "ZOOKEEPER_OBSERVER_START_ID": -1,
+    "ZOOKEEPER_LOCAL_SESSION_RO": "false"
 }
 
 
@@ -34,7 +35,8 @@ def get_global_cluster():
                   "ZOOKEEPER_CLUSTER_SIZE",
                   "ZOOKEEPER_VERSION",
                   "ZOOKEEPER_OBSERVER_START_ID",
-                  "ZOOKEEPER_JAAS_AUTH"]
+                  "ZOOKEEPER_JAAS_AUTH",
+                  "ZOOKEEPER_LOCAL_SESSION_RO"]
     }
     if CLUSTER is not None:
         if CLUSTER_CONF == cluster_conf:
@@ -61,9 +63,16 @@ def get_global_cluster():
         "For deb package installations this is /usr/share/java")
 
     if ZK_VERSION >= (3, 5):
+        ZOOKEEPER_LOCAL_SESSION_RO = cluster_conf.get(
+            "ZOOKEEPER_LOCAL_SESSION_RO"
+        )
         additional_configuration_entries = [
             "4lw.commands.whitelist=*",
-            "reconfigEnabled=true"
+            "reconfigEnabled=true",
+            # required to avoid session validation error
+            # in read only test
+            "localSessionsEnabled=" + ZOOKEEPER_LOCAL_SESSION_RO,
+            "localSessionsUpgradingEnabled=" + ZOOKEEPER_LOCAL_SESSION_RO
         ]
         # If defined, this sets the superuser password to "test"
         additional_java_system_properties = [
diff --git a/kazoo/tests/conftest.py b/kazoo/tests/conftest.py
new file mode 100644
index 0000000..bd43772
--- /dev/null
+++ b/kazoo/tests/conftest.py
@@ -0,0 +1,10 @@
+import logging
+
+log = logging.getLogger(__name__)
+
+
+def pytest_exception_interact(node, call, report):
+    cluster = node._testcase.cluster
+    log.error('Zookeeper cluster logs:')
+    for logs in cluster.get_logs():
+        log.error(logs)
diff --git a/kazoo/tests/test_connection.py b/kazoo/tests/test__connection.py
index d62f7f7..c600d59 100644
--- a/kazoo/tests/test_connection.py
+++ b/kazoo/tests/test__connection.py
@@ -257,6 +257,7 @@ class TestConnectionDrop(KazooTestCase):
 
 class TestReadOnlyMode(KazooTestCase):
     def setUp(self):
+        os.environ["ZOOKEEPER_LOCAL_SESSION_RO"] = "true"
         self.setup_zookeeper(read_only=True)
         skip = False
         if CI_ZK_VERSION and CI_ZK_VERSION < (3, 4):
@@ -272,6 +273,7 @@ class TestReadOnlyMode(KazooTestCase):
 
     def tearDown(self):
         self.client.stop()
+        os.environ.pop('ZOOKEEPER_LOCAL_SESSION_RO', None)
 
     def test_read_only(self):
         from kazoo.exceptions import NotReadOnlyCallError
@@ -288,9 +290,24 @@ class TestReadOnlyMode(KazooTestCase):
                 ev.set()
 
         try:
-            self.cluster[1].stop()
-            self.cluster[2].stop()
-            ev.wait(6)
+            # stopping both nodes at the same time
+            # else the test seems flaky when on CI hosts
+            zk_stop_threads = []
+            zk_stop_threads.append(
+                threading.Thread(
+                    target=self.cluster[1].stop, daemon=True
+                )
+            )
+            zk_stop_threads.append(
+                threading.Thread(
+                    target=self.cluster[2].stop, daemon=True
+                )
+            )
+            for thread in zk_stop_threads:
+                thread.start()
+            for thread in zk_stop_threads:
+                thread.join()
+            ev.wait(15)
             assert ev.is_set()
             assert client.client_state == KeeperState.CONNECTED_RO
 
diff --git a/kazoo/tests/test_sasl.py b/kazoo/tests/test_sasl.py
index 2ee7120..3cca854 100644
--- a/kazoo/tests/test_sasl.py
+++ b/kazoo/tests/test_sasl.py
@@ -31,6 +31,7 @@ class TestLegacySASLDigestAuthentication(KazooTestHarness):
 
     def tearDown(self):
         self.teardown_zookeeper()
+        os.environ.pop('ZOOKEEPER_JAAS_AUTH', None)
 
     def test_connect_sasl_auth(self):
         from kazoo.security import make_acl
@@ -80,6 +81,7 @@ class TestSASLDigestAuthentication(KazooTestHarness):
 
     def tearDown(self):
         self.teardown_zookeeper()
+        os.environ.pop('ZOOKEEPER_JAAS_AUTH', None)
 
     def test_connect_sasl_auth(self):
         from kazoo.security import make_acl
@@ -145,6 +147,7 @@ class TestSASLGSSAPIAuthentication(KazooTestHarness):
 
     def tearDown(self):
         self.teardown_zookeeper()
+        os.environ.pop('ZOOKEEPER_JAAS_AUTH', None)
 
     def test_connect_gssapi_auth(self):
         from kazoo.security import make_acl
diff --git a/setup.cfg b/setup.cfg
index babe067..ba528cd 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -20,6 +20,8 @@ classifiers =
     Programming Language :: Python :: 3
     Programming Language :: Python :: 3.7
     Programming Language :: Python :: 3.8
+    Programming Language :: Python :: 3.9
+    Programming Language :: Python :: 3.10
     Programming Language :: Python :: Implementation :: CPython
     Programming Language :: Python :: Implementation :: PyPy
     Topic :: Communications