feat(tests): update Zookeeper 3.5.2-alpha to 3.5.4-beta

Version of Zookeeper is upgraded from 3.5.2-alpha to 3.5.4-beta for automated tests.
Reconfig now needs superuser authentification, a test is added to cover this feature.
Additionnal configuration and jvm parameters can be added when initializing
ManagedZookeeper. This is needed to ensure reconfig's tests to pass and can be used
in the future for similar needs.

Closes #477

4 files changed, 73 insertions, 17 deletions

diff --git a/.travis.yml b/.travis.yml
index e4aa004..23191e4 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -19,31 +19,43 @@ matrix:
   - python: '2.7'
     env: ZOOKEEPER_VERSION=3.4.13 TOX_VENV=py27
   - python: '2.7'
+    env: ZOOKEEPER_VERSION=3.5.4-beta TOX_VENV=py27
+  - python: '2.7'
     env: ZOOKEEPER_VERSION=3.3.6 TOX_VENV=py27-gevent
   - python: '2.7'
     env: ZOOKEEPER_VERSION=3.4.13 TOX_VENV=py27-gevent
   - python: '2.7'
+    env: ZOOKEEPER_VERSION=3.5.4-beta TOX_VENV=py27-gevent
+  - python: '2.7'
     env: ZOOKEEPER_VERSION=3.3.6 TOX_VENV=py27-eventlet
   - python: '2.7'
     env: ZOOKEEPER_VERSION=3.4.13 TOX_VENV=py27-eventlet
   - python: '2.7'
-    env: ZOOKEEPER_VERSION=3.5.2-alpha TOX_VENV=py27
+    env: ZOOKEEPER_VERSION=3.5.4-beta TOX_VENV=py27-eventlet
   - python: '3.4'
     env: ZOOKEEPER_VERSION=3.3.6 TOX_VENV=py34
   - python: '3.4'
     env: ZOOKEEPER_VERSION=3.4.13 TOX_VENV=py34
+  - python: '3.4'
+    env: ZOOKEEPER_VERSION=3.5.4-beta TOX_VENV=py34
   - python: '3.5'
     env: ZOOKEEPER_VERSION=3.3.6 TOX_VENV=py35
   - python: '3.5'
     env: ZOOKEEPER_VERSION=3.4.13 TOX_VENV=py35
+  - python: '3.5'
+    env: ZOOKEEPER_VERSION=3.5.4-beta TOX_VENV=py35
   - python: '3.6'
     env: ZOOKEEPER_VERSION=3.3.6 TOX_VENV=py36
   - python: '3.6'
     env: ZOOKEEPER_VERSION=3.4.13 TOX_VENV=py36
+  - python: '3.6'
+    env: ZOOKEEPER_VERSION=3.5.4-beta TOX_VENV=py36
   - python: pypy
     env: ZOOKEEPER_VERSION=3.3.6 TOX_VENV=pypy
   - python: pypy
     env: ZOOKEEPER_VERSION=3.4.13 TOX_VENV=pypy
+  - python: 'pypy'
+    env: ZOOKEEPER_VERSION=3.5.4-beta TOX_VENV=pypy
 notifications:
   email: false
 install:
diff --git a/kazoo/testing/common.py b/kazoo/testing/common.py
index c4aa0b9..3f0218c 100644
--- a/kazoo/testing/common.py
+++ b/kazoo/testing/common.py
@@ -74,7 +74,8 @@ class ManagedZooKeeper(object):
     future, we may want to do that, especially when run in a
     Hudson/Buildbot context, to ensure more test robustness."""
 
-    def __init__(self, software_path, server_info, peers=(), classpath=None):
+    def __init__(self, software_path, server_info, peers=(), classpath=None,
+                 configuration_entries=[], java_system_properties=[]):
         """Define the ZooKeeper test instance.
 
         @param install_path: The path to the install for ZK
@@ -87,6 +88,8 @@ class ManagedZooKeeper(object):
         self.peers = peers
         self.working_path = tempfile.mkdtemp()
         self._running = False
+        self.configuration_entries = configuration_entries
+        self.java_system_properties = java_system_properties
 
     def run(self):
         """Run the ZooKeeper instance under a temporary directory.
@@ -117,9 +120,12 @@ clientPort=%s
 maxClientCnxns=0
 admin.serverPort=%s
 authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider
+%s
 """ % (to_java_compatible_path(data_path),
        self.server_info.client_port,
-       self.server_info.admin_port))  # NOQA
+       self.server_info.admin_port,
+       "\n".join(self.configuration_entries)))  # NOQA
+
 
         # setup a replicated setup if peers are specified
         if self.peers:
@@ -175,7 +181,7 @@ log4j.appender.ROLLINGFILE.File=""" + to_java_compatible_path(  # NOQA
 
             # JAAS configuration for SASL authentication
             "-Djava.security.auth.login.config=%s" % jass_config_path,
-
+        ] + self.java_system_properties + [
             "org.apache.zookeeper.server.quorum.QuorumPeerMain",
             config_path,
         ]
@@ -265,11 +271,14 @@ log4j.appender.ROLLINGFILE.File=""" + to_java_compatible_path(  # NOQA
 class ZookeeperCluster(object):
 
     def __init__(self, install_path=None, classpath=None,
-                 size=3, port_offset=20000, observer_start_id=-1):
+                 size=3, port_offset=20000, observer_start_id=-1,
+                 configuration_entries=[],
+                 java_system_properties=[]):
         self._install_path = install_path
         self._classpath = classpath
         self._servers = []
 
+
         # Calculate ports and peer group
         port = port_offset
         peers = []
@@ -292,7 +301,9 @@ class ZookeeperCluster(object):
             self._servers.append(
                 ManagedZooKeeper(
                     self._install_path, server_info, server_peers,
-                    classpath=self._classpath))
+                    classpath=self._classpath,
+                    configuration_entries=configuration_entries,
+                    java_system_properties=java_system_properties))
 
     def __getitem__(self, k):
         return self._servers[k]
diff --git a/kazoo/testing/harness.py b/kazoo/testing/harness.py
index 76adcf3..de56795 100644
--- a/kazoo/testing/harness.py
+++ b/kazoo/testing/harness.py
@@ -13,7 +13,6 @@ from kazoo.protocol.states import (
 )
 from kazoo.testing.common import ZookeeperCluster
 
-
 log = logging.getLogger(__name__)
 
 CLUSTER = None
@@ -26,20 +25,41 @@ def get_global_cluster():
         ZK_CLASSPATH = os.environ.get("ZOOKEEPER_CLASSPATH")
         ZK_PORT_OFFSET = int(os.environ.get("ZOOKEEPER_PORT_OFFSET", 20000))
         ZK_CLUSTER_SIZE = int(os.environ.get("ZOOKEEPER_CLUSTER_SIZE", 3))
+        ZK_VERSION = os.environ.get("ZOOKEEPER_VERSION")
+        if '-' in ZK_VERSION:
+            # Ignore pre-release markers like -alpha
+            ZK_VERSION = ZK_VERSION.split('-')[0]
+        ZK_VERSION = tuple([int(n) for n in ZK_VERSION.split('.')])
+
         ZK_OBSERVER_START_ID = int(
             os.environ.get("ZOOKEEPER_OBSERVER_START_ID", -1))
 
-        assert ZK_HOME or ZK_CLASSPATH, (
-            "Either ZOOKEEPER_PATH or ZOOKEEPER_CLASSPATH environment "
-            "variable must be defined.\n"
+        assert ZK_HOME or ZK_CLASSPATH or ZK_VERSION, (
+            "Either ZOOKEEPER_PATH or ZOOKEEPER_CLASSPATH or "
+            "ZOOKEEPER_VERSION environment variable must be defined.\n"
             "For deb package installations this is /usr/share/java")
 
+        if ZK_VERSION >= (3, 5):
+            additional_configuration_entries = [
+                "4lw.commands.whitelist=*",
+                "reconfigEnabled=true"
+            ]
+            # If defines, this sets the superuser password to "test"
+            additional_java_system_properties = [
+                "-Dzookeeper.DigestAuthenticationProvider.superDigest="
+                "super:D/InIHSb7yEEbrWz8b9l71RjZJU="
+            ]
+        else:
+            additional_configuration_entries = []
+            additional_java_system_properties = []
         CLUSTER = ZookeeperCluster(
             install_path=ZK_HOME,
             classpath=ZK_CLASSPATH,
             port_offset=ZK_PORT_OFFSET,
             size=ZK_CLUSTER_SIZE,
-            observer_start_id=ZK_OBSERVER_START_ID
+            observer_start_id=ZK_OBSERVER_START_ID,
+            configuration_entries=additional_configuration_entries,
+            java_system_properties=additional_java_system_properties
         )
         atexit.register(lambda cluster: cluster.terminate(), CLUSTER)
     return CLUSTER
diff --git a/kazoo/tests/test_client.py b/kazoo/tests/test_client.py
index 0dad864..e22261d 100644
--- a/kazoo/tests/test_client.py
+++ b/kazoo/tests/test_client.py
@@ -1204,9 +1204,9 @@ class TestNonChrootClient(KazooTestCase):
 
 
 class TestReconfig(KazooTestCase):
-
     def setUp(self):
         KazooTestCase.setUp(self)
+
         if TRAVIS_ZK_VERSION:
             version = TRAVIS_ZK_VERSION
         else:
@@ -1214,12 +1214,25 @@ class TestReconfig(KazooTestCase):
         if not version or version < (3, 5):
             raise SkipTest("Must use Zookeeper 3.5 or above")
 
+    def test_no_super_auth(self):
+        self.assertRaises(NoAuthError,
+                          self.client.reconfig,
+                          joining='server.999=0.0.0.0:1234:2345:observer;3456',
+                          leaving=None,
+                          new_members=None)
+
     def test_add_remove_observer(self):
         def free_sock_port():
             s = socket.socket()
             s.bind(('', 0))
             return s, s.getsockname()[1]
 
+        username = "super"
+        password = "test"
+        digest_auth = "%s:%s" % (username, password)
+        client = self._get_client(auth_data=[('digest', digest_auth)])
+        client.start()
+
         # get ports for election, zab and client endpoints. we need to use
         # ports for which we'd immediately get a RST upon connect(); otherwise
         # the cluster could crash if it gets a SocketTimeoutException:
@@ -1230,18 +1243,18 @@ class TestReconfig(KazooTestCase):
 
         joining = 'server.100=0.0.0.0:%d:%d:observer;0.0.0.0:%d' % (
             port1, port2, port3)
-        data, _ = self.client.reconfig(joining=joining,
+        data, _ = client.reconfig(joining=joining,
                                        leaving=None,
                                        new_members=None)
-        self.assertIn(joining, data)
+        self.assertIn(joining.encode('utf8'), data)
 
-        data, _ = self.client.reconfig(joining=None,
+        data, _ = client.reconfig(joining=None,
                                        leaving='100',
                                        new_members=None)
-        self.assertNotIn(joining, data)
+        self.assertNotIn(joining.encode('utf8'), data)
 
         # try to add it again, but a config number in the future
-        curver = int(data.split('\n')[-1].split('=')[1], base=16)
+        curver = int(data.decode().split('\n')[-1].split('=')[1], base=16)
         self.assertRaises(BadVersionError,
                           self.client.reconfig,
                           joining=joining,