From 3033091530b8f0aba13a1b8d031a7297e54006c2 Mon Sep 17 00:00:00 2001 From: Stephen Date: Sat, 2 Jun 2018 11:46:37 +0200 Subject: feat(tests): update Zookeeper 3.5.2-alpha to 3.5.4-beta Version of Zookeeper is upgraded from 3.5.2-alpha to 3.5.4-beta for automated tests. Reconfig now needs superuser authentification, a test is added to cover this feature. Additionnal configuration and jvm parameters can be added when initializing ManagedZookeeper. This is needed to ensure reconfig's tests to pass and can be used in the future for similar needs. Closes #477 --- .travis.yml | 14 +++++++++++++- kazoo/testing/common.py | 21 ++++++++++++++++----- kazoo/testing/harness.py | 30 +++++++++++++++++++++++++----- kazoo/tests/test_client.py | 25 +++++++++++++++++++------ 4 files changed, 73 insertions(+), 17 deletions(-) diff --git a/.travis.yml b/.travis.yml index e4aa004..23191e4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,32 +18,44 @@ matrix: env: ZOOKEEPER_VERSION=3.3.6 TOX_VENV=py27 - python: '2.7' env: ZOOKEEPER_VERSION=3.4.13 TOX_VENV=py27 + - python: '2.7' + env: ZOOKEEPER_VERSION=3.5.4-beta TOX_VENV=py27 - python: '2.7' env: ZOOKEEPER_VERSION=3.3.6 TOX_VENV=py27-gevent - python: '2.7' env: ZOOKEEPER_VERSION=3.4.13 TOX_VENV=py27-gevent + - python: '2.7' + env: ZOOKEEPER_VERSION=3.5.4-beta TOX_VENV=py27-gevent - python: '2.7' env: ZOOKEEPER_VERSION=3.3.6 TOX_VENV=py27-eventlet - python: '2.7' env: ZOOKEEPER_VERSION=3.4.13 TOX_VENV=py27-eventlet - python: '2.7' - env: ZOOKEEPER_VERSION=3.5.2-alpha TOX_VENV=py27 + env: ZOOKEEPER_VERSION=3.5.4-beta TOX_VENV=py27-eventlet - python: '3.4' env: ZOOKEEPER_VERSION=3.3.6 TOX_VENV=py34 - python: '3.4' env: ZOOKEEPER_VERSION=3.4.13 TOX_VENV=py34 + - python: '3.4' + env: ZOOKEEPER_VERSION=3.5.4-beta TOX_VENV=py34 - python: '3.5' env: ZOOKEEPER_VERSION=3.3.6 TOX_VENV=py35 - python: '3.5' env: ZOOKEEPER_VERSION=3.4.13 TOX_VENV=py35 + - python: '3.5' + env: ZOOKEEPER_VERSION=3.5.4-beta TOX_VENV=py35 - python: '3.6' env: ZOOKEEPER_VERSION=3.3.6 TOX_VENV=py36 - python: '3.6' env: ZOOKEEPER_VERSION=3.4.13 TOX_VENV=py36 + - python: '3.6' + env: ZOOKEEPER_VERSION=3.5.4-beta TOX_VENV=py36 - python: pypy env: ZOOKEEPER_VERSION=3.3.6 TOX_VENV=pypy - python: pypy env: ZOOKEEPER_VERSION=3.4.13 TOX_VENV=pypy + - python: 'pypy' + env: ZOOKEEPER_VERSION=3.5.4-beta TOX_VENV=pypy notifications: email: false install: diff --git a/kazoo/testing/common.py b/kazoo/testing/common.py index c4aa0b9..3f0218c 100644 --- a/kazoo/testing/common.py +++ b/kazoo/testing/common.py @@ -74,7 +74,8 @@ class ManagedZooKeeper(object): future, we may want to do that, especially when run in a Hudson/Buildbot context, to ensure more test robustness.""" - def __init__(self, software_path, server_info, peers=(), classpath=None): + def __init__(self, software_path, server_info, peers=(), classpath=None, + configuration_entries=[], java_system_properties=[]): """Define the ZooKeeper test instance. @param install_path: The path to the install for ZK @@ -87,6 +88,8 @@ class ManagedZooKeeper(object): self.peers = peers self.working_path = tempfile.mkdtemp() self._running = False + self.configuration_entries = configuration_entries + self.java_system_properties = java_system_properties def run(self): """Run the ZooKeeper instance under a temporary directory. @@ -117,9 +120,12 @@ clientPort=%s maxClientCnxns=0 admin.serverPort=%s authProvider.1=org.apache.zookeeper.server.auth.SASLAuthenticationProvider +%s """ % (to_java_compatible_path(data_path), self.server_info.client_port, - self.server_info.admin_port)) # NOQA + self.server_info.admin_port, + "\n".join(self.configuration_entries))) # NOQA + # setup a replicated setup if peers are specified if self.peers: @@ -175,7 +181,7 @@ log4j.appender.ROLLINGFILE.File=""" + to_java_compatible_path( # NOQA # JAAS configuration for SASL authentication "-Djava.security.auth.login.config=%s" % jass_config_path, - + ] + self.java_system_properties + [ "org.apache.zookeeper.server.quorum.QuorumPeerMain", config_path, ] @@ -265,11 +271,14 @@ log4j.appender.ROLLINGFILE.File=""" + to_java_compatible_path( # NOQA class ZookeeperCluster(object): def __init__(self, install_path=None, classpath=None, - size=3, port_offset=20000, observer_start_id=-1): + size=3, port_offset=20000, observer_start_id=-1, + configuration_entries=[], + java_system_properties=[]): self._install_path = install_path self._classpath = classpath self._servers = [] + # Calculate ports and peer group port = port_offset peers = [] @@ -292,7 +301,9 @@ class ZookeeperCluster(object): self._servers.append( ManagedZooKeeper( self._install_path, server_info, server_peers, - classpath=self._classpath)) + classpath=self._classpath, + configuration_entries=configuration_entries, + java_system_properties=java_system_properties)) def __getitem__(self, k): return self._servers[k] diff --git a/kazoo/testing/harness.py b/kazoo/testing/harness.py index 76adcf3..de56795 100644 --- a/kazoo/testing/harness.py +++ b/kazoo/testing/harness.py @@ -13,7 +13,6 @@ from kazoo.protocol.states import ( ) from kazoo.testing.common import ZookeeperCluster - log = logging.getLogger(__name__) CLUSTER = None @@ -26,20 +25,41 @@ def get_global_cluster(): ZK_CLASSPATH = os.environ.get("ZOOKEEPER_CLASSPATH") ZK_PORT_OFFSET = int(os.environ.get("ZOOKEEPER_PORT_OFFSET", 20000)) ZK_CLUSTER_SIZE = int(os.environ.get("ZOOKEEPER_CLUSTER_SIZE", 3)) + ZK_VERSION = os.environ.get("ZOOKEEPER_VERSION") + if '-' in ZK_VERSION: + # Ignore pre-release markers like -alpha + ZK_VERSION = ZK_VERSION.split('-')[0] + ZK_VERSION = tuple([int(n) for n in ZK_VERSION.split('.')]) + ZK_OBSERVER_START_ID = int( os.environ.get("ZOOKEEPER_OBSERVER_START_ID", -1)) - assert ZK_HOME or ZK_CLASSPATH, ( - "Either ZOOKEEPER_PATH or ZOOKEEPER_CLASSPATH environment " - "variable must be defined.\n" + assert ZK_HOME or ZK_CLASSPATH or ZK_VERSION, ( + "Either ZOOKEEPER_PATH or ZOOKEEPER_CLASSPATH or " + "ZOOKEEPER_VERSION environment variable must be defined.\n" "For deb package installations this is /usr/share/java") + if ZK_VERSION >= (3, 5): + additional_configuration_entries = [ + "4lw.commands.whitelist=*", + "reconfigEnabled=true" + ] + # If defines, this sets the superuser password to "test" + additional_java_system_properties = [ + "-Dzookeeper.DigestAuthenticationProvider.superDigest=" + "super:D/InIHSb7yEEbrWz8b9l71RjZJU=" + ] + else: + additional_configuration_entries = [] + additional_java_system_properties = [] CLUSTER = ZookeeperCluster( install_path=ZK_HOME, classpath=ZK_CLASSPATH, port_offset=ZK_PORT_OFFSET, size=ZK_CLUSTER_SIZE, - observer_start_id=ZK_OBSERVER_START_ID + observer_start_id=ZK_OBSERVER_START_ID, + configuration_entries=additional_configuration_entries, + java_system_properties=additional_java_system_properties ) atexit.register(lambda cluster: cluster.terminate(), CLUSTER) return CLUSTER diff --git a/kazoo/tests/test_client.py b/kazoo/tests/test_client.py index 0dad864..e22261d 100644 --- a/kazoo/tests/test_client.py +++ b/kazoo/tests/test_client.py @@ -1204,9 +1204,9 @@ class TestNonChrootClient(KazooTestCase): class TestReconfig(KazooTestCase): - def setUp(self): KazooTestCase.setUp(self) + if TRAVIS_ZK_VERSION: version = TRAVIS_ZK_VERSION else: @@ -1214,12 +1214,25 @@ class TestReconfig(KazooTestCase): if not version or version < (3, 5): raise SkipTest("Must use Zookeeper 3.5 or above") + def test_no_super_auth(self): + self.assertRaises(NoAuthError, + self.client.reconfig, + joining='server.999=0.0.0.0:1234:2345:observer;3456', + leaving=None, + new_members=None) + def test_add_remove_observer(self): def free_sock_port(): s = socket.socket() s.bind(('', 0)) return s, s.getsockname()[1] + username = "super" + password = "test" + digest_auth = "%s:%s" % (username, password) + client = self._get_client(auth_data=[('digest', digest_auth)]) + client.start() + # get ports for election, zab and client endpoints. we need to use # ports for which we'd immediately get a RST upon connect(); otherwise # the cluster could crash if it gets a SocketTimeoutException: @@ -1230,18 +1243,18 @@ class TestReconfig(KazooTestCase): joining = 'server.100=0.0.0.0:%d:%d:observer;0.0.0.0:%d' % ( port1, port2, port3) - data, _ = self.client.reconfig(joining=joining, + data, _ = client.reconfig(joining=joining, leaving=None, new_members=None) - self.assertIn(joining, data) + self.assertIn(joining.encode('utf8'), data) - data, _ = self.client.reconfig(joining=None, + data, _ = client.reconfig(joining=None, leaving='100', new_members=None) - self.assertNotIn(joining, data) + self.assertNotIn(joining.encode('utf8'), data) # try to add it again, but a config number in the future - curver = int(data.split('\n')[-1].split('=')[1], base=16) + curver = int(data.decode().split('\n')[-1].split('=')[1], base=16) self.assertRaises(BadVersionError, self.client.reconfig, joining=joining, -- cgit v1.2.1