From 63ebe693174a4e6ec314e48d12fcdf3f8401eec6 Mon Sep 17 00:00:00 2001 From: Chayim Date: Thu, 21 Oct 2021 08:55:32 +0300 Subject: tox integrations with invoke and docker (#1632) --- .github/workflows/integration.yaml | 56 ++++++++++++++- .gitignore | 1 + CONTRIBUTING.rst | 41 ++++++----- Dockerfile | 9 --- Makefile | 14 ---- dev_requirements.txt | 5 ++ docker-entry.sh | 19 ----- docker/replica/Dockerfile | 7 ++ docker/replica/redis.conf | 4 ++ docker/slave/Dockerfile | 7 -- docker/slave/redis.conf | 4 -- tasks.py | 60 ++++++++++++++++ tox.ini | 138 +++++++++++++++++++++++++++++-------- 13 files changed, 263 insertions(+), 102 deletions(-) delete mode 100644 Dockerfile delete mode 100644 Makefile create mode 100644 dev_requirements.txt delete mode 100755 docker-entry.sh create mode 100644 docker/replica/Dockerfile create mode 100644 docker/replica/redis.conf delete mode 100644 docker/slave/Dockerfile delete mode 100644 docker/slave/redis.conf create mode 100644 tasks.py diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index f08a2c2..2618c33 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -2,12 +2,62 @@ name: CI on: push: + paths-ignore: + - 'docs/**' + - '**/*.rst' + - '**/*.md' pull_request: + paths-ignore: + - 'docs/**' + - '**/*.rst' + - '**/*.md' jobs: - integration: + + lint: + name: Code linters + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: install python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: run code linters + run: | + pip install -r dev_requirements.txt + invoke linters + + run-tests: + runs-on: ubuntu-latest + strategy: + max-parallel: 6 + matrix: + python-version: ['3.6', '3.7', '3.8', '3.9', '3.10', 'pypy-3.7'] + env: + ACTIONS_ALLOW_UNSECURE_COMMANDS: true + name: Python ${{ matrix.python-version }} tests + steps: + - uses: actions/checkout@v2 + - name: install python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: run tests + run: | + pip install -r dev_requirements.txt + invoke tests + + build_package: + name: Validate building and installing the package runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: test - run: make test + - name: install python + uses: actions/setup-python@v2 + with: + python-version: 3.9 + - name: build and install + run: | + pip install invoke + invoke package diff --git a/.gitignore b/.gitignore index b59bcdf..05c3846 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,4 @@ vagrant/.vagrant env venv coverage.xml +.venv diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index edaa7b6..ba1ddfc 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -29,25 +29,36 @@ Here's how to get started with your code contribution: 1. Create your own fork of redis-py 2. Do the changes in your fork -3. If you need a development environment, run ``make dev`` -4. While developing, make sure the tests pass by running ``make test`` +3. Create a virtualenv and install the development dependencies from the dev_requirements.txt file: + a. python -m venv .venv + b. source .venv/bin/activate + c. pip install -r dev_requirements.txt +3. If you need a development environment, run ``invoke devenv`` +4. While developing, make sure the tests pass by running ``invoke tests`` 5. If you like the change and think the project could use it, send a pull request +To see what else is part of the automation, run ``invoke -l`` + The Development Environment --------------------------- -Running ``make dev`` will create a Docker-based development environment that starts the following containers: +Running ``invoke devenv`` installs the development dependencies specified in the dev_requirements.txt. It starts all of the dockers used by this project, and leaves them running. These can be easily cleaned up with ``invoke clean``. NOTE: it is assumed that the user running these tests, can execute docker and its various commands. * A master Redis node -* A slave Redis node +* A Redis replica node * Three sentinel Redis nodes -* A test container +* A multi-python docker, with your source code mounted in /data -The slave is a replica of the master node, using the `leader-follower replication `_ feature. +The replica node, is a replica of the master node, using the `leader-follower replication `_ feature. The sentinels monitor the master node in a `sentinel high-availability configuration `_. -Meanwhile, the `test` container hosts the code from your checkout of ``redis-py`` and allows running tests against many Python versions. +Testing +------- + +Each run of tox starts and stops the various dockers required. Sometimes things get stuck, an ``invoke clean`` can help. + +Continuous Integration uses these same wrappers to run all of these tests against multiple versions of python. Feel free to test your changes against all the python versions supported, as declared by the tox.ini file (eg: tox -e py39). If you have the various python versions on your desktop, you can run *tox* by itself, to test all supported versions. Alternatively, as your source code is mounted in the **lots-of-pythons** docker, you can start exploring from there, with all supported python versions! Docker Tips ^^^^^^^^^^^ @@ -56,17 +67,17 @@ Following are a few tips that can help you work with the Docker-based developmen To get a bash shell inside of a container: -``$ docker-compose run /bin/bash`` - -**Note**: The term "service" refers to the "services" defined in the ``docker-compose.yml`` file: "master", "slave", "sentinel_1", "sentinel_2", "sentinel_3", "test". +``$ docker run -it /bin/bash`` + +**Note**: The term "service" refers to the "services" defined in the ``tox.ini`` file at the top of the repo: "master", "replicaof", "sentinel_1", "sentinel_2", "sentinel_3". Containers run a minimal Debian image that probably lacks tools you want to use. To install packages, first get a bash session (see previous tip) and then run: ``$ apt update && apt install `` -You can see the combined logging output of all containers like this: +You can see the logging output of a containers like this: -``$ docker-compose logs`` +``$ docker logs -f `` The command `make test` runs all tests in all tested Python environments. To run the tests in a single environment, like Python 3.6, use a command like this: @@ -81,13 +92,11 @@ Our test suite uses ``pytest``. You can run a specific test suite against a spec Troubleshooting ^^^^^^^^^^^^^^^ If you get any errors when running ``make dev`` or ``make test``, make sure that you -are using supported versions of Docker and docker-compose. +are using supported versions of Docker. -The included Dockerfiles and docker-compose.yml file work with the following -versions of Docker and docker-compose: +Please try at least versions of Docker. * Docker 19.03.12 -* docker-compose 1.26.2 How to Report a Bug ------------------- diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 2ceb725..0000000 --- a/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM fkrull/multi-python:latest - -RUN apt update \ - && apt install -y pypy pypy-dev pypy3-dev \ - && rm -rf /var/lib/apt/lists/* - -WORKDIR /redis-py - -COPY . /redis-py diff --git a/Makefile b/Makefile deleted file mode 100644 index 0d09643..0000000 --- a/Makefile +++ /dev/null @@ -1,14 +0,0 @@ -.PHONY: base clean dev test - -base: - docker build -t redis-py-base docker/base - -dev: base - docker-compose up -d --build - -test: dev - docker-compose run --rm test /redis-py/docker-entry.sh - -clean: - docker-compose stop - docker-compose rm -f diff --git a/dev_requirements.txt b/dev_requirements.txt new file mode 100644 index 0000000..2648127 --- /dev/null +++ b/dev_requirements.txt @@ -0,0 +1,5 @@ +flake8>=3.9.2 +pytest==6.2.5 +tox==3.24.4 +tox-docker==3.1.0 +invoke==1.6.0 diff --git a/docker-entry.sh b/docker-entry.sh deleted file mode 100755 index dc119dc..0000000 --- a/docker-entry.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/bin/bash - -# This is the entrypoint for "make test". It invokes Tox. If running -# outside the CI environment, it disables uploading the coverage report to codecov - -set -eu - -REDIS_MASTER="${REDIS_MASTER_HOST}":"${REDIS_MASTER_PORT}" -echo "Testing against Redis Server: ${REDIS_MASTER}" - -# skip the "codecov" env if not running on Travis -if [ "${GITHUB_ACTIONS}" = true ] ; then - echo "Skipping codecov" - export TOX_SKIP_ENV="codecov" -fi - -# use the wait-for-it util to ensure the server is running before invoking Tox -util/wait-for-it.sh ${REDIS_MASTER} -- tox -- --redis-url=redis://"${REDIS_MASTER}"/9 - diff --git a/docker/replica/Dockerfile b/docker/replica/Dockerfile new file mode 100644 index 0000000..7b6bdbe --- /dev/null +++ b/docker/replica/Dockerfile @@ -0,0 +1,7 @@ +FROM redis-py-base:latest + +COPY redis.conf /usr/local/etc/redis/redis.conf + +EXPOSE 6380 + +CMD [ "redis-server", "/usr/local/etc/redis/redis.conf" ] diff --git a/docker/replica/redis.conf b/docker/replica/redis.conf new file mode 100644 index 0000000..4a1dcd7 --- /dev/null +++ b/docker/replica/redis.conf @@ -0,0 +1,4 @@ +bind replica 127.0.0.1 +port 6380 +save "" +replicaof master 6379 diff --git a/docker/slave/Dockerfile b/docker/slave/Dockerfile deleted file mode 100644 index 7b6bdbe..0000000 --- a/docker/slave/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM redis-py-base:latest - -COPY redis.conf /usr/local/etc/redis/redis.conf - -EXPOSE 6380 - -CMD [ "redis-server", "/usr/local/etc/redis/redis.conf" ] diff --git a/docker/slave/redis.conf b/docker/slave/redis.conf deleted file mode 100644 index 629ac70..0000000 --- a/docker/slave/redis.conf +++ /dev/null @@ -1,4 +0,0 @@ -bind slave 127.0.0.1 -port 6380 -save "" -slaveof master 6379 diff --git a/tasks.py b/tasks.py new file mode 100644 index 0000000..15e983b --- /dev/null +++ b/tasks.py @@ -0,0 +1,60 @@ +import os +import shutil +from invoke import task, run + +with open('tox.ini') as fp: + lines = fp.read().split("\n") + dockers = [line.split("=")[1].strip() for line in lines + if line.find("name") != -1] + + +@task +def devenv(c): + """Builds a development environment: downloads, and starts all dockers + specified in the tox.ini file. + """ + clean(c) + cmd = 'tox -e devenv' + for d in dockers: + cmd += " --docker-dont-stop={}".format(d) + print("Running: {}".format(cmd)) + run(cmd) + + +@task +def linters(c): + """Run code linters""" + run("flake8") + + +@task +def all_tests(c): + """Run all linters, and tests in redis-py. This assumes you have all + the python versions specified in the tox.ini file. + """ + linters(c) + tests(c) + + +@task +def tests(c): + """Run the redis-py test suite against the current python, + with and without hiredis. + """ + run("tox -e plain -e hiredis") + + +@task +def clean(c): + """Stop all dockers, and clean up the built binaries, if generated.""" + if os.path.isdir("build"): + shutil.rmtree("build") + if os.path.isdir("dist"): + shutil.rmtree("dist") + run("docker rm -f {}".format(' '.join(dockers))) + + +@task +def package(c): + """Create the python packages""" + run("python setup.py build install") diff --git a/tox.ini b/tox.ini index db7492d..bfe937f 100644 --- a/tox.ini +++ b/tox.ini @@ -2,50 +2,128 @@ addopts = -s [tox] -minversion = 2.4 -envlist = {py35,py36,py37,py38,py39,pypy3}-{plain,hiredis}, flake8, covreport, codecov +minversion = 3.2.0 +requires = tox-docker +envlist = {py35,py36,py37,py38,py39,pypy3}-{plain,hiredis}, flake8 + +[docker:master] +name = master +image = redis:6.2-bullseye +ports = + 6379:6379/tcp +healtcheck_cmd = python -c "import socket;print(True) if 0 == socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('127.0.0.1',6379)) else False" +volumes = + bind:rw:{toxinidir}/docker/master/redis.conf:/usr/local/etc/redis/redis.conf + +[docker:replica] +name = replica +image = redis:6.2-bullseye +links = + master:master +ports = + 6380:6380/tcp +healtcheck_cmd = python -c "import socket;print(True) if 0 == socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('127.0.0.1',6380)) else False" +volumes = + bind:rw:{toxinidir}/docker/replica/redis.conf:/usr/local/etc/redis/redis.conf + +[docker:sentinel_1] +name = sentinel_1 +image = redis:6.2-bullseye +links = + master:master +ports = + 26379:26379/tcp +healtcheck_cmd = python -c "import socket;print(True) if 0 == socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('127.0.0.1',26379)) else False" +volumes = + bind:rw:{toxinidir}/docker/sentinel_1/sentinel.conf:/usr/local/etc/redis/sentinel.conf + +[docker:sentinel_2] +name = sentinel_2 +image = redis:6.2-bullseye +links = + master:master +ports = + 26380:26380/tcp +healtcheck_cmd = python -c "import socket;print(True) if 0 == socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('127.0.0.1',26380)) else False" +volumes = + bind:rw:{toxinidir}/docker/sentinel_2/sentinel.conf:/usr/local/etc/redis/sentinel.conf + +[docker:sentinel_3] +name = sentinel_3 +image = redis:6.2-bullseye +links = + master:master +ports = + 26381:26381/tcp +healtcheck_cmd = python -c "import socket;print(True) if 0 == socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('127.0.0.1',26381)) else False" +volumes = + bind:rw:{toxinidir}/docker/sentinel_3/sentinel.conf:/usr/local/etc/redis/sentinel.conf + +[docker:redismod] +name = redismod +image = redislabs/redismod:edge +ports = + 16379:16379/tcp +healtcheck_cmd = python -c "import socket;print(True) if 0 == socket.socket(socket.AF_INET, socket.SOCK_STREAM).connect_ex(('127.0.0.1',16379)) else False" + +[docker:lots-of-pythons] +name = lots-of-pythons +image = redisfab/lots-of-pythons +volumes = + bind:rw:{toxinidir}:/data [testenv] -deps = - coverage - pytest >= 2.7.0 +deps = -r {toxinidir}/dev_requirements.txt +docker = + master + replica + sentinel_1 + sentinel_2 + sentinel_3 + redismod extras = hiredis: hiredis commands = - {envpython} -b -m coverage run -p -m pytest -W always {posargs} - {envpython} -b -m coverage combine --append + pytest -W always {posargs} + +[testenv:devenv] +skipsdist = true +skip_install = true +deps = -r {toxinidir}/dev_requirements.txt +docker = + master + replica + sentinel_1 + sentinel_2 + sentinel_3 + redismod + lots-of-pythons +commands = echo [testenv:flake8] -basepython = python3.6 -deps = flake8 +deps_files = dev_requirements.txt commands = flake8 skipsdist = true skip_install = true -[testenv:pypy-plain] -basepython = pypy - -[testenv:pypy-hiredis] -basepython = pypy - [testenv:pypy3-plain] basepython = pypy3 [testenv:pypy3-hiredis] basepython = pypy3 -[testenv:codecov] -deps = codecov -commands = codecov -passenv = - REDIS_* - CI - CI_* - CODECOV_* - SHIPPABLE - GITHUB_* - VCS_* - -[testenv:covreport] -deps = coverage -commands = coverage report +#[testenv:codecov] +#deps = codecov +#commands = codecov +#passenv = +# REDIS_* +# CI +# CI_* +# CODECOV_* +# SHIPPABLE +# GITHUB_* +# VCS_* +# +#[testenv:covreport] +#deps = coverage +#commands = coverage report -- cgit v1.2.1