From 5c55bedfb187ec62e4b15e997e86727d70d334f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bern=C3=A1t=20G=C3=A1bor?= Date: Sat, 28 Nov 2020 11:33:35 +0000 Subject: tests: add test for the sequential run failing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Bernát Gábor --- tests/session/cmd/run/test_tox_env_runner.py | 90 ----------------------- tests/session/cmd/test_sequential.py | 105 +++++++++++++++++++++++++++ 2 files changed, 105 insertions(+), 90 deletions(-) delete mode 100644 tests/session/cmd/run/test_tox_env_runner.py create mode 100644 tests/session/cmd/test_sequential.py diff --git a/tests/session/cmd/run/test_tox_env_runner.py b/tests/session/cmd/run/test_tox_env_runner.py deleted file mode 100644 index 91970f1a..00000000 --- a/tests/session/cmd/run/test_tox_env_runner.py +++ /dev/null @@ -1,90 +0,0 @@ -import json -import re -import sys -from typing import Any, Dict, List, Tuple - -import pytest -from virtualenv.discovery.py_info import PythonInfo - -from tox import __version__ -from tox.pytest import ToxProjectCreator - - -def test_ignore_cmd(tox_project: ToxProjectCreator) -> None: - cmd = [ - "- python -c 'import sys; print(\"magic fail\", file=sys.stderr); sys.exit(1)'", - "python -c 'import sys; print(\"magic pass\"); sys.exit(0)'", - ] - project = tox_project({"tox.ini": f"[tox]\nenvlist=py\nno_package=true\n[testenv]\ncommands={cmd[0]}\n {cmd[1]}"}) - outcome = project.run("r", "-e", "py") - outcome.assert_success() - assert "magic pass" in outcome.out - assert "magic fail" in outcome.err - - -@pytest.mark.timeout(60) -@pytest.mark.integration -def test_result_json_run_one(tox_project: ToxProjectCreator) -> None: - cmd = [ - "- python -c 'import sys; print(\"magic fail\", file=sys.stderr); sys.exit(1)'", - "python -c 'import sys; print(\"magic pass\"); sys.exit(0)'", - ] - project = tox_project( - { - "tox.ini": f"[tox]\nenvlist=py\n[testenv]\npackage=wheel\ncommands={cmd[0]}\n {cmd[1]}", - "setup.py": "from setuptools import setup\nsetup(name='a', version='1.0', py_modules=['run']," - "install_requires=['setuptools>44'])", - "run.py": "print('run')", - "pyproject.toml": '[build-system]\nrequires=["setuptools","wheel"]\nbuild-backend="setuptools.build_meta"', - } - ) - log = project.path / "log.json" - outcome = project.run("r", "-vv", "-e", "py", "--result-json", str(log)) - outcome.assert_success() - with log.open("rt") as file_handler: - log_report = json.load(file_handler) - - py_info = PythonInfo.current_system() - host_python = { - "executable": py_info.system_executable, - "extra_version_info": None, - "implementation": py_info.implementation, - "is_64": py_info.architecture == 64, - "sysplatform": py_info.platform, - "version": py_info.version, - "version_info": list(py_info.version_info), - } - packaging_setup = get_cmd_exit_run_id(log_report, ".package-py", "setup") - assert packaging_setup == [(0, "install"), (0, "build requires"), (0, "freeze"), (0, "package meta")] - packaging_test = get_cmd_exit_run_id(log_report, ".package-py", "test") - assert packaging_test == [(0, "build")] - packaging_installed = log_report["testenvs"][".package-py"].pop("installed_packages") - assert {i[: i.find("==")] for i in packaging_installed} == {"pip", "setuptools", "wheel"} - - py_setup = get_cmd_exit_run_id(log_report, "py", "setup") - assert py_setup == [(0, "install"), (0, "install"), (0, "freeze")] # install => 1 dep and 1 package - py_test = get_cmd_exit_run_id(log_report, "py", "test") - assert py_test == [(1, "commands[0]"), (0, "commands[1]")] - packaging_installed = log_report["testenvs"]["py"].pop("installed_packages") - expected_pkg = {"pip", "setuptools", "wheel", "a"} - assert {i[: i.find("==")] if "@" not in i else "a" for i in packaging_installed} == expected_pkg - install_package = log_report["testenvs"]["py"].pop("installpkg") - assert re.match("^[a-fA-F0-9]{64}$", install_package.pop("sha256")) - assert install_package == {"basename": "a-1.0-py3-none-any.whl", "type": "file"} - - expected = { - "reportversion": "1", - "toxversion": __version__, - "platform": sys.platform, - "testenvs": { - "py": {"python": host_python}, - ".package-py": {"python": host_python}, - }, - } - assert "host" in log_report - assert log_report.pop("host") - assert log_report == expected - - -def get_cmd_exit_run_id(report: Dict[str, Any], name: str, group: str) -> List[Tuple[int, str]]: - return [(i["retcode"], i["run_id"]) for i in report["testenvs"][name].pop(group)] diff --git a/tests/session/cmd/test_sequential.py b/tests/session/cmd/test_sequential.py new file mode 100644 index 00000000..ff01b151 --- /dev/null +++ b/tests/session/cmd/test_sequential.py @@ -0,0 +1,105 @@ +import json +import re +import sys +from typing import Any, Dict, List, Tuple + +import pytest +from re_assert import Matches +from virtualenv.discovery.py_info import PythonInfo + +from tox import __version__ +from tox.pytest import ToxProjectCreator + + +def test_run_ignore_cmd_exit_code(tox_project: ToxProjectCreator) -> None: + cmd = [ + "- python -c 'import sys; print(\"magic fail\", file=sys.stderr); sys.exit(1)'", + "python -c 'import sys; print(\"magic pass\"); sys.exit(0)'", + ] + project = tox_project({"tox.ini": f"[tox]\nno_package=true\n[testenv]\ncommands={cmd[0]}\n {cmd[1]}"}) + outcome = project.run("r", "-e", "py") + outcome.assert_success() + assert "magic pass" in outcome.out + assert "magic fail" in outcome.err + + +def test_run_sequential_fail(tox_project: ToxProjectCreator) -> None: + def _cmd(value: int) -> str: + return f"python -c 'import sys; print(\"exit {value}\"); sys.exit({value})'" + + ini = f"[tox]\nenv_list=a,b\nno_package=true\n[testenv:a]\ncommands={_cmd(1)}\n[testenv:b]\ncommands={_cmd(0)}" + project = tox_project({"tox.ini": ini}) + outcome = project.run("r", "-e", "a,b") + outcome.assert_failed() + reports = outcome.out.splitlines()[-3:] + assert Matches(r" evaluation failed :\( \(.* seconds\)") == reports[-1] + assert Matches(r" b: OK \(.*=setup\[.*\]\+cmd\[.*\] seconds\)") == reports[-2] + assert Matches(r" a: FAIL code 1\(.*=setup\[.*\]\+cmd\[.*\] seconds\)") == reports[-3] + + +@pytest.mark.timeout(60) +@pytest.mark.integration +def test_result_json_sequential(tox_project: ToxProjectCreator) -> None: + cmd = [ + "- python -c 'import sys; print(\"magic fail\", file=sys.stderr); sys.exit(1)'", + "python -c 'import sys; print(\"magic pass\"); sys.exit(0)'", + ] + project = tox_project( + { + "tox.ini": f"[tox]\nenvlist=py\n[testenv]\npackage=wheel\ncommands={cmd[0]}\n {cmd[1]}", + "setup.py": "from setuptools import setup\nsetup(name='a', version='1.0', py_modules=['run']," + "install_requires=['setuptools>44'])", + "run.py": "print('run')", + "pyproject.toml": '[build-system]\nrequires=["setuptools","wheel"]\nbuild-backend="setuptools.build_meta"', + } + ) + log = project.path / "log.json" + outcome = project.run("r", "-vv", "-e", "py", "--result-json", str(log)) + outcome.assert_success() + with log.open("rt") as file_handler: + log_report = json.load(file_handler) + + py_info = PythonInfo.current_system() + host_python = { + "executable": py_info.system_executable, + "extra_version_info": None, + "implementation": py_info.implementation, + "is_64": py_info.architecture == 64, + "sysplatform": py_info.platform, + "version": py_info.version, + "version_info": list(py_info.version_info), + } + packaging_setup = get_cmd_exit_run_id(log_report, ".package-py", "setup") + assert packaging_setup == [(0, "install"), (0, "build requires"), (0, "freeze"), (0, "package meta")] + packaging_test = get_cmd_exit_run_id(log_report, ".package-py", "test") + assert packaging_test == [(0, "build")] + packaging_installed = log_report["testenvs"][".package-py"].pop("installed_packages") + assert {i[: i.find("==")] for i in packaging_installed} == {"pip", "setuptools", "wheel"} + + py_setup = get_cmd_exit_run_id(log_report, "py", "setup") + assert py_setup == [(0, "install"), (0, "install"), (0, "freeze")] # install => 1 dep and 1 package + py_test = get_cmd_exit_run_id(log_report, "py", "test") + assert py_test == [(1, "commands[0]"), (0, "commands[1]")] + packaging_installed = log_report["testenvs"]["py"].pop("installed_packages") + expected_pkg = {"pip", "setuptools", "wheel", "a"} + assert {i[: i.find("==")] if "@" not in i else "a" for i in packaging_installed} == expected_pkg + install_package = log_report["testenvs"]["py"].pop("installpkg") + assert re.match("^[a-fA-F0-9]{64}$", install_package.pop("sha256")) + assert install_package == {"basename": "a-1.0-py3-none-any.whl", "type": "file"} + + expected = { + "reportversion": "1", + "toxversion": __version__, + "platform": sys.platform, + "testenvs": { + "py": {"python": host_python}, + ".package-py": {"python": host_python}, + }, + } + assert "host" in log_report + assert log_report.pop("host") + assert log_report == expected + + +def get_cmd_exit_run_id(report: Dict[str, Any], name: str, group: str) -> List[Tuple[int, str]]: + return [(i["retcode"], i["run_id"]) for i in report["testenvs"][name].pop(group)] -- cgit v1.2.1