diff options
author | Tristan Maat <tristan.maat@codethink.co.uk> | 2017-12-20 11:58:18 +0000 |
---|---|---|
committer | Tristan Maat <tristan.maat@codethink.co.uk> | 2018-02-07 13:03:28 +0000 |
commit | e4e713ee0860f3de5a7dec4487e01f5d1d6dd0bb (patch) | |
tree | 657df200fe258a858ca2a3188d410d982867b2b0 | |
parent | b351de2a403893e02192bfefa0705442dc6a7980 (diff) | |
download | buildstream-e4e713ee0860f3de5a7dec4487e01f5d1d6dd0bb.tar.gz |
Add test utilities for integration tests
-rw-r--r-- | .gitignore | 2 | ||||
-rw-r--r-- | tests/testutils/__init__.py | 2 | ||||
-rw-r--r-- | tests/testutils/integration.py | 40 | ||||
-rw-r--r-- | tests/testutils/runcli.py | 114 |
4 files changed, 132 insertions, 26 deletions
diff --git a/.gitignore b/.gitignore index e4b6605c0..bf6214070 100644 --- a/.gitignore +++ b/.gitignore @@ -10,7 +10,7 @@ tests/**/*.pyc .eggs # Some testing related things -cache/ +integration-cache/ tmp .coverage .coverage.* diff --git a/tests/testutils/__init__.py b/tests/testutils/__init__.py index 9fc450a28..7e5b792a2 100644 --- a/tests/testutils/__init__.py +++ b/tests/testutils/__init__.py @@ -1,3 +1,3 @@ -from .runcli import cli +from .runcli import cli, cli_integration from .repo import create_repo, ALL_REPO_KINDS from .artifactshare import create_artifact_share diff --git a/tests/testutils/integration.py b/tests/testutils/integration.py new file mode 100644 index 000000000..a8c54e574 --- /dev/null +++ b/tests/testutils/integration.py @@ -0,0 +1,40 @@ +import os + +from buildstream import _yaml + + +# Recursively call .format() on all files in the given directory. +# +# This modifies the original files. +# +def format_files(directory, *args, **kwargs): + for dirname, _, filenames in os.walk(directory): + for filename in filenames: + with open(os.path.join(dirname, filename), 'r') as f: + template = f.read() + + element = template.format(*args, **kwargs) + + with open(os.path.join(dirname, filename), 'w') as f: + f.write(element) + + +# Return a list of files relative to the given directory +def walk_dir(root): + for dirname, dirnames, filenames in os.walk(root): + # print path to all subdirectories first. + for subdirname in dirnames: + yield os.path.join(dirname, subdirname)[len(root):] + + # print path to all filenames. + for filename in filenames: + yield os.path.join(dirname, filename)[len(root):] + + +# Ensure that a directory contains the given filenames. +def assert_contains(directory, expected): + missing = set(expected) + missing.difference_update(walk_dir(directory)) + if len(missing) > 0: + raise AssertionError("Missing {} expected elements from list: {}" + .format(len(missing), missing)) diff --git a/tests/testutils/runcli.py b/tests/testutils/runcli.py index 6738efc7e..bc33a4a2a 100644 --- a/tests/testutils/runcli.py +++ b/tests/testutils/runcli.py @@ -4,6 +4,7 @@ import sys import shutil import itertools import traceback +import subprocess from contextlib import contextmanager, ExitStack from ruamel import yaml import pytest @@ -15,7 +16,7 @@ import pytest # CliRunner convenience API (click.testing module) does not support # separation of stdout/stderr. # -from _pytest.capture import MultiCapture, SysCapture +from _pytest.capture import MultiCapture, FDCapture from tests.testutils.site import IS_LINUX @@ -159,11 +160,16 @@ class Result(): class Cli(): - def __init__(self, directory, verbose=True): + def __init__(self, directory, verbose=True, default_options=None): self.directory = directory self.config = None self.verbose = verbose + if default_options is None: + default_options = [] + + self.default_options = default_options + # configure(): # # Serializes a user configuration into a buildstream.conf @@ -173,7 +179,11 @@ class Cli(): # config (dict): The user configuration to use # def configure(self, config): - self.config = config + if self.config is None: + self.config = {} + + for key, val in config.items(): + self.config[key] = val def remove_artifact_from_cache(self, project, element_name): cache_dir = os.path.join(project, 'cache', 'artifacts') @@ -199,9 +209,14 @@ class Cli(): # env (dict): Environment variables to temporarily set during the test # args (list): A list of arguments to pass buildstream # - def run(self, configure=True, project=None, silent=False, env=None, cwd=None, args=None): + def run(self, configure=True, project=None, silent=False, env=None, + cwd=None, options=None, args=None): if args is None: args = [] + if options is None: + options = [] + + options = self.default_options + options with ExitStack() as stack: bst_args = ['--no-colors'] @@ -218,6 +233,9 @@ class Cli(): if project: bst_args += ['--directory', project] + for option, value in options: + bst_args += ['--option', option, value] + bst_args += args if cwd is not None: @@ -256,29 +274,36 @@ class Cli(): exception = None exit_code = 0 - capture = MultiCapture(Capture=SysCapture) - capture.start_capturing() - - try: - cli.main(args=args or (), prog_name=cli.name, **extra) - except SystemExit as e: - if e.code != 0: - exception = e + # Temporarily redirect sys.stdin to /dev/null to ensure that + # Popen doesn't attempt to read pytest's dummy stdin. + old_stdin = sys.stdin + with open(os.devnull) as devnull: + sys.stdin = devnull - exc_info = sys.exc_info() + capture = MultiCapture(out=True, err=True, in_=False, Capture=FDCapture) + capture.start_capturing() - exit_code = e.code - if not isinstance(exit_code, int): - sys.stdout.write(str(exit_code)) - sys.stdout.write('\n') - exit_code = 1 - except Exception as e: - exception = e - exit_code = -1 - exc_info = sys.exc_info() - finally: - sys.stdout.flush() + try: + cli.main(args=args or (), prog_name=cli.name, **extra) + except SystemExit as e: + if e.code != 0: + exception = e + + exc_info = sys.exc_info() + + exit_code = e.code + if not isinstance(exit_code, int): + sys.stdout.write(str(exit_code)) + sys.stdout.write('\n') + exit_code = 1 + except Exception as e: + exception = e + exit_code = -1 + exc_info = sys.exc_info() + finally: + sys.stdout.flush() + sys.stdin = old_stdin out, err = capture.readouterr() capture.stop_capturing() @@ -343,6 +368,22 @@ class Cli(): return result.output.splitlines() +class CliIntegration(Cli): + def run(self, *args, **kwargs): + + # Set the project_dir variable in our project.conf for + # relative tar imports + project_conf = os.path.join(kwargs['project'], 'project.conf') + + with open(project_conf) as f: + config = f.read() + config = config.format(project_dir=kwargs['project']) + with open(project_conf, 'w') as f: + f.write(config) + + return super().run(*args, **kwargs) + + # Main fixture # # Use result = cli.run([arg1, arg2]) to run buildstream commands @@ -354,6 +395,31 @@ def cli(tmpdir): return Cli(directory) +# A variant of the main fixture that keeps persistent artifact and +# source caches. +# +# It also does not use the click test runner to avoid deadlock issues +# when running `bst shell`, but unfortunately cannot produce nice +# stacktraces. +@pytest.fixture() +def cli_integration(tmpdir): + directory = os.path.join(str(tmpdir), 'cache') + os.makedirs(directory) + + if os.environ.get('BST_FORCE_BACKEND') == 'unix': + fixture = CliIntegration(directory, default_options=[('linux', 'False')]) + else: + fixture = CliIntegration(directory) + + # We want to cache sources for integration tests more permanently, + # to avoid downloading the huge base-sdk repeatedly + fixture.configure({ + 'sourcedir': os.path.join(os.getcwd(), 'integration-cache', 'sources') + }) + + return fixture + + @contextmanager def chdir(directory): old_dir = os.getcwd() |