diff options
author | Phil Dawson <phil.dawson@codethink.co.uk> | 2019-03-26 09:49:17 +0000 |
---|---|---|
committer | Phil Dawson <phil.dawson@codethink.co.uk> | 2019-04-12 16:06:19 +0100 |
commit | 16a28c0316a64189cb8c396c43b570764a83d33b (patch) | |
tree | b0cf6a513b6f3da080fc5f5563625e624f0548d1 /tests | |
parent | c30f4a56e1f802f338e4b248b2ba925e479d3418 (diff) | |
download | buildstream-16a28c0316a64189cb8c396c43b570764a83d33b.tar.gz |
Make templated source tests available in buildstream.plugintestutils
Diffstat (limited to 'tests')
38 files changed, 25 insertions, 1657 deletions
diff --git a/tests/conftest.py b/tests/conftest.py index 30577870f..fefd2f755 100755 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 # # Copyright (C) 2018 Codethink Limited +# Copyright (C) 2019 Bloomberg Finance LLP # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public @@ -23,6 +24,14 @@ import shutil import tempfile import pytest from buildstream._platform.platform import Platform +from buildstream.plugintestutils import register_repo_kind, sourcetests_collection_hook + +from tests.testutils.repo.git import Git +from tests.testutils.repo.bzr import Bzr +from tests.testutils.repo.ostree import OSTree +from tests.testutils.repo.tar import Tar +from tests.testutils.repo.zip import Zip + # # This file is loaded by pytest, we use it to add a custom @@ -155,3 +164,19 @@ def clean_platform_cache(): @pytest.fixture(autouse=True) def ensure_platform_cache_is_clean(): clean_platform_cache() + + +################################################# +# Setup for templated source tests # +################################################# +register_repo_kind('git', Git) +register_repo_kind('bzr', Bzr) +register_repo_kind('ostree', OSTree) +register_repo_kind('tar', Tar) +register_repo_kind('zip', Zip) + + +# This hook enables pytest to collect the templated source tests from +# buildstream.plugintestutils +def pytest_sessionstart(session): + sourcetests_collection_hook(session) diff --git a/tests/sources/__init__.py b/tests/sources/__init__.py index 155ecc064..02a755074 100644 --- a/tests/sources/__init__.py +++ b/tests/sources/__init__.py @@ -1,18 +1,5 @@ import os -from buildstream.plugintestutils import register_repo_kind -from tests.testutils.repo.git import Git -from tests.testutils.repo.bzr import Bzr -from tests.testutils.repo.ostree import OSTree -from tests.testutils.repo.tar import Tar -from tests.testutils.repo.zip import Zip - -register_repo_kind('git', Git) -register_repo_kind('bzr', Bzr) -register_repo_kind('ostree', OSTree) -register_repo_kind('tar', Tar) -register_repo_kind('zip', Zip) - def list_dir_contents(srcdir): contents = set() diff --git a/tests/sources/generic/__init__.py b/tests/sources/generic/__init__.py deleted file mode 100644 index e69de29bb..000000000 --- a/tests/sources/generic/__init__.py +++ /dev/null diff --git a/tests/sources/generic/build_checkout.py b/tests/sources/generic/build_checkout.py deleted file mode 100644 index 074be8429..000000000 --- a/tests/sources/generic/build_checkout.py +++ /dev/null @@ -1,83 +0,0 @@ -# -# Copyright (C) 2018 Codethink Limited -# Copyright (C) 2019 Bloomberg Finance LP -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see <http://www.gnu.org/licenses/>. -# - -# Pylint doesn't play well with fixtures and dependency injection from pytest -# pylint: disable=redefined-outer-name - -import os -import pytest - -from buildstream.plugintestutils import create_repo, ALL_REPO_KINDS -from buildstream.plugintestutils import cli # pylint: disable=unused-import -from buildstream import _yaml - -# Project directory -TOP_DIR = os.path.dirname(os.path.realpath(__file__)) -DATA_DIR = os.path.join(TOP_DIR, 'project') - -fetch_build_checkout_combos = \ - [("strict", kind) for kind in ALL_REPO_KINDS] + \ - [("non-strict", kind) for kind in ALL_REPO_KINDS] - - -def strict_args(args, strict): - if strict != "strict": - return ['--no-strict', *args] - return args - - -@pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("strict,kind", fetch_build_checkout_combos) -def test_fetch_build_checkout(cli, tmpdir, datafiles, strict, kind): - checkout = os.path.join(cli.directory, 'checkout') - project = str(datafiles) - dev_files_path = os.path.join(project, 'files', 'dev-files') - element_path = os.path.join(project, 'elements') - element_name = 'build-test-{}.bst'.format(kind) - - # Create our repo object of the given source type with - # the dev files, and then collect the initial ref. - # - repo = create_repo(kind, str(tmpdir)) - ref = repo.create(dev_files_path) - - # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - repo.source_config(ref=ref) - ] - } - _yaml.dump(element, - os.path.join(element_path, - element_name)) - - assert cli.get_element_state(project, element_name) == 'fetch needed' - result = cli.run(project=project, args=strict_args(['build', element_name], strict)) - result.assert_success() - assert cli.get_element_state(project, element_name) == 'cached' - - # Now check it out - result = cli.run(project=project, args=strict_args([ - 'artifact', 'checkout', element_name, '--directory', checkout - ], strict)) - result.assert_success() - - # Check that the pony.h include from files/dev-files exists - filename = os.path.join(checkout, 'usr', 'include', 'pony.h') - assert os.path.exists(filename) diff --git a/tests/sources/generic/fetch.py b/tests/sources/generic/fetch.py deleted file mode 100644 index 52c853398..000000000 --- a/tests/sources/generic/fetch.py +++ /dev/null @@ -1,110 +0,0 @@ -# -# Copyright (C) 2018 Codethink Limited -# Copyright (C) 2019 Bloomberg Finance LP -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see <http://www.gnu.org/licenses/>. -# - -# Pylint doesn't play well with fixtures and dependency injection from pytest -# pylint: disable=redefined-outer-name - -import os -import pytest - -from .._utils import generate_junction -from buildstream.plugintestutils import create_repo, ALL_REPO_KINDS -from tests.frontend import configure_project - -from buildstream.plugintestutils import cli # pylint: disable=unused-import -from buildstream import _yaml - - -# Project directory -TOP_DIR = os.path.dirname(os.path.realpath(__file__)) -DATA_DIR = os.path.join(TOP_DIR, 'project') - - -@pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) -def test_fetch(cli, tmpdir, datafiles, kind): - project = str(datafiles) - bin_files_path = os.path.join(project, 'files', 'bin-files') - element_path = os.path.join(project, 'elements') - element_name = 'fetch-test-{}.bst'.format(kind) - - # Create our repo object of the given source type with - # the bin files, and then collect the initial ref. - # - repo = create_repo(kind, str(tmpdir)) - ref = repo.create(bin_files_path) - - # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - repo.source_config(ref=ref) - ] - } - _yaml.dump(element, - os.path.join(element_path, - element_name)) - - # Assert that a fetch is needed - assert cli.get_element_state(project, element_name) == 'fetch needed' - - # Now try to fetch it - result = cli.run(project=project, args=['source', 'fetch', element_name]) - result.assert_success() - - # Assert that we are now buildable because the source is - # now cached. - assert cli.get_element_state(project, element_name) == 'buildable' - - -@pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) -@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) -def test_fetch_cross_junction(cli, tmpdir, datafiles, ref_storage, kind): - project = str(datafiles) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') - - import_etc_path = os.path.join(subproject_path, 'elements', 'import-etc-repo.bst') - etc_files_path = os.path.join(subproject_path, 'files', 'etc-files') - - repo = create_repo(kind, str(tmpdir.join('import-etc'))) - ref = repo.create(etc_files_path) - - element = { - 'kind': 'import', - 'sources': [ - repo.source_config(ref=(ref if ref_storage == 'inline' else None)) - ] - } - _yaml.dump(element, import_etc_path) - - configure_project(project, { - 'ref-storage': ref_storage - }) - - generate_junction(tmpdir, subproject_path, junction_path, store_ref=(ref_storage == 'inline')) - - if ref_storage == 'project.refs': - result = cli.run(project=project, args=['source', 'track', 'junction.bst']) - result.assert_success() - result = cli.run(project=project, args=['source', 'track', 'junction.bst:import-etc.bst']) - result.assert_success() - - result = cli.run(project=project, args=['source', 'fetch', 'junction.bst:import-etc.bst']) - result.assert_success() diff --git a/tests/sources/generic/mirror.py b/tests/sources/generic/mirror.py deleted file mode 100644 index 5315d184f..000000000 --- a/tests/sources/generic/mirror.py +++ /dev/null @@ -1,428 +0,0 @@ -# -# Copyright (C) 2018 Codethink Limited -# Copyright (C) 2019 Bloomberg Finance LP -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see <http://www.gnu.org/licenses/>. -# - -# Pylint doesn't play well with fixtures and dependency injection from pytest -# pylint: disable=redefined-outer-name - -import os -import pytest - -from .._utils import generate_junction - -from buildstream.plugintestutils import create_repo, ALL_REPO_KINDS -from buildstream.plugintestutils import cli # pylint: disable=unused-import -from buildstream import _yaml -from buildstream._exceptions import ErrorDomain - -# Project directory -TOP_DIR = os.path.dirname(os.path.realpath(__file__)) -DATA_DIR = os.path.join(TOP_DIR, 'project') - - -@pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) -def test_mirror_fetch(cli, tmpdir, datafiles, kind): - bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr') - dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr') - upstream_repodir = os.path.join(str(tmpdir), 'upstream') - mirror_repodir = os.path.join(str(tmpdir), 'mirror') - project_dir = os.path.join(str(tmpdir), 'project') - os.makedirs(project_dir) - element_dir = os.path.join(project_dir, 'elements') - - # Create repo objects of the upstream and mirror - upstream_repo = create_repo(kind, upstream_repodir) - upstream_repo.create(bin_files_path) - mirror_repo = upstream_repo.copy(mirror_repodir) - upstream_ref = upstream_repo.create(dev_files_path) - - element = { - 'kind': 'import', - 'sources': [ - upstream_repo.source_config(ref=upstream_ref) - ] - } - element_name = 'test.bst' - element_path = os.path.join(element_dir, element_name) - full_repo = element['sources'][0]['url'] - upstream_map, repo_name = os.path.split(full_repo) - alias = 'foo-' + kind - aliased_repo = alias + ':' + repo_name - element['sources'][0]['url'] = aliased_repo - full_mirror = mirror_repo.source_config()['url'] - mirror_map, _ = os.path.split(full_mirror) - os.makedirs(element_dir) - _yaml.dump(element, element_path) - - project = { - 'name': 'test', - 'element-path': 'elements', - 'aliases': { - alias: upstream_map + "/" - }, - 'mirrors': [ - { - 'name': 'middle-earth', - 'aliases': { - alias: [mirror_map + "/"], - }, - }, - ] - } - project_file = os.path.join(project_dir, 'project.conf') - _yaml.dump(project, project_file) - - # No obvious ways of checking that the mirror has been fetched - # But at least we can be sure it succeeds - result = cli.run(project=project_dir, args=['source', 'fetch', element_name]) - result.assert_success() - - -@pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) -def test_mirror_fetch_upstream_absent(cli, tmpdir, datafiles, kind): - dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr') - upstream_repodir = os.path.join(str(tmpdir), 'upstream') - mirror_repodir = os.path.join(str(tmpdir), 'mirror') - project_dir = os.path.join(str(tmpdir), 'project') - os.makedirs(project_dir) - element_dir = os.path.join(project_dir, 'elements') - - # Create repo objects of the upstream and mirror - upstream_repo = create_repo(kind, upstream_repodir) - ref = upstream_repo.create(dev_files_path) - mirror_repo = upstream_repo.copy(mirror_repodir) - - element = { - 'kind': 'import', - 'sources': [ - upstream_repo.source_config(ref=ref) - ] - } - - element_name = 'test.bst' - element_path = os.path.join(element_dir, element_name) - full_repo = element['sources'][0]['url'] - _, repo_name = os.path.split(full_repo) - alias = 'foo-' + kind - aliased_repo = alias + ':' + repo_name - element['sources'][0]['url'] = aliased_repo - full_mirror = mirror_repo.source_config()['url'] - mirror_map, _ = os.path.split(full_mirror) - os.makedirs(element_dir) - _yaml.dump(element, element_path) - - project = { - 'name': 'test', - 'element-path': 'elements', - 'aliases': { - alias: 'http://www.example.com/' - }, - 'mirrors': [ - { - 'name': 'middle-earth', - 'aliases': { - alias: [mirror_map + "/"], - }, - }, - ] - } - project_file = os.path.join(project_dir, 'project.conf') - _yaml.dump(project, project_file) - - result = cli.run(project=project_dir, args=['source', 'fetch', element_name]) - result.assert_success() - - -@pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) -def test_mirror_from_includes(cli, tmpdir, datafiles, kind): - bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr') - upstream_repodir = os.path.join(str(tmpdir), 'upstream') - mirror_repodir = os.path.join(str(tmpdir), 'mirror') - project_dir = os.path.join(str(tmpdir), 'project') - os.makedirs(project_dir) - element_dir = os.path.join(project_dir, 'elements') - - # Create repo objects of the upstream and mirror - upstream_repo = create_repo(kind, upstream_repodir) - upstream_ref = upstream_repo.create(bin_files_path) - mirror_repo = upstream_repo.copy(mirror_repodir) - - element = { - 'kind': 'import', - 'sources': [ - upstream_repo.source_config(ref=upstream_ref) - ] - } - element_name = 'test.bst' - element_path = os.path.join(element_dir, element_name) - full_repo = element['sources'][0]['url'] - upstream_map, repo_name = os.path.split(full_repo) - alias = 'foo-' + kind - aliased_repo = alias + ':' + repo_name - element['sources'][0]['url'] = aliased_repo - full_mirror = mirror_repo.source_config()['url'] - mirror_map, _ = os.path.split(full_mirror) - os.makedirs(element_dir) - _yaml.dump(element, element_path) - - config_project_dir = str(tmpdir.join('config')) - os.makedirs(config_project_dir, exist_ok=True) - config_project = { - 'name': 'config' - } - _yaml.dump(config_project, os.path.join(config_project_dir, 'project.conf')) - extra_mirrors = { - 'mirrors': [ - { - 'name': 'middle-earth', - 'aliases': { - alias: [mirror_map + "/"], - } - } - ] - } - _yaml.dump(extra_mirrors, os.path.join(config_project_dir, 'mirrors.yml')) - generate_junction(str(tmpdir.join('config_repo')), - config_project_dir, - os.path.join(element_dir, 'config.bst')) - - project = { - 'name': 'test', - 'element-path': 'elements', - 'aliases': { - alias: upstream_map + "/" - }, - '(@)': [ - 'config.bst:mirrors.yml' - ] - } - project_file = os.path.join(project_dir, 'project.conf') - _yaml.dump(project, project_file) - - # Now make the upstream unavailable. - os.rename(upstream_repo.repo, '{}.bak'.format(upstream_repo.repo)) - result = cli.run(project=project_dir, args=['source', 'fetch', element_name]) - result.assert_success() - - -@pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) -def test_mirror_junction_from_includes(cli, tmpdir, datafiles, kind): - bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr') - upstream_repodir = os.path.join(str(tmpdir), 'upstream') - mirror_repodir = os.path.join(str(tmpdir), 'mirror') - project_dir = os.path.join(str(tmpdir), 'project') - os.makedirs(project_dir) - element_dir = os.path.join(project_dir, 'elements') - - # Create repo objects of the upstream and mirror - upstream_repo = create_repo(kind, upstream_repodir) - upstream_ref = upstream_repo.create(bin_files_path) - mirror_repo = upstream_repo.copy(mirror_repodir) - - element = { - 'kind': 'junction', - 'sources': [ - upstream_repo.source_config(ref=upstream_ref) - ] - } - element_name = 'test.bst' - element_path = os.path.join(element_dir, element_name) - full_repo = element['sources'][0]['url'] - upstream_map, repo_name = os.path.split(full_repo) - alias = 'foo-' + kind - aliased_repo = alias + ':' + repo_name - element['sources'][0]['url'] = aliased_repo - full_mirror = mirror_repo.source_config()['url'] - mirror_map, _ = os.path.split(full_mirror) - os.makedirs(element_dir) - _yaml.dump(element, element_path) - - config_project_dir = str(tmpdir.join('config')) - os.makedirs(config_project_dir, exist_ok=True) - config_project = { - 'name': 'config' - } - _yaml.dump(config_project, os.path.join(config_project_dir, 'project.conf')) - extra_mirrors = { - 'mirrors': [ - { - 'name': 'middle-earth', - 'aliases': { - alias: [mirror_map + "/"], - } - } - ] - } - _yaml.dump(extra_mirrors, os.path.join(config_project_dir, 'mirrors.yml')) - generate_junction(str(tmpdir.join('config_repo')), - config_project_dir, - os.path.join(element_dir, 'config.bst')) - - project = { - 'name': 'test', - 'element-path': 'elements', - 'aliases': { - alias: upstream_map + "/" - }, - '(@)': [ - 'config.bst:mirrors.yml' - ] - } - project_file = os.path.join(project_dir, 'project.conf') - _yaml.dump(project, project_file) - - # Now make the upstream unavailable. - os.rename(upstream_repo.repo, '{}.bak'.format(upstream_repo.repo)) - result = cli.run(project=project_dir, args=['source', 'fetch', element_name]) - result.assert_main_error(ErrorDomain.STREAM, None) - # Now make the upstream available again. - os.rename('{}.bak'.format(upstream_repo.repo), upstream_repo.repo) - result = cli.run(project=project_dir, args=['source', 'fetch', element_name]) - result.assert_success() - - -@pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) -def test_mirror_track_upstream_present(cli, tmpdir, datafiles, kind): - bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr') - dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr') - upstream_repodir = os.path.join(str(tmpdir), 'upstream') - mirror_repodir = os.path.join(str(tmpdir), 'mirror') - project_dir = os.path.join(str(tmpdir), 'project') - os.makedirs(project_dir) - element_dir = os.path.join(project_dir, 'elements') - - # Create repo objects of the upstream and mirror - upstream_repo = create_repo(kind, upstream_repodir) - upstream_repo.create(bin_files_path) - mirror_repo = upstream_repo.copy(mirror_repodir) - upstream_ref = upstream_repo.create(dev_files_path) - - element = { - 'kind': 'import', - 'sources': [ - upstream_repo.source_config(ref=upstream_ref) - ] - } - - element_name = 'test.bst' - element_path = os.path.join(element_dir, element_name) - full_repo = element['sources'][0]['url'] - upstream_map, repo_name = os.path.split(full_repo) - alias = 'foo-' + kind - aliased_repo = alias + ':' + repo_name - element['sources'][0]['url'] = aliased_repo - full_mirror = mirror_repo.source_config()['url'] - mirror_map, _ = os.path.split(full_mirror) - os.makedirs(element_dir) - _yaml.dump(element, element_path) - - project = { - 'name': 'test', - 'element-path': 'elements', - 'aliases': { - alias: upstream_map + "/" - }, - 'mirrors': [ - { - 'name': 'middle-earth', - 'aliases': { - alias: [mirror_map + "/"], - }, - }, - ] - } - project_file = os.path.join(project_dir, 'project.conf') - _yaml.dump(project, project_file) - - result = cli.run(project=project_dir, args=['source', 'track', element_name]) - result.assert_success() - - # Tracking tries upstream first. Check the ref is from upstream. - new_element = _yaml.load(element_path) - source = _yaml.node_get(new_element, dict, 'sources', [0]) - if 'ref' in source: - assert _yaml.node_get(source, str, 'ref') == upstream_ref - - -@pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) -def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind): - bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr') - dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr') - upstream_repodir = os.path.join(str(tmpdir), 'upstream') - mirror_repodir = os.path.join(str(tmpdir), 'mirror') - project_dir = os.path.join(str(tmpdir), 'project') - os.makedirs(project_dir) - element_dir = os.path.join(project_dir, 'elements') - - # Create repo objects of the upstream and mirror - upstream_repo = create_repo(kind, upstream_repodir) - upstream_ref = upstream_repo.create(bin_files_path) - mirror_repo = upstream_repo.copy(mirror_repodir) - mirror_ref = upstream_ref - upstream_ref = upstream_repo.create(dev_files_path) - - element = { - 'kind': 'import', - 'sources': [ - upstream_repo.source_config(ref=upstream_ref) - ] - } - - element_name = 'test.bst' - element_path = os.path.join(element_dir, element_name) - full_repo = element['sources'][0]['url'] - _, repo_name = os.path.split(full_repo) - alias = 'foo-' + kind - aliased_repo = alias + ':' + repo_name - element['sources'][0]['url'] = aliased_repo - full_mirror = mirror_repo.source_config()['url'] - mirror_map, _ = os.path.split(full_mirror) - os.makedirs(element_dir) - _yaml.dump(element, element_path) - - project = { - 'name': 'test', - 'element-path': 'elements', - 'aliases': { - alias: 'http://www.example.com/' - }, - 'mirrors': [ - { - 'name': 'middle-earth', - 'aliases': { - alias: [mirror_map + "/"], - }, - }, - ] - } - project_file = os.path.join(project_dir, 'project.conf') - _yaml.dump(project, project_file) - - result = cli.run(project=project_dir, args=['source', 'track', element_name]) - result.assert_success() - - # Check that tracking fell back to the mirror - new_element = _yaml.load(element_path) - source = _yaml.node_get(new_element, dict, 'sources', [0]) - if 'ref' in source: - assert _yaml.node_get(source, str, 'ref') == mirror_ref diff --git a/tests/sources/generic/project/elements/base.bst b/tests/sources/generic/project/elements/base.bst deleted file mode 100644 index 428afa736..000000000 --- a/tests/sources/generic/project/elements/base.bst +++ /dev/null @@ -1,5 +0,0 @@ -# elements/base.bst - -kind: stack -depends: - - base/base-alpine.bst diff --git a/tests/sources/generic/project/elements/base/base-alpine.bst b/tests/sources/generic/project/elements/base/base-alpine.bst deleted file mode 100644 index c5833095d..000000000 --- a/tests/sources/generic/project/elements/base/base-alpine.bst +++ /dev/null @@ -1,17 +0,0 @@ -kind: import - -description: | - Alpine Linux base for tests - - Generated using the `tests/integration-tests/base/generate-base.sh` script. - -sources: - - kind: tar - base-dir: '' - (?): - - arch == "x86-64": - ref: 3eb559250ba82b64a68d86d0636a6b127aa5f6d25d3601a79f79214dc9703639 - url: "alpine:integration-tests-base.v1.x86_64.tar.xz" - - arch == "aarch64": - ref: 431fb5362032ede6f172e70a3258354a8fd71fcbdeb1edebc0e20968c792329a - url: "alpine:integration-tests-base.v1.aarch64.tar.xz" diff --git a/tests/sources/generic/project/elements/import-bin.bst b/tests/sources/generic/project/elements/import-bin.bst deleted file mode 100644 index a847c0c23..000000000 --- a/tests/sources/generic/project/elements/import-bin.bst +++ /dev/null @@ -1,4 +0,0 @@ -kind: import -sources: -- kind: local - path: files/bin-files diff --git a/tests/sources/generic/project/elements/import-dev.bst b/tests/sources/generic/project/elements/import-dev.bst deleted file mode 100644 index 152a54667..000000000 --- a/tests/sources/generic/project/elements/import-dev.bst +++ /dev/null @@ -1,4 +0,0 @@ -kind: import -sources: -- kind: local - path: files/dev-files diff --git a/tests/sources/generic/project/elements/multiple_targets/dependency/horsey.bst b/tests/sources/generic/project/elements/multiple_targets/dependency/horsey.bst deleted file mode 100644 index bd1ffae9c..000000000 --- a/tests/sources/generic/project/elements/multiple_targets/dependency/horsey.bst +++ /dev/null @@ -1,3 +0,0 @@ -kind: autotools -depends: - - multiple_targets/dependency/pony.bst diff --git a/tests/sources/generic/project/elements/multiple_targets/dependency/pony.bst b/tests/sources/generic/project/elements/multiple_targets/dependency/pony.bst deleted file mode 100644 index 3c29b4ea1..000000000 --- a/tests/sources/generic/project/elements/multiple_targets/dependency/pony.bst +++ /dev/null @@ -1 +0,0 @@ -kind: autotools diff --git a/tests/sources/generic/project/elements/multiple_targets/dependency/zebry.bst b/tests/sources/generic/project/elements/multiple_targets/dependency/zebry.bst deleted file mode 100644 index 98447ab52..000000000 --- a/tests/sources/generic/project/elements/multiple_targets/dependency/zebry.bst +++ /dev/null @@ -1,3 +0,0 @@ -kind: autotools -depends: - - multiple_targets/dependency/horsey.bst diff --git a/tests/sources/generic/project/elements/multiple_targets/order/0.bst b/tests/sources/generic/project/elements/multiple_targets/order/0.bst deleted file mode 100644 index a99be06a0..000000000 --- a/tests/sources/generic/project/elements/multiple_targets/order/0.bst +++ /dev/null @@ -1,7 +0,0 @@ -kind: autotools -description: Root node -depends: - - multiple_targets/order/2.bst - - multiple_targets/order/3.bst - - filename: multiple_targets/order/run.bst - type: runtime diff --git a/tests/sources/generic/project/elements/multiple_targets/order/1.bst b/tests/sources/generic/project/elements/multiple_targets/order/1.bst deleted file mode 100644 index 82b507a62..000000000 --- a/tests/sources/generic/project/elements/multiple_targets/order/1.bst +++ /dev/null @@ -1,4 +0,0 @@ -kind: autotools -description: Root node -depends: - - multiple_targets/order/9.bst diff --git a/tests/sources/generic/project/elements/multiple_targets/order/2.bst b/tests/sources/generic/project/elements/multiple_targets/order/2.bst deleted file mode 100644 index ee1afae20..000000000 --- a/tests/sources/generic/project/elements/multiple_targets/order/2.bst +++ /dev/null @@ -1,4 +0,0 @@ -kind: autotools -description: First dependency level -depends: - - multiple_targets/order/3.bst diff --git a/tests/sources/generic/project/elements/multiple_targets/order/3.bst b/tests/sources/generic/project/elements/multiple_targets/order/3.bst deleted file mode 100644 index 4c3a23dab..000000000 --- a/tests/sources/generic/project/elements/multiple_targets/order/3.bst +++ /dev/null @@ -1,6 +0,0 @@ -kind: autotools -description: Second dependency level -depends: - - multiple_targets/order/4.bst - - multiple_targets/order/5.bst - - multiple_targets/order/6.bst diff --git a/tests/sources/generic/project/elements/multiple_targets/order/4.bst b/tests/sources/generic/project/elements/multiple_targets/order/4.bst deleted file mode 100644 index b663a0b52..000000000 --- a/tests/sources/generic/project/elements/multiple_targets/order/4.bst +++ /dev/null @@ -1,2 +0,0 @@ -kind: autotools -description: Third level dependency diff --git a/tests/sources/generic/project/elements/multiple_targets/order/5.bst b/tests/sources/generic/project/elements/multiple_targets/order/5.bst deleted file mode 100644 index b9efcf71b..000000000 --- a/tests/sources/generic/project/elements/multiple_targets/order/5.bst +++ /dev/null @@ -1,2 +0,0 @@ -kind: autotools -description: Fifth level dependency diff --git a/tests/sources/generic/project/elements/multiple_targets/order/6.bst b/tests/sources/generic/project/elements/multiple_targets/order/6.bst deleted file mode 100644 index 6c19d04e3..000000000 --- a/tests/sources/generic/project/elements/multiple_targets/order/6.bst +++ /dev/null @@ -1,4 +0,0 @@ -kind: autotools -description: Fourth level dependency -depends: - - multiple_targets/order/5.bst diff --git a/tests/sources/generic/project/elements/multiple_targets/order/7.bst b/tests/sources/generic/project/elements/multiple_targets/order/7.bst deleted file mode 100644 index 6805b3e6d..000000000 --- a/tests/sources/generic/project/elements/multiple_targets/order/7.bst +++ /dev/null @@ -1,4 +0,0 @@ -kind: autotools -description: Third level dependency -depends: - - multiple_targets/order/6.bst diff --git a/tests/sources/generic/project/elements/multiple_targets/order/8.bst b/tests/sources/generic/project/elements/multiple_targets/order/8.bst deleted file mode 100644 index b8d8964a0..000000000 --- a/tests/sources/generic/project/elements/multiple_targets/order/8.bst +++ /dev/null @@ -1,4 +0,0 @@ -kind: autotools -description: Second level dependency -depends: - - multiple_targets/order/7.bst diff --git a/tests/sources/generic/project/elements/multiple_targets/order/9.bst b/tests/sources/generic/project/elements/multiple_targets/order/9.bst deleted file mode 100644 index cc13bf3f0..000000000 --- a/tests/sources/generic/project/elements/multiple_targets/order/9.bst +++ /dev/null @@ -1,4 +0,0 @@ -kind: autotools -description: First level dependency -depends: - - multiple_targets/order/8.bst diff --git a/tests/sources/generic/project/elements/multiple_targets/order/run.bst b/tests/sources/generic/project/elements/multiple_targets/order/run.bst deleted file mode 100644 index 9b3d2446c..000000000 --- a/tests/sources/generic/project/elements/multiple_targets/order/run.bst +++ /dev/null @@ -1,2 +0,0 @@ -kind: autotools -description: Not a root node, yet built at the same time as root nodes diff --git a/tests/sources/generic/project/files/bar b/tests/sources/generic/project/files/bar deleted file mode 100644 index e69de29bb..000000000 --- a/tests/sources/generic/project/files/bar +++ /dev/null diff --git a/tests/sources/generic/project/files/bin-files/usr/bin/hello b/tests/sources/generic/project/files/bin-files/usr/bin/hello deleted file mode 100755 index f534a4083..000000000 --- a/tests/sources/generic/project/files/bin-files/usr/bin/hello +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -echo "Hello !" diff --git a/tests/sources/generic/project/files/dev-files/usr/include/pony.h b/tests/sources/generic/project/files/dev-files/usr/include/pony.h deleted file mode 100644 index 40bd0c2e7..000000000 --- a/tests/sources/generic/project/files/dev-files/usr/include/pony.h +++ /dev/null @@ -1,12 +0,0 @@ -#ifndef __PONY_H__ -#define __PONY_H__ - -#define PONY_BEGIN "Once upon a time, there was a pony." -#define PONY_END "And they lived happily ever after, the end." - -#define MAKE_PONY(story) \ - PONY_BEGIN \ - story \ - PONY_END - -#endif /* __PONY_H__ */ diff --git a/tests/sources/generic/project/files/etc-files/etc/buildstream/config b/tests/sources/generic/project/files/etc-files/etc/buildstream/config deleted file mode 100644 index 04204c7c9..000000000 --- a/tests/sources/generic/project/files/etc-files/etc/buildstream/config +++ /dev/null @@ -1 +0,0 @@ -config diff --git a/tests/sources/generic/project/files/foo b/tests/sources/generic/project/files/foo deleted file mode 100644 index e69de29bb..000000000 --- a/tests/sources/generic/project/files/foo +++ /dev/null diff --git a/tests/sources/generic/project/files/source-bundle/llamas.txt b/tests/sources/generic/project/files/source-bundle/llamas.txt deleted file mode 100644 index f98b24871..000000000 --- a/tests/sources/generic/project/files/source-bundle/llamas.txt +++ /dev/null @@ -1 +0,0 @@ -llamas diff --git a/tests/sources/generic/project/files/sub-project/elements/import-etc.bst b/tests/sources/generic/project/files/sub-project/elements/import-etc.bst deleted file mode 100644 index f0171990e..000000000 --- a/tests/sources/generic/project/files/sub-project/elements/import-etc.bst +++ /dev/null @@ -1,4 +0,0 @@ -kind: import -sources: -- kind: local - path: files/etc-files diff --git a/tests/sources/generic/project/files/sub-project/files/etc-files/etc/animal.conf b/tests/sources/generic/project/files/sub-project/files/etc-files/etc/animal.conf deleted file mode 100644 index db8c36cba..000000000 --- a/tests/sources/generic/project/files/sub-project/files/etc-files/etc/animal.conf +++ /dev/null @@ -1 +0,0 @@ -animal=Pony diff --git a/tests/sources/generic/project/files/sub-project/project.conf b/tests/sources/generic/project/files/sub-project/project.conf deleted file mode 100644 index bbb8414a3..000000000 --- a/tests/sources/generic/project/files/sub-project/project.conf +++ /dev/null @@ -1,4 +0,0 @@ -# Project config for frontend build test -name: subtest - -element-path: elements diff --git a/tests/sources/generic/project/project.conf b/tests/sources/generic/project/project.conf deleted file mode 100644 index 05b68bfeb..000000000 --- a/tests/sources/generic/project/project.conf +++ /dev/null @@ -1,27 +0,0 @@ -# Project config for frontend build test -name: test -element-path: elements -aliases: - alpine: https://bst-integration-test-images.ams3.cdn.digitaloceanspaces.com/ - project_dir: file://{project_dir} -options: - linux: - type: bool - description: Whether to expect a linux platform - default: True - arch: - type: arch - description: Current architecture - values: - - x86-64 - - aarch64 -split-rules: - test: - - | - /tests - - | - /tests/* - -fatal-warnings: -- bad-element-suffix -- bad-characters-in-name diff --git a/tests/sources/generic/source_determinism.py b/tests/sources/generic/source_determinism.py deleted file mode 100644 index bd83da007..000000000 --- a/tests/sources/generic/source_determinism.py +++ /dev/null @@ -1,119 +0,0 @@ -# -# Copyright (C) 2018 Codethink Limited -# Copyright (C) 2019 Bloomberg Finance LP -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see <http://www.gnu.org/licenses/>. -# - -# Pylint doesn't play well with fixtures and dependency injection from pytest -# pylint: disable=redefined-outer-name - -import os -import pytest - -from .._utils.site import HAVE_SANDBOX - -from buildstream.plugintestutils import create_repo, ALL_REPO_KINDS -from buildstream.plugintestutils import cli # pylint: disable=unused-import -from buildstream import _yaml - -# Project directory -TOP_DIR = os.path.dirname(os.path.realpath(__file__)) -DATA_DIR = os.path.join(TOP_DIR, 'project') - - -def create_test_file(*path, mode=0o644, content='content\n'): - path = os.path.join(*path) - os.makedirs(os.path.dirname(path), exist_ok=True) - with open(path, 'w') as f: - f.write(content) - os.fchmod(f.fileno(), mode) - - -def create_test_directory(*path, mode=0o644): - create_test_file(*path, '.keep', content='') - path = os.path.join(*path) - os.chmod(path, mode) - - -@pytest.mark.integration -@pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("kind", ['local', *ALL_REPO_KINDS]) -@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') -def test_deterministic_source_umask(cli, tmpdir, datafiles, kind): - project = str(datafiles) - element_name = 'list.bst' - element_path = os.path.join(project, 'elements', element_name) - repodir = os.path.join(str(tmpdir), 'repo') - sourcedir = os.path.join(project, 'source') - - create_test_file(sourcedir, 'a.txt', mode=0o700) - create_test_file(sourcedir, 'b.txt', mode=0o755) - create_test_file(sourcedir, 'c.txt', mode=0o600) - create_test_file(sourcedir, 'd.txt', mode=0o400) - create_test_file(sourcedir, 'e.txt', mode=0o644) - create_test_file(sourcedir, 'f.txt', mode=0o4755) - create_test_file(sourcedir, 'g.txt', mode=0o2755) - create_test_file(sourcedir, 'h.txt', mode=0o1755) - create_test_directory(sourcedir, 'dir-a', mode=0o0700) - create_test_directory(sourcedir, 'dir-c', mode=0o0755) - create_test_directory(sourcedir, 'dir-d', mode=0o4755) - create_test_directory(sourcedir, 'dir-e', mode=0o2755) - create_test_directory(sourcedir, 'dir-f', mode=0o1755) - - if kind == 'local': - source = {'kind': 'local', - 'path': 'source'} - else: - repo = create_repo(kind, repodir) - ref = repo.create(sourcedir) - source = repo.source_config(ref=ref) - element = { - 'kind': 'manual', - 'depends': [ - { - 'filename': 'base.bst', - 'type': 'build' - } - ], - 'sources': [ - source - ], - 'config': { - 'install-commands': [ - 'ls -l >"%{install-root}/ls-l"' - ] - } - } - _yaml.dump(element, element_path) - - def get_value_for_umask(umask): - checkoutdir = os.path.join(str(tmpdir), 'checkout-{}'.format(umask)) - - old_umask = os.umask(umask) - - try: - result = cli.run(project=project, args=['build', element_name]) - result.assert_success() - - result = cli.run(project=project, args=['artifact', 'checkout', element_name, '--directory', checkoutdir]) - result.assert_success() - - with open(os.path.join(checkoutdir, 'ls-l'), 'r') as f: - return f.read() - finally: - os.umask(old_umask) - cli.remove_artifact_from_cache(project, element_name) - - assert get_value_for_umask(0o022) == get_value_for_umask(0o077) diff --git a/tests/sources/generic/track.py b/tests/sources/generic/track.py deleted file mode 100644 index 334a8f679..000000000 --- a/tests/sources/generic/track.py +++ /dev/null @@ -1,423 +0,0 @@ -# -# Copyright (C) 2018 Codethink Limited -# Copyright (C) 2019 Bloomberg Finance LP -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see <http://www.gnu.org/licenses/>. -# - -# Pylint doesn't play well with fixtures and dependency injection from pytest -# pylint: disable=redefined-outer-name - -import os -import pytest - -from tests.frontend import configure_project -from .._utils import generate_junction - -from buildstream.plugintestutils import create_repo, ALL_REPO_KINDS -from buildstream.plugintestutils import cli # pylint: disable=unused-import -from buildstream import _yaml -from buildstream._exceptions import ErrorDomain - - -# Project directory -TOP_DIR = os.path.dirname(os.path.realpath(__file__)) -DATA_DIR = os.path.join(TOP_DIR, 'project') - - -def generate_element(repo, element_path, dep_name=None): - element = { - 'kind': 'import', - 'sources': [ - repo.source_config() - ] - } - if dep_name: - element['depends'] = [dep_name] - - _yaml.dump(element, element_path) - - -@pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) -@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) -def test_track(cli, tmpdir, datafiles, ref_storage, kind): - project = str(datafiles) - dev_files_path = os.path.join(project, 'files', 'dev-files') - element_path = os.path.join(project, 'elements') - element_name = 'track-test-{}.bst'.format(kind) - - configure_project(project, { - 'ref-storage': ref_storage - }) - - # Create our repo object of the given source type with - # the dev files, and then collect the initial ref. - # - repo = create_repo(kind, str(tmpdir)) - repo.create(dev_files_path) - - # Generate the element - generate_element(repo, os.path.join(element_path, element_name)) - - # Assert that a fetch is needed - assert cli.get_element_state(project, element_name) == 'no reference' - - # Now first try to track it - result = cli.run(project=project, args=['source', 'track', element_name]) - result.assert_success() - - # And now fetch it: The Source has probably already cached the - # latest ref locally, but it is not required to have cached - # the associated content of the latest ref at track time, that - # is the job of fetch. - result = cli.run(project=project, args=['source', 'fetch', element_name]) - result.assert_success() - - # Assert that we are now buildable because the source is - # now cached. - assert cli.get_element_state(project, element_name) == 'buildable' - - # Assert there was a project.refs created, depending on the configuration - if ref_storage == 'project.refs': - assert os.path.exists(os.path.join(project, 'project.refs')) - else: - assert not os.path.exists(os.path.join(project, 'project.refs')) - - -# NOTE: -# -# This test checks that recursive tracking works by observing -# element states after running a recursive tracking operation. -# -# However, this test is ALSO valuable as it stresses the source -# plugins in a situation where many source plugins are operating -# at once on the same backing repository. -# -# Do not change this test to use a separate 'Repo' per element -# as that would defeat the purpose of the stress test, otherwise -# please refactor that aspect into another test. -# -@pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("amount", [(1), (10)]) -@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) -def test_track_recurse(cli, tmpdir, datafiles, kind, amount): - project = str(datafiles) - dev_files_path = os.path.join(project, 'files', 'dev-files') - element_path = os.path.join(project, 'elements') - - # Try to actually launch as many fetch jobs as possible at the same time - # - # This stresses the Source plugins and helps to ensure that - # they handle concurrent access to the store correctly. - cli.configure({ - 'scheduler': { - 'fetchers': amount, - } - }) - - # Create our repo object of the given source type with - # the dev files, and then collect the initial ref. - # - repo = create_repo(kind, str(tmpdir)) - repo.create(dev_files_path) - - # Write out our test targets - element_names = [] - last_element_name = None - for i in range(amount + 1): - element_name = 'track-test-{}-{}.bst'.format(kind, i + 1) - filename = os.path.join(element_path, element_name) - - element_names.append(element_name) - - generate_element(repo, filename, dep_name=last_element_name) - last_element_name = element_name - - # Assert that a fetch is needed - states = cli.get_element_states(project, [last_element_name]) - for element_name in element_names: - assert states[element_name] == 'no reference' - - # Now first try to track it - result = cli.run(project=project, args=[ - 'source', 'track', '--deps', 'all', - last_element_name]) - result.assert_success() - - # And now fetch it: The Source has probably already cached the - # latest ref locally, but it is not required to have cached - # the associated content of the latest ref at track time, that - # is the job of fetch. - result = cli.run(project=project, args=[ - 'source', 'fetch', '--deps', 'all', - last_element_name]) - result.assert_success() - - # Assert that the base is buildable and the rest are waiting - states = cli.get_element_states(project, [last_element_name]) - for element_name in element_names: - if element_name == element_names[0]: - assert states[element_name] == 'buildable' - else: - assert states[element_name] == 'waiting' - - -@pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) -def test_track_recurse_except(cli, tmpdir, datafiles, kind): - project = str(datafiles) - dev_files_path = os.path.join(project, 'files', 'dev-files') - element_path = os.path.join(project, 'elements') - element_dep_name = 'track-test-dep-{}.bst'.format(kind) - element_target_name = 'track-test-target-{}.bst'.format(kind) - - # Create our repo object of the given source type with - # the dev files, and then collect the initial ref. - # - repo = create_repo(kind, str(tmpdir)) - repo.create(dev_files_path) - - # Write out our test targets - generate_element(repo, os.path.join(element_path, element_dep_name)) - generate_element(repo, os.path.join(element_path, element_target_name), - dep_name=element_dep_name) - - # Assert that a fetch is needed - states = cli.get_element_states(project, [element_target_name]) - assert states[element_dep_name] == 'no reference' - assert states[element_target_name] == 'no reference' - - # Now first try to track it - result = cli.run(project=project, args=[ - 'source', 'track', '--deps', 'all', '--except', element_dep_name, - element_target_name]) - result.assert_success() - - # And now fetch it: The Source has probably already cached the - # latest ref locally, but it is not required to have cached - # the associated content of the latest ref at track time, that - # is the job of fetch. - result = cli.run(project=project, args=[ - 'source', 'fetch', '--deps', 'none', - element_target_name]) - result.assert_success() - - # Assert that the dependency is buildable and the target is waiting - states = cli.get_element_states(project, [element_target_name]) - assert states[element_dep_name] == 'no reference' - assert states[element_target_name] == 'waiting' - - -@pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) -@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) -def test_cross_junction(cli, tmpdir, datafiles, ref_storage, kind): - project = str(datafiles) - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(project, 'elements', 'junction.bst') - etc_files = os.path.join(subproject_path, 'files', 'etc-files') - repo_element_path = os.path.join(subproject_path, 'elements', - 'import-etc-repo.bst') - - configure_project(project, { - 'ref-storage': ref_storage - }) - - repo = create_repo(kind, str(tmpdir.join('element_repo'))) - repo.create(etc_files) - - generate_element(repo, repo_element_path) - - generate_junction(str(tmpdir.join('junction_repo')), - subproject_path, junction_path, store_ref=False) - - # Track the junction itself first. - result = cli.run(project=project, args=['source', 'track', 'junction.bst']) - result.assert_success() - - assert cli.get_element_state(project, 'junction.bst:import-etc-repo.bst') == 'no reference' - - # Track the cross junction element. -J is not given, it is implied. - result = cli.run(project=project, args=['source', 'track', 'junction.bst:import-etc-repo.bst']) - - if ref_storage == 'inline': - # This is not allowed to track cross junction without project.refs. - result.assert_main_error(ErrorDomain.PIPELINE, 'untrackable-sources') - else: - result.assert_success() - - assert cli.get_element_state(project, 'junction.bst:import-etc-repo.bst') == 'buildable' - - assert os.path.exists(os.path.join(project, 'project.refs')) - - -@pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) -@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) -def test_track_include(cli, tmpdir, datafiles, ref_storage, kind): - project = str(datafiles) - dev_files_path = os.path.join(project, 'files', 'dev-files') - element_path = os.path.join(project, 'elements') - element_name = 'track-test-{}.bst'.format(kind) - - configure_project(project, { - 'ref-storage': ref_storage - }) - - # Create our repo object of the given source type with - # the dev files, and then collect the initial ref. - # - repo = create_repo(kind, str(tmpdir)) - ref = repo.create(dev_files_path) - - # Generate the element - element = { - 'kind': 'import', - '(@)': ['elements/sources.yml'] - } - sources = { - 'sources': [ - repo.source_config() - ] - } - - _yaml.dump(element, os.path.join(element_path, element_name)) - _yaml.dump(sources, os.path.join(element_path, 'sources.yml')) - - # Assert that a fetch is needed - assert cli.get_element_state(project, element_name) == 'no reference' - - # Now first try to track it - result = cli.run(project=project, args=['source', 'track', element_name]) - result.assert_success() - - # And now fetch it: The Source has probably already cached the - # latest ref locally, but it is not required to have cached - # the associated content of the latest ref at track time, that - # is the job of fetch. - result = cli.run(project=project, args=['source', 'fetch', element_name]) - result.assert_success() - - # Assert that we are now buildable because the source is - # now cached. - assert cli.get_element_state(project, element_name) == 'buildable' - - # Assert there was a project.refs created, depending on the configuration - if ref_storage == 'project.refs': - assert os.path.exists(os.path.join(project, 'project.refs')) - else: - assert not os.path.exists(os.path.join(project, 'project.refs')) - - new_sources = _yaml.load(os.path.join(element_path, 'sources.yml')) - - # Get all of the sources - assert 'sources' in new_sources - sources_list = _yaml.node_get(new_sources, list, 'sources') - assert len(sources_list) == 1 - - # Get the first source from the sources list - new_source = _yaml.node_get(new_sources, dict, 'sources', indices=[0]) - assert 'ref' in new_source - assert ref == _yaml.node_get(new_source, str, 'ref') - - -@pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) -@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) -def test_track_include_junction(cli, tmpdir, datafiles, ref_storage, kind): - project = str(datafiles) - dev_files_path = os.path.join(project, 'files', 'dev-files') - element_path = os.path.join(project, 'elements') - element_name = 'track-test-{}.bst'.format(kind) - subproject_path = os.path.join(project, 'files', 'sub-project') - sub_element_path = os.path.join(subproject_path, 'elements') - junction_path = os.path.join(element_path, 'junction.bst') - - configure_project(project, { - 'ref-storage': ref_storage - }) - - # Create our repo object of the given source type with - # the dev files, and then collect the initial ref. - # - repo = create_repo(kind, str(tmpdir.join('element_repo'))) - repo.create(dev_files_path) - - # Generate the element - element = { - 'kind': 'import', - '(@)': ['junction.bst:elements/sources.yml'] - } - sources = { - 'sources': [ - repo.source_config() - ] - } - - _yaml.dump(element, os.path.join(element_path, element_name)) - _yaml.dump(sources, os.path.join(sub_element_path, 'sources.yml')) - - generate_junction(str(tmpdir.join('junction_repo')), - subproject_path, junction_path, store_ref=True) - - result = cli.run(project=project, args=['source', 'track', 'junction.bst']) - result.assert_success() - - # Assert that a fetch is needed - assert cli.get_element_state(project, element_name) == 'no reference' - - # Now first try to track it - result = cli.run(project=project, args=['source', 'track', element_name]) - - # Assert there was a project.refs created, depending on the configuration - if ref_storage == 'inline': - # FIXME: We should expect an error. But only a warning is emitted - # result.assert_main_error(ErrorDomain.SOURCE, 'tracking-junction-fragment') - - assert 'junction.bst:elements/sources.yml: Cannot track source in a fragment from a junction' in result.stderr - else: - assert os.path.exists(os.path.join(project, 'project.refs')) - - # And now fetch it: The Source has probably already cached the - # latest ref locally, but it is not required to have cached - # the associated content of the latest ref at track time, that - # is the job of fetch. - result = cli.run(project=project, args=['source', 'fetch', element_name]) - result.assert_success() - - # Assert that we are now buildable because the source is - # now cached. - assert cli.get_element_state(project, element_name) == 'buildable' - - -@pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) -@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) -def test_track_junction_included(cli, tmpdir, datafiles, ref_storage, kind): - project = str(datafiles) - element_path = os.path.join(project, 'elements') - subproject_path = os.path.join(project, 'files', 'sub-project') - junction_path = os.path.join(element_path, 'junction.bst') - - configure_project(project, { - 'ref-storage': ref_storage, - '(@)': ['junction.bst:test.yml'] - }) - - generate_junction(str(tmpdir.join('junction_repo')), - subproject_path, junction_path, store_ref=False) - - result = cli.run(project=project, args=['source', 'track', 'junction.bst']) - result.assert_success() diff --git a/tests/sources/generic/track_cross_junction.py b/tests/sources/generic/track_cross_junction.py deleted file mode 100644 index 68db91756..000000000 --- a/tests/sources/generic/track_cross_junction.py +++ /dev/null @@ -1,187 +0,0 @@ -# -# Copyright (C) 2018 Codethink Limited -# Copyright (C) 2019 Bloomberg Finance LP -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see <http://www.gnu.org/licenses/>. -# - -# Pylint doesn't play well with fixtures and dependency injection from pytest -# pylint: disable=redefined-outer-name - -import os -import pytest - -from .._utils import generate_junction - -from buildstream.plugintestutils import create_repo, ALL_REPO_KINDS -from buildstream.plugintestutils import cli # pylint: disable=unused-import -from buildstream import _yaml - -# Project directory -TOP_DIR = os.path.dirname(os.path.realpath(__file__)) -DATA_DIR = os.path.join(TOP_DIR, 'project') - - -def generate_element(repo, element_path, dep_name=None): - element = { - 'kind': 'import', - 'sources': [ - repo.source_config() - ] - } - if dep_name: - element['depends'] = [dep_name] - - _yaml.dump(element, element_path) - - -def generate_import_element(tmpdir, kind, project, name): - element_name = 'import-{}.bst'.format(name) - repo_element_path = os.path.join(project, 'elements', element_name) - files = str(tmpdir.join("imported_files_{}".format(name))) - os.makedirs(files) - - with open(os.path.join(files, '{}.txt'.format(name)), 'w') as f: - f.write(name) - - repo = create_repo(kind, str(tmpdir.join('element_{}_repo'.format(name)))) - repo.create(files) - - generate_element(repo, repo_element_path) - - return element_name - - -def generate_project(tmpdir, name, config=None): - if config is None: - config = {} - - project_name = 'project-{}'.format(name) - subproject_path = os.path.join(str(tmpdir.join(project_name))) - os.makedirs(os.path.join(subproject_path, 'elements')) - - project_conf = { - 'name': name, - 'element-path': 'elements' - } - project_conf.update(config) - _yaml.dump(project_conf, os.path.join(subproject_path, 'project.conf')) - - return project_name, subproject_path - - -def generate_simple_stack(project, name, dependencies): - element_name = '{}.bst'.format(name) - element_path = os.path.join(project, 'elements', element_name) - element = { - 'kind': 'stack', - 'depends': dependencies - } - _yaml.dump(element, element_path) - - return element_name - - -def generate_cross_element(project, subproject_name, import_name): - basename, _ = os.path.splitext(import_name) - return generate_simple_stack(project, 'import-{}-{}'.format(subproject_name, basename), - [{ - 'junction': '{}.bst'.format(subproject_name), - 'filename': import_name - }]) - - -@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) -def test_cross_junction_multiple_projects(cli, tmpdir, kind): - tmpdir = tmpdir.join(kind) - - # Generate 3 projects: main, a, b - _, project = generate_project(tmpdir, 'main', {'ref-storage': 'project.refs'}) - project_a, project_a_path = generate_project(tmpdir, 'a') - project_b, project_b_path = generate_project(tmpdir, 'b') - - # Generate an element with a trackable source for each project - element_a = generate_import_element(tmpdir, kind, project_a_path, 'a') - element_b = generate_import_element(tmpdir, kind, project_b_path, 'b') - element_c = generate_import_element(tmpdir, kind, project, 'c') - - # Create some indirections to the elements with dependencies to test --deps - stack_a = generate_simple_stack(project_a_path, 'stack-a', [element_a]) - stack_b = generate_simple_stack(project_b_path, 'stack-b', [element_b]) - - # Create junctions for projects a and b in main. - junction_a = '{}.bst'.format(project_a) - junction_a_path = os.path.join(project, 'elements', junction_a) - generate_junction(tmpdir.join('repo_a'), project_a_path, junction_a_path, store_ref=False) - - junction_b = '{}.bst'.format(project_b) - junction_b_path = os.path.join(project, 'elements', junction_b) - generate_junction(tmpdir.join('repo_b'), project_b_path, junction_b_path, store_ref=False) - - # Track the junctions. - result = cli.run(project=project, args=['source', 'track', junction_a, junction_b]) - result.assert_success() - - # Import elements from a and b in to main. - imported_a = generate_cross_element(project, project_a, stack_a) - imported_b = generate_cross_element(project, project_b, stack_b) - - # Generate a top level stack depending on everything - all_bst = generate_simple_stack(project, 'all', [imported_a, imported_b, element_c]) - - # Track without following junctions. But explicitly also track the elements in project a. - result = cli.run(project=project, args=['source', 'track', - '--deps', 'all', - all_bst, - '{}:{}'.format(junction_a, stack_a)]) - result.assert_success() - - # Elements in project b should not be tracked. But elements in project a and main should. - expected = [element_c, - '{}:{}'.format(junction_a, element_a)] - assert set(result.get_tracked_elements()) == set(expected) - - -@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) -def test_track_exceptions(cli, tmpdir, kind): - tmpdir = tmpdir.join(kind) - - _, project = generate_project(tmpdir, 'main', {'ref-storage': 'project.refs'}) - project_a, project_a_path = generate_project(tmpdir, 'a') - - element_a = generate_import_element(tmpdir, kind, project_a_path, 'a') - element_b = generate_import_element(tmpdir, kind, project_a_path, 'b') - - all_bst = generate_simple_stack(project_a_path, 'all', [element_a, - element_b]) - - junction_a = '{}.bst'.format(project_a) - junction_a_path = os.path.join(project, 'elements', junction_a) - generate_junction(tmpdir.join('repo_a'), project_a_path, junction_a_path, store_ref=False) - - result = cli.run(project=project, args=['source', 'track', junction_a]) - result.assert_success() - - imported_b = generate_cross_element(project, project_a, element_b) - indirection = generate_simple_stack(project, 'indirection', [imported_b]) - - result = cli.run(project=project, - args=['source', 'track', '--deps', 'all', - '--except', indirection, - '{}:{}'.format(junction_a, all_bst), imported_b]) - result.assert_success() - - expected = ['{}:{}'.format(junction_a, element_a), - '{}:{}'.format(junction_a, element_b)] - assert set(result.get_tracked_elements()) == set(expected) diff --git a/tests/sources/generic/workspace.py b/tests/sources/generic/workspace.py deleted file mode 100644 index 4da3a5b78..000000000 --- a/tests/sources/generic/workspace.py +++ /dev/null @@ -1,161 +0,0 @@ -# -# Copyright (C) 2018 Codethink Limited -# Copyright (C) 2019 Bloomberg Finance LP -# -# This program is free software; you can redistribute it and/or -# modify it under the terms of the GNU Lesser General Public -# License as published by the Free Software Foundation; either -# version 2 of the License, or (at your option) any later version. -# -# This library is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -# Lesser General Public License for more details. -# -# You should have received a copy of the GNU Lesser General Public -# License along with this library. If not, see <http://www.gnu.org/licenses/>. -# - -# Pylint doesn't play well with fixtures and dependency injection from pytest -# pylint: disable=redefined-outer-name - -import os -import shutil -import pytest - -from buildstream.plugintestutils import create_repo, ALL_REPO_KINDS -from buildstream.plugintestutils import cli # pylint: disable=unused-import -from buildstream import _yaml - -# Project directory -TOP_DIR = os.path.dirname(os.path.realpath(__file__)) -DATA_DIR = os.path.join(TOP_DIR, 'project') - - -class WorkspaceCreator(): - def __init__(self, cli, tmpdir, datafiles, project_path=None): - self.cli = cli - self.tmpdir = tmpdir - self.datafiles = datafiles - - if not project_path: - project_path = str(datafiles) - else: - shutil.copytree(str(datafiles), project_path) - - self.project_path = project_path - self.bin_files_path = os.path.join(project_path, 'files', 'bin-files') - - self.workspace_cmd = os.path.join(self.project_path, 'workspace_cmd') - - def create_workspace_element(self, kind, track, suffix='', workspace_dir=None, - element_attrs=None): - element_name = 'workspace-test-{}{}.bst'.format(kind, suffix) - element_path = os.path.join(self.project_path, 'elements') - if not workspace_dir: - workspace_dir = os.path.join(self.workspace_cmd, element_name) - if workspace_dir[-4:] == '.bst': - workspace_dir = workspace_dir[:-4] - - # Create our repo object of the given source type with - # the bin files, and then collect the initial ref. - repo = create_repo(kind, str(self.tmpdir)) - ref = repo.create(self.bin_files_path) - if track: - ref = None - - # Write out our test target - element = { - 'kind': 'import', - 'sources': [ - repo.source_config(ref=ref) - ] - } - if element_attrs: - element = {**element, **element_attrs} - _yaml.dump(element, - os.path.join(element_path, - element_name)) - return element_name, element_path, workspace_dir - - def create_workspace_elements(self, kinds, track, suffixs=None, workspace_dir_usr=None, - element_attrs=None): - - element_tuples = [] - - if suffixs is None: - suffixs = ['', ] * len(kinds) - else: - if len(suffixs) != len(kinds): - raise "terable error" - - for suffix, kind in zip(suffixs, kinds): - element_name, _, workspace_dir = \ - self.create_workspace_element(kind, track, suffix, workspace_dir_usr, - element_attrs) - element_tuples.append((element_name, workspace_dir)) - - # Assert that there is no reference, a track & fetch is needed - states = self.cli.get_element_states(self.project_path, [ - e for e, _ in element_tuples - ]) - if track: - assert not any(states[e] != 'no reference' for e, _ in element_tuples) - else: - assert not any(states[e] != 'fetch needed' for e, _ in element_tuples) - - return element_tuples - - def open_workspaces(self, kinds, track, suffixs=None, workspace_dir=None, - element_attrs=None, no_checkout=False): - - element_tuples = self.create_workspace_elements(kinds, track, suffixs, workspace_dir, - element_attrs) - os.makedirs(self.workspace_cmd, exist_ok=True) - - # Now open the workspace, this should have the effect of automatically - # tracking & fetching the source from the repo. - args = ['workspace', 'open'] - if track: - args.append('--track') - if no_checkout: - args.append('--no-checkout') - if workspace_dir is not None: - assert len(element_tuples) == 1, "test logic error" - _, workspace_dir = element_tuples[0] - args.extend(['--directory', workspace_dir]) - - args.extend([element_name for element_name, workspace_dir_suffix in element_tuples]) - result = self.cli.run(cwd=self.workspace_cmd, project=self.project_path, args=args) - - result.assert_success() - - if not no_checkout: - # Assert that we are now buildable because the source is now cached. - states = self.cli.get_element_states(self.project_path, [ - e for e, _ in element_tuples - ]) - assert not any(states[e] != 'buildable' for e, _ in element_tuples) - - # Check that the executable hello file is found in each workspace - for _, workspace in element_tuples: - filename = os.path.join(workspace, 'usr', 'bin', 'hello') - assert os.path.exists(filename) - - return element_tuples - - -def open_workspace(cli, tmpdir, datafiles, kind, track, suffix='', workspace_dir=None, - project_path=None, element_attrs=None, no_checkout=False): - workspace_object = WorkspaceCreator(cli, tmpdir, datafiles, project_path) - workspaces = workspace_object.open_workspaces((kind, ), track, (suffix, ), workspace_dir, - element_attrs, no_checkout) - assert len(workspaces) == 1 - element_name, workspace = workspaces[0] - return element_name, workspace_object.project_path, workspace - - -@pytest.mark.datafiles(DATA_DIR) -@pytest.mark.parametrize("kind", ALL_REPO_KINDS) -def test_open(cli, tmpdir, datafiles, kind): - open_workspace(cli, tmpdir, datafiles, kind, False) |