# # Copyright (C) 2019 Bloomberg Finance LP # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see . # # Authors: # Raoul Hidalgo Charman # # Pylint doesn't play well with fixtures and dependency injection from pytest # pylint: disable=redefined-outer-name import os import shutil import pytest from buildstream._context import Context from buildstream._exceptions import ErrorDomain from buildstream._project import Project from buildstream import _yaml from buildstream.testing import cli # pylint: disable=unused-import from buildstream.testing import create_repo from tests.testutils import create_artifact_share DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project") def message_handler(message, context): pass @pytest.mark.datafiles(DATA_DIR) def test_source_push(cli, tmpdir, datafiles): cache_dir = os.path.join(str(tmpdir), 'cache') project_dir = str(datafiles) with create_artifact_share(os.path.join(str(tmpdir), 'sourceshare')) as share: user_config_file = str(tmpdir.join('buildstream.conf')) user_config = { 'scheduler': { 'pushers': 1 }, 'source-caches': { 'url': share.repo, 'push': True, }, 'cachedir': cache_dir, } _yaml.dump(_yaml.node_sanitize(user_config), filename=user_config_file) cli.configure(user_config) repo = create_repo('git', str(tmpdir)) ref = repo.create(os.path.join(project_dir, 'files')) element_path = os.path.join(project_dir, 'elements') element_name = 'push.bst' element = { 'kind': 'import', 'sources': [repo.source_config(ref=ref)] } _yaml.dump(element, os.path.join(element_path, element_name)) # get the source object context = Context() context.load(config=user_config_file) context.set_message_handler(message_handler) project = Project(project_dir, context) project.ensure_fully_loaded() element = project.load_elements(['push.bst'])[0] assert not element._source_cached() source = list(element.sources())[0] # check we don't have it in the current cache cas = context.get_cascache() assert not cas.contains(source._get_source_name()) # build the element, this should fetch and then push the source to the # remote res = cli.run(project=project_dir, args=['build', 'push.bst']) res.assert_success() assert "Pushed source" in res.stderr # check that we've got the remote locally now sourcecache = context.sourcecache assert sourcecache.contains(source) # check that's the remote CAS now has it digest = share.cas.resolve_ref(source._get_source_name()) assert share.has_object(digest) @pytest.mark.datafiles(DATA_DIR) def test_push_pull(cli, datafiles, tmpdir): project_dir = str(datafiles) cache_dir = os.path.join(str(tmpdir), 'cache') with create_artifact_share(os.path.join(str(tmpdir), 'sourceshare')) as share: user_config_file = str(tmpdir.join('buildstream.conf')) user_config = { 'scheduler': { 'pushers': 1 }, 'source-caches': { 'url': share.repo, 'push': True, }, 'cachedir': cache_dir, } _yaml.dump(_yaml.node_sanitize(user_config), filename=user_config_file) cli.configure(user_config) # create repo to pull from repo = create_repo('git', str(tmpdir)) ref = repo.create(os.path.join(project_dir, 'files')) element_path = os.path.join(project_dir, 'elements') element_name = 'push.bst' element = { 'kind': 'import', 'sources': [repo.source_config(ref=ref)] } _yaml.dump(element, os.path.join(element_path, element_name)) res = cli.run(project=project_dir, args=['build', 'push.bst']) res.assert_success() # remove local cache dir, and repo files and check it all works shutil.rmtree(cache_dir) os.makedirs(cache_dir) shutil.rmtree(repo.repo) # check it's pulls from the share res = cli.run(project=project_dir, args=['build', 'push.bst']) res.assert_success() @pytest.mark.datafiles(DATA_DIR) def test_push_fail(cli, tmpdir, datafiles): project_dir = str(datafiles) cache_dir = os.path.join(str(tmpdir), 'cache') # set up config with remote that we'll take down with create_artifact_share(os.path.join(str(tmpdir), 'sourceshare')) as share: remote = share.repo user_config_file = str(tmpdir.join('buildstream.conf')) user_config = { 'scheduler': { 'pushers': 1 }, 'source-caches': { 'url': share.repo, 'push': True, }, 'cachedir': cache_dir, } _yaml.dump(_yaml.node_sanitize(user_config), filename=user_config_file) cli.configure(user_config) # create repo to pull from repo = create_repo('git', str(tmpdir)) ref = repo.create(os.path.join(project_dir, 'files')) element_path = os.path.join(project_dir, 'elements') element_name = 'push.bst' element = { 'kind': 'import', 'sources': [repo.source_config(ref=ref)] } _yaml.dump(element, os.path.join(element_path, element_name)) # build and check that it fails to set up the remote res = cli.run(project=project_dir, args=['build', 'push.bst']) res.assert_success() assert "Failed to initialize remote {}".format(remote) in res.stderr assert "Pushing" not in res.stderr assert "Pushed" not in res.stderr @pytest.mark.datafiles(DATA_DIR) def test_source_push_build_fail(cli, tmpdir, datafiles): project_dir = str(datafiles) cache_dir = os.path.join(str(tmpdir), 'cache') with create_artifact_share(os.path.join(str(tmpdir), 'share')) as share: user_config = { 'scheduler': { 'pushers': 1 }, 'source-caches': { 'url': share.repo, 'push': True, }, 'cachedir': cache_dir, } cli.configure(user_config) repo = create_repo('git', str(tmpdir)) ref = repo.create(os.path.join(project_dir, 'files')) element_path = os.path.join(project_dir, 'elements') element_name = 'always-fail.bst' element = { 'kind': 'always_fail', 'sources': [repo.source_config(ref=ref)] } _yaml.dump(element, os.path.join(element_path, element_name)) res = cli.run(project=project_dir, args=['build', 'always-fail.bst']) res.assert_main_error(ErrorDomain.STREAM, None) res.assert_task_error(ErrorDomain.ELEMENT, None) # Sources are not pushed as the build queue is before the source push # queue. assert "Pushed source " not in res.stderr