# # Copyright (C) 2018 Codethink Limited # Copyright (C) 2018 Bloomberg Finance LP # # This program is free software; you can redistribute it and/or # modify it under the terms of the GNU Lesser General Public # License as published by the Free Software Foundation; either # version 2 of the License, or (at your option) any later version. # # This library is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU # Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public # License along with this library. If not, see . # # Authors: Tristan Van Berkom # Jonathan Maw # William Salmon # import os import pytest import subprocess from buildstream._exceptions import ErrorDomain from buildstream import _yaml from buildstream.plugin import CoreWarnings from tests.testutils import cli, create_repo from tests.testutils.site import HAVE_GIT DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), 'git', ) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) def test_fetch_bad_ref(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) # Create the repo from 'repofiles' subdir repo = create_repo('git', str(tmpdir)) ref = repo.create(os.path.join(project, 'repofiles')) # Write out our test target with a bad ref element = { 'kind': 'import', 'sources': [ repo.source_config(ref='5') ] } _yaml.dump(element, os.path.join(project, 'target.bst')) # Assert that fetch raises an error here result = cli.run(project=project, args=[ 'fetch', 'target.bst' ]) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.SOURCE, None) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) def test_submodule_fetch_checkout(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) checkoutdir = os.path.join(str(tmpdir), "checkout") # Create the submodule first from the 'subrepofiles' subdir subrepo = create_repo('git', str(tmpdir), 'subrepo') subref = subrepo.create(os.path.join(project, 'subrepofiles')) # Create the repo from 'repofiles' subdir repo = create_repo('git', str(tmpdir)) ref = repo.create(os.path.join(project, 'repofiles')) # Add a submodule pointing to the one we created ref = repo.add_submodule('subdir', 'file://' + subrepo.repo) # Write out our test target element = { 'kind': 'import', 'sources': [ repo.source_config(ref=ref) ] } _yaml.dump(element, os.path.join(project, 'target.bst')) # Fetch, build, checkout result = cli.run(project=project, args=['fetch', 'target.bst']) result.assert_success() result = cli.run(project=project, args=['build', 'target.bst']) result.assert_success() result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir]) result.assert_success() # Assert we checked out both files at their expected location assert os.path.exists(os.path.join(checkoutdir, 'file.txt')) assert os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt')) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) def test_submodule_fetch_source_enable_explicit(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) checkoutdir = os.path.join(str(tmpdir), "checkout") # Create the submodule first from the 'subrepofiles' subdir subrepo = create_repo('git', str(tmpdir), 'subrepo') subrepo.create(os.path.join(project, 'subrepofiles')) # Create the repo from 'repofiles' subdir repo = create_repo('git', str(tmpdir)) ref = repo.create(os.path.join(project, 'repofiles')) # Add a submodule pointing to the one we created ref = repo.add_submodule('subdir', 'file://' + subrepo.repo) # Write out our test target element = { 'kind': 'import', 'sources': [ repo.source_config(ref=ref, checkout_submodules=True) ] } _yaml.dump(element, os.path.join(project, 'target.bst')) # Fetch, build, checkout result = cli.run(project=project, args=['fetch', 'target.bst']) result.assert_success() result = cli.run(project=project, args=['build', 'target.bst']) result.assert_success() result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir]) result.assert_success() # Assert we checked out both files at their expected location assert os.path.exists(os.path.join(checkoutdir, 'file.txt')) assert os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt')) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) def test_submodule_fetch_source_disable(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) checkoutdir = os.path.join(str(tmpdir), "checkout") # Create the submodule first from the 'subrepofiles' subdir subrepo = create_repo('git', str(tmpdir), 'subrepo') subrepo.create(os.path.join(project, 'subrepofiles')) # Create the repo from 'repofiles' subdir repo = create_repo('git', str(tmpdir)) ref = repo.create(os.path.join(project, 'repofiles')) # Add a submodule pointing to the one we created ref = repo.add_submodule('subdir', 'file://' + subrepo.repo) # Write out our test target element = { 'kind': 'import', 'sources': [ repo.source_config(ref=ref, checkout_submodules=False) ] } _yaml.dump(element, os.path.join(project, 'target.bst')) # Fetch, build, checkout result = cli.run(project=project, args=['fetch', 'target.bst']) result.assert_success() result = cli.run(project=project, args=['build', 'target.bst']) result.assert_success() result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir]) result.assert_success() # Assert we checked out both files at their expected location assert os.path.exists(os.path.join(checkoutdir, 'file.txt')) assert not os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt')) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) def test_submodule_fetch_submodule_does_override(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) checkoutdir = os.path.join(str(tmpdir), "checkout") # Create the submodule first from the 'subrepofiles' subdir subrepo = create_repo('git', str(tmpdir), 'subrepo') subrepo.create(os.path.join(project, 'subrepofiles')) # Create the repo from 'repofiles' subdir repo = create_repo('git', str(tmpdir)) ref = repo.create(os.path.join(project, 'repofiles')) # Add a submodule pointing to the one we created ref = repo.add_submodule('subdir', 'file://' + subrepo.repo, checkout=True) # Write out our test target element = { 'kind': 'import', 'sources': [ repo.source_config(ref=ref, checkout_submodules=False) ] } _yaml.dump(element, os.path.join(project, 'target.bst')) # Fetch, build, checkout result = cli.run(project=project, args=['fetch', 'target.bst']) result.assert_success() result = cli.run(project=project, args=['build', 'target.bst']) result.assert_success() result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir]) result.assert_success() # Assert we checked out both files at their expected location assert os.path.exists(os.path.join(checkoutdir, 'file.txt')) assert os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt')) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) def test_submodule_fetch_submodule_individual_checkout(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) checkoutdir = os.path.join(str(tmpdir), "checkout") # Create the submodule first from the 'subrepofiles' subdir subrepo = create_repo('git', str(tmpdir), 'subrepo') subrepo.create(os.path.join(project, 'subrepofiles')) # Create another submodule from the 'othersubrepofiles' subdir other_subrepo = create_repo('git', str(tmpdir), 'othersubrepo') other_subrepo.create(os.path.join(project, 'othersubrepofiles')) # Create the repo from 'repofiles' subdir repo = create_repo('git', str(tmpdir)) ref = repo.create(os.path.join(project, 'repofiles')) # Add a submodule pointing to the one we created ref = repo.add_submodule('subdir', 'file://' + subrepo.repo, checkout=False) ref = repo.add_submodule('othersubdir', 'file://' + other_subrepo.repo) # Write out our test target element = { 'kind': 'import', 'sources': [ repo.source_config(ref=ref, checkout_submodules=True) ] } _yaml.dump(element, os.path.join(project, 'target.bst')) # Fetch, build, checkout result = cli.run(project=project, args=['fetch', 'target.bst']) result.assert_success() result = cli.run(project=project, args=['build', 'target.bst']) result.assert_success() result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir]) result.assert_success() # Assert we checked out files at their expected location assert os.path.exists(os.path.join(checkoutdir, 'file.txt')) assert not os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt')) assert os.path.exists(os.path.join(checkoutdir, 'othersubdir', 'unicornfile.txt')) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) def test_submodule_fetch_submodule_individual_checkout_explicit(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) checkoutdir = os.path.join(str(tmpdir), "checkout") # Create the submodule first from the 'subrepofiles' subdir subrepo = create_repo('git', str(tmpdir), 'subrepo') subrepo.create(os.path.join(project, 'subrepofiles')) # Create another submodule from the 'othersubrepofiles' subdir other_subrepo = create_repo('git', str(tmpdir), 'othersubrepo') other_subrepo.create(os.path.join(project, 'othersubrepofiles')) # Create the repo from 'repofiles' subdir repo = create_repo('git', str(tmpdir)) ref = repo.create(os.path.join(project, 'repofiles')) # Add a submodule pointing to the one we created ref = repo.add_submodule('subdir', 'file://' + subrepo.repo, checkout=False) ref = repo.add_submodule('othersubdir', 'file://' + other_subrepo.repo, checkout=True) # Write out our test target element = { 'kind': 'import', 'sources': [ repo.source_config(ref=ref, checkout_submodules=True) ] } _yaml.dump(element, os.path.join(project, 'target.bst')) # Fetch, build, checkout result = cli.run(project=project, args=['fetch', 'target.bst']) result.assert_success() result = cli.run(project=project, args=['build', 'target.bst']) result.assert_success() result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir]) result.assert_success() # Assert we checked out files at their expected location assert os.path.exists(os.path.join(checkoutdir, 'file.txt')) assert not os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt')) assert os.path.exists(os.path.join(checkoutdir, 'othersubdir', 'unicornfile.txt')) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.datafiles(os.path.join(DATA_DIR, 'project-override')) def test_submodule_fetch_project_override(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) checkoutdir = os.path.join(str(tmpdir), "checkout") # Create the submodule first from the 'subrepofiles' subdir subrepo = create_repo('git', str(tmpdir), 'subrepo') subrepo.create(os.path.join(project, 'subrepofiles')) # Create the repo from 'repofiles' subdir repo = create_repo('git', str(tmpdir)) ref = repo.create(os.path.join(project, 'repofiles')) # Add a submodule pointing to the one we created ref = repo.add_submodule('subdir', 'file://' + subrepo.repo) # Write out our test target element = { 'kind': 'import', 'sources': [ repo.source_config(ref=ref) ] } _yaml.dump(element, os.path.join(project, 'target.bst')) # Fetch, build, checkout result = cli.run(project=project, args=['fetch', 'target.bst']) result.assert_success() result = cli.run(project=project, args=['build', 'target.bst']) result.assert_success() result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir]) result.assert_success() # Assert we checked out both files at their expected location assert os.path.exists(os.path.join(checkoutdir, 'file.txt')) assert not os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt')) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) def test_submodule_track_ignore_inconsistent(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) # Create the repo from 'repofiles' subdir repo = create_repo('git', str(tmpdir)) ref = repo.create(os.path.join(project, 'repofiles')) # Write out our test target element = { 'kind': 'import', 'sources': [ repo.source_config(ref=ref) ] } _yaml.dump(element, os.path.join(project, 'target.bst')) # Now add a .gitmodules file with an inconsistent submodule, # we are calling this inconsistent because the file was created # but `git submodule add` was never called, so there is no reference # associated to the submodule. # repo.add_file(os.path.join(project, 'inconsistent-submodule', '.gitmodules')) # Fetch should work, we're not yet at the offending ref result = cli.run(project=project, args=['fetch', 'target.bst']) result.assert_success() # Track will encounter an inconsistent submodule without any ref result = cli.run(project=project, args=['track', 'target.bst']) result.assert_success() # Assert that we are just fine without it, and emit a warning to the user. assert "Ignoring inconsistent submodule" in result.stderr @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) def test_submodule_track_no_ref_or_track(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) # Create the repo from 'repofiles' subdir repo = create_repo('git', str(tmpdir)) ref = repo.create(os.path.join(project, 'repofiles')) # Write out our test target gitsource = repo.source_config(ref=None) gitsource.pop('track') element = { 'kind': 'import', 'sources': [ gitsource ] } _yaml.dump(element, os.path.join(project, 'target.bst')) # Track will encounter an inconsistent submodule without any ref result = cli.run(project=project, args=['show', 'target.bst']) result.assert_main_error(ErrorDomain.SOURCE, "missing-track-and-ref") result.assert_task_error(None, None) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) def test_ref_not_in_track_warn(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) # Create the repo from 'repofiles', create a branch without latest commit repo = create_repo('git', str(tmpdir)) ref = repo.create(os.path.join(project, 'repofiles')) gitsource = repo.source_config(ref=ref) # Overwrite the track value to the added branch gitsource['track'] = 'foo' # Write out our test target element = { 'kind': 'import', 'sources': [ gitsource ] } _yaml.dump(element, os.path.join(project, 'target.bst')) # Assert the warning is raised as ref is not in branch foo. # Assert warning not error to the user, when not set as fatal. result = cli.run(project=project, args=['build', 'target.bst']) assert "The ref provided for the element does not exist locally" in result.stderr @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) def test_ref_not_in_track_warn_error(cli, tmpdir, datafiles): project = os.path.join(datafiles.dirname, datafiles.basename) # Add fatal-warnings ref-not-in-track to project.conf project_template = { "name": "foo", "fatal-warnings": [CoreWarnings.REF_NOT_IN_TRACK] } _yaml.dump(project_template, os.path.join(project, 'project.conf')) # Create the repo from 'repofiles', create a branch without latest commit repo = create_repo('git', str(tmpdir)) ref = repo.create(os.path.join(project, 'repofiles')) gitsource = repo.source_config(ref=ref) # Overwrite the track value to the added branch gitsource['track'] = 'foo' # Write out our test target element = { 'kind': 'import', 'sources': [ gitsource ] } _yaml.dump(element, os.path.join(project, 'target.bst')) # Assert that build raises a warning here that is captured # as plugin error, due to the fatal warning being set result = cli.run(project=project, args=['build', 'target.bst']) result.assert_main_error(ErrorDomain.STREAM, None) result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.REF_NOT_IN_TRACK) @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) @pytest.mark.parametrize("ref_format", ['sha1', 'git-describe']) @pytest.mark.parametrize("tag,extra_commit", [(False, False), (True, False), (True, True)]) def test_track_fetch(cli, tmpdir, datafiles, ref_format, tag, extra_commit): project = os.path.join(datafiles.dirname, datafiles.basename) # Create the repo from 'repofiles' subdir repo = create_repo('git', str(tmpdir)) ref = repo.create(os.path.join(project, 'repofiles')) if tag: repo.add_tag('tag') if extra_commit: repo.add_commit() # Write out our test target element = { 'kind': 'import', 'sources': [ repo.source_config() ] } element['sources'][0]['ref-format'] = ref_format element_path = os.path.join(project, 'target.bst') _yaml.dump(element, element_path) # Track it result = cli.run(project=project, args=['track', 'target.bst']) result.assert_success() element = _yaml.load(element_path) new_ref = element['sources'][0]['ref'] if ref_format == 'git-describe' and tag: # Check and strip prefix prefix = 'tag-{}-g'.format(0 if not extra_commit else 1) assert new_ref.startswith(prefix) new_ref = new_ref[len(prefix):] # 40 chars for SHA-1 assert len(new_ref) == 40 # Fetch it result = cli.run(project=project, args=['fetch', 'target.bst']) result.assert_success() @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) @pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) @pytest.mark.parametrize("tag_type", [('annotated'), ('lightweight')]) def test_git_describe(cli, tmpdir, datafiles, ref_storage, tag_type): project = str(datafiles) project_config = _yaml.load(os.path.join(project, 'project.conf')) project_config['ref-storage'] = ref_storage _yaml.dump(_yaml.node_sanitize(project_config), os.path.join(project, 'project.conf')) repofiles = os.path.join(str(tmpdir), 'repofiles') os.makedirs(repofiles, exist_ok=True) file0 = os.path.join(repofiles, 'file0') with open(file0, 'w') as f: f.write('test\n') repo = create_repo('git', str(tmpdir)) def tag(name): if tag_type == 'annotated': repo.add_annotated_tag(name, name) else: repo.add_tag(name) ref = repo.create(repofiles) tag('uselesstag') file1 = os.path.join(str(tmpdir), 'file1') with open(file1, 'w') as f: f.write('test\n') repo.add_file(file1) tag('tag1') file2 = os.path.join(str(tmpdir), 'file2') with open(file2, 'w') as f: f.write('test\n') repo.branch('branch2') repo.add_file(file2) tag('tag2') repo.checkout('master') file3 = os.path.join(str(tmpdir), 'file3') with open(file3, 'w') as f: f.write('test\n') repo.add_file(file3) repo.merge('branch2') config = repo.source_config() config['track'] = repo.latest_commit() config['track-tags'] = True # Write out our test target element = { 'kind': 'import', 'sources': [ config ], } element_path = os.path.join(project, 'target.bst') _yaml.dump(element, element_path) if ref_storage == 'inline': result = cli.run(project=project, args=['track', 'target.bst']) result.assert_success() else: result = cli.run(project=project, args=['track', 'target.bst', '--deps', 'all']) result.assert_success() if ref_storage == 'inline': element = _yaml.load(element_path) tags = _yaml.node_sanitize(element['sources'][0]['tags']) assert len(tags) == 2 for tag in tags: assert 'tag' in tag assert 'commit' in tag assert 'annotated' in tag assert tag['annotated'] == (tag_type == 'annotated') assert set([(tag['tag'], tag['commit']) for tag in tags]) == set([('tag1', repo.rev_parse('tag1^{commit}')), ('tag2', repo.rev_parse('tag2^{commit}'))]) checkout = os.path.join(str(tmpdir), 'checkout') result = cli.run(project=project, args=['build', 'target.bst']) result.assert_success() result = cli.run(project=project, args=['checkout', 'target.bst', checkout]) result.assert_success() if tag_type == 'annotated': options = [] else: options = ['--tags'] describe = subprocess.check_output(['git', 'describe'] + options, cwd=checkout).decode('ascii') assert describe.startswith('tag2-2-') describe_fp = subprocess.check_output(['git', 'describe', '--first-parent'] + options, cwd=checkout).decode('ascii') assert describe_fp.startswith('tag1-2-') tags = subprocess.check_output(['git', 'tag'], cwd=checkout).decode('ascii') tags = set(tags.splitlines()) assert tags == set(['tag1', 'tag2']) p = subprocess.run(['git', 'log', repo.rev_parse('uselesstag')], cwd=checkout) assert p.returncode != 0 @pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") @pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) def test_default_do_not_track_tags(cli, tmpdir, datafiles): project = str(datafiles) project_config = _yaml.load(os.path.join(project, 'project.conf')) project_config['ref-storage'] = 'inline' _yaml.dump(_yaml.node_sanitize(project_config), os.path.join(project, 'project.conf')) repofiles = os.path.join(str(tmpdir), 'repofiles') os.makedirs(repofiles, exist_ok=True) file0 = os.path.join(repofiles, 'file0') with open(file0, 'w') as f: f.write('test\n') repo = create_repo('git', str(tmpdir)) ref = repo.create(repofiles) repo.add_tag('tag') config = repo.source_config() config['track'] = repo.latest_commit() # Write out our test target element = { 'kind': 'import', 'sources': [ config ], } element_path = os.path.join(project, 'target.bst') _yaml.dump(element, element_path) result = cli.run(project=project, args=['track', 'target.bst']) result.assert_success() element = _yaml.load(element_path) assert 'tags' not in element['sources'][0]