#
# Copyright (C) 2018 Codethink Limited
# Copyright (C) 2018 Bloomberg Finance LP
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see .
#
# Authors: Tristan Van Berkom
# Jonathan Maw
# William Salmon
#
import os
import pytest
import subprocess
from buildstream._exceptions import ErrorDomain
from buildstream import _yaml
from buildstream.plugin import CoreWarnings
from tests.testutils import cli, create_repo
from tests.testutils.site import HAVE_GIT
DATA_DIR = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
'git',
)
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
def test_fetch_bad_ref(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
# Create the repo from 'repofiles' subdir
repo = create_repo('git', str(tmpdir))
ref = repo.create(os.path.join(project, 'repofiles'))
# Write out our test target with a bad ref
element = {
'kind': 'import',
'sources': [
repo.source_config(ref='5')
]
}
_yaml.dump(element, os.path.join(project, 'target.bst'))
# Assert that fetch raises an error here
result = cli.run(project=project, args=[
'fetch', 'target.bst'
])
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.SOURCE, None)
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
def test_submodule_fetch_checkout(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the submodule first from the 'subrepofiles' subdir
subrepo = create_repo('git', str(tmpdir), 'subrepo')
subref = subrepo.create(os.path.join(project, 'subrepofiles'))
# Create the repo from 'repofiles' subdir
repo = create_repo('git', str(tmpdir))
ref = repo.create(os.path.join(project, 'repofiles'))
# Add a submodule pointing to the one we created
ref = repo.add_submodule('subdir', 'file://' + subrepo.repo)
# Write out our test target
element = {
'kind': 'import',
'sources': [
repo.source_config(ref=ref)
]
}
_yaml.dump(element, os.path.join(project, 'target.bst'))
# Fetch, build, checkout
result = cli.run(project=project, args=['fetch', 'target.bst'])
result.assert_success()
result = cli.run(project=project, args=['build', 'target.bst'])
result.assert_success()
result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir])
result.assert_success()
# Assert we checked out both files at their expected location
assert os.path.exists(os.path.join(checkoutdir, 'file.txt'))
assert os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt'))
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
def test_submodule_fetch_source_enable_explicit(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the submodule first from the 'subrepofiles' subdir
subrepo = create_repo('git', str(tmpdir), 'subrepo')
subrepo.create(os.path.join(project, 'subrepofiles'))
# Create the repo from 'repofiles' subdir
repo = create_repo('git', str(tmpdir))
ref = repo.create(os.path.join(project, 'repofiles'))
# Add a submodule pointing to the one we created
ref = repo.add_submodule('subdir', 'file://' + subrepo.repo)
# Write out our test target
element = {
'kind': 'import',
'sources': [
repo.source_config(ref=ref, checkout_submodules=True)
]
}
_yaml.dump(element, os.path.join(project, 'target.bst'))
# Fetch, build, checkout
result = cli.run(project=project, args=['fetch', 'target.bst'])
result.assert_success()
result = cli.run(project=project, args=['build', 'target.bst'])
result.assert_success()
result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir])
result.assert_success()
# Assert we checked out both files at their expected location
assert os.path.exists(os.path.join(checkoutdir, 'file.txt'))
assert os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt'))
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
def test_submodule_fetch_source_disable(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the submodule first from the 'subrepofiles' subdir
subrepo = create_repo('git', str(tmpdir), 'subrepo')
subrepo.create(os.path.join(project, 'subrepofiles'))
# Create the repo from 'repofiles' subdir
repo = create_repo('git', str(tmpdir))
ref = repo.create(os.path.join(project, 'repofiles'))
# Add a submodule pointing to the one we created
ref = repo.add_submodule('subdir', 'file://' + subrepo.repo)
# Write out our test target
element = {
'kind': 'import',
'sources': [
repo.source_config(ref=ref, checkout_submodules=False)
]
}
_yaml.dump(element, os.path.join(project, 'target.bst'))
# Fetch, build, checkout
result = cli.run(project=project, args=['fetch', 'target.bst'])
result.assert_success()
result = cli.run(project=project, args=['build', 'target.bst'])
result.assert_success()
result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir])
result.assert_success()
# Assert we checked out both files at their expected location
assert os.path.exists(os.path.join(checkoutdir, 'file.txt'))
assert not os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt'))
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
def test_submodule_fetch_submodule_does_override(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the submodule first from the 'subrepofiles' subdir
subrepo = create_repo('git', str(tmpdir), 'subrepo')
subrepo.create(os.path.join(project, 'subrepofiles'))
# Create the repo from 'repofiles' subdir
repo = create_repo('git', str(tmpdir))
ref = repo.create(os.path.join(project, 'repofiles'))
# Add a submodule pointing to the one we created
ref = repo.add_submodule('subdir', 'file://' + subrepo.repo, checkout=True)
# Write out our test target
element = {
'kind': 'import',
'sources': [
repo.source_config(ref=ref, checkout_submodules=False)
]
}
_yaml.dump(element, os.path.join(project, 'target.bst'))
# Fetch, build, checkout
result = cli.run(project=project, args=['fetch', 'target.bst'])
result.assert_success()
result = cli.run(project=project, args=['build', 'target.bst'])
result.assert_success()
result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir])
result.assert_success()
# Assert we checked out both files at their expected location
assert os.path.exists(os.path.join(checkoutdir, 'file.txt'))
assert os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt'))
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
def test_submodule_fetch_submodule_individual_checkout(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the submodule first from the 'subrepofiles' subdir
subrepo = create_repo('git', str(tmpdir), 'subrepo')
subrepo.create(os.path.join(project, 'subrepofiles'))
# Create another submodule from the 'othersubrepofiles' subdir
other_subrepo = create_repo('git', str(tmpdir), 'othersubrepo')
other_subrepo.create(os.path.join(project, 'othersubrepofiles'))
# Create the repo from 'repofiles' subdir
repo = create_repo('git', str(tmpdir))
ref = repo.create(os.path.join(project, 'repofiles'))
# Add a submodule pointing to the one we created
ref = repo.add_submodule('subdir', 'file://' + subrepo.repo, checkout=False)
ref = repo.add_submodule('othersubdir', 'file://' + other_subrepo.repo)
# Write out our test target
element = {
'kind': 'import',
'sources': [
repo.source_config(ref=ref, checkout_submodules=True)
]
}
_yaml.dump(element, os.path.join(project, 'target.bst'))
# Fetch, build, checkout
result = cli.run(project=project, args=['fetch', 'target.bst'])
result.assert_success()
result = cli.run(project=project, args=['build', 'target.bst'])
result.assert_success()
result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir])
result.assert_success()
# Assert we checked out files at their expected location
assert os.path.exists(os.path.join(checkoutdir, 'file.txt'))
assert not os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt'))
assert os.path.exists(os.path.join(checkoutdir, 'othersubdir', 'unicornfile.txt'))
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
def test_submodule_fetch_submodule_individual_checkout_explicit(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the submodule first from the 'subrepofiles' subdir
subrepo = create_repo('git', str(tmpdir), 'subrepo')
subrepo.create(os.path.join(project, 'subrepofiles'))
# Create another submodule from the 'othersubrepofiles' subdir
other_subrepo = create_repo('git', str(tmpdir), 'othersubrepo')
other_subrepo.create(os.path.join(project, 'othersubrepofiles'))
# Create the repo from 'repofiles' subdir
repo = create_repo('git', str(tmpdir))
ref = repo.create(os.path.join(project, 'repofiles'))
# Add a submodule pointing to the one we created
ref = repo.add_submodule('subdir', 'file://' + subrepo.repo, checkout=False)
ref = repo.add_submodule('othersubdir', 'file://' + other_subrepo.repo, checkout=True)
# Write out our test target
element = {
'kind': 'import',
'sources': [
repo.source_config(ref=ref, checkout_submodules=True)
]
}
_yaml.dump(element, os.path.join(project, 'target.bst'))
# Fetch, build, checkout
result = cli.run(project=project, args=['fetch', 'target.bst'])
result.assert_success()
result = cli.run(project=project, args=['build', 'target.bst'])
result.assert_success()
result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir])
result.assert_success()
# Assert we checked out files at their expected location
assert os.path.exists(os.path.join(checkoutdir, 'file.txt'))
assert not os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt'))
assert os.path.exists(os.path.join(checkoutdir, 'othersubdir', 'unicornfile.txt'))
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'project-override'))
def test_submodule_fetch_project_override(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
checkoutdir = os.path.join(str(tmpdir), "checkout")
# Create the submodule first from the 'subrepofiles' subdir
subrepo = create_repo('git', str(tmpdir), 'subrepo')
subrepo.create(os.path.join(project, 'subrepofiles'))
# Create the repo from 'repofiles' subdir
repo = create_repo('git', str(tmpdir))
ref = repo.create(os.path.join(project, 'repofiles'))
# Add a submodule pointing to the one we created
ref = repo.add_submodule('subdir', 'file://' + subrepo.repo)
# Write out our test target
element = {
'kind': 'import',
'sources': [
repo.source_config(ref=ref)
]
}
_yaml.dump(element, os.path.join(project, 'target.bst'))
# Fetch, build, checkout
result = cli.run(project=project, args=['fetch', 'target.bst'])
result.assert_success()
result = cli.run(project=project, args=['build', 'target.bst'])
result.assert_success()
result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir])
result.assert_success()
# Assert we checked out both files at their expected location
assert os.path.exists(os.path.join(checkoutdir, 'file.txt'))
assert not os.path.exists(os.path.join(checkoutdir, 'subdir', 'ponyfile.txt'))
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
def test_submodule_track_ignore_inconsistent(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
# Create the repo from 'repofiles' subdir
repo = create_repo('git', str(tmpdir))
ref = repo.create(os.path.join(project, 'repofiles'))
# Write out our test target
element = {
'kind': 'import',
'sources': [
repo.source_config(ref=ref)
]
}
_yaml.dump(element, os.path.join(project, 'target.bst'))
# Now add a .gitmodules file with an inconsistent submodule,
# we are calling this inconsistent because the file was created
# but `git submodule add` was never called, so there is no reference
# associated to the submodule.
#
repo.add_file(os.path.join(project, 'inconsistent-submodule', '.gitmodules'))
# Fetch should work, we're not yet at the offending ref
result = cli.run(project=project, args=['fetch', 'target.bst'])
result.assert_success()
# Track will encounter an inconsistent submodule without any ref
result = cli.run(project=project, args=['track', 'target.bst'])
result.assert_success()
# Assert that we are just fine without it, and emit a warning to the user.
assert "Ignoring inconsistent submodule" in result.stderr
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
def test_submodule_track_no_ref_or_track(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
# Create the repo from 'repofiles' subdir
repo = create_repo('git', str(tmpdir))
ref = repo.create(os.path.join(project, 'repofiles'))
# Write out our test target
gitsource = repo.source_config(ref=None)
gitsource.pop('track')
element = {
'kind': 'import',
'sources': [
gitsource
]
}
_yaml.dump(element, os.path.join(project, 'target.bst'))
# Track will encounter an inconsistent submodule without any ref
result = cli.run(project=project, args=['show', 'target.bst'])
result.assert_main_error(ErrorDomain.SOURCE, "missing-track-and-ref")
result.assert_task_error(None, None)
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
def test_ref_not_in_track_warn(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
# Create the repo from 'repofiles', create a branch without latest commit
repo = create_repo('git', str(tmpdir))
ref = repo.create(os.path.join(project, 'repofiles'))
gitsource = repo.source_config(ref=ref)
# Overwrite the track value to the added branch
gitsource['track'] = 'foo'
# Write out our test target
element = {
'kind': 'import',
'sources': [
gitsource
]
}
_yaml.dump(element, os.path.join(project, 'target.bst'))
# Assert the warning is raised as ref is not in branch foo.
# Assert warning not error to the user, when not set as fatal.
result = cli.run(project=project, args=['build', 'target.bst'])
assert "The ref provided for the element does not exist locally" in result.stderr
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
def test_ref_not_in_track_warn_error(cli, tmpdir, datafiles):
project = os.path.join(datafiles.dirname, datafiles.basename)
# Add fatal-warnings ref-not-in-track to project.conf
project_template = {
"name": "foo",
"fatal-warnings": [CoreWarnings.REF_NOT_IN_TRACK]
}
_yaml.dump(project_template, os.path.join(project, 'project.conf'))
# Create the repo from 'repofiles', create a branch without latest commit
repo = create_repo('git', str(tmpdir))
ref = repo.create(os.path.join(project, 'repofiles'))
gitsource = repo.source_config(ref=ref)
# Overwrite the track value to the added branch
gitsource['track'] = 'foo'
# Write out our test target
element = {
'kind': 'import',
'sources': [
gitsource
]
}
_yaml.dump(element, os.path.join(project, 'target.bst'))
# Assert that build raises a warning here that is captured
# as plugin error, due to the fatal warning being set
result = cli.run(project=project, args=['build', 'target.bst'])
result.assert_main_error(ErrorDomain.STREAM, None)
result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.REF_NOT_IN_TRACK)
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
@pytest.mark.parametrize("ref_format", ['sha1', 'git-describe'])
@pytest.mark.parametrize("tag,extra_commit", [(False, False), (True, False), (True, True)])
def test_track_fetch(cli, tmpdir, datafiles, ref_format, tag, extra_commit):
project = os.path.join(datafiles.dirname, datafiles.basename)
# Create the repo from 'repofiles' subdir
repo = create_repo('git', str(tmpdir))
ref = repo.create(os.path.join(project, 'repofiles'))
if tag:
repo.add_tag('tag')
if extra_commit:
repo.add_commit()
# Write out our test target
element = {
'kind': 'import',
'sources': [
repo.source_config()
]
}
element['sources'][0]['ref-format'] = ref_format
element_path = os.path.join(project, 'target.bst')
_yaml.dump(element, element_path)
# Track it
result = cli.run(project=project, args=['track', 'target.bst'])
result.assert_success()
element = _yaml.load(element_path)
new_ref = element['sources'][0]['ref']
if ref_format == 'git-describe' and tag:
# Check and strip prefix
prefix = 'tag-{}-g'.format(0 if not extra_commit else 1)
assert new_ref.startswith(prefix)
new_ref = new_ref[len(prefix):]
# 40 chars for SHA-1
assert len(new_ref) == 40
# Fetch it
result = cli.run(project=project, args=['fetch', 'target.bst'])
result.assert_success()
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
@pytest.mark.parametrize("tag_type", [('annotated'), ('lightweight')])
def test_git_describe(cli, tmpdir, datafiles, ref_storage, tag_type):
project = str(datafiles)
project_config = _yaml.load(os.path.join(project, 'project.conf'))
project_config['ref-storage'] = ref_storage
_yaml.dump(_yaml.node_sanitize(project_config), os.path.join(project, 'project.conf'))
repofiles = os.path.join(str(tmpdir), 'repofiles')
os.makedirs(repofiles, exist_ok=True)
file0 = os.path.join(repofiles, 'file0')
with open(file0, 'w') as f:
f.write('test\n')
repo = create_repo('git', str(tmpdir))
def tag(name):
if tag_type == 'annotated':
repo.add_annotated_tag(name, name)
else:
repo.add_tag(name)
ref = repo.create(repofiles)
tag('uselesstag')
file1 = os.path.join(str(tmpdir), 'file1')
with open(file1, 'w') as f:
f.write('test\n')
repo.add_file(file1)
tag('tag1')
file2 = os.path.join(str(tmpdir), 'file2')
with open(file2, 'w') as f:
f.write('test\n')
repo.branch('branch2')
repo.add_file(file2)
tag('tag2')
repo.checkout('master')
file3 = os.path.join(str(tmpdir), 'file3')
with open(file3, 'w') as f:
f.write('test\n')
repo.add_file(file3)
repo.merge('branch2')
config = repo.source_config()
config['track'] = repo.latest_commit()
config['track-tags'] = True
# Write out our test target
element = {
'kind': 'import',
'sources': [
config
],
}
element_path = os.path.join(project, 'target.bst')
_yaml.dump(element, element_path)
if ref_storage == 'inline':
result = cli.run(project=project, args=['track', 'target.bst'])
result.assert_success()
else:
result = cli.run(project=project, args=['track', 'target.bst', '--deps', 'all'])
result.assert_success()
if ref_storage == 'inline':
element = _yaml.load(element_path)
tags = _yaml.node_sanitize(element['sources'][0]['tags'])
assert len(tags) == 2
for tag in tags:
assert 'tag' in tag
assert 'commit' in tag
assert 'annotated' in tag
assert tag['annotated'] == (tag_type == 'annotated')
assert set([(tag['tag'], tag['commit']) for tag in tags]) == set([('tag1', repo.rev_parse('tag1^{commit}')),
('tag2', repo.rev_parse('tag2^{commit}'))])
checkout = os.path.join(str(tmpdir), 'checkout')
result = cli.run(project=project, args=['build', 'target.bst'])
result.assert_success()
result = cli.run(project=project, args=['checkout', 'target.bst', checkout])
result.assert_success()
if tag_type == 'annotated':
options = []
else:
options = ['--tags']
describe = subprocess.check_output(['git', 'describe'] + options,
cwd=checkout).decode('ascii')
assert describe.startswith('tag2-2-')
describe_fp = subprocess.check_output(['git', 'describe', '--first-parent'] + options,
cwd=checkout).decode('ascii')
assert describe_fp.startswith('tag1-2-')
tags = subprocess.check_output(['git', 'tag'],
cwd=checkout).decode('ascii')
tags = set(tags.splitlines())
assert tags == set(['tag1', 'tag2'])
p = subprocess.run(['git', 'log', repo.rev_parse('uselesstag')],
cwd=checkout)
assert p.returncode != 0
@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
def test_default_do_not_track_tags(cli, tmpdir, datafiles):
project = str(datafiles)
project_config = _yaml.load(os.path.join(project, 'project.conf'))
project_config['ref-storage'] = 'inline'
_yaml.dump(_yaml.node_sanitize(project_config), os.path.join(project, 'project.conf'))
repofiles = os.path.join(str(tmpdir), 'repofiles')
os.makedirs(repofiles, exist_ok=True)
file0 = os.path.join(repofiles, 'file0')
with open(file0, 'w') as f:
f.write('test\n')
repo = create_repo('git', str(tmpdir))
ref = repo.create(repofiles)
repo.add_tag('tag')
config = repo.source_config()
config['track'] = repo.latest_commit()
# Write out our test target
element = {
'kind': 'import',
'sources': [
config
],
}
element_path = os.path.join(project, 'target.bst')
_yaml.dump(element, element_path)
result = cli.run(project=project, args=['track', 'target.bst'])
result.assert_success()
element = _yaml.load(element_path)
assert 'tags' not in element['sources'][0]