diff options
-rw-r--r-- | NEWS | 4 | ||||
-rw-r--r-- | buildstream/_versions.py | 2 | ||||
-rw-r--r-- | buildstream/plugins/sources/git.py | 216 | ||||
-rw-r--r-- | tests/cachekey/project/sources/git3.bst | 12 | ||||
-rw-r--r-- | tests/cachekey/project/sources/git3.expected | 1 | ||||
-rw-r--r-- | tests/cachekey/project/target.bst | 1 | ||||
-rw-r--r-- | tests/cachekey/project/target.expected | 2 | ||||
-rw-r--r-- | tests/sources/git.py | 153 | ||||
-rw-r--r-- | tests/testutils/repo/git.py | 14 |
9 files changed, 390 insertions, 15 deletions
@@ -74,6 +74,10 @@ buildstream 1.3.1 o Add sandbox API for command batching and use it for build, script, and compose elements. + o BREAKING CHANGE: The `git` plugin does not create a local `.git` + repository by default. If `git describe` is required to work, the + plugin has now a tag tracking feature instead. This can be enabled + by setting 'track-tags'. ================= buildstream 1.1.5 diff --git a/buildstream/_versions.py b/buildstream/_versions.py index 842ac8bf8..42dcb1a31 100644 --- a/buildstream/_versions.py +++ b/buildstream/_versions.py @@ -23,7 +23,7 @@ # This version is bumped whenever enhancements are made # to the `project.conf` format or the core element format. # -BST_FORMAT_VERSION = 18 +BST_FORMAT_VERSION = 19 # The base BuildStream artifact version diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py index 8586ebcf8..e3dded1d4 100644 --- a/buildstream/plugins/sources/git.py +++ b/buildstream/plugins/sources/git.py @@ -76,6 +76,56 @@ git - stage files from a git repository url: upstream:baz.git checkout: False + # Enable tag tracking. + # + # This causes the `tags` metadata to be populated automatically + # as a result of tracking the git source. + # + # By default this is 'False'. + # + track-tags: True + + # If the list of tags below is set, then a lightweight dummy + # git repository will be staged along with the content at + # build time. + # + # This is useful for a growing number of modules which use + # `git describe` at build time in order to determine the version + # which will be encoded into the built software. + # + # The 'tags' below is considered as a part of the git source + # reference and will be stored in the 'project.refs' file if + # that has been selected as your project's ref-storage. + # + # Migration notes: + # + # If you are upgrading from BuildStream 1.2, which used to + # stage the entire repository by default, you will notice that + # some modules which use `git describe` are broken, and will + # need to enable this feature in order to fix them. + # + # If you need to enable this feature without changing the + # the specific commit that you are building, then we recommend + # the following migration steps for any git sources where + # `git describe` is required: + # + # o Enable `track-tags` feature + # o Set the `track` parameter to the desired commit sha which + # the current `ref` points to + # o Run `bst track` for these elements, this will result in + # populating the `tags` portion of the refs without changing + # the refs + # o Restore the `track` parameter to the branches which you have + # previously been tracking afterwards. + # + tags: + - tag: lightweight-example + commit: 04ad0dc656cb7cc6feb781aa13bdbf1d67d0af78 + annotated: false + - tag: annotated-example + commit: 10abe77fe8d77385d86f225b503d9185f4ef7f3a + annotated: true + See :ref:`built-in functionality doumentation <core_source_builtins>` for details on common configuration options for sources. @@ -95,6 +145,7 @@ import re import shutil from collections.abc import Mapping from io import StringIO +from tempfile import TemporaryFile from configparser import RawConfigParser @@ -115,13 +166,14 @@ INCONSISTENT_SUBMODULE = "inconsistent-submodules" # class GitMirror(SourceFetcher): - def __init__(self, source, path, url, ref, *, primary=False): + def __init__(self, source, path, url, ref, *, primary=False, tags=[]): super().__init__() self.source = source self.path = path self.url = url self.ref = ref + self.tags = tags self.primary = primary self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(url)) self.mark_download_url(url) @@ -214,7 +266,7 @@ class GitMirror(SourceFetcher): raise SourceError("{}: expected ref '{}' was not found in git repository: '{}'" .format(self.source, self.ref, self.url)) - def latest_commit(self, tracking): + def latest_commit_with_tags(self, tracking, track_tags=False): _, output = self.source.check_output( [self.source.host_git, 'rev-parse', tracking], fail="Unable to find commit for specified branch name '{}'".format(tracking), @@ -230,7 +282,28 @@ class GitMirror(SourceFetcher): if exit_code == 0: ref = output.rstrip('\n') - return ref + if not track_tags: + return ref, [] + + tags = set() + for options in [[], ['--first-parent'], ['--tags'], ['--tags', '--first-parent']]: + exit_code, output = self.source.check_output( + [self.source.host_git, 'describe', '--abbrev=0', ref] + options, + cwd=self.mirror) + if exit_code == 0: + tag = output.strip() + _, commit_ref = self.source.check_output( + [self.source.host_git, 'rev-parse', tag + '^{commit}'], + fail="Unable to resolve tag '{}'".format(tag), + cwd=self.mirror) + exit_code = self.source.call( + [self.source.host_git, 'cat-file', 'tag', tag], + cwd=self.mirror) + annotated = (exit_code == 0) + + tags.add((tag, commit_ref.strip(), annotated)) + + return ref, list(tags) def stage(self, directory, track=None): fullpath = os.path.join(directory, self.path) @@ -246,13 +319,15 @@ class GitMirror(SourceFetcher): fail="Failed to checkout git ref {}".format(self.ref), cwd=fullpath) + # Remove .git dir + shutil.rmtree(os.path.join(fullpath, ".git")) + + self._rebuild_git(fullpath) + # Check that the user specified ref exists in the track if provided & not already tracked if track: self.assert_ref_in_track(fullpath, track) - # Remove .git dir - shutil.rmtree(os.path.join(fullpath, ".git")) - def init_workspace(self, directory, track=None): fullpath = os.path.join(directory, self.path) url = self.source.translate_url(self.url) @@ -359,6 +434,78 @@ class GitMirror(SourceFetcher): .format(self.source, self.ref, track, self.url), detail=detail, warning_token=CoreWarnings.REF_NOT_IN_TRACK) + def _rebuild_git(self, fullpath): + if not self.tags: + return + + with self.source.tempdir() as tmpdir: + included = set() + shallow = set() + for _, commit_ref, _ in self.tags: + + _, out = self.source.check_output([self.source.host_git, 'rev-list', + '--boundary', '{}..{}'.format(commit_ref, self.ref)], + fail="Failed to get git history {}..{} in directory: {}" + .format(commit_ref, self.ref, fullpath), + fail_temporarily=True, + cwd=self.mirror) + for line in out.splitlines(): + rev = line.lstrip('-') + if line[0] == '-': + shallow.add(rev) + else: + included.add(rev) + + shallow -= included + included |= shallow + + self.source.call([self.source.host_git, 'init'], + fail="Cannot initialize git repository: {}".format(fullpath), + cwd=fullpath) + + for rev in included: + with TemporaryFile(dir=tmpdir) as commit_file: + self.source.call([self.source.host_git, 'cat-file', 'commit', rev], + stdout=commit_file, + fail="Failed to get commit {}".format(rev), + cwd=self.mirror) + commit_file.seek(0, 0) + self.source.call([self.source.host_git, 'hash-object', '-w', '-t', 'commit', '--stdin'], + stdin=commit_file, + fail="Failed to add commit object {}".format(rev), + cwd=fullpath) + + with open(os.path.join(fullpath, '.git', 'shallow'), 'w') as shallow_file: + for rev in shallow: + shallow_file.write('{}\n'.format(rev)) + + for tag, commit_ref, annotated in self.tags: + if annotated: + with TemporaryFile(dir=tmpdir) as tag_file: + tag_data = 'object {}\ntype commit\ntag {}\n'.format(commit_ref, tag) + tag_file.write(tag_data.encode('ascii')) + tag_file.seek(0, 0) + _, tag_ref = self.source.check_output( + [self.source.host_git, 'hash-object', '-w', '-t', + 'tag', '--stdin'], + stdin=tag_file, + fail="Failed to add tag object {}".format(tag), + cwd=fullpath) + + self.source.call([self.source.host_git, 'tag', tag, tag_ref.strip()], + fail="Failed to tag: {}".format(tag), + cwd=fullpath) + else: + self.source.call([self.source.host_git, 'tag', tag, commit_ref], + fail="Failed to tag: {}".format(tag), + cwd=fullpath) + + with open(os.path.join(fullpath, '.git', 'HEAD'), 'w') as head: + self.source.call([self.source.host_git, 'rev-parse', self.ref], + stdout=head, + fail="Failed to parse commit {}".format(self.ref), + cwd=self.mirror) + class GitSource(Source): # pylint: disable=attribute-defined-outside-init @@ -366,11 +513,20 @@ class GitSource(Source): def configure(self, node): ref = self.node_get_member(node, str, 'ref', None) - config_keys = ['url', 'track', 'ref', 'submodules', 'checkout-submodules', 'ref-format'] + config_keys = ['url', 'track', 'ref', 'submodules', + 'checkout-submodules', 'ref-format', + 'track-tags', 'tags'] self.node_validate(node, config_keys + Source.COMMON_CONFIG_KEYS) + tags_node = self.node_get_member(node, list, 'tags', []) + for tag_node in tags_node: + self.node_validate(tag_node, ['tag', 'commit', 'annotated']) + + tags = self._load_tags(node) + self.track_tags = self.node_get_member(node, bool, 'track-tags', False) + self.original_url = self.node_get_member(node, str, 'url') - self.mirror = GitMirror(self, '', self.original_url, ref, primary=True) + self.mirror = GitMirror(self, '', self.original_url, ref, tags=tags, primary=True) self.tracking = self.node_get_member(node, str, 'track', None) self.ref_format = self.node_get_member(node, str, 'ref-format', 'sha1') @@ -417,6 +573,9 @@ class GitSource(Source): # the ref, if the user changes the alias to fetch the same sources # from another location, it should not affect the cache key. key = [self.original_url, self.mirror.ref] + if self.mirror.tags: + tags = {tag: (commit, annotated) for tag, commit, annotated in self.mirror.tags} + key.append({'tags': tags}) # Only modify the cache key with checkout_submodules if it's something # other than the default behaviour. @@ -442,12 +601,33 @@ class GitSource(Source): def load_ref(self, node): self.mirror.ref = self.node_get_member(node, str, 'ref', None) + self.mirror.tags = self._load_tags(node) def get_ref(self): - return self.mirror.ref - - def set_ref(self, ref, node): - node['ref'] = self.mirror.ref = ref + return self.mirror.ref, self.mirror.tags + + def set_ref(self, ref_data, node): + if not ref_data: + self.mirror.ref = None + if 'ref' in node: + del node['ref'] + self.mirror.tags = [] + if 'tags' in node: + del node['tags'] + else: + ref, tags = ref_data + node['ref'] = self.mirror.ref = ref + self.mirror.tags = tags + if tags: + node['tags'] = [] + for tag, commit_ref, annotated in tags: + data = {'tag': tag, + 'commit': commit_ref, + 'annotated': annotated} + node['tags'].append(data) + else: + if 'tags' in node: + del node['tags'] def track(self): @@ -470,7 +650,7 @@ class GitSource(Source): self.mirror._fetch() # Update self.mirror.ref and node.ref from the self.tracking branch - ret = self.mirror.latest_commit(self.tracking) + ret = self.mirror.latest_commit_with_tags(self.tracking, self.track_tags) # Set tracked attribute, parameter for if self.mirror.assert_ref_in_track is needed self.tracked = True @@ -556,6 +736,16 @@ class GitSource(Source): self.submodules = submodules + def _load_tags(self, node): + tags = [] + tags_node = self.node_get_member(node, list, 'tags', []) + for tag_node in tags_node: + tag = self.node_get_member(tag_node, str, 'tag') + commit_ref = self.node_get_member(tag_node, str, 'commit') + annotated = self.node_get_member(tag_node, bool, 'annotated') + tags.append((tag, commit_ref, annotated)) + return tags + # Plugin entry point def setup(): diff --git a/tests/cachekey/project/sources/git3.bst b/tests/cachekey/project/sources/git3.bst new file mode 100644 index 000000000..b331a3af3 --- /dev/null +++ b/tests/cachekey/project/sources/git3.bst @@ -0,0 +1,12 @@ +kind: import +sources: +- kind: git + url: https://example.com/git/repo.git + ref: 6ac68af3e80b7b17c23a3c65233043550a7fa685 + tags: + - tag: lightweight + commit: 0a3917d57477ee9afe7be49a0e8a76f56d176df1 + annotated: false + - tag: annotated + commit: 68c7f0bd386684742c41ec2a54ce2325e3922f6c + annotated: true diff --git a/tests/cachekey/project/sources/git3.expected b/tests/cachekey/project/sources/git3.expected new file mode 100644 index 000000000..b383ccbfc --- /dev/null +++ b/tests/cachekey/project/sources/git3.expected @@ -0,0 +1 @@ +6a25f539bd8629a36399c58efd2f5c9c117feb845076a37dc321b55d456932b6
\ No newline at end of file diff --git a/tests/cachekey/project/target.bst b/tests/cachekey/project/target.bst index d96645da8..8c5c12723 100644 --- a/tests/cachekey/project/target.bst +++ b/tests/cachekey/project/target.bst @@ -7,6 +7,7 @@ depends: - sources/bzr1.bst - sources/git1.bst - sources/git2.bst +- sources/git3.bst - sources/local1.bst - sources/local2.bst - sources/ostree1.bst diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected index 640133e23..0c89af6bb 100644 --- a/tests/cachekey/project/target.expected +++ b/tests/cachekey/project/target.expected @@ -1 +1 @@ -125d9e7dcf4f49e5f80d85b7f144b43ed43186064afc2e596e57f26cce679cf5
\ No newline at end of file +bc99c288f855ac2619787f0067223f7812d2e10a9d2c7f2bf47de7113c0fd25c
\ No newline at end of file diff --git a/tests/sources/git.py b/tests/sources/git.py index 7ab32a6b5..462200f97 100644 --- a/tests/sources/git.py +++ b/tests/sources/git.py @@ -22,6 +22,7 @@ import os import pytest +import subprocess from buildstream._exceptions import ErrorDomain from buildstream import _yaml @@ -523,3 +524,155 @@ def test_track_fetch(cli, tmpdir, datafiles, ref_format, tag, extra_commit): # Fetch it result = cli.run(project=project, args=['fetch', 'target.bst']) result.assert_success() + + +@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) +@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) +@pytest.mark.parametrize("tag_type", [('annotated'), ('lightweight')]) +def test_git_describe(cli, tmpdir, datafiles, ref_storage, tag_type): + project = str(datafiles) + + project_config = _yaml.load(os.path.join(project, 'project.conf')) + project_config['ref-storage'] = ref_storage + _yaml.dump(_yaml.node_sanitize(project_config), os.path.join(project, 'project.conf')) + + repofiles = os.path.join(str(tmpdir), 'repofiles') + os.makedirs(repofiles, exist_ok=True) + file0 = os.path.join(repofiles, 'file0') + with open(file0, 'w') as f: + f.write('test\n') + + repo = create_repo('git', str(tmpdir)) + + def tag(name): + if tag_type == 'annotated': + repo.add_annotated_tag(name, name) + else: + repo.add_tag(name) + + ref = repo.create(repofiles) + tag('uselesstag') + + file1 = os.path.join(str(tmpdir), 'file1') + with open(file1, 'w') as f: + f.write('test\n') + repo.add_file(file1) + tag('tag1') + + file2 = os.path.join(str(tmpdir), 'file2') + with open(file2, 'w') as f: + f.write('test\n') + repo.branch('branch2') + repo.add_file(file2) + tag('tag2') + + repo.checkout('master') + file3 = os.path.join(str(tmpdir), 'file3') + with open(file3, 'w') as f: + f.write('test\n') + repo.add_file(file3) + + repo.merge('branch2') + + config = repo.source_config() + config['track'] = repo.latest_commit() + config['track-tags'] = True + + # Write out our test target + element = { + 'kind': 'import', + 'sources': [ + config + ], + } + element_path = os.path.join(project, 'target.bst') + _yaml.dump(element, element_path) + + if ref_storage == 'inline': + result = cli.run(project=project, args=['track', 'target.bst']) + result.assert_success() + else: + result = cli.run(project=project, args=['track', 'target.bst', '--deps', 'all']) + result.assert_success() + + if ref_storage == 'inline': + element = _yaml.load(element_path) + tags = _yaml.node_sanitize(element['sources'][0]['tags']) + assert len(tags) == 2 + for tag in tags: + assert 'tag' in tag + assert 'commit' in tag + assert 'annotated' in tag + assert tag['annotated'] == (tag_type == 'annotated') + + assert set([(tag['tag'], tag['commit']) for tag in tags]) == set([('tag1', repo.rev_parse('tag1^{commit}')), + ('tag2', repo.rev_parse('tag2^{commit}'))]) + + checkout = os.path.join(str(tmpdir), 'checkout') + + result = cli.run(project=project, args=['build', 'target.bst']) + result.assert_success() + result = cli.run(project=project, args=['checkout', 'target.bst', checkout]) + result.assert_success() + + if tag_type == 'annotated': + options = [] + else: + options = ['--tags'] + describe = subprocess.check_output(['git', 'describe'] + options, + cwd=checkout).decode('ascii') + assert describe.startswith('tag2-2-') + + describe_fp = subprocess.check_output(['git', 'describe', '--first-parent'] + options, + cwd=checkout).decode('ascii') + assert describe_fp.startswith('tag1-2-') + + tags = subprocess.check_output(['git', 'tag'], + cwd=checkout).decode('ascii') + tags = set(tags.splitlines()) + assert tags == set(['tag1', 'tag2']) + + p = subprocess.run(['git', 'log', repo.rev_parse('uselesstag')], + cwd=checkout) + assert p.returncode != 0 + + +@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available") +@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template')) +def test_default_do_not_track_tags(cli, tmpdir, datafiles): + project = str(datafiles) + + project_config = _yaml.load(os.path.join(project, 'project.conf')) + project_config['ref-storage'] = 'inline' + _yaml.dump(_yaml.node_sanitize(project_config), os.path.join(project, 'project.conf')) + + repofiles = os.path.join(str(tmpdir), 'repofiles') + os.makedirs(repofiles, exist_ok=True) + file0 = os.path.join(repofiles, 'file0') + with open(file0, 'w') as f: + f.write('test\n') + + repo = create_repo('git', str(tmpdir)) + + ref = repo.create(repofiles) + repo.add_tag('tag') + + config = repo.source_config() + config['track'] = repo.latest_commit() + + # Write out our test target + element = { + 'kind': 'import', + 'sources': [ + config + ], + } + element_path = os.path.join(project, 'target.bst') + _yaml.dump(element, element_path) + + result = cli.run(project=project, args=['track', 'target.bst']) + result.assert_success() + + element = _yaml.load(element_path) + assert 'tags' not in element['sources'][0] diff --git a/tests/testutils/repo/git.py b/tests/testutils/repo/git.py index bc2dae691..fe3ebd547 100644 --- a/tests/testutils/repo/git.py +++ b/tests/testutils/repo/git.py @@ -45,6 +45,9 @@ class Git(Repo): def add_tag(self, tag): self._run_git('tag', tag) + def add_annotated_tag(self, tag, message): + self._run_git('tag', '-a', tag, '-m', message) + def add_commit(self): self._run_git('commit', '--allow-empty', '-m', 'Additional commit') return self.latest_commit() @@ -95,3 +98,14 @@ class Git(Repo): def branch(self, branch_name): self._run_git('checkout', '-b', branch_name) + + def checkout(self, commit): + self._run_git('checkout', commit) + + def merge(self, commit): + self._run_git('merge', '-m', 'Merge', commit) + return self.latest_commit() + + def rev_parse(self, rev): + output = self._run_git('rev-parse', rev, stdout=subprocess.PIPE).stdout + return output.decode('UTF-8').strip() |