git source plugin: Track git tags and save them to reproduce a minimum shallow repository

Instead of tag information being fetched which can change with time,
they are tracked and saved in the projects.refs/.bst. Then we re-tag
automatically the closest tag so that `git describe` works and is
reproducible.

This new feature is opt-in with the new `track-tags` configuration,
and must be used to fix modules which are broken by our new policy
of omitting the `.git/` repository when staging git sources.

This fixes issue #487

8 files changed, 386 insertions, 15 deletions

diff --git a/buildstream/_versions.py b/buildstream/_versions.py
index 842ac8bf8..42dcb1a31 100644
--- a/buildstream/_versions.py
+++ b/buildstream/_versions.py
@@ -23,7 +23,7 @@
 # This version is bumped whenever enhancements are made
 # to the `project.conf` format or the core element format.
 #
-BST_FORMAT_VERSION = 18
+BST_FORMAT_VERSION = 19
 
 
 # The base BuildStream artifact version
diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py
index 8586ebcf8..e3dded1d4 100644
--- a/buildstream/plugins/sources/git.py
+++ b/buildstream/plugins/sources/git.py
@@ -76,6 +76,56 @@ git - stage files from a git repository
        url: upstream:baz.git
        checkout: False
 
+   # Enable tag tracking.
+   #
+   # This causes the `tags` metadata to be populated automatically
+   # as a result of tracking the git source.
+   #
+   # By default this is 'False'.
+   #
+   track-tags: True
+
+   # If the list of tags below is set, then a lightweight dummy
+   # git repository will be staged along with the content at
+   # build time.
+   #
+   # This is useful for a growing number of modules which use
+   # `git describe` at build time in order to determine the version
+   # which will be encoded into the built software.
+   #
+   # The 'tags' below is considered as a part of the git source
+   # reference and will be stored in the 'project.refs' file if
+   # that has been selected as your project's ref-storage.
+   #
+   # Migration notes:
+   #
+   #   If you are upgrading from BuildStream 1.2, which used to
+   #   stage the entire repository by default, you will notice that
+   #   some modules which use `git describe` are broken, and will
+   #   need to enable this feature in order to fix them.
+   #
+   #   If you need to enable this feature without changing the
+   #   the specific commit that you are building, then we recommend
+   #   the following migration steps for any git sources where
+   #   `git describe` is required:
+   #
+   #     o Enable `track-tags` feature
+   #     o Set the `track` parameter to the desired commit sha which
+   #       the current `ref` points to
+   #     o Run `bst track` for these elements, this will result in
+   #       populating the `tags` portion of the refs without changing
+   #       the refs
+   #     o Restore the `track` parameter to the branches which you have
+   #       previously been tracking afterwards.
+   #
+   tags:
+   - tag: lightweight-example
+     commit: 04ad0dc656cb7cc6feb781aa13bdbf1d67d0af78
+     annotated: false
+   - tag: annotated-example
+     commit: 10abe77fe8d77385d86f225b503d9185f4ef7f3a
+     annotated: true
+
 See :ref:`built-in functionality doumentation <core_source_builtins>` for
 details on common configuration options for sources.
 
@@ -95,6 +145,7 @@ import re
 import shutil
 from collections.abc import Mapping
 from io import StringIO
+from tempfile import TemporaryFile
 
 from configparser import RawConfigParser
 
@@ -115,13 +166,14 @@ INCONSISTENT_SUBMODULE = "inconsistent-submodules"
 #
 class GitMirror(SourceFetcher):
 
-    def __init__(self, source, path, url, ref, *, primary=False):
+    def __init__(self, source, path, url, ref, *, primary=False, tags=[]):
 
         super().__init__()
         self.source = source
         self.path = path
         self.url = url
         self.ref = ref
+        self.tags = tags
         self.primary = primary
         self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(url))
         self.mark_download_url(url)
@@ -214,7 +266,7 @@ class GitMirror(SourceFetcher):
             raise SourceError("{}: expected ref '{}' was not found in git repository: '{}'"
                               .format(self.source, self.ref, self.url))
 
-    def latest_commit(self, tracking):
+    def latest_commit_with_tags(self, tracking, track_tags=False):
         _, output = self.source.check_output(
             [self.source.host_git, 'rev-parse', tracking],
             fail="Unable to find commit for specified branch name '{}'".format(tracking),
@@ -230,7 +282,28 @@ class GitMirror(SourceFetcher):
             if exit_code == 0:
                 ref = output.rstrip('\n')
 
-        return ref
+        if not track_tags:
+            return ref, []
+
+        tags = set()
+        for options in [[], ['--first-parent'], ['--tags'], ['--tags', '--first-parent']]:
+            exit_code, output = self.source.check_output(
+                [self.source.host_git, 'describe', '--abbrev=0', ref] + options,
+                cwd=self.mirror)
+            if exit_code == 0:
+                tag = output.strip()
+                _, commit_ref = self.source.check_output(
+                    [self.source.host_git, 'rev-parse', tag + '^{commit}'],
+                    fail="Unable to resolve tag '{}'".format(tag),
+                    cwd=self.mirror)
+                exit_code = self.source.call(
+                    [self.source.host_git, 'cat-file', 'tag', tag],
+                    cwd=self.mirror)
+                annotated = (exit_code == 0)
+
+                tags.add((tag, commit_ref.strip(), annotated))
+
+        return ref, list(tags)
 
     def stage(self, directory, track=None):
         fullpath = os.path.join(directory, self.path)
@@ -246,13 +319,15 @@ class GitMirror(SourceFetcher):
                          fail="Failed to checkout git ref {}".format(self.ref),
                          cwd=fullpath)
 
+        # Remove .git dir
+        shutil.rmtree(os.path.join(fullpath, ".git"))
+
+        self._rebuild_git(fullpath)
+
         # Check that the user specified ref exists in the track if provided & not already tracked
         if track:
             self.assert_ref_in_track(fullpath, track)
 
-        # Remove .git dir
-        shutil.rmtree(os.path.join(fullpath, ".git"))
-
     def init_workspace(self, directory, track=None):
         fullpath = os.path.join(directory, self.path)
         url = self.source.translate_url(self.url)
@@ -359,6 +434,78 @@ class GitMirror(SourceFetcher):
                          .format(self.source, self.ref, track, self.url),
                          detail=detail, warning_token=CoreWarnings.REF_NOT_IN_TRACK)
 
+    def _rebuild_git(self, fullpath):
+        if not self.tags:
+            return
+
+        with self.source.tempdir() as tmpdir:
+            included = set()
+            shallow = set()
+            for _, commit_ref, _ in self.tags:
+
+                _, out = self.source.check_output([self.source.host_git, 'rev-list',
+                                                   '--boundary', '{}..{}'.format(commit_ref, self.ref)],
+                                                  fail="Failed to get git history {}..{} in directory: {}"
+                                                  .format(commit_ref, self.ref, fullpath),
+                                                  fail_temporarily=True,
+                                                  cwd=self.mirror)
+                for line in out.splitlines():
+                    rev = line.lstrip('-')
+                    if line[0] == '-':
+                        shallow.add(rev)
+                    else:
+                        included.add(rev)
+
+            shallow -= included
+            included |= shallow
+
+            self.source.call([self.source.host_git, 'init'],
+                             fail="Cannot initialize git repository: {}".format(fullpath),
+                             cwd=fullpath)
+
+            for rev in included:
+                with TemporaryFile(dir=tmpdir) as commit_file:
+                    self.source.call([self.source.host_git, 'cat-file', 'commit', rev],
+                                     stdout=commit_file,
+                                     fail="Failed to get commit {}".format(rev),
+                                     cwd=self.mirror)
+                    commit_file.seek(0, 0)
+                    self.source.call([self.source.host_git, 'hash-object', '-w', '-t', 'commit', '--stdin'],
+                                     stdin=commit_file,
+                                     fail="Failed to add commit object {}".format(rev),
+                                     cwd=fullpath)
+
+            with open(os.path.join(fullpath, '.git', 'shallow'), 'w') as shallow_file:
+                for rev in shallow:
+                    shallow_file.write('{}\n'.format(rev))
+
+            for tag, commit_ref, annotated in self.tags:
+                if annotated:
+                    with TemporaryFile(dir=tmpdir) as tag_file:
+                        tag_data = 'object {}\ntype commit\ntag {}\n'.format(commit_ref, tag)
+                        tag_file.write(tag_data.encode('ascii'))
+                        tag_file.seek(0, 0)
+                        _, tag_ref = self.source.check_output(
+                            [self.source.host_git, 'hash-object', '-w', '-t',
+                             'tag', '--stdin'],
+                            stdin=tag_file,
+                            fail="Failed to add tag object {}".format(tag),
+                            cwd=fullpath)
+
+                    self.source.call([self.source.host_git, 'tag', tag, tag_ref.strip()],
+                                     fail="Failed to tag: {}".format(tag),
+                                     cwd=fullpath)
+                else:
+                    self.source.call([self.source.host_git, 'tag', tag, commit_ref],
+                                     fail="Failed to tag: {}".format(tag),
+                                     cwd=fullpath)
+
+            with open(os.path.join(fullpath, '.git', 'HEAD'), 'w') as head:
+                self.source.call([self.source.host_git, 'rev-parse', self.ref],
+                                 stdout=head,
+                                 fail="Failed to parse commit {}".format(self.ref),
+                                 cwd=self.mirror)
+
 
 class GitSource(Source):
     # pylint: disable=attribute-defined-outside-init
@@ -366,11 +513,20 @@ class GitSource(Source):
     def configure(self, node):
         ref = self.node_get_member(node, str, 'ref', None)
 
-        config_keys = ['url', 'track', 'ref', 'submodules', 'checkout-submodules', 'ref-format']
+        config_keys = ['url', 'track', 'ref', 'submodules',
+                       'checkout-submodules', 'ref-format',
+                       'track-tags', 'tags']
         self.node_validate(node, config_keys + Source.COMMON_CONFIG_KEYS)
 
+        tags_node = self.node_get_member(node, list, 'tags', [])
+        for tag_node in tags_node:
+            self.node_validate(tag_node, ['tag', 'commit', 'annotated'])
+
+        tags = self._load_tags(node)
+        self.track_tags = self.node_get_member(node, bool, 'track-tags', False)
+
         self.original_url = self.node_get_member(node, str, 'url')
-        self.mirror = GitMirror(self, '', self.original_url, ref, primary=True)
+        self.mirror = GitMirror(self, '', self.original_url, ref, tags=tags, primary=True)
         self.tracking = self.node_get_member(node, str, 'track', None)
 
         self.ref_format = self.node_get_member(node, str, 'ref-format', 'sha1')
@@ -417,6 +573,9 @@ class GitSource(Source):
         # the ref, if the user changes the alias to fetch the same sources
         # from another location, it should not affect the cache key.
         key = [self.original_url, self.mirror.ref]
+        if self.mirror.tags:
+            tags = {tag: (commit, annotated) for tag, commit, annotated in self.mirror.tags}
+            key.append({'tags': tags})
 
         # Only modify the cache key with checkout_submodules if it's something
         # other than the default behaviour.
@@ -442,12 +601,33 @@ class GitSource(Source):
 
     def load_ref(self, node):
         self.mirror.ref = self.node_get_member(node, str, 'ref', None)
+        self.mirror.tags = self._load_tags(node)
 
     def get_ref(self):
-        return self.mirror.ref
-
-    def set_ref(self, ref, node):
-        node['ref'] = self.mirror.ref = ref
+        return self.mirror.ref, self.mirror.tags
+
+    def set_ref(self, ref_data, node):
+        if not ref_data:
+            self.mirror.ref = None
+            if 'ref' in node:
+                del node['ref']
+            self.mirror.tags = []
+            if 'tags' in node:
+                del node['tags']
+        else:
+            ref, tags = ref_data
+            node['ref'] = self.mirror.ref = ref
+            self.mirror.tags = tags
+            if tags:
+                node['tags'] = []
+                for tag, commit_ref, annotated in tags:
+                    data = {'tag': tag,
+                            'commit': commit_ref,
+                            'annotated': annotated}
+                    node['tags'].append(data)
+            else:
+                if 'tags' in node:
+                    del node['tags']
 
     def track(self):
 
@@ -470,7 +650,7 @@ class GitSource(Source):
             self.mirror._fetch()
 
             # Update self.mirror.ref and node.ref from the self.tracking branch
-            ret = self.mirror.latest_commit(self.tracking)
+            ret = self.mirror.latest_commit_with_tags(self.tracking, self.track_tags)
 
         # Set tracked attribute, parameter for if self.mirror.assert_ref_in_track is needed
         self.tracked = True
@@ -556,6 +736,16 @@ class GitSource(Source):
 
         self.submodules = submodules
 
+    def _load_tags(self, node):
+        tags = []
+        tags_node = self.node_get_member(node, list, 'tags', [])
+        for tag_node in tags_node:
+            tag = self.node_get_member(tag_node, str, 'tag')
+            commit_ref = self.node_get_member(tag_node, str, 'commit')
+            annotated = self.node_get_member(tag_node, bool, 'annotated')
+            tags.append((tag, commit_ref, annotated))
+        return tags
+
 
 # Plugin entry point
 def setup():
diff --git a/tests/cachekey/project/sources/git3.bst b/tests/cachekey/project/sources/git3.bst
new file mode 100644
index 000000000..b331a3af3
--- /dev/null
+++ b/tests/cachekey/project/sources/git3.bst
@@ -0,0 +1,12 @@
+kind: import
+sources:
+- kind: git
+  url: https://example.com/git/repo.git
+  ref: 6ac68af3e80b7b17c23a3c65233043550a7fa685
+  tags:
+  - tag: lightweight
+    commit: 0a3917d57477ee9afe7be49a0e8a76f56d176df1
+    annotated: false
+  - tag: annotated
+    commit: 68c7f0bd386684742c41ec2a54ce2325e3922f6c
+    annotated: true
diff --git a/tests/cachekey/project/sources/git3.expected b/tests/cachekey/project/sources/git3.expected
new file mode 100644
index 000000000..b383ccbfc
--- /dev/null
+++ b/tests/cachekey/project/sources/git3.expected
@@ -0,0 +1 @@
+6a25f539bd8629a36399c58efd2f5c9c117feb845076a37dc321b55d456932b6
\ No newline at end of file
diff --git a/tests/cachekey/project/target.bst b/tests/cachekey/project/target.bst
index d96645da8..8c5c12723 100644
--- a/tests/cachekey/project/target.bst
+++ b/tests/cachekey/project/target.bst
@@ -7,6 +7,7 @@ depends:
 - sources/bzr1.bst
 - sources/git1.bst
 - sources/git2.bst
+- sources/git3.bst
 - sources/local1.bst
 - sources/local2.bst
 - sources/ostree1.bst
diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected
index 640133e23..0c89af6bb 100644
--- a/tests/cachekey/project/target.expected
+++ b/tests/cachekey/project/target.expected
@@ -1 +1 @@
-125d9e7dcf4f49e5f80d85b7f144b43ed43186064afc2e596e57f26cce679cf5
\ No newline at end of file
+bc99c288f855ac2619787f0067223f7812d2e10a9d2c7f2bf47de7113c0fd25c
\ No newline at end of file
diff --git a/tests/sources/git.py b/tests/sources/git.py
index 7ab32a6b5..462200f97 100644
--- a/tests/sources/git.py
+++ b/tests/sources/git.py
@@ -22,6 +22,7 @@
 
 import os
 import pytest
+import subprocess
 
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
@@ -523,3 +524,155 @@ def test_track_fetch(cli, tmpdir, datafiles, ref_format, tag, extra_commit):
     # Fetch it
     result = cli.run(project=project, args=['fetch', 'target.bst'])
     result.assert_success()
+
+
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
+@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
+@pytest.mark.parametrize("tag_type", [('annotated'), ('lightweight')])
+def test_git_describe(cli, tmpdir, datafiles, ref_storage, tag_type):
+    project = str(datafiles)
+
+    project_config = _yaml.load(os.path.join(project, 'project.conf'))
+    project_config['ref-storage'] = ref_storage
+    _yaml.dump(_yaml.node_sanitize(project_config), os.path.join(project, 'project.conf'))
+
+    repofiles = os.path.join(str(tmpdir), 'repofiles')
+    os.makedirs(repofiles, exist_ok=True)
+    file0 = os.path.join(repofiles, 'file0')
+    with open(file0, 'w') as f:
+        f.write('test\n')
+
+    repo = create_repo('git', str(tmpdir))
+
+    def tag(name):
+        if tag_type == 'annotated':
+            repo.add_annotated_tag(name, name)
+        else:
+            repo.add_tag(name)
+
+    ref = repo.create(repofiles)
+    tag('uselesstag')
+
+    file1 = os.path.join(str(tmpdir), 'file1')
+    with open(file1, 'w') as f:
+        f.write('test\n')
+    repo.add_file(file1)
+    tag('tag1')
+
+    file2 = os.path.join(str(tmpdir), 'file2')
+    with open(file2, 'w') as f:
+        f.write('test\n')
+    repo.branch('branch2')
+    repo.add_file(file2)
+    tag('tag2')
+
+    repo.checkout('master')
+    file3 = os.path.join(str(tmpdir), 'file3')
+    with open(file3, 'w') as f:
+        f.write('test\n')
+    repo.add_file(file3)
+
+    repo.merge('branch2')
+
+    config = repo.source_config()
+    config['track'] = repo.latest_commit()
+    config['track-tags'] = True
+
+    # Write out our test target
+    element = {
+        'kind': 'import',
+        'sources': [
+            config
+        ],
+    }
+    element_path = os.path.join(project, 'target.bst')
+    _yaml.dump(element, element_path)
+
+    if ref_storage == 'inline':
+        result = cli.run(project=project, args=['track', 'target.bst'])
+        result.assert_success()
+    else:
+        result = cli.run(project=project, args=['track', 'target.bst', '--deps', 'all'])
+        result.assert_success()
+
+    if ref_storage == 'inline':
+        element = _yaml.load(element_path)
+        tags = _yaml.node_sanitize(element['sources'][0]['tags'])
+        assert len(tags) == 2
+        for tag in tags:
+            assert 'tag' in tag
+            assert 'commit' in tag
+            assert 'annotated' in tag
+            assert tag['annotated'] == (tag_type == 'annotated')
+
+        assert set([(tag['tag'], tag['commit']) for tag in tags]) == set([('tag1', repo.rev_parse('tag1^{commit}')),
+                                                                          ('tag2', repo.rev_parse('tag2^{commit}'))])
+
+    checkout = os.path.join(str(tmpdir), 'checkout')
+
+    result = cli.run(project=project, args=['build', 'target.bst'])
+    result.assert_success()
+    result = cli.run(project=project, args=['checkout', 'target.bst', checkout])
+    result.assert_success()
+
+    if tag_type == 'annotated':
+        options = []
+    else:
+        options = ['--tags']
+    describe = subprocess.check_output(['git', 'describe'] + options,
+                                       cwd=checkout).decode('ascii')
+    assert describe.startswith('tag2-2-')
+
+    describe_fp = subprocess.check_output(['git', 'describe', '--first-parent'] + options,
+                                          cwd=checkout).decode('ascii')
+    assert describe_fp.startswith('tag1-2-')
+
+    tags = subprocess.check_output(['git', 'tag'],
+                                   cwd=checkout).decode('ascii')
+    tags = set(tags.splitlines())
+    assert tags == set(['tag1', 'tag2'])
+
+    p = subprocess.run(['git', 'log', repo.rev_parse('uselesstag')],
+                       cwd=checkout)
+    assert p.returncode != 0
+
+
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
+def test_default_do_not_track_tags(cli, tmpdir, datafiles):
+    project = str(datafiles)
+
+    project_config = _yaml.load(os.path.join(project, 'project.conf'))
+    project_config['ref-storage'] = 'inline'
+    _yaml.dump(_yaml.node_sanitize(project_config), os.path.join(project, 'project.conf'))
+
+    repofiles = os.path.join(str(tmpdir), 'repofiles')
+    os.makedirs(repofiles, exist_ok=True)
+    file0 = os.path.join(repofiles, 'file0')
+    with open(file0, 'w') as f:
+        f.write('test\n')
+
+    repo = create_repo('git', str(tmpdir))
+
+    ref = repo.create(repofiles)
+    repo.add_tag('tag')
+
+    config = repo.source_config()
+    config['track'] = repo.latest_commit()
+
+    # Write out our test target
+    element = {
+        'kind': 'import',
+        'sources': [
+            config
+        ],
+    }
+    element_path = os.path.join(project, 'target.bst')
+    _yaml.dump(element, element_path)
+
+    result = cli.run(project=project, args=['track', 'target.bst'])
+    result.assert_success()
+
+    element = _yaml.load(element_path)
+    assert 'tags' not in element['sources'][0]
diff --git a/tests/testutils/repo/git.py b/tests/testutils/repo/git.py
index bc2dae691..fe3ebd547 100644
--- a/tests/testutils/repo/git.py
+++ b/tests/testutils/repo/git.py
@@ -45,6 +45,9 @@ class Git(Repo):
     def add_tag(self, tag):
         self._run_git('tag', tag)
 
+    def add_annotated_tag(self, tag, message):
+        self._run_git('tag', '-a', tag, '-m', message)
+
     def add_commit(self):
         self._run_git('commit', '--allow-empty', '-m', 'Additional commit')
         return self.latest_commit()
@@ -95,3 +98,14 @@ class Git(Repo):
 
     def branch(self, branch_name):
         self._run_git('checkout', '-b', branch_name)
+
+    def checkout(self, commit):
+        self._run_git('checkout', commit)
+
+    def merge(self, commit):
+        self._run_git('merge', '-m', 'Merge', commit)
+        return self.latest_commit()
+
+    def rev_parse(self, rev):
+        output = self._run_git('rev-parse', rev, stdout=subprocess.PIPE).stdout
+        return output.decode('UTF-8').strip()