Merge branch 'tpollard/483' into 'master'

plugins/git.py: Warn if ref is not in given track

See merge request BuildStream/buildstream!564

2 files changed, 109 insertions, 4 deletions

diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py
index f45a23bfc..cc6d35cd0 100644
--- a/buildstream/plugins/sources/git.py
+++ b/buildstream/plugins/sources/git.py
@@ -74,6 +74,9 @@ This plugin provides the following configurable warnings:
 
 - 'git:inconsistent-submodule' - A submodule was found to be missing from the underlying git repository.
 
+This plugin also utilises the following configurable core plugin warnings:
+
+- 'ref-not-in-track' - The provided ref was not found in the provided track in the element's git repository.
 """
 
 import os
@@ -87,6 +90,7 @@ from configparser import RawConfigParser
 
 from buildstream import Source, SourceError, Consistency, SourceFetcher
 from buildstream import utils
+from buildstream.plugin import CoreWarnings
 
 GIT_MODULES = '.gitmodules'
 
@@ -203,7 +207,7 @@ class GitMirror(SourceFetcher):
             cwd=self.mirror)
         return output.rstrip('\n')
 
-    def stage(self, directory):
+    def stage(self, directory, track=None):
         fullpath = os.path.join(directory, self.path)
 
         # Using --shared here avoids copying the objects into the checkout, in any
@@ -217,10 +221,14 @@ class GitMirror(SourceFetcher):
                          fail="Failed to checkout git ref {}".format(self.ref),
                          cwd=fullpath)
 
+        # Check that the user specified ref exists in the track if provided & not already tracked
+        if track:
+            self.assert_ref_in_track(fullpath, track)
+
         # Remove .git dir
         shutil.rmtree(os.path.join(fullpath, ".git"))
 
-    def init_workspace(self, directory):
+    def init_workspace(self, directory, track=None):
         fullpath = os.path.join(directory, self.path)
         url = self.source.translate_url(self.url)
 
@@ -236,6 +244,10 @@ class GitMirror(SourceFetcher):
                          fail="Failed to checkout git ref {}".format(self.ref),
                          cwd=fullpath)
 
+        # Check that the user specified ref exists in the track if provided & not already tracked
+        if track:
+            self.assert_ref_in_track(fullpath, track)
+
     # List the submodules (path/url tuples) present at the given ref of this repo
     def submodule_list(self):
         modules = "{}:{}".format(self.ref, GIT_MODULES)
@@ -300,6 +312,28 @@ class GitMirror(SourceFetcher):
 
             return None
 
+    # Assert that ref exists in track, if track has been specified.
+    def assert_ref_in_track(self, fullpath, track):
+        _, branch = self.source.check_output([self.source.host_git, 'branch', '--list', track,
+                                              '--contains', self.ref],
+                                             cwd=fullpath,)
+        if branch:
+            return True
+        else:
+            _, tag = self.source.check_output([self.source.host_git, 'tag', '--list', track,
+                                               '--contains', self.ref],
+                                              cwd=fullpath,)
+            if tag:
+                return True
+
+        detail = "The ref provided for the element does not exist locally in the provided track branch / tag " + \
+                 "'{}'.\nYou may wish to track the element to update the ref from '{}' ".format(track, track) + \
+                 "with `bst track`,\nor examine the upstream at '{}' for the specific ref.".format(self.url)
+
+        self.source.warn("{}: expected ref '{}' was not found in given track '{}' for staged repository: '{}'\n"
+                         .format(self.source, self.ref, track, self.url),
+                         detail=detail, warning_token=CoreWarnings.REF_NOT_IN_TRACK)
+
 
 class GitSource(Source):
     # pylint: disable=attribute-defined-outside-init
@@ -342,6 +376,7 @@ class GitSource(Source):
                 self.submodule_checkout_overrides[path] = checkout
 
         self.mark_download_url(self.original_url)
+        self.tracked = False
 
     def preflight(self):
         # Check if git is installed, get the binary at the same time
@@ -407,6 +442,8 @@ class GitSource(Source):
             # Update self.mirror.ref and node.ref from the self.tracking branch
             ret = self.mirror.latest_commit(self.tracking)
 
+        # Set tracked attribute, parameter for if self.mirror.assert_ref_in_track is needed
+        self.tracked = True
         return ret
 
     def init_workspace(self, directory):
@@ -414,7 +451,7 @@ class GitSource(Source):
         self.refresh_submodules()
 
         with self.timed_activity('Setting up workspace "{}"'.format(directory), silent_nested=True):
-            self.mirror.init_workspace(directory)
+            self.mirror.init_workspace(directory, track=(self.tracking if not self.tracked else None))
             for mirror in self.submodules:
                 mirror.init_workspace(directory)
 
@@ -430,7 +467,7 @@ class GitSource(Source):
         # Stage the main repo in the specified directory
         #
         with self.timed_activity("Staging {}".format(self.mirror.url), silent_nested=True):
-            self.mirror.stage(directory)
+            self.mirror.stage(directory, track=(self.tracking if not self.tracked else None))
             for mirror in self.submodules:
                 if mirror.path in self.submodule_checkout_overrides:
                     checkout = self.submodule_checkout_overrides[mirror.path]
diff --git a/tests/sources/git.py b/tests/sources/git.py
index 781d6d4d1..8f6074fae 100644
--- a/tests/sources/git.py
+++ b/tests/sources/git.py
@@ -25,6 +25,7 @@ import pytest
 
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
+from buildstream.plugin import CoreWarnings
 
 from tests.testutils import cli, create_repo
 from tests.testutils.site import HAVE_GIT
@@ -408,3 +409,70 @@ def test_submodule_track_no_ref_or_track(cli, tmpdir, datafiles):
     result = cli.run(project=project, args=['show', 'target.bst'])
     result.assert_main_error(ErrorDomain.SOURCE, "missing-track-and-ref")
     result.assert_task_error(None, None)
+
+
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
+def test_ref_not_in_track_warn(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+
+    # Create the repo from 'repofiles', create a branch without latest commit
+    repo = create_repo('git', str(tmpdir))
+    ref = repo.create(os.path.join(project, 'repofiles'))
+
+    gitsource = repo.source_config(ref=ref)
+
+    # Overwrite the track value to the added branch
+    gitsource['track'] = 'foo'
+
+    # Write out our test target
+    element = {
+        'kind': 'import',
+        'sources': [
+            gitsource
+        ]
+    }
+    _yaml.dump(element, os.path.join(project, 'target.bst'))
+
+    # Assert the warning is raised as ref is not in branch foo.
+    # Assert warning not error to the user, when not set as fatal.
+    result = cli.run(project=project, args=['build', 'target.bst'])
+    assert "The ref provided for the element does not exist locally" in result.stderr
+
+
+@pytest.mark.skipif(HAVE_GIT is False, reason="git is not available")
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'template'))
+def test_ref_not_in_track_warn_error(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+
+    # Add fatal-warnings ref-not-in-track to project.conf
+    project_template = {
+        "name": "foo",
+        "fatal-warnings": [CoreWarnings.REF_NOT_IN_TRACK]
+    }
+
+    _yaml.dump(project_template, os.path.join(project, 'project.conf'))
+
+    # Create the repo from 'repofiles', create a branch without latest commit
+    repo = create_repo('git', str(tmpdir))
+    ref = repo.create(os.path.join(project, 'repofiles'))
+
+    gitsource = repo.source_config(ref=ref)
+
+    # Overwrite the track value to the added branch
+    gitsource['track'] = 'foo'
+
+    # Write out our test target
+    element = {
+        'kind': 'import',
+        'sources': [
+            gitsource
+        ]
+    }
+    _yaml.dump(element, os.path.join(project, 'target.bst'))
+
+    # Assert that build raises a warning here that is captured
+    # as plugin error, due to the fatal warning being set
+    result = cli.run(project=project, args=['build', 'target.bst'])
+    result.assert_main_error(ErrorDomain.STREAM, None)
+    result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.REF_NOT_IN_TRACK)