diff options
-rw-r--r-- | buildstream/plugins/sources/git.py | 102 |
1 files changed, 57 insertions, 45 deletions
diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py index e68de81cf..1ba9993e1 100644 --- a/buildstream/plugins/sources/git.py +++ b/buildstream/plugins/sources/git.py @@ -78,7 +78,7 @@ from io import StringIO from configparser import RawConfigParser -from buildstream import Source, SourceError, Consistency +from buildstream import Source, SourceError, Consistency, SourceFetcher from buildstream import utils GIT_MODULES = '.gitmodules' @@ -88,18 +88,22 @@ GIT_MODULES = '.gitmodules' # for the primary git source and also for each submodule it # might have at a given time # -class GitMirror(): +class GitMirror(SourceFetcher): def __init__(self, source, path, url, ref): + super().__init__() self.source = source self.path = path - self.url = source.translate_url(url) + self.url = url self.ref = ref - self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(self.url)) + self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(url)) + + alias, _ = self.source.split_aliased_url(self.url) + self.set_alias(alias) # Ensures that the mirror exists - def ensure(self): + def ensure(self, alias_override=None): # Unfortunately, git does not know how to only clone just a specific ref, # so we have to download all of those gigs even if we only need a couple @@ -112,20 +116,49 @@ class GitMirror(): # system configured tmpdir is not on the same partition. # with self.source.tempdir() as tmpdir: - self.source.call([self.source.host_git, 'clone', '--mirror', '-n', self.url, tmpdir], - fail="Failed to clone git repository {}".format(self.url)) + url = self.source.translate_url(self.url, alias_override) + self.source.call([self.source.host_git, 'clone', '--mirror', '-n', url, tmpdir], + fail="Failed to clone git repository {}".format(url)) try: shutil.move(tmpdir, self.mirror) except (shutil.Error, OSError) as e: raise SourceError("{}: Failed to move cloned git repository {} from '{}' to '{}'" - .format(self.source, self.url, tmpdir, self.mirror)) from e + .format(self.source, url, tmpdir, self.mirror)) from e + + def _fetch(self, alias_override=None): + url = self.source.translate_url(self.url, alias_override) + + if alias_override: + remote_name = utils.url_directory_name(alias_override) + _, remotes = self.source.check_output( + [self.source.host_git, 'remote'], + fail="Failed to retrieve list of remotes in {}".format(self.mirror), + cwd=self.mirror + ) + if remote_name not in remotes: + self.source.call( + [self.source.host_git, 'remote', 'add', remote_name, url], + fail="Failed to add remote {} with url {}".format(remote_name, url), + cwd=self.mirror + ) + else: + remote_name = "origin" - def fetch(self): - self.source.call([self.source.host_git, 'fetch', 'origin', '--prune'], - fail="Failed to fetch from remote git repository: {}".format(self.url), + self.source.call([self.source.host_git, 'fetch', remote_name, '--prune'], + fail="Failed to fetch from remote git repository: {}".format(url), cwd=self.mirror) + def fetch(self, alias_override=None): + self.ensure(alias_override) + if not self.has_ref(): + self._fetch(alias_override) + self.assert_ref() + + def get_alias(self): + alias, _ = self.source.split_aliased_url(self.url) + return alias + def has_ref(self): if not self.ref: return False @@ -168,12 +201,13 @@ class GitMirror(): def init_workspace(self, directory): fullpath = os.path.join(directory, self.path) + url = self.source.translate_url(self.url) self.source.call([self.source.host_git, 'clone', '--no-checkout', self.mirror, fullpath], fail="Failed to clone git mirror {} in directory: {}".format(self.mirror, fullpath)) - self.source.call([self.source.host_git, 'remote', 'set-url', 'origin', self.url], - fail='Failed to add remote origin "{}"'.format(self.url), + self.source.call([self.source.host_git, 'remote', 'set-url', 'origin', url], + fail='Failed to add remote origin "{}"'.format(url), cwd=fullpath) self.source.call([self.source.host_git, 'checkout', '--force', self.ref], @@ -273,6 +307,10 @@ class GitSource(Source): checkout = self.node_get_member(submodule, bool, 'checkout') self.submodule_checkout_overrides[path] = checkout + # Set the alias, because it's not implicitly set by a translate_url call. + alias, _ = self.split_aliased_url(self.original_url) + self.set_alias(alias) + def preflight(self): # Check if git is installed, get the binary at the same time self.host_git = utils.get_host_tool('git') @@ -324,31 +362,13 @@ class GitSource(Source): .format(self.tracking, self.mirror.url), silent_nested=True): self.mirror.ensure() - self.mirror.fetch() + self.mirror._fetch() # Update self.mirror.ref and node.ref from the self.tracking branch ret = self.mirror.latest_commit(self.tracking) return ret - def fetch(self): - - with self.timed_activity("Fetching {}".format(self.mirror.url), silent_nested=True): - - # Here we are only interested in ensuring that our mirror contains - # the self.mirror.ref commit. - self.mirror.ensure() - if not self.mirror.has_ref(): - self.mirror.fetch() - - self.mirror.assert_ref() - - # Here after performing any fetches, we need to also ensure that - # we've cached the desired refs in our mirrors of submodules. - # - self.refresh_submodules() - self.fetch_submodules() - def init_workspace(self, directory): # XXX: may wish to refactor this as some code dupe with stage() self.refresh_submodules() @@ -380,6 +400,10 @@ class GitSource(Source): if checkout: mirror.stage(directory) + def get_source_fetchers(self): + self.refresh_submodules() + return [self.mirror] + self.submodules + ########################################################### # Local Functions # ########################################################### @@ -401,6 +425,7 @@ class GitSource(Source): # Assumes that we have our mirror and we have the ref which we point to # def refresh_submodules(self): + self.mirror.ensure() submodules = [] # XXX Here we should issue a warning if either: @@ -422,19 +447,6 @@ class GitSource(Source): self.submodules = submodules - # Ensures that we have mirrored git repositories for all - # the submodules existing at the given commit of the main git source. - # - # Also ensure that these mirrors have the required commits - # referred to at the given commit of the main git source. - # - def fetch_submodules(self): - for mirror in self.submodules: - mirror.ensure() - if not mirror.has_ref(): - mirror.fetch() - mirror.assert_ref() - # Plugin entry point def setup(): |