From de73cc0b7076d5c6d60ea771a3ff388a7eb4153c Mon Sep 17 00:00:00 2001 From: Valentin David Date: Tue, 14 Aug 2018 15:27:25 +0200 Subject: Fix ostree repository mirroring Ostree mirrors were not sharing the same local repository, so it was impossible the request refs from the right local repository when data was fetched from a mirror rather than upstream. Instead of having several repository with one remote each, we now have one repository with several remotes. This fixes #538. --- buildstream/plugins/sources/ostree.py | 30 ++++++++++++++++++++---------- tests/frontend/mirror.py | 4 ---- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/buildstream/plugins/sources/ostree.py b/buildstream/plugins/sources/ostree.py index 3a841c488..6266731bf 100644 --- a/buildstream/plugins/sources/ostree.py +++ b/buildstream/plugins/sources/ostree.py @@ -71,7 +71,7 @@ class OSTreeSource(Source): self.ref = self.node_get_member(node, str, 'ref', None) self.tracking = self.node_get_member(node, str, 'track', None) self.mirror = os.path.join(self.get_mirror_directory(), - utils.url_directory_name(self.url)) + utils.url_directory_name(self.original_url)) # (optional) Not all repos are signed. But if they are, get the gpg key self.gpg_key_path = None @@ -104,10 +104,11 @@ class OSTreeSource(Source): return None self.ensure() + remote_name = self.ensure_remote(self.url) with self.timed_activity("Fetching tracking ref '{}' from origin: {}" .format(self.tracking, self.url)): try: - _ostree.fetch(self.repo, ref=self.tracking, progress=self.progress) + _ostree.fetch(self.repo, remote=remote_name, ref=self.tracking, progress=self.progress) except OSTreeError as e: raise SourceError("{}: Failed to fetch tracking ref '{}' from origin {}\n\n{}" .format(self, self.tracking, self.url, e)) from e @@ -116,11 +117,12 @@ class OSTreeSource(Source): def fetch(self): self.ensure() + remote_name = self.ensure_remote(self.url) if not _ostree.exists(self.repo, self.ref): with self.timed_activity("Fetching remote ref: {} from origin: {}" .format(self.ref, self.url)): try: - _ostree.fetch(self.repo, ref=self.ref, progress=self.progress) + _ostree.fetch(self.repo, remote=remote_name, ref=self.ref, progress=self.progress) except OSTreeError as e: raise SourceError("{}: Failed to fetch ref '{}' from origin: {}\n\n{}" .format(self, self.ref, self.url, e)) from e @@ -171,14 +173,22 @@ class OSTreeSource(Source): self.status("Creating local mirror for {}".format(self.url)) self.repo = _ostree.ensure(self.mirror, True) - gpg_key = None - if self.gpg_key_path: - gpg_key = 'file://' + self.gpg_key_path - try: - _ostree.configure_remote(self.repo, "origin", self.url, key_url=gpg_key) - except OSTreeError as e: - raise SourceError("{}: Failed to configure origin {}\n\n{}".format(self, self.url, e)) from e + def ensure_remote(self, url): + if self.original_url == self.url: + remote_name = 'origin' + else: + remote_name = utils.url_directory_name(url) + + gpg_key = None + if self.gpg_key_path: + gpg_key = 'file://' + self.gpg_key_path + + try: + _ostree.configure_remote(self.repo, remote_name, url, key_url=gpg_key) + except OSTreeError as e: + raise SourceError("{}: Failed to configure origin {}\n\n{}".format(self, self.url, e)) from e + return remote_name def progress(self, percent, message): self.status(message) diff --git a/tests/frontend/mirror.py b/tests/frontend/mirror.py index 93e270b4d..f6031cad8 100644 --- a/tests/frontend/mirror.py +++ b/tests/frontend/mirror.py @@ -466,10 +466,6 @@ def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind): @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS]) def test_mirror_from_includes(cli, tmpdir, datafiles, kind): - if kind == 'ostree': - # FIXME: Mirroring fallback fails with ostree - pytest.skip("Bug #538 - ostree mirror fallback breaks assertion") - bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr') upstream_repodir = os.path.join(str(tmpdir), 'upstream') mirror_repodir = os.path.join(str(tmpdir), 'mirror') -- cgit v1.2.1