From bbf65daff319f707c6010bfc8aeddd19dd1c6e16 Mon Sep 17 00:00:00 2001 From: Benjamin Schubert Date: Mon, 15 Jul 2019 18:27:22 +0100 Subject: downloadablefilesource: Cache 'mirror_dir' and 'default_mirror_file' This allows faster access to those two files and reduces the time spent in their getters by more than 50%, which can be 2-3s on medium projects --- src/buildstream/_utils.pyx | 3 +++ .../plugins/sources/_downloadablefilesource.py | 23 +++++++++++++--------- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/buildstream/_utils.pyx b/src/buildstream/_utils.pyx index 3a3a97017..6605cc82a 100644 --- a/src/buildstream/_utils.pyx +++ b/src/buildstream/_utils.pyx @@ -50,3 +50,6 @@ cdef Py_UNICODE _transl(Py_UNICODE x): if ("a" <= x <= "z") or ("A" <= x <= "Z") or ("0" <= x <= "9") or x == "%": return x return "_" + + +# get_mirror_directory in configure of _downlaodablefilesource diff --git a/src/buildstream/plugins/sources/_downloadablefilesource.py b/src/buildstream/plugins/sources/_downloadablefilesource.py index 10418691e..2d53f8a56 100644 --- a/src/buildstream/plugins/sources/_downloadablefilesource.py +++ b/src/buildstream/plugins/sources/_downloadablefilesource.py @@ -75,11 +75,14 @@ class DownloadableFileSource(Source): COMMON_CONFIG_KEYS = Source.COMMON_CONFIG_KEYS + ['url', 'ref', 'etag'] __urlopener = None + __default_mirror_file = None def configure(self, node): self.original_url = node.get_str('url') self.ref = node.get_str('ref', None) self.url = self.translate_url(self.original_url) + self._mirror_dir = os.path.join(self.get_mirror_directory(), + utils.url_directory_name(self.original_url)) self._warn_deprecated_etag(node) def preflight(self): @@ -149,7 +152,7 @@ class DownloadableFileSource(Source): self.warn('{} "etag" is deprecated and ignored.'.format(provenance)) def _get_etag(self, ref): - etagfilename = os.path.join(self._get_mirror_dir(), '{}.etag'.format(ref)) + etagfilename = os.path.join(self._mirror_dir, '{}.etag'.format(ref)) if os.path.exists(etagfilename): with open(etagfilename, 'r') as etagfile: return etagfile.read() @@ -157,7 +160,7 @@ class DownloadableFileSource(Source): return None def _store_etag(self, ref, etag): - etagfilename = os.path.join(self._get_mirror_dir(), '{}.etag'.format(ref)) + etagfilename = os.path.join(self._mirror_dir, '{}.etag'.format(ref)) with utils.save_file_atomic(etagfilename) as etagfile: etagfile.write(etag) @@ -192,8 +195,8 @@ class DownloadableFileSource(Source): shutil.copyfileobj(response, dest) # Make sure url-specific mirror dir exists. - if not os.path.isdir(self._get_mirror_dir()): - os.makedirs(self._get_mirror_dir()) + if not os.path.isdir(self._mirror_dir): + os.makedirs(self._mirror_dir) # Store by sha256sum sha256 = utils.sha256sum(local_file) @@ -220,12 +223,14 @@ class DownloadableFileSource(Source): raise SourceError("{}: Error mirroring {}: {}" .format(self, self.url, e), temporary=True) from e - def _get_mirror_dir(self): - return os.path.join(self.get_mirror_directory(), - utils.url_directory_name(self.original_url)) - def _get_mirror_file(self, sha=None): - return os.path.join(self._get_mirror_dir(), sha or self.ref) + if sha is not None: + return os.path.join(self._mirror_dir, sha) + + if self.__default_mirror_file is None: + self.__default_mirror_file = os.path.join(self._mirror_dir, self.ref) + + return self.__default_mirror_file def __get_urlopener(self): if not DownloadableFileSource.__urlopener: -- cgit v1.2.1