From 364455436aded34ce6f843dfa68a32c14aaf01ea Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Sat, 7 Mar 2015 17:48:32 +0000 Subject: Fix bug in build system cache access This fixes a bug that causes morph to run build system detection even though the build system has already been cached. The cache is accessed before the key has been computed (absref isn't known until after we've called resolve_ref) so it always misses. --- morphlib/sourceresolver.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index 387d2e0d..9255c2de 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -185,6 +185,8 @@ class SourceResolver(object): reponame, ref) return ref, self._resolved_trees[(reponame, ref)] + logging.debug('tree (%s, %s) not in cache', reponame, ref) + absref = None if self.lrc.has_repo(reponame): repo = self.lrc.get_repo(reponame) @@ -428,9 +430,6 @@ class SourceResolver(object): morphology = self._get_morphology(*definition_key) buildsystem = None - if chunk_key in self._resolved_buildsystems: - buildsystem = self._resolved_buildsystems[chunk_key] - if morphology is None and buildsystem is None: # This is a slow operation (looking for a file in Git repo may # potentially require cloning the whole thing). @@ -438,12 +437,21 @@ class SourceResolver(object): chunk_key = (chunk_repo, absref, filename) morphology = self._get_morphology(*chunk_key) + if chunk_key in self._resolved_buildsystems: + logging.debug('Build system for %s is cached', str(chunk_key)) + self.status(msg='Build system for %s is cached' % str(chunk_key), + chatty=True) + buildsystem = self._resolved_buildsystems[chunk_key] + if morphology is None: if buildsystem is None: buildsystem = self._detect_build_system(*chunk_key) + if buildsystem is None: raise MorphologyNotFoundError(filename) else: + logging.debug('Caching build system for chunk with key %s', + chunk_key) self._resolved_buildsystems[chunk_key] = buildsystem morphology = self._create_morphology_for_build_system( buildsystem, morph_name) -- cgit v1.2.1 From 8fcf5f3c68e24b96347aee0819ab18cf97b48721 Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Sat, 7 Mar 2015 21:24:50 +0000 Subject: Minimise repo cache fetches If we know the build system for a chunk is cached then we also know that it doesn't have a chunk morph, so we don't need to look for one. (We only cache the build systems of chunks that don't have chunk morphs) --- morphlib/sourceresolver.py | 70 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 49 insertions(+), 21 deletions(-) diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index 9255c2de..bdbf30b8 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -417,25 +417,39 @@ class SourceResolver(object): return chunk_queue + def _generate_morph_and_cache_buildsystem(self, + definition_key, chunk_key, + buildsystem, + morph_name): # pragma: no cover + logging.debug('Caching build system for chunk with key %s', chunk_key) + + self._resolved_buildsystems[chunk_key] = buildsystem + + morphology = self._create_morphology_for_build_system(buildsystem, + morph_name) + self._resolved_morphologies[definition_key] = morphology + return morphology + def process_chunk(self, definition_repo, definition_ref, chunk_repo, chunk_ref, filename, visit): # pragma: no cover absref = None tree = None - - definition_key = (definition_repo, definition_ref, filename) chunk_key = None + buildsystem = None morph_name = os.path.splitext(os.path.basename(filename))[0] + # Get morphology from definitions repo + definition_key = (definition_repo, definition_ref, filename) morphology = self._get_morphology(*definition_key) - buildsystem = None - if morphology is None and buildsystem is None: - # This is a slow operation (looking for a file in Git repo may - # potentially require cloning the whole thing). + if morphology: absref, tree = self._resolve_ref(chunk_repo, chunk_ref) - chunk_key = (chunk_repo, absref, filename) - morphology = self._get_morphology(*chunk_key) + visit(chunk_repo, chunk_ref, filename, absref, tree, morphology) + return + + absref, tree = self._resolve_ref(chunk_repo, chunk_ref) + chunk_key = (chunk_repo, absref, filename) if chunk_key in self._resolved_buildsystems: logging.debug('Build system for %s is cached', str(chunk_key)) @@ -443,22 +457,36 @@ class SourceResolver(object): chatty=True) buildsystem = self._resolved_buildsystems[chunk_key] - if morphology is None: - if buildsystem is None: - buildsystem = self._detect_build_system(*chunk_key) - - if buildsystem is None: - raise MorphologyNotFoundError(filename) + # If the build system for this chunk is cached then: + # * the chunk does not have a chunk morph + # (so we don't need to look for one) + # + # * a suitable (generated) morphology may already be cached. + # + # If the morphology is not already cached we can generate it + # from the build-system and cache it. + if definition_key in self._resolved_morphologies: + morphology = self._resolved_morphologies[definition_key] else: - logging.debug('Caching build system for chunk with key %s', - chunk_key) - self._resolved_buildsystems[chunk_key] = buildsystem - morphology = self._create_morphology_for_build_system( - buildsystem, morph_name) + morphology = self._generate_morph_and_cache_buildsystem( + definition_key, chunk_key, buildsystem, morph_name) + else: + logging.debug('Build system for %s is NOT cached', str(chunk_key)) + # build-system not cached, look for morphology in chunk repo + # this can be slow (we may need to clone the repo from a remote) + morphology = self._get_morphology(*chunk_key) + + if morphology != None: self._resolved_morphologies[definition_key] = morphology + else: + # This chunk doesn't have a chunk morph + buildsystem = self._detect_build_system(*chunk_key) - if not absref or not tree: - absref, tree = self._resolve_ref(chunk_repo, chunk_ref) + if buildsystem is None: + raise MorphologyNotFoundError(filename) + else: + morphology = self._generate_morph_and_cache_buildsystem( + definition_key, chunk_key, buildsystem, morph_name) visit(chunk_repo, chunk_ref, filename, absref, tree, morphology) -- cgit v1.2.1