diff options
Diffstat (limited to 'morphlib/sourceresolver.py')
| -rw-r--r-- | morphlib/sourceresolver.py | 116 |
1 files changed, 86 insertions, 30 deletions
diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index 387d2e0d..1e64c23a 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see <http://www.gnu.org/licenses/>. import collections @@ -32,7 +31,7 @@ tree_cache_filename = 'trees.cache.pickle' buildsystem_cache_size = 10000 buildsystem_cache_filename = 'detected-chunk-buildsystems.cache.pickle' -not_supported_versions = [] +supported_versions = [0, 1] class PickleCacheManager(object): # pragma: no cover '''Cache manager for PyLRU that reads and writes to Pickle files. @@ -185,6 +184,8 @@ class SourceResolver(object): reponame, ref) return ref, self._resolved_trees[(reponame, ref)] + logging.debug('tree (%s, %s) not in cache', reponame, ref) + absref = None if self.lrc.has_repo(reponame): repo = self.lrc.get_repo(reponame) @@ -205,7 +206,7 @@ class SourceResolver(object): reponame=reponame, ref=ref, chatty=True) - except BaseException, e: + except BaseException as e: logging.warning('Caught (and ignored) exception: %s' % str(e)) if absref is None: @@ -345,6 +346,29 @@ class SourceResolver(object): loader.set_defaults(morph) return morph + def _parse_version_file(self, version_file): # pragma : no cover + '''Parse VERSION file and return the version of the format if: + + VERSION is a YAML file + and it's a dict + and has the key 'version' + and the type stored in the 'version' key is an int + and that int is not in the supported format + + otherwise returns None + + ''' + version = None + + yaml_obj = yaml.safe_load(version_file) + if yaml_obj is not None: + if type(yaml_obj) is dict: + if 'version' in yaml_obj.keys(): + if type(yaml_obj['version']) is int: + version = yaml_obj['version'] + + return version + def _check_version_file(self,definitions_repo, definitions_absref): # pragma: no cover version_file = self._get_file_contents( @@ -353,13 +377,10 @@ class SourceResolver(object): if version_file is None: return - try: - version = yaml.safe_load(version_file)['version'] - except (yaml.error.YAMLError, KeyError, TypeError): - version = 0 - - if version in not_supported_versions: - raise UnknownVersionError(version) + version = self._parse_version_file(version_file) + if version is not None: + if version not in supported_versions: + raise UnknownVersionError(version) def _process_definitions_with_children(self, system_filenames, definitions_repo, @@ -415,42 +436,77 @@ class SourceResolver(object): return chunk_queue + def _generate_morph_and_cache_buildsystem(self, + definition_key, chunk_key, + buildsystem, + morph_name): # pragma: no cover + logging.debug('Caching build system for chunk with key %s', chunk_key) + + self._resolved_buildsystems[chunk_key] = buildsystem + + morphology = self._create_morphology_for_build_system(buildsystem, + morph_name) + self._resolved_morphologies[definition_key] = morphology + return morphology + def process_chunk(self, definition_repo, definition_ref, chunk_repo, chunk_ref, filename, visit): # pragma: no cover absref = None tree = None - - definition_key = (definition_repo, definition_ref, filename) chunk_key = None + buildsystem = None morph_name = os.path.splitext(os.path.basename(filename))[0] + # Get morphology from definitions repo + definition_key = (definition_repo, definition_ref, filename) morphology = self._get_morphology(*definition_key) - buildsystem = None + + if morphology: + absref, tree = self._resolve_ref(chunk_repo, chunk_ref) + visit(chunk_repo, chunk_ref, filename, absref, tree, morphology) + return + + absref, tree = self._resolve_ref(chunk_repo, chunk_ref) + chunk_key = (chunk_repo, absref, filename) if chunk_key in self._resolved_buildsystems: + logging.debug('Build system for %s is cached', str(chunk_key)) + self.status(msg='Build system for %(chunk)s is cached', + chunk=str(chunk_key), + chatty=True) buildsystem = self._resolved_buildsystems[chunk_key] - if morphology is None and buildsystem is None: - # This is a slow operation (looking for a file in Git repo may - # potentially require cloning the whole thing). - absref, tree = self._resolve_ref(chunk_repo, chunk_ref) - chunk_key = (chunk_repo, absref, filename) + # If the build system for this chunk is cached then: + # * the chunk does not have a chunk morph + # (so we don't need to look for one) + # + # * a suitable (generated) morphology may already be cached. + # + # If the morphology is not already cached we can generate it + # from the build-system and cache it. + if definition_key in self._resolved_morphologies: + morphology = self._resolved_morphologies[definition_key] + else: + morphology = self._generate_morph_and_cache_buildsystem( + definition_key, chunk_key, buildsystem, morph_name) + else: + logging.debug('Build system for %s is NOT cached', str(chunk_key)) + # build-system not cached, look for morphology in chunk repo + # this can be slow (we may need to clone the repo from a remote) morphology = self._get_morphology(*chunk_key) - if morphology is None: - if buildsystem is None: - buildsystem = self._detect_build_system(*chunk_key) - if buildsystem is None: - raise MorphologyNotFoundError(filename) - else: - self._resolved_buildsystems[chunk_key] = buildsystem - morphology = self._create_morphology_for_build_system( - buildsystem, morph_name) + if morphology != None: self._resolved_morphologies[definition_key] = morphology + else: + # This chunk doesn't have a chunk morph + buildsystem = self._detect_build_system(*chunk_key) - if not absref or not tree: - absref, tree = self._resolve_ref(chunk_repo, chunk_ref) + if buildsystem is None: + raise MorphologyNotFoundError(filename) + else: + morphology = self._generate_morph_and_cache_buildsystem( + definition_key, chunk_key, buildsystem, morph_name) visit(chunk_repo, chunk_ref, filename, absref, tree, morphology) |