From 593556017d4b8e5e862d92f1a56dc086138b2acc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 27 Feb 2015 15:23:09 +0000 Subject: morphlib/sourceresolver.py: Add _get_file_contents_from_repo() And make _get_morphology_from_repo() use it --- morphlib/sourceresolver.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'morphlib/sourceresolver.py') diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index 22e643d2..6347861e 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -219,7 +219,8 @@ class SourceResolver(object): else: return None - def _get_morphology_from_repo(self, loader, reponame, sha1, filename): + def _get_file_contents_from_repo(self, reponame, + sha1, filename): # pragma: no cover if self.lrc.has_repo(reponame): self.status(msg="Looking for %(reponame)s:%(filename)s in the " "local repo cache.", @@ -227,21 +228,26 @@ class SourceResolver(object): try: repo = self.lrc.get_repo(reponame) text = repo.read_file(filename, sha1) - morph = loader.load_from_string(text) except IOError: - morph = None + text = None elif self.rrc is not None: self.status(msg="Looking for %(reponame)s:%(filename)s in the " "remote repo cache.", reponame=reponame, filename=filename, chatty=True) try: text = self.rrc.cat_file(reponame, sha1, filename) - morph = loader.load_from_string(text) except morphlib.remoterepocache.CatFileError: - morph = None + text = None else: # pragma: no cover repo = self.cache_repo_locally(reponame) text = repo.read_file(filename, sha1) + + return text + + def _get_morphology_from_repo(self, loader, reponame, sha1, filename): + text = self._get_file_contents_from_repo(self, reponame, sha1, filename) + + if file not None: morph = loader.load_from_string(text) return morph -- cgit v1.2.1 From 174a0d7c67f15e9ce6c34e8f3e7d6e0d9acd3f6e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 27 Feb 2015 15:36:49 +0000 Subject: morphlib/sourceresolver.py: Add _get_file_contents_from_definitions() And make _get_morphology_from_definitions() use it --- morphlib/sourceresolver.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) (limited to 'morphlib/sourceresolver.py') diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index 6347861e..dd020803 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -212,10 +212,19 @@ class SourceResolver(object): return absref, tree + def _get_file_contents_from_definitions(self, + filename): # pragma: no cover + if os.path.exists(filename): + with open(filename) as f: + return f.read() + else: + return None + def _get_morphology_from_definitions(self, loader, filename): # pragma: no cover if os.path.exists(filename): - return loader.load_from_file(filename) + text = self._get_file_contents_from_definitions(filename) + return self.load_from_string(text, filename=filename) else: return None -- cgit v1.2.1 From 5fa101c0a1a817a58fc88306b028fb61d5f1589b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 27 Feb 2015 15:48:13 +0000 Subject: morphlib/sourceresolver.py: Add _get_file_contents() And make _get_morphology() use it --- morphlib/sourceresolver.py | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) (limited to 'morphlib/sourceresolver.py') diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index dd020803..cd183ee0 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -261,18 +261,13 @@ class SourceResolver(object): return morph - def _get_morphology(self, reponame, sha1, filename): - '''Read the morphology at the specified location. + def _get_file_contents(self, reponame, sha1, filename): # pragma: no cover + '''Read the file at the specified location. Returns None if the file does not exist in the specified commit. ''' - key = (reponame, sha1, filename) - if key in self._resolved_morphologies: - return self._resolved_morphologies[key] - - loader = morphlib.morphloader.MorphologyLoader() - morph = None + text = None if reponame == self._definitions_repo and \ sha1 == self._definitions_absref: # pragma: no cover @@ -280,11 +275,27 @@ class SourceResolver(object): # we can quickly read definitions files from. defs_filename = os.path.join(self._definitions_checkout_dir, filename) - morph = self._get_morphology_from_definitions(loader, - defs_filename) + text = self._get_file_contents_from_definitions(defs_filename) else: - morph = self._get_morphology_from_repo(loader, reponame, sha1, - filename) + text = self._get_file_contents_from_repo(reponame, sha1, filename) + + return text + + def _get_morphology(self, reponame, sha1, filename): # pragma: no cover + '''Read the morphology at the specified location. + + Returns None if the file does not exist in the specified commit. + + ''' + key = (reponame, sha1, filename) + if key in self._resolved_morphologies: + return self._resolved_morphologies[key] + + loader = morphlib.morphloader.MorphologyLoader() + morph = None + + text = self._get_file_contents(reponame, sha1, filename) + morph = loader.load_from_string(text) if morph is None: return None -- cgit v1.2.1 From ea2c67c0da04a7add7d27370202c707c2cce4ffc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 27 Feb 2015 16:05:17 +0000 Subject: morphlib/sourceresolver.py: Do not duplicate what is already done in load_from_string() --- morphlib/sourceresolver.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) (limited to 'morphlib/sourceresolver.py') diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index cd183ee0..5be373a8 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -292,19 +292,14 @@ class SourceResolver(object): return self._resolved_morphologies[key] loader = morphlib.morphloader.MorphologyLoader() - morph = None text = self._get_file_contents(reponame, sha1, filename) morph = loader.load_from_string(text) - if morph is None: - return None - else: - loader.validate(morph) - loader.set_commands(morph) - loader.set_defaults(morph) + if morph is not None: self._resolved_morphologies[key] = morph - return morph + + return morph def _detect_build_system(self, reponame, sha1, expected_filename): '''Attempt to detect buildsystem of the given commit. -- cgit v1.2.1 From 6b291f5e9027a1707a1525bf1fea6efa159b14f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 27 Feb 2015 15:48:51 +0000 Subject: morphlib/sourceresolver.py: Remove not used functions _get_morphology_from_definitions() and _get_morphology_from_repo() --- morphlib/sourceresolver.py | 16 ---------------- 1 file changed, 16 deletions(-) (limited to 'morphlib/sourceresolver.py') diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index 5be373a8..f571d76b 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -220,14 +220,6 @@ class SourceResolver(object): else: return None - def _get_morphology_from_definitions(self, loader, - filename): # pragma: no cover - if os.path.exists(filename): - text = self._get_file_contents_from_definitions(filename) - return self.load_from_string(text, filename=filename) - else: - return None - def _get_file_contents_from_repo(self, reponame, sha1, filename): # pragma: no cover if self.lrc.has_repo(reponame): @@ -253,14 +245,6 @@ class SourceResolver(object): return text - def _get_morphology_from_repo(self, loader, reponame, sha1, filename): - text = self._get_file_contents_from_repo(self, reponame, sha1, filename) - - if file not None: - morph = loader.load_from_string(text) - - return morph - def _get_file_contents(self, reponame, sha1, filename): # pragma: no cover '''Read the file at the specified location. -- cgit v1.2.1 From c2d6bf758845076eca6eb71af09df77165993270 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Mon, 2 Mar 2015 19:14:57 +0000 Subject: morphlib/sourceresolver.py: Check and parse VERSION file Only fail if: - VERSION file exists - and its a yaml file - and its a dict - and has the key 'version' - and the contents of the key 'version' is an int - and that int is in the list of non_supported_versions (empty at the moment) --- morphlib/sourceresolver.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) (limited to 'morphlib/sourceresolver.py') diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index f571d76b..387d2e0d 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -21,6 +21,7 @@ import os import pylru import shutil import tempfile +import yaml import cliapp @@ -31,6 +32,7 @@ tree_cache_filename = 'trees.cache.pickle' buildsystem_cache_size = 10000 buildsystem_cache_filename = 'detected-chunk-buildsystems.cache.pickle' +not_supported_versions = [] class PickleCacheManager(object): # pragma: no cover '''Cache manager for PyLRU that reads and writes to Pickle files. @@ -89,6 +91,11 @@ class MorphologyNotFoundError(SourceResolverError): # pragma: no cover SourceResolverError.__init__( self, "Couldn't find morphology: %s" % filename) +class UnknownVersionError(SourceResolverError): # pragma: no cover + def __init__(self, version): + SourceResolverError.__init__( + self, "Definitions format version %s is not supported" % version) + class SourceResolver(object): '''Provides a way of resolving the set of sources for a given system. @@ -338,6 +345,22 @@ class SourceResolver(object): loader.set_defaults(morph) return morph + def _check_version_file(self,definitions_repo, + definitions_absref): # pragma: no cover + version_file = self._get_file_contents( + definitions_repo, definitions_absref, 'VERSION') + + if version_file is None: + return + + try: + version = yaml.safe_load(version_file)['version'] + except (yaml.error.YAMLError, KeyError, TypeError): + version = 0 + + if version in not_supported_versions: + raise UnknownVersionError(version) + def _process_definitions_with_children(self, system_filenames, definitions_repo, definitions_ref, @@ -347,6 +370,8 @@ class SourceResolver(object): definitions_queue = collections.deque(system_filenames) chunk_queue = set() + self._check_version_file(definitions_repo, definitions_absref) + while definitions_queue: filename = definitions_queue.popleft() -- cgit v1.2.1