diff options
Diffstat (limited to 'morphlib')
-rw-r--r-- | morphlib/sourceresolver.py | 196 | ||||
-rw-r--r-- | morphlib/sourceresolver_tests.py | 80 |
2 files changed, 164 insertions, 112 deletions
diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index 3b87407f..a065e831 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -174,13 +174,7 @@ class SourceResolver(object): self.update = update_repos self.status = status_cb - self._resolved_trees = {} - self._resolved_morphologies = {} - self._resolved_buildsystems = {} - - self._definitions_checkout_dir = None - - def _resolve_ref(self, reponame, ref): # pragma: no cover + def _resolve_ref(self, resolved_trees, reponame, ref): # pragma: no cover '''Resolves commit and tree sha1s of the ref in a repo and returns it. If update is True then this has the side-effect of updating or cloning @@ -193,10 +187,10 @@ class SourceResolver(object): # The Baserock reference definitions use absolute refs so, and, if the # absref is cached, we can short-circuit all this code. - if (reponame, ref) in self._resolved_trees: + if (reponame, ref) in resolved_trees: logging.debug('Returning tree (%s, %s) from tree cache', reponame, ref) - return ref, self._resolved_trees[(reponame, ref)] + return ref, resolved_trees[(reponame, ref)] logging.debug('tree (%s, %s) not in cache', reponame, ref) @@ -230,14 +224,15 @@ class SourceResolver(object): logging.debug('Writing tree to cache with ref (%s, %s)', reponame, absref) - self._resolved_trees[(reponame, absref)] = tree + resolved_trees[(reponame, absref)] = tree return absref, tree - def _get_file_contents_from_definitions(self, + def _get_file_contents_from_definitions(self, definitions_checkout_dir, filename): # pragma: no cover - if os.path.exists(filename): - with open(filename) as f: + fp = os.path.join(definitions_checkout_dir, filename) + if os.path.exists(fp): + with open(fp) as f: return f.read() else: return None @@ -267,7 +262,9 @@ class SourceResolver(object): return text - def _get_file_contents(self, reponame, sha1, filename): # pragma: no cover + def _get_file_contents(self, definitions_checkout_dir, definitions_repo, + definitions_absref, reponame, sha1, + filename): # pragma: no cover '''Read the file at the specified location. Returns None if the file does not exist in the specified commit. @@ -275,35 +272,34 @@ class SourceResolver(object): ''' text = None - if reponame == self._definitions_repo and \ - sha1 == self._definitions_absref: # pragma: no cover - # There is a temporary local checkout of the definitions repo which - # we can quickly read definitions files from. - defs_filename = os.path.join(self._definitions_checkout_dir, - filename) - text = self._get_file_contents_from_definitions(defs_filename) + if reponame == definitions_repo and \ + sha1 == definitions_absref: # pragma: no cover + text = self._get_file_contents_from_definitions( + definitions_checkout_dir, filename) else: text = self._get_file_contents_from_repo(reponame, sha1, filename) return text - def _get_morphology(self, reponame, sha1, filename): # pragma: no cover + def _get_morphology(self, resolved_morphologies, definitions_checkout_dir, + definitions_repo, definitions_absref, morph_loader, + reponame, sha1, filename): # pragma: no cover '''Read the morphology at the specified location. Returns None if the file does not exist in the specified commit. ''' key = (reponame, sha1, filename) - if key in self._resolved_morphologies: - return self._resolved_morphologies[key] - - loader = morphlib.morphloader.MorphologyLoader() + if key in resolved_morphologies: + return resolved_morphologies[key] - text = self._get_file_contents(reponame, sha1, filename) - morph = loader.load_from_string(text, filename) + text = self._get_file_contents(definitions_checkout_dir, + definitions_repo, definitions_absref, + reponame, sha1, filename) + morph = morph_loader.load_from_string(text, filename) if morph is not None: - self._resolved_morphologies[key] = morph + resolved_morphologies[key] = morph return morph @@ -350,16 +346,17 @@ class SourceResolver(object): return buildsystem - def _create_morphology_for_build_system(self, buildsystem, + @staticmethod + def _create_morphology_for_build_system(morph_loader, buildsystem, morph_name): # pragma: no cover - loader = morphlib.morphloader.MorphologyLoader() morph = buildsystem.get_morphology(morph_name) - loader.validate(morph) - loader.set_commands(morph) - loader.set_defaults(morph) + morph_loader.validate(morph) + morph_loader.set_commands(morph) + morph_loader.set_defaults(morph) return morph - def _parse_version_file(self, version_file): # pragma : no cover + @staticmethod + def _parse_version_file(version_file): # pragma : no cover '''Parse VERSION file and return the version of the format if: VERSION is a YAML file @@ -380,10 +377,9 @@ class SourceResolver(object): else None) - def _check_version_file(self, definitions_repo, - definitions_absref): # pragma: no cover - version_file = self._get_file_contents(definitions_repo, - definitions_absref, 'VERSION') + def _check_version_file(self, definitions_checkout_dir): # pragma: no cover + version_file = self._get_file_contents_from_definitions( + definitions_checkout_dir, 'VERSION') if version_file == None: return 0 # Assume version 0 if no version file @@ -398,23 +394,31 @@ class SourceResolver(object): return version - def _process_definitions_with_children(self, system_filenames, + def _process_definitions_with_children(self, + resolved_morphologies, + definitions_checkout_dir, definitions_repo, definitions_ref, definitions_absref, definitions_tree, + definitions_version, + morph_loader, + system_filenames, visit): # pragma: no cover definitions_queue = collections.deque(system_filenames) chunk_queue = set() - definitions_version = self._check_version_file(definitions_repo, - definitions_absref) + def get_morphology(repo, sha1, filename): + return self._get_morphology(resolved_morphologies, + definitions_checkout_dir, + definitions_repo, definitions_absref, + morph_loader, repo, sha1, filename) while definitions_queue: filename = definitions_queue.popleft() - morphology = self._get_morphology( - definitions_repo, definitions_absref, filename) + morphology = get_morphology(definitions_repo, definitions_absref, + filename) if morphology is None: raise MorphologyNotFoundError(filename) @@ -453,9 +457,9 @@ class SourceResolver(object): # Now, does this path actually exist? path = c['morph'] - morphology = self._get_morphology(definitions_repo, - definitions_absref, - path) + morphology = get_morphology(definitions_repo, + definitions_absref, + path) if morphology is None: if definitions_version > 1: raise MorphologyReferenceNotFoundError( @@ -471,21 +475,27 @@ class SourceResolver(object): return chunk_queue - def _generate_morph_and_cache_buildsystem(self, + @classmethod + def _generate_morph_and_cache_buildsystem(cls, resolved_morphologies, + resolved_buildsystems, + morph_loader, definition_key, chunk_key, buildsystem, morph_name): # pragma: no cover logging.debug('Caching build system for chunk with key %s', chunk_key) - self._resolved_buildsystems[chunk_key] = buildsystem.name + resolved_buildsystems[chunk_key] = buildsystem.name - morphology = self._create_morphology_for_build_system(buildsystem, - morph_name) - self._resolved_morphologies[definition_key] = morphology + morphology = cls._create_morphology_for_build_system( + morph_loader, buildsystem, morph_name) + resolved_morphologies[definition_key] = morphology return morphology - def process_chunk(self, definition_repo, definition_ref, chunk_repo, - chunk_ref, filename, visit): # pragma: no cover + def process_chunk(self, resolved_morphologies, resolved_trees, + resolved_buildsystems, definitions_checkout_dir, + definitions_repo, definitions_absref, morph_loader, + chunk_repo, chunk_ref, filename, + visit): # pragma: no cover absref = None tree = None chunk_key = None @@ -493,24 +503,36 @@ class SourceResolver(object): morph_name = os.path.splitext(os.path.basename(filename))[0] + def get_morphology(repo, sha1, filename): + return self._get_morphology( + resolved_morphologies, definitions_checkout_dir, + definitions_repo, definitions_absref, morph_loader, + repo, sha1, filename) + # Get morphology from definitions repo - definition_key = (definition_repo, definition_ref, filename) - morphology = self._get_morphology(*definition_key) + definition_key = (definitions_repo, definitions_absref, filename) + morphology = get_morphology(*definition_key) if morphology: - absref, tree = self._resolve_ref(chunk_repo, chunk_ref) + absref, tree = self._resolve_ref(resolved_trees, chunk_repo, + chunk_ref) visit(chunk_repo, chunk_ref, filename, absref, tree, morphology) return - absref, tree = self._resolve_ref(chunk_repo, chunk_ref) + absref, tree = self._resolve_ref(resolved_trees, chunk_repo, chunk_ref) chunk_key = (chunk_repo, absref, filename) - if chunk_key in self._resolved_buildsystems: + def generate_morph_and_cache_buildsystem(buildsystem): + return self._generate_morph_and_cache_buildsystem( + resolved_morphologies, resolved_buildsystems, morph_loader, + definition_key, chunk_key, buildsystem, morph_name) + + if chunk_key in resolved_buildsystems: logging.debug('Build system for %s is cached', str(chunk_key)) self.status(msg='Build system for %(chunk)s is cached', chunk=str(chunk_key), chatty=True) - buildsystem_name = self._resolved_buildsystems[chunk_key] + buildsystem_name = resolved_buildsystems[chunk_key] buildsystem = morphlib.buildsystem.lookup_build_system( buildsystem_name) @@ -522,19 +544,18 @@ class SourceResolver(object): # # If the morphology is not already cached we can generate it # from the build-system and cache it. - if definition_key in self._resolved_morphologies: - morphology = self._resolved_morphologies[definition_key] + if definition_key in resolved_morphologies: + morphology = resolved_morphologies[definition_key] else: - morphology = self._generate_morph_and_cache_buildsystem( - definition_key, chunk_key, buildsystem, morph_name) + morphology = generate_morph_and_cache_buildsystem(buildsystem) else: logging.debug('Build system for %s is NOT cached', str(chunk_key)) # build-system not cached, look for morphology in chunk repo # this can be slow (we may need to clone the repo from a remote) - morphology = self._get_morphology(*chunk_key) + morphology = get_morphology(*chunk_key) if morphology != None: - self._resolved_morphologies[definition_key] = morphology + resolved_morphologies[definition_key] = morphology else: # This chunk doesn't have a chunk morph buildsystem = self._detect_build_system(*chunk_key) @@ -542,8 +563,8 @@ class SourceResolver(object): if buildsystem is None: raise MorphologyNotFoundError(filename) else: - morphology = self._generate_morph_and_cache_buildsystem( - definition_key, chunk_key, buildsystem, morph_name) + morphology = generate_morph_and_cache_buildsystem( + buildsystem) visit(chunk_repo, chunk_ref, filename, absref, tree, morphology) @@ -551,39 +572,46 @@ class SourceResolver(object): system_filenames, visit=lambda rn, rf, fn, arf, m: None, definitions_original_ref=None): # pragma: no cover - # Resolve the (repo, ref) pair for the definitions repo, cache result. - definitions_absref, definitions_tree = self._resolve_ref( - definitions_repo, definitions_ref) - if definitions_original_ref: - definitions_ref = definitions_original_ref + resolved_morphologies = {} with morphlib.util.temp_dir() as definitions_checkout_dir, \ self.tree_cache_manager.open() as resolved_trees, \ self.buildsystem_cache_manager.open() as resolved_buildsystems: - # FIXME: not an ideal way of passing this info across - self._definitions_checkout_dir = definitions_checkout_dir - self._resolved_trees = resolved_trees - self._resolved_buildsystems = resolved_buildsystems - self._definitions_repo = definitions_repo - self._definitions_absref = definitions_absref + + # Resolve the repo, ref pair for definitions repo, cache result + definitions_absref, definitions_tree = self._resolve_ref( + resolved_trees, definitions_repo, definitions_ref) + + if definitions_original_ref: + definitions_ref = definitions_original_ref definitions_cached_repo = self.lrc.get_updated_repo( repo_name=definitions_repo, ref=definitions_absref) definitions_cached_repo.extract_commit( - definitions_absref, self._definitions_checkout_dir) + definitions_absref, definitions_checkout_dir) + + definitions_version = self._check_version_file( + definitions_checkout_dir) + morph_loader = morphlib.morphloader.MorphologyLoader() # First, process the system and its stratum morphologies. These # will all live in the same Git repository, and will point to # various chunk morphologies. chunk_queue = self._process_definitions_with_children( - system_filenames, definitions_repo, definitions_ref, - definitions_absref, definitions_tree, visit) + resolved_morphologies, definitions_checkout_dir, + definitions_repo, definitions_ref, definitions_absref, + definitions_tree, definitions_version, morph_loader, + system_filenames, visit) # Now process all the chunks involved in the build. for repo, ref, filename in chunk_queue: - self.process_chunk(definitions_repo, definitions_absref, repo, - ref, filename, visit) + self.process_chunk(resolved_morphologies, resolved_trees, + resolved_buildsystems, + definitions_checkout_dir, + definitions_repo, definitions_absref, + morph_loader, repo, ref, filename, + visit) def create_source_pool(lrc, rrc, repo, ref, filenames, cachedir, diff --git a/morphlib/sourceresolver_tests.py b/morphlib/sourceresolver_tests.py index 6cd4ddf7..956ec40c 100644 --- a/morphlib/sourceresolver_tests.py +++ b/morphlib/sourceresolver_tests.py @@ -174,9 +174,6 @@ class SourceResolverTests(unittest.TestCase): self.lsr = SourceResolver(self.lrc, None, tree_cache_manager, buildsystem_cache_manager, True, status) - self.sr._definitions_repo = None - self.lsr._definitions_repo = None - def tearDown(self): shutil.rmtree(self.cachedir) @@ -216,23 +213,31 @@ class SourceResolverTests(unittest.TestCase): def test_gets_morph_from_local_repo(self): self.lr.list_files = self.localmorph - morph = self.sr._get_morphology('reponame', 'sha1', - 'chunk.morph') + morph = self.sr._get_morphology( + {}, None, None, None, + morphlib.morphloader.MorphologyLoader(), 'reponame', + 'sha1', 'chunk.morph') self.assertEqual('chunk', morph['name']) def test_gets_morph_from_cache(self): self.lr.list_files = self.localmorph - morph_from_repo = self.sr._get_morphology('reponame', 'sha1', - 'chunk.morph') - morph_from_cache = self.sr._get_morphology('reponame', 'sha1', - 'chunk.morph') + morph_from_repo = self.sr._get_morphology( + {}, None, None, None, + morphlib.morphloader.MorphologyLoader(), 'reponame', + 'sha1', 'chunk.morph') + morph_from_cache = self.sr._get_morphology( + {}, None, None, None, + morphlib.morphloader.MorphologyLoader(), 'reponame', + 'sha1', 'chunk.morph') self.assertEqual(morph_from_repo, morph_from_cache) def test_gets_morph_from_remote_repo(self): self.rrc.ls_tree = self.remotemorph self.lrc.has_repo = self.doesnothaverepo - morph = self.sr._get_morphology('reponame', 'sha1', - 'remote-chunk.morph') + morph = self.sr._get_morphology( + {}, None, None, None, + morphlib.morphloader.MorphologyLoader(), 'reponame', + 'sha1', 'remote-chunk.morph') self.assertEqual('remote-chunk', morph['name']) def test_autodetects_local_morphology(self): @@ -261,13 +266,19 @@ class SourceResolverTests(unittest.TestCase): def test_returns_none_when_no_local_morph(self): self.lr.read_file = self.nolocalfile - morph = self.sr._get_morphology('reponame', 'sha1', 'unreached.morph') + morph = self.sr._get_morphology( + {}, None, None, None, + morphlib.morphloader.MorphologyLoader(), 'reponame', + 'sha1', 'unreached.morph') self.assertEqual(morph, None) def test_returns_none_when_fails_no_remote_morph(self): self.lrc.has_repo = self.doesnothaverepo self.rrc.cat_file = self.noremotefile - morph = self.sr._get_morphology('reponame', 'sha1', 'unreached.morph') + morph = self.sr._get_morphology( + {}, None, None, None, + morphlib.morphloader.MorphologyLoader(), 'reponame', + 'sha1', 'unreached.morph') self.assertEqual(morph, None) def test_raises_error_when_repo_does_not_exist(self): @@ -284,13 +295,16 @@ class SourceResolverTests(unittest.TestCase): 'reponame', 'sha1', 'undetected.morph') def test_raises_error_when_name_mismatches(self): - self.assertRaises(morphlib.Error, self.sr._get_morphology, - 'reponame', 'sha1', 'name-mismatch.morph') + self.assertRaises(morphlib.Error, self.sr._get_morphology, {}, + None, None, None, morphlib.morphloader.MorphologyLoader(), + 'reponame', 'sha1', 'name-mismatch.morph') def test_looks_locally_with_no_remote(self): self.lr.list_files = self.localmorph - morph = self.lsr._get_morphology('reponame', 'sha1', - 'chunk.morph') + morph = self.lsr._get_morphology( + {}, None, None, None, + morphlib.morphloader.MorphologyLoader(), 'reponame', + 'sha1', 'chunk.morph') self.assertEqual('chunk', morph['name']) def test_autodetects_locally_with_no_remote(self): @@ -303,37 +317,47 @@ class SourceResolverTests(unittest.TestCase): def test_succeeds_when_local_not_cached_and_no_remote(self): self.lrc.has_repo = self.doesnothaverepo self.lr.list_files = self.localmorph - morph = self.sr._get_morphology('reponame', 'sha1', - 'chunk.morph') + morph = self.sr._get_morphology( + {}, None, None, None, + morphlib.morphloader.MorphologyLoader(), 'reponame', + 'sha1', 'chunk.morph') self.assertEqual('chunk', morph['name']) def test_arch_is_validated(self): self.lr.arch = 'unknown' - self.assertRaises(morphlib.Error, self.sr._get_morphology, - 'reponame', 'sha1', 'system.morph') + self.assertRaises(morphlib.Error, self.sr._get_morphology, {}, + None, None, None, morphlib.morphloader.MorphologyLoader(), + 'reponame', 'sha1', 'system.morph') def test_arch_arm_defaults_to_le(self): self.lr.arch = 'armv7' - morph = self.sr._get_morphology('reponame', 'sha1', 'system.morph') + morph = self.sr._get_morphology( + {}, None, None, None, + morphlib.morphloader.MorphologyLoader(), 'reponame', + 'sha1', 'system.morph') self.assertEqual(morph['arch'], 'armv7l') def test_fails_on_parse_error(self): - self.assertRaises(morphlib.Error, self.sr._get_morphology, - 'reponame', 'sha1', 'parse-error.morph') + self.assertRaises(morphlib.Error, self.sr._get_morphology, {}, + None, None, None, morphlib.morphloader.MorphologyLoader(), + 'reponame', 'sha1', 'parse-error.morph') def test_fails_on_no_bdeps_or_bootstrap(self): self.assertRaises( morphlib.morphloader.NoStratumBuildDependenciesError, - self.sr._get_morphology, 'reponame', 'sha1', + self.sr._get_morphology, {}, None, None, None, + morphlib.morphloader.MorphologyLoader(), 'reponame', 'sha1', 'stratum-no-bdeps-no-bootstrap.morph') def test_succeeds_on_bdeps_no_bootstrap(self): - self.sr._get_morphology( - 'reponame', 'sha1', + self.sr._get_morphology({}, None, None, None, + morphlib.morphloader.MorphologyLoader(), 'reponame', 'sha1', 'stratum-bdeps-no-bootstrap.morph') def test_fails_on_empty_stratum(self): self.assertRaises( morphlib.morphloader.EmptyStratumError, - self.sr._get_morphology, 'reponame', 'sha1', 'stratum-empty.morph') + self.sr._get_morphology, {}, None, None, None, + morphlib.morphloader.MorphologyLoader(), 'reponame', 'sha1', + 'stratum-empty.morph') |