diff options
| -rw-r--r-- | morphlib/buildbranch.py | 2 | ||||
| -rw-r--r-- | morphlib/builder.py | 4 | ||||
| -rw-r--r-- | morphlib/extensions.py | 2 | ||||
| -rw-r--r-- | morphlib/plugins/cross-bootstrap_plugin.py | 4 | ||||
| -rw-r--r-- | morphlib/plugins/list_artifacts_plugin.py | 3 | ||||
| -rw-r--r-- | morphlib/sourceresolver.py | 75 | ||||
| -rw-r--r-- | morphlib/writeexts.py | 2 |
7 files changed, 65 insertions, 27 deletions
diff --git a/morphlib/buildbranch.py b/morphlib/buildbranch.py index 07fd9774..6c736c7b 100644 --- a/morphlib/buildbranch.py +++ b/morphlib/buildbranch.py @@ -114,7 +114,7 @@ class BuildBranch(object): for morphology in morphologies: loader.unset_defaults(morphology) sha1 = gd.store_blob(loader.save_to_string(morphology)) - yield 0100644, sha1, morphology.filename + yield 0o100644, sha1, morphology.filename def inject_build_refs(self, loader, use_local_repos, inject_cb=lambda **kwargs: None): diff --git a/morphlib/builder.py b/morphlib/builder.py index 85f775b1..94eeacb8 100644 --- a/morphlib/builder.py +++ b/morphlib/builder.py @@ -405,7 +405,7 @@ class ChunkBuilder(BuilderBase): with morphlib.savefile.SaveFile(script_path, 'w') as f: f.write("#!/bin/sh\nset -xeu\n") f.write(script) - os.chmod(script_path, 0555) + os.chmod(script_path, 0o555) rel_script_path = os.path.join(SYSTEM_INTEGRATION_PATH, script_name) @@ -623,7 +623,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover f.write('SUPPORT_URL="http://wiki.baserock.org/mailinglist"\n') f.write('BUG_REPORT_URL="http://wiki.baserock.org/mailinglist"\n') - os.chmod(os_release_file, 0644) + os.chmod(os_release_file, 0o644) def run_system_integration_commands(self, rootdir): # pragma: no cover ''' Run the system integration commands ''' diff --git a/morphlib/extensions.py b/morphlib/extensions.py index 8fb8dd49..aa0622b8 100644 --- a/morphlib/extensions.py +++ b/morphlib/extensions.py @@ -148,7 +148,7 @@ class get_extension_filename(): fd, ext_filename = tempfile.mkstemp() os.write(fd, ext_contents) os.close(fd) - os.chmod(ext_filename, 0700) + os.chmod(ext_filename, 0o700) self.delete = True self.ext_filename = ext_filename diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py index 46e33f7f..e1f566c2 100644 --- a/morphlib/plugins/cross-bootstrap_plugin.py +++ b/morphlib/plugins/cross-bootstrap_plugin.py @@ -110,7 +110,7 @@ class BootstrapSystemBuilder(morphlib.builder.BuilderBase): chunk_script = os.path.join(path, 'src', 'build-%s' % name) with morphlib.savefile.SaveFile(chunk_script, 'w') as f: self.write_chunk_build_script(s, f) - os.chmod(chunk_script, 0777) + os.chmod(chunk_script, 0o777) def write_build_script(self, path): '''Output a script to run build on the bootstrap target''' @@ -135,7 +135,7 @@ class BootstrapSystemBuilder(morphlib.builder.BuilderBase): f.write('if [ -e /sbin/ldconfig ]; then /sbin/ldconfig; fi\n') f.write(driver_footer) - os.chmod(driver_script, 0777) + os.chmod(driver_script, 0o777) def write_chunk_build_script(self, source, f): m = source.morphology diff --git a/morphlib/plugins/list_artifacts_plugin.py b/morphlib/plugins/list_artifacts_plugin.py index ed8951b8..b7a5e080 100644 --- a/morphlib/plugins/list_artifacts_plugin.py +++ b/morphlib/plugins/list_artifacts_plugin.py @@ -16,6 +16,7 @@ # # See: <http://wiki.baserock.org/guides/release-process> for more information. +from __future__ import print_function import cliapp import morphlib @@ -67,7 +68,7 @@ class ListArtifactsPlugin(cliapp.Plugin): artifact_files.update(system_artifact_files) for artifact_file in sorted(artifact_files): - print artifact_file + print(artifact_file) def list_artifacts_for_system(self, repo, ref, system_filename): '''List all artifact files in the build graph of a single system.''' diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index fcf4793d..0b8ab5db 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -184,6 +184,8 @@ class SourceResolver(object): reponame, ref) return ref, self._resolved_trees[(reponame, ref)] + logging.debug('tree (%s, %s) not in cache', reponame, ref) + absref = None if self.lrc.has_repo(reponame): repo = self.lrc.get_repo(reponame) @@ -414,42 +416,77 @@ class SourceResolver(object): return chunk_queue + def _generate_morph_and_cache_buildsystem(self, + definition_key, chunk_key, + buildsystem, + morph_name): # pragma: no cover + logging.debug('Caching build system for chunk with key %s', chunk_key) + + self._resolved_buildsystems[chunk_key] = buildsystem + + morphology = self._create_morphology_for_build_system(buildsystem, + morph_name) + self._resolved_morphologies[definition_key] = morphology + return morphology + def process_chunk(self, definition_repo, definition_ref, chunk_repo, chunk_ref, filename, visit): # pragma: no cover absref = None tree = None - - definition_key = (definition_repo, definition_ref, filename) chunk_key = None + buildsystem = None morph_name = os.path.splitext(os.path.basename(filename))[0] + # Get morphology from definitions repo + definition_key = (definition_repo, definition_ref, filename) morphology = self._get_morphology(*definition_key) - buildsystem = None + + if morphology: + absref, tree = self._resolve_ref(chunk_repo, chunk_ref) + visit(chunk_repo, chunk_ref, filename, absref, tree, morphology) + return + + absref, tree = self._resolve_ref(chunk_repo, chunk_ref) + chunk_key = (chunk_repo, absref, filename) if chunk_key in self._resolved_buildsystems: + logging.debug('Build system for %s is cached', str(chunk_key)) + self.status(msg='Build system for %(chunk)s is cached', + chunk=str(chunk_key), + chatty=True) buildsystem = self._resolved_buildsystems[chunk_key] - if morphology is None and buildsystem is None: - # This is a slow operation (looking for a file in Git repo may - # potentially require cloning the whole thing). - absref, tree = self._resolve_ref(chunk_repo, chunk_ref) - chunk_key = (chunk_repo, absref, filename) + # If the build system for this chunk is cached then: + # * the chunk does not have a chunk morph + # (so we don't need to look for one) + # + # * a suitable (generated) morphology may already be cached. + # + # If the morphology is not already cached we can generate it + # from the build-system and cache it. + if definition_key in self._resolved_morphologies: + morphology = self._resolved_morphologies[definition_key] + else: + morphology = self._generate_morph_and_cache_buildsystem( + definition_key, chunk_key, buildsystem, morph_name) + else: + logging.debug('Build system for %s is NOT cached', str(chunk_key)) + # build-system not cached, look for morphology in chunk repo + # this can be slow (we may need to clone the repo from a remote) morphology = self._get_morphology(*chunk_key) - if morphology is None: - if buildsystem is None: - buildsystem = self._detect_build_system(*chunk_key) - if buildsystem is None: - raise MorphologyNotFoundError(filename) - else: - self._resolved_buildsystems[chunk_key] = buildsystem - morphology = self._create_morphology_for_build_system( - buildsystem, morph_name) + if morphology != None: self._resolved_morphologies[definition_key] = morphology + else: + # This chunk doesn't have a chunk morph + buildsystem = self._detect_build_system(*chunk_key) - if not absref or not tree: - absref, tree = self._resolve_ref(chunk_repo, chunk_ref) + if buildsystem is None: + raise MorphologyNotFoundError(filename) + else: + morphology = self._generate_morph_and_cache_buildsystem( + definition_key, chunk_key, buildsystem, morph_name) visit(chunk_repo, chunk_ref, filename, absref, tree, morphology) diff --git a/morphlib/writeexts.py b/morphlib/writeexts.py index 6eec4465..3382767c 100644 --- a/morphlib/writeexts.py +++ b/morphlib/writeexts.py @@ -359,7 +359,7 @@ class WriteExtension(cliapp.Application): self.status(msg='Creating %s subvolume' % state_subdir) subvolume = os.path.join(mountpoint, 'state', state_subdir) cliapp.runcmd(['btrfs', 'subvolume', 'create', subvolume]) - os.chmod(subvolume, 0755) + os.chmod(subvolume, 0o755) existing_state_dir = os.path.join(system_dir, state_subdir) files = [] |