diff options
author | Paul Sherwood <paul.sherwood@codethink.co.uk> | 2014-10-27 13:28:53 +0000 |
---|---|---|
committer | Paul Sherwood <paul.sherwood@codethink.co.uk> | 2014-10-27 13:28:53 +0000 |
commit | 79c06711f47d39bdff6d84c2736d88b99ef35b9c (patch) | |
tree | 4a0774f36f9034d8d3a1d7f4d010fad7a80f2cd8 /morphlib | |
parent | ee8b0047ae5a5e7c64f32197cdf20d552a49cd9e (diff) | |
parent | 5e823a4f36f30c59a473f22f824dfd6f6c66c89f (diff) | |
download | morph-79c06711f47d39bdff6d84c2736d88b99ef35b9c.tar.gz |
Merge branch 'master' of git://git.baserock.org/baserock/baserock/morph into HEAD
Diffstat (limited to 'morphlib')
-rw-r--r-- | morphlib/app.py | 12 | ||||
-rw-r--r-- | morphlib/buildbranch.py | 16 | ||||
-rw-r--r-- | morphlib/buildcommand.py | 44 | ||||
-rw-r--r-- | morphlib/builder2.py | 80 | ||||
-rw-r--r-- | morphlib/extensions.py | 6 | ||||
-rw-r--r-- | morphlib/plugins/branch_and_merge_plugin.py | 2 | ||||
-rw-r--r-- | morphlib/plugins/build_plugin.py | 11 | ||||
-rw-r--r-- | morphlib/plugins/deploy_plugin.py | 4 | ||||
-rw-r--r-- | morphlib/stagingarea.py | 24 |
9 files changed, 130 insertions, 69 deletions
diff --git a/morphlib/app.py b/morphlib/app.py index 9ab102b3..c3c9c970 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -290,7 +290,8 @@ class Morph(cliapp.Application): morphlib.util.sanitise_morphology_path(args[2])) args = args[3:] - def create_source_pool(self, lrc, rrc, repo, ref, filename): + def create_source_pool(self, lrc, rrc, repo, ref, filename, + original_ref=None): pool = morphlib.sourcepool.SourcePool() def add_to_pool(reponame, ref, filename, absref, tree, morphology): @@ -302,7 +303,8 @@ class Morph(cliapp.Application): self.traverse_morphs(repo, ref, [filename], lrc, rrc, update=not self.settings['no-git-update'], - visit=add_to_pool) + visit=add_to_pool, + definitions_original_ref=original_ref) return pool def resolve_ref(self, lrc, rrc, reponame, ref, update=True): @@ -346,7 +348,8 @@ class Morph(cliapp.Application): def traverse_morphs(self, definitions_repo, definitions_ref, system_filenames, lrc, rrc, update=True, - visit=lambda rn, rf, fn, arf, m: None): + visit=lambda rn, rf, fn, arf, m: None, + definitions_original_ref=None): morph_factory = morphlib.morphologyfactory.MorphologyFactory(lrc, rrc, self) definitions_queue = collections.deque(system_filenames) @@ -359,6 +362,9 @@ class Morph(cliapp.Application): definitions_absref, definitions_tree = self.resolve_ref( lrc, rrc, definitions_repo, definitions_ref, update) + if definitions_original_ref: + definitions_ref = definitions_original_ref + while definitions_queue: filename = definitions_queue.popleft() diff --git a/morphlib/buildbranch.py b/morphlib/buildbranch.py index 638350e3..cfc4a67f 100644 --- a/morphlib/buildbranch.py +++ b/morphlib/buildbranch.py @@ -254,6 +254,10 @@ class BuildBranch(object): return self._sb.get_config('branch.name') @property + def root_commit(self): + return self._root.resolve_ref_to_commit(self.root_ref) + + @property def root_local_repo_url(self): return urlparse.urljoin('file://', self._root.dirname) @@ -291,7 +295,13 @@ def pushed_build_branch(bb, loader, changes_need_pushing, name, email, unpushed = any(bb.get_unpushed_branches()) if not changes_made and not unpushed: - yield bb.root_repo_url, bb.root_ref + # We resolve the system branch ref to the commit SHA1 here, so that + # the build uses whatever commit the user's copy of the root repo + # considers the head of that branch to be. If we returned a named + # ref, we risk building what the remote considers the head of that + # branch to be instead, and we also trigger a needless update in + # the cached copy of the root repo. + yield bb.root_repo_url, bb.root_commit, bb.root_ref return def report_inject(gd): @@ -318,6 +328,6 @@ def pushed_build_branch(bb, loader, changes_need_pushing, name, email, remote=remote.get_push_url(), chatty=True) bb.push_build_branches(push_cb=report_push) - yield bb.root_repo_url, bb.root_build_ref + yield bb.root_repo_url, bb.root_build_ref, bb.root_build_ref else: - yield bb.root_local_repo_url, bb.root_build_ref + yield bb.root_local_repo_url, bb.root_build_ref, bb.root_build_ref diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 2aec5e08..544d88d8 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -50,21 +50,24 @@ class BuildCommand(object): self.lac, self.rac = self.new_artifact_caches() self.lrc, self.rrc = self.new_repo_caches() - def build(self, args): - '''Build triplets specified on command line.''' + def build(self, repo_name, ref, filename, original_ref=None): + '''Build a given system morphology.''' - self.app.status(msg='Build starts', chatty=True) + self.app.status( + msg='Building %(repo_name)s %(ref)s %(filename)s', + repo_name=repo_name, ref=ref, filename=filename) - for repo_name, ref, filename in self.app.itertriplets(args): - self.app.status(msg='Building %(repo_name)s %(ref)s %(filename)s', - repo_name=repo_name, ref=ref, filename=filename) - self.app.status(msg='Deciding on task order') - srcpool = self.create_source_pool(repo_name, ref, filename) - self.validate_sources(srcpool) - root_artifact = self.resolve_artifacts(srcpool) - self.build_in_order(root_artifact) + self.app.status(msg='Deciding on task order') + srcpool = self.create_source_pool( + repo_name, ref, filename, original_ref) + self.validate_sources(srcpool) + root_artifact = self.resolve_artifacts(srcpool) + self.build_in_order(root_artifact) - self.app.status(msg='Build ends successfully') + self.app.status( + msg='Build of %(repo_name)s %(ref)s %(filename)s ended ' + 'successfully', + repo_name=repo_name, ref=ref, filename=filename) def new_artifact_caches(self): '''Create interfaces for the build artifact caches. @@ -82,7 +85,7 @@ class BuildCommand(object): return morphlib.buildenvironment.BuildEnvironment(self.app.settings, arch) - def create_source_pool(self, repo_name, ref, filename): + def create_source_pool(self, repo_name, ref, filename, original_ref=None): '''Find the source objects required for building a the given artifact The SourcePool will contain every stratum and chunk dependency of the @@ -92,7 +95,8 @@ class BuildCommand(object): ''' self.app.status(msg='Creating source pool', chatty=True) srcpool = self.app.create_source_pool( - self.lrc, self.rrc, repo_name, ref, filename) + self.lrc, self.rrc, repo_name, ref, filename, + original_ref=original_ref) return srcpool @@ -184,7 +188,7 @@ class BuildCommand(object): def _validate_cross_morphology_references(self, srcpool): '''Perform validation across all morphologies involved in the build''' - stratum_names = [] + stratum_names = {} for src in srcpool: kind = src.morphology['kind'] @@ -208,15 +212,11 @@ class BuildCommand(object): # and Ref specified. if src.morphology['kind'] == 'stratum': name = src.name - ref = src.sha1[:7] - self.app.status(msg='Stratum [%(name)s] version is %(ref)s', - name=name, ref=ref) if name in stratum_names: raise morphlib.Error( - "Conflicting versions of stratum '%s' appear in the " - "build. Check the contents of the system against the " - "build-depends of the strata." % name) - stratum_names.append(name) + "Multiple strata produce a '%s' artifact: %s and %s" % + (name, stratum_names[name].filename, src.filename)) + stratum_names[name] = src def _validate_cross_refs_for_system(self, src, srcpool): self._validate_cross_refs_for_xxx( diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 9cd3a074..8615ed59 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -28,6 +28,7 @@ import time import traceback import subprocess import tempfile +import textwrap import gzip import cliapp @@ -341,6 +342,7 @@ class ChunkBuilder(BuilderBase): relative_builddir = self.staging_area.relative(builddir) relative_destdir = self.staging_area.relative(destdir) + ccache_dir = self.staging_area.ccache_dir(self.source) extra_env = { 'DESTDIR': relative_destdir } steps = [ @@ -390,7 +392,8 @@ class ChunkBuilder(BuilderBase): cwd=relative_builddir, stdout=stdout or subprocess.PIPE, stderr=subprocess.STDOUT, - logfile=logfilepath) + logfile=logfilepath, + ccache_dir=ccache_dir) if stdout: stdout.flush() @@ -644,6 +647,53 @@ class SystemBuilder(BuilderBase): # pragma: no cover os.chmod(os_release_file, 0644) + def _chroot_runcmd(self, rootdir, to_mount, env, *args): + # We need to do mounts in a different namespace. Unfortunately + # this means we have to in-line the mount commands in the + # command-line. + command = textwrap.dedent(r''' + mount --make-rprivate / + rootdir="$1" + shift + ''') + cmdargs = [rootdir] + + # We need to mount all the specified mounts in the namespace, + # we don't need to unmount them before exiting, as they'll be + # unmounted when the namespace is no longer used. + command += textwrap.dedent(r''' + while true; do + case "$1" in + --) + shift + break + ;; + *) + mount_point="$1" + mount_type="$2" + mount_source="$3" + shift 3 + path="$rootdir/$mount_point" + mount -t "$mount_type" "$mount_source" "$path" + ;; + esac + done + ''') + for mount_opts in to_mount: + cmdargs.extend(mount_opts) + cmdargs.append('--') + + command += textwrap.dedent(r''' + exec chroot "$rootdir" "$@" + ''') + cmdargs.extend(args) + + # The single - is just a shell convention to fill $0 when using -c, + # since ordinarily $0 contains the program name. + cmdline = ['unshare', '--mount', '--', 'sh', '-ec', command, '-'] + cmdline.extend(cmdargs) + self.app.runcmd(cmdline, env=env) + def run_system_integration_commands(self, rootdir): # pragma: no cover ''' Run the system integration commands ''' @@ -655,44 +705,26 @@ class SystemBuilder(BuilderBase): # pragma: no cover 'PATH': '/bin:/usr/bin:/sbin:/usr/sbin' } - self.app.status(msg='Running the system integration commands', - error=True) + self.app.status(msg='Running the system integration commands') - mounted = [] to_mount = ( ('proc', 'proc', 'none'), ('dev/shm', 'tmpfs', 'none'), + ('tmp', 'tmpfs', 'none'), ) - try: for mount_point, mount_type, source in to_mount: - logging.debug('Mounting %s in system root filesystem' - % mount_point) path = os.path.join(rootdir, mount_point) if not os.path.exists(path): os.makedirs(path) - morphlib.fsutils.mount(self.app.runcmd, source, path, - mount_type) - mounted.append(path) - - # The single - is just a shell convention to fill $0 when using -c, - # since ordinarily $0 contains the program name. - # -- is used to indicate the end of options for run-parts, - # we don't want SYSTEM_INTEGRATION_PATH to be interpreted - # as an option if it happens to begin with a - - self.app.runcmd(['chroot', rootdir, 'sh', '-c', - 'cd / && run-parts -- "$1"', '-', SYSTEM_INTEGRATION_PATH], - env=env) + for bin in sorted(os.listdir(sys_integration_dir)): + self._chroot_runcmd(rootdir, to_mount, env, + os.path.join(SYSTEM_INTEGRATION_PATH, bin)) except BaseException, e: self.app.status( msg='Error while running system integration commands', error=True) raise - finally: - for mount_path in reversed(mounted): - logging.debug('Unmounting %s in system root filesystem' - % mount_path) - morphlib.fsutils.unmount(self.app.runcmd, mount_path) class Builder(object): # pragma: no cover diff --git a/morphlib/extensions.py b/morphlib/extensions.py index af6ba279..6b81e116 100644 --- a/morphlib/extensions.py +++ b/morphlib/extensions.py @@ -223,7 +223,11 @@ class ExtensionSubprocess(object): def close_read_end(): os.close(log_read_fd) p = subprocess.Popen( - [filename] + args, cwd=cwd, env=new_env, + # We unshare and mount --make-rprivate so mounts done by write + # extensions can't interfere with the rest of the system. + ['unshare', '-m', '--', '/bin/sh', '-c', + 'mount --make-rprivate / && exec "$@"', '-', filename] + args, + cwd=cwd, env=new_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=close_read_end) os.close(log_write_fd) diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index d816fb90..5531f7f6 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -36,7 +36,7 @@ class BranchAndMergePlugin(cliapp.Plugin): self.app.add_subcommand( 'branch', self.branch, arg_synopsis='REPO NEW [OLD]') self.app.add_subcommand( - 'edit', self.edit, arg_synopsis='SYSTEM STRATUM [CHUNK]') + 'edit', self.edit, arg_synopsis='CHUNK') self.app.add_subcommand( 'show-system-branch', self.show_system_branch, arg_synopsis='') self.app.add_subcommand( diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py index 64630c2b..218bd819 100644 --- a/morphlib/plugins/build_plugin.py +++ b/morphlib/plugins/build_plugin.py @@ -56,7 +56,8 @@ class BuildPlugin(cliapp.Plugin): build_command = morphlib.buildcommand.InitiatorBuildCommand( self.app, addr, port) - build_command.build(args) + for repo_name, ref, filename in self.app.itertriplets(args): + build_command.build(repo_name, ref, filename) def distbuild(self, args): '''Distbuild a system image in the current system branch @@ -116,7 +117,8 @@ class BuildPlugin(cliapp.Plugin): self.app.settings['cachedir-min-space']) build_command = morphlib.buildcommand.BuildCommand(self.app) - build_command.build(args) + for repo_name, ref, filename in self.app.itertriplets(args): + build_command.build(repo_name, ref, filename) def build(self, args): '''Build a system image in the current system branch @@ -189,5 +191,6 @@ class BuildPlugin(cliapp.Plugin): bb, loader=loader, changes_need_pushing=push, name=name, email=email, build_uuid=build_uuid, status=self.app.status) - with pbb as (repo, ref): - build_command.build([repo, ref, system_filename]) + with pbb as (repo, commit, original_ref): + build_command.build(repo, commit, system_filename, + original_ref=original_ref) diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 2bc53a0d..e795e637 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -328,14 +328,14 @@ class DeployPlugin(cliapp.Plugin): bb, loader=loader, changes_need_pushing=False, name=name, email=email, build_uuid=build_uuid, status=self.app.status) - with pbb as (repo, ref): + with pbb as (repo, commit, original_ref): # Create a tempdir for this deployment to work in deploy_tempdir = tempfile.mkdtemp( dir=os.path.join(self.app.settings['tempdir'], 'deployments')) try: for system in cluster_morphology['systems']: self.deploy_system(build_command, deploy_tempdir, - root_repo_dir, repo, ref, system, + root_repo_dir, repo, commit, system, env_vars, deployments, parent_location='') finally: diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index bfe0a716..25e33b3f 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -192,11 +192,10 @@ class StagingArea(object): shutil.rmtree(self.dirname) to_mount = ( - ('proc', 'proc', 'none'), ('dev/shm', 'tmpfs', 'none'), ) - def mount_ccachedir(self, source): #pragma: no cover + def ccache_dir(self, source): #pragma: no cover ccache_dir = self._app.settings['compiler-cache-dir'] if not os.path.isdir(ccache_dir): os.makedirs(ccache_dir) @@ -223,10 +222,7 @@ class StagingArea(object): # to avoid breaking when faced with an empty staging area. if not os.path.isdir(ccache_destdir): os.makedirs(ccache_destdir) - # Mount it into the staging-area - self._app.runcmd(['mount', '--bind', ccache_repodir, - ccache_destdir]) - return ccache_destdir + return ccache_repodir def do_mounts(self, setup_mounts): # pragma: no cover if not setup_mounts: @@ -257,9 +253,6 @@ class StagingArea(object): self.do_mounts(setup_mounts) - if not self._app.settings['no-ccache']: - self.mounted.append(self.mount_ccachedir(source)) - return builddir, destdir def chroot_close(self): # pragma: no cover @@ -284,6 +277,7 @@ class StagingArea(object): del kwargs['cwd'] else: cwd = '/' + ccache_dir = kwargs.pop('ccache_dir', None) chroot_dir = self.dirname if self.use_chroot else '/' temp_dir = kwargs["env"].get("TMPDIR", "/tmp") @@ -304,6 +298,18 @@ class StagingArea(object): if not os.path.islink(d): real_argv += ['--mount-readonly', self.relative(d)] + if self.use_chroot: + proc_target = os.path.join(self.dirname, 'proc') + if not os.path.exists(proc_target): + os.makedirs(proc_target) + real_argv += ['--mount-proc', self.relative(proc_target)] + + if ccache_dir and not self._app.settings['no-ccache']: + ccache_target = os.path.join( + self.dirname, kwargs['env']['CCACHE_DIR'].lstrip('/')) + real_argv += ['--mount-bind', ccache_dir, + self.relative(ccache_target)] + real_argv += [chroot_dir] real_argv += argv |