diff options
-rw-r--r-- | morphlib/app.py | 12 | ||||
-rw-r--r-- | morphlib/buildbranch.py | 16 | ||||
-rw-r--r-- | morphlib/buildcommand.py | 44 | ||||
-rw-r--r-- | morphlib/builder2.py | 80 | ||||
-rw-r--r-- | morphlib/extensions.py | 6 | ||||
-rw-r--r-- | morphlib/plugins/branch_and_merge_plugin.py | 2 | ||||
-rw-r--r-- | morphlib/plugins/build_plugin.py | 11 | ||||
-rw-r--r-- | morphlib/plugins/deploy_plugin.py | 4 | ||||
-rw-r--r-- | morphlib/stagingarea.py | 24 | ||||
-rw-r--r-- | scripts/test-shell.c | 91 | ||||
-rw-r--r-- | yarns/building.yarn | 30 | ||||
-rw-r--r-- | yarns/implementations.yarn | 8 | ||||
-rw-r--r-- | yarns/splitting.yarn | 1 |
13 files changed, 229 insertions, 100 deletions
diff --git a/morphlib/app.py b/morphlib/app.py index 9ab102b3..c3c9c970 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -290,7 +290,8 @@ class Morph(cliapp.Application): morphlib.util.sanitise_morphology_path(args[2])) args = args[3:] - def create_source_pool(self, lrc, rrc, repo, ref, filename): + def create_source_pool(self, lrc, rrc, repo, ref, filename, + original_ref=None): pool = morphlib.sourcepool.SourcePool() def add_to_pool(reponame, ref, filename, absref, tree, morphology): @@ -302,7 +303,8 @@ class Morph(cliapp.Application): self.traverse_morphs(repo, ref, [filename], lrc, rrc, update=not self.settings['no-git-update'], - visit=add_to_pool) + visit=add_to_pool, + definitions_original_ref=original_ref) return pool def resolve_ref(self, lrc, rrc, reponame, ref, update=True): @@ -346,7 +348,8 @@ class Morph(cliapp.Application): def traverse_morphs(self, definitions_repo, definitions_ref, system_filenames, lrc, rrc, update=True, - visit=lambda rn, rf, fn, arf, m: None): + visit=lambda rn, rf, fn, arf, m: None, + definitions_original_ref=None): morph_factory = morphlib.morphologyfactory.MorphologyFactory(lrc, rrc, self) definitions_queue = collections.deque(system_filenames) @@ -359,6 +362,9 @@ class Morph(cliapp.Application): definitions_absref, definitions_tree = self.resolve_ref( lrc, rrc, definitions_repo, definitions_ref, update) + if definitions_original_ref: + definitions_ref = definitions_original_ref + while definitions_queue: filename = definitions_queue.popleft() diff --git a/morphlib/buildbranch.py b/morphlib/buildbranch.py index 638350e3..cfc4a67f 100644 --- a/morphlib/buildbranch.py +++ b/morphlib/buildbranch.py @@ -254,6 +254,10 @@ class BuildBranch(object): return self._sb.get_config('branch.name') @property + def root_commit(self): + return self._root.resolve_ref_to_commit(self.root_ref) + + @property def root_local_repo_url(self): return urlparse.urljoin('file://', self._root.dirname) @@ -291,7 +295,13 @@ def pushed_build_branch(bb, loader, changes_need_pushing, name, email, unpushed = any(bb.get_unpushed_branches()) if not changes_made and not unpushed: - yield bb.root_repo_url, bb.root_ref + # We resolve the system branch ref to the commit SHA1 here, so that + # the build uses whatever commit the user's copy of the root repo + # considers the head of that branch to be. If we returned a named + # ref, we risk building what the remote considers the head of that + # branch to be instead, and we also trigger a needless update in + # the cached copy of the root repo. + yield bb.root_repo_url, bb.root_commit, bb.root_ref return def report_inject(gd): @@ -318,6 +328,6 @@ def pushed_build_branch(bb, loader, changes_need_pushing, name, email, remote=remote.get_push_url(), chatty=True) bb.push_build_branches(push_cb=report_push) - yield bb.root_repo_url, bb.root_build_ref + yield bb.root_repo_url, bb.root_build_ref, bb.root_build_ref else: - yield bb.root_local_repo_url, bb.root_build_ref + yield bb.root_local_repo_url, bb.root_build_ref, bb.root_build_ref diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 2aec5e08..544d88d8 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -50,21 +50,24 @@ class BuildCommand(object): self.lac, self.rac = self.new_artifact_caches() self.lrc, self.rrc = self.new_repo_caches() - def build(self, args): - '''Build triplets specified on command line.''' + def build(self, repo_name, ref, filename, original_ref=None): + '''Build a given system morphology.''' - self.app.status(msg='Build starts', chatty=True) + self.app.status( + msg='Building %(repo_name)s %(ref)s %(filename)s', + repo_name=repo_name, ref=ref, filename=filename) - for repo_name, ref, filename in self.app.itertriplets(args): - self.app.status(msg='Building %(repo_name)s %(ref)s %(filename)s', - repo_name=repo_name, ref=ref, filename=filename) - self.app.status(msg='Deciding on task order') - srcpool = self.create_source_pool(repo_name, ref, filename) - self.validate_sources(srcpool) - root_artifact = self.resolve_artifacts(srcpool) - self.build_in_order(root_artifact) + self.app.status(msg='Deciding on task order') + srcpool = self.create_source_pool( + repo_name, ref, filename, original_ref) + self.validate_sources(srcpool) + root_artifact = self.resolve_artifacts(srcpool) + self.build_in_order(root_artifact) - self.app.status(msg='Build ends successfully') + self.app.status( + msg='Build of %(repo_name)s %(ref)s %(filename)s ended ' + 'successfully', + repo_name=repo_name, ref=ref, filename=filename) def new_artifact_caches(self): '''Create interfaces for the build artifact caches. @@ -82,7 +85,7 @@ class BuildCommand(object): return morphlib.buildenvironment.BuildEnvironment(self.app.settings, arch) - def create_source_pool(self, repo_name, ref, filename): + def create_source_pool(self, repo_name, ref, filename, original_ref=None): '''Find the source objects required for building a the given artifact The SourcePool will contain every stratum and chunk dependency of the @@ -92,7 +95,8 @@ class BuildCommand(object): ''' self.app.status(msg='Creating source pool', chatty=True) srcpool = self.app.create_source_pool( - self.lrc, self.rrc, repo_name, ref, filename) + self.lrc, self.rrc, repo_name, ref, filename, + original_ref=original_ref) return srcpool @@ -184,7 +188,7 @@ class BuildCommand(object): def _validate_cross_morphology_references(self, srcpool): '''Perform validation across all morphologies involved in the build''' - stratum_names = [] + stratum_names = {} for src in srcpool: kind = src.morphology['kind'] @@ -208,15 +212,11 @@ class BuildCommand(object): # and Ref specified. if src.morphology['kind'] == 'stratum': name = src.name - ref = src.sha1[:7] - self.app.status(msg='Stratum [%(name)s] version is %(ref)s', - name=name, ref=ref) if name in stratum_names: raise morphlib.Error( - "Conflicting versions of stratum '%s' appear in the " - "build. Check the contents of the system against the " - "build-depends of the strata." % name) - stratum_names.append(name) + "Multiple strata produce a '%s' artifact: %s and %s" % + (name, stratum_names[name].filename, src.filename)) + stratum_names[name] = src def _validate_cross_refs_for_system(self, src, srcpool): self._validate_cross_refs_for_xxx( diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 9cd3a074..8615ed59 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -28,6 +28,7 @@ import time import traceback import subprocess import tempfile +import textwrap import gzip import cliapp @@ -341,6 +342,7 @@ class ChunkBuilder(BuilderBase): relative_builddir = self.staging_area.relative(builddir) relative_destdir = self.staging_area.relative(destdir) + ccache_dir = self.staging_area.ccache_dir(self.source) extra_env = { 'DESTDIR': relative_destdir } steps = [ @@ -390,7 +392,8 @@ class ChunkBuilder(BuilderBase): cwd=relative_builddir, stdout=stdout or subprocess.PIPE, stderr=subprocess.STDOUT, - logfile=logfilepath) + logfile=logfilepath, + ccache_dir=ccache_dir) if stdout: stdout.flush() @@ -644,6 +647,53 @@ class SystemBuilder(BuilderBase): # pragma: no cover os.chmod(os_release_file, 0644) + def _chroot_runcmd(self, rootdir, to_mount, env, *args): + # We need to do mounts in a different namespace. Unfortunately + # this means we have to in-line the mount commands in the + # command-line. + command = textwrap.dedent(r''' + mount --make-rprivate / + rootdir="$1" + shift + ''') + cmdargs = [rootdir] + + # We need to mount all the specified mounts in the namespace, + # we don't need to unmount them before exiting, as they'll be + # unmounted when the namespace is no longer used. + command += textwrap.dedent(r''' + while true; do + case "$1" in + --) + shift + break + ;; + *) + mount_point="$1" + mount_type="$2" + mount_source="$3" + shift 3 + path="$rootdir/$mount_point" + mount -t "$mount_type" "$mount_source" "$path" + ;; + esac + done + ''') + for mount_opts in to_mount: + cmdargs.extend(mount_opts) + cmdargs.append('--') + + command += textwrap.dedent(r''' + exec chroot "$rootdir" "$@" + ''') + cmdargs.extend(args) + + # The single - is just a shell convention to fill $0 when using -c, + # since ordinarily $0 contains the program name. + cmdline = ['unshare', '--mount', '--', 'sh', '-ec', command, '-'] + cmdline.extend(cmdargs) + self.app.runcmd(cmdline, env=env) + def run_system_integration_commands(self, rootdir): # pragma: no cover ''' Run the system integration commands ''' @@ -655,44 +705,26 @@ class SystemBuilder(BuilderBase): # pragma: no cover 'PATH': '/bin:/usr/bin:/sbin:/usr/sbin' } - self.app.status(msg='Running the system integration commands', - error=True) + self.app.status(msg='Running the system integration commands') - mounted = [] to_mount = ( ('proc', 'proc', 'none'), ('dev/shm', 'tmpfs', 'none'), + ('tmp', 'tmpfs', 'none'), ) - try: for mount_point, mount_type, source in to_mount: - logging.debug('Mounting %s in system root filesystem' - % mount_point) path = os.path.join(rootdir, mount_point) if not os.path.exists(path): os.makedirs(path) - morphlib.fsutils.mount(self.app.runcmd, source, path, - mount_type) - mounted.append(path) - - # The single - is just a shell convention to fill $0 when using -c, - # since ordinarily $0 contains the program name. - # -- is used to indicate the end of options for run-parts, - # we don't want SYSTEM_INTEGRATION_PATH to be interpreted - # as an option if it happens to begin with a - - self.app.runcmd(['chroot', rootdir, 'sh', '-c', - 'cd / && run-parts -- "$1"', '-', SYSTEM_INTEGRATION_PATH], - env=env) + for bin in sorted(os.listdir(sys_integration_dir)): + self._chroot_runcmd(rootdir, to_mount, env, + os.path.join(SYSTEM_INTEGRATION_PATH, bin)) except BaseException, e: self.app.status( msg='Error while running system integration commands', error=True) raise - finally: - for mount_path in reversed(mounted): - logging.debug('Unmounting %s in system root filesystem' - % mount_path) - morphlib.fsutils.unmount(self.app.runcmd, mount_path) class Builder(object): # pragma: no cover diff --git a/morphlib/extensions.py b/morphlib/extensions.py index af6ba279..6b81e116 100644 --- a/morphlib/extensions.py +++ b/morphlib/extensions.py @@ -223,7 +223,11 @@ class ExtensionSubprocess(object): def close_read_end(): os.close(log_read_fd) p = subprocess.Popen( - [filename] + args, cwd=cwd, env=new_env, + # We unshare and mount --make-rprivate so mounts done by write + # extensions can't interfere with the rest of the system. + ['unshare', '-m', '--', '/bin/sh', '-c', + 'mount --make-rprivate / && exec "$@"', '-', filename] + args, + cwd=cwd, env=new_env, stdout=subprocess.PIPE, stderr=subprocess.PIPE, preexec_fn=close_read_end) os.close(log_write_fd) diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index d816fb90..5531f7f6 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -36,7 +36,7 @@ class BranchAndMergePlugin(cliapp.Plugin): self.app.add_subcommand( 'branch', self.branch, arg_synopsis='REPO NEW [OLD]') self.app.add_subcommand( - 'edit', self.edit, arg_synopsis='SYSTEM STRATUM [CHUNK]') + 'edit', self.edit, arg_synopsis='CHUNK') self.app.add_subcommand( 'show-system-branch', self.show_system_branch, arg_synopsis='') self.app.add_subcommand( diff --git a/morphlib/plugins/build_plugin.py b/morphlib/plugins/build_plugin.py index 64630c2b..218bd819 100644 --- a/morphlib/plugins/build_plugin.py +++ b/morphlib/plugins/build_plugin.py @@ -56,7 +56,8 @@ class BuildPlugin(cliapp.Plugin): build_command = morphlib.buildcommand.InitiatorBuildCommand( self.app, addr, port) - build_command.build(args) + for repo_name, ref, filename in self.app.itertriplets(args): + build_command.build(repo_name, ref, filename) def distbuild(self, args): '''Distbuild a system image in the current system branch @@ -116,7 +117,8 @@ class BuildPlugin(cliapp.Plugin): self.app.settings['cachedir-min-space']) build_command = morphlib.buildcommand.BuildCommand(self.app) - build_command.build(args) + for repo_name, ref, filename in self.app.itertriplets(args): + build_command.build(repo_name, ref, filename) def build(self, args): '''Build a system image in the current system branch @@ -189,5 +191,6 @@ class BuildPlugin(cliapp.Plugin): bb, loader=loader, changes_need_pushing=push, name=name, email=email, build_uuid=build_uuid, status=self.app.status) - with pbb as (repo, ref): - build_command.build([repo, ref, system_filename]) + with pbb as (repo, commit, original_ref): + build_command.build(repo, commit, system_filename, + original_ref=original_ref) diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 2bc53a0d..e795e637 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -328,14 +328,14 @@ class DeployPlugin(cliapp.Plugin): bb, loader=loader, changes_need_pushing=False, name=name, email=email, build_uuid=build_uuid, status=self.app.status) - with pbb as (repo, ref): + with pbb as (repo, commit, original_ref): # Create a tempdir for this deployment to work in deploy_tempdir = tempfile.mkdtemp( dir=os.path.join(self.app.settings['tempdir'], 'deployments')) try: for system in cluster_morphology['systems']: self.deploy_system(build_command, deploy_tempdir, - root_repo_dir, repo, ref, system, + root_repo_dir, repo, commit, system, env_vars, deployments, parent_location='') finally: diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index bfe0a716..25e33b3f 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -192,11 +192,10 @@ class StagingArea(object): shutil.rmtree(self.dirname) to_mount = ( - ('proc', 'proc', 'none'), ('dev/shm', 'tmpfs', 'none'), ) - def mount_ccachedir(self, source): #pragma: no cover + def ccache_dir(self, source): #pragma: no cover ccache_dir = self._app.settings['compiler-cache-dir'] if not os.path.isdir(ccache_dir): os.makedirs(ccache_dir) @@ -223,10 +222,7 @@ class StagingArea(object): # to avoid breaking when faced with an empty staging area. if not os.path.isdir(ccache_destdir): os.makedirs(ccache_destdir) - # Mount it into the staging-area - self._app.runcmd(['mount', '--bind', ccache_repodir, - ccache_destdir]) - return ccache_destdir + return ccache_repodir def do_mounts(self, setup_mounts): # pragma: no cover if not setup_mounts: @@ -257,9 +253,6 @@ class StagingArea(object): self.do_mounts(setup_mounts) - if not self._app.settings['no-ccache']: - self.mounted.append(self.mount_ccachedir(source)) - return builddir, destdir def chroot_close(self): # pragma: no cover @@ -284,6 +277,7 @@ class StagingArea(object): del kwargs['cwd'] else: cwd = '/' + ccache_dir = kwargs.pop('ccache_dir', None) chroot_dir = self.dirname if self.use_chroot else '/' temp_dir = kwargs["env"].get("TMPDIR", "/tmp") @@ -304,6 +298,18 @@ class StagingArea(object): if not os.path.islink(d): real_argv += ['--mount-readonly', self.relative(d)] + if self.use_chroot: + proc_target = os.path.join(self.dirname, 'proc') + if not os.path.exists(proc_target): + os.makedirs(proc_target) + real_argv += ['--mount-proc', self.relative(proc_target)] + + if ccache_dir and not self._app.settings['no-ccache']: + ccache_target = os.path.join( + self.dirname, kwargs['env']['CCACHE_DIR'].lstrip('/')) + real_argv += ['--mount-bind', ccache_dir, + self.relative(ccache_target)] + real_argv += [chroot_dir] real_argv += argv diff --git a/scripts/test-shell.c b/scripts/test-shell.c index e3ef1ff1..2f8ecbd8 100644 --- a/scripts/test-shell.c +++ b/scripts/test-shell.c @@ -107,44 +107,73 @@ cleanup: return ret; } -int main(int argc, char *argv[]) { +int copy_file_objects(FILE *source, FILE *target) { + char buffer[BUFSIZ]; + size_t read; + do { + read = fread(buffer, 1, sizeof(buffer), source); + fwrite(buffer, 1, read, target); + } while (!feof(source)); + return ferror(source) ? -1 : 0; +} + +int run_commands(FILE *cmdstream){ int ret = 1; - if (argc != 3 || strcmp(argv[1], "-c") != 0) { - fprintf(stderr, "Usage: %s -c COMMAND\n", argv[0]); - return 1; - } - size_t cmdlen = strlen(argv[2]); - FILE *cmdstream = fmemopen(argv[2], cmdlen, "r"); - { - ssize_t read; - size_t len = 0; - char *line = NULL; + ssize_t read; + size_t len = 0; + char *line = NULL; - ret = 0; - while ((read = getline(&line, &len, cmdstream)) != -1) { - if (line[read - 1] == '\n') line[read - 1] = '\0'; - if (strcmp(line, "copy files") == 0) { - /* Recursively copy contents of current dir to DESTDIR */ - if (nftw(".", copy_entry, 20, FTW_PHYS)) { - ret = 1; - break; - } - } else if (strcmp(line, "false") == 0 || - strstr(line, "false ") == line) { + ret = 0; + while ((read = getline(&line, &len, cmdstream)) != -1) { + if (line[read - 1] == '\n') line[read - 1] = '\0'; + if (strcmp(line, "copy files") == 0) { + /* Recursively copy contents of current dir to DESTDIR */ + if (nftw(".", copy_entry, 20, FTW_PHYS)) { ret = 1; break; - } else if (strstr(line, "echo ") == line) { - if (puts(line + sizeof("echo ") - 1) == EOF){ - perror("echo"); - ret = 1; - break; - } - } else { - ret = 127; + } + } else if (strcmp(line, "false") == 0 || + strstr(line, "false ") == line) { + ret = 1; + break; + } else if (strstr(line, "echo ") == line) { + if (puts(line + sizeof("echo ") - 1) == EOF){ + perror("echo"); + ret = 1; break; } + } else if (strstr(line, "create file ") == line) { + char const *filename = line + sizeof("create file ") -1; + FILE *outfile = fopen(filename, "w"); + if (copy_file_objects(cmdstream, outfile) < 0) { + ret = 1; + fclose(outfile); + break; + } + fclose(outfile); + } else if (line[0] == '#' || strstr(line, "set ") == line) { + /* Comments and set commands are ignored */ + continue; + } else { + fprintf(stderr, "Unrecognized command: %s\n", line); + ret = 127; + break; } - free(line); } + free(line); return ret; } + +int main(int argc, char *argv[]) { + if (argc == 3 && strcmp(argv[1], "-c") == 0) { + size_t cmdlen = strlen(argv[2]); + FILE *cmdstream = fmemopen(argv[2], cmdlen, "r"); + return run_commands(cmdstream); + } else if (argc == 2) { + FILE *cmdstream = fopen(argv[1], "r"); + return run_commands(cmdstream); + } else { + fprintf(stderr, "Usage: %s -c COMMAND|%s SCRIPT\n", argv[0], argv[0]); + return 1; + } +} diff --git a/yarns/building.yarn b/yarns/building.yarn index 52742ac8..253b3b3c 100644 --- a/yarns/building.yarn +++ b/yarns/building.yarn @@ -9,6 +9,36 @@ Morph Building Tests THEN morph build the system systems/base-system.morph of the branch master FINALLY the git server is shut down +System integrations +------------------- + +`system-integration` is a field in chunk morphologies that allows you to +have some scripts run at system artifact construction time, because some +things need to be done after every chunk is built, such as `ldconfig`, +so every library path in `/etc/ld.so.conf` can be found, and it can look +up libraries more quickly. + + SCENARIO using system integrations + GIVEN a workspace + AND a git server + WHEN the user checks out the system branch called master + AND the user attempts to build the system systems/test-system.morph in branch master + THEN morph succeeded + +In our example, we have a system integration that creates /etc/passwd, +so when we deploy the system, we can check whether it exists. + + GIVEN a cluster called test-cluster.morph in system branch master + AND a system in cluster test-cluster.morph in branch master called test-system + AND system test-system in cluster test-cluster.morph in branch master builds systems/test-system.morph + AND system test-system in cluster test-cluster.morph in branch master has deployment type: tar + WHEN the user attempts to deploy the cluster test-cluster.morph in branch master with options test-system.location="$DATADIR/test.tar" + THEN morph succeeded + AND tarball test.tar contains etc/passwd + +Distbuilding +------------ + SCENARIO distbuilding ASSUMING the morph-cache-server can be run GIVEN a workspace diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index c6d245d0..6110148e 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -300,6 +300,14 @@ another to hold a chunk. install-commands: - copy files + system-integration: + test-chunk-bins: + 00-passwd: + - | + create file /etc/passwd + root:x:0:0:Super user:/root:/bin/sh + daemon:x:1:1:daemon:/usr/sbin:/bin/sh + nobody:x:65534:65534:nobody:/nonexistent:/bin/false EOF install -m644 -D /dev/stdin << 'EOF' "stage2-chunk.morph" diff --git a/yarns/splitting.yarn b/yarns/splitting.yarn index 2726d294..75831e53 100644 --- a/yarns/splitting.yarn +++ b/yarns/splitting.yarn @@ -66,6 +66,7 @@ we've successfully excluded it, we won't have those files. AND tarball test.tar doesn't contain lib/libtest.a AND tarball test.tar doesn't contain man/man3/test.3.gz + FINALLY the git server is shut down As a consequence of how dependencies are generated, if we select strata to go into our system, such that there are chunk artifacts that are not |