From 694a481360edeb0146c02362385af59f0371eb97 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 22 Oct 2014 18:46:20 +0000 Subject: test-shell: add create file command --- scripts/test-shell.c | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/scripts/test-shell.c b/scripts/test-shell.c index e3ef1ff1..d1a10e5b 100644 --- a/scripts/test-shell.c +++ b/scripts/test-shell.c @@ -107,6 +107,17 @@ cleanup: return ret; } +int copy_file_objects(FILE *source, FILE *target) { + char buffer[BUFSIZ]; + size_t read; + do { + read = fread(buffer, 1, sizeof(buffer), source); + fprintf(stderr, "Read: %*s\n", read, buffer); + fwrite(buffer, 1, read, target); + } while (!feof(source)); + return ferror(source) ? -1 : 0; +} + int main(int argc, char *argv[]) { int ret = 1; if (argc != 3 || strcmp(argv[1], "-c") != 0) { @@ -139,6 +150,15 @@ int main(int argc, char *argv[]) { ret = 1; break; } + } else if (strstr(line, "create file ") == line) { + char const *filename = line + sizeof("create file ") -1; + FILE *outfile = fopen(filename, "w"); + if (copy_file_objects(cmdstream, outfile) < 0) { + ret = 1; + fclose(outfile); + break; + } + fclose(outfile); } else { ret = 127; break; -- cgit v1.2.1 From a54e12e1ab19c3e952d9c76b3aed59738a4af98c Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 22 Oct 2014 19:03:40 +0000 Subject: test-shell: Allow to be used in system-integrations It now works when passed a file, rather than a command to run, and ignores comments and set commands, so the generated preamble is accepted. --- scripts/test-shell.c | 88 +++++++++++++++++++++++++++++----------------------- 1 file changed, 49 insertions(+), 39 deletions(-) diff --git a/scripts/test-shell.c b/scripts/test-shell.c index d1a10e5b..7e600388 100644 --- a/scripts/test-shell.c +++ b/scripts/test-shell.c @@ -118,53 +118,63 @@ int copy_file_objects(FILE *source, FILE *target) { return ferror(source) ? -1 : 0; } -int main(int argc, char *argv[]) { +int run_commands(FILE *cmdstream){ int ret = 1; - if (argc != 3 || strcmp(argv[1], "-c") != 0) { - fprintf(stderr, "Usage: %s -c COMMAND\n", argv[0]); - return 1; - } - size_t cmdlen = strlen(argv[2]); - FILE *cmdstream = fmemopen(argv[2], cmdlen, "r"); - { - ssize_t read; - size_t len = 0; - char *line = NULL; + ssize_t read; + size_t len = 0; + char *line = NULL; - ret = 0; - while ((read = getline(&line, &len, cmdstream)) != -1) { - if (line[read - 1] == '\n') line[read - 1] = '\0'; - if (strcmp(line, "copy files") == 0) { - /* Recursively copy contents of current dir to DESTDIR */ - if (nftw(".", copy_entry, 20, FTW_PHYS)) { - ret = 1; - break; - } - } else if (strcmp(line, "false") == 0 || - strstr(line, "false ") == line) { + ret = 0; + while ((read = getline(&line, &len, cmdstream)) != -1) { + if (line[read - 1] == '\n') line[read - 1] = '\0'; + if (strcmp(line, "copy files") == 0) { + /* Recursively copy contents of current dir to DESTDIR */ + if (nftw(".", copy_entry, 20, FTW_PHYS)) { ret = 1; break; - } else if (strstr(line, "echo ") == line) { - if (puts(line + sizeof("echo ") - 1) == EOF){ - perror("echo"); - ret = 1; - break; - } - } else if (strstr(line, "create file ") == line) { - char const *filename = line + sizeof("create file ") -1; - FILE *outfile = fopen(filename, "w"); - if (copy_file_objects(cmdstream, outfile) < 0) { - ret = 1; - fclose(outfile); - break; - } + } + } else if (strcmp(line, "false") == 0 || + strstr(line, "false ") == line) { + ret = 1; + break; + } else if (strstr(line, "echo ") == line) { + if (puts(line + sizeof("echo ") - 1) == EOF){ + perror("echo"); + ret = 1; + break; + } + } else if (strstr(line, "create file ") == line) { + char const *filename = line + sizeof("create file ") -1; + FILE *outfile = fopen(filename, "w"); + if (copy_file_objects(cmdstream, outfile) < 0) { + ret = 1; fclose(outfile); - } else { - ret = 127; break; } + fclose(outfile); + } else if (line[0] == '#' || strstr(line, "set ") == line) { + /* Comments and set commands are ignored */ + continue; + } else { + fprintf(stderr, "Unrecognized command: %s\n", line); + ret = 127; + break; } - free(line); } + free(line); return ret; } + +int main(int argc, char *argv[]) { + if (argc == 3 && strcmp(argv[1], "-c") == 0) { + size_t cmdlen = strlen(argv[2]); + FILE *cmdstream = fmemopen(argv[2], cmdlen, "r"); + return run_commands(cmdstream); + } else if (argc == 2) { + FILE *cmdstream = fopen(argv[1], "r"); + return run_commands(cmdstream); + } else { + fprintf(stderr, "Usage: %s -c COMMAND|%s SCRIPT\n", argv[0], argv[0]); + return 1; + } +} -- cgit v1.2.1 From 94676362e1fc7e2b0ca500a1d0da16ae5c2fc300 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 22 Oct 2014 19:05:33 +0000 Subject: yarns: Fix missing FINALLY --- yarns/splitting.yarn | 1 + 1 file changed, 1 insertion(+) diff --git a/yarns/splitting.yarn b/yarns/splitting.yarn index 2726d294..75831e53 100644 --- a/yarns/splitting.yarn +++ b/yarns/splitting.yarn @@ -66,6 +66,7 @@ we've successfully excluded it, we won't have those files. AND tarball test.tar doesn't contain lib/libtest.a AND tarball test.tar doesn't contain man/man3/test.3.gz + FINALLY the git server is shut down As a consequence of how dependencies are generated, if we select strata to go into our system, such that there are chunk artifacts that are not -- cgit v1.2.1 From 7e027adeb6b40c02c04c78c5a6957c7d79bba75c Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 22 Oct 2014 19:06:17 +0000 Subject: Run system-integrations in a namespace All temporary mounts should be done inside a namespace, so that they don't interfere with other namespaces. system-integrations may need capabilities that regular builds don't have, so they're chrooted, rather than linux-user-chrooted, which means it's more complicated to do namespaces. In the absence of a better command for it, we can do this with an in-lined shell script. This also stops us using the run-parts inside the system, and executes the integrations directly. --- morphlib/builder2.py | 76 ++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 53 insertions(+), 23 deletions(-) diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 9cd3a074..596cd645 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -28,6 +28,7 @@ import time import traceback import subprocess import tempfile +import textwrap import gzip import cliapp @@ -644,6 +645,53 @@ class SystemBuilder(BuilderBase): # pragma: no cover os.chmod(os_release_file, 0644) + def _chroot_runcmd(self, rootdir, to_mount, env, *args): + # We need to do mounts in a different namespace. Unfortunately + # this means we have to in-line the mount commands in the + # command-line. + command = textwrap.dedent(r''' + mount --make-rprivate / + rootdir="$1" + shift + ''') + cmdargs = [rootdir] + + # We need to mount all the specified mounts in the namespace, + # we don't need to unmount them before exiting, as they'll be + # unmounted when the namespace is no longer used. + command += textwrap.dedent(r''' + while true; do + case "$1" in + --) + shift + break + ;; + *) + mount_point="$1" + mount_type="$2" + mount_source="$3" + shift 3 + path="$rootdir/$mount_point" + mount -t "$mount_type" "$mount_source" "$path" + ;; + esac + done + ''') + for mount_opts in to_mount: + cmdargs.extend(mount_opts) + cmdargs.append('--') + + command += textwrap.dedent(r''' + exec chroot "$rootdir" "$@" + ''') + cmdargs.extend(args) + + # The single - is just a shell convention to fill $0 when using -c, + # since ordinarily $0 contains the program name. + cmdline = ['unshare', '--mount', '--', 'sh', '-ec', command, '-'] + cmdline.extend(cmdargs) + self.app.runcmd(cmdline, env=env) + def run_system_integration_commands(self, rootdir): # pragma: no cover ''' Run the system integration commands ''' @@ -655,44 +703,26 @@ class SystemBuilder(BuilderBase): # pragma: no cover 'PATH': '/bin:/usr/bin:/sbin:/usr/sbin' } - self.app.status(msg='Running the system integration commands', - error=True) + self.app.status(msg='Running the system integration commands') - mounted = [] to_mount = ( ('proc', 'proc', 'none'), ('dev/shm', 'tmpfs', 'none'), + ('tmp', 'tmpfs', 'none'), ) - try: for mount_point, mount_type, source in to_mount: - logging.debug('Mounting %s in system root filesystem' - % mount_point) path = os.path.join(rootdir, mount_point) if not os.path.exists(path): os.makedirs(path) - morphlib.fsutils.mount(self.app.runcmd, source, path, - mount_type) - mounted.append(path) - - # The single - is just a shell convention to fill $0 when using -c, - # since ordinarily $0 contains the program name. - # -- is used to indicate the end of options for run-parts, - # we don't want SYSTEM_INTEGRATION_PATH to be interpreted - # as an option if it happens to begin with a - - self.app.runcmd(['chroot', rootdir, 'sh', '-c', - 'cd / && run-parts -- "$1"', '-', SYSTEM_INTEGRATION_PATH], - env=env) + for bin in sorted(os.listdir(sys_integration_dir)): + self._chroot_runcmd(rootdir, to_mount, env, + os.path.join(SYSTEM_INTEGRATION_PATH, bin)) except BaseException, e: self.app.status( msg='Error while running system integration commands', error=True) raise - finally: - for mount_path in reversed(mounted): - logging.debug('Unmounting %s in system root filesystem' - % mount_path) - morphlib.fsutils.unmount(self.app.runcmd, mount_path) class Builder(object): # pragma: no cover -- cgit v1.2.1 From 8b37ecabaa5cb30c55f6ea2492d1e0ac3f612748 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 22 Oct 2014 19:13:20 +0000 Subject: yarns: Add yarns for system-integrations These were previously completely untested, so their unsafe mounting wasn't noticed, but now both are fixed. --- yarns/building.yarn | 30 ++++++++++++++++++++++++++++++ yarns/implementations.yarn | 8 ++++++++ 2 files changed, 38 insertions(+) diff --git a/yarns/building.yarn b/yarns/building.yarn index 52742ac8..253b3b3c 100644 --- a/yarns/building.yarn +++ b/yarns/building.yarn @@ -9,6 +9,36 @@ Morph Building Tests THEN morph build the system systems/base-system.morph of the branch master FINALLY the git server is shut down +System integrations +------------------- + +`system-integration` is a field in chunk morphologies that allows you to +have some scripts run at system artifact construction time, because some +things need to be done after every chunk is built, such as `ldconfig`, +so every library path in `/etc/ld.so.conf` can be found, and it can look +up libraries more quickly. + + SCENARIO using system integrations + GIVEN a workspace + AND a git server + WHEN the user checks out the system branch called master + AND the user attempts to build the system systems/test-system.morph in branch master + THEN morph succeeded + +In our example, we have a system integration that creates /etc/passwd, +so when we deploy the system, we can check whether it exists. + + GIVEN a cluster called test-cluster.morph in system branch master + AND a system in cluster test-cluster.morph in branch master called test-system + AND system test-system in cluster test-cluster.morph in branch master builds systems/test-system.morph + AND system test-system in cluster test-cluster.morph in branch master has deployment type: tar + WHEN the user attempts to deploy the cluster test-cluster.morph in branch master with options test-system.location="$DATADIR/test.tar" + THEN morph succeeded + AND tarball test.tar contains etc/passwd + +Distbuilding +------------ + SCENARIO distbuilding ASSUMING the morph-cache-server can be run GIVEN a workspace diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index c6d245d0..6110148e 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -300,6 +300,14 @@ another to hold a chunk. install-commands: - copy files + system-integration: + test-chunk-bins: + 00-passwd: + - | + create file /etc/passwd + root:x:0:0:Super user:/root:/bin/sh + daemon:x:1:1:daemon:/usr/sbin:/bin/sh + nobody:x:65534:65534:nobody:/nonexistent:/bin/false EOF install -m644 -D /dev/stdin << 'EOF' "stage2-chunk.morph" -- cgit v1.2.1