summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRichard Maw <richard.maw@codethink.co.uk>2014-10-24 14:27:55 +0000
committerRichard Maw <richard.maw@codethink.co.uk>2014-10-24 14:27:55 +0000
commit0ed18258e2fa760eae780fee741bb95eeb467bc4 (patch)
tree1b15ed4f440e0b9cd8bfc30ef53b43778d85d6a5
parent8036c2ddfdbf9f6b35ea76244b98a10b1d76c55b (diff)
parent8b37ecabaa5cb30c55f6ea2492d1e0ac3f612748 (diff)
downloadmorph-0ed18258e2fa760eae780fee741bb95eeb467bc4.tar.gz
Merge branch 'baserock/richardmaw/test-system-integrations'
Reviewed-by: Sam Thursfield Reviewed-by: Pedro Alvarez Reviewed-by: Daniel Silverstone
-rw-r--r--morphlib/builder2.py76
-rw-r--r--scripts/test-shell.c91
-rw-r--r--yarns/building.yarn30
-rw-r--r--yarns/implementations.yarn8
-rw-r--r--yarns/splitting.yarn1
5 files changed, 152 insertions, 54 deletions
diff --git a/morphlib/builder2.py b/morphlib/builder2.py
index 9cd3a074..596cd645 100644
--- a/morphlib/builder2.py
+++ b/morphlib/builder2.py
@@ -28,6 +28,7 @@ import time
import traceback
import subprocess
import tempfile
+import textwrap
import gzip
import cliapp
@@ -644,6 +645,53 @@ class SystemBuilder(BuilderBase): # pragma: no cover
os.chmod(os_release_file, 0644)
+ def _chroot_runcmd(self, rootdir, to_mount, env, *args):
+ # We need to do mounts in a different namespace. Unfortunately
+ # this means we have to in-line the mount commands in the
+ # command-line.
+ command = textwrap.dedent(r'''
+ mount --make-rprivate /
+ rootdir="$1"
+ shift
+ ''')
+ cmdargs = [rootdir]
+
+ # We need to mount all the specified mounts in the namespace,
+ # we don't need to unmount them before exiting, as they'll be
+ # unmounted when the namespace is no longer used.
+ command += textwrap.dedent(r'''
+ while true; do
+ case "$1" in
+ --)
+ shift
+ break
+ ;;
+ *)
+ mount_point="$1"
+ mount_type="$2"
+ mount_source="$3"
+ shift 3
+ path="$rootdir/$mount_point"
+ mount -t "$mount_type" "$mount_source" "$path"
+ ;;
+ esac
+ done
+ ''')
+ for mount_opts in to_mount:
+ cmdargs.extend(mount_opts)
+ cmdargs.append('--')
+
+ command += textwrap.dedent(r'''
+ exec chroot "$rootdir" "$@"
+ ''')
+ cmdargs.extend(args)
+
+ # The single - is just a shell convention to fill $0 when using -c,
+ # since ordinarily $0 contains the program name.
+ cmdline = ['unshare', '--mount', '--', 'sh', '-ec', command, '-']
+ cmdline.extend(cmdargs)
+ self.app.runcmd(cmdline, env=env)
+
def run_system_integration_commands(self, rootdir): # pragma: no cover
''' Run the system integration commands '''
@@ -655,44 +703,26 @@ class SystemBuilder(BuilderBase): # pragma: no cover
'PATH': '/bin:/usr/bin:/sbin:/usr/sbin'
}
- self.app.status(msg='Running the system integration commands',
- error=True)
+ self.app.status(msg='Running the system integration commands')
- mounted = []
to_mount = (
('proc', 'proc', 'none'),
('dev/shm', 'tmpfs', 'none'),
+ ('tmp', 'tmpfs', 'none'),
)
-
try:
for mount_point, mount_type, source in to_mount:
- logging.debug('Mounting %s in system root filesystem'
- % mount_point)
path = os.path.join(rootdir, mount_point)
if not os.path.exists(path):
os.makedirs(path)
- morphlib.fsutils.mount(self.app.runcmd, source, path,
- mount_type)
- mounted.append(path)
-
- # The single - is just a shell convention to fill $0 when using -c,
- # since ordinarily $0 contains the program name.
- # -- is used to indicate the end of options for run-parts,
- # we don't want SYSTEM_INTEGRATION_PATH to be interpreted
- # as an option if it happens to begin with a -
- self.app.runcmd(['chroot', rootdir, 'sh', '-c',
- 'cd / && run-parts -- "$1"', '-', SYSTEM_INTEGRATION_PATH],
- env=env)
+ for bin in sorted(os.listdir(sys_integration_dir)):
+ self._chroot_runcmd(rootdir, to_mount, env,
+ os.path.join(SYSTEM_INTEGRATION_PATH, bin))
except BaseException, e:
self.app.status(
msg='Error while running system integration commands',
error=True)
raise
- finally:
- for mount_path in reversed(mounted):
- logging.debug('Unmounting %s in system root filesystem'
- % mount_path)
- morphlib.fsutils.unmount(self.app.runcmd, mount_path)
class Builder(object): # pragma: no cover
diff --git a/scripts/test-shell.c b/scripts/test-shell.c
index e3ef1ff1..2f8ecbd8 100644
--- a/scripts/test-shell.c
+++ b/scripts/test-shell.c
@@ -107,44 +107,73 @@ cleanup:
return ret;
}
-int main(int argc, char *argv[]) {
+int copy_file_objects(FILE *source, FILE *target) {
+ char buffer[BUFSIZ];
+ size_t read;
+ do {
+ read = fread(buffer, 1, sizeof(buffer), source);
+ fwrite(buffer, 1, read, target);
+ } while (!feof(source));
+ return ferror(source) ? -1 : 0;
+}
+
+int run_commands(FILE *cmdstream){
int ret = 1;
- if (argc != 3 || strcmp(argv[1], "-c") != 0) {
- fprintf(stderr, "Usage: %s -c COMMAND\n", argv[0]);
- return 1;
- }
- size_t cmdlen = strlen(argv[2]);
- FILE *cmdstream = fmemopen(argv[2], cmdlen, "r");
- {
- ssize_t read;
- size_t len = 0;
- char *line = NULL;
+ ssize_t read;
+ size_t len = 0;
+ char *line = NULL;
- ret = 0;
- while ((read = getline(&line, &len, cmdstream)) != -1) {
- if (line[read - 1] == '\n') line[read - 1] = '\0';
- if (strcmp(line, "copy files") == 0) {
- /* Recursively copy contents of current dir to DESTDIR */
- if (nftw(".", copy_entry, 20, FTW_PHYS)) {
- ret = 1;
- break;
- }
- } else if (strcmp(line, "false") == 0 ||
- strstr(line, "false ") == line) {
+ ret = 0;
+ while ((read = getline(&line, &len, cmdstream)) != -1) {
+ if (line[read - 1] == '\n') line[read - 1] = '\0';
+ if (strcmp(line, "copy files") == 0) {
+ /* Recursively copy contents of current dir to DESTDIR */
+ if (nftw(".", copy_entry, 20, FTW_PHYS)) {
ret = 1;
break;
- } else if (strstr(line, "echo ") == line) {
- if (puts(line + sizeof("echo ") - 1) == EOF){
- perror("echo");
- ret = 1;
- break;
- }
- } else {
- ret = 127;
+ }
+ } else if (strcmp(line, "false") == 0 ||
+ strstr(line, "false ") == line) {
+ ret = 1;
+ break;
+ } else if (strstr(line, "echo ") == line) {
+ if (puts(line + sizeof("echo ") - 1) == EOF){
+ perror("echo");
+ ret = 1;
break;
}
+ } else if (strstr(line, "create file ") == line) {
+ char const *filename = line + sizeof("create file ") -1;
+ FILE *outfile = fopen(filename, "w");
+ if (copy_file_objects(cmdstream, outfile) < 0) {
+ ret = 1;
+ fclose(outfile);
+ break;
+ }
+ fclose(outfile);
+ } else if (line[0] == '#' || strstr(line, "set ") == line) {
+ /* Comments and set commands are ignored */
+ continue;
+ } else {
+ fprintf(stderr, "Unrecognized command: %s\n", line);
+ ret = 127;
+ break;
}
- free(line);
}
+ free(line);
return ret;
}
+
+int main(int argc, char *argv[]) {
+ if (argc == 3 && strcmp(argv[1], "-c") == 0) {
+ size_t cmdlen = strlen(argv[2]);
+ FILE *cmdstream = fmemopen(argv[2], cmdlen, "r");
+ return run_commands(cmdstream);
+ } else if (argc == 2) {
+ FILE *cmdstream = fopen(argv[1], "r");
+ return run_commands(cmdstream);
+ } else {
+ fprintf(stderr, "Usage: %s -c COMMAND|%s SCRIPT\n", argv[0], argv[0]);
+ return 1;
+ }
+}
diff --git a/yarns/building.yarn b/yarns/building.yarn
index 52742ac8..253b3b3c 100644
--- a/yarns/building.yarn
+++ b/yarns/building.yarn
@@ -9,6 +9,36 @@ Morph Building Tests
THEN morph build the system systems/base-system.morph of the branch master
FINALLY the git server is shut down
+System integrations
+-------------------
+
+`system-integration` is a field in chunk morphologies that allows you to
+have some scripts run at system artifact construction time, because some
+things need to be done after every chunk is built, such as `ldconfig`,
+so every library path in `/etc/ld.so.conf` can be found, and it can look
+up libraries more quickly.
+
+ SCENARIO using system integrations
+ GIVEN a workspace
+ AND a git server
+ WHEN the user checks out the system branch called master
+ AND the user attempts to build the system systems/test-system.morph in branch master
+ THEN morph succeeded
+
+In our example, we have a system integration that creates /etc/passwd,
+so when we deploy the system, we can check whether it exists.
+
+ GIVEN a cluster called test-cluster.morph in system branch master
+ AND a system in cluster test-cluster.morph in branch master called test-system
+ AND system test-system in cluster test-cluster.morph in branch master builds systems/test-system.morph
+ AND system test-system in cluster test-cluster.morph in branch master has deployment type: tar
+ WHEN the user attempts to deploy the cluster test-cluster.morph in branch master with options test-system.location="$DATADIR/test.tar"
+ THEN morph succeeded
+ AND tarball test.tar contains etc/passwd
+
+Distbuilding
+------------
+
SCENARIO distbuilding
ASSUMING the morph-cache-server can be run
GIVEN a workspace
diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn
index c6d245d0..6110148e 100644
--- a/yarns/implementations.yarn
+++ b/yarns/implementations.yarn
@@ -300,6 +300,14 @@ another to hold a chunk.
install-commands:
- copy files
+ system-integration:
+ test-chunk-bins:
+ 00-passwd:
+ - |
+ create file /etc/passwd
+ root:x:0:0:Super user:/root:/bin/sh
+ daemon:x:1:1:daemon:/usr/sbin:/bin/sh
+ nobody:x:65534:65534:nobody:/nonexistent:/bin/false
EOF
install -m644 -D /dev/stdin << 'EOF' "stage2-chunk.morph"
diff --git a/yarns/splitting.yarn b/yarns/splitting.yarn
index 2726d294..75831e53 100644
--- a/yarns/splitting.yarn
+++ b/yarns/splitting.yarn
@@ -66,6 +66,7 @@ we've successfully excluded it, we won't have those files.
AND tarball test.tar doesn't contain lib/libtest.a
AND tarball test.tar doesn't contain man/man3/test.3.gz
+ FINALLY the git server is shut down
As a consequence of how dependencies are generated, if we select strata
to go into our system, such that there are chunk artifacts that are not