From 501a039172d066d0b0ea9071c5d82467e89de9a6 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 12 Nov 2014 12:54:56 +0000 Subject: test-shell: Report write failures --- scripts/test-shell.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/test-shell.c b/scripts/test-shell.c index 2f8ecbd8..f4ac6bfa 100644 --- a/scripts/test-shell.c +++ b/scripts/test-shell.c @@ -10,6 +10,7 @@ #include #include #include +#include char *readlinka(char const *path){ size_t buflen = BUFSIZ; @@ -113,8 +114,8 @@ int copy_file_objects(FILE *source, FILE *target) { do { read = fread(buffer, 1, sizeof(buffer), source); fwrite(buffer, 1, read, target); - } while (!feof(source)); - return ferror(source) ? -1 : 0; + } while (!feof(source) && !feof(target)); + return (ferror(source) || ferror(target)) ? -1 : 0; } int run_commands(FILE *cmdstream){ @@ -145,8 +146,14 @@ int run_commands(FILE *cmdstream){ } else if (strstr(line, "create file ") == line) { char const *filename = line + sizeof("create file ") -1; FILE *outfile = fopen(filename, "w"); + if (outfile == NULL){ + ret = 1; + err(errno, "Opening %s for write failed", filename); + break; + } if (copy_file_objects(cmdstream, outfile) < 0) { ret = 1; + err(errno, "Writing to %s failed", filename); fclose(outfile); break; } -- cgit v1.2.1 From e82909361a50f5046295a1453f48735752318696 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 12 Nov 2014 12:56:28 +0000 Subject: yarns: Report stderr on morph failure --- yarns/implementations.yarn | 3 ++- yarns/morph.shell-lib | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index 6110148e..e1ae271f 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -34,7 +34,8 @@ we can test it later in a THEN step. case $(cat "$DATADIR/morph-exit") in 0) echo "Morph succeeded!" ;; - *) die "Morph should have succeeded, but didn't. Unexpected failure!" + *) cat "$DATADIR/result-latest" >&2 + die "Morph should have succeeded, but didn't. Unexpected failure!" ;; esac diff --git a/yarns/morph.shell-lib b/yarns/morph.shell-lib index 9c13e449..d9c9eff6 100644 --- a/yarns/morph.shell-lib +++ b/yarns/morph.shell-lib @@ -39,9 +39,13 @@ run_morph() set +e "$SRCDIR"/morph --verbose \ --cachedir-min-space=0 --tempdir-min-space=0 \ - --no-default-config --config "$DATADIR/morph.conf" "$@" \ - 2> "$DATADIR/result-$1" > "$DATADIR/out-$1" + --no-default-config --config "$DATADIR/morph.conf" \ + --log="$DATADIR/log-$1" \ + "$@" 2> "$DATADIR/result-$1" > "$DATADIR/out-$1" local exit_code="$?" + for o in log result out; do + ln -sf "$o-$1" "$DATADIR/$o-latest" + done cat "$DATADIR/out-$1" cat "$DATADIR/result-$1" >&2 return "$exit_code" -- cgit v1.2.1 From 824b1600d41d8906e87fa98cc78d776e406c19e5 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 12 Nov 2014 14:14:46 +0000 Subject: Make invert_paths work more reliably for writable-all --- morphlib/fsutils.py | 67 ++++++++++++++++++++++------------------------- morphlib/fsutils_tests.py | 25 ++++++++++++++---- 2 files changed, 51 insertions(+), 41 deletions(-) diff --git a/morphlib/fsutils.py b/morphlib/fsutils.py index 6d651171..8a4128d9 100644 --- a/morphlib/fsutils.py +++ b/morphlib/fsutils.py @@ -97,48 +97,43 @@ def invert_paths(tree_walker, paths): ''' - def is_subpath(prefix, path): - prefix_components = prefix.split(os.sep) - path_components = path.split(os.sep) - return path_components[:len(prefix_components)] == prefix_components + def normpath(path): + if path == '.': + return path + path = os.path.normpath(path) + if not os.path.isabs(path): + path = os.path.join('.', path) + return path + def any_paths_are_subpath_of(prefix): + prefix = normpath(prefix) + norm_paths = (normpath(path) for path in paths) + return any(path[:len(prefix)] == prefix + for path in norm_paths) + + def path_is_listed(path): + return any(normpath(path) == normpath(other) + for other in paths) for dirpath, dirnames, filenames in tree_walker: - if any(p == dirpath for p in paths): # pragma: no cover - # Dir is an exact match for a path - # don't recurse any further - # Don't yield it, since we don't return listed paths - continue - dn_copy = list(dirnames) - for subdir in dn_copy: - subdirpath = os.path.join(dirpath, subdir) - - if any(p == subdirpath for p in paths): - # Subdir is an exact match for a path - # Don't recurse into it, so remove from list - # Don't yield it, since we don't return listed paths - dirnames.remove(subdir) - elif any(is_subpath(subdirpath, p) for p in paths): - # This directory is a parent directory of one - # of our paths - # Recurse into it, so don't remove it from the list - # Don't yield it, since we don't return listed paths - pass - else: - # This directory is neither one marked for writing, - # nor a parent of a file marked for writing - # Don't recurse, so remove it from the list - # Yield it, since we return listed paths - dirnames.remove(subdir) - yield subdirpath + if path_is_listed(dirpath): + # No subpaths need to be considered + del dirnames[:] + del filenames[:] + elif any_paths_are_subpath_of(dirpath): + # Subpaths may be marked, or may not, need to leave this + # writable, so don't yield, but we don't cull. + pass + else: + # not listed as a parent or an exact match, needs to be + # yielded, but we don't need to consider subdirs, so can cull + yield dirpath + del dirnames[:] + del filenames[:] for filename in filenames: fullpath = os.path.join(dirpath, filename) - if any(is_subpath(p, fullpath) for p in paths): - # The file path is a child of one of the paths - # or is equal. - # Don't yield because either it is one of the specified - # paths, or is a file in a directory specified by a path + if path_is_listed(fullpath): pass else: yield fullpath diff --git a/morphlib/fsutils_tests.py b/morphlib/fsutils_tests.py index 7b159665..47a4488e 100644 --- a/morphlib/fsutils_tests.py +++ b/morphlib/fsutils_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013, 2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -51,11 +51,26 @@ class InvertPathsTests(unittest.TestCase): }, } - def test_flat_lists_single_files(self): + def test_flat_lists_top_dir(self): walker = dummy_top_down_walker('.', self.flat_tree) - self.assertEqual(sorted(["./foo", "./bar", "./baz"]), + self.assertEqual(["."], sorted(morphlib.fsutils.invert_paths(walker, []))) + def test_flat_skips_all_with_root_pased(self): + walker = dummy_top_down_walker('.', self.flat_tree) + self.assertEqual([], + list(morphlib.fsutils.invert_paths(walker, ['.']))) + + def test_flat_lists_top_dir(self): + walker = dummy_top_down_walker('.', self.nested_tree) + self.assertEqual(["."], + sorted(morphlib.fsutils.invert_paths(walker, []))) + + def test_flat_skips_all_with_root_pased(self): + walker = dummy_top_down_walker('.', self.nested_tree) + self.assertEqual([], + list(morphlib.fsutils.invert_paths(walker, ['.']))) + def test_flat_excludes_listed_files(self): walker = dummy_top_down_walker('.', self.flat_tree) self.assertTrue( @@ -95,5 +110,5 @@ class InvertPathsTests(unittest.TestCase): "./tmp/morph/staging/inst", "./tmp", ])) - expected = ("./bin",) - self.assertEqual(sorted(found), sorted(expected)) + expected = ["./bin"] + self.assertEqual(sorted(found), expected) -- cgit v1.2.1 From 40424d89f62ce6f268c4920a050b1f42ed2f5ac2 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 12 Nov 2014 12:35:35 +0000 Subject: Unrevert "Make system-integration commands use containerised_cmdline" The bug in the generated command lines has been fixed, so we can have this cleanup back now. --- morphlib/builder2.py | 60 +++++----------------------------------------------- 1 file changed, 5 insertions(+), 55 deletions(-) diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 8615ed59..f71f21db 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -28,7 +28,6 @@ import time import traceback import subprocess import tempfile -import textwrap import gzip import cliapp @@ -647,53 +646,6 @@ class SystemBuilder(BuilderBase): # pragma: no cover os.chmod(os_release_file, 0644) - def _chroot_runcmd(self, rootdir, to_mount, env, *args): - # We need to do mounts in a different namespace. Unfortunately - # this means we have to in-line the mount commands in the - # command-line. - command = textwrap.dedent(r''' - mount --make-rprivate / - rootdir="$1" - shift - ''') - cmdargs = [rootdir] - - # We need to mount all the specified mounts in the namespace, - # we don't need to unmount them before exiting, as they'll be - # unmounted when the namespace is no longer used. - command += textwrap.dedent(r''' - while true; do - case "$1" in - --) - shift - break - ;; - *) - mount_point="$1" - mount_type="$2" - mount_source="$3" - shift 3 - path="$rootdir/$mount_point" - mount -t "$mount_type" "$mount_source" "$path" - ;; - esac - done - ''') - for mount_opts in to_mount: - cmdargs.extend(mount_opts) - cmdargs.append('--') - - command += textwrap.dedent(r''' - exec chroot "$rootdir" "$@" - ''') - cmdargs.extend(args) - - # The single - is just a shell convention to fill $0 when using -c, - # since ordinarily $0 contains the program name. - cmdline = ['unshare', '--mount', '--', 'sh', '-ec', command, '-'] - cmdline.extend(cmdargs) - self.app.runcmd(cmdline, env=env) - def run_system_integration_commands(self, rootdir): # pragma: no cover ''' Run the system integration commands ''' @@ -708,18 +660,16 @@ class SystemBuilder(BuilderBase): # pragma: no cover self.app.status(msg='Running the system integration commands') to_mount = ( - ('proc', 'proc', 'none'), ('dev/shm', 'tmpfs', 'none'), ('tmp', 'tmpfs', 'none'), ) try: - for mount_point, mount_type, source in to_mount: - path = os.path.join(rootdir, mount_point) - if not os.path.exists(path): - os.makedirs(path) for bin in sorted(os.listdir(sys_integration_dir)): - self._chroot_runcmd(rootdir, to_mount, env, - os.path.join(SYSTEM_INTEGRATION_PATH, bin)) + self.app.runcmd( + morphlib.util.containerised_cmdline( + [os.path.join(SYSTEM_INTEGRATION_PATH, bin)], + root=rootdir, mounts=to_mount, mount_proc=True), + env=env) except BaseException, e: self.app.status( msg='Error while running system integration commands', -- cgit v1.2.1