summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRichard Maw <richard.maw@codethink.co.uk>2014-11-14 14:02:02 +0000
committerRichard Maw <richard.maw@codethink.co.uk>2014-11-14 14:02:02 +0000
commit9b0b5206a25c1d874d6e17952c4385838e57563e (patch)
tree675456f07c011d97d6c023603d01db0ebc03ea9a
parent2a019b5fee65a738bdd38717b0b84eea638a776d (diff)
parent40424d89f62ce6f268c4920a050b1f42ed2f5ac2 (diff)
downloadmorph-9b0b5206a25c1d874d6e17952c4385838e57563e.tar.gz
Merge branch 'baserock/richardmaw/fixup-ro-integrations' of git://git.baserock.org/baserock/baserock/morph
Reviewed-by: Sam Thursfield Reviewed-by: Pedro Alvarez
-rw-r--r--morphlib/builder2.py60
-rw-r--r--morphlib/fsutils.py67
-rw-r--r--morphlib/fsutils_tests.py25
-rw-r--r--scripts/test-shell.c11
-rw-r--r--yarns/implementations.yarn3
-rw-r--r--yarns/morph.shell-lib8
6 files changed, 73 insertions, 101 deletions
diff --git a/morphlib/builder2.py b/morphlib/builder2.py
index 8615ed59..f71f21db 100644
--- a/morphlib/builder2.py
+++ b/morphlib/builder2.py
@@ -28,7 +28,6 @@ import time
import traceback
import subprocess
import tempfile
-import textwrap
import gzip
import cliapp
@@ -647,53 +646,6 @@ class SystemBuilder(BuilderBase): # pragma: no cover
os.chmod(os_release_file, 0644)
- def _chroot_runcmd(self, rootdir, to_mount, env, *args):
- # We need to do mounts in a different namespace. Unfortunately
- # this means we have to in-line the mount commands in the
- # command-line.
- command = textwrap.dedent(r'''
- mount --make-rprivate /
- rootdir="$1"
- shift
- ''')
- cmdargs = [rootdir]
-
- # We need to mount all the specified mounts in the namespace,
- # we don't need to unmount them before exiting, as they'll be
- # unmounted when the namespace is no longer used.
- command += textwrap.dedent(r'''
- while true; do
- case "$1" in
- --)
- shift
- break
- ;;
- *)
- mount_point="$1"
- mount_type="$2"
- mount_source="$3"
- shift 3
- path="$rootdir/$mount_point"
- mount -t "$mount_type" "$mount_source" "$path"
- ;;
- esac
- done
- ''')
- for mount_opts in to_mount:
- cmdargs.extend(mount_opts)
- cmdargs.append('--')
-
- command += textwrap.dedent(r'''
- exec chroot "$rootdir" "$@"
- ''')
- cmdargs.extend(args)
-
- # The single - is just a shell convention to fill $0 when using -c,
- # since ordinarily $0 contains the program name.
- cmdline = ['unshare', '--mount', '--', 'sh', '-ec', command, '-']
- cmdline.extend(cmdargs)
- self.app.runcmd(cmdline, env=env)
-
def run_system_integration_commands(self, rootdir): # pragma: no cover
''' Run the system integration commands '''
@@ -708,18 +660,16 @@ class SystemBuilder(BuilderBase): # pragma: no cover
self.app.status(msg='Running the system integration commands')
to_mount = (
- ('proc', 'proc', 'none'),
('dev/shm', 'tmpfs', 'none'),
('tmp', 'tmpfs', 'none'),
)
try:
- for mount_point, mount_type, source in to_mount:
- path = os.path.join(rootdir, mount_point)
- if not os.path.exists(path):
- os.makedirs(path)
for bin in sorted(os.listdir(sys_integration_dir)):
- self._chroot_runcmd(rootdir, to_mount, env,
- os.path.join(SYSTEM_INTEGRATION_PATH, bin))
+ self.app.runcmd(
+ morphlib.util.containerised_cmdline(
+ [os.path.join(SYSTEM_INTEGRATION_PATH, bin)],
+ root=rootdir, mounts=to_mount, mount_proc=True),
+ env=env)
except BaseException, e:
self.app.status(
msg='Error while running system integration commands',
diff --git a/morphlib/fsutils.py b/morphlib/fsutils.py
index 6d651171..8a4128d9 100644
--- a/morphlib/fsutils.py
+++ b/morphlib/fsutils.py
@@ -97,48 +97,43 @@ def invert_paths(tree_walker, paths):
'''
- def is_subpath(prefix, path):
- prefix_components = prefix.split(os.sep)
- path_components = path.split(os.sep)
- return path_components[:len(prefix_components)] == prefix_components
+ def normpath(path):
+ if path == '.':
+ return path
+ path = os.path.normpath(path)
+ if not os.path.isabs(path):
+ path = os.path.join('.', path)
+ return path
+ def any_paths_are_subpath_of(prefix):
+ prefix = normpath(prefix)
+ norm_paths = (normpath(path) for path in paths)
+ return any(path[:len(prefix)] == prefix
+ for path in norm_paths)
+
+ def path_is_listed(path):
+ return any(normpath(path) == normpath(other)
+ for other in paths)
for dirpath, dirnames, filenames in tree_walker:
- if any(p == dirpath for p in paths): # pragma: no cover
- # Dir is an exact match for a path
- # don't recurse any further
- # Don't yield it, since we don't return listed paths
- continue
- dn_copy = list(dirnames)
- for subdir in dn_copy:
- subdirpath = os.path.join(dirpath, subdir)
-
- if any(p == subdirpath for p in paths):
- # Subdir is an exact match for a path
- # Don't recurse into it, so remove from list
- # Don't yield it, since we don't return listed paths
- dirnames.remove(subdir)
- elif any(is_subpath(subdirpath, p) for p in paths):
- # This directory is a parent directory of one
- # of our paths
- # Recurse into it, so don't remove it from the list
- # Don't yield it, since we don't return listed paths
- pass
- else:
- # This directory is neither one marked for writing,
- # nor a parent of a file marked for writing
- # Don't recurse, so remove it from the list
- # Yield it, since we return listed paths
- dirnames.remove(subdir)
- yield subdirpath
+ if path_is_listed(dirpath):
+ # No subpaths need to be considered
+ del dirnames[:]
+ del filenames[:]
+ elif any_paths_are_subpath_of(dirpath):
+ # Subpaths may be marked, or may not, need to leave this
+ # writable, so don't yield, but we don't cull.
+ pass
+ else:
+ # not listed as a parent or an exact match, needs to be
+ # yielded, but we don't need to consider subdirs, so can cull
+ yield dirpath
+ del dirnames[:]
+ del filenames[:]
for filename in filenames:
fullpath = os.path.join(dirpath, filename)
- if any(is_subpath(p, fullpath) for p in paths):
- # The file path is a child of one of the paths
- # or is equal.
- # Don't yield because either it is one of the specified
- # paths, or is a file in a directory specified by a path
+ if path_is_listed(fullpath):
pass
else:
yield fullpath
diff --git a/morphlib/fsutils_tests.py b/morphlib/fsutils_tests.py
index 7b159665..47a4488e 100644
--- a/morphlib/fsutils_tests.py
+++ b/morphlib/fsutils_tests.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2013 Codethink Limited
+# Copyright (C) 2013, 2014 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -51,11 +51,26 @@ class InvertPathsTests(unittest.TestCase):
},
}
- def test_flat_lists_single_files(self):
+ def test_flat_lists_top_dir(self):
walker = dummy_top_down_walker('.', self.flat_tree)
- self.assertEqual(sorted(["./foo", "./bar", "./baz"]),
+ self.assertEqual(["."],
sorted(morphlib.fsutils.invert_paths(walker, [])))
+ def test_flat_skips_all_with_root_pased(self):
+ walker = dummy_top_down_walker('.', self.flat_tree)
+ self.assertEqual([],
+ list(morphlib.fsutils.invert_paths(walker, ['.'])))
+
+ def test_flat_lists_top_dir(self):
+ walker = dummy_top_down_walker('.', self.nested_tree)
+ self.assertEqual(["."],
+ sorted(morphlib.fsutils.invert_paths(walker, [])))
+
+ def test_flat_skips_all_with_root_pased(self):
+ walker = dummy_top_down_walker('.', self.nested_tree)
+ self.assertEqual([],
+ list(morphlib.fsutils.invert_paths(walker, ['.'])))
+
def test_flat_excludes_listed_files(self):
walker = dummy_top_down_walker('.', self.flat_tree)
self.assertTrue(
@@ -95,5 +110,5 @@ class InvertPathsTests(unittest.TestCase):
"./tmp/morph/staging/inst",
"./tmp",
]))
- expected = ("./bin",)
- self.assertEqual(sorted(found), sorted(expected))
+ expected = ["./bin"]
+ self.assertEqual(sorted(found), expected)
diff --git a/scripts/test-shell.c b/scripts/test-shell.c
index 2f8ecbd8..f4ac6bfa 100644
--- a/scripts/test-shell.c
+++ b/scripts/test-shell.c
@@ -10,6 +10,7 @@
#include <stdint.h>
#include <ftw.h>
#include <errno.h>
+#include <err.h>
char *readlinka(char const *path){
size_t buflen = BUFSIZ;
@@ -113,8 +114,8 @@ int copy_file_objects(FILE *source, FILE *target) {
do {
read = fread(buffer, 1, sizeof(buffer), source);
fwrite(buffer, 1, read, target);
- } while (!feof(source));
- return ferror(source) ? -1 : 0;
+ } while (!feof(source) && !feof(target));
+ return (ferror(source) || ferror(target)) ? -1 : 0;
}
int run_commands(FILE *cmdstream){
@@ -145,8 +146,14 @@ int run_commands(FILE *cmdstream){
} else if (strstr(line, "create file ") == line) {
char const *filename = line + sizeof("create file ") -1;
FILE *outfile = fopen(filename, "w");
+ if (outfile == NULL){
+ ret = 1;
+ err(errno, "Opening %s for write failed", filename);
+ break;
+ }
if (copy_file_objects(cmdstream, outfile) < 0) {
ret = 1;
+ err(errno, "Writing to %s failed", filename);
fclose(outfile);
break;
}
diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn
index 6110148e..e1ae271f 100644
--- a/yarns/implementations.yarn
+++ b/yarns/implementations.yarn
@@ -34,7 +34,8 @@ we can test it later in a THEN step.
case $(cat "$DATADIR/morph-exit") in
0) echo "Morph succeeded!"
;;
- *) die "Morph should have succeeded, but didn't. Unexpected failure!"
+ *) cat "$DATADIR/result-latest" >&2
+ die "Morph should have succeeded, but didn't. Unexpected failure!"
;;
esac
diff --git a/yarns/morph.shell-lib b/yarns/morph.shell-lib
index 9c13e449..d9c9eff6 100644
--- a/yarns/morph.shell-lib
+++ b/yarns/morph.shell-lib
@@ -39,9 +39,13 @@ run_morph()
set +e
"$SRCDIR"/morph --verbose \
--cachedir-min-space=0 --tempdir-min-space=0 \
- --no-default-config --config "$DATADIR/morph.conf" "$@" \
- 2> "$DATADIR/result-$1" > "$DATADIR/out-$1"
+ --no-default-config --config "$DATADIR/morph.conf" \
+ --log="$DATADIR/log-$1" \
+ "$@" 2> "$DATADIR/result-$1" > "$DATADIR/out-$1"
local exit_code="$?"
+ for o in log result out; do
+ ln -sf "$o-$1" "$DATADIR/$o-latest"
+ done
cat "$DATADIR/out-$1"
cat "$DATADIR/result-$1" >&2
return "$exit_code"