diff options
author | Jannis Pohlmann <jannis.pohlmann@codethink.co.uk> | 2012-11-30 12:35:29 +0000 |
---|---|---|
committer | Jannis Pohlmann <jannis.pohlmann@codethink.co.uk> | 2012-11-30 12:35:29 +0000 |
commit | 8cd95022df4a01a9ffa2f36a4118d0ace8ddbdc0 (patch) | |
tree | b95c413aaabedd0255e49d27010514b1d1afcca9 /morphlib | |
parent | c859d70d86423a52bc7053abf64e9ca21f62a487 (diff) | |
download | morph-8cd95022df4a01a9ffa2f36a4118d0ace8ddbdc0.tar.gz |
Revert "Merge branch 'liw/hardlink-staging-area-pre-rebase'"
This reverts commit c859d70d86423a52bc7053abf64e9ca21f62a487, reversing
changes made to 3085a672d1e8b5177be33f0463385de72a0ef5bf.
Unfortunately, hardlinking and linux-user-chroot both break builds
in various ways. Hardlinking breaks the bootstrap process by creating
symlinks like /usr/libexec that can then not be overwritten with real
files by install scripts from morphologies like gcc.
linux-user-chroot caused problems by breaking privileged operations such
as chgrp and CAP_SETFCAP. As a consequence, chunks like util-linux and
libcap can no longer be built.
Diffstat (limited to 'morphlib')
-rw-r--r-- | morphlib/builder2.py | 32 | ||||
-rw-r--r-- | morphlib/stagingarea.py | 71 | ||||
-rw-r--r-- | morphlib/stagingarea_tests.py | 19 |
3 files changed, 33 insertions, 89 deletions
diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 28a5e0ba..c7d25e1a 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -259,31 +259,53 @@ class ChunkBuilder(BuilderBase): def build_and_cache(self): # pragma: no cover with self.build_watch('overall-build'): - - builddir, destdir = \ - self.staging_area.chroot_open(self.artifact.source) + mounted = self.do_mounts() log_name = None try: + builddir = self.staging_area.builddir(self.artifact.source) self.get_sources(builddir) + destdir = self.staging_area.destdir(self.artifact.source) with self.local_artifact_cache.put_source_metadata( self.artifact.source, self.artifact.cache_key, 'build-log') as log: log_name = log.real_filename self.run_commands(builddir, destdir, log) except: - self.staging_area.chroot_close() + self.do_unmounts(mounted) if log_name: with open(log_name) as f: for line in f: logging.error('OUTPUT FROM FAILED BUILD: %s' % line.rstrip('\n')) raise - self.staging_area.chroot_close() + self.do_unmounts(mounted) built_artifacts = self.assemble_chunk_artifacts(destdir) self.save_build_times() return built_artifacts + to_mount = ( + ('proc', 'proc', 'none'), + ('dev/shm', 'tmpfs', 'none'), + ) + + def do_mounts(self): # pragma: no cover + mounted = [] + if not self.setup_mounts: + return mounted + for mount_point, mount_type, source in ChunkBuilder.to_mount: + logging.debug('Mounting %s in staging area' % mount_point) + path = os.path.join(self.staging_area.dirname, mount_point) + if not os.path.exists(path): + os.makedirs(path) + self.app.runcmd(['mount', '-t', mount_type, source, path]) + mounted.append(path) + return mounted + + def do_unmounts(self, mounted): # pragma: no cover + for path in mounted: + logging.debug('Unmounting %s in staging area' % path) + morphlib.fsutils.unmount(self.app.runcmd, path) def get_sources(self, srcdir): # pragma: no cover '''Get sources from git to a source directory, for building.''' diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index 77eeab7d..a87b45c3 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -40,8 +40,6 @@ class StagingArea(object): self._app = app self.dirname = dirname self.tempdir = tempdir - self.builddirname = None - self.destdirname = None # Wrapper to be overridden by unit tests. def _mkdir(self, dirname): # pragma: no cover @@ -92,21 +90,7 @@ class StagingArea(object): logging.debug('Installing artifact %s' % getattr(handle, 'name', 'unknown name')) - - unpacked_artifact = os.path.join( - self._app.settings['cachedir'], - 'artifacts', - os.path.basename(handle.name) + '.d') - if not os.path.exists(unpacked_artifact): - self._mkdir(unpacked_artifact) - morphlib.bins.unpack_binary_from_file( - handle, unpacked_artifact + '/') - - if not os.path.exists(self.dirname): - self._mkdir(self.dirname) - - self._app.runcmd( - ['cp', '-al', unpacked_artifact + '/.', self.dirname + '/.']) + morphlib.bins.unpack_binary_from_file(handle, self.dirname) def remove(self): '''Remove the entire staging area. @@ -119,61 +103,14 @@ class StagingArea(object): shutil.rmtree(self.dirname) - def chroot_open(self, source): # pragma: no cover - '''Setup staging area for use as a chroot.''' - - assert self.builddirname == None and self.destdirname == None - - builddir = self.builddir(source) - destdir = self.destdir(source) - self.builddirname = self.relative(builddir).lstrip('/') - self.destdirname = self.relative(destdir).lstrip('/') - - for mount_point in ['proc','dev/shm']: - path = os.path.join(self.dirname, mount_point) - if not os.path.exists(path): - os.makedirs(path) - - return builddir, destdir - - def chroot_close(self): # pragma: no cover - '''Undo changes by chroot_open. - - This should be called after the staging area is no longer needed. - - ''' - def runcmd(self, argv, **kwargs): # pragma: no cover - '''Run a command in the staging area.''' - + '''Run a command in a chroot in the staging area.''' cwd = kwargs.get('cwd') or '/' if 'cwd' in kwargs: cwd = kwargs['cwd'] del kwargs['cwd'] else: cwd = '/' - - if self._app.settings['staging-chroot']: - entries = os.listdir(self.dirname) - - friends = [self.builddirname, self.destdirname, - 'dev', 'proc', 'tmp'] - for friend in friends: - if friend in friends: - entries.remove(friend) - - real_argv = ['linux-user-chroot'] - - for entry in entries: - real_argv += ['--mount-readonly',"/"+entry] - - real_argv += ['--mount-proc','/proc'] - real_argv += ['--mount-bind','/dev/shm','/dev/shm'] - real_argv += [self.dirname] - else: - real_argv = ['chroot', '/'] - - real_argv += ['sh', '-c', 'cd "$1" && shift && exec "$@"', '--', cwd] - real_argv += argv - + real_argv = ['chroot', self.dirname, 'sh', '-c', + 'cd "$1" && shift && exec "$@"', '--', cwd] + argv return self._app.runcmd(real_argv, **kwargs) diff --git a/morphlib/stagingarea_tests.py b/morphlib/stagingarea_tests.py index 15131272..3230b9e3 100644 --- a/morphlib/stagingarea_tests.py +++ b/morphlib/stagingarea_tests.py @@ -14,7 +14,6 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -import cliapp import os import shutil import tarfile @@ -32,28 +31,14 @@ class FakeSource(object): } -class FakeApplication(object): - - def __init__(self, cachedir): - self.settings = { - 'cachedir': cachedir, - } - - def runcmd(self, *args, **kwargs): - cliapp.runcmd(*args, **kwargs) - - class StagingAreaTests(unittest.TestCase): def setUp(self): self.tempdir = tempfile.mkdtemp() - self.cachedir = os.path.join(self.tempdir, 'cachedir') - os.mkdir(self.cachedir) - os.mkdir(os.path.join(self.cachedir, 'artifacts')) self.staging = os.path.join(self.tempdir, 'staging') self.created_dirs = [] - self.sa = morphlib.stagingarea.StagingArea( - FakeApplication(self.cachedir), self.staging, self.staging) + self.sa = morphlib.stagingarea.StagingArea(object(), self.staging, + self.staging) def tearDown(self): shutil.rmtree(self.tempdir) |