diff options
author | Lars Wirzenius <lars.wirzenius@codethink.co.uk> | 2012-11-29 16:08:20 +0000 |
---|---|---|
committer | Lars Wirzenius <lars.wirzenius@codethink.co.uk> | 2012-11-29 16:08:20 +0000 |
commit | c859d70d86423a52bc7053abf64e9ca21f62a487 (patch) | |
tree | 8a714de08c4a6ac12f37fd840ac58c70c5f9b8da | |
parent | 3085a672d1e8b5177be33f0463385de72a0ef5bf (diff) | |
parent | 0b4655904bd0293055cd7e150d89124c2a32e6b9 (diff) | |
download | morph-c859d70d86423a52bc7053abf64e9ca21f62a487.tar.gz |
Merge branch 'liw/hardlink-staging-area-pre-rebase'
This is based on, and incorporates, Joe Burmeister's changes to fill
staging areas using hardlinking, for speed reasons. I have reformatted
the code a bit, and fixed it to pass the test suite, and fixed a bug
in how commands are run in the staging area chroot (preserving the
environment).
-rw-r--r-- | morphlib/builder2.py | 32 | ||||
-rw-r--r-- | morphlib/stagingarea.py | 71 | ||||
-rw-r--r-- | morphlib/stagingarea_tests.py | 19 |
3 files changed, 89 insertions, 33 deletions
diff --git a/morphlib/builder2.py b/morphlib/builder2.py index c7d25e1a..28a5e0ba 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -259,53 +259,31 @@ class ChunkBuilder(BuilderBase): def build_and_cache(self): # pragma: no cover with self.build_watch('overall-build'): - mounted = self.do_mounts() + + builddir, destdir = \ + self.staging_area.chroot_open(self.artifact.source) log_name = None try: - builddir = self.staging_area.builddir(self.artifact.source) self.get_sources(builddir) - destdir = self.staging_area.destdir(self.artifact.source) with self.local_artifact_cache.put_source_metadata( self.artifact.source, self.artifact.cache_key, 'build-log') as log: log_name = log.real_filename self.run_commands(builddir, destdir, log) except: - self.do_unmounts(mounted) + self.staging_area.chroot_close() if log_name: with open(log_name) as f: for line in f: logging.error('OUTPUT FROM FAILED BUILD: %s' % line.rstrip('\n')) raise - self.do_unmounts(mounted) + self.staging_area.chroot_close() built_artifacts = self.assemble_chunk_artifacts(destdir) self.save_build_times() return built_artifacts - to_mount = ( - ('proc', 'proc', 'none'), - ('dev/shm', 'tmpfs', 'none'), - ) - - def do_mounts(self): # pragma: no cover - mounted = [] - if not self.setup_mounts: - return mounted - for mount_point, mount_type, source in ChunkBuilder.to_mount: - logging.debug('Mounting %s in staging area' % mount_point) - path = os.path.join(self.staging_area.dirname, mount_point) - if not os.path.exists(path): - os.makedirs(path) - self.app.runcmd(['mount', '-t', mount_type, source, path]) - mounted.append(path) - return mounted - - def do_unmounts(self, mounted): # pragma: no cover - for path in mounted: - logging.debug('Unmounting %s in staging area' % path) - morphlib.fsutils.unmount(self.app.runcmd, path) def get_sources(self, srcdir): # pragma: no cover '''Get sources from git to a source directory, for building.''' diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index a87b45c3..77eeab7d 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -40,6 +40,8 @@ class StagingArea(object): self._app = app self.dirname = dirname self.tempdir = tempdir + self.builddirname = None + self.destdirname = None # Wrapper to be overridden by unit tests. def _mkdir(self, dirname): # pragma: no cover @@ -90,7 +92,21 @@ class StagingArea(object): logging.debug('Installing artifact %s' % getattr(handle, 'name', 'unknown name')) - morphlib.bins.unpack_binary_from_file(handle, self.dirname) + + unpacked_artifact = os.path.join( + self._app.settings['cachedir'], + 'artifacts', + os.path.basename(handle.name) + '.d') + if not os.path.exists(unpacked_artifact): + self._mkdir(unpacked_artifact) + morphlib.bins.unpack_binary_from_file( + handle, unpacked_artifact + '/') + + if not os.path.exists(self.dirname): + self._mkdir(self.dirname) + + self._app.runcmd( + ['cp', '-al', unpacked_artifact + '/.', self.dirname + '/.']) def remove(self): '''Remove the entire staging area. @@ -103,14 +119,61 @@ class StagingArea(object): shutil.rmtree(self.dirname) + def chroot_open(self, source): # pragma: no cover + '''Setup staging area for use as a chroot.''' + + assert self.builddirname == None and self.destdirname == None + + builddir = self.builddir(source) + destdir = self.destdir(source) + self.builddirname = self.relative(builddir).lstrip('/') + self.destdirname = self.relative(destdir).lstrip('/') + + for mount_point in ['proc','dev/shm']: + path = os.path.join(self.dirname, mount_point) + if not os.path.exists(path): + os.makedirs(path) + + return builddir, destdir + + def chroot_close(self): # pragma: no cover + '''Undo changes by chroot_open. + + This should be called after the staging area is no longer needed. + + ''' + def runcmd(self, argv, **kwargs): # pragma: no cover - '''Run a command in a chroot in the staging area.''' + '''Run a command in the staging area.''' + cwd = kwargs.get('cwd') or '/' if 'cwd' in kwargs: cwd = kwargs['cwd'] del kwargs['cwd'] else: cwd = '/' - real_argv = ['chroot', self.dirname, 'sh', '-c', - 'cd "$1" && shift && exec "$@"', '--', cwd] + argv + + if self._app.settings['staging-chroot']: + entries = os.listdir(self.dirname) + + friends = [self.builddirname, self.destdirname, + 'dev', 'proc', 'tmp'] + for friend in friends: + if friend in friends: + entries.remove(friend) + + real_argv = ['linux-user-chroot'] + + for entry in entries: + real_argv += ['--mount-readonly',"/"+entry] + + real_argv += ['--mount-proc','/proc'] + real_argv += ['--mount-bind','/dev/shm','/dev/shm'] + real_argv += [self.dirname] + else: + real_argv = ['chroot', '/'] + + real_argv += ['sh', '-c', 'cd "$1" && shift && exec "$@"', '--', cwd] + real_argv += argv + return self._app.runcmd(real_argv, **kwargs) diff --git a/morphlib/stagingarea_tests.py b/morphlib/stagingarea_tests.py index 3230b9e3..15131272 100644 --- a/morphlib/stagingarea_tests.py +++ b/morphlib/stagingarea_tests.py @@ -14,6 +14,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import cliapp import os import shutil import tarfile @@ -31,14 +32,28 @@ class FakeSource(object): } +class FakeApplication(object): + + def __init__(self, cachedir): + self.settings = { + 'cachedir': cachedir, + } + + def runcmd(self, *args, **kwargs): + cliapp.runcmd(*args, **kwargs) + + class StagingAreaTests(unittest.TestCase): def setUp(self): self.tempdir = tempfile.mkdtemp() + self.cachedir = os.path.join(self.tempdir, 'cachedir') + os.mkdir(self.cachedir) + os.mkdir(os.path.join(self.cachedir, 'artifacts')) self.staging = os.path.join(self.tempdir, 'staging') self.created_dirs = [] - self.sa = morphlib.stagingarea.StagingArea(object(), self.staging, - self.staging) + self.sa = morphlib.stagingarea.StagingArea( + FakeApplication(self.cachedir), self.staging, self.staging) def tearDown(self): shutil.rmtree(self.tempdir) |