diff options
author | Jonathan Maw <jonathan.maw@codethink.co.uk> | 2013-02-28 11:49:30 +0000 |
---|---|---|
committer | Jonathan Maw <jonathan.maw@codethink.co.uk> | 2013-02-28 11:49:30 +0000 |
commit | f33163d678ff74b41125654fabaf84e33f160f86 (patch) | |
tree | a7de4b0d203c6cdbf93a52870dcde33918f56164 | |
parent | 163d7e1dd8c84b93ddbb03b27f1a602cd660a402 (diff) | |
parent | 4474a163f78c8e1ea6cee9e594b5b8f7679ddb3f (diff) | |
download | morph-f33163d678ff74b41125654fabaf84e33f160f86.tar.gz |
Merge branch 'jonathanmaw/hardlink-staging-area-rebase'
Reviewed-by: Lars Wirzenius <lars.wirzenius@codethink.co.uk>
-rw-r--r-- | morphlib/builder2.py | 94 | ||||
-rw-r--r-- | morphlib/morph2.py | 1 | ||||
-rw-r--r-- | morphlib/plugins/branch_and_merge_plugin.py | 1 | ||||
-rw-r--r-- | morphlib/stagingarea.py | 172 | ||||
-rw-r--r-- | morphlib/stagingarea_tests.py | 26 |
5 files changed, 223 insertions, 71 deletions
diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 68768699..82a95820 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -20,6 +20,7 @@ import json import logging import os import shutil +import stat import time from collections import defaultdict import tarfile @@ -27,7 +28,6 @@ import traceback import subprocess import tempfile import gzip -from urlparse import urlparse import cliapp @@ -266,91 +266,57 @@ class ChunkBuilder(BuilderBase): else: return morphology[which] + def create_devices(self, destdir): # pragma: no cover + '''Creates device nodes if the morphology specifies them''' + morphology = self.artifact.source.morphology + perms_mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO + if 'devices' in morphology and morphology['devices'] is not None: + for dev in morphology['devices']: + destfile = os.path.join(destdir, './' + dev['filename']) + mode = int(dev['permissions'], 8) & perms_mask + if dev['type'] == 'c': + mode = mode | stat.S_IFCHR + elif dev['type'] == 'b': + mode = mode | stat.S_IFBLK + else: + raise IOError('Cannot create device node %s,' + 'unrecognized device type "%s"' + % (destfile, dev['type'])) + self.app.status(msg="Creating device node %s" + % destfile) + os.mknod(destfile, mode, + os.makedev(dev['major'], dev['minor'])) + os.chown(destfile, dev['uid'], dev['gid']) + def build_and_cache(self): # pragma: no cover with self.build_watch('overall-build'): - mounted = self.do_mounts() + + builddir, destdir = \ + self.staging_area.chroot_open(self.artifact.source, + self.setup_mounts) log_name = None try: - builddir = self.staging_area.builddir(self.artifact.source) self.get_sources(builddir) - destdir = self.staging_area.destdir(self.artifact.source) with self.local_artifact_cache.put_source_metadata( self.artifact.source, self.artifact.cache_key, 'build-log') as log: log_name = log.real_filename self.run_commands(builddir, destdir, log) + self.create_devices(destdir) except: - self.do_unmounts(mounted) + self.staging_area.chroot_close() if log_name: with open(log_name) as f: for line in f: logging.error('OUTPUT FROM FAILED BUILD: %s' % line.rstrip('\n')) raise - self.do_unmounts(mounted) + self.staging_area.chroot_close() built_artifacts = self.assemble_chunk_artifacts(destdir) self.save_build_times() return built_artifacts - to_mount = ( - ('proc', 'proc', 'none'), - ('dev/shm', 'tmpfs', 'none'), - ) - - def mount_ccachedir(self): #pragma: no cover - ccache_dir = self.app.settings['compiler-cache-dir'] - if not os.path.isdir(ccache_dir): - os.makedirs(ccache_dir) - # Get a path for the repo's ccache - ccache_url = self.artifact.source.repo.url - ccache_path = urlparse(ccache_url).path - ccache_repobase = os.path.basename(ccache_path) - if ':' in ccache_repobase: # the basename is a repo-alias - resolver = morphlib.repoaliasresolver.RepoAliasResolver( - self.app.settings['repo-alias']) - ccache_url = resolver.pull_url(ccache_repobase) - ccache_path = urlparse(ccache_url).path - ccache_repobase = os.path.basename(ccache_path) - if ccache_repobase.endswith('.git'): - ccache_repobase = ccache_repobase[:-len('.git')] - - ccache_repodir = os.path.join(ccache_dir, ccache_repobase) - # Make sure that directory exists - if not os.path.isdir(ccache_repodir): - os.mkdir(ccache_repodir) - # Get the destination path - ccache_destdir= os.path.join(self.staging_area.tempdir, - 'tmp', 'ccache') - # Make sure that the destination exists. We'll create /tmp if necessary - # to avoid breaking when faced with an empty staging area. - if not os.path.isdir(ccache_destdir): - os.makedirs(ccache_destdir) - # Mount it into the staging-area - self.app.runcmd(['mount', '--bind', ccache_repodir, - ccache_destdir]) - return ccache_destdir - - def do_mounts(self): # pragma: no cover - mounted = [] - if not self.setup_mounts: - return mounted - for mount_point, mount_type, source in ChunkBuilder.to_mount: - logging.debug('Mounting %s in staging area' % mount_point) - path = os.path.join(self.staging_area.dirname, mount_point) - if not os.path.exists(path): - os.makedirs(path) - self.app.runcmd(['mount', '-t', mount_type, source, path]) - mounted.append(path) - if not self.app.settings['no-ccache']: - mounted.append(self.mount_ccachedir()) - return mounted - - def do_unmounts(self, mounted): # pragma: no cover - for path in mounted: - logging.debug('Unmounting %s in staging area' % path) - morphlib.fsutils.unmount(self.app.runcmd, path) - def get_sources(self, srcdir): # pragma: no cover '''Get sources from git to a source directory, for building.''' diff --git a/morphlib/morph2.py b/morphlib/morph2.py index a95312c6..ee58ecdc 100644 --- a/morphlib/morph2.py +++ b/morphlib/morph2.py @@ -36,6 +36,7 @@ class Morphology(object): ('build-commands', None), ('test-commands', None), ('install-commands', None), + ('devices', None), ('chunks', []), ('max-jobs', None), ('build-system', 'manual') diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index f9595f98..cbc41acf 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -348,6 +348,7 @@ class BranchAndMergePlugin(cliapp.Plugin): 'install-commands', 'max-jobs', 'chunks', + 'devices', ] } diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index a87b45c3..f930f9d7 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,6 +17,8 @@ import logging import os import shutil +import stat +from urlparse import urlparse import morphlib @@ -40,6 +42,10 @@ class StagingArea(object): self._app = app self.dirname = dirname self.tempdir = tempdir + self.builddirname = None + self.destdirname = None + self.mounted = None + self._bind_readonly_mount = None # Wrapper to be overridden by unit tests. def _mkdir(self, dirname): # pragma: no cover @@ -80,6 +86,52 @@ class StagingArea(object): assert filename.startswith(dirname) return filename[len(dirname) - 1:] # include leading slash + def hardlink_all_files(self, srcpath, destpath): # pragma: no cover + '''Hardlink every file in the path to the staging-area + + If an exception is raised, the staging-area is indeterminate. + + ''' + + file_stat = os.lstat(srcpath) + mode = file_stat.st_mode + + if stat.S_ISDIR(mode): + # Ensure directory exists in destination, then recurse. + if not os.path.exists(destpath): + os.makedirs(destpath) + dest_stat = os.stat(os.path.realpath(destpath)) + if not stat.S_ISDIR(dest_stat.st_mode): + raise IOError('Destination not a directory. source has %s' + ' destination has %s' % (srcpath, destpath)) + + for entry in os.listdir(srcpath): + self.hardlink_all_files(os.path.join(srcpath, entry), + os.path.join(destpath, entry)) + elif stat.S_ISLNK(mode): + # Copy the symlink. + if os.path.exists(destpath): + os.remove(destpath) + os.symlink(os.readlink(srcpath), destpath) + + elif stat.S_ISREG(mode): + # Hardlink the file. + if os.path.exists(destpath): + os.remove(destpath) + os.link(srcpath, destpath) + + elif stat.S_ISCHR(mode) or stat.S_ISBLK(mode): + # Block or character device. Put contents of st_dev in a mknod. + if os.path.exists(destpath): + os.remove(destpath) + os.mknod(destpath, file_stat.st_mode, file_stat.st_rdev) + os.chmod(destpath, file_stat.st_mode) + + else: + # Unsupported type. + raise IOError('Cannot extract %s into staging-area. Unsupported' + ' type.' % srcpath) + def install_artifact(self, handle): '''Install a build artifact into the staging area. @@ -90,7 +142,19 @@ class StagingArea(object): logging.debug('Installing artifact %s' % getattr(handle, 'name', 'unknown name')) - morphlib.bins.unpack_binary_from_file(handle, self.dirname) + + unpacked_artifact = os.path.join( + self._app.settings['tempdir'], + os.path.basename(handle.name) + '.d') + if not os.path.exists(unpacked_artifact): + self._mkdir(unpacked_artifact) + morphlib.bins.unpack_binary_from_file( + handle, unpacked_artifact + '/') + + if not os.path.exists(self.dirname): + self._mkdir(self.dirname) + + self.hardlink_all_files(unpacked_artifact, self.dirname) def remove(self): '''Remove the entire staging area. @@ -103,14 +167,114 @@ class StagingArea(object): shutil.rmtree(self.dirname) + to_mount = ( + ('proc', 'proc', 'none'), + ('dev/shm', 'tmpfs', 'none'), + ) + + def mount_ccachedir(self, source): #pragma: no cover + ccache_dir = self._app.settings['compiler-cache-dir'] + if not os.path.isdir(ccache_dir): + os.makedirs(ccache_dir) + # Get a path for the repo's ccache + ccache_url = source.repo.url + ccache_path = urlparse(ccache_url).path + ccache_repobase = os.path.basename(ccache_path) + if ':' in ccache_repobase: # the basename is a repo-alias + resolver = morphlib.repoaliasresolver.RepoAliasResolver( + self._app.settings['repo-alias']) + ccache_url = resolver.pull_url(ccache_repobase) + ccache_path = urlparse(ccache_url).path + ccache_repobase = os.path.basename(ccache_path) + if ccache_repobase.endswith('.git'): + ccache_repobase = ccache_repobase[:-len('.git')] + + ccache_repodir = os.path.join(ccache_dir, ccache_repobase) + # Make sure that directory exists + if not os.path.isdir(ccache_repodir): + os.mkdir(ccache_repodir) + # Get the destination path + ccache_destdir= os.path.join(self.tempdir, + 'tmp', 'ccache') + # Make sure that the destination exists. We'll create /tmp if necessary + # to avoid breaking when faced with an empty staging area. + if not os.path.isdir(ccache_destdir): + os.makedirs(ccache_destdir) + # Mount it into the staging-area + self._app.runcmd(['mount', '--bind', ccache_repodir, + ccache_destdir]) + return ccache_destdir + + def do_mounts(self, setup_mounts): # pragma: no cover + self.mounted = [] + if not setup_mounts: + return + for mount_point, mount_type, source in self.to_mount: + logging.debug('Mounting %s in staging area' % mount_point) + path = os.path.join(self.dirname, mount_point) + if not os.path.exists(path): + os.makedirs(path) + self._app.runcmd(['mount', '-t', mount_type, source, path]) + self.mounted.append(path) + return + + def do_unmounts(self): # pragma: no cover + for path in reversed(self.mounted): + logging.debug('Unmounting %s in staging area' % path) + morphlib.fsutils.unmount(self._app.runcmd, path) + + def chroot_open(self, source, setup_mounts): # pragma: no cover + '''Setup staging area for use as a chroot.''' + + assert self.builddirname == None and self.destdirname == None + + builddir = self.builddir(source) + destdir = self.destdir(source) + self.builddirname = self.relative(builddir).lstrip('/') + self.destdirname = self.relative(destdir).lstrip('/') + + self.do_mounts(setup_mounts) + + if not self._app.settings['no-ccache']: + self.mounted.append(self.mount_ccachedir(source)) + + return builddir, destdir + + def chroot_close(self): # pragma: no cover + '''Undo changes by chroot_open. + + This should be called after the staging area is no longer needed. + + ''' + + self.do_unmounts() + def runcmd(self, argv, **kwargs): # pragma: no cover '''Run a command in a chroot in the staging area.''' + cwd = kwargs.get('cwd') or '/' if 'cwd' in kwargs: cwd = kwargs['cwd'] del kwargs['cwd'] else: cwd = '/' - real_argv = ['chroot', self.dirname, 'sh', '-c', - 'cd "$1" && shift && exec "$@"', '--', cwd] + argv + if self._app.settings['staging-chroot']: + not_readonly_dirs = [self.builddirname, self.destdirname, + 'dev', 'proc', 'tmp'] + dirs = os.listdir(self.dirname) + for excluded_dir in not_readonly_dirs: + dirs.remove(excluded_dir) + + real_argv = ['linux-user-chroot'] + + for entry in dirs: + real_argv += ['--mount-readonly', '/'+entry] + + real_argv += [self.dirname] + else: + real_argv = ['chroot', '/'] + + real_argv += ['sh', '-c', 'cd "$1" && shift && exec "$@"', '--', cwd] + real_argv += argv + return self._app.runcmd(real_argv, **kwargs) diff --git a/morphlib/stagingarea_tests.py b/morphlib/stagingarea_tests.py index 3230b9e3..313226d2 100644 --- a/morphlib/stagingarea_tests.py +++ b/morphlib/stagingarea_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,6 +14,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import cliapp import os import shutil import tarfile @@ -31,14 +32,33 @@ class FakeSource(object): } +class FakeApplication(object): + + def __init__(self, cachedir, tempdir): + self.settings = { + 'cachedir': cachedir, + 'tempdir': tempdir, + } + + def runcmd(self, *args, **kwargs): + return cliapp.runcmd(*args, **kwargs) + + def runcmd_unchecked(self, *args, **kwargs): + return cliapp.runcmd_unchecked(*args, **kwargs) + + class StagingAreaTests(unittest.TestCase): def setUp(self): self.tempdir = tempfile.mkdtemp() + self.cachedir = os.path.join(self.tempdir, 'cachedir') + os.mkdir(self.cachedir) + os.mkdir(os.path.join(self.cachedir, 'artifacts')) self.staging = os.path.join(self.tempdir, 'staging') self.created_dirs = [] - self.sa = morphlib.stagingarea.StagingArea(object(), self.staging, - self.staging) + self.sa = morphlib.stagingarea.StagingArea( + FakeApplication(self.cachedir, self.tempdir), + self.staging, self.staging) def tearDown(self): shutil.rmtree(self.tempdir) |