diff options
-rw-r--r-- | morphlib/__init__.py | 2 | ||||
-rw-r--r-- | morphlib/bins.py | 29 | ||||
-rw-r--r-- | morphlib/builder2.py | 94 | ||||
-rw-r--r-- | morphlib/extractedtarball.py | 61 | ||||
-rw-r--r-- | morphlib/morph2.py | 1 | ||||
-rw-r--r-- | morphlib/mountableimage.py | 81 | ||||
-rw-r--r-- | morphlib/plugins/artifact_inspection_plugin.py | 294 | ||||
-rw-r--r-- | morphlib/plugins/branch_and_merge_plugin.py | 1 | ||||
-rw-r--r-- | morphlib/plugins/trebuchet_plugin.py | 60 | ||||
-rw-r--r-- | morphlib/stagingarea.py | 172 | ||||
-rw-r--r-- | morphlib/stagingarea_tests.py | 26 | ||||
-rw-r--r-- | tests.as-root/run-in-artifact-propagates-exit-code.exit | 1 | ||||
-rwxr-xr-x | tests.as-root/run-in-artifact-propagates-exit-code.script | 33 | ||||
-rw-r--r-- | tests.as-root/run-in-artifact-propagates-exit-code.stderr | 3 | ||||
-rwxr-xr-x | tests.as-root/run-in-artifact-with-different-artifacts.script | 47 | ||||
-rw-r--r-- | tests.as-root/run-in-artifact-with-different-artifacts.stderr | 1 | ||||
-rw-r--r-- | tests.as-root/run-in-artifact-with-different-artifacts.stdout | 14 | ||||
-rw-r--r-- | without-test-modules | 3 |
18 files changed, 794 insertions, 129 deletions
diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 271fa977..b20c3f01 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -53,10 +53,12 @@ import builder2 import cachedir import cachedrepo import cachekeycomputer +import extractedtarball import fsutils import git import localartifactcache import localrepocache +import mountableimage import morph2 import morphologyfactory import remoteartifactcache diff --git a/morphlib/bins.py b/morphlib/bins.py index 622aa165..ba5f713f 100644 --- a/morphlib/bins.py +++ b/morphlib/bins.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2012 Codethink Limited +# Copyright (C) 2011-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -21,6 +21,7 @@ Binaries are chunks, strata, and system images. ''' +import cliapp import logging import os import re @@ -29,6 +30,11 @@ import stat import shutil import tarfile +import morphlib + +from morphlib.extractedtarball import ExtractedTarball +from morphlib.mountableimage import MountableImage + # Work around http://bugs.python.org/issue16477 def safe_makefile(self, tarinfo, targetpath): @@ -211,3 +217,24 @@ def unpack_binary_from_file(f, dirname): # pragma: no cover def unpack_binary(filename, dirname): with open(filename, "rb") as f: unpack_binary_from_file(f, dirname) + + +class ArtifactNotMountableError(cliapp.AppException): # pragma: no cover + + def __init__(self, filename): + cliapp.AppException.__init__( + self, 'Artifact %s cannot be extracted or mounted' % filename) + + +def call_in_artifact_directory(app, filename, callback): # pragma: no cover + '''Call a function in a directory the artifact is extracted/mounted in.''' + + try: + with ExtractedTarball(app, filename) as dirname: + callback(dirname) + except tarfile.TarError: + try: + with MountableImage(app, filename) as dirname: + callback(dirname) + except (IOError, OSError): + raise ArtifactNotMountableError(filename) diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 68768699..82a95820 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -20,6 +20,7 @@ import json import logging import os import shutil +import stat import time from collections import defaultdict import tarfile @@ -27,7 +28,6 @@ import traceback import subprocess import tempfile import gzip -from urlparse import urlparse import cliapp @@ -266,91 +266,57 @@ class ChunkBuilder(BuilderBase): else: return morphology[which] + def create_devices(self, destdir): # pragma: no cover + '''Creates device nodes if the morphology specifies them''' + morphology = self.artifact.source.morphology + perms_mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO + if 'devices' in morphology and morphology['devices'] is not None: + for dev in morphology['devices']: + destfile = os.path.join(destdir, './' + dev['filename']) + mode = int(dev['permissions'], 8) & perms_mask + if dev['type'] == 'c': + mode = mode | stat.S_IFCHR + elif dev['type'] == 'b': + mode = mode | stat.S_IFBLK + else: + raise IOError('Cannot create device node %s,' + 'unrecognized device type "%s"' + % (destfile, dev['type'])) + self.app.status(msg="Creating device node %s" + % destfile) + os.mknod(destfile, mode, + os.makedev(dev['major'], dev['minor'])) + os.chown(destfile, dev['uid'], dev['gid']) + def build_and_cache(self): # pragma: no cover with self.build_watch('overall-build'): - mounted = self.do_mounts() + + builddir, destdir = \ + self.staging_area.chroot_open(self.artifact.source, + self.setup_mounts) log_name = None try: - builddir = self.staging_area.builddir(self.artifact.source) self.get_sources(builddir) - destdir = self.staging_area.destdir(self.artifact.source) with self.local_artifact_cache.put_source_metadata( self.artifact.source, self.artifact.cache_key, 'build-log') as log: log_name = log.real_filename self.run_commands(builddir, destdir, log) + self.create_devices(destdir) except: - self.do_unmounts(mounted) + self.staging_area.chroot_close() if log_name: with open(log_name) as f: for line in f: logging.error('OUTPUT FROM FAILED BUILD: %s' % line.rstrip('\n')) raise - self.do_unmounts(mounted) + self.staging_area.chroot_close() built_artifacts = self.assemble_chunk_artifacts(destdir) self.save_build_times() return built_artifacts - to_mount = ( - ('proc', 'proc', 'none'), - ('dev/shm', 'tmpfs', 'none'), - ) - - def mount_ccachedir(self): #pragma: no cover - ccache_dir = self.app.settings['compiler-cache-dir'] - if not os.path.isdir(ccache_dir): - os.makedirs(ccache_dir) - # Get a path for the repo's ccache - ccache_url = self.artifact.source.repo.url - ccache_path = urlparse(ccache_url).path - ccache_repobase = os.path.basename(ccache_path) - if ':' in ccache_repobase: # the basename is a repo-alias - resolver = morphlib.repoaliasresolver.RepoAliasResolver( - self.app.settings['repo-alias']) - ccache_url = resolver.pull_url(ccache_repobase) - ccache_path = urlparse(ccache_url).path - ccache_repobase = os.path.basename(ccache_path) - if ccache_repobase.endswith('.git'): - ccache_repobase = ccache_repobase[:-len('.git')] - - ccache_repodir = os.path.join(ccache_dir, ccache_repobase) - # Make sure that directory exists - if not os.path.isdir(ccache_repodir): - os.mkdir(ccache_repodir) - # Get the destination path - ccache_destdir= os.path.join(self.staging_area.tempdir, - 'tmp', 'ccache') - # Make sure that the destination exists. We'll create /tmp if necessary - # to avoid breaking when faced with an empty staging area. - if not os.path.isdir(ccache_destdir): - os.makedirs(ccache_destdir) - # Mount it into the staging-area - self.app.runcmd(['mount', '--bind', ccache_repodir, - ccache_destdir]) - return ccache_destdir - - def do_mounts(self): # pragma: no cover - mounted = [] - if not self.setup_mounts: - return mounted - for mount_point, mount_type, source in ChunkBuilder.to_mount: - logging.debug('Mounting %s in staging area' % mount_point) - path = os.path.join(self.staging_area.dirname, mount_point) - if not os.path.exists(path): - os.makedirs(path) - self.app.runcmd(['mount', '-t', mount_type, source, path]) - mounted.append(path) - if not self.app.settings['no-ccache']: - mounted.append(self.mount_ccachedir()) - return mounted - - def do_unmounts(self, mounted): # pragma: no cover - for path in mounted: - logging.debug('Unmounting %s in staging area' % path) - morphlib.fsutils.unmount(self.app.runcmd, path) - def get_sources(self, srcdir): # pragma: no cover '''Get sources from git to a source directory, for building.''' diff --git a/morphlib/extractedtarball.py b/morphlib/extractedtarball.py new file mode 100644 index 00000000..e435b1ef --- /dev/null +++ b/morphlib/extractedtarball.py @@ -0,0 +1,61 @@ +# Copyright (C) 2012-2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import gzip +import os +import tempfile +import shutil + +import morphlib + + +class ExtractedTarball(object): # pragma: no cover + + '''Tarball extracted in a temporary directory. + + This can be used e.g. to inspect the contents of a rootfs tarball. + + ''' + def __init__(self, app, tarball): + self.app = app + self.tarball = tarball + + def setup(self): + self.app.status(msg='Preparing tarball %(tarball)s', + tarball=os.path.basename(self.tarball), chatty=True) + self.app.status(msg=' Extracting...', chatty=True) + self.tempdir = tempfile.mkdtemp(dir=self.app.settings['tempdir']) + try: + morphlib.bins.unpack_binary(self.tarball, self.tempdir) + except: + shutil.rmtree(self.tempdir) + raise + return self.tempdir + + def cleanup(self): + self.app.status(msg='Cleanup extracted tarball %(tarball)s', + tarball=os.path.basename(self.tarball), chatty=True) + try: + shutil.rmtree(self.tempdir) + except: + pass + + def __enter__(self): + return self.setup() + + def __exit__(self, exctype, excvalue, exctraceback): + self.cleanup() diff --git a/morphlib/morph2.py b/morphlib/morph2.py index a95312c6..ee58ecdc 100644 --- a/morphlib/morph2.py +++ b/morphlib/morph2.py @@ -36,6 +36,7 @@ class Morphology(object): ('build-commands', None), ('test-commands', None), ('install-commands', None), + ('devices', None), ('chunks', []), ('max-jobs', None), ('build-system', 'manual') diff --git a/morphlib/mountableimage.py b/morphlib/mountableimage.py new file mode 100644 index 00000000..3d29a516 --- /dev/null +++ b/morphlib/mountableimage.py @@ -0,0 +1,81 @@ +# Copyright (C) 2012-2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import os +import tempfile +import gzip + +import morphlib + + +class MountableImage(object): # pragma: no cover + + '''Mountable image (deals with decompression). + + Note, this is a read-only mount in the sense that the decompressed + image is not then recompressed after, instead any changes are discarded. + + ''' + def __init__(self, app, artifact_path): + self.app = app + self.artifact_path = artifact_path + + def setup(self, path): + self.app.status(msg='Preparing image %(path)s', path=path, chatty=True) + self.app.status(msg=' Decompressing...', chatty=True) + (tempfd, self.temp_path) = \ + tempfile.mkstemp(dir=self.app.settings['tempdir']) + + try: + with os.fdopen(tempfd, "wb") as outfh: + infh = gzip.open(path, "rb") + morphlib.util.copyfileobj(infh, outfh) + infh.close() + except: + os.unlink(self.temp_path) + raise + self.app.status(msg=' Mounting image at %(path)s', + path=self.temp_path, chatty=True) + part = morphlib.fsutils.setup_device_mapping(self.app.runcmd, + self.temp_path) + mount_point = tempfile.mkdtemp(dir=self.app.settings['tempdir']) + morphlib.fsutils.mount(self.app.runcmd, part, mount_point) + self.mount_point = mount_point + return mount_point + + def cleanup(self, path, mount_point): + self.app.status(msg='Clearing down image at %(path)s', path=path, + chatty=True) + try: + morphlib.fsutils.unmount(self.app.runcmd, mount_point) + except: + pass + try: + morphlib.fsutils.undo_device_mapping(self.app.runcmd, path) + except: + pass + try: + os.rmdir(mount_point) + os.unlink(path) + except: + pass + + def __enter__(self): + return self.setup(self.artifact_path) + + def __exit__(self, exctype, excvalue, exctraceback): + self.cleanup(self.temp_path, self.mount_point) diff --git a/morphlib/plugins/artifact_inspection_plugin.py b/morphlib/plugins/artifact_inspection_plugin.py new file mode 100644 index 00000000..569fbb8a --- /dev/null +++ b/morphlib/plugins/artifact_inspection_plugin.py @@ -0,0 +1,294 @@ +# Copyright (C) 2012-2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import glob +import json +import os +import re + +import morphlib + +from morphlib.bins import call_in_artifact_directory +from morphlib.extractedtarball import ExtractedTarball +from morphlib.mountableimage import MountableImage + + +class NotASystemArtifactError(cliapp.AppException): + + def __init__(self, artifact): + cliapp.AppException.__init__( + self, '%s is not a system artifact' % artifact) + + +class ProjectVersionGuesser(object): + + def __init__(self, app, lrc, rrc, interesting_files): + self.app = app + self.lrc = lrc + self.rrc = rrc + self.interesting_files = interesting_files + + def file_contents(self, repo, ref, tree): + filenames = [x for x in self.interesting_files if x in tree] + if filenames: + if self.lrc.has_repo(repo): + repository = self.lrc.get_repo(repo) + for filename in filenames: + yield filename, repository.cat(ref, filename) + elif self.rrc: + for filename in filenames: + yield filename, self.rrc.cat_file(repo, ref, filename) + + +class AutotoolsVersionGuesser(ProjectVersionGuesser): + + def __init__(self, app, lrc, rrc): + ProjectVersionGuesser.__init__(self, app, lrc, rrc, [ + 'configure.ac', + 'configure.in', + 'configure.ac.in', + 'configure.in.in', + ]) + + def guess_version(self, repo, ref, tree): + version = None + for filename, data in self.file_contents(repo, ref, tree): + # First, try to grep for AC_INIT() + version = self._check_ac_init(data) + if version: + self.app.status( + msg='%(repo)s: Version of %(ref)s detected ' + 'via %(filename)s:AC_INIT: %(version)s', + repo=repo, ref=ref, filename=filename, + version=version, chatty=True) + break + + # Then, try running autoconf against the configure script + version = self._check_autoconf_package_version(filename, data) + if version: + self.app.status( + msg='%(repo)s: Version of %(ref)s detected ' + 'by processing %(filename)s: %(version)s', + repo=repo, ref=ref, filename=filename, + version=version, chatty=True) + break + return version + + def _check_ac_init(self, data): + data = data.replace('\n', ' ') + for macro in ['AC_INIT', 'AM_INIT_AUTOMAKE']: + pattern = r'.*%s\((.*?)\).*' % macro + if not re.match(pattern, data): + continue + acinit = re.sub(pattern, r'\1', data) + if acinit: + version = acinit.split(',') + if macro == 'AM_INIT_AUTOMAKE' and len(version) == 1: + continue + version = version[0] if len(version) == 1 else version[1] + version = re.sub('[\[\]]', '', version).strip() + version = version.split()[0] + if version: + if version and version[0].isdigit(): + return version + return None + + def _check_autoconf_package_version(self, filename, data): + tempdir = morphlib.tempdir.Tempdir(self.app.settings['tempdir']) + with open(tempdir.join(filename), 'w') as f: + f.write(data) + exit_code, output, errors = self.app.runcmd_unchecked( + ['autoconf', filename], + ['grep', '^PACKAGE_VERSION='], + ['cut', '-d=', '-f2'], + ['sed', "s/'//g"], + cwd=tempdir.dirname) + tempdir.remove() + version = None + if output: + output = output.strip() + if output and output[0].isdigit(): + version = output + if exit_code != 0: + self.app.status( + msg='%(repo)s: Failed to detect version from ' + '%(ref)s:%(filename)s', + repo=repo, ref=ref, filename=filename, chatty=True) + return version + + +class VersionGuesser(object): + + def __init__(self, app): + self.app = app + self.lrc, self.rrc = morphlib.util.new_repo_caches(app) + self.guessers = [ + AutotoolsVersionGuesser(app, self.lrc, self.rrc) + ] + + def guess_version(self, repo, ref): + self.app.status(msg='%(repo)s: Guessing version of %(ref)s', + repo=repo, ref=ref, chatty=True) + version = None + try: + if self.lrc.has_repo(repo): + repository = self.lrc.get_repo(repo) + if not self.app.settings['no-git-update']: + repository.update() + tree = repository.ls_tree(ref) + elif self.rrc: + repository = None + tree = self.rrc.ls_tree(repo, ref) + else: + return None + for guesser in self.guessers: + version = guesser.guess_version(repo, ref, tree) + if version: + break + except cliapp.AppException, err: + self.app.status(msg='%(repo)s: Failed to list files in %(ref)s', + repo=repo, ref=ref, chatty=True) + return version + + +class ManifestGenerator(object): + + def __init__(self, app): + self.app = app + self.version_guesser = VersionGuesser(app) + + def generate(self, artifact, dirname): + # Try to find a directory with baserock metadata files. + metadirs = [ + os.path.join(dirname, 'factory', 'baserock'), + os.path.join(dirname, 'baserock') + ] + existing_metadirs = [x for x in metadirs if os.path.isdir(x)] + if not existing_metadirs: + raise NotASystemArtifactError(artifact) + metadir = existing_metadirs[0] + + # Collect all meta information about the system, its strata + # and its chunks that we are interested in. + artifacts = [] + for basename in glob.glob(os.path.join(metadir, '*.meta')): + metafile = os.path.join(metadir, basename) + metadata = json.load(open(metafile)) + + # Try to guess the version of this artifact + version = self.version_guesser.guess_version( + metadata['repo'], metadata['sha1']) or '-' + + artifacts.append({ + 'cache-key': metadata['cache-key'], + 'name': metadata['artifact-name'], + 'kind': metadata['kind'], + 'sha1': metadata['sha1'], + 'original_ref': metadata['original_ref'], + 'repo': metadata['repo'], + 'morphology': metadata['morphology'], + 'version': version, + }) + + # Generate a format string for dumping the information. + fmt = self._generate_output_format(artifacts) + + # Print information about system, strata and chunks. + self._print_artifacts(fmt, artifacts, 'system') + self._print_artifacts(fmt, artifacts, 'stratum') + self._print_artifacts(fmt, artifacts, 'chunk') + + def _generate_output_format(self, artifacts): + colwidths = {} + for artifact in artifacts: + for key, value in artifact.iteritems(): + colwidths[key] = max(colwidths.get(key, 0), len(value)) + + colwidths['first'] = sum([colwidths['cache-key'], + colwidths['kind'], + colwidths['name']]) + 1 + + return 'artifact=%%-%is\t' \ + 'version=%%-%is\t' \ + 'commit=%%-%is\t' \ + 'repository=%%-%is\t' \ + 'ref=%%-%is\t' \ + 'morphology=%%-%is\n' % ( + len('artifact=') + colwidths['first'], + len('version=') + colwidths['version'], + len('commit=') + colwidths['sha1'], + len('repository=') + colwidths['repo'], + len('ref=') + colwidths['original_ref'], + len('morphology=') + colwidths['morphology']) + + def _print_artifacts(self, fmt, artifacts, kind): + for artifact in sorted(artifacts, key=lambda x: x['name']): + if artifact['kind'] == kind: + self.app.output.write(fmt % ( + '%s.%s.%s' % (artifact['cache-key'], + artifact['kind'], + artifact['name']), + artifact['version'], + artifact['sha1'], + artifact['repo'], + artifact['original_ref'], + artifact['morphology'])) + + +class ArtifactInspectionPlugin(cliapp.Plugin): + + def enable(self): + self.app.add_subcommand('run-in-artifact', + self.run_in_artifact, + arg_synopsis='ARTIFACT CMD') + self.app.add_subcommand('generate-manifest', + self.generate_manifest, + arg_synopsis='ROOTFS_ARTIFACT') + + def disable(self): + pass + + def run_in_artifact(self, args): + '''Run a command inside an extracted/mounted chunk or system.''' + + if len(args) < 2: + raise cliapp.AppException( + 'run-in-artifact requires arguments: a chunk or ' + 'system artifact and a a shell command') + + artifact, cmd = args[0], args[1:] + + def run_command_in_dir(dirname): + output = self.app.runcmd(cmd, cwd=dirname) + self.app.output.write(output) + + call_in_artifact_directory(self.app, artifact, run_command_in_dir) + + def generate_manifest(self, args): + '''Generate a content manifest for a system image.''' + + if len(args) != 1: + raise cliapp.AppException('morph generate-manifest expects ' + 'a system image as its input') + + artifact = args[0] + + def generate_manifest(dirname): + generator = ManifestGenerator(self.app) + generator.generate(artifact, dirname) + + call_in_artifact_directory(self.app, artifact, generate_manifest) diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index f9595f98..cbc41acf 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -348,6 +348,7 @@ class BranchAndMergePlugin(cliapp.Plugin): 'install-commands', 'max-jobs', 'chunks', + 'devices', ] } diff --git a/morphlib/plugins/trebuchet_plugin.py b/morphlib/plugins/trebuchet_plugin.py index 2bdf4c3c..1ebffbf4 100644 --- a/morphlib/plugins/trebuchet_plugin.py +++ b/morphlib/plugins/trebuchet_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -21,62 +21,8 @@ import gzip import morphlib -class MountableImage(object): - '''Mountable image (deals with decompression). - - Note, this is a read-only mount in the sense that the decompressed - image is not then recompressed after, instead any changes are discarded. - - ''' - def __init__(self, app, artifact_path): - self.app = app - self.artifact_path = artifact_path - - def setup(self, path): - self.app.status(msg='Preparing image %(path)s', path=path) - self.app.status(msg=' Decompressing...', chatty=True) - (tempfd, self.temp_path) = \ - tempfile.mkstemp(dir=self.app.settings['tempdir']) - - try: - with os.fdopen(tempfd, "wb") as outfh: - infh = gzip.open(path, "rb") - morphlib.util.copyfileobj(infh, outfh) - infh.close() - except: - os.unlink(self.temp_path) - raise - self.app.status(msg=' Mounting image at %(path)s', - path=self.temp_path, chatty=True) - part = morphlib.fsutils.setup_device_mapping(self.app.runcmd, - self.temp_path) - mount_point = tempfile.mkdtemp(dir=self.app.settings['tempdir']) - morphlib.fsutils.mount(self.app.runcmd, part, mount_point) - self.mount_point = mount_point - return mount_point - - def cleanup(self, path, mount_point): - self.app.status(msg='Clearing down image at %(path)s', path=path, - chatty=True) - try: - morphlib.fsutils.unmount(self.app.runcmd, mount_point) - except: - pass - try: - morphlib.fsutils.undo_device_mapping(self.app.runcmd, path) - except: - pass - try: - os.rmdir(mount_point) - os.unlink(path) - except: - pass - - def __enter__(self): - return self.setup(self.artifact_path) - - def __exit__(self, exctype, excvalue, exctraceback): - self.cleanup(self.temp_path, self.mount_point) +from morphlib.mountableimage import MountableImage + class TrebuchetPlugin(cliapp.Plugin): diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index a87b45c3..f930f9d7 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,6 +17,8 @@ import logging import os import shutil +import stat +from urlparse import urlparse import morphlib @@ -40,6 +42,10 @@ class StagingArea(object): self._app = app self.dirname = dirname self.tempdir = tempdir + self.builddirname = None + self.destdirname = None + self.mounted = None + self._bind_readonly_mount = None # Wrapper to be overridden by unit tests. def _mkdir(self, dirname): # pragma: no cover @@ -80,6 +86,52 @@ class StagingArea(object): assert filename.startswith(dirname) return filename[len(dirname) - 1:] # include leading slash + def hardlink_all_files(self, srcpath, destpath): # pragma: no cover + '''Hardlink every file in the path to the staging-area + + If an exception is raised, the staging-area is indeterminate. + + ''' + + file_stat = os.lstat(srcpath) + mode = file_stat.st_mode + + if stat.S_ISDIR(mode): + # Ensure directory exists in destination, then recurse. + if not os.path.exists(destpath): + os.makedirs(destpath) + dest_stat = os.stat(os.path.realpath(destpath)) + if not stat.S_ISDIR(dest_stat.st_mode): + raise IOError('Destination not a directory. source has %s' + ' destination has %s' % (srcpath, destpath)) + + for entry in os.listdir(srcpath): + self.hardlink_all_files(os.path.join(srcpath, entry), + os.path.join(destpath, entry)) + elif stat.S_ISLNK(mode): + # Copy the symlink. + if os.path.exists(destpath): + os.remove(destpath) + os.symlink(os.readlink(srcpath), destpath) + + elif stat.S_ISREG(mode): + # Hardlink the file. + if os.path.exists(destpath): + os.remove(destpath) + os.link(srcpath, destpath) + + elif stat.S_ISCHR(mode) or stat.S_ISBLK(mode): + # Block or character device. Put contents of st_dev in a mknod. + if os.path.exists(destpath): + os.remove(destpath) + os.mknod(destpath, file_stat.st_mode, file_stat.st_rdev) + os.chmod(destpath, file_stat.st_mode) + + else: + # Unsupported type. + raise IOError('Cannot extract %s into staging-area. Unsupported' + ' type.' % srcpath) + def install_artifact(self, handle): '''Install a build artifact into the staging area. @@ -90,7 +142,19 @@ class StagingArea(object): logging.debug('Installing artifact %s' % getattr(handle, 'name', 'unknown name')) - morphlib.bins.unpack_binary_from_file(handle, self.dirname) + + unpacked_artifact = os.path.join( + self._app.settings['tempdir'], + os.path.basename(handle.name) + '.d') + if not os.path.exists(unpacked_artifact): + self._mkdir(unpacked_artifact) + morphlib.bins.unpack_binary_from_file( + handle, unpacked_artifact + '/') + + if not os.path.exists(self.dirname): + self._mkdir(self.dirname) + + self.hardlink_all_files(unpacked_artifact, self.dirname) def remove(self): '''Remove the entire staging area. @@ -103,14 +167,114 @@ class StagingArea(object): shutil.rmtree(self.dirname) + to_mount = ( + ('proc', 'proc', 'none'), + ('dev/shm', 'tmpfs', 'none'), + ) + + def mount_ccachedir(self, source): #pragma: no cover + ccache_dir = self._app.settings['compiler-cache-dir'] + if not os.path.isdir(ccache_dir): + os.makedirs(ccache_dir) + # Get a path for the repo's ccache + ccache_url = source.repo.url + ccache_path = urlparse(ccache_url).path + ccache_repobase = os.path.basename(ccache_path) + if ':' in ccache_repobase: # the basename is a repo-alias + resolver = morphlib.repoaliasresolver.RepoAliasResolver( + self._app.settings['repo-alias']) + ccache_url = resolver.pull_url(ccache_repobase) + ccache_path = urlparse(ccache_url).path + ccache_repobase = os.path.basename(ccache_path) + if ccache_repobase.endswith('.git'): + ccache_repobase = ccache_repobase[:-len('.git')] + + ccache_repodir = os.path.join(ccache_dir, ccache_repobase) + # Make sure that directory exists + if not os.path.isdir(ccache_repodir): + os.mkdir(ccache_repodir) + # Get the destination path + ccache_destdir= os.path.join(self.tempdir, + 'tmp', 'ccache') + # Make sure that the destination exists. We'll create /tmp if necessary + # to avoid breaking when faced with an empty staging area. + if not os.path.isdir(ccache_destdir): + os.makedirs(ccache_destdir) + # Mount it into the staging-area + self._app.runcmd(['mount', '--bind', ccache_repodir, + ccache_destdir]) + return ccache_destdir + + def do_mounts(self, setup_mounts): # pragma: no cover + self.mounted = [] + if not setup_mounts: + return + for mount_point, mount_type, source in self.to_mount: + logging.debug('Mounting %s in staging area' % mount_point) + path = os.path.join(self.dirname, mount_point) + if not os.path.exists(path): + os.makedirs(path) + self._app.runcmd(['mount', '-t', mount_type, source, path]) + self.mounted.append(path) + return + + def do_unmounts(self): # pragma: no cover + for path in reversed(self.mounted): + logging.debug('Unmounting %s in staging area' % path) + morphlib.fsutils.unmount(self._app.runcmd, path) + + def chroot_open(self, source, setup_mounts): # pragma: no cover + '''Setup staging area for use as a chroot.''' + + assert self.builddirname == None and self.destdirname == None + + builddir = self.builddir(source) + destdir = self.destdir(source) + self.builddirname = self.relative(builddir).lstrip('/') + self.destdirname = self.relative(destdir).lstrip('/') + + self.do_mounts(setup_mounts) + + if not self._app.settings['no-ccache']: + self.mounted.append(self.mount_ccachedir(source)) + + return builddir, destdir + + def chroot_close(self): # pragma: no cover + '''Undo changes by chroot_open. + + This should be called after the staging area is no longer needed. + + ''' + + self.do_unmounts() + def runcmd(self, argv, **kwargs): # pragma: no cover '''Run a command in a chroot in the staging area.''' + cwd = kwargs.get('cwd') or '/' if 'cwd' in kwargs: cwd = kwargs['cwd'] del kwargs['cwd'] else: cwd = '/' - real_argv = ['chroot', self.dirname, 'sh', '-c', - 'cd "$1" && shift && exec "$@"', '--', cwd] + argv + if self._app.settings['staging-chroot']: + not_readonly_dirs = [self.builddirname, self.destdirname, + 'dev', 'proc', 'tmp'] + dirs = os.listdir(self.dirname) + for excluded_dir in not_readonly_dirs: + dirs.remove(excluded_dir) + + real_argv = ['linux-user-chroot'] + + for entry in dirs: + real_argv += ['--mount-readonly', '/'+entry] + + real_argv += [self.dirname] + else: + real_argv = ['chroot', '/'] + + real_argv += ['sh', '-c', 'cd "$1" && shift && exec "$@"', '--', cwd] + real_argv += argv + return self._app.runcmd(real_argv, **kwargs) diff --git a/morphlib/stagingarea_tests.py b/morphlib/stagingarea_tests.py index 3230b9e3..313226d2 100644 --- a/morphlib/stagingarea_tests.py +++ b/morphlib/stagingarea_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,6 +14,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import cliapp import os import shutil import tarfile @@ -31,14 +32,33 @@ class FakeSource(object): } +class FakeApplication(object): + + def __init__(self, cachedir, tempdir): + self.settings = { + 'cachedir': cachedir, + 'tempdir': tempdir, + } + + def runcmd(self, *args, **kwargs): + return cliapp.runcmd(*args, **kwargs) + + def runcmd_unchecked(self, *args, **kwargs): + return cliapp.runcmd_unchecked(*args, **kwargs) + + class StagingAreaTests(unittest.TestCase): def setUp(self): self.tempdir = tempfile.mkdtemp() + self.cachedir = os.path.join(self.tempdir, 'cachedir') + os.mkdir(self.cachedir) + os.mkdir(os.path.join(self.cachedir, 'artifacts')) self.staging = os.path.join(self.tempdir, 'staging') self.created_dirs = [] - self.sa = morphlib.stagingarea.StagingArea(object(), self.staging, - self.staging) + self.sa = morphlib.stagingarea.StagingArea( + FakeApplication(self.cachedir, self.tempdir), + self.staging, self.staging) def tearDown(self): shutil.rmtree(self.tempdir) diff --git a/tests.as-root/run-in-artifact-propagates-exit-code.exit b/tests.as-root/run-in-artifact-propagates-exit-code.exit new file mode 100644 index 00000000..d00491fd --- /dev/null +++ b/tests.as-root/run-in-artifact-propagates-exit-code.exit @@ -0,0 +1 @@ +1 diff --git a/tests.as-root/run-in-artifact-propagates-exit-code.script b/tests.as-root/run-in-artifact-propagates-exit-code.script new file mode 100755 index 00000000..d815c73d --- /dev/null +++ b/tests.as-root/run-in-artifact-propagates-exit-code.script @@ -0,0 +1,33 @@ +#!/bin/bash +# +# Copyright (C) 2012-2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +## Test that 'run-in-artifact' propagates the exit code of its command. + +set -eu + +. "$SRCDIR/tests.as-root/lib" + +# Build first image. Remember the stratum. +"$SRCDIR/scripts/test-morph" build-morphology \ + test:morphs master linux-system + +system=$(find "$DATADIR/cache/artifacts" -maxdepth 1 -name '*.system.*-rootfs') + +# Run 'run-in-artifact' with the system artifact. The command will fail +# and this should result in an exit code of 1 in the test. +"$SRCDIR/scripts/test-morph" run-in-artifact "$system" -- ls i-do-not-exist diff --git a/tests.as-root/run-in-artifact-propagates-exit-code.stderr b/tests.as-root/run-in-artifact-propagates-exit-code.stderr new file mode 100644 index 00000000..98aa5450 --- /dev/null +++ b/tests.as-root/run-in-artifact-propagates-exit-code.stderr @@ -0,0 +1,3 @@ +ERROR: Command failed: ls i-do-not-exist +ls: i-do-not-exist: No such file or directory + diff --git a/tests.as-root/run-in-artifact-with-different-artifacts.script b/tests.as-root/run-in-artifact-with-different-artifacts.script new file mode 100755 index 00000000..0b14f527 --- /dev/null +++ b/tests.as-root/run-in-artifact-with-different-artifacts.script @@ -0,0 +1,47 @@ +#!/bin/bash +# +# Copyright (C) 2012-2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +## Test the 'run-in-artifact' command with different types of artifacts. + +set -eu + +. "$SRCDIR/tests.as-root/lib" + +# Build first image. Remember the stratum. +"$SRCDIR/scripts/test-morph" build-morphology \ + test:morphs master linux-system + +system=$(find "$DATADIR/cache/artifacts" -maxdepth 1 -name '*.system.*-rootfs') +chunk=$(find "$DATADIR/cache/artifacts" -maxdepth 1 -name '*.chunk.linux') +stratum=$(find "$DATADIR/cache/artifacts" -maxdepth 1 \ + -name '*.stratum.linux-stratum') + +# Run 'run-in-artifact' with the system artifact. +echo "System:" +"$SRCDIR/scripts/test-morph" run-in-artifact "$system" -- ls factory/baserock/ +echo + +# Run 'run-in-artifact' with the chunk artifact. +echo "Chunk:" +"$SRCDIR/scripts/test-morph" run-in-artifact "$chunk" -- ls baserock/ +echo + +# Run 'run-in-artifact' with the statum artifact. +echo "Stratum:" +"$SRCDIR/scripts/test-morph" run-in-artifact "$stratum" -- ls baserock/ \ + || echo "Failed" diff --git a/tests.as-root/run-in-artifact-with-different-artifacts.stderr b/tests.as-root/run-in-artifact-with-different-artifacts.stderr new file mode 100644 index 00000000..44e70c38 --- /dev/null +++ b/tests.as-root/run-in-artifact-with-different-artifacts.stderr @@ -0,0 +1 @@ +ERROR: Artifact TMP/cache/artifacts/293fc1b78dd2af221ae7de246ff5a59df476165704b7e366230ac8ed4c16d1b7.stratum.linux-stratum cannot be extracted or mounted diff --git a/tests.as-root/run-in-artifact-with-different-artifacts.stdout b/tests.as-root/run-in-artifact-with-different-artifacts.stdout new file mode 100644 index 00000000..281ab109 --- /dev/null +++ b/tests.as-root/run-in-artifact-with-different-artifacts.stdout @@ -0,0 +1,14 @@ +System: +hello-stratum.meta +hello.meta +linux-stratum.meta +linux-system-rootfs.meta +linux.meta +tools-stratum.meta +tools.meta + +Chunk: +linux.meta + +Stratum: +Failed diff --git a/without-test-modules b/without-test-modules index 5bcdb025..cc901f32 100644 --- a/without-test-modules +++ b/without-test-modules @@ -5,6 +5,9 @@ morphlib/tester.py morphlib/git.py morphlib/fsutils.py morphlib/app.py +morphlib/mountableimage.py +morphlib/extractedtarball.py +morphlib/plugins/artifact_inspection_plugin.py morphlib/plugins/hello_plugin.py morphlib/plugins/graphing_plugin.py morphlib/plugins/syslinux-disk-systembuilder_plugin.py |