Merge branch 'master' of git://git.baserock.org/baserock/baserock/morph

18 files changed, 794 insertions, 129 deletions

diff --git a/morphlib/__init__.py b/morphlib/__init__.py
index 271fa977..b20c3f01 100644
--- a/morphlib/__init__.py
+++ b/morphlib/__init__.py
@@ -53,10 +53,12 @@ import builder2
 import cachedir
 import cachedrepo
 import cachekeycomputer
+import extractedtarball
 import fsutils
 import git
 import localartifactcache
 import localrepocache
+import mountableimage
 import morph2
 import morphologyfactory
 import remoteartifactcache
diff --git a/morphlib/bins.py b/morphlib/bins.py
index 622aa165..ba5f713f 100644
--- a/morphlib/bins.py
+++ b/morphlib/bins.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2011-2012  Codethink Limited
+# Copyright (C) 2011-2013  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -21,6 +21,7 @@ Binaries are chunks, strata, and system images.
 '''
 
 
+import cliapp
 import logging
 import os
 import re
@@ -29,6 +30,11 @@ import stat
 import shutil
 import tarfile
 
+import morphlib
+
+from morphlib.extractedtarball import ExtractedTarball
+from morphlib.mountableimage import MountableImage
+
 
 # Work around http://bugs.python.org/issue16477
 def safe_makefile(self, tarinfo, targetpath):
@@ -211,3 +217,24 @@ def unpack_binary_from_file(f, dirname):  # pragma: no cover
 def unpack_binary(filename, dirname):
     with open(filename, "rb") as f:
         unpack_binary_from_file(f, dirname)
+
+
+class ArtifactNotMountableError(cliapp.AppException): # pragma: no cover
+
+    def __init__(self, filename):
+        cliapp.AppException.__init__(
+                self, 'Artifact %s cannot be extracted or mounted' % filename)
+
+
+def call_in_artifact_directory(app, filename, callback): # pragma: no cover
+    '''Call a function in a directory the artifact is extracted/mounted in.'''
+
+    try:
+        with ExtractedTarball(app, filename) as dirname:
+            callback(dirname)
+    except tarfile.TarError:
+        try:
+            with MountableImage(app, filename) as dirname:
+                callback(dirname)
+        except (IOError, OSError):
+            raise ArtifactNotMountableError(filename)
diff --git a/morphlib/builder2.py b/morphlib/builder2.py
index 68768699..82a95820 100644
--- a/morphlib/builder2.py
+++ b/morphlib/builder2.py
@@ -20,6 +20,7 @@ import json
 import logging
 import os
 import shutil
+import stat
 import time
 from collections import defaultdict
 import tarfile
@@ -27,7 +28,6 @@ import traceback
 import subprocess
 import tempfile
 import gzip
-from urlparse import urlparse
 
 import cliapp
 
@@ -266,91 +266,57 @@ class ChunkBuilder(BuilderBase):
         else:
             return morphology[which]
 
+    def create_devices(self, destdir): # pragma: no cover
+        '''Creates device nodes if the morphology specifies them'''
+        morphology = self.artifact.source.morphology
+        perms_mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO
+        if 'devices' in morphology and morphology['devices'] is not None:
+            for dev in morphology['devices']:
+                destfile = os.path.join(destdir, './' + dev['filename'])
+                mode = int(dev['permissions'], 8) & perms_mask
+                if dev['type'] == 'c':
+                    mode = mode | stat.S_IFCHR
+                elif dev['type'] == 'b':
+                    mode = mode | stat.S_IFBLK
+                else:
+                    raise IOError('Cannot create device node %s,'
+                                  'unrecognized device type "%s"'
+                                  % (destfile, dev['type']))
+                self.app.status(msg="Creating device node %s"
+                                    % destfile)
+                os.mknod(destfile, mode,
+                         os.makedev(dev['major'], dev['minor']))
+                os.chown(destfile, dev['uid'], dev['gid'])
+
     def build_and_cache(self):  # pragma: no cover
         with self.build_watch('overall-build'):
-            mounted = self.do_mounts()
+
+            builddir, destdir = \
+                self.staging_area.chroot_open(self.artifact.source,
+                                              self.setup_mounts)
             log_name = None
             try:
-                builddir = self.staging_area.builddir(self.artifact.source)
                 self.get_sources(builddir)
-                destdir = self.staging_area.destdir(self.artifact.source)
                 with self.local_artifact_cache.put_source_metadata(
                         self.artifact.source, self.artifact.cache_key,
                         'build-log') as log:
                     log_name = log.real_filename
                     self.run_commands(builddir, destdir, log)
+                    self.create_devices(destdir)
             except:
-                self.do_unmounts(mounted)
+                self.staging_area.chroot_close()
                 if log_name:
                     with open(log_name) as f:
                         for line in f:
                             logging.error('OUTPUT FROM FAILED BUILD: %s' %
                                           line.rstrip('\n'))
                 raise
-            self.do_unmounts(mounted)
+            self.staging_area.chroot_close()
             built_artifacts = self.assemble_chunk_artifacts(destdir)
 
         self.save_build_times()
         return built_artifacts
 
-    to_mount = (
-        ('proc',    'proc',  'none'),
-        ('dev/shm', 'tmpfs', 'none'),
-    )
-
-    def mount_ccachedir(self): #pragma: no cover
-        ccache_dir = self.app.settings['compiler-cache-dir']
-        if not os.path.isdir(ccache_dir):
-            os.makedirs(ccache_dir)
-        # Get a path for the repo's ccache
-        ccache_url = self.artifact.source.repo.url
-        ccache_path = urlparse(ccache_url).path
-        ccache_repobase = os.path.basename(ccache_path)
-        if ':' in ccache_repobase: # the basename is a repo-alias
-            resolver = morphlib.repoaliasresolver.RepoAliasResolver(
-                self.app.settings['repo-alias'])
-            ccache_url = resolver.pull_url(ccache_repobase)
-            ccache_path = urlparse(ccache_url).path
-            ccache_repobase = os.path.basename(ccache_path)
-        if ccache_repobase.endswith('.git'):
-            ccache_repobase = ccache_repobase[:-len('.git')]
-
-        ccache_repodir = os.path.join(ccache_dir, ccache_repobase)
-        # Make sure that directory exists
-        if not os.path.isdir(ccache_repodir):
-            os.mkdir(ccache_repodir)
-        # Get the destination path
-        ccache_destdir= os.path.join(self.staging_area.tempdir,
-                                     'tmp', 'ccache')
-        # Make sure that the destination exists. We'll create /tmp if necessary
-        # to avoid breaking when faced with an empty staging area.
-        if not os.path.isdir(ccache_destdir):
-            os.makedirs(ccache_destdir)
-        # Mount it into the staging-area
-        self.app.runcmd(['mount', '--bind', ccache_repodir,
-                         ccache_destdir])
-        return ccache_destdir
-
-    def do_mounts(self):  # pragma: no cover
-        mounted = []
-        if not self.setup_mounts:
-            return mounted
-        for mount_point, mount_type, source in ChunkBuilder.to_mount:
-            logging.debug('Mounting %s in staging area' % mount_point)
-            path = os.path.join(self.staging_area.dirname, mount_point)
-            if not os.path.exists(path):
-                os.makedirs(path)
-            self.app.runcmd(['mount', '-t', mount_type, source, path])
-            mounted.append(path)
-        if not self.app.settings['no-ccache']:
-            mounted.append(self.mount_ccachedir())
-        return mounted
-
-    def do_unmounts(self, mounted):  # pragma: no cover
-        for path in mounted:
-            logging.debug('Unmounting %s in staging area' % path)
-            morphlib.fsutils.unmount(self.app.runcmd, path)
-
     def get_sources(self, srcdir):  # pragma: no cover
         '''Get sources from git to a source directory, for building.'''
 
diff --git a/morphlib/extractedtarball.py b/morphlib/extractedtarball.py
new file mode 100644
index 00000000..e435b1ef
--- /dev/null
+++ b/morphlib/extractedtarball.py
@@ -0,0 +1,61 @@
+# Copyright (C) 2012-2013  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import cliapp
+import gzip
+import os
+import tempfile
+import shutil
+
+import morphlib
+
+
+class ExtractedTarball(object): # pragma: no cover
+
+    '''Tarball extracted in a temporary directory.
+
+    This can be used e.g. to inspect the contents of a rootfs tarball.
+
+    '''
+    def __init__(self, app, tarball):
+        self.app = app
+        self.tarball = tarball
+
+    def setup(self):
+        self.app.status(msg='Preparing tarball %(tarball)s',
+                        tarball=os.path.basename(self.tarball), chatty=True)
+        self.app.status(msg='  Extracting...', chatty=True)
+        self.tempdir = tempfile.mkdtemp(dir=self.app.settings['tempdir'])
+        try:
+            morphlib.bins.unpack_binary(self.tarball, self.tempdir)
+        except:
+            shutil.rmtree(self.tempdir)
+            raise
+        return self.tempdir
+
+    def cleanup(self):
+        self.app.status(msg='Cleanup extracted tarball %(tarball)s',
+                        tarball=os.path.basename(self.tarball), chatty=True)
+        try:
+            shutil.rmtree(self.tempdir)
+        except:
+            pass
+
+    def __enter__(self):
+        return self.setup()
+
+    def __exit__(self, exctype, excvalue, exctraceback):
+        self.cleanup()
diff --git a/morphlib/morph2.py b/morphlib/morph2.py
index a95312c6..ee58ecdc 100644
--- a/morphlib/morph2.py
+++ b/morphlib/morph2.py
@@ -36,6 +36,7 @@ class Morphology(object):
             ('build-commands', None),
             ('test-commands', None),
             ('install-commands', None),
+            ('devices', None),
             ('chunks', []),
             ('max-jobs', None),
             ('build-system', 'manual')
diff --git a/morphlib/mountableimage.py b/morphlib/mountableimage.py
new file mode 100644
index 00000000..3d29a516
--- /dev/null
+++ b/morphlib/mountableimage.py
@@ -0,0 +1,81 @@
+# Copyright (C) 2012-2013  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import cliapp
+import os
+import tempfile
+import gzip
+
+import morphlib
+
+
+class MountableImage(object): # pragma: no cover
+
+    '''Mountable image (deals with decompression).
+
+    Note, this is a read-only mount in the sense that the decompressed
+    image is not then recompressed after, instead any changes are discarded.
+
+    '''
+    def __init__(self, app, artifact_path):
+        self.app = app
+        self.artifact_path = artifact_path
+
+    def setup(self, path):
+        self.app.status(msg='Preparing image %(path)s', path=path, chatty=True)
+        self.app.status(msg='  Decompressing...', chatty=True)
+        (tempfd, self.temp_path) = \
+            tempfile.mkstemp(dir=self.app.settings['tempdir'])
+
+        try:
+            with os.fdopen(tempfd, "wb") as outfh:
+                infh = gzip.open(path, "rb")
+                morphlib.util.copyfileobj(infh, outfh)
+                infh.close()
+        except:
+            os.unlink(self.temp_path)
+            raise
+        self.app.status(msg='  Mounting image at %(path)s',
+                        path=self.temp_path, chatty=True)
+        part = morphlib.fsutils.setup_device_mapping(self.app.runcmd,
+                                                     self.temp_path)
+        mount_point = tempfile.mkdtemp(dir=self.app.settings['tempdir'])
+        morphlib.fsutils.mount(self.app.runcmd, part, mount_point)
+        self.mount_point = mount_point
+        return mount_point
+
+    def cleanup(self, path, mount_point):
+        self.app.status(msg='Clearing down image at %(path)s', path=path,
+                        chatty=True)
+        try:
+            morphlib.fsutils.unmount(self.app.runcmd, mount_point)
+        except:
+            pass
+        try:
+            morphlib.fsutils.undo_device_mapping(self.app.runcmd, path)
+        except:
+            pass
+        try:
+            os.rmdir(mount_point)
+            os.unlink(path)
+        except:
+            pass
+
+    def __enter__(self):
+        return self.setup(self.artifact_path)
+
+    def __exit__(self, exctype, excvalue, exctraceback):
+        self.cleanup(self.temp_path, self.mount_point)
diff --git a/morphlib/plugins/artifact_inspection_plugin.py b/morphlib/plugins/artifact_inspection_plugin.py
new file mode 100644
index 00000000..569fbb8a
--- /dev/null
+++ b/morphlib/plugins/artifact_inspection_plugin.py
@@ -0,0 +1,294 @@
+# Copyright (C) 2012-2013  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import cliapp
+import glob
+import json
+import os
+import re
+
+import morphlib
+
+from morphlib.bins import call_in_artifact_directory
+from morphlib.extractedtarball import ExtractedTarball
+from morphlib.mountableimage import MountableImage
+
+
+class NotASystemArtifactError(cliapp.AppException):
+
+    def __init__(self, artifact):
+        cliapp.AppException.__init__(
+                self, '%s is not a system artifact' % artifact)
+
+
+class ProjectVersionGuesser(object):
+
+    def __init__(self, app, lrc, rrc, interesting_files):
+        self.app = app
+        self.lrc = lrc
+        self.rrc = rrc
+        self.interesting_files = interesting_files
+
+    def file_contents(self, repo, ref, tree):
+        filenames = [x for x in self.interesting_files if x in tree]
+        if filenames:
+            if self.lrc.has_repo(repo):
+                repository = self.lrc.get_repo(repo)
+                for filename in filenames:
+                    yield filename, repository.cat(ref, filename)
+            elif self.rrc:
+                for filename in filenames:
+                    yield filename, self.rrc.cat_file(repo, ref, filename)
+
+
+class AutotoolsVersionGuesser(ProjectVersionGuesser):
+
+    def __init__(self, app, lrc, rrc):
+        ProjectVersionGuesser.__init__(self, app, lrc, rrc, [
+            'configure.ac',
+            'configure.in',
+            'configure.ac.in',
+            'configure.in.in',
+        ])
+
+    def guess_version(self, repo, ref, tree):
+        version = None
+        for filename, data in self.file_contents(repo, ref, tree):
+            # First, try to grep for AC_INIT()
+            version = self._check_ac_init(data)
+            if version:
+                self.app.status(
+                        msg='%(repo)s: Version of %(ref)s detected '
+                            'via %(filename)s:AC_INIT: %(version)s',
+                        repo=repo, ref=ref, filename=filename,
+                        version=version, chatty=True)
+                break
+
+            # Then, try running autoconf against the configure script
+            version = self._check_autoconf_package_version(filename, data)
+            if version:
+                self.app.status(
+                        msg='%(repo)s: Version of %(ref)s detected '
+                            'by processing %(filename)s: %(version)s',
+                        repo=repo, ref=ref, filename=filename,
+                        version=version, chatty=True)
+                break
+        return version
+
+    def _check_ac_init(self, data):
+        data = data.replace('\n', ' ')
+        for macro in ['AC_INIT', 'AM_INIT_AUTOMAKE']:
+            pattern = r'.*%s\((.*?)\).*' % macro
+            if not re.match(pattern, data):
+                continue
+            acinit = re.sub(pattern, r'\1', data)
+            if acinit:
+                version = acinit.split(',')
+                if macro == 'AM_INIT_AUTOMAKE' and len(version) == 1:
+                    continue
+                version = version[0] if len(version) == 1 else version[1]
+                version = re.sub('[\[\]]', '', version).strip()
+                version = version.split()[0]
+                if version:
+                    if version and version[0].isdigit():
+                        return version
+        return None
+
+    def _check_autoconf_package_version(self, filename, data):
+        tempdir = morphlib.tempdir.Tempdir(self.app.settings['tempdir'])
+        with open(tempdir.join(filename), 'w') as f:
+            f.write(data)
+        exit_code, output, errors = self.app.runcmd_unchecked(
+                ['autoconf', filename],
+                ['grep', '^PACKAGE_VERSION='],
+                ['cut', '-d=', '-f2'],
+                ['sed', "s/'//g"],
+                cwd=tempdir.dirname)
+        tempdir.remove()
+        version = None
+        if output:
+            output = output.strip()
+            if output and output[0].isdigit():
+                version = output
+        if exit_code != 0:
+            self.app.status(
+                    msg='%(repo)s: Failed to detect version from '
+                        '%(ref)s:%(filename)s',
+                    repo=repo, ref=ref, filename=filename, chatty=True)
+        return version
+
+
+class VersionGuesser(object):
+
+    def __init__(self, app):
+        self.app = app
+        self.lrc, self.rrc = morphlib.util.new_repo_caches(app)
+        self.guessers = [
+            AutotoolsVersionGuesser(app, self.lrc, self.rrc)
+        ]
+
+    def guess_version(self, repo, ref):
+        self.app.status(msg='%(repo)s: Guessing version of %(ref)s',
+                        repo=repo, ref=ref, chatty=True)
+        version = None
+        try:
+            if self.lrc.has_repo(repo):
+                repository = self.lrc.get_repo(repo)
+                if not self.app.settings['no-git-update']:
+                    repository.update()
+                tree = repository.ls_tree(ref)
+            elif self.rrc:
+                repository = None
+                tree = self.rrc.ls_tree(repo, ref)
+            else:
+                return None
+            for guesser in self.guessers:
+                version = guesser.guess_version(repo, ref, tree)
+                if version:
+                    break
+        except cliapp.AppException, err:
+            self.app.status(msg='%(repo)s: Failed to list files in %(ref)s',
+                            repo=repo, ref=ref, chatty=True)
+        return version
+
+
+class ManifestGenerator(object):
+
+    def __init__(self, app):
+        self.app = app
+        self.version_guesser = VersionGuesser(app)
+
+    def generate(self, artifact, dirname):
+        # Try to find a directory with baserock metadata files.
+        metadirs = [
+            os.path.join(dirname, 'factory', 'baserock'),
+            os.path.join(dirname, 'baserock')
+        ]
+        existing_metadirs = [x for x in metadirs if os.path.isdir(x)]
+        if not existing_metadirs:
+            raise NotASystemArtifactError(artifact)
+        metadir = existing_metadirs[0]
+
+        # Collect all meta information about the system, its strata
+        # and its chunks that we are interested in.
+        artifacts = []
+        for basename in glob.glob(os.path.join(metadir, '*.meta')):
+            metafile = os.path.join(metadir, basename)
+            metadata = json.load(open(metafile))
+
+            # Try to guess the version of this artifact
+            version = self.version_guesser.guess_version(
+                    metadata['repo'], metadata['sha1']) or '-'
+        
+            artifacts.append({
+                'cache-key': metadata['cache-key'],
+                'name': metadata['artifact-name'],
+                'kind': metadata['kind'],
+                'sha1': metadata['sha1'],
+                'original_ref': metadata['original_ref'],
+                'repo': metadata['repo'],
+                'morphology': metadata['morphology'],
+                'version': version,
+            })
+        
+        # Generate a format string for dumping the information.
+        fmt = self._generate_output_format(artifacts)
+        
+        # Print information about system, strata and chunks.
+        self._print_artifacts(fmt, artifacts, 'system')
+        self._print_artifacts(fmt, artifacts, 'stratum')
+        self._print_artifacts(fmt, artifacts, 'chunk')
+
+    def _generate_output_format(self, artifacts):
+        colwidths = {}
+        for artifact in artifacts:
+            for key, value in artifact.iteritems():
+                colwidths[key] = max(colwidths.get(key, 0), len(value))
+
+        colwidths['first'] = sum([colwidths['cache-key'],
+                                  colwidths['kind'],
+                                  colwidths['name']]) + 1
+
+        return 'artifact=%%-%is\t' \
+               'version=%%-%is\t' \
+               'commit=%%-%is\t' \
+               'repository=%%-%is\t' \
+               'ref=%%-%is\t' \
+               'morphology=%%-%is\n' % (
+                len('artifact=') + colwidths['first'],
+                len('version=') + colwidths['version'],
+                len('commit=') + colwidths['sha1'],
+                len('repository=') + colwidths['repo'],
+                len('ref=') + colwidths['original_ref'],
+                len('morphology=') + colwidths['morphology'])
+
+    def _print_artifacts(self, fmt, artifacts, kind):
+        for artifact in sorted(artifacts, key=lambda x: x['name']):
+            if artifact['kind'] == kind:
+                self.app.output.write(fmt % (
+                    '%s.%s.%s' % (artifact['cache-key'],
+                                  artifact['kind'],
+                                  artifact['name']),
+                    artifact['version'],
+                    artifact['sha1'],
+                    artifact['repo'],
+                    artifact['original_ref'],
+                    artifact['morphology']))
+
+
+class ArtifactInspectionPlugin(cliapp.Plugin):
+
+    def enable(self):
+        self.app.add_subcommand('run-in-artifact',
+                                self.run_in_artifact,
+                                arg_synopsis='ARTIFACT CMD')
+        self.app.add_subcommand('generate-manifest',
+                                self.generate_manifest,
+                                arg_synopsis='ROOTFS_ARTIFACT')
+
+    def disable(self):
+        pass
+
+    def run_in_artifact(self, args):
+        '''Run a command inside an extracted/mounted chunk or system.'''
+
+        if len(args) < 2:
+            raise cliapp.AppException(
+                    'run-in-artifact requires arguments: a chunk or '
+                    'system artifact and a a shell command')
+
+        artifact, cmd = args[0], args[1:]
+
+        def run_command_in_dir(dirname):
+            output = self.app.runcmd(cmd, cwd=dirname)
+            self.app.output.write(output)
+
+        call_in_artifact_directory(self.app, artifact, run_command_in_dir)
+
+    def generate_manifest(self, args):
+        '''Generate a content manifest for a system image.'''
+
+        if len(args) != 1:
+            raise cliapp.AppException('morph generate-manifest expects '
+                                      'a system image as its input')
+
+        artifact = args[0]
+
+        def generate_manifest(dirname):
+            generator = ManifestGenerator(self.app)
+            generator.generate(artifact, dirname)
+
+        call_in_artifact_directory(self.app, artifact, generate_manifest)
diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py
index f9595f98..cbc41acf 100644
--- a/morphlib/plugins/branch_and_merge_plugin.py
+++ b/morphlib/plugins/branch_and_merge_plugin.py
@@ -348,6 +348,7 @@ class BranchAndMergePlugin(cliapp.Plugin):
                 'install-commands',
                 'max-jobs',
                 'chunks',
+                'devices',
             ]
         }
         
diff --git a/morphlib/plugins/trebuchet_plugin.py b/morphlib/plugins/trebuchet_plugin.py
index 2bdf4c3c..1ebffbf4 100644
--- a/morphlib/plugins/trebuchet_plugin.py
+++ b/morphlib/plugins/trebuchet_plugin.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012  Codethink Limited
+# Copyright (C) 2012-2013  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -21,62 +21,8 @@ import gzip
 
 import morphlib
 
-class MountableImage(object):
-    '''Mountable image (deals with decompression).
-
-    Note, this is a read-only mount in the sense that the decompressed
-    image is not then recompressed after, instead any changes are discarded.
-
-    '''
-    def __init__(self, app, artifact_path):
-        self.app = app
-        self.artifact_path = artifact_path
-
-    def setup(self, path):
-        self.app.status(msg='Preparing image %(path)s', path=path)
-        self.app.status(msg='  Decompressing...', chatty=True)
-        (tempfd, self.temp_path) = \
-            tempfile.mkstemp(dir=self.app.settings['tempdir'])
-
-        try:
-            with os.fdopen(tempfd, "wb") as outfh:
-                infh = gzip.open(path, "rb")
-                morphlib.util.copyfileobj(infh, outfh)
-                infh.close()
-        except:
-            os.unlink(self.temp_path)
-            raise
-        self.app.status(msg='  Mounting image at %(path)s',
-                        path=self.temp_path, chatty=True)
-        part = morphlib.fsutils.setup_device_mapping(self.app.runcmd,
-                                                     self.temp_path)
-        mount_point = tempfile.mkdtemp(dir=self.app.settings['tempdir'])
-        morphlib.fsutils.mount(self.app.runcmd, part, mount_point)
-        self.mount_point = mount_point
-        return mount_point
-
-    def cleanup(self, path, mount_point):
-        self.app.status(msg='Clearing down image at %(path)s', path=path,
-                        chatty=True)
-        try:
-            morphlib.fsutils.unmount(self.app.runcmd, mount_point)
-        except:
-            pass
-        try:
-            morphlib.fsutils.undo_device_mapping(self.app.runcmd, path)
-        except:
-            pass
-        try:
-            os.rmdir(mount_point)
-            os.unlink(path)
-        except:
-            pass
-
-    def __enter__(self):
-        return self.setup(self.artifact_path)
-
-    def __exit__(self, exctype, excvalue, exctraceback):
-        self.cleanup(self.temp_path, self.mount_point)
+from morphlib.mountableimage import MountableImage
+
 
 class TrebuchetPlugin(cliapp.Plugin):
 
diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py
index a87b45c3..f930f9d7 100644
--- a/morphlib/stagingarea.py
+++ b/morphlib/stagingarea.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012  Codethink Limited
+# Copyright (C) 2012,2013  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -17,6 +17,8 @@
 import logging
 import os
 import shutil
+import stat
+from urlparse import urlparse
 
 import morphlib
 
@@ -40,6 +42,10 @@ class StagingArea(object):
         self._app = app
         self.dirname = dirname
         self.tempdir = tempdir
+        self.builddirname = None
+        self.destdirname = None
+        self.mounted = None
+        self._bind_readonly_mount = None
 
     # Wrapper to be overridden by unit tests.
     def _mkdir(self, dirname):  # pragma: no cover
@@ -80,6 +86,52 @@ class StagingArea(object):
         assert filename.startswith(dirname)
         return filename[len(dirname) - 1:]  # include leading slash
 
+    def hardlink_all_files(self, srcpath, destpath): # pragma: no cover
+        '''Hardlink every file in the path to the staging-area
+
+        If an exception is raised, the staging-area is indeterminate.
+
+        '''
+
+        file_stat = os.lstat(srcpath)
+        mode = file_stat.st_mode
+
+        if stat.S_ISDIR(mode):
+            # Ensure directory exists in destination, then recurse.
+            if not os.path.exists(destpath):
+                os.makedirs(destpath)
+            dest_stat = os.stat(os.path.realpath(destpath))
+            if not stat.S_ISDIR(dest_stat.st_mode):
+                raise IOError('Destination not a directory. source has %s'
+                              ' destination has %s' % (srcpath, destpath))
+
+            for entry in os.listdir(srcpath):
+                self.hardlink_all_files(os.path.join(srcpath, entry),
+                                        os.path.join(destpath, entry))
+        elif stat.S_ISLNK(mode):
+            # Copy the symlink.
+            if os.path.exists(destpath):
+                os.remove(destpath)
+            os.symlink(os.readlink(srcpath), destpath)
+
+        elif stat.S_ISREG(mode):
+            # Hardlink the file.
+            if os.path.exists(destpath):
+                os.remove(destpath)
+            os.link(srcpath, destpath)
+
+        elif stat.S_ISCHR(mode) or stat.S_ISBLK(mode):
+            # Block or character device. Put contents of st_dev in a mknod.
+            if os.path.exists(destpath):
+                os.remove(destpath)
+            os.mknod(destpath, file_stat.st_mode, file_stat.st_rdev)
+            os.chmod(destpath, file_stat.st_mode)
+
+        else:
+            # Unsupported type.
+            raise IOError('Cannot extract %s into staging-area. Unsupported'
+                          ' type.' % srcpath)
+
     def install_artifact(self, handle):
         '''Install a build artifact into the staging area.
 
@@ -90,7 +142,19 @@ class StagingArea(object):
 
         logging.debug('Installing artifact %s' %
                       getattr(handle, 'name', 'unknown name'))
-        morphlib.bins.unpack_binary_from_file(handle, self.dirname)
+
+        unpacked_artifact = os.path.join(
+            self._app.settings['tempdir'],
+            os.path.basename(handle.name) + '.d')
+        if not os.path.exists(unpacked_artifact):
+            self._mkdir(unpacked_artifact)
+            morphlib.bins.unpack_binary_from_file(
+                handle, unpacked_artifact + '/')
+
+        if not os.path.exists(self.dirname):
+            self._mkdir(self.dirname)
+
+        self.hardlink_all_files(unpacked_artifact, self.dirname)
 
     def remove(self):
         '''Remove the entire staging area.
@@ -103,14 +167,114 @@ class StagingArea(object):
 
         shutil.rmtree(self.dirname)
 
+    to_mount = (
+        ('proc',    'proc',  'none'),
+        ('dev/shm', 'tmpfs', 'none'),
+    )
+
+    def mount_ccachedir(self, source): #pragma: no cover
+        ccache_dir = self._app.settings['compiler-cache-dir']
+        if not os.path.isdir(ccache_dir):
+            os.makedirs(ccache_dir)
+        # Get a path for the repo's ccache
+        ccache_url = source.repo.url
+        ccache_path = urlparse(ccache_url).path
+        ccache_repobase = os.path.basename(ccache_path)
+        if ':' in ccache_repobase: # the basename is a repo-alias
+            resolver = morphlib.repoaliasresolver.RepoAliasResolver(
+                self._app.settings['repo-alias'])
+            ccache_url = resolver.pull_url(ccache_repobase)
+            ccache_path = urlparse(ccache_url).path
+            ccache_repobase = os.path.basename(ccache_path)
+        if ccache_repobase.endswith('.git'):
+            ccache_repobase = ccache_repobase[:-len('.git')]
+
+        ccache_repodir = os.path.join(ccache_dir, ccache_repobase)
+        # Make sure that directory exists
+        if not os.path.isdir(ccache_repodir):
+            os.mkdir(ccache_repodir)
+        # Get the destination path
+        ccache_destdir= os.path.join(self.tempdir,
+                                     'tmp', 'ccache')
+        # Make sure that the destination exists. We'll create /tmp if necessary
+        # to avoid breaking when faced with an empty staging area.
+        if not os.path.isdir(ccache_destdir):
+            os.makedirs(ccache_destdir)
+        # Mount it into the staging-area
+        self._app.runcmd(['mount', '--bind', ccache_repodir,
+                         ccache_destdir])
+        return ccache_destdir
+
+    def do_mounts(self, setup_mounts):  # pragma: no cover
+        self.mounted = []
+        if not setup_mounts:
+            return
+        for mount_point, mount_type, source in self.to_mount:
+            logging.debug('Mounting %s in staging area' % mount_point)
+            path = os.path.join(self.dirname, mount_point)
+            if not os.path.exists(path):
+                os.makedirs(path)
+            self._app.runcmd(['mount', '-t', mount_type, source, path])
+            self.mounted.append(path)
+        return
+
+    def do_unmounts(self):  # pragma: no cover
+        for path in reversed(self.mounted):
+            logging.debug('Unmounting %s in staging area' % path)
+            morphlib.fsutils.unmount(self._app.runcmd, path)
+
+    def chroot_open(self, source, setup_mounts): # pragma: no cover
+        '''Setup staging area for use as a chroot.'''
+
+        assert self.builddirname == None and self.destdirname == None
+
+        builddir = self.builddir(source)
+        destdir = self.destdir(source)
+        self.builddirname = self.relative(builddir).lstrip('/')
+        self.destdirname = self.relative(destdir).lstrip('/')
+
+        self.do_mounts(setup_mounts)
+
+        if not self._app.settings['no-ccache']:
+            self.mounted.append(self.mount_ccachedir(source))
+
+        return builddir, destdir
+
+    def chroot_close(self): # pragma: no cover
+        '''Undo changes by chroot_open.
+
+        This should be called after the staging area is no longer needed.
+
+        '''
+
+        self.do_unmounts()
+
     def runcmd(self, argv, **kwargs):  # pragma: no cover
         '''Run a command in a chroot in the staging area.'''
+
         cwd = kwargs.get('cwd') or '/'
         if 'cwd' in kwargs:
             cwd = kwargs['cwd']
             del kwargs['cwd']
         else:
             cwd = '/'
-        real_argv = ['chroot', self.dirname, 'sh', '-c',
-                     'cd "$1" && shift && exec "$@"', '--', cwd] + argv
+        if self._app.settings['staging-chroot']:
+            not_readonly_dirs = [self.builddirname, self.destdirname,
+                                 'dev', 'proc', 'tmp']
+            dirs = os.listdir(self.dirname)
+            for excluded_dir in not_readonly_dirs:
+                dirs.remove(excluded_dir)
+
+            real_argv = ['linux-user-chroot']
+
+            for entry in dirs:
+                real_argv += ['--mount-readonly', '/'+entry]
+
+            real_argv += [self.dirname]
+        else:
+            real_argv = ['chroot', '/']
+
+        real_argv += ['sh', '-c', 'cd "$1" && shift && exec "$@"', '--', cwd]
+        real_argv += argv
+
         return self._app.runcmd(real_argv, **kwargs)
diff --git a/morphlib/stagingarea_tests.py b/morphlib/stagingarea_tests.py
index 3230b9e3..313226d2 100644
--- a/morphlib/stagingarea_tests.py
+++ b/morphlib/stagingarea_tests.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012  Codethink Limited
+# Copyright (C) 2012,2013  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -14,6 +14,7 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 
+import cliapp
 import os
 import shutil
 import tarfile
@@ -31,14 +32,33 @@ class FakeSource(object):
         }
 
 
+class FakeApplication(object):
+
+    def __init__(self, cachedir, tempdir):
+        self.settings = {
+            'cachedir': cachedir,
+            'tempdir': tempdir,
+        }
+
+    def runcmd(self, *args, **kwargs):
+        return cliapp.runcmd(*args, **kwargs)
+
+    def runcmd_unchecked(self, *args, **kwargs):
+        return cliapp.runcmd_unchecked(*args, **kwargs)
+
+
 class StagingAreaTests(unittest.TestCase):
 
     def setUp(self):
         self.tempdir = tempfile.mkdtemp()
+        self.cachedir = os.path.join(self.tempdir, 'cachedir')
+        os.mkdir(self.cachedir)
+        os.mkdir(os.path.join(self.cachedir, 'artifacts'))
         self.staging = os.path.join(self.tempdir, 'staging')
         self.created_dirs = []
-        self.sa = morphlib.stagingarea.StagingArea(object(), self.staging,
-                                                   self.staging)
+        self.sa = morphlib.stagingarea.StagingArea(
+            FakeApplication(self.cachedir, self.tempdir),
+            self.staging, self.staging)
 
     def tearDown(self):
         shutil.rmtree(self.tempdir)
diff --git a/tests.as-root/run-in-artifact-propagates-exit-code.exit b/tests.as-root/run-in-artifact-propagates-exit-code.exit
new file mode 100644
index 00000000..d00491fd
--- /dev/null
+++ b/tests.as-root/run-in-artifact-propagates-exit-code.exit
@@ -0,0 +1 @@
+1
diff --git a/tests.as-root/run-in-artifact-propagates-exit-code.script b/tests.as-root/run-in-artifact-propagates-exit-code.script
new file mode 100755
index 00000000..d815c73d
--- /dev/null
+++ b/tests.as-root/run-in-artifact-propagates-exit-code.script
@@ -0,0 +1,33 @@
+#!/bin/bash
+#
+# Copyright (C) 2012-2013  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+## Test that 'run-in-artifact' propagates the exit code of its command.
+
+set -eu
+
+. "$SRCDIR/tests.as-root/lib"
+
+# Build first image. Remember the stratum.
+"$SRCDIR/scripts/test-morph" build-morphology \
+    test:morphs master linux-system
+
+system=$(find "$DATADIR/cache/artifacts" -maxdepth 1 -name '*.system.*-rootfs')
+
+# Run 'run-in-artifact' with the system artifact. The command will fail
+# and this should result in an exit code of 1 in the test.
+"$SRCDIR/scripts/test-morph" run-in-artifact "$system" -- ls i-do-not-exist
diff --git a/tests.as-root/run-in-artifact-propagates-exit-code.stderr b/tests.as-root/run-in-artifact-propagates-exit-code.stderr
new file mode 100644
index 00000000..98aa5450
--- /dev/null
+++ b/tests.as-root/run-in-artifact-propagates-exit-code.stderr
@@ -0,0 +1,3 @@
+ERROR: Command failed: ls i-do-not-exist
+ls: i-do-not-exist: No such file or directory
+
diff --git a/tests.as-root/run-in-artifact-with-different-artifacts.script b/tests.as-root/run-in-artifact-with-different-artifacts.script
new file mode 100755
index 00000000..0b14f527
--- /dev/null
+++ b/tests.as-root/run-in-artifact-with-different-artifacts.script
@@ -0,0 +1,47 @@
+#!/bin/bash
+#
+# Copyright (C) 2012-2013  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+## Test the 'run-in-artifact' command with different types of artifacts.
+
+set -eu
+
+. "$SRCDIR/tests.as-root/lib"
+
+# Build first image. Remember the stratum.
+"$SRCDIR/scripts/test-morph" build-morphology \
+    test:morphs master linux-system
+
+system=$(find "$DATADIR/cache/artifacts" -maxdepth 1 -name '*.system.*-rootfs')
+chunk=$(find "$DATADIR/cache/artifacts" -maxdepth 1 -name '*.chunk.linux')
+stratum=$(find "$DATADIR/cache/artifacts" -maxdepth 1 \
+    -name '*.stratum.linux-stratum')
+
+# Run 'run-in-artifact' with the system artifact.
+echo "System:"
+"$SRCDIR/scripts/test-morph" run-in-artifact "$system" -- ls factory/baserock/
+echo
+
+# Run 'run-in-artifact' with the chunk artifact.
+echo "Chunk:"
+"$SRCDIR/scripts/test-morph" run-in-artifact "$chunk" -- ls baserock/
+echo
+
+# Run 'run-in-artifact' with the statum artifact.
+echo "Stratum:"
+"$SRCDIR/scripts/test-morph" run-in-artifact "$stratum" -- ls baserock/ \
+    || echo "Failed"
diff --git a/tests.as-root/run-in-artifact-with-different-artifacts.stderr b/tests.as-root/run-in-artifact-with-different-artifacts.stderr
new file mode 100644
index 00000000..44e70c38
--- /dev/null
+++ b/tests.as-root/run-in-artifact-with-different-artifacts.stderr
@@ -0,0 +1 @@
+ERROR: Artifact TMP/cache/artifacts/293fc1b78dd2af221ae7de246ff5a59df476165704b7e366230ac8ed4c16d1b7.stratum.linux-stratum cannot be extracted or mounted
diff --git a/tests.as-root/run-in-artifact-with-different-artifacts.stdout b/tests.as-root/run-in-artifact-with-different-artifacts.stdout
new file mode 100644
index 00000000..281ab109
--- /dev/null
+++ b/tests.as-root/run-in-artifact-with-different-artifacts.stdout
@@ -0,0 +1,14 @@
+System:
+hello-stratum.meta
+hello.meta
+linux-stratum.meta
+linux-system-rootfs.meta
+linux.meta
+tools-stratum.meta
+tools.meta
+
+Chunk:
+linux.meta
+
+Stratum:
+Failed
diff --git a/without-test-modules b/without-test-modules
index 5bcdb025..cc901f32 100644
--- a/without-test-modules
+++ b/without-test-modules
@@ -5,6 +5,9 @@ morphlib/tester.py
 morphlib/git.py
 morphlib/fsutils.py
 morphlib/app.py
+morphlib/mountableimage.py
+morphlib/extractedtarball.py
+morphlib/plugins/artifact_inspection_plugin.py
 morphlib/plugins/hello_plugin.py
 morphlib/plugins/graphing_plugin.py
 morphlib/plugins/syslinux-disk-systembuilder_plugin.py