Merge branch 'baserock/richardmaw/parallelism-improvements'

This stripped out the commit from the patch series on the mailing list
to remove /dev/shm.

Reviewed-by: Sam Thursfield
Reviewed-by: Daniel Silverstone

3 files changed, 23 insertions, 11 deletions

diff --git a/morphlib/builder2.py b/morphlib/builder2.py
index 596cd645..8615ed59 100644
--- a/morphlib/builder2.py
+++ b/morphlib/builder2.py
@@ -342,6 +342,7 @@ class ChunkBuilder(BuilderBase):
 
         relative_builddir = self.staging_area.relative(builddir)
         relative_destdir = self.staging_area.relative(destdir)
+        ccache_dir = self.staging_area.ccache_dir(self.source)
         extra_env = { 'DESTDIR': relative_destdir }
 
         steps = [
@@ -391,7 +392,8 @@ class ChunkBuilder(BuilderBase):
                                     cwd=relative_builddir,
                                     stdout=stdout or subprocess.PIPE,
                                     stderr=subprocess.STDOUT,
-                                    logfile=logfilepath)
+                                    logfile=logfilepath,
+                                    ccache_dir=ccache_dir)
 
                         if stdout:
                             stdout.flush()
diff --git a/morphlib/extensions.py b/morphlib/extensions.py
index af6ba279..6b81e116 100644
--- a/morphlib/extensions.py
+++ b/morphlib/extensions.py
@@ -223,7 +223,11 @@ class ExtensionSubprocess(object):
             def close_read_end():
                 os.close(log_read_fd)
             p = subprocess.Popen(
-                [filename] + args, cwd=cwd, env=new_env,
+                # We unshare and mount --make-rprivate so mounts done by write
+                # extensions can't interfere with the rest of the system.
+                ['unshare', '-m', '--', '/bin/sh', '-c',
+                 'mount --make-rprivate / && exec "$@"', '-', filename] + args,
+                cwd=cwd, env=new_env,
                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                 preexec_fn=close_read_end)
             os.close(log_write_fd)
diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py
index bfe0a716..25e33b3f 100644
--- a/morphlib/stagingarea.py
+++ b/morphlib/stagingarea.py
@@ -192,11 +192,10 @@ class StagingArea(object):
         shutil.rmtree(self.dirname)
 
     to_mount = (
-        ('proc',    'proc',  'none'),
         ('dev/shm', 'tmpfs', 'none'),
     )
 
-    def mount_ccachedir(self, source): #pragma: no cover
+    def ccache_dir(self, source): #pragma: no cover
         ccache_dir = self._app.settings['compiler-cache-dir']
         if not os.path.isdir(ccache_dir):
             os.makedirs(ccache_dir)
@@ -223,10 +222,7 @@ class StagingArea(object):
         # to avoid breaking when faced with an empty staging area.
         if not os.path.isdir(ccache_destdir):
             os.makedirs(ccache_destdir)
-        # Mount it into the staging-area
-        self._app.runcmd(['mount', '--bind', ccache_repodir,
-                         ccache_destdir])
-        return ccache_destdir
+        return ccache_repodir
 
     def do_mounts(self, setup_mounts):  # pragma: no cover
         if not setup_mounts:
@@ -257,9 +253,6 @@ class StagingArea(object):
 
         self.do_mounts(setup_mounts)
 
-        if not self._app.settings['no-ccache']:
-            self.mounted.append(self.mount_ccachedir(source))
-
         return builddir, destdir
 
     def chroot_close(self): # pragma: no cover
@@ -284,6 +277,7 @@ class StagingArea(object):
             del kwargs['cwd']
         else:
             cwd = '/'
+        ccache_dir = kwargs.pop('ccache_dir', None)
 
         chroot_dir = self.dirname if self.use_chroot else '/'
         temp_dir = kwargs["env"].get("TMPDIR", "/tmp")
@@ -304,6 +298,18 @@ class StagingArea(object):
             if not os.path.islink(d):
                 real_argv += ['--mount-readonly', self.relative(d)]
 
+        if self.use_chroot:
+            proc_target = os.path.join(self.dirname, 'proc')
+            if not os.path.exists(proc_target):
+                os.makedirs(proc_target)
+            real_argv += ['--mount-proc', self.relative(proc_target)]
+
+        if ccache_dir and not self._app.settings['no-ccache']:
+            ccache_target = os.path.join(
+                    self.dirname, kwargs['env']['CCACHE_DIR'].lstrip('/'))
+            real_argv += ['--mount-bind', ccache_dir,
+                          self.relative(ccache_target)]
+
         real_argv += [chroot_dir]
 
         real_argv += argv