Use overlayfs when building systems

This will allow us to cache systems as a list of chunks and a small
filesystem delta, rather than a massive tarball.

3 files changed, 47 insertions, 7 deletions

diff --git a/morphlib/builder.py b/morphlib/builder.py
index 1bf4d454..40df10c3 100644
--- a/morphlib/builder.py
+++ b/morphlib/builder.py
@@ -555,13 +555,25 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
 
             for a_name, artifact in self.source.artifacts.iteritems():
                 handle = self.local_artifact_cache.put(artifact)
-
                 try:
                     fs_root = self.staging_area.destdir(self.source)
                     self.unpack_strata(fs_root)
-                    self.write_metadata(fs_root, a_name)
-                    self.run_system_integration_commands(fs_root)
-                    unslashy_root = fs_root[1:]
+                    overlay_root = self.staging_area.overlay_upperdir(
+                        self.source)
+                    editable_root = self.staging_area.overlaydir(self.source)
+                    workdir = os.path.join(self.staging_area.dirname,
+                                           'overlayfs-workdir')
+                    if not os.path.exists(workdir):
+                        os.makedirs(workdir)
+                    options = '-olowerdir=%s,upperdir=%s,workdir=%s' % \
+                        (fs_root, overlay_root, workdir)
+                    morphlib.fsutils.mount(self.app.runcmd,
+                                           'overlay-%s' % a_name,
+                                           editable_root,
+                                           'overlay', options)
+                    self.write_metadata(editable_root, a_name)
+                    self.run_system_integration_commands(editable_root)
+                    unslashy_root = editable_root[1:]
                     def uproot_info(info):
                         info.name = relpath(info.name, unslashy_root)
                         if info.islnk():
@@ -571,15 +583,21 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
                     tar = tarfile.open(fileobj=handle, mode="w", name=a_name)
                     self.app.status(msg='Constructing tarball of rootfs',
                                     chatty=True)
-                    tar.add(fs_root, recursive=True, filter=uproot_info)
+                    tar.add(editable_root, recursive=True, filter=uproot_info)
                     tar.close()
                 except BaseException as e:
                     logging.error(traceback.format_exc())
                     self.app.status(msg='Error while building system',
                                     error=True)
+                    if editable_root and os.path.exists(editable_root):
+                        morphlib.fsutils.unmount(self.app.runcmd,
+                                                 editable_root)
                     handle.abort()
                     raise
                 else:
+                    if editable_root and os.path.exists(editable_root):
+                        morphlib.fsutils.unmount(self.app.runcmd,
+                                                 editable_root)
                     handle.close()
 
         self.save_build_times()
diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py
index fd3f6881..eaa8b54b 100644
--- a/morphlib/stagingarea.py
+++ b/morphlib/stagingarea.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2014  Codethink Limited
+# Copyright (C) 2012-2015  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -88,6 +88,14 @@ class StagingArea(object):
 
         return self._dir_for_source(source, 'inst')
 
+    def overlay_upperdir(self, source):
+        '''Create a directory to be upperdir for overlayfs, and return it.'''
+        return self._dir_for_source(source, 'overlay_upper')
+
+    def overlaydir(self, source):
+        '''Create a directory to be a mount point for overlayfs, return it'''
+        return self._dir_for_source(source, 'overlay')
+
     def relative(self, filename):
         '''Return a filename relative to the staging area.'''
 
diff --git a/morphlib/stagingarea_tests.py b/morphlib/stagingarea_tests.py
index dc43e4f6..f64cf2d4 100644
--- a/morphlib/stagingarea_tests.py
+++ b/morphlib/stagingarea_tests.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2014  Codethink Limited
+# Copyright (C) 2012-2015  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -119,6 +119,20 @@ class StagingAreaTests(unittest.TestCase):
         self.assertEqual(self.created_dirs, [dirname])
         self.assertTrue(dirname.startswith(self.staging))
 
+    def test_creates_overlay_upper_directory(self):
+        source = FakeSource()
+        self.sa._mkdir = self.fake_mkdir
+        dirname = self.sa.overlay_upperdir(source)
+        self.assertEqual(self.created_dirs, [dirname])
+        self.assertTrue(dirname.startswith(self.staging))
+
+    def test_creates_overlay_directory(self):
+        source = FakeSource()
+        self.sa._mkdir = self.fake_mkdir
+        dirname = self.sa.overlaydir(source)
+        self.assertEqual(self.created_dirs, [dirname])
+        self.assertTrue(dirname.startswith(self.staging))
+
     def test_makes_relative_name(self):
         filename = os.path.join(self.staging, 'foobar')
         self.assertEqual(self.sa.relative(filename), '/foobar')