Use OSTree for hardlink and artifact cache and a CoW unionfs to make system artifacts fasterbaserock/richardmaw/ostree-squash

This replaces the artifact cache and the hardlink cache with an OSTree
repository, which is a great performance improvement when the cache
directory and temporary directory are on the same filesystem.

Additionally it can de-duplicate file contents.

When we construct system artifacts deploy them, the staging area needs
to be writable, so OSTree on its own is insufficient, as its hardlinks
require the root to be kept read-only.

To handle this we use either the in-kernel overlayfs or unionfs-fuse,
though there is no automatic fall-back and it needs to be specified
manually.

To support distributed building, the artifact cache is extended to
support an OSTree repository.

Unfortunately cross-bootstrap is not expected to work with these changes
at this point in time.

IMPORTANT NOTE: We are well aware that this patch is too large to be
comprehensible. We intend to revert and apply a cleaned up series when
it is ready.

Change-Id: I693bb752500dab3c6db3b97393689239ae7071a8

1 files changed, 59 insertions, 5 deletions

diff --git a/morphlib/util.py b/morphlib/util.py
index 904dc355..8179e523 100644
--- a/morphlib/util.py
+++ b/morphlib/util.py
@@ -14,16 +14,21 @@
 
 import contextlib
 import itertools
+import json
+import logging
 import os
 import pipes
 import re
+import stat
 import subprocess
-import textwrap
 import sys
+import textwrap
 
+import cliapp
 import fs.osfs
 
 import morphlib
+from morphlib.artifactcachereference import ArtifactCacheReference
 
 '''Utility functions for morph.'''
 
@@ -120,7 +125,7 @@ def get_git_resolve_cache_server(settings): # pragma: no cover
     return None
 
 
-def new_artifact_caches(settings):  # pragma: no cover
+def new_artifact_caches(settings, status_cb=None):  # pragma: no cover
     '''Create new objects for local and remote artifact caches.
 
     This includes creating the directories on disk, if missing.
@@ -132,12 +137,17 @@ def new_artifact_caches(settings):  # pragma: no cover
     if not os.path.exists(artifact_cachedir):
         os.mkdir(artifact_cachedir)
 
-    lac = morphlib.localartifactcache.LocalArtifactCache(
-            fs.osfs.OSFS(artifact_cachedir))
+    mode = settings['ostree-repo-mode']
+    lac = morphlib.ostreeartifactcache.OSTreeArtifactCache(
+        artifact_cachedir, mode=mode, status_cb=status_cb)
 
     rac_url = get_artifact_cache_server(settings)
     rac = None
-    if rac_url:
+    # We let 'none' here specify 'don't use a remote artifact cache'.
+    # The 'artifact-cache-server' setting defaults to a calue based
+    # on 'trove-host' so it's not enough to specify --artifact-cache-server=''
+    # if you want to force Morph to ignore cached artifacts.
+    if rac_url and rac_url.lower() != 'none':
         rac = morphlib.remoteartifactcache.RemoteArtifactCache(rac_url)
     return lac, rac
 
@@ -691,3 +701,47 @@ def write_from_dict(filepath, d, validate=lambda x, y: True): #pragma: no cover
 
         os.fchown(f.fileno(), 0, 0)
         os.fchmod(f.fileno(), 0644)
+
+
+def create_devices(morphology, destdir): # pragma: no cover
+    '''Creates device nodes if the morphology specifies them'''
+    perms_mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO
+    if 'devices' in morphology and morphology['devices'] is not None:
+        for dev in morphology['devices']:
+            destfile = os.path.join(destdir, './' + dev['filename'])
+            mode = int(dev['permissions'], 8) & perms_mask
+            if dev['type'] == 'c':
+                mode = mode | stat.S_IFCHR
+            elif dev['type'] == 'b':
+                mode = mode | stat.S_IFBLK
+            else:
+                raise IOError('Cannot create device node %s,'
+                              'unrecognized device type "%s"'
+                              % (destfile, dev['type']))
+            parent = os.path.dirname(destfile)
+            if not os.path.exists(parent):
+                os.makedirs(parent)
+            if not os.path.exists(destfile):
+                logging.debug("Creating device node %s" % destfile)
+                os.mknod(destfile, mode,
+                         os.makedev(dev['major'], dev['minor']))
+                os.chown(destfile, dev['uid'], dev['gid'])
+
+
+def get_stratum_contents(cache, stratum_artifact): # pragma: no cover
+    '''Load a stratum from a local artifact cache.
+
+    Returns a list of ArtifactCacheReference instances for the chunks
+    contained in the stratum.
+
+    '''
+
+    with open(cache.get(stratum_artifact), 'r') as stratum_file:
+        try:
+            artifact_list = json.load(stratum_file,
+                                        encoding='unicode-escape')
+        except ValueError as e:
+            raise cliapp.AppException(
+                'Corruption detected: %s while loading %s' %
+                (e, cache.artifact_filename(stratum_artifact)))
+    return [ArtifactCacheReference(a) for a in artifact_list]