Morph build 590a8150f91843c3a291168f2106ddba

System branch: master

10 files changed, 103 insertions, 43 deletions

diff --git a/morph-cache-server b/morph-cache-server
index 6c7665aa..938ecb1f 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -38,6 +38,7 @@ defaults = {
     'artifact-dir': '/var/cache/morph-cache-server/artifacts',
     'port': 8080,
     'ostree-port': 12324,
+    'ostree-repo-mode': 'archive_z2',
 }
 
 
@@ -68,6 +69,11 @@ class MorphCacheServer(cliapp.Application):
                              'path to the artifact cache directory',
                              metavar='PATH',
                              default=defaults['artifact-dir'])
+        self.settings.string(['ostree-repo-mode'],
+                             'mode of the ostree artifact cache - either '
+                             'archive_z2 or bare, for servers and users '
+                             'respectively.',
+                             default=defaults['ostree-repo-mode'])
         self.settings.boolean(['direct-mode'],
                               'cache directories are directly managed')
         self.settings.boolean(['enable-writes'],
@@ -78,7 +84,8 @@ class MorphCacheServer(cliapp.Application):
 
     def _fetch_artifacts(self, server, cacheid, artifacts):
         ret = {}
-        cache = OSTreeArtifactCache(self.settings['artifact-dir'])
+        cache = OSTreeArtifactCache(self.settings['artifact-dir'],
+                                    mode=self.settings['ostree-repo-mode'])
         remote = RemoteArtifactCache('http://%s/' % server)
         try:
             for artifact in artifacts:
@@ -274,7 +281,8 @@ class MorphCacheServer(cliapp.Application):
         @app.get('/artifacts')
         def artifact():
             basename = self._unescape_parameter(request.query.filename)
-            cache = OSTreeArtifactCache(self.settings['artifact-dir'])
+            cache = OSTreeArtifactCache(self.settings['artifact-dir'],
+                                        mode=self.settings['ostree-repo-mode'])
             try:
                 cachekey, kind, name = basename.split('.', 2)
                 a = ArtifactCacheReference(basename)
@@ -320,7 +328,8 @@ class MorphCacheServer(cliapp.Application):
 
             logging.debug('Received a POST request for /artifacts')
 
-            cache = OSTreeArtifactCache(self.settings['artifact-dir'])
+            cache = OSTreeArtifactCache(self.settings['artifact-dir'],
+                                        mode=self.settings['ostree-repo-mode'])
             for basename in artifacts:
                 if basename.startswith('/'):
                     response.status = 500
diff --git a/morphlib/app.py b/morphlib/app.py
index f7c07726..c367cafb 100644
--- a/morphlib/app.py
+++ b/morphlib/app.py
@@ -127,6 +127,15 @@ class Morph(cliapp.Application):
                              'supported at this time.',
                              default='overlayfs',
                              group=group_advanced)
+        self.settings.string(['ostree-repo-mode'],
+                             'Mode for OSTree artifact cache repository. If '
+                             'things will need to pull from your cache, this '
+                             'needs to be "archive_z2". Otherwise use '
+                             '"bare". Note that archive_z2 will cause things '
+                             'involving the artifact cache (building and/or '
+                             'deploying) to be slow.',
+                             default='bare',
+                             group=group_advanced)
 
         group_build = 'Build Options'
         self.settings.integer(['max-jobs'],
diff --git a/morphlib/exts/sysroot.write b/morphlib/exts/sysroot.write
index 0ad8d630..cc50f9dc 100755
--- a/morphlib/exts/sysroot.write
+++ b/morphlib/exts/sysroot.write
@@ -23,4 +23,5 @@ mkdir -p "$2"
 # Previously we would (cd "$1" && find -print0 | cpio -0pumd "$absolute_path")
 # to do this, but the source directory is disposable anyway, so we can move
 # its contents to save time
-find "$1" -maxdepth 1 -mindepth 1 -exec mv {} "$2/." +
+# find "$1" -maxdepth 1 -mindepth 1 -exec mv {} "$2/." +
+(cd "$1" && find -print0 | cpio -0pumd "$2")
diff --git a/morphlib/ostree.py b/morphlib/ostree.py
index 64cf48e8..45514e2f 100644
--- a/morphlib/ostree.py
+++ b/morphlib/ostree.py
@@ -1,35 +1,70 @@
+# Copyright (C) 2013-2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-2 =*=
+
+
 from gi.repository import OSTree
 from gi.repository import Gio
 from gi.repository import GLib
 
 import os
+import logging
 
 
 class OSTreeRepo(object):
 
     """Class to wrap the OSTree API."""
 
-    OSTREE_GIO_FAST_QUERYINFO = 'standard::name,standard::type,standard::' \
-                                'size,standard::is-symlink,standard::syml' \
-                                'ink-target,unix::device,unix::inode,unix' \
-                                '::mode,unix::uid,unix::gid,unix::rdev'
     G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS = Gio.FileQueryInfoFlags(1)
-    cancellable = Gio.Cancellable.new()
+    OSTREE_GIO_FAST_QUERYINFO = (
+        'standard::name,'
+        'standard::type,'
+        'standard::size,'
+        'standard::is-symlink,'
+        'standard::symlink-target,'
+        'unix::device,'
+        'unix::inode,'
+        'unix::mode,'
+        'unix::uid,'
+        'unix::gid,'
+        'unix::rdev')
     
-    def __init__(self, path, disable_fsync=True):
+    def __init__(self, path, disable_fsync=True, mode='bare'):
         self.path = path
-        self.repo = self._open_repo(path, disable_fsync)
+        self.repo = self._open_repo(path, disable_fsync, mode)
 
-    def _open_repo(self, path, disable_fsync=True):
+    def _open_repo(self, path, disable_fsync=True, mode='bare'):
         """Create and open and OSTree.Repo, and return it."""
         repo_dir = Gio.file_new_for_path(path)
         repo = OSTree.Repo.new(repo_dir)
+        logging.debug('using %s' % mode)
+        if mode == 'bare':
+            mode = OSTree.RepoMode.BARE
+        elif mode == 'archive_z2':
+            mode = OSTree.RepoMode.ARCHIVE_Z2
+        else:
+            raise Exception('Mode %s is not supported' % mode)
+
         try:
-            repo.open(self.cancellable)
+            repo.open(None)
+            logging.debug('opened')
         except GLib.GError:
             if not os.path.exists(path):
                 os.makedirs(path)
-            repo.create(OSTree.RepoMode.BARE, self.cancellable)
+            logging.debug('failed to open, creating')
+            repo.create(mode, None)
         repo.set_disable_fsync(disable_fsync)
         return repo
     
@@ -47,13 +82,13 @@ class OSTreeRepo(object):
     
     def read_commit(self, branch):
         """Return an OSTree.RepoFile representing a committed tree."""
-        return self.repo.read_commit(branch, self.cancellable)[1]
+        return self.repo.read_commit(branch, None)[1]
     
     def query_info(self, file_object):
         """Quickly return a Gio.FileInfo for file_object."""
         return file_object.query_info(self.OSTREE_GIO_FAST_QUERYINFO,
                                       self.G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS,
-                                      self.cancellable)
+                                      None)
     
     def checkout(self, branch, destdir):
         """Checkout branch into destdir."""
@@ -68,23 +103,23 @@ class OSTreeRepo(object):
                                 OSTree.RepoCheckoutOverwriteMode.UNION_FILES,
                                 checkout, commit,commit_info, None)
 
-    def commit(self, subject, srcdir, branch):
-        """Commit the contents of 'srcdir' to 'branch'."""
-        self.repo.prepare_transaction(self.cancellable)
-        parent = self.resolve_rev(branch)
-        if parent:
-            parent_root = self.read_commit(parent)
+    def commit(self, subject, srcdir, branch, body=''):
+        """Commit the contents of 'srcdir' to 'branch'.
+        
+        The subject parameter is the title of the commit message, and the
+        body parameter is the body of the commit message.
 
+        """
+        self.repo.prepare_transaction(None)
+        parent = self.resolve_rev(branch)
         mtree = OSTree.MutableTree()
         src = Gio.file_new_for_path(srcdir)
-        self.repo.write_directory_to_mtree(src, mtree, None, self.cancellable)
-        root = self.repo.write_mtree(mtree, self.cancellable)[1]
-        if parent and root.equal(parent_root):
-            return
-        checksum = self.repo.write_commit(parent, subject, '', None,
-                                          root, self.cancellable)[1]
+        self.repo.write_directory_to_mtree(src, mtree, None, None)
+        root = self.repo.write_mtree(mtree, None)[1]
+        checksum = self.repo.write_commit(parent, subject, body,
+                                          None, root, None)[1]
         self.repo.transaction_set_ref(None, branch, checksum)
-        stats = self.repo.commit_transaction(self.cancellable)
+        stats = self.repo.commit_transaction(None)
 
     def cat_file(self, ref, path):
         """Return the file descriptor of path at ref."""
@@ -93,9 +128,12 @@ class OSTreeRepo(object):
         ret, content, etag = relative.load_contents()
         return content
 
-    def list_refs(self, resolved=False):
+    def list_refs(self, ref=None, resolved=False):
         """Return a list of all refs in the repo."""
-        refs = self.repo.list_refs()[1]
+        if ref:
+            refs = self.repo.list_refs(ref)[1]
+        else:
+            refs = self.repo.list_refs()[1]
         if not resolved:
             return refs.keys()
         return refs
@@ -104,7 +142,7 @@ class OSTreeRepo(object):
         """Remove refspec from the repo."""
         if not self.list_refs(ref):
             raise Exception("Failed to delete ref, it doesn't exist")
-        self.repo.set_ref_immediate(None, ref, None, self.cancellable)
+        self.repo.set_ref_immediate(None, ref, None, None)
 
     def prune(self):
         """Remove unreachable objects from the repo."""
@@ -120,7 +158,7 @@ class OSTreeRepo(object):
 
     def remove_remote(self, name):
         """Remove a remote with a given name."""
-        self.repo.remote_delete(name, self.cancellable)
+        self.repo.remote_delete(name, None)
 
     def get_remote_url(self, name):
         """Return the URL for a remote."""
diff --git a/morphlib/ostreeartifactcache.py b/morphlib/ostreeartifactcache.py
index 307f6f50..ea659c8f 100644
--- a/morphlib/ostreeartifactcache.py
+++ b/morphlib/ostreeartifactcache.py
@@ -30,9 +30,9 @@ from morphlib.artifactcachereference import ArtifactCacheReference
 class OSTreeArtifactCache(object):
     """Class to provide the artifact cache API using an OSTree repo."""
 
-    def __init__(self, cachedir):
+    def __init__(self, cachedir, mode):
         repo_dir = os.path.join(cachedir, 'repo')
-        self.repo = morphlib.ostree.OSTreeRepo(repo_dir)
+        self.repo = morphlib.ostree.OSTreeRepo(repo_dir, mode=mode)
         self.cachedir = cachedir
 
     def _get_file_from_remote(self, artifact, remote, metadata_name=None):
diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py
index 08589ea6..cdbb303f 100644
--- a/morphlib/plugins/branch_and_merge_plugin.py
+++ b/morphlib/plugins/branch_and_merge_plugin.py
@@ -623,6 +623,7 @@ class BranchAndMergePlugin(cliapp.Plugin):
 
         smd = morphlib.systemmetadatadir.SystemMetadataDir(path)
         metadata = smd.values()
+        logging.debug(metadata)
         systems = [md for md in metadata
                    if 'kind' in md and md['kind'] == 'system']
 
diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py
index 1d5c1184..efe6735b 100644
--- a/morphlib/plugins/deploy_plugin.py
+++ b/morphlib/plugins/deploy_plugin.py
@@ -544,7 +544,7 @@ class DeployPlugin(cliapp.Plugin):
     def checkout_stratum(self, path, artifact, lac, rac):
         with open(lac.get(artifact), 'r') as stratum:
             chunks = [ArtifactCacheReference(c) for c in json.load(stratum)]
-        #morphlib.builder.download_depends(chunks, lac, rac)
+        morphlib.builder.download_depends(chunks, lac, rac)
         for chunk in chunks:
             self.app.status(msg='Checkout chunk %(name)s.',
                             name=chunk.basename(), chatty=True)
@@ -557,7 +557,7 @@ class DeployPlugin(cliapp.Plugin):
 
     def checkout_strata(self, path, artifact, lac, rac):
         deps = artifact.source.dependencies
-        #morphlib.builder.download_depends(deps, lac, rac)
+        morphlib.builder.download_depends(deps, lac, rac)
         for stratum in deps:
             self.checkout_stratum(path, stratum, lac, rac)
         morphlib.builder.ldconfig(self.app.runcmd, path)
diff --git a/morphlib/util.py b/morphlib/util.py
index 00111ff7..8566345d 100644
--- a/morphlib/util.py
+++ b/morphlib/util.py
@@ -131,10 +131,11 @@ def new_artifact_caches(settings):  # pragma: no cover
     if not os.path.exists(artifact_cachedir):
         os.mkdir(artifact_cachedir)
 
-    #lac = morphlib.localartifactcache.LocalArtifactCache(
-    #        fs.osfs.OSFS(artifact_cachedir))
-
-    lac = morphlib.ostreeartifactcache.OSTreeArtifactCache(artifact_cachedir)
+    mode = settings['ostree-repo-mode']
+    import logging
+    logging.debug(mode)
+    lac = morphlib.ostreeartifactcache.OSTreeArtifactCache(artifact_cachedir,
+                                                           mode=mode)
 
     rac_url = get_artifact_cache_server(settings)
     rac = None
diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn
index 3277075e..fb110ffd 100644
--- a/yarns/implementations.yarn
+++ b/yarns/implementations.yarn
@@ -1121,7 +1121,8 @@ Distbuild
     exec "$SRCDIR/morph" --quiet \
         --cachedir-min-space=0 --tempdir-min-space=0 \
         --no-default-config --config "$DATADIR/morph.conf" \
-        --cachedir "$DATADIR/distbuild-worker-cache" "$@"
+        --cachedir "$DATADIR/distbuild-worker-cache" \
+        --ostree-repo-mode="archive_z2" "$@"
     EOF
 
     IMPLEMENTS FINALLY the distbuild worker is terminated
diff --git a/yarns/morph.shell-lib b/yarns/morph.shell-lib
index 4f345a4a..3252b30f 100644
--- a/yarns/morph.shell-lib
+++ b/yarns/morph.shell-lib
@@ -39,7 +39,7 @@ run_morph()
         "$SRCDIR"/morph --verbose \
             --cachedir-min-space=0 --tempdir-min-space=0 \
             --no-default-config --config "$DATADIR/morph.conf" \
-            --log="$DATADIR/log-$1"  \
+            --log="$DATADIR/log-$1" --ostree-repo-mode="archive_z2" \
             "$@" 2> "$DATADIR/result-$1" > "$DATADIR/out-$1"
         local exit_code="$?"
         for o in log result out; do