From 1328d72e1953cc3ffa8a3e8596399a6588b5f904 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Tue, 31 Mar 2015 14:57:33 +0000 Subject: Morph build 590a8150f91843c3a291168f2106ddba System branch: master --- morphlib/app.py | 9 +++ morphlib/exts/sysroot.write | 3 +- morphlib/ostree.py | 96 ++++++++++++++++++++--------- morphlib/ostreeartifactcache.py | 4 +- morphlib/plugins/branch_and_merge_plugin.py | 1 + morphlib/plugins/deploy_plugin.py | 4 +- morphlib/util.py | 9 +-- 7 files changed, 88 insertions(+), 38 deletions(-) (limited to 'morphlib') diff --git a/morphlib/app.py b/morphlib/app.py index f7c07726..c367cafb 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -127,6 +127,15 @@ class Morph(cliapp.Application): 'supported at this time.', default='overlayfs', group=group_advanced) + self.settings.string(['ostree-repo-mode'], + 'Mode for OSTree artifact cache repository. If ' + 'things will need to pull from your cache, this ' + 'needs to be "archive_z2". Otherwise use ' + '"bare". Note that archive_z2 will cause things ' + 'involving the artifact cache (building and/or ' + 'deploying) to be slow.', + default='bare', + group=group_advanced) group_build = 'Build Options' self.settings.integer(['max-jobs'], diff --git a/morphlib/exts/sysroot.write b/morphlib/exts/sysroot.write index 0ad8d630..cc50f9dc 100755 --- a/morphlib/exts/sysroot.write +++ b/morphlib/exts/sysroot.write @@ -23,4 +23,5 @@ mkdir -p "$2" # Previously we would (cd "$1" && find -print0 | cpio -0pumd "$absolute_path") # to do this, but the source directory is disposable anyway, so we can move # its contents to save time -find "$1" -maxdepth 1 -mindepth 1 -exec mv {} "$2/." + +# find "$1" -maxdepth 1 -mindepth 1 -exec mv {} "$2/." + +(cd "$1" && find -print0 | cpio -0pumd "$2") diff --git a/morphlib/ostree.py b/morphlib/ostree.py index 64cf48e8..45514e2f 100644 --- a/morphlib/ostree.py +++ b/morphlib/ostree.py @@ -1,35 +1,70 @@ +# Copyright (C) 2013-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# =*= License: GPL-2 =*= + + from gi.repository import OSTree from gi.repository import Gio from gi.repository import GLib import os +import logging class OSTreeRepo(object): """Class to wrap the OSTree API.""" - OSTREE_GIO_FAST_QUERYINFO = 'standard::name,standard::type,standard::' \ - 'size,standard::is-symlink,standard::syml' \ - 'ink-target,unix::device,unix::inode,unix' \ - '::mode,unix::uid,unix::gid,unix::rdev' G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS = Gio.FileQueryInfoFlags(1) - cancellable = Gio.Cancellable.new() + OSTREE_GIO_FAST_QUERYINFO = ( + 'standard::name,' + 'standard::type,' + 'standard::size,' + 'standard::is-symlink,' + 'standard::symlink-target,' + 'unix::device,' + 'unix::inode,' + 'unix::mode,' + 'unix::uid,' + 'unix::gid,' + 'unix::rdev') - def __init__(self, path, disable_fsync=True): + def __init__(self, path, disable_fsync=True, mode='bare'): self.path = path - self.repo = self._open_repo(path, disable_fsync) + self.repo = self._open_repo(path, disable_fsync, mode) - def _open_repo(self, path, disable_fsync=True): + def _open_repo(self, path, disable_fsync=True, mode='bare'): """Create and open and OSTree.Repo, and return it.""" repo_dir = Gio.file_new_for_path(path) repo = OSTree.Repo.new(repo_dir) + logging.debug('using %s' % mode) + if mode == 'bare': + mode = OSTree.RepoMode.BARE + elif mode == 'archive_z2': + mode = OSTree.RepoMode.ARCHIVE_Z2 + else: + raise Exception('Mode %s is not supported' % mode) + try: - repo.open(self.cancellable) + repo.open(None) + logging.debug('opened') except GLib.GError: if not os.path.exists(path): os.makedirs(path) - repo.create(OSTree.RepoMode.BARE, self.cancellable) + logging.debug('failed to open, creating') + repo.create(mode, None) repo.set_disable_fsync(disable_fsync) return repo @@ -47,13 +82,13 @@ class OSTreeRepo(object): def read_commit(self, branch): """Return an OSTree.RepoFile representing a committed tree.""" - return self.repo.read_commit(branch, self.cancellable)[1] + return self.repo.read_commit(branch, None)[1] def query_info(self, file_object): """Quickly return a Gio.FileInfo for file_object.""" return file_object.query_info(self.OSTREE_GIO_FAST_QUERYINFO, self.G_FILE_QUERY_INFO_NOFOLLOW_SYMLINKS, - self.cancellable) + None) def checkout(self, branch, destdir): """Checkout branch into destdir.""" @@ -68,23 +103,23 @@ class OSTreeRepo(object): OSTree.RepoCheckoutOverwriteMode.UNION_FILES, checkout, commit,commit_info, None) - def commit(self, subject, srcdir, branch): - """Commit the contents of 'srcdir' to 'branch'.""" - self.repo.prepare_transaction(self.cancellable) - parent = self.resolve_rev(branch) - if parent: - parent_root = self.read_commit(parent) + def commit(self, subject, srcdir, branch, body=''): + """Commit the contents of 'srcdir' to 'branch'. + + The subject parameter is the title of the commit message, and the + body parameter is the body of the commit message. + """ + self.repo.prepare_transaction(None) + parent = self.resolve_rev(branch) mtree = OSTree.MutableTree() src = Gio.file_new_for_path(srcdir) - self.repo.write_directory_to_mtree(src, mtree, None, self.cancellable) - root = self.repo.write_mtree(mtree, self.cancellable)[1] - if parent and root.equal(parent_root): - return - checksum = self.repo.write_commit(parent, subject, '', None, - root, self.cancellable)[1] + self.repo.write_directory_to_mtree(src, mtree, None, None) + root = self.repo.write_mtree(mtree, None)[1] + checksum = self.repo.write_commit(parent, subject, body, + None, root, None)[1] self.repo.transaction_set_ref(None, branch, checksum) - stats = self.repo.commit_transaction(self.cancellable) + stats = self.repo.commit_transaction(None) def cat_file(self, ref, path): """Return the file descriptor of path at ref.""" @@ -93,9 +128,12 @@ class OSTreeRepo(object): ret, content, etag = relative.load_contents() return content - def list_refs(self, resolved=False): + def list_refs(self, ref=None, resolved=False): """Return a list of all refs in the repo.""" - refs = self.repo.list_refs()[1] + if ref: + refs = self.repo.list_refs(ref)[1] + else: + refs = self.repo.list_refs()[1] if not resolved: return refs.keys() return refs @@ -104,7 +142,7 @@ class OSTreeRepo(object): """Remove refspec from the repo.""" if not self.list_refs(ref): raise Exception("Failed to delete ref, it doesn't exist") - self.repo.set_ref_immediate(None, ref, None, self.cancellable) + self.repo.set_ref_immediate(None, ref, None, None) def prune(self): """Remove unreachable objects from the repo.""" @@ -120,7 +158,7 @@ class OSTreeRepo(object): def remove_remote(self, name): """Remove a remote with a given name.""" - self.repo.remote_delete(name, self.cancellable) + self.repo.remote_delete(name, None) def get_remote_url(self, name): """Return the URL for a remote.""" diff --git a/morphlib/ostreeartifactcache.py b/morphlib/ostreeartifactcache.py index 307f6f50..ea659c8f 100644 --- a/morphlib/ostreeartifactcache.py +++ b/morphlib/ostreeartifactcache.py @@ -30,9 +30,9 @@ from morphlib.artifactcachereference import ArtifactCacheReference class OSTreeArtifactCache(object): """Class to provide the artifact cache API using an OSTree repo.""" - def __init__(self, cachedir): + def __init__(self, cachedir, mode): repo_dir = os.path.join(cachedir, 'repo') - self.repo = morphlib.ostree.OSTreeRepo(repo_dir) + self.repo = morphlib.ostree.OSTreeRepo(repo_dir, mode=mode) self.cachedir = cachedir def _get_file_from_remote(self, artifact, remote, metadata_name=None): diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index 08589ea6..cdbb303f 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -623,6 +623,7 @@ class BranchAndMergePlugin(cliapp.Plugin): smd = morphlib.systemmetadatadir.SystemMetadataDir(path) metadata = smd.values() + logging.debug(metadata) systems = [md for md in metadata if 'kind' in md and md['kind'] == 'system'] diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 1d5c1184..efe6735b 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -544,7 +544,7 @@ class DeployPlugin(cliapp.Plugin): def checkout_stratum(self, path, artifact, lac, rac): with open(lac.get(artifact), 'r') as stratum: chunks = [ArtifactCacheReference(c) for c in json.load(stratum)] - #morphlib.builder.download_depends(chunks, lac, rac) + morphlib.builder.download_depends(chunks, lac, rac) for chunk in chunks: self.app.status(msg='Checkout chunk %(name)s.', name=chunk.basename(), chatty=True) @@ -557,7 +557,7 @@ class DeployPlugin(cliapp.Plugin): def checkout_strata(self, path, artifact, lac, rac): deps = artifact.source.dependencies - #morphlib.builder.download_depends(deps, lac, rac) + morphlib.builder.download_depends(deps, lac, rac) for stratum in deps: self.checkout_stratum(path, stratum, lac, rac) morphlib.builder.ldconfig(self.app.runcmd, path) diff --git a/morphlib/util.py b/morphlib/util.py index 00111ff7..8566345d 100644 --- a/morphlib/util.py +++ b/morphlib/util.py @@ -131,10 +131,11 @@ def new_artifact_caches(settings): # pragma: no cover if not os.path.exists(artifact_cachedir): os.mkdir(artifact_cachedir) - #lac = morphlib.localartifactcache.LocalArtifactCache( - # fs.osfs.OSFS(artifact_cachedir)) - - lac = morphlib.ostreeartifactcache.OSTreeArtifactCache(artifact_cachedir) + mode = settings['ostree-repo-mode'] + import logging + logging.debug(mode) + lac = morphlib.ostreeartifactcache.OSTreeArtifactCache(artifact_cachedir, + mode=mode) rac_url = get_artifact_cache_server(settings) rac = None -- cgit v1.2.1