path: root/buildstream/_ostree.py

#
#  Copyright (C) 2017 Codethink Limited
#
#  This program is free software; you can redistribute it and/or
#  modify it under the terms of the GNU Lesser General Public
#  License as published by the Free Software Foundation; either
#  version 2 of the License, or (at your option) any later version.
#
#  This library is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
#  Lesser General Public License for more details.
#
#  You should have received a copy of the GNU Lesser General Public
#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
#
#  Authors:
#        Jürg Billeter <juerg.billeter@codethink.co.uk>
#        Andrew Leeming <andrew.leeming@codethink.co.uk>
#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
#
# Code based on Jürg's artifact cache and Andrew's ostree plugin
#

# Disable pylint warnings that are not appicable to this module
# pylint: disable=bad-exception-context,catching-non-exception

import os
from collections import namedtuple

import gi
from gi.repository.GLib import Variant, VariantDict

from ._exceptions import BstError, ErrorDomain

# pylint: disable=wrong-import-position,wrong-import-order
gi.require_version('OSTree', '1.0')
from gi.repository import GLib, Gio, OSTree  # nopep8


# For users of this file, they must expect (except) it.
class OSTreeError(BstError):
    def __init__(self, message, reason=None):
        super().__init__(message, domain=ErrorDomain.UTIL, reason=reason)


# ensure()
#
# Args:
#    path (str): The file path to where the desired repo should be
#    compress (bool): use compression or not when creating
#
# Returns: an OSTree.Repo
def ensure(path, compress):

    # create also succeeds on existing repository
    repo = OSTree.Repo.new(Gio.File.new_for_path(path))
    mode = OSTree.RepoMode.ARCHIVE_Z2 if compress \
        else OSTree.RepoMode.BARE_USER

    repo.create(mode)

    # Disble OSTree's built in minimum-disk-space check.
    config = repo.copy_config()
    config.set_string('core', 'min-free-space-percent', '0')
    repo.write_config(config)
    repo.reload_config()

    return repo


# checkout()
#
# Checkout the content at 'commit' from 'repo' in
# the specified 'path'
#
# Args:
#    repo (OSTree.Repo): The repo
#    path (str): The checkout path
#    commit_ (str): The commit checksum to checkout
#    user (boot): Whether to checkout in user mode
#
def checkout(repo, path, commit_, user=False):

    # Check out a full copy of an OSTree at a given ref to some directory.
    #
    # Note: OSTree does not like updating directories inline/sync, therefore
    # make sure you checkout to a clean directory or add additional code to support
    # union mode or (if it exists) file replacement/update.
    #
    # Returns True on success
    #
    # cli exmaple:
    #   ostree --repo=repo checkout --user-mode runtime/org.freedesktop.Sdk/x86_64/1.4 foo
    os.makedirs(os.path.dirname(path), exist_ok=True)

    options = OSTree.RepoCheckoutAtOptions()

    # For repos which contain root owned files, we need
    # to checkout with OSTree.RepoCheckoutMode.USER
    #
    # This will reassign uid/gid and also munge the
    # permission bits a bit.
    if user:
        options.mode = OSTree.RepoCheckoutMode.USER

    # Using AT_FDCWD value from fcntl.h
    #
    # This will be ignored if the passed path is an absolute path,
    # if path is a relative path then it will be appended to the
    # current working directory.
    AT_FDCWD = -100
    try:
        repo.checkout_at(options, AT_FDCWD, path, commit_)
    except GLib.GError as e:
        raise OSTreeError("Failed to checkout commit '{}': {}".format(commit_, e.message)) from e


# commit():
#
# Commit built artifact to cache.
#
# Files are all recorded with uid/gid 0
#
# Args:
#    repo (OSTree.Repo): The repo
#    dir_ (str): The source directory to commit to the repo
#    refs (list): A list of symbolic references (tag) for the commit
#
def commit(repo, dir_, refs):

    def commit_filter(repo, path, file_info):

        # For now, just set everything in the repo as uid/gid 0
        #
        # In the future we'll want to extract virtualized file
        # attributes from a fuse layer and use that.
        #
        file_info.set_attribute_uint32('unix::uid', 0)
        file_info.set_attribute_uint32('unix::gid', 0)

        return OSTree.RepoCommitFilterResult.ALLOW

    commit_modifier = OSTree.RepoCommitModifier.new(
        OSTree.RepoCommitModifierFlags.NONE, commit_filter)

    repo.prepare_transaction()
    try:
        # add tree to repository
        mtree = OSTree.MutableTree.new()
        repo.write_directory_to_mtree(Gio.File.new_for_path(dir_),
                                      mtree, commit_modifier)
        _, root = repo.write_mtree(mtree)

        # create root commit object, no parent, no branch
        _, rev = repo.write_commit(None, None, None, None, root)

        # create refs
        for ref in refs:
            repo.transaction_set_ref(None, ref, rev)

        # complete repo transaction
        repo.commit_transaction(None)
    except GLib.GError as e:

        # Reraise any error as a buildstream error
        repo.abort_transaction()
        raise OSTreeError(e.message) from e


# set_ref():
#
# Set symbolic reference to specified revision.
#
# Args:
#    repo (OSTree.Repo): The repo
#    ref (str): A symbolic reference (tag) for the commit
#    rev (str): Commit checksum
#
def set_ref(repo, ref, rev):

    repo.prepare_transaction()
    try:
        repo.transaction_set_ref(None, ref, rev)

        # complete repo transaction
        repo.commit_transaction(None)
    except:
        repo.abort_transaction()
        raise


# exists():
#
# Checks wether a given commit or symbolic ref exists and
# is locally cached in the specified repo.
#
# Args:
#    repo (OSTree.Repo): The repo
#    ref (str): A commit checksum or symbolic ref
#
# Returns:
#    (bool): Whether 'ref' is valid in 'repo'
#
def exists(repo, ref):

    # Get the commit checksum, this will:
    #
    #  o Return a commit checksum if ref is a symbolic branch
    #  o Return the same commit checksum if ref is a valid commit checksum
    #  o Return None if the ostree repo doesnt know this ref.
    #
    ref = checksum(repo, ref)
    if ref is None:
        return False

    # If we do have a ref which the ostree knows about, this does
    # not mean we necessarily have the object locally (we may just
    # have some metadata about it, this can happen).
    #
    # Use has_object() only with a resolved valid commit checksum
    # to check if we actually have the object locally.
    _, has_object = repo.has_object(OSTree.ObjectType.COMMIT, ref, None)
    return has_object


# remove():
#
# Removes the given commit or symbolic ref from the repo.
#
# Args:
#    repo (OSTree.Repo): The repo
#    ref (str): A commit checksum or symbolic ref
#    defer_prune (bool): Whether to defer pruning to the caller. NOTE:
#                        The space won't be freed until you manually
#                        call repo.prune.
#
# Returns:
#    (int|None) The amount of space pruned from the repository in
#               Bytes, or None if defer_prune is True
#
def remove(repo, ref, *, defer_prune=False):

    # Get the commit checksum, this will:
    #
    #  o Return a commit checksum if ref is a symbolic branch
    #  o Return the same commit checksum if ref is a valid commit checksum
    #  o Return None if the ostree repo doesnt know this ref.
    #
    check = checksum(repo, ref)
    if check is None:
        raise OSTreeError("Could not find artifact for ref '{}'".format(ref))

    repo.set_ref_immediate(None, ref, None)

    if not defer_prune:
        _, _, _, pruned = repo.prune(OSTree.RepoPruneFlags.REFS_ONLY, -1)
        return pruned

    return None


# checksum():
#
# Returns the commit checksum for a given symbolic ref,
# which might be a branch or tag. If it is a branch,
# the latest commit checksum for the given branch is returned.
#
# Args:
#    repo (OSTree.Repo): The repo
#    ref (str): The symbolic ref
#
# Returns:
#    (str): The commit checksum, or None if ref does not exist.
#
def checksum(repo, ref):

    _, checksum_ = repo.resolve_rev(ref, True)
    return checksum_


OSTREE_GIO_FAST_QUERYINFO = ("standard::name,standard::type,standard::size,"
                             "standard::is-symlink,standard::symlink-target,"
                             "unix::device,unix::inode,unix::mode,unix::uid,"
                             "unix::gid,unix::rdev")


DiffItem = namedtuple('DiffItem', ['src', 'src_info',
                                   'target', 'target_info',
                                   'src_checksum', 'target_checksum'])


# diff_dirs():
#
# Compute the difference between directory a and b as 3 separate sets
# of OSTree.DiffItem.
#
# This is more-or-less a direct port of OSTree.diff_dirs (which cannot
# be used via PyGobject), but does not support options.
#
# Args:
#    a (Gio.File): The first directory for the comparison.
#    b (Gio.File): The second directory for the comparison.
#
# Returns:
#    (modified, removed, added)
#
def diff_dirs(a, b):
    # get_file_checksum():
    #
    # Helper to compute the checksum of an arbitrary file (different
    # objects have different methods to compute these).
    #
    def get_file_checksum(f, f_info):
        if isinstance(f, OSTree.RepoFile):
            return f.get_checksum()
        else:
            contents = None
            if f_info.get_file_type() == Gio.FileType.REGULAR:
                contents = f.read()

            csum = OSTree.checksum_file_from_input(f_info, None, contents,
                                                   OSTree.ObjectType.FILE)
            return OSTree.checksum_from_bytes(csum)

    # diff_files():
    #
    # Helper to compute a diff between two files.
    #
    def diff_files(a, a_info, b, b_info):
        checksum_a = get_file_checksum(a, a_info)
        checksum_b = get_file_checksum(b, b_info)

        if checksum_a != checksum_b:
            return DiffItem(a, a_info, b, b_info, checksum_a, checksum_b)

        return None

    # diff_add_dir_recurse():
    #
    # Helper to collect all files in a directory recursively.
    #
    def diff_add_dir_recurse(d):
        added = []

        dir_enum = d.enumerate_children(OSTREE_GIO_FAST_QUERYINFO,
                                        Gio.FileQueryInfoFlags.NOFOLLOW_SYMLINKS)

        for child_info in dir_enum:
            name = child_info.get_name()
            child = d.get_child(name)
            added.append(child)

            if child_info.get_file_type() == Gio.FileType.DIRECTORY:
                added.extend(diff_add_dir_recurse(child))

        return added

    modified = []
    removed = []
    added = []

    child_a_info = a.query_info(OSTREE_GIO_FAST_QUERYINFO,
                                Gio.FileQueryInfoFlags.NOFOLLOW_SYMLINKS)
    child_b_info = b.query_info(OSTREE_GIO_FAST_QUERYINFO,
                                Gio.FileQueryInfoFlags.NOFOLLOW_SYMLINKS)

    # If both are directories and have the same checksum, we know that
    # none of the underlying files changed, so we can save time.
    if (child_a_info.get_file_type() == Gio.FileType.DIRECTORY and
            child_b_info.get_file_type() == Gio.FileType.DIRECTORY and
            isinstance(a, OSTree.RepoFileClass) and
            isinstance(b, OSTree.RepoFileClass)):
        if a.tree_get_contents_checksum() == b.tree_get_contents_checksum():
            return modified, removed, added

    # We walk through 'a' first
    dir_enum = a.enumerate_children(OSTREE_GIO_FAST_QUERYINFO,
                                    Gio.FileQueryInfoFlags.NOFOLLOW_SYMLINKS)
    for child_a_info in dir_enum:
        name = child_a_info.get_name()

        child_a = a.get_child(name)
        child_a_type = child_a_info.get_file_type()

        try:
            child_b = b.get_child(name)
            child_b_info = child_b.query_info(OSTREE_GIO_FAST_QUERYINFO,
                                              Gio.FileQueryInfoFlags.NOFOLLOW_SYMLINKS)
        except GLib.Error as e:
            # If the file does not exist in b, it has been removed
            if e.matches(Gio.io_error_quark(), Gio.IOErrorEnum.NOT_FOUND):
                removed.append(child_a)
                continue
            else:
                raise

        # If the files differ but are of different types, we report a
        # modification, saving a bit of time because we won't need a
        # checksum
        child_b_type = child_b_info.get_file_type()
        if child_a_type != child_b_type:
            diff_item = DiffItem(child_a, child_a_info,
                                 child_b, child_b_info,
                                 None, None)
            modified.append(diff_item)
        # Finally, we compute checksums and compare the file contents directly
        else:
            diff_item = diff_files(child_a, child_a_info, child_b, child_b_info)

            if diff_item:
                modified.append(diff_item)

            # If the files are both directories, we recursively use
            # this function to find differences - saving time if they
            # are equal.
            if child_a_type == Gio.FileType.DIRECTORY:
                subdir = diff_dirs(child_a, child_b)
                modified.extend(subdir[0])
                removed.extend(subdir[1])
                added.extend(subdir[2])

    # Now we walk through 'b' to find any files that were added
    dir_enum = b.enumerate_children(OSTREE_GIO_FAST_QUERYINFO,
                                    Gio.FileQueryInfoFlags.NOFOLLOW_SYMLINKS)
    for child_b_info in dir_enum:
        name = child_b_info.get_name()

        child_b = b.get_child(name)

        try:
            child_a = a.get_child(name)
            child_a_info = child_a.query_info(OSTREE_GIO_FAST_QUERYINFO,
                                              Gio.FileQueryInfoFlags.NOFOLLOW_SYMLINKS)
        except GLib.Error as e:
            # If the file does not exist in 'a', it was added.
            if e.matches(Gio.io_error_quark(), Gio.IOErrorEnum.NOT_FOUND):
                added.append(child_b)
                if child_b_info.get_file_type() == Gio.FileType.DIRECTORY:
                    added.extend(diff_add_dir_recurse(child_b))
                continue
            else:
                raise

    return modified, removed, added


# fetch()
#
# Fetch new objects from a remote, if configured
#
# Args:
#    repo (OSTree.Repo): The repo
#    remote (str): An optional remote name, defaults to 'origin'
#    ref (str): An optional ref to fetch, will reduce the amount of objects fetched
#    progress (callable): An optional progress callback
#
# Note that a commit checksum or a branch reference are both
# valid options for the 'ref' parameter. Using the ref parameter
# can save a lot of bandwidth but mirroring the full repo is
# still possible.
#
def fetch(repo, remote="origin", ref=None, progress=None):
    # Fetch metadata of the repo from a remote
    #
    # cli example:
    #  ostree --repo=repo pull --mirror freedesktop:runtime/org.freedesktop.Sdk/x86_64/1.4
    def progress_callback(info):
        status = async_progress.get_status()
        outstanding_fetches = async_progress.get_uint('outstanding-fetches')
        bytes_transferred = async_progress.get_uint64('bytes-transferred')
        fetched = async_progress.get_uint('fetched')
        requested = async_progress.get_uint('requested')

        if status:
            progress(0.0, status)
        elif outstanding_fetches > 0:
            formatted_bytes = GLib.format_size_full(bytes_transferred, 0)
            if requested == 0:
                percent = 0.0
            else:
                percent = (fetched * 1.0 / requested) * 100

            progress(percent,
                     "Receiving objects: {:d}% ({:d}/{:d}) {}".format(int(percent), fetched,
                                                                      requested, formatted_bytes))
        else:
            progress(100.0, "Writing Objects")

    async_progress = None
    if progress is not None:
        async_progress = OSTree.AsyncProgress.new()
        async_progress.connect('changed', progress_callback)

    # FIXME: This hangs the process and ignores keyboard interrupt,
    #        fix this using the Gio.Cancellable
    refs = None
    if ref is not None:
        refs = [ref]

    try:
        repo.pull(remote,
                  refs,
                  OSTree.RepoPullFlags.MIRROR,
                  async_progress,
                  None)  # Gio.Cancellable
    except GLib.GError as e:
        if ref is not None:
            raise OSTreeError("Failed to fetch ref '{}' from '{}': {}".format(ref, remote, e.message)) from e
        else:
            raise OSTreeError("Failed to fetch from '{}': {}".format(remote, e.message)) from e


# configure_remote():
#
# Ensures a remote is setup to a given url.
#
# Args:
#    repo (OSTree.Repo): The repo
#    remote (str): The name of the remote
#    url (str): The url of the remote ostree repo
#    key_url (str): The optional url of a GPG key (should be a local file)
#
def configure_remote(repo, remote, url, key_url=None):
    # Add a remote OSTree repo. If no key is given, we disable gpg checking.
    #
    # cli exmaple:
    #   wget https://sdk.gnome.org/keys/gnome-sdk.gpg
    #   ostree --repo=repo --gpg-import=gnome-sdk.gpg remote add freedesktop https://sdk.gnome.org/repo
    options = None  # or GLib.Variant of type a{sv}
    if key_url is None:
        vd = VariantDict.new()
        vd.insert_value('gpg-verify', Variant.new_boolean(False))
        options = vd.end()

    try:
        repo.remote_change(None,      # Optional OSTree.Sysroot
                           OSTree.RepoRemoteChange.ADD_IF_NOT_EXISTS,
                           remote,    # Remote name
                           url,       # Remote url
                           options,   # Remote options
                           None)      # Optional Gio.Cancellable
    except GLib.GError as e:
        raise OSTreeError("Failed to configure remote '{}': {}".format(remote, e.message)) from e

    # Remote needs to exist before adding key
    if key_url is not None:
        try:
            gfile = Gio.File.new_for_uri(key_url)
            stream = gfile.read()
            repo.remote_gpg_import(remote, stream, None, 0, None)
        except GLib.GError as e:
            raise OSTreeError("Failed to add gpg key from url '{}': {}".format(key_url, e.message)) from e


# list_artifacts():
#
# List cached artifacts in Least Recently Modified (LRM) order.
#
# Returns:
#     (list) - A list of refs in LRM order
#
def list_artifacts(repo):
    # string of: /path/to/repo/refs/heads
    ref_heads = os.path.join(repo.get_path().get_path(), 'refs', 'heads')

    # obtain list of <project>/<element>/<key>
    refs = _list_all_refs(repo).keys()

    mtimes = []
    for ref in refs:
        ref_path = os.path.join(ref_heads, ref)
        if os.path.exists(ref_path):
            # Obtain the mtime (the time a file was last modified)
            mtimes.append(os.path.getmtime(ref_path))

    # NOTE: Sorted will sort from earliest to latest, thus the
    # first element of this list will be the file modified earliest.
    return [ref for _, ref in sorted(zip(mtimes, refs))]


# _list_all_refs():
#
# Create a list of all refs.
#
# Args:
#    repo (OSTree.Repo): The repo
#
# Returns:
#    (dict): A dict of refs to checksums.
#
def _list_all_refs(repo):
    try:
        _, refs = repo.list_refs(None)
        return refs
    except GLib.GError as e:
        raise OSTreeError(message=e.message) from e