path: root/ybd/rpm.py

import os
import re
import sys
from glob import glob
from collections import Mapping
from cache import cache_key, get_cache, md5
from app import log, timer
import assembly
import time
import app
import sandbox
import shutil
import yaml
import repos
import requests
import tempfile
import utils
import subprocess


# Because rpm is otherwise totally broken
# NOTE: _build_name_fmt would ordinary be
# %%{NAME}-%%{VERSION}-%%{RELEASE}.%%{ARCH}.rpm
# but we are pulling them out of a cache with a different naming scheme now.
#
common_rpm_args = (
  '--dbpath=/var/lib/rpm '
  '--define "_rpmconfigdir /usr/lib/rpm" '
  '--define "_rpmlock_path /var/lib/rpm/.rpm.lock" '
  '--define "_fileattrsdir /usr/lib/rpm/fileattrs" '
  '--define "_build_name_fmt %%{NAME}.rpm" '
  '--define "_rpmfilename %{_build_name_fmt}" '
  '--define "_tmppath /tmp" '
  '--define "_unpackaged_files_terminate_build 0" '
  '--define "__spec_clean_cmd echo" ')


# foreach_def
# @dn: The target definition or name
# @callback: The callback to call
# @user_data: Something to be passed to @callback along with the definition
#
# Iterates over each chunk which is to be included
# in the target @dn in order of dependency, the passed
# @dn definition will be the last one called.
#
def foreach_def(dn, callback, user_data, traversed=None, whitelist=None):
    if traversed is None:
        traversed = {}

    if type(dn) is not dict:
        dn = app.defs.get(dn)

    # if we can't calculate cache key, we can't create this component
    if cache_key(dn) is False:
        if 'tried' not in dn:
            log(dn, 'No cache_key, so skipping compose')
            dn['tried'] = True
        yield False, None

    # if dn is already cached, we're done
    if not get_cache(dn):
        log('SMTH', '%s is not cached, try again sucker !' % dn['name'],
            exit=True)

    systems = dn.get('systems', [])
    for system in systems:
        for s in system.get('subsystems', []):
            subsystem = app.defs.get(s['path'])
            for res in foreach_def(
                    subsystem, callback, user_data, traversed, whitelist):
                yield res[0], res[1]
        for res in foreach_def(
                system['path'], callback, user_data, traversed, whitelist):
            yield res[0], res[1]

    contents = dn.get('contents', [])
    for it in contents:
        item = app.defs.get(it)

        if traversed.get(item.get('name')):
            continue

        if item.get('build-mode', 'staging') != 'bootstrap':

            if not traversed.get(item.get('name')):
                for res in foreach_def(
                        item, callback, user_data, traversed, whitelist):
                    yield res[0], res[1]

    result = True
    if whitelist is None or dn.get('name') in whitelist:
        result = callback(dn, user_data)

    traversed[dn.get('name')] = True
    yield True, result


def expand_macro(system, dn, text):
    # Macros are identified by use of %
    if r'%' not in text:
        return text

    defines = ['--define "{} {}"'.format(ent['key'], ent['value'])
               for ent in dn['rpm-metadata'].get('defines', [])]
    defines.append('--define "{} {}"'.format('name', dn['name']))
    command = ('rpm ' + common_rpm_args + ' '.join(defines) +
               ' --eval=%s' % text)
    env_vars = sandbox.env_vars_for_build(system)
    # Force subprocess to return output, we will log after the call.
    _, output, _ = sandbox.run_sandboxed(
        system, command, env_vars, run_logged=False, print_command=False)
    with open(system['log'], 'a') as logfile:
        logfile.write(output)
    return output.strip()


def is_main_package(system, dn, package):
    return expand_macro(system, dn, package) == dn['name']


def generate_spec(dn, stage_dir, metafile, output, name, system):
    with open(metafile, "r") as metafile_f:
        metadata = yaml.safe_load(metafile_f)

    with open(output, "w") as output_f:
        rpm_metadata = dn['rpm-metadata']
        package_list = rpm_metadata.get('packages', [])

        description = 'No Description'

        if metadata.get('description') is not None:
            description = metadata.get('description')

        # Write out the package header first
        output_f.write('Name: %s\n' % name)
        output_f.write('Summary: %s\n' % description)
        output_f.write('License: %s\n' % 'Undetermined')
        output_f.write('Prefix: %s\n' % dn.get('prefix', '/usr'))
        list_fields = ('Requires', 'Conflicts', 'Provides', 'Obsoletes')
        # Add all the main package Provides:
        for package in package_list:
            if is_main_package(system, dn, package['name']):
                vendor = next((d['vendor'] for d in (package, rpm_metadata)
                               if 'vendor' in d), None)
                if vendor:
                    output_f.write('Vendor: %s\n' % vendor)
                version = next((d['version'] for d in (package, rpm_metadata)
                                if 'version' in d), None)
                if version:
                    output_f.write('Version: %s\n' % version)
                else:
                    app.log(dn, "Main package has no version!")
                    return False
                release = next((d['release'] for d in (package, rpm_metadata)
                                if 'release' in d), None)
                if release:
                    output_f.write('Release: {}git{}\n'
                                   .format(release, dn['sha']))
                else:
                    app.log(dn, "Main package has no release!")
                    return False
                epoch = next((d['epoch'] for d in (package, rpm_metadata)
                              if 'epoch' in d), None)
                if epoch:
                    output_f.write('Epoch: %s\n' % epoch)
                for field in list_fields:
                    field_l = field.lower()
                    if field_l in package:
                        for item in package[field_l]:
                            output_f.write('%s: %s\n' % (field, item))
        output_f.write('\n')

        output_f.write('%%description\n')
        output_f.write('%s\n' % description)
        output_f.write('\n')

        for package in package_list:
            if not is_main_package(system, dn, package['name']):
                # Sub-Package header
                output_f.write('%%package -n %s\n' % package['name'])
                output_f.write('Summary: %s\n' % description)
                if 'vendor' in package:
                    output_f.write('Vendor: %s\n' % package['vendor'])
                if 'version' in package:
                    output_f.write('Version: %s\n' % package['version'])
                if 'release' in package:
                    output_f.write('Release: {}git{}\n'
                                   .format(package['release'], dn['sha']))
                if 'epoch' in package:
                    output_f.write('Epoch: %s\n' % package['epoch'])
                for field in list_fields:
                    field_l = field.lower()
                    if field_l in package:
                        for item in package[field_l]:
                            output_f.write('%s: %s\n' % (field, item))
                output_f.write('\n')
                output_f.write('%%description -n %s\n' % package['name'])
                output_f.write('%s\n' % description)
                output_f.write('\n')

            for cmd in ['pre', 'preun', 'post', 'postun']:
                if cmd in package:
                    if 'interpreter' in package[cmd]:
                        interp_str = ' -p %s' % package[cmd]['interpreter']
                    else:
                        # To be explicit, this might default to /bin/sh,
                        # but that's more verbose than is strictly needed.
                        interp_str = ''
                    output_f.write('%%%s -n %s%s\n'
                                   % (cmd, package['name'], interp_str))
                    if 'body' in package[cmd]:
                        # Expecting it to be a single multi-line command,
                        # since running as separate commands discards env.
                        output_f.write(package[cmd]['body'] + '\n')
                    output_f.write('\n')

            # Skip writing files if nonexistent/empty
            if type(package.get('files')) is not list:
                continue

            metafile = package.get('metafile', '').lstrip(os.sep)
            if metafile:
                metafile = " -f " + metafile

            # Sub-Package files
            if package['name'] == name:
                output_f.write('%%files%s\n' % metafile)
            else:
                output_f.write('%%files -n %s%s\n' %
                               (package['name'], metafile))
            for filename in package['files']:
                output_f.write(filename + "\n")
            output_f.write('\n')

    return True


def extract_defines(dn):
    meta = dn.get('rpm-metadata')
    if meta is None or isinstance(meta, list):
        return ''

    defines = meta.get('defines')
    if defines is None:
        return ''

    strings = []
    for define in defines:
        key = define.get('key')
        value = define.get('value')

        strings.append(' --define "{} {}"'.format(key, value))

    return ''.join(strings)


def extract_metafiles(system, dn, instdir, metadir):
    packages = dn['rpm-metadata'].get('packages', [])
    for package in packages:
        metafile = package.get('metafile', '').lstrip(os.sep)
        metafile = expand_macro(system, dn, metafile)
        if metafile:
            shutil.move(os.path.join(instdir, metafile), metadir)


def get_package_names_with_fields(system, dn, fields):
    # Ignores subpackages without files sections as those aren't generated.
    if 'rpm-metadata' not in dn:
        app.log(dn, "Tried to get package names for definition "
                "without rpm-metadata!")
        sys.exit(1)
    if 'packages' not in dn['rpm-metadata']:
        app.log(dn, "Tried to get package names for definition "
                "with no packages in rpm-metadata!")
        sys.exit(1)
    if any(all(key in package for key in ('path', 'files'))
           for package in dn['rpm-metadata']['packages']):
        app.log(dn, "ERROR: Cannot mix 'path' (prebuilts) and 'files' "
                "(generated) in rpm-metadata")
        sys.exit(1)
    return {expand_macro(system, dn, package['name'])
            for package in dn['rpm-metadata']['packages']
            if any(field in package for field in fields)}


def get_generated_package_names(system, dn):
    return get_package_names_with_fields(system, dn, ['files', 'path'])


def get_remote_rpm(dn, pkgfilename):
    # Defaults to allowing rpms to be fetched from kbas.
    # Override kbas-upload to prevent this.
    if 'rpm' not in app.config.get('kbas-upload', 'rpm'):
        return False
    if 'kbas-url' not in app.config:
        return False
    try:
        app.log(dn, 'Try downloading', pkgfilename)
        url = "{}get/{}".format(app.config['kbas-url'], pkgfilename)
        response = requests.get(url=url, stream=True)
    except Exception as e:
        app.config.pop('kbas-url')
        app.log(dn, "WARNING: Failure to call artifact server: ", e)
        return False

    if response.status_code == 200:
        try:
            tempfile.tempdir = app.config['tmp']
            tmpdir = tempfile.mkdtemp()
            cachefile = os.path.join(tmpdir, pkgfilename)
            with open(cachefile, 'wb') as f:
                f.write(response.content)
            cache_dst = os.path.join(app.config['artifacts'], pkgfilename)
            os.rename(cachefile, cache_dst)
            size = os.path.getsize(cache_dst)
            size = re.sub("(\d)(?=(\d{3})+(?!\d))", r"\1,", "%d" % size)
            app.log(dn, 'Cached %s bytes as' % size, pkgfilename)
            return True
        except Exception as e:
            app.log(dn, "WARNING: Failed to download {}: {}"
                    .format(pkgfilename, e))
    return False


def get_cache_pkgfilename(pkgname, dn):
    return "{}.{}.rpm".format(pkgname, cache_key(dn))


def all_rpms_cached(system, dn):
    for pkgname in get_generated_package_names(system, dn):
        pkgfn = get_cache_pkgfilename(pkgname, dn)
        cached_path = os.path.join(app.config['artifacts'], pkgfn)
        if (not os.path.exists(cached_path)
                and not get_remote_rpm(dn, pkgfn)):
            return False
    return True


def stage_prebuilts(dn, system):
    for package in dn['rpm-metadata']['packages']:
        if 'path' not in package:
            continue
        basedir = os.path.join(system['sandbox'], '%s.inst' % dn['name'])
        cache_path = os.path.join(
            system['sandbox'], 'RPMS', '%s.rpm' % package['name'])
        rpm_glob = sorted(glob(
            os.path.join(basedir, package['path'].lstrip('/'))))
        if not rpm_glob:
            log(dn, 'Failed to find prebuilt rpm matching', package['path'])
            sys.exit(1)
        if len(rpm_glob) > 1:
            exit = app.config.get('check-definitions') == 'exit'
            log(dn, 'WARNING: glob pattern %s matches multiple paths' %
                package['path'], exit=exit)
            log(dn, 'WARNING: Taking %s from glob match' % rpm_glob[0])
        os.rename(rpm_glob[0], cache_path)


def compose_rpm(dn, userdata):
    # a chunk lacking in rpm-metadata is not an error.
    # It means this should be skipped without doing anything.
    if 'rpm-metadata' not in dn:
        return True
    if not all_rpms_cached(userdata['system'], dn):
        if not package_one_rpm(dn, userdata):
            return False

        if not cache_generated_rpms(userdata['system'], dn):
            return False

    return True


def upload_generated_rpm(dn, cachefile):
    filename = os.path.basename(cachefile)
    url = app.config['kbas-url'] + 'upload'
    params = {"filename": filename,
              "password": app.config['kbas-password'],
              "checksum": md5(cachefile)}
    with open(cachefile, 'rb') as f:
        try:
            response = requests.post(url=url, data=params, files={"file": f})
            if response.status_code == 201:
                app.log(dn, 'Uploaded %s to kbas' % filename)
                return
            if response.status_code == 777:
                app.log(dn, 'Reproduced %s at' % md5(cachefile), filename)
                app.config['reproduced'].append([md5(cachefile), filename])
                return
            if response.status_code == 405:
                app.log(dn, 'Artifact server already has', filename)
                return
            app.log(dn, 'Artifact server problem:', response.status_code)
        except:
            pass
        app.log(dn, 'Failed to upload', filename)


def cache_generated_rpms(system, dn):
    rpms_dir = os.path.join(system['sandbox'], 'RPMS')
    for pkg in get_generated_package_names(system, dn):
        pkgfile = "{}.rpm".format(pkg)
        pkgpath = os.path.join(rpms_dir, pkgfile)
        cachepath = os.path.join(app.config['artifacts'],
                                 get_cache_pkgfilename(pkg, dn))
        if not os.path.exists(pkgpath):
            app.log(dn, "Can't extract rpms, {} is missing!".format(pkgpath))
            return False
        os.rename(pkgpath, cachepath)

        # Upload the cached rpm, if applicable
        if (app.config.get('kbas-password', 'insecure') != 'insecure'
                and 'kbas-url' in app.config
                and 'rpm' in app.config.get('kbas-upload', 'rpm')):
            with app.timer(dn, 'Upload {}'.format(pkgfile)):
                upload_generated_rpm(dn, cachepath)

    return True


def package_one_rpm(dn, userdata):

    system = userdata['system']
    kind = dn.get('kind')
    name = dn.get('name')

    if kind == 'chunk' or kind is None:

        with timer(name):
            subdir = '%s.inst' % name
            fulldir = os.path.join(system['sandbox'], subdir)
            metadir = os.path.join(system['sandbox'], '%s.meta' % name)
            baserockdir = os.path.join(fulldir, 'baserock')

            # Install the chunk we're gonna package under subdir
            sandbox.install(system, dn, subdir)

            # Move the baserock directory out of the way,
            # we don't package the metadata
            shutil.move(baserockdir, metadir)

            stage_prebuilts(dn, system)

            # Generate the specfile in the metadir, note that we use
            # the metadata for the given package from the system metadata
            # directory, not the metadata for the specific chunk artifact

            # XXX Right now the chunk's individual metadata is richer, it
            # includes the desciption, change this to use the system metadata
            # for that chunk later !!!
            metafile = os.path.join(metadir, '%s.meta' % name)
            specfile = os.path.join(metadir, '%s.spec' % name)
            success = True

            if not any(to_build for to_build
                       in dn['rpm-metadata']['packages']
                       if 'files' in to_build):
                log(dn, 'WARNING: Nothing to package in %s, skipping' % name)
                return success

            if generate_spec(dn, fulldir, metafile, specfile,
                             name, system):
                defines = extract_defines(dn)
                extract_metafiles(system, dn, fulldir, metadir)

                # XXX Now we gonna run rpmbuild in the sandbox !!!
                command = ('rpmbuild ' + common_rpm_args +
                           ' --buildroot=/%s.inst' % name +
                           ' --define "_builddir /%s.meta"' % name +
                           ' --define "_rpmdir /RPMS"' + defines +
                           ' --target %s' % app.config['cpu'] +
                           ' -bb /%s.meta/%s.spec' % (name, name))
                env_vars = sandbox.env_vars_for_build(system)

                # Keep building all the rpms we can even if one fails,
                # we'd rather see everything that failed in a log at once.
                success, _, _ = sandbox.run_sandboxed(
                    system, command, env_vars, exit_on_error=False)
            else:
                success = False
                app.log(dn, "Failed to generate spec, not generating rpm")

            if success:
                app.log(dn, "Removing sandbox dir", fulldir, verbose=True)
                shutil.rmtree(fulldir)
                shutil.rmtree(metadir)
            else:
                app.log(dn, "Failed rpm build sandbox dir:", fulldir)

            return success
    return True


def rpm_deployment_filename(dn, rpmpath):
    # Reads rpm's headers to construct its filename.
    out = ""
    qf = '--queryformat=%{name}-%{version}-%{release}.%{arch}.rpm'
    try:
        with open(os.devnull, 'w') as fnull:
            out = subprocess.check_output(['rpm', '-q', '-p', rpmpath, qf],
                                          stderr=fnull)
    except subprocess.CalledProcessError:
        out = ""
    if not out:
        app.log(dn, "ERROR: Failed to generate name from %s headers" % rpmpath)
        sys.exit(1)
    return out


def deploy_rpm(dn, userdata):
    # Skip this if there's no rpm-metadata
    if 'rpm-metadata' not in dn:
        return True

    for pkgname in get_generated_package_names(userdata['system'], dn):
        pkgfn = get_cache_pkgfilename(pkgname, dn)
        cached_path = os.path.join(app.config['artifacts'], pkgfn)
        if not os.path.exists(cached_path):
            app.log(dn, "WARNING: Missing cached file {}".format(cached_path))
            return False
        dstfilename = rpm_deployment_filename(dn, cached_path)
        dstpath = os.path.join(userdata['dstdir'], dstfilename)
        shutil.copyfile(cached_path, dstpath)
        app.log(dn, "Copied %s to deployment area" % dstfilename, verbose=True)
    return True


def deploy_rpms(system, whitelist=None):
    dstdir = os.path.join(app.config['deployment'], 'RPMs', cache_key(system))
    utils.makedirs(dstdir, exist_ok=True)
    userdata = {'system': system, 'dstdir': dstdir}
    deploy_results = foreach_def(system, deploy_rpm, userdata,
                                 whitelist=whitelist)
    errors = any(not t[1] for t in deploy_results)
    if errors:
        app.log(system, "ERROR: Failed to deploy all RPMs!")
        sys.exit(1)
    app.log(system, "Finished deploying RPMs to %s!" % dstdir)


# package_rpms
# @system: The system to package rpms for
# @whitelist: A whitelist of chunk names to package rpms for
#
# This function will first stage the given @system, which
# must have an rpm installation, and then it will use the
# metadata in the system's baserock directory to package
# each individual chunk, by staging those chunks one by one
# and packaging them in a chroot.
#
# Care will be taken to build the rpms in the order of their
# dependencies, this should allow rpm to infer package dependencies
# correctly
#
def package_rpms(system, whitelist=None):

    if type(system) is not dict:
        system = app.defs.get(system)

    with sandbox.setup(system):
        assembly.install_contents(system, assembly.compose)
        if system.get('kind') != "system":
            assembly.install_dependencies(system, assembly.compose)

        # Fail now if missing `rpm` or `rpmbuild`
        env_vars = sandbox.env_vars_for_build(system)
        test_cmds = ['rpm --version', 'rpmbuild --version']
        for cmd in test_cmds:
            error = sandbox.run_sandboxed(
                system, cmd, env_vars, exit_on_error=False)
            if error:
                log('RPM-BUILD', 'To generate rpms, appropriate rpm and '
                    'rpmbuild binaries are required', exit=True)

        # First initialize the db
        rpmdb_path = os.path.join(system['sandbox'], 'var', 'lib', 'rpm')
        rpmdir = os.path.join(system['sandbox'], 'RPMS')
        command = 'rpm ' + common_rpm_args + ' --initdb'
        # os.path.exists turned out to be unreliable, so ensuring the
        # directories exist is done via catching any EEXIST errors.
        utils.makedirs(rpmdb_path, exist_ok=True)
        utils.makedirs(rpmdir, exist_ok=True)

        sandbox.run_sandboxed(system, command, env_vars)

        # Package each rpm in order of build dependency
        package_results = foreach_def(
            system, compose_rpm, {'system': system},
            whitelist=whitelist)
        errors = any(not t[1] for t in package_results)
        if errors:
            log(system, 'ERROR: Failed to successfully generate all rpms!')
            sys.exit(1)

        deploy_rpms(system, whitelist)