path: root/morphlib/morphologyfactory.py

# Copyright (C) 2012-2014  Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.


import base64
import logging

import morphlib
import cliapp


class MorphologyFactoryError(cliapp.AppException):
    pass


class AutodetectError(MorphologyFactoryError):
    def __init__(self, repo_name, ref, filename):
        MorphologyFactoryError.__init__(
            self, "Failed to determine the build system of repo %s at "
                  "ref %s: was looking for %s" % (repo_name, ref, filename))


class NotcachedError(MorphologyFactoryError):
    def __init__(self, repo_name):
        MorphologyFactoryError.__init__(
            self, "Repository %s is not cached locally and there is no "
                  "remote cache specified" % repo_name)


class StratumError(MorphologyFactoryError):
    pass


class NoChunkBuildDependsError(StratumError):
    def __init__(self, stratum, chunk):
        StratumError.__init__(
            self, 'No build dependencies in stratum %s for chunk %s '
                  '(build-depends is a mandatory field)' % (stratum, chunk))


class EmptyStratumError(StratumError):

    def __init__(self, stratum):
        cliapp.AppException.__init__(self,
            "Stratum %s is empty (has no dependencies)" % stratum)


class NoStratumBuildDependsError(StratumError):
    def __init__(self, stratum):
        StratumError.__init__(
            self, 'Stratum %s has no build-dependencies listed '
                  'and has no bootstrap chunks.' % stratum)


class MorphologyFactory(object):

    '''An way of creating morphologies which will provide a default'''

    def __init__(self, local_repo_cache, remote_repo_cache=None, app=None):
        self._lrc = local_repo_cache
        self._rrc = remote_repo_cache
        self._app = app

    def status(self, *args, **kwargs):  # pragma: no cover
        if self._app is not None:
            self._app.status(*args, **kwargs)

    def _read_local_repo(self, reponame, sha1, filename):
        '''Fetch file list and maybe morphology text from local repo cache.'''
        text = None
        repo = self._lrc.get_repo(reponame)
        file_list = repo.ls_tree(sha1)
        if filename in file_list:
            text = repo.cat(sha1, filename)
        return file_list, text

    def _read_remote_repo(self, reponame, sha1, filename):
        '''Fetch file list and maybe morphology text from remote repo cache.'''
        text = None
        file_list = self._rrc.ls_tree(reponame, sha1)
        if filename in file_list:
            self.status(msg="Retrieving %(reponame)s %(sha1)s %(filename)s"
                        " from the remote artifact cache.",
                        reponame=reponame, sha1=sha1, filename=filename,
                        chatty=True)
            text = self._rrc.cat_file(reponame, sha1, filename)
        return file_list, text

    def _parse_or_generate_morphology(self, reponame, sha1, filename,
                                      file_list, text):
        '''Parse text or autodetect morphology contents based on file list.'''
        if text is None:
            bs = morphlib.buildsystem.detect_build_system(file_list)
            if bs is None:
                raise AutodetectError(reponame, sha1, filename)
            # TODO consider changing how morphs are located to be by morph
            #      name rather than filename, it would save creating a
            #      filename only to strip it back to its morph name again
            #      and would allow future changes like morphologies being
            #      stored as git metadata instead of as a file in the repo
            morph_name = filename[:-len('.morph')]
            text = bs.get_morphology_text(morph_name)

        try:
            morphology = morphlib.morph2.Morphology(text)
        except morphlib.YAMLError as e: # pragma: no cover
            raise morphlib.Error("Error parsing %s: %s" %
                                 (filename, str(e)))

        if filename != morphology['name'] + '.morph':
            raise morphlib.Error(
                "Name %s does not match basename of morphology file %s" %
                (morphology['name'], filename))

        method_name = '_check_and_tweak_%s' % morphology['kind']
        if hasattr(self, method_name):
            method = getattr(self, method_name)
            method(morphology, reponame, sha1, filename)

        return morphology

    def get_morphology(self, reponame, sha1, filename):
        if self._lrc.has_repo(reponame):
            file_list, text = self._read_local_repo(reponame, sha1, filename)
        elif self._rrc is not None:
            file_list, text = self._read_remote_repo(reponame, sha1, filename)
        else:
            raise NotcachedError(reponame)

        morphology = self._parse_or_generate_morphology(reponame, sha1,
                                                        filename, file_list,
                                                        text)
        return morphology

    def get_morphologies(self, resolved_refs, resolved_morphologies, triplets):
        '''Fetch morphologies that are not already resolved.

        The resolved_morphologies dict is updated with the new morphologies.

        All refs in the list of triplets are assumed to be stored in the
        resolved_refs dict already.

        '''
        text_dict = {}
        file_list_dict = {}
        to_read = dict()
        to_autodetect = set()
        for triplet in triplets:
            if triplet in resolved_morphologies:
                continue

            reponame, ref, filename = triplet
            absref, tree = resolved_refs[(reponame, ref)]

            if self._lrc.has_repo(reponame):
                text = self._read_local_repo(
                    reponame, absref, filename)
                text_dict[(reponame, ref, filename)] = text
            if self._rrc is None:
                raise NotcachedError(reponame)
            else:
                repourl = self._rrc._resolver.pull_url(reponame)
                remote_triplet = (repourl, absref, filename)
                #print 'to_read[%s,%s,%s] set' % remote_triplet
                to_read[remote_triplet] = triplet

        if len(to_read) > 0:
            self.status(msg='Fetching %i morphologies from remote repo cache' %
                        len(to_read))
            result = self._rrc.cat_file_multiple(to_read.keys())
            for item in result:
                remote_triplet = (item['repo'], item['ref'], item['filename'])
                triplet = to_read[remote_triplet]
                if 'data' in item:
                    text = base64.decodestring(item['data'])
                    text_dict[triplet] = text
                else:
                    logging.debug('Remote cache: %s', item)
                    to_autodetect.add(triplet)

        for triplet in to_autodetect:
            reponame, ref, filename = triplet
            self.status(msg='Fetching file list for %s from remote repo cache'
                        % reponame, chatty=True)
            absref, tree = resolved_refs[(reponame, ref)]
            file_list = self._rrc.ls_tree(reponame, absref)
            assert file_list is not None
            file_list_dict[triplet] = file_list

        for triplet in triplets:
            if triplet in resolved_morphologies:
                continue
            reponame, ref, filename = triplet
            absref, tree = resolved_refs[(reponame, ref)]
            file_list = file_list_dict.get(triplet, [])
            text = text_dict.get(triplet, None)
            assert text or file_list
            morphology = self._parse_or_generate_morphology(
                reponame, absref, filename, file_list, text)
            resolved_morphologies[triplet] = morphology

    def _check_and_tweak_system(self, morphology, reponame, sha1, filename):
        '''Check and tweak a system morphology.'''

        if morphology['arch'] is None:  # pragma: no cover
            raise morphlib.Error('No arch specified in system %s '
                                 '(arch is a mandatory field)' %
                                 filename)

        if morphology['arch'] == 'armv7':
            morphology._dict['arch'] = 'armv7l'

        if morphology['arch'] not in morphlib.valid_archs:
            raise morphlib.Error('Unknown arch %s. This version of Morph '
                                 'supports the following architectures: %s' %
                                 (morphology['arch'],
                                  ', '.join(morphlib.valid_archs)))

        name = morphology['name']
        morphology.builds_artifacts = [name + '-rootfs']

        morphology.needs_artifact_metadata_cached = False

        morphlib.morphloader.MorphologyLoader._validate_stratum_specs_fields(
            morphology, 'strata')
        morphlib.morphloader.MorphologyLoader._set_stratum_specs_defaults(
            morphology, 'strata')

    def _check_and_tweak_stratum(self, morphology, reponame, sha1, filename):
        '''Check and tweak a stratum morphology.'''

        if len(morphology['chunks']) == 0:
            raise EmptyStratumError(morphology['name'])

        for source in morphology['chunks']:
            if source.get('build-depends', None) is None:
                name = source.get('name', source.get('repo', 'unknown'))
                raise NoChunkBuildDependsError(filename, name)

        if (len(morphology['build-depends'] or []) == 0 and
            not any(c.get('build-mode') in ('bootstrap', 'test')
                    for c in morphology['chunks'])):
            raise NoStratumBuildDependsError(filename)

        morphology.builds_artifacts = [morphology['name']]
        morphology.needs_artifact_metadata_cached = True

        morphlib.morphloader.MorphologyLoader._validate_stratum_specs_fields(
            morphology, 'build-depends')
        morphlib.morphloader.MorphologyLoader._set_stratum_specs_defaults(
            morphology, 'build-depends')

    def _check_and_tweak_chunk(self, morphology, reponame, sha1, filename):
        '''Check and tweak a chunk morphology.'''

        if 'products' in morphology and len(morphology['products']) > 1:
            morphology.builds_artifacts = [d['artifact']
                                           for d in morphology['products']]
        else:
            morphology.builds_artifacts = [morphology['name']]

        morphology.needs_artifact_metadata_cached = False

        morphlib.morphloader.MorphologyLoader._validate_chunk(morphology)