# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import base64 import logging import morphlib import cliapp class MorphologyFactoryError(cliapp.AppException): pass class AutodetectError(MorphologyFactoryError): def __init__(self, repo_name, ref, filename): MorphologyFactoryError.__init__( self, "Failed to determine the build system of repo %s at " "ref %s: was looking for %s" % (repo_name, ref, filename)) class NotcachedError(MorphologyFactoryError): def __init__(self, repo_name): MorphologyFactoryError.__init__( self, "Repository %s is not cached locally and there is no " "remote cache specified" % repo_name) class StratumError(MorphologyFactoryError): pass class NoChunkBuildDependsError(StratumError): def __init__(self, stratum, chunk): StratumError.__init__( self, 'No build dependencies in stratum %s for chunk %s ' '(build-depends is a mandatory field)' % (stratum, chunk)) class EmptyStratumError(StratumError): def __init__(self, stratum): cliapp.AppException.__init__(self, "Stratum %s is empty (has no dependencies)" % stratum) class NoStratumBuildDependsError(StratumError): def __init__(self, stratum): StratumError.__init__( self, 'Stratum %s has no build-dependencies listed ' 'and has no bootstrap chunks.' % stratum) class MorphologyFactory(object): '''An way of creating morphologies which will provide a default''' def __init__(self, local_repo_cache, remote_repo_cache=None, app=None): self._lrc = local_repo_cache self._rrc = remote_repo_cache self._app = app def status(self, *args, **kwargs): # pragma: no cover if self._app is not None: self._app.status(*args, **kwargs) def _read_local_repo(self, reponame, sha1, filename): '''Fetch file list and maybe morphology text from local repo cache.''' text = None repo = self._lrc.get_repo(reponame) file_list = repo.ls_tree(sha1) if filename in file_list: text = repo.cat(sha1, filename) return file_list, text def _read_remote_repo(self, reponame, sha1, filename): '''Fetch file list and maybe morphology text from remote repo cache.''' text = None file_list = self._rrc.ls_tree(reponame, sha1) if filename in file_list: self.status(msg="Retrieving %(reponame)s %(sha1)s %(filename)s" " from the remote artifact cache.", reponame=reponame, sha1=sha1, filename=filename, chatty=True) text = self._rrc.cat_file(reponame, sha1, filename) return file_list, text def _parse_or_generate_morphology(self, reponame, sha1, filename, file_list, text): '''Parse text or autodetect morphology contents based on file list.''' if text is None: bs = morphlib.buildsystem.detect_build_system(file_list) if bs is None: raise AutodetectError(reponame, sha1, filename) # TODO consider changing how morphs are located to be by morph # name rather than filename, it would save creating a # filename only to strip it back to its morph name again # and would allow future changes like morphologies being # stored as git metadata instead of as a file in the repo morph_name = filename[:-len('.morph')] text = bs.get_morphology_text(morph_name) try: morphology = morphlib.morph2.Morphology(text) except morphlib.YAMLError as e: # pragma: no cover raise morphlib.Error("Error parsing %s: %s" % (filename, str(e))) if filename != morphology['name'] + '.morph': raise morphlib.Error( "Name %s does not match basename of morphology file %s" % (morphology['name'], filename)) method_name = '_check_and_tweak_%s' % morphology['kind'] if hasattr(self, method_name): method = getattr(self, method_name) method(morphology, reponame, sha1, filename) return morphology def get_morphology(self, reponame, sha1, filename): if self._lrc.has_repo(reponame): file_list, text = self._read_local_repo(reponame, sha1, filename) elif self._rrc is not None: file_list, text = self._read_remote_repo(reponame, sha1, filename) else: raise NotcachedError(reponame) morphology = self._parse_or_generate_morphology(reponame, sha1, filename, file_list, text) return morphology def get_morphologies(self, resolved_refs, resolved_morphologies, triplets): '''Fetch morphologies that are not already resolved. The resolved_morphologies dict is updated with the new morphologies. All refs in the list of triplets are assumed to be stored in the resolved_refs dict already. ''' text_dict = {} file_list_dict = {} to_read = dict() to_autodetect = set() for triplet in triplets: if triplet in resolved_morphologies: continue reponame, ref, filename = triplet absref, tree = resolved_refs[(reponame, ref)] if self._lrc.has_repo(reponame): text = self._read_local_repo( reponame, absref, filename) text_dict[(reponame, ref, filename)] = text if self._rrc is None: raise NotcachedError(reponame) else: repourl = self._rrc._resolver.pull_url(reponame) remote_triplet = (repourl, absref, filename) #print 'to_read[%s,%s,%s] set' % remote_triplet to_read[remote_triplet] = triplet if len(to_read) > 0: self.status(msg='Fetching %i morphologies from remote repo cache' % len(to_read)) result = self._rrc.cat_file_multiple(to_read.keys()) for item in result: remote_triplet = (item['repo'], item['ref'], item['filename']) triplet = to_read[remote_triplet] if 'data' in item: text = base64.decodestring(item['data']) text_dict[triplet] = text else: logging.debug('Remote cache: %s', item) to_autodetect.add(triplet) for triplet in to_autodetect: reponame, ref, filename = triplet self.status(msg='Fetching file list for %s from remote repo cache' % reponame, chatty=True) absref, tree = resolved_refs[(reponame, ref)] file_list = self._rrc.ls_tree(reponame, absref) assert file_list is not None file_list_dict[triplet] = file_list for triplet in triplets: if triplet in resolved_morphologies: continue reponame, ref, filename = triplet absref, tree = resolved_refs[(reponame, ref)] file_list = file_list_dict.get(triplet, []) text = text_dict.get(triplet, None) assert text or file_list morphology = self._parse_or_generate_morphology( reponame, absref, filename, file_list, text) resolved_morphologies[triplet] = morphology def _check_and_tweak_system(self, morphology, reponame, sha1, filename): '''Check and tweak a system morphology.''' if morphology['arch'] is None: # pragma: no cover raise morphlib.Error('No arch specified in system %s ' '(arch is a mandatory field)' % filename) if morphology['arch'] == 'armv7': morphology._dict['arch'] = 'armv7l' if morphology['arch'] not in morphlib.valid_archs: raise morphlib.Error('Unknown arch %s. This version of Morph ' 'supports the following architectures: %s' % (morphology['arch'], ', '.join(morphlib.valid_archs))) name = morphology['name'] morphology.builds_artifacts = [name + '-rootfs'] morphology.needs_artifact_metadata_cached = False morphlib.morphloader.MorphologyLoader._validate_stratum_specs_fields( morphology, 'strata') morphlib.morphloader.MorphologyLoader._set_stratum_specs_defaults( morphology, 'strata') def _check_and_tweak_stratum(self, morphology, reponame, sha1, filename): '''Check and tweak a stratum morphology.''' if len(morphology['chunks']) == 0: raise EmptyStratumError(morphology['name']) for source in morphology['chunks']: if source.get('build-depends', None) is None: name = source.get('name', source.get('repo', 'unknown')) raise NoChunkBuildDependsError(filename, name) if (len(morphology['build-depends'] or []) == 0 and not any(c.get('build-mode') in ('bootstrap', 'test') for c in morphology['chunks'])): raise NoStratumBuildDependsError(filename) morphology.builds_artifacts = [morphology['name']] morphology.needs_artifact_metadata_cached = True morphlib.morphloader.MorphologyLoader._validate_stratum_specs_fields( morphology, 'build-depends') morphlib.morphloader.MorphologyLoader._set_stratum_specs_defaults( morphology, 'build-depends') def _check_and_tweak_chunk(self, morphology, reponame, sha1, filename): '''Check and tweak a chunk morphology.''' if 'products' in morphology and len(morphology['products']) > 1: morphology.builds_artifacts = [d['artifact'] for d in morphology['products']] else: morphology.builds_artifacts = [morphology['name']] morphology.needs_artifact_metadata_cached = False morphlib.morphloader.MorphologyLoader._validate_chunk(morphology)