From bacc5a9862d64a057762724810e7412128bdc002 Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Wed, 19 Aug 2015 17:13:10 +0100 Subject: Make to_chunk stage optional Most CPAN distributions can be installed with a standard set of commands so we don't want to generate chunk morphs for each chunk. This also fixes stratum build dependency generation Change-Id: Ica51721b9b3809f91b8016c959202694d4bc6ea7 --- baserockimport/mainloop.py | 90 +++++++++++++++++++++++----------------------- baserockimport/package.py | 8 +++-- 2 files changed, 50 insertions(+), 48 deletions(-) diff --git a/baserockimport/mainloop.py b/baserockimport/mainloop.py index c31bd0e..0571e5c 100644 --- a/baserockimport/mainloop.py +++ b/baserockimport/mainloop.py @@ -1,4 +1,6 @@ -# Copyright (C) 2014 Codethink Limited +# -*- coding: utf-8 -*- +# +# Copyright © 2014, 2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,6 +21,7 @@ import morphlib import networkx import json +import itertools import logging import os import tempfile @@ -100,13 +103,15 @@ class ImportLoop(object): ''' - def __init__(self, app, goal_kind, goal_name, goal_version): + def __init__(self, app, goal_kind, goal_name, goal_version, + generate_chunk_morphs=True): '''Set up an ImportLoop to process dependencies of one goal package.''' self.app = app self.goal_kind = goal_kind self.goal_name = goal_name self.goal_version = goal_version + self.generate_chunk_morphs = generate_chunk_morphs self.lorry_set = baserockimport.lorryset.LorrySet( self.app.settings['lorries-dir']) @@ -204,7 +209,7 @@ class ImportLoop(object): # 1. Make the source code available. lorry = self._find_or_create_lorry_file(kind, name, version) - source_repo, url = self._fetch_or_update_source(lorry) + source_repo, repo_url = self._fetch_or_update_source(lorry) checked_out_version, ref = self._checkout_source_version_for_package( source_repo, package) @@ -224,17 +229,24 @@ class ImportLoop(object): ref, source_repo.resolve_ref_to_commit(ref)) # 2. Create a chunk morphology with build instructions. + sha1 = source_repo.resolve_ref_to_commit(ref) + chunk_morph = None - chunk_morph = self._find_or_create_chunk_morph( - kind, name, checked_out_version, source_repo, url, ref) + if self.generate_chunk_morphs: + chunk_morph = self._find_or_create_chunk_morph( + kind, name, checked_out_version, + source_repo, repo_url, sha1) if self.app.settings['use-local-sources']: - chunk_morph.repo_url = 'file://' + source_repo.dirname + package.repo_url = 'file://' + source_repo.dirname else: reponame = lorry.keys()[0] - chunk_morph.repo_url = 'upstream:%s' % reponame + package.repo_url = 'upstream:%s' % reponame + + package.ref = sha1 + package.named_ref = ref - package.set_morphology(chunk_morph) + package.morphology = chunk_morph # 3. Calculate the dependencies of this package. @@ -431,10 +443,9 @@ class ImportLoop(object): return version, ref def _find_or_create_chunk_morph(self, kind, name, version, source_repo, - repo_url, named_ref): + repo_url, sha1): morphology_filename = 'strata/%s/%s-%s.morph' % ( self.goal_name, name, version) - sha1 = source_repo.resolve_ref_to_commit(named_ref) def generate_morphology(): morphology = self._generate_chunk_morph_for_package( @@ -463,10 +474,6 @@ class ImportLoop(object): morphology_filename) morphology = generate_morphology() - morphology.repo_url = repo_url - morphology.ref = sha1 - morphology.named_ref = named_ref - return morphology def _generate_chunk_morph_for_package(self, source_repo, kind, name, @@ -572,52 +579,43 @@ class ImportLoop(object): ignore_errors=False): self.app.status(msg='Generating stratum morph for %s' % goal_name) + def get_build_deps(kind, deps): + return deps[kind].get('build-dependencies', {}) + chunk_entries = [] for package in self._sort_chunks_by_build_order(graph): - m = package.morphology - - if m is None: - if ignore_errors: - logging.warn( - 'Ignoring %s because there is no chunk morphology.') - continue - else: - raise cliapp.AppException('No morphology for %s' % package) + morphology = package.morphology - def format_build_dep(kind, name, version): - dep_package = find( - graph, lambda p: p.match(kind, name, version)) - return '%s-%s' % (name, dep_package.version_in_use) + entry = { + 'name': package.name, + 'repo': package.repo_url, + 'ref': package.ref, + 'unpetrify-ref': package.named_ref, + } - def get_build_deps(morphology, kind): - field = 'x-build-dependencies-%s' % kind - return morphology.get(field, {}) + if morphology is not None: + entry['morph'] = morphology.filename build_depends = [] - for kind in self.importers: - for name, version in get_build_deps(m, kind).iteritems(): - build_depends.extend( - format_build_dep(kind, name, version)) - entry = { - 'name': m['name'], - 'repo': m.repo_url, - 'ref': m.ref, - 'unpetrify-ref': m.named_ref, - 'morph': m.filename, - } + ds = package.dependencies + build_depends = list(itertools.chain.from_iterable( + get_build_deps(kind, ds).keys() + for kind in self.importers) + if ds is not None else []) if build_depends: entry['build-depends'] = build_depends - chunk_entries.append(entry) - kwargs = self.importers[kind]['kwargs'] + strata = itertools.chain.from_iterable( + self.importers[kind]['kwargs'].get('strata', []) + for kind in self.importers) + + stratum_build_depends = [{'morph': stratum} for stratum in strata] - stratum_build_depends = ( - [{'morph': stratum} for stratum in kwargs['strata']] - if 'strata' in kwargs else []) + assert stratum_build_depends stratum_name = goal_name stratum = { diff --git a/baserockimport/package.py b/baserockimport/package.py index 1739292..361022f 100644 --- a/baserockimport/package.py +++ b/baserockimport/package.py @@ -1,4 +1,6 @@ -# Copyright (C) 2014 Codethink Limited +# -*- coding: utf-8 -*- +# +# Copyright © 2014, 2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -13,7 +15,6 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - class Package(object): '''A package in the processing queue. @@ -27,6 +28,9 @@ class Package(object): self.version = version self.required_by = [] self.morphology = None + self.repo_url = None + self.ref = None + self.named_ref = None self.dependencies = None self.is_build_dep = False self.version_in_use = version -- cgit v1.2.1