Make to_chunk stage optional

Most CPAN distributions can be installed with a standard
set of commands so we don't want to generate chunk morphs
for each chunk.

This also fixes stratum build dependency generation

Change-Id: Ica51721b9b3809f91b8016c959202694d4bc6ea7

2 files changed, 50 insertions, 48 deletions

diff --git a/baserockimport/mainloop.py b/baserockimport/mainloop.py
index c31bd0e..0571e5c 100644
--- a/baserockimport/mainloop.py
+++ b/baserockimport/mainloop.py
@@ -1,4 +1,6 @@
-# Copyright (C) 2014  Codethink Limited
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014, 2015  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -19,6 +21,7 @@ import morphlib
 import networkx
 
 import json
+import itertools
 import logging
 import os
 import tempfile
@@ -100,13 +103,15 @@ class ImportLoop(object):
 
     '''
 
-    def __init__(self, app, goal_kind, goal_name, goal_version):
+    def __init__(self, app, goal_kind, goal_name, goal_version,
+                 generate_chunk_morphs=True):
         '''Set up an ImportLoop to process dependencies of one goal package.'''
 
         self.app = app
         self.goal_kind = goal_kind
         self.goal_name = goal_name
         self.goal_version = goal_version
+        self.generate_chunk_morphs = generate_chunk_morphs
 
         self.lorry_set = baserockimport.lorryset.LorrySet(
             self.app.settings['lorries-dir'])
@@ -204,7 +209,7 @@ class ImportLoop(object):
         # 1. Make the source code available.
 
         lorry = self._find_or_create_lorry_file(kind, name, version)
-        source_repo, url = self._fetch_or_update_source(lorry)
+        source_repo, repo_url = self._fetch_or_update_source(lorry)
 
         checked_out_version, ref = self._checkout_source_version_for_package(
             source_repo, package)
@@ -224,17 +229,24 @@ class ImportLoop(object):
                 ref, source_repo.resolve_ref_to_commit(ref))
 
         # 2. Create a chunk morphology with build instructions.
+        sha1 = source_repo.resolve_ref_to_commit(ref)
+        chunk_morph = None
 
-        chunk_morph = self._find_or_create_chunk_morph(
-            kind, name, checked_out_version, source_repo, url, ref)
+        if self.generate_chunk_morphs:
+            chunk_morph = self._find_or_create_chunk_morph(
+                            kind, name, checked_out_version,
+                            source_repo, repo_url, sha1)
 
         if self.app.settings['use-local-sources']:
-            chunk_morph.repo_url = 'file://' + source_repo.dirname
+            package.repo_url = 'file://' + source_repo.dirname
         else:
             reponame = lorry.keys()[0]
-            chunk_morph.repo_url = 'upstream:%s' % reponame
+            package.repo_url = 'upstream:%s' % reponame
+
+        package.ref = sha1
+        package.named_ref = ref
 
-        package.set_morphology(chunk_morph)
+        package.morphology = chunk_morph
 
         # 3. Calculate the dependencies of this package.
 
@@ -431,10 +443,9 @@ class ImportLoop(object):
         return version, ref
 
     def _find_or_create_chunk_morph(self, kind, name, version, source_repo,
-                                    repo_url, named_ref):
+                                    repo_url, sha1):
         morphology_filename = 'strata/%s/%s-%s.morph' % (
             self.goal_name, name, version)
-        sha1 = source_repo.resolve_ref_to_commit(named_ref)
 
         def generate_morphology():
             morphology = self._generate_chunk_morph_for_package(
@@ -463,10 +474,6 @@ class ImportLoop(object):
                                   morphology_filename)
                     morphology = generate_morphology()
 
-        morphology.repo_url = repo_url
-        morphology.ref = sha1
-        morphology.named_ref = named_ref
-
         return morphology
 
     def _generate_chunk_morph_for_package(self, source_repo, kind, name,
@@ -572,52 +579,43 @@ class ImportLoop(object):
                           ignore_errors=False):
         self.app.status(msg='Generating stratum morph for %s' % goal_name)
 
+        def get_build_deps(kind, deps):
+            return deps[kind].get('build-dependencies', {})
+
         chunk_entries = []
 
         for package in self._sort_chunks_by_build_order(graph):
-            m = package.morphology
-
-            if m is None:
-                if ignore_errors:
-                    logging.warn(
-                        'Ignoring %s because there is no chunk morphology.')
-                    continue
-                else:
-                    raise cliapp.AppException('No morphology for %s' % package)
+            morphology = package.morphology
 
-            def format_build_dep(kind, name, version):
-                dep_package = find(
-                    graph, lambda p: p.match(kind, name, version))
-                return '%s-%s' % (name, dep_package.version_in_use)
+            entry = {
+                'name': package.name,
+                'repo': package.repo_url,
+                'ref': package.ref,
+                'unpetrify-ref': package.named_ref,
+            }
 
-            def get_build_deps(morphology, kind):
-                field = 'x-build-dependencies-%s' % kind
-                return morphology.get(field, {})
+            if morphology is not None:
+                entry['morph'] = morphology.filename
 
             build_depends = []
-            for kind in self.importers:
-                for name, version in get_build_deps(m, kind).iteritems():
-                    build_depends.extend(
-                        format_build_dep(kind, name, version))
 
-            entry = {
-                'name': m['name'],
-                'repo': m.repo_url,
-                'ref': m.ref,
-                'unpetrify-ref': m.named_ref,
-                'morph': m.filename,
-            }
+            ds = package.dependencies
+            build_depends = list(itertools.chain.from_iterable(
+                                    get_build_deps(kind, ds).keys()
+                                        for kind in self.importers)
+                                        if ds is not None else [])
 
             if build_depends:
                 entry['build-depends'] = build_depends
-
             chunk_entries.append(entry)
 
-        kwargs = self.importers[kind]['kwargs']
+        strata = itertools.chain.from_iterable(
+                    self.importers[kind]['kwargs'].get('strata', [])
+                        for kind in self.importers)
+
+        stratum_build_depends = [{'morph': stratum} for stratum in strata]
 
-        stratum_build_depends = (
-            [{'morph': stratum} for stratum in kwargs['strata']]
-                if 'strata' in kwargs else [])
+        assert stratum_build_depends
 
         stratum_name = goal_name
         stratum = {
diff --git a/baserockimport/package.py b/baserockimport/package.py
index 1739292..361022f 100644
--- a/baserockimport/package.py
+++ b/baserockimport/package.py
@@ -1,4 +1,6 @@
-# Copyright (C) 2014  Codethink Limited
+# -*- coding: utf-8 -*-
+#
+# Copyright © 2014, 2015  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -13,7 +15,6 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-
 class Package(object):
     '''A package in the processing queue.
 
@@ -27,6 +28,9 @@ class Package(object):
         self.version = version
         self.required_by = []
         self.morphology = None
+        self.repo_url = None
+        self.ref = None
+        self.named_ref = None
         self.dependencies = None
         self.is_build_dep = False
         self.version_in_use = version