Merge branch 'baserock/richardmaw/fix-distbuild-v3'

Reviewed-by: Sam Thursfield
Reviewed-by: Richard Ipsum
Reviewed-by: Pedro Alvarez

5 files changed, 38 insertions, 107 deletions

diff --git a/morphlib/artifactsplitrule.py b/morphlib/artifactsplitrule.py
index cf0d1060..1511d694 100644
--- a/morphlib/artifactsplitrule.py
+++ b/morphlib/artifactsplitrule.py
@@ -230,7 +230,7 @@ DEFAULT_STRATUM_RULES = [
 ]
 
 
-def unify_chunk_matches(morphology):
+def unify_chunk_matches(morphology, default_rules=DEFAULT_CHUNK_RULES):
     '''Create split rules including defaults and per-chunk rules.
 
     With rules specified in the morphology's 'products' field and the
@@ -246,7 +246,7 @@ def unify_chunk_matches(morphology):
         split_rules.add(ca_name, FileMatch(patterns))
 
     name = morphology['name']
-    for suffix, patterns in DEFAULT_CHUNK_RULES:
+    for suffix, patterns in default_rules:
         ca_name = name + suffix
         # Explicit rules override the default rules. This is an all-or-nothing
         # override: there is no way to extend the default split rules right now
@@ -257,7 +257,7 @@ def unify_chunk_matches(morphology):
     return split_rules
 
 
-def unify_stratum_matches(morphology):
+def unify_stratum_matches(morphology, default_rules=DEFAULT_STRATUM_RULES):
     '''Create split rules including defaults and per-stratum rules.
 
     With rules specified in the chunk spec's 'artifacts' fields, the
@@ -284,7 +284,7 @@ def unify_stratum_matches(morphology):
                                for d in morphology.get('products', {})):
         match_split_rules.add(sta_name, ArtifactMatch(patterns))
 
-    for suffix, patterns in DEFAULT_STRATUM_RULES:
+    for suffix, patterns in default_rules:
         sta_name = morphology['name'] + suffix
         # Explicit rules override the default rules. This is an all-or-nothing
         # override: there is no way to extend the default split rules right now
diff --git a/morphlib/builder2.py b/morphlib/builder2.py
index 20cae225..9cd3a074 100644
--- a/morphlib/builder2.py
+++ b/morphlib/builder2.py
@@ -162,46 +162,6 @@ def get_stratum_files(f, lac):  # pragma: no cover
         cf.close()
 
 
-def get_overlaps(source, constituents, lac):  # pragma: no cover
-    # check whether strata overlap
-    installed = defaultdict(set)
-    for dep in constituents:
-        handle = lac.get(dep)
-        if source.morphology['kind'] == 'stratum':
-            for filename in get_chunk_files(handle):
-                installed[filename].add(dep)
-        elif source.morphology['kind'] == 'system':
-            for filename in get_stratum_files(handle, lac):
-                installed[filename].add(dep)
-        handle.close()
-    overlaps = defaultdict(set)
-    for filename, artifacts in installed.iteritems():
-        if len(artifacts) > 1:
-            overlaps[frozenset(artifacts)].add(filename)
-    return overlaps
-
-
-def log_overlaps(overlaps):  # pragma: no cover
-    for overlapping, files in sorted(overlaps.iteritems()):
-        logging.warning('  Artifacts %s overlap with files:' %
-                        ', '.join(sorted(a.name for a in overlapping)))
-        for filename in sorted(files):
-            logging.warning('    %s' % filename)
-
-
-def write_overlap_metadata(artifact, overlaps, lac):  # pragma: no cover
-    f = lac.put_artifact_metadata(artifact, 'overlaps')
-    # the big list comprehension is because json can't serialize
-    # artifacts, sets or dicts with non-string keys
-    json.dump(
-        [
-            [
-                [a.name for a in afs], list(files)
-            ] for afs, files in overlaps.iteritems()
-        ], f, indent=4, encoding='unicode-escape')
-    f.close()
-
-
 class BuilderBase(object):
 
     '''Base class for building artifacts.'''
@@ -559,18 +519,6 @@ class StratumBuilder(BuilderBase):
                     download_depends(constituents,
                                      self.local_artifact_cache,
                                      self.remote_artifact_cache)
-                    # check for chunk overlaps
-                    overlaps = get_overlaps(self.source, constituents,
-                                            self.local_artifact_cache)
-                    if len(overlaps) > 0:
-                        logging.warning(
-                            'Overlaps in stratum artifact %s detected' %a_name)
-                        log_overlaps(overlaps)
-                        self.app.status(msg='Overlaps in stratum artifact '
-                                            '%(stratum_name)s detected',
-                                        stratum_name=a_name, error=True)
-                        write_overlap_metadata(a, overlaps,
-                                               self.local_artifact_cache)
 
             with self.build_watch('create-chunk-list'):
                 lac = self.local_artifact_cache
@@ -674,18 +622,6 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
                                      self.remote_artifact_cache)
                     f.close()
 
-                # check whether the strata overlap
-                overlaps = get_overlaps(self.source, self.source.dependencies,
-                                        self.local_artifact_cache)
-                if len(overlaps) > 0:
-                    self.app.status(msg='Overlaps in system artifact '
-                                        '%(artifact_name)s detected',
-                                    artifact_name=a_name,
-                                    error=True)
-                    log_overlaps(overlaps)
-                    write_overlap_metadata(a, overlaps,
-                                           self.local_artifact_cache)
-
                 # unpack it from the local artifact cache
                 for stratum_artifact in self.source.dependencies:
                     self.unpack_one_stratum(stratum_artifact, path)
diff --git a/morphlib/plugins/artifact_inspection_plugin.py b/morphlib/plugins/artifact_inspection_plugin.py
index 6eeece77..74645f41 100644
--- a/morphlib/plugins/artifact_inspection_plugin.py
+++ b/morphlib/plugins/artifact_inspection_plugin.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2013  Codethink Limited
+# Copyright (C) 2012-2014 Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -257,9 +257,6 @@ class ManifestGenerator(object):
 class ArtifactInspectionPlugin(cliapp.Plugin):
 
     def enable(self):
-        self.app.add_subcommand('run-in-artifact',
-                                self.run_in_artifact,
-                                arg_synopsis='ARTIFACT CMD')
         self.app.add_subcommand('generate-manifest',
                                 self.generate_manifest,
                                 arg_synopsis='SYSTEM-ARTIFACT')
@@ -267,35 +264,6 @@ class ArtifactInspectionPlugin(cliapp.Plugin):
     def disable(self):
         pass
 
-    def run_in_artifact(self, args):
-        '''Run a command inside an extracted/mounted chunk or system.
-
-        Command line arguments:
-
-        * `ARTIFACT` is a filename for the artifact.
-        * `CMD` is the command to run.
-
-        run-in-artifact unpacks an artifact, and runs a command in
-        the temporary directory it was unpacked to.
-
-        The command must be given to Morph as a single argument, so
-        use shell quoting appropriately.
-
-        '''
-
-        if len(args) < 2:
-            raise cliapp.AppException(
-                    'run-in-artifact requires arguments: a chunk or '
-                    'system artifact and a a shell command')
-
-        artifact, cmd = args[0], args[1:]
-
-        def run_command_in_dir(dirname):
-            output = self.app.runcmd(cmd, cwd=dirname)
-            self.app.output.write(output)
-
-        call_in_artifact_directory(self.app, artifact, run_command_in_dir)
-
     def generate_manifest(self, args):
         '''Generate a content manifest for a system image.
 
diff --git a/morphlib/plugins/distbuild_plugin.py b/morphlib/plugins/distbuild_plugin.py
index 50ab7eeb..653eeae8 100644
--- a/morphlib/plugins/distbuild_plugin.py
+++ b/morphlib/plugins/distbuild_plugin.py
@@ -103,7 +103,7 @@ class WorkerBuild(cliapp.Plugin):
         self.app.subcommands['gc']([])
 
         arch = artifact.arch
-        bc.build_artifact(artifact, bc.new_build_env(arch))
+        bc.build_source(artifact.source, bc.new_build_env(arch))
 
     def is_system_artifact(self, filename):
         return re.match(r'^[0-9a-fA-F]{64}\.system\.', filename)
@@ -121,6 +121,11 @@ class WorkerDaemon(cliapp.Plugin):
             'listen for connections on PORT',
             default=3434,
             group=group_distbuild)
+        self.app.settings.string(
+            ['worker-daemon-port-file'],
+            'write port used by worker-daemon to FILE',
+            default='',
+            group=group_distbuild)
         self.app.add_subcommand(
             'worker-daemon',
             self.worker_daemon,
@@ -136,7 +141,9 @@ class WorkerDaemon(cliapp.Plugin):
 
         address = self.app.settings['worker-daemon-address']
         port = self.app.settings['worker-daemon-port']
-        router = distbuild.ListenServer(address, port, distbuild.JsonRouter)
+        port_file = self.app.settings['worker-daemon-port-file']
+        router = distbuild.ListenServer(address, port, distbuild.JsonRouter,
+                                        port_file=port_file)
         loop = distbuild.MainLoop()
         loop.add_state_machine(router)
         loop.run()
@@ -156,6 +163,16 @@ class ControllerDaemon(cliapp.Plugin):
             'listen for initiator connections on PORT',
             default=7878,
             group=group_distbuild)
+        self.app.settings.string(
+            ['controller-initiator-port-file'],
+            'write the port to listen for initiator connections to FILE',
+            default='',
+            group=group_distbuild)
+        self.app.settings.string(
+            ['initiator-step-output-dir'],
+            'write build output to files in DIR',
+            default='.',
+            group=group_distbuild)
 
         self.app.settings.string(
             ['controller-helper-address'],
@@ -167,6 +184,11 @@ class ControllerDaemon(cliapp.Plugin):
             'listen for helper connections on PORT',
             default=5656,
             group=group_distbuild)
+        self.app.settings.string(
+            ['controller-helper-port-file'],
+            'write the port to listen for helper connections to FILE',
+            default='',
+            group=group_distbuild)
 
         self.app.settings.string_list(
             ['worker'],
@@ -218,8 +240,10 @@ class ControllerDaemon(cliapp.Plugin):
         listener_specs = [
             # address, port, class to initiate on connection, class init args
             ('controller-helper-address', 'controller-helper-port', 
+             'controller-helper-port-file',
              distbuild.HelperRouter, []),
             ('controller-initiator-address', 'controller-initiator-port',
+             'controller-initiator-port-file',
              distbuild.InitiatorConnection, 
              [artifact_cache_server, morph_instance]),
         ]
@@ -229,11 +253,12 @@ class ControllerDaemon(cliapp.Plugin):
         queuer = distbuild.WorkerBuildQueuer()
         loop.add_state_machine(queuer)
 
-        for addr, port, sm, extra_args in listener_specs:
+        for addr, port, port_file, sm, extra_args in listener_specs:
             addr = self.app.settings[addr]
             port = self.app.settings[port]
+            port_file = self.app.settings[port_file]
             listener = distbuild.ListenServer(
-                addr, port, sm, extra_args=extra_args)
+                addr, port, sm, extra_args=extra_args, port_file=port_file)
             loop.add_state_machine(listener)
 
         for worker in self.app.settings['worker']:
diff --git a/morphlib/plugins/list_artifacts_plugin.py b/morphlib/plugins/list_artifacts_plugin.py
index 8074206b..61c8d160 100644
--- a/morphlib/plugins/list_artifacts_plugin.py
+++ b/morphlib/plugins/list_artifacts_plugin.py
@@ -105,11 +105,13 @@ class ListArtifactsPlugin(cliapp.Plugin):
             self.app.settings, system_artifact.source.morphology['arch'])
         ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env)
 
-        artifact_files = set()
-        for artifact in system_artifact.walk():
+        for source in set(a.source for a in system_artifact.walk()):
             artifact.cache_key = ckc.compute_key(artifact)
             artifact.cache_id = ckc.get_cache_id(artifact)
 
+        artifact_files = set()
+        for artifact in system_artifact.walk():
+
             artifact_files.add(artifact.basename())
 
             if artifact.source.morphology.needs_artifact_metadata_cached: