diff options
author | Richard Maw <richard.maw@gmail.com> | 2014-10-08 13:17:00 +0000 |
---|---|---|
committer | Richard Maw <richard.maw@gmail.com> | 2014-10-08 13:17:00 +0000 |
commit | 8b85d60e9294713097928d52b8ca4cfe5d5f801a (patch) | |
tree | 0de7707ca779b47f7c3882b5627cb3dedaf09a6a /morphlib | |
parent | d3be16e282bfdf7a8db8538339719161725b5cad (diff) | |
parent | 9034cd11ee2b9c050f2301c84627a3d7b2e67895 (diff) | |
download | morph-8b85d60e9294713097928d52b8ca4cfe5d5f801a.tar.gz |
Merge branch 'baserock/richardmaw/fix-distbuild-v3'
Reviewed-by: Sam Thursfield
Reviewed-by: Richard Ipsum
Reviewed-by: Pedro Alvarez
Diffstat (limited to 'morphlib')
-rw-r--r-- | morphlib/artifactsplitrule.py | 8 | ||||
-rw-r--r-- | morphlib/builder2.py | 64 | ||||
-rw-r--r-- | morphlib/plugins/artifact_inspection_plugin.py | 34 | ||||
-rw-r--r-- | morphlib/plugins/distbuild_plugin.py | 33 | ||||
-rw-r--r-- | morphlib/plugins/list_artifacts_plugin.py | 6 |
5 files changed, 38 insertions, 107 deletions
diff --git a/morphlib/artifactsplitrule.py b/morphlib/artifactsplitrule.py index cf0d1060..1511d694 100644 --- a/morphlib/artifactsplitrule.py +++ b/morphlib/artifactsplitrule.py @@ -230,7 +230,7 @@ DEFAULT_STRATUM_RULES = [ ] -def unify_chunk_matches(morphology): +def unify_chunk_matches(morphology, default_rules=DEFAULT_CHUNK_RULES): '''Create split rules including defaults and per-chunk rules. With rules specified in the morphology's 'products' field and the @@ -246,7 +246,7 @@ def unify_chunk_matches(morphology): split_rules.add(ca_name, FileMatch(patterns)) name = morphology['name'] - for suffix, patterns in DEFAULT_CHUNK_RULES: + for suffix, patterns in default_rules: ca_name = name + suffix # Explicit rules override the default rules. This is an all-or-nothing # override: there is no way to extend the default split rules right now @@ -257,7 +257,7 @@ def unify_chunk_matches(morphology): return split_rules -def unify_stratum_matches(morphology): +def unify_stratum_matches(morphology, default_rules=DEFAULT_STRATUM_RULES): '''Create split rules including defaults and per-stratum rules. With rules specified in the chunk spec's 'artifacts' fields, the @@ -284,7 +284,7 @@ def unify_stratum_matches(morphology): for d in morphology.get('products', {})): match_split_rules.add(sta_name, ArtifactMatch(patterns)) - for suffix, patterns in DEFAULT_STRATUM_RULES: + for suffix, patterns in default_rules: sta_name = morphology['name'] + suffix # Explicit rules override the default rules. This is an all-or-nothing # override: there is no way to extend the default split rules right now diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 20cae225..9cd3a074 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -162,46 +162,6 @@ def get_stratum_files(f, lac): # pragma: no cover cf.close() -def get_overlaps(source, constituents, lac): # pragma: no cover - # check whether strata overlap - installed = defaultdict(set) - for dep in constituents: - handle = lac.get(dep) - if source.morphology['kind'] == 'stratum': - for filename in get_chunk_files(handle): - installed[filename].add(dep) - elif source.morphology['kind'] == 'system': - for filename in get_stratum_files(handle, lac): - installed[filename].add(dep) - handle.close() - overlaps = defaultdict(set) - for filename, artifacts in installed.iteritems(): - if len(artifacts) > 1: - overlaps[frozenset(artifacts)].add(filename) - return overlaps - - -def log_overlaps(overlaps): # pragma: no cover - for overlapping, files in sorted(overlaps.iteritems()): - logging.warning(' Artifacts %s overlap with files:' % - ', '.join(sorted(a.name for a in overlapping))) - for filename in sorted(files): - logging.warning(' %s' % filename) - - -def write_overlap_metadata(artifact, overlaps, lac): # pragma: no cover - f = lac.put_artifact_metadata(artifact, 'overlaps') - # the big list comprehension is because json can't serialize - # artifacts, sets or dicts with non-string keys - json.dump( - [ - [ - [a.name for a in afs], list(files) - ] for afs, files in overlaps.iteritems() - ], f, indent=4, encoding='unicode-escape') - f.close() - - class BuilderBase(object): '''Base class for building artifacts.''' @@ -559,18 +519,6 @@ class StratumBuilder(BuilderBase): download_depends(constituents, self.local_artifact_cache, self.remote_artifact_cache) - # check for chunk overlaps - overlaps = get_overlaps(self.source, constituents, - self.local_artifact_cache) - if len(overlaps) > 0: - logging.warning( - 'Overlaps in stratum artifact %s detected' %a_name) - log_overlaps(overlaps) - self.app.status(msg='Overlaps in stratum artifact ' - '%(stratum_name)s detected', - stratum_name=a_name, error=True) - write_overlap_metadata(a, overlaps, - self.local_artifact_cache) with self.build_watch('create-chunk-list'): lac = self.local_artifact_cache @@ -674,18 +622,6 @@ class SystemBuilder(BuilderBase): # pragma: no cover self.remote_artifact_cache) f.close() - # check whether the strata overlap - overlaps = get_overlaps(self.source, self.source.dependencies, - self.local_artifact_cache) - if len(overlaps) > 0: - self.app.status(msg='Overlaps in system artifact ' - '%(artifact_name)s detected', - artifact_name=a_name, - error=True) - log_overlaps(overlaps) - write_overlap_metadata(a, overlaps, - self.local_artifact_cache) - # unpack it from the local artifact cache for stratum_artifact in self.source.dependencies: self.unpack_one_stratum(stratum_artifact, path) diff --git a/morphlib/plugins/artifact_inspection_plugin.py b/morphlib/plugins/artifact_inspection_plugin.py index 6eeece77..74645f41 100644 --- a/morphlib/plugins/artifact_inspection_plugin.py +++ b/morphlib/plugins/artifact_inspection_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -257,9 +257,6 @@ class ManifestGenerator(object): class ArtifactInspectionPlugin(cliapp.Plugin): def enable(self): - self.app.add_subcommand('run-in-artifact', - self.run_in_artifact, - arg_synopsis='ARTIFACT CMD') self.app.add_subcommand('generate-manifest', self.generate_manifest, arg_synopsis='SYSTEM-ARTIFACT') @@ -267,35 +264,6 @@ class ArtifactInspectionPlugin(cliapp.Plugin): def disable(self): pass - def run_in_artifact(self, args): - '''Run a command inside an extracted/mounted chunk or system. - - Command line arguments: - - * `ARTIFACT` is a filename for the artifact. - * `CMD` is the command to run. - - run-in-artifact unpacks an artifact, and runs a command in - the temporary directory it was unpacked to. - - The command must be given to Morph as a single argument, so - use shell quoting appropriately. - - ''' - - if len(args) < 2: - raise cliapp.AppException( - 'run-in-artifact requires arguments: a chunk or ' - 'system artifact and a a shell command') - - artifact, cmd = args[0], args[1:] - - def run_command_in_dir(dirname): - output = self.app.runcmd(cmd, cwd=dirname) - self.app.output.write(output) - - call_in_artifact_directory(self.app, artifact, run_command_in_dir) - def generate_manifest(self, args): '''Generate a content manifest for a system image. diff --git a/morphlib/plugins/distbuild_plugin.py b/morphlib/plugins/distbuild_plugin.py index 50ab7eeb..653eeae8 100644 --- a/morphlib/plugins/distbuild_plugin.py +++ b/morphlib/plugins/distbuild_plugin.py @@ -103,7 +103,7 @@ class WorkerBuild(cliapp.Plugin): self.app.subcommands['gc']([]) arch = artifact.arch - bc.build_artifact(artifact, bc.new_build_env(arch)) + bc.build_source(artifact.source, bc.new_build_env(arch)) def is_system_artifact(self, filename): return re.match(r'^[0-9a-fA-F]{64}\.system\.', filename) @@ -121,6 +121,11 @@ class WorkerDaemon(cliapp.Plugin): 'listen for connections on PORT', default=3434, group=group_distbuild) + self.app.settings.string( + ['worker-daemon-port-file'], + 'write port used by worker-daemon to FILE', + default='', + group=group_distbuild) self.app.add_subcommand( 'worker-daemon', self.worker_daemon, @@ -136,7 +141,9 @@ class WorkerDaemon(cliapp.Plugin): address = self.app.settings['worker-daemon-address'] port = self.app.settings['worker-daemon-port'] - router = distbuild.ListenServer(address, port, distbuild.JsonRouter) + port_file = self.app.settings['worker-daemon-port-file'] + router = distbuild.ListenServer(address, port, distbuild.JsonRouter, + port_file=port_file) loop = distbuild.MainLoop() loop.add_state_machine(router) loop.run() @@ -156,6 +163,16 @@ class ControllerDaemon(cliapp.Plugin): 'listen for initiator connections on PORT', default=7878, group=group_distbuild) + self.app.settings.string( + ['controller-initiator-port-file'], + 'write the port to listen for initiator connections to FILE', + default='', + group=group_distbuild) + self.app.settings.string( + ['initiator-step-output-dir'], + 'write build output to files in DIR', + default='.', + group=group_distbuild) self.app.settings.string( ['controller-helper-address'], @@ -167,6 +184,11 @@ class ControllerDaemon(cliapp.Plugin): 'listen for helper connections on PORT', default=5656, group=group_distbuild) + self.app.settings.string( + ['controller-helper-port-file'], + 'write the port to listen for helper connections to FILE', + default='', + group=group_distbuild) self.app.settings.string_list( ['worker'], @@ -218,8 +240,10 @@ class ControllerDaemon(cliapp.Plugin): listener_specs = [ # address, port, class to initiate on connection, class init args ('controller-helper-address', 'controller-helper-port', + 'controller-helper-port-file', distbuild.HelperRouter, []), ('controller-initiator-address', 'controller-initiator-port', + 'controller-initiator-port-file', distbuild.InitiatorConnection, [artifact_cache_server, morph_instance]), ] @@ -229,11 +253,12 @@ class ControllerDaemon(cliapp.Plugin): queuer = distbuild.WorkerBuildQueuer() loop.add_state_machine(queuer) - for addr, port, sm, extra_args in listener_specs: + for addr, port, port_file, sm, extra_args in listener_specs: addr = self.app.settings[addr] port = self.app.settings[port] + port_file = self.app.settings[port_file] listener = distbuild.ListenServer( - addr, port, sm, extra_args=extra_args) + addr, port, sm, extra_args=extra_args, port_file=port_file) loop.add_state_machine(listener) for worker in self.app.settings['worker']: diff --git a/morphlib/plugins/list_artifacts_plugin.py b/morphlib/plugins/list_artifacts_plugin.py index 8074206b..61c8d160 100644 --- a/morphlib/plugins/list_artifacts_plugin.py +++ b/morphlib/plugins/list_artifacts_plugin.py @@ -105,11 +105,13 @@ class ListArtifactsPlugin(cliapp.Plugin): self.app.settings, system_artifact.source.morphology['arch']) ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) - artifact_files = set() - for artifact in system_artifact.walk(): + for source in set(a.source for a in system_artifact.walk()): artifact.cache_key = ckc.compute_key(artifact) artifact.cache_id = ckc.get_cache_id(artifact) + artifact_files = set() + for artifact in system_artifact.walk(): + artifact_files.add(artifact.basename()) if artifact.source.morphology.needs_artifact_metadata_cached: |