diff options
Diffstat (limited to 'morphlib')
-rw-r--r-- | morphlib/artifactresolver.py | 6 | ||||
-rw-r--r-- | morphlib/artifactresolver_tests.py | 16 | ||||
-rw-r--r-- | morphlib/artifactsplitrule.py | 8 | ||||
-rw-r--r-- | morphlib/buildcommand.py | 57 | ||||
-rw-r--r-- | morphlib/builder2.py | 64 | ||||
-rw-r--r-- | morphlib/cachekeycomputer_tests.py | 2 | ||||
-rwxr-xr-x | morphlib/exts/ssh-rsync.check | 7 | ||||
-rwxr-xr-x | morphlib/exts/ssh-rsync.write | 9 | ||||
-rw-r--r-- | morphlib/gitversion.py | 6 | ||||
-rw-r--r-- | morphlib/plugins/artifact_inspection_plugin.py | 34 | ||||
-rw-r--r-- | morphlib/plugins/distbuild_plugin.py | 33 | ||||
-rw-r--r-- | morphlib/plugins/list_artifacts_plugin.py | 6 | ||||
-rw-r--r-- | morphlib/util.py | 15 | ||||
-rwxr-xr-x | morphlib/xfer-hole | 10 |
14 files changed, 111 insertions, 162 deletions
diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index 5deb25b7..8c8b37d0 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -52,7 +52,11 @@ class ArtifactResolver(object): self._added_artifacts = None self._source_pool = None - def resolve_artifacts(self, source_pool): + def resolve_root_artifacts(self, source_pool): #pragma: no cover + return [a for a in self._resolve_artifacts(source_pool) + if not a.dependents] + + def _resolve_artifacts(self, source_pool): self._source_pool = source_pool self._added_artifacts = set() diff --git a/morphlib/artifactresolver_tests.py b/morphlib/artifactresolver_tests.py index 89f30010..b958da4f 100644 --- a/morphlib/artifactresolver_tests.py +++ b/morphlib/artifactresolver_tests.py @@ -82,7 +82,7 @@ class ArtifactResolverTests(unittest.TestCase): def test_resolve_artifacts_using_an_empty_pool(self): pool = morphlib.sourcepool.SourcePool() - artifacts = self.resolver.resolve_artifacts(pool) + artifacts = self.resolver._resolve_artifacts(pool) self.assertEqual(len(artifacts), 0) def test_resolve_single_chunk_with_no_subartifacts(self): @@ -95,7 +95,7 @@ class ArtifactResolverTests(unittest.TestCase): for source in sources: pool.add(source) - artifacts = self.resolver.resolve_artifacts(pool) + artifacts = self.resolver._resolve_artifacts(pool) self.assertEqual(len(artifacts), sum(len(s.split_rules.artifacts) for s in pool)) @@ -116,7 +116,7 @@ class ArtifactResolverTests(unittest.TestCase): for source in sources: pool.add(source) - artifacts = self.resolver.resolve_artifacts(pool) + artifacts = self.resolver._resolve_artifacts(pool) self.assertEqual(len(artifacts), sum(len(s.split_rules.artifacts) for s in pool)) @@ -136,7 +136,7 @@ class ArtifactResolverTests(unittest.TestCase): for source in sources: pool.add(source) - artifacts = self.resolver.resolve_artifacts(pool) + artifacts = self.resolver._resolve_artifacts(pool) artifacts.sort(key=lambda a: a.name) self.assertEqual(len(artifacts), @@ -167,7 +167,7 @@ class ArtifactResolverTests(unittest.TestCase): for stratum in stratum_sources: pool.add(stratum) - artifacts = self.resolver.resolve_artifacts(pool) + artifacts = self.resolver._resolve_artifacts(pool) all_artifacts = set() for s in pool: all_artifacts.update(s.split_rules.artifacts) @@ -216,7 +216,7 @@ class ArtifactResolverTests(unittest.TestCase): for stratum in stratum_sources: pool.add(stratum) - artifacts = self.resolver.resolve_artifacts(pool) + artifacts = self.resolver._resolve_artifacts(pool) self.assertEqual( set(artifacts), @@ -278,7 +278,7 @@ class ArtifactResolverTests(unittest.TestCase): pool.add(stratum2) self.assertRaises(morphlib.artifactresolver.MutualDependencyError, - self.resolver.resolve_artifacts, pool) + self.resolver._resolve_artifacts, pool) def test_detection_of_chunk_dependencies_in_invalid_order(self): pool = morphlib.sourcepool.SourcePool() @@ -321,7 +321,7 @@ class ArtifactResolverTests(unittest.TestCase): pool.add(chunk2) self.assertRaises(morphlib.artifactresolver.DependencyOrderError, - self.resolver.resolve_artifacts, pool) + self.resolver._resolve_artifacts, pool) # TODO: Expand test suite to include better dependency checking, many diff --git a/morphlib/artifactsplitrule.py b/morphlib/artifactsplitrule.py index cf0d1060..1511d694 100644 --- a/morphlib/artifactsplitrule.py +++ b/morphlib/artifactsplitrule.py @@ -230,7 +230,7 @@ DEFAULT_STRATUM_RULES = [ ] -def unify_chunk_matches(morphology): +def unify_chunk_matches(morphology, default_rules=DEFAULT_CHUNK_RULES): '''Create split rules including defaults and per-chunk rules. With rules specified in the morphology's 'products' field and the @@ -246,7 +246,7 @@ def unify_chunk_matches(morphology): split_rules.add(ca_name, FileMatch(patterns)) name = morphology['name'] - for suffix, patterns in DEFAULT_CHUNK_RULES: + for suffix, patterns in default_rules: ca_name = name + suffix # Explicit rules override the default rules. This is an all-or-nothing # override: there is no way to extend the default split rules right now @@ -257,7 +257,7 @@ def unify_chunk_matches(morphology): return split_rules -def unify_stratum_matches(morphology): +def unify_stratum_matches(morphology, default_rules=DEFAULT_STRATUM_RULES): '''Create split rules including defaults and per-stratum rules. With rules specified in the chunk spec's 'artifacts' fields, the @@ -284,7 +284,7 @@ def unify_stratum_matches(morphology): for d in morphology.get('products', {})): match_split_rules.add(sta_name, ArtifactMatch(patterns)) - for suffix, patterns in DEFAULT_STRATUM_RULES: + for suffix, patterns in default_rules: sta_name = morphology['name'] + suffix # Explicit rules override the default rules. This is an all-or-nothing # override: there is no way to extend the default split rules right now diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index c5ba05e5..2aec5e08 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -143,6 +143,21 @@ class BuildCommand(object): for spec in src.morphology['chunks']): raise morphlib.Error('No non-bootstrap chunks found.') + def _compute_cache_keys(self, root_artifact): + arch = root_artifact.source.morphology['arch'] + self.app.status(msg='Creating build environment for %(arch)s', + arch=arch, chatty=True) + build_env = self.new_build_env(arch) + + self.app.status(msg='Computing cache keys', chatty=True) + ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) + + for source in set(a.source for a in root_artifact.walk()): + source.cache_key = ckc.compute_key(source) + source.cache_id = ckc.get_cache_id(source) + + root_artifact.build_env = build_env + def resolve_artifacts(self, srcpool): '''Resolve the artifacts that will be built for a set of sources''' @@ -150,38 +165,20 @@ class BuildCommand(object): ar = morphlib.artifactresolver.ArtifactResolver() self.app.status(msg='Resolving artifacts', chatty=True) - artifacts = ar.resolve_artifacts(srcpool) + root_artifacts = ar.resolve_root_artifacts(srcpool) - self.app.status(msg='Computing build order', chatty=True) - root_artifacts = self._find_root_artifacts(artifacts) if len(root_artifacts) > 1: - # Validate root artifacts, since validation covers errors - # such as trying to build a chunk or stratum directly, - # and this is one cause for having multiple root artifacts + # Validate root artifacts to give a more useful error message for root_artifact in root_artifacts: self._validate_root_artifact(root_artifact) raise MultipleRootArtifactsError(root_artifacts) - root_artifact = root_artifacts[0] - # Validate the root artifact here, since it's a costly function - # to finalise it, so any pre finalisation validation is better - # done before that happens, but we also don't want to expose - # the root artifact until it's finalised. + root_artifact = root_artifacts[0] self.app.status(msg='Validating root artifact', chatty=True) self._validate_root_artifact(root_artifact) - arch = root_artifact.source.morphology['arch'] - self.app.status(msg='Creating build environment for %(arch)s', - arch=arch, chatty=True) - build_env = self.new_build_env(arch) + self._compute_cache_keys(root_artifact) - self.app.status(msg='Computing cache keys', chatty=True) - ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) - for source in set(a.source for a in artifacts): - source.cache_key = ckc.compute_key(source) - source.cache_id = ckc.get_cache_id(source) - - root_artifact.build_env = build_env return root_artifact def _validate_cross_morphology_references(self, srcpool): @@ -251,17 +248,6 @@ class BuildCommand(object): other.morphology['kind'], wanted)) - def _find_root_artifacts(self, artifacts): - '''Find all the root artifacts among a set of artifacts in a DAG. - - It would be nice if the ArtifactResolver would return its results in a - more useful order to save us from needing to do this -- the root object - is known already since that's the one the user asked us to build. - - ''' - - return [a for a in artifacts if not a.dependents] - @staticmethod def get_ordered_sources(artifacts): ordered_sources = [] @@ -370,8 +356,9 @@ class BuildCommand(object): self.remove_staging_area(staging_area) td = datetime.datetime.now() - starttime - td_string = "%02d:%02d:%02d" % (td.seconds/3600, - td.seconds%3600/60, td.seconds%60) + hours, remainder = divmod(int(td.total_seconds()), 60*60) + minutes, seconds = divmod(remainder, 60) + td_string = "%02d:%02d:%02d" % (hours, minutes, seconds) self.app.status(msg="Elapsed time %(duration)s", duration=td_string) def get_recursive_deps(self, artifacts): diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 20cae225..9cd3a074 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -162,46 +162,6 @@ def get_stratum_files(f, lac): # pragma: no cover cf.close() -def get_overlaps(source, constituents, lac): # pragma: no cover - # check whether strata overlap - installed = defaultdict(set) - for dep in constituents: - handle = lac.get(dep) - if source.morphology['kind'] == 'stratum': - for filename in get_chunk_files(handle): - installed[filename].add(dep) - elif source.morphology['kind'] == 'system': - for filename in get_stratum_files(handle, lac): - installed[filename].add(dep) - handle.close() - overlaps = defaultdict(set) - for filename, artifacts in installed.iteritems(): - if len(artifacts) > 1: - overlaps[frozenset(artifacts)].add(filename) - return overlaps - - -def log_overlaps(overlaps): # pragma: no cover - for overlapping, files in sorted(overlaps.iteritems()): - logging.warning(' Artifacts %s overlap with files:' % - ', '.join(sorted(a.name for a in overlapping))) - for filename in sorted(files): - logging.warning(' %s' % filename) - - -def write_overlap_metadata(artifact, overlaps, lac): # pragma: no cover - f = lac.put_artifact_metadata(artifact, 'overlaps') - # the big list comprehension is because json can't serialize - # artifacts, sets or dicts with non-string keys - json.dump( - [ - [ - [a.name for a in afs], list(files) - ] for afs, files in overlaps.iteritems() - ], f, indent=4, encoding='unicode-escape') - f.close() - - class BuilderBase(object): '''Base class for building artifacts.''' @@ -559,18 +519,6 @@ class StratumBuilder(BuilderBase): download_depends(constituents, self.local_artifact_cache, self.remote_artifact_cache) - # check for chunk overlaps - overlaps = get_overlaps(self.source, constituents, - self.local_artifact_cache) - if len(overlaps) > 0: - logging.warning( - 'Overlaps in stratum artifact %s detected' %a_name) - log_overlaps(overlaps) - self.app.status(msg='Overlaps in stratum artifact ' - '%(stratum_name)s detected', - stratum_name=a_name, error=True) - write_overlap_metadata(a, overlaps, - self.local_artifact_cache) with self.build_watch('create-chunk-list'): lac = self.local_artifact_cache @@ -674,18 +622,6 @@ class SystemBuilder(BuilderBase): # pragma: no cover self.remote_artifact_cache) f.close() - # check whether the strata overlap - overlaps = get_overlaps(self.source, self.source.dependencies, - self.local_artifact_cache) - if len(overlaps) > 0: - self.app.status(msg='Overlaps in system artifact ' - '%(artifact_name)s detected', - artifact_name=a_name, - error=True) - log_overlaps(overlaps) - write_overlap_metadata(a, overlaps, - self.local_artifact_cache) - # unpack it from the local artifact cache for stratum_artifact in self.source.dependencies: self.unpack_one_stratum(stratum_artifact, path) diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py index 55936f94..57739983 100644 --- a/morphlib/cachekeycomputer_tests.py +++ b/morphlib/cachekeycomputer_tests.py @@ -99,7 +99,7 @@ class CacheKeyComputerTests(unittest.TestCase): "USER": "foouser", "USERNAME": "foouser"}, 'dummy') self.artifact_resolver = morphlib.artifactresolver.ArtifactResolver() - self.artifacts = self.artifact_resolver.resolve_artifacts( + self.artifacts = self.artifact_resolver._resolve_artifacts( self.source_pool) self.ckc = morphlib.cachekeycomputer.CacheKeyComputer(self.build_env) diff --git a/morphlib/exts/ssh-rsync.check b/morphlib/exts/ssh-rsync.check index 6a776ce9..11446c28 100755 --- a/morphlib/exts/ssh-rsync.check +++ b/morphlib/exts/ssh-rsync.check @@ -18,8 +18,9 @@ import cliapp -import morphlib.writeexts +import os +import morphlib.writeexts class SshRsyncCheckExtension(morphlib.writeexts.WriteExtension): def process_args(self, args): @@ -33,6 +34,10 @@ class SshRsyncCheckExtension(morphlib.writeexts.WriteExtension): 'Baserock machines. It cannot be used for an initial ' 'deployment.') + if os.environ.get('VERSION_LABEL', '') == '': + raise cliapp.AppException( + 'A VERSION_LABEL must be set when deploying an upgrade.') + location = args[0] self.check_ssh_connectivity(location) self.check_is_baserock_system(location) diff --git a/morphlib/exts/ssh-rsync.write b/morphlib/exts/ssh-rsync.write index c139b6c0..775619ec 100755 --- a/morphlib/exts/ssh-rsync.write +++ b/morphlib/exts/ssh-rsync.write @@ -54,10 +54,13 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.upgrade_remote_system(location, temp_root) def upgrade_remote_system(self, location, temp_root): - self.complete_fstab_for_btrfs_layout(temp_root) - root_disk = self.find_root_disk(location) - version_label = os.environ.get('VERSION_LABEL') + uuid = cliapp.ssh_runcmd(location, ['blkid', '-s', 'UUID', '-o', + 'value', root_disk]).strip() + + self.complete_fstab_for_btrfs_layout(temp_root, uuid) + + version_label = os.environ['VERSION_LABEL'] autostart = self.get_environment_boolean('AUTOSTART') self.status(msg='Creating remote mount point') diff --git a/morphlib/gitversion.py b/morphlib/gitversion.py index b1f82da6..c593c330 100644 --- a/morphlib/gitversion.py +++ b/morphlib/gitversion.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013 - 2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -49,7 +49,9 @@ except IOError, e: return o[0].strip() try: - version = run_git('describe', '--always', '--dirty=-unreproducible') + version = run_git('describe', '--abbrev=40', '--always', + '--dirty=-unreproducible', + '--match=DO-NOT-MATCH-ANY-TAGS') commit = run_git('rev-parse', 'HEAD^{commit}') tree = run_git('rev-parse', 'HEAD^{tree}') ref = run_git('rev-parse', '--symbolic-full-name', 'HEAD') diff --git a/morphlib/plugins/artifact_inspection_plugin.py b/morphlib/plugins/artifact_inspection_plugin.py index 6eeece77..74645f41 100644 --- a/morphlib/plugins/artifact_inspection_plugin.py +++ b/morphlib/plugins/artifact_inspection_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -257,9 +257,6 @@ class ManifestGenerator(object): class ArtifactInspectionPlugin(cliapp.Plugin): def enable(self): - self.app.add_subcommand('run-in-artifact', - self.run_in_artifact, - arg_synopsis='ARTIFACT CMD') self.app.add_subcommand('generate-manifest', self.generate_manifest, arg_synopsis='SYSTEM-ARTIFACT') @@ -267,35 +264,6 @@ class ArtifactInspectionPlugin(cliapp.Plugin): def disable(self): pass - def run_in_artifact(self, args): - '''Run a command inside an extracted/mounted chunk or system. - - Command line arguments: - - * `ARTIFACT` is a filename for the artifact. - * `CMD` is the command to run. - - run-in-artifact unpacks an artifact, and runs a command in - the temporary directory it was unpacked to. - - The command must be given to Morph as a single argument, so - use shell quoting appropriately. - - ''' - - if len(args) < 2: - raise cliapp.AppException( - 'run-in-artifact requires arguments: a chunk or ' - 'system artifact and a a shell command') - - artifact, cmd = args[0], args[1:] - - def run_command_in_dir(dirname): - output = self.app.runcmd(cmd, cwd=dirname) - self.app.output.write(output) - - call_in_artifact_directory(self.app, artifact, run_command_in_dir) - def generate_manifest(self, args): '''Generate a content manifest for a system image. diff --git a/morphlib/plugins/distbuild_plugin.py b/morphlib/plugins/distbuild_plugin.py index 50ab7eeb..653eeae8 100644 --- a/morphlib/plugins/distbuild_plugin.py +++ b/morphlib/plugins/distbuild_plugin.py @@ -103,7 +103,7 @@ class WorkerBuild(cliapp.Plugin): self.app.subcommands['gc']([]) arch = artifact.arch - bc.build_artifact(artifact, bc.new_build_env(arch)) + bc.build_source(artifact.source, bc.new_build_env(arch)) def is_system_artifact(self, filename): return re.match(r'^[0-9a-fA-F]{64}\.system\.', filename) @@ -121,6 +121,11 @@ class WorkerDaemon(cliapp.Plugin): 'listen for connections on PORT', default=3434, group=group_distbuild) + self.app.settings.string( + ['worker-daemon-port-file'], + 'write port used by worker-daemon to FILE', + default='', + group=group_distbuild) self.app.add_subcommand( 'worker-daemon', self.worker_daemon, @@ -136,7 +141,9 @@ class WorkerDaemon(cliapp.Plugin): address = self.app.settings['worker-daemon-address'] port = self.app.settings['worker-daemon-port'] - router = distbuild.ListenServer(address, port, distbuild.JsonRouter) + port_file = self.app.settings['worker-daemon-port-file'] + router = distbuild.ListenServer(address, port, distbuild.JsonRouter, + port_file=port_file) loop = distbuild.MainLoop() loop.add_state_machine(router) loop.run() @@ -156,6 +163,16 @@ class ControllerDaemon(cliapp.Plugin): 'listen for initiator connections on PORT', default=7878, group=group_distbuild) + self.app.settings.string( + ['controller-initiator-port-file'], + 'write the port to listen for initiator connections to FILE', + default='', + group=group_distbuild) + self.app.settings.string( + ['initiator-step-output-dir'], + 'write build output to files in DIR', + default='.', + group=group_distbuild) self.app.settings.string( ['controller-helper-address'], @@ -167,6 +184,11 @@ class ControllerDaemon(cliapp.Plugin): 'listen for helper connections on PORT', default=5656, group=group_distbuild) + self.app.settings.string( + ['controller-helper-port-file'], + 'write the port to listen for helper connections to FILE', + default='', + group=group_distbuild) self.app.settings.string_list( ['worker'], @@ -218,8 +240,10 @@ class ControllerDaemon(cliapp.Plugin): listener_specs = [ # address, port, class to initiate on connection, class init args ('controller-helper-address', 'controller-helper-port', + 'controller-helper-port-file', distbuild.HelperRouter, []), ('controller-initiator-address', 'controller-initiator-port', + 'controller-initiator-port-file', distbuild.InitiatorConnection, [artifact_cache_server, morph_instance]), ] @@ -229,11 +253,12 @@ class ControllerDaemon(cliapp.Plugin): queuer = distbuild.WorkerBuildQueuer() loop.add_state_machine(queuer) - for addr, port, sm, extra_args in listener_specs: + for addr, port, port_file, sm, extra_args in listener_specs: addr = self.app.settings[addr] port = self.app.settings[port] + port_file = self.app.settings[port_file] listener = distbuild.ListenServer( - addr, port, sm, extra_args=extra_args) + addr, port, sm, extra_args=extra_args, port_file=port_file) loop.add_state_machine(listener) for worker in self.app.settings['worker']: diff --git a/morphlib/plugins/list_artifacts_plugin.py b/morphlib/plugins/list_artifacts_plugin.py index 8074206b..61c8d160 100644 --- a/morphlib/plugins/list_artifacts_plugin.py +++ b/morphlib/plugins/list_artifacts_plugin.py @@ -105,11 +105,13 @@ class ListArtifactsPlugin(cliapp.Plugin): self.app.settings, system_artifact.source.morphology['arch']) ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) - artifact_files = set() - for artifact in system_artifact.walk(): + for source in set(a.source for a in system_artifact.walk()): artifact.cache_key = ckc.compute_key(artifact) artifact.cache_id = ckc.get_cache_id(artifact) + artifact_files = set() + for artifact in system_artifact.walk(): + artifact_files.add(artifact.basename()) if artifact.source.morphology.needs_artifact_metadata_cached: diff --git a/morphlib/util.py b/morphlib/util.py index dc3dd474..cc8ce88d 100644 --- a/morphlib/util.py +++ b/morphlib/util.py @@ -142,7 +142,18 @@ def new_artifact_caches(settings): # pragma: no cover def combine_aliases(app): # pragma: no cover - '''Create a full repo-alias set from the app's settings.''' + '''Create a full repo-alias set from the app's settings. + + The standard 'baserock:' and 'upstream:' keyed URLs use whatever trove was + set as 'trove-host'. + + Every trove listed in 'trove-ids' has its own repo-alias created in + addition to the defaults. We assume these require authenticated access, so + the keyed URL expansions for these troves are ssh:// URLs for both read and + write access. This can be overridden by the user if they calculate the full + repo-alias string and set it in their config manually. + + ''' trove_host = app.settings['trove-host'] trove_ids = app.settings['trove-id'] repo_aliases = app.settings['repo-alias'] @@ -173,7 +184,7 @@ def combine_aliases(app): # pragma: no cover m.group('prefix'), _expand(m.group('pull'), m.group('path')), _expand(m.group('push'), m.group('path'))) - elif '=' not in trove_id: + elif '=' not in trove_id and trove_id not in alias_map: alias_map[trove_id] = "%s=%s#%s" % ( trove_id, _expand('ssh', trove_id), diff --git a/morphlib/xfer-hole b/morphlib/xfer-hole index 0d4cee7a..22ee06bf 100755 --- a/morphlib/xfer-hole +++ b/morphlib/xfer-hole @@ -120,8 +120,14 @@ def make_xfer_instructions(fd): def copy_slice_from_file(to, fd, start, end): safe_lseek(fd, start, os.SEEK_SET) - data = os.read(fd, end - start) - to.write(data) + nbytes = end - start + max_at_a_time = 1024**2 + while nbytes > 0: + data = os.read(fd, min(nbytes, max_at_a_time)) + if not data: + break + to.write(data) + nbytes -= len(data) for kind, start, end in make_xfer_instructions(fd): |