summaryrefslogtreecommitdiff
path: root/morphlib
diff options
context:
space:
mode:
Diffstat (limited to 'morphlib')
-rw-r--r--morphlib/artifactresolver.py6
-rw-r--r--morphlib/artifactresolver_tests.py16
-rw-r--r--morphlib/artifactsplitrule.py8
-rw-r--r--morphlib/buildcommand.py57
-rw-r--r--morphlib/builder2.py64
-rw-r--r--morphlib/cachekeycomputer_tests.py2
-rwxr-xr-xmorphlib/exts/ssh-rsync.check7
-rwxr-xr-xmorphlib/exts/ssh-rsync.write9
-rw-r--r--morphlib/gitversion.py6
-rw-r--r--morphlib/plugins/artifact_inspection_plugin.py34
-rw-r--r--morphlib/plugins/distbuild_plugin.py33
-rw-r--r--morphlib/plugins/list_artifacts_plugin.py6
-rw-r--r--morphlib/util.py15
-rwxr-xr-xmorphlib/xfer-hole10
14 files changed, 111 insertions, 162 deletions
diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py
index 5deb25b7..8c8b37d0 100644
--- a/morphlib/artifactresolver.py
+++ b/morphlib/artifactresolver.py
@@ -52,7 +52,11 @@ class ArtifactResolver(object):
self._added_artifacts = None
self._source_pool = None
- def resolve_artifacts(self, source_pool):
+ def resolve_root_artifacts(self, source_pool): #pragma: no cover
+ return [a for a in self._resolve_artifacts(source_pool)
+ if not a.dependents]
+
+ def _resolve_artifacts(self, source_pool):
self._source_pool = source_pool
self._added_artifacts = set()
diff --git a/morphlib/artifactresolver_tests.py b/morphlib/artifactresolver_tests.py
index 89f30010..b958da4f 100644
--- a/morphlib/artifactresolver_tests.py
+++ b/morphlib/artifactresolver_tests.py
@@ -82,7 +82,7 @@ class ArtifactResolverTests(unittest.TestCase):
def test_resolve_artifacts_using_an_empty_pool(self):
pool = morphlib.sourcepool.SourcePool()
- artifacts = self.resolver.resolve_artifacts(pool)
+ artifacts = self.resolver._resolve_artifacts(pool)
self.assertEqual(len(artifacts), 0)
def test_resolve_single_chunk_with_no_subartifacts(self):
@@ -95,7 +95,7 @@ class ArtifactResolverTests(unittest.TestCase):
for source in sources:
pool.add(source)
- artifacts = self.resolver.resolve_artifacts(pool)
+ artifacts = self.resolver._resolve_artifacts(pool)
self.assertEqual(len(artifacts),
sum(len(s.split_rules.artifacts) for s in pool))
@@ -116,7 +116,7 @@ class ArtifactResolverTests(unittest.TestCase):
for source in sources:
pool.add(source)
- artifacts = self.resolver.resolve_artifacts(pool)
+ artifacts = self.resolver._resolve_artifacts(pool)
self.assertEqual(len(artifacts),
sum(len(s.split_rules.artifacts) for s in pool))
@@ -136,7 +136,7 @@ class ArtifactResolverTests(unittest.TestCase):
for source in sources:
pool.add(source)
- artifacts = self.resolver.resolve_artifacts(pool)
+ artifacts = self.resolver._resolve_artifacts(pool)
artifacts.sort(key=lambda a: a.name)
self.assertEqual(len(artifacts),
@@ -167,7 +167,7 @@ class ArtifactResolverTests(unittest.TestCase):
for stratum in stratum_sources:
pool.add(stratum)
- artifacts = self.resolver.resolve_artifacts(pool)
+ artifacts = self.resolver._resolve_artifacts(pool)
all_artifacts = set()
for s in pool: all_artifacts.update(s.split_rules.artifacts)
@@ -216,7 +216,7 @@ class ArtifactResolverTests(unittest.TestCase):
for stratum in stratum_sources:
pool.add(stratum)
- artifacts = self.resolver.resolve_artifacts(pool)
+ artifacts = self.resolver._resolve_artifacts(pool)
self.assertEqual(
set(artifacts),
@@ -278,7 +278,7 @@ class ArtifactResolverTests(unittest.TestCase):
pool.add(stratum2)
self.assertRaises(morphlib.artifactresolver.MutualDependencyError,
- self.resolver.resolve_artifacts, pool)
+ self.resolver._resolve_artifacts, pool)
def test_detection_of_chunk_dependencies_in_invalid_order(self):
pool = morphlib.sourcepool.SourcePool()
@@ -321,7 +321,7 @@ class ArtifactResolverTests(unittest.TestCase):
pool.add(chunk2)
self.assertRaises(morphlib.artifactresolver.DependencyOrderError,
- self.resolver.resolve_artifacts, pool)
+ self.resolver._resolve_artifacts, pool)
# TODO: Expand test suite to include better dependency checking, many
diff --git a/morphlib/artifactsplitrule.py b/morphlib/artifactsplitrule.py
index cf0d1060..1511d694 100644
--- a/morphlib/artifactsplitrule.py
+++ b/morphlib/artifactsplitrule.py
@@ -230,7 +230,7 @@ DEFAULT_STRATUM_RULES = [
]
-def unify_chunk_matches(morphology):
+def unify_chunk_matches(morphology, default_rules=DEFAULT_CHUNK_RULES):
'''Create split rules including defaults and per-chunk rules.
With rules specified in the morphology's 'products' field and the
@@ -246,7 +246,7 @@ def unify_chunk_matches(morphology):
split_rules.add(ca_name, FileMatch(patterns))
name = morphology['name']
- for suffix, patterns in DEFAULT_CHUNK_RULES:
+ for suffix, patterns in default_rules:
ca_name = name + suffix
# Explicit rules override the default rules. This is an all-or-nothing
# override: there is no way to extend the default split rules right now
@@ -257,7 +257,7 @@ def unify_chunk_matches(morphology):
return split_rules
-def unify_stratum_matches(morphology):
+def unify_stratum_matches(morphology, default_rules=DEFAULT_STRATUM_RULES):
'''Create split rules including defaults and per-stratum rules.
With rules specified in the chunk spec's 'artifacts' fields, the
@@ -284,7 +284,7 @@ def unify_stratum_matches(morphology):
for d in morphology.get('products', {})):
match_split_rules.add(sta_name, ArtifactMatch(patterns))
- for suffix, patterns in DEFAULT_STRATUM_RULES:
+ for suffix, patterns in default_rules:
sta_name = morphology['name'] + suffix
# Explicit rules override the default rules. This is an all-or-nothing
# override: there is no way to extend the default split rules right now
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index c5ba05e5..2aec5e08 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -143,6 +143,21 @@ class BuildCommand(object):
for spec in src.morphology['chunks']):
raise morphlib.Error('No non-bootstrap chunks found.')
+ def _compute_cache_keys(self, root_artifact):
+ arch = root_artifact.source.morphology['arch']
+ self.app.status(msg='Creating build environment for %(arch)s',
+ arch=arch, chatty=True)
+ build_env = self.new_build_env(arch)
+
+ self.app.status(msg='Computing cache keys', chatty=True)
+ ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env)
+
+ for source in set(a.source for a in root_artifact.walk()):
+ source.cache_key = ckc.compute_key(source)
+ source.cache_id = ckc.get_cache_id(source)
+
+ root_artifact.build_env = build_env
+
def resolve_artifacts(self, srcpool):
'''Resolve the artifacts that will be built for a set of sources'''
@@ -150,38 +165,20 @@ class BuildCommand(object):
ar = morphlib.artifactresolver.ArtifactResolver()
self.app.status(msg='Resolving artifacts', chatty=True)
- artifacts = ar.resolve_artifacts(srcpool)
+ root_artifacts = ar.resolve_root_artifacts(srcpool)
- self.app.status(msg='Computing build order', chatty=True)
- root_artifacts = self._find_root_artifacts(artifacts)
if len(root_artifacts) > 1:
- # Validate root artifacts, since validation covers errors
- # such as trying to build a chunk or stratum directly,
- # and this is one cause for having multiple root artifacts
+ # Validate root artifacts to give a more useful error message
for root_artifact in root_artifacts:
self._validate_root_artifact(root_artifact)
raise MultipleRootArtifactsError(root_artifacts)
- root_artifact = root_artifacts[0]
- # Validate the root artifact here, since it's a costly function
- # to finalise it, so any pre finalisation validation is better
- # done before that happens, but we also don't want to expose
- # the root artifact until it's finalised.
+ root_artifact = root_artifacts[0]
self.app.status(msg='Validating root artifact', chatty=True)
self._validate_root_artifact(root_artifact)
- arch = root_artifact.source.morphology['arch']
- self.app.status(msg='Creating build environment for %(arch)s',
- arch=arch, chatty=True)
- build_env = self.new_build_env(arch)
+ self._compute_cache_keys(root_artifact)
- self.app.status(msg='Computing cache keys', chatty=True)
- ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env)
- for source in set(a.source for a in artifacts):
- source.cache_key = ckc.compute_key(source)
- source.cache_id = ckc.get_cache_id(source)
-
- root_artifact.build_env = build_env
return root_artifact
def _validate_cross_morphology_references(self, srcpool):
@@ -251,17 +248,6 @@ class BuildCommand(object):
other.morphology['kind'],
wanted))
- def _find_root_artifacts(self, artifacts):
- '''Find all the root artifacts among a set of artifacts in a DAG.
-
- It would be nice if the ArtifactResolver would return its results in a
- more useful order to save us from needing to do this -- the root object
- is known already since that's the one the user asked us to build.
-
- '''
-
- return [a for a in artifacts if not a.dependents]
-
@staticmethod
def get_ordered_sources(artifacts):
ordered_sources = []
@@ -370,8 +356,9 @@ class BuildCommand(object):
self.remove_staging_area(staging_area)
td = datetime.datetime.now() - starttime
- td_string = "%02d:%02d:%02d" % (td.seconds/3600,
- td.seconds%3600/60, td.seconds%60)
+ hours, remainder = divmod(int(td.total_seconds()), 60*60)
+ minutes, seconds = divmod(remainder, 60)
+ td_string = "%02d:%02d:%02d" % (hours, minutes, seconds)
self.app.status(msg="Elapsed time %(duration)s", duration=td_string)
def get_recursive_deps(self, artifacts):
diff --git a/morphlib/builder2.py b/morphlib/builder2.py
index 20cae225..9cd3a074 100644
--- a/morphlib/builder2.py
+++ b/morphlib/builder2.py
@@ -162,46 +162,6 @@ def get_stratum_files(f, lac): # pragma: no cover
cf.close()
-def get_overlaps(source, constituents, lac): # pragma: no cover
- # check whether strata overlap
- installed = defaultdict(set)
- for dep in constituents:
- handle = lac.get(dep)
- if source.morphology['kind'] == 'stratum':
- for filename in get_chunk_files(handle):
- installed[filename].add(dep)
- elif source.morphology['kind'] == 'system':
- for filename in get_stratum_files(handle, lac):
- installed[filename].add(dep)
- handle.close()
- overlaps = defaultdict(set)
- for filename, artifacts in installed.iteritems():
- if len(artifacts) > 1:
- overlaps[frozenset(artifacts)].add(filename)
- return overlaps
-
-
-def log_overlaps(overlaps): # pragma: no cover
- for overlapping, files in sorted(overlaps.iteritems()):
- logging.warning(' Artifacts %s overlap with files:' %
- ', '.join(sorted(a.name for a in overlapping)))
- for filename in sorted(files):
- logging.warning(' %s' % filename)
-
-
-def write_overlap_metadata(artifact, overlaps, lac): # pragma: no cover
- f = lac.put_artifact_metadata(artifact, 'overlaps')
- # the big list comprehension is because json can't serialize
- # artifacts, sets or dicts with non-string keys
- json.dump(
- [
- [
- [a.name for a in afs], list(files)
- ] for afs, files in overlaps.iteritems()
- ], f, indent=4, encoding='unicode-escape')
- f.close()
-
-
class BuilderBase(object):
'''Base class for building artifacts.'''
@@ -559,18 +519,6 @@ class StratumBuilder(BuilderBase):
download_depends(constituents,
self.local_artifact_cache,
self.remote_artifact_cache)
- # check for chunk overlaps
- overlaps = get_overlaps(self.source, constituents,
- self.local_artifact_cache)
- if len(overlaps) > 0:
- logging.warning(
- 'Overlaps in stratum artifact %s detected' %a_name)
- log_overlaps(overlaps)
- self.app.status(msg='Overlaps in stratum artifact '
- '%(stratum_name)s detected',
- stratum_name=a_name, error=True)
- write_overlap_metadata(a, overlaps,
- self.local_artifact_cache)
with self.build_watch('create-chunk-list'):
lac = self.local_artifact_cache
@@ -674,18 +622,6 @@ class SystemBuilder(BuilderBase): # pragma: no cover
self.remote_artifact_cache)
f.close()
- # check whether the strata overlap
- overlaps = get_overlaps(self.source, self.source.dependencies,
- self.local_artifact_cache)
- if len(overlaps) > 0:
- self.app.status(msg='Overlaps in system artifact '
- '%(artifact_name)s detected',
- artifact_name=a_name,
- error=True)
- log_overlaps(overlaps)
- write_overlap_metadata(a, overlaps,
- self.local_artifact_cache)
-
# unpack it from the local artifact cache
for stratum_artifact in self.source.dependencies:
self.unpack_one_stratum(stratum_artifact, path)
diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py
index 55936f94..57739983 100644
--- a/morphlib/cachekeycomputer_tests.py
+++ b/morphlib/cachekeycomputer_tests.py
@@ -99,7 +99,7 @@ class CacheKeyComputerTests(unittest.TestCase):
"USER": "foouser",
"USERNAME": "foouser"}, 'dummy')
self.artifact_resolver = morphlib.artifactresolver.ArtifactResolver()
- self.artifacts = self.artifact_resolver.resolve_artifacts(
+ self.artifacts = self.artifact_resolver._resolve_artifacts(
self.source_pool)
self.ckc = morphlib.cachekeycomputer.CacheKeyComputer(self.build_env)
diff --git a/morphlib/exts/ssh-rsync.check b/morphlib/exts/ssh-rsync.check
index 6a776ce9..11446c28 100755
--- a/morphlib/exts/ssh-rsync.check
+++ b/morphlib/exts/ssh-rsync.check
@@ -18,8 +18,9 @@
import cliapp
-import morphlib.writeexts
+import os
+import morphlib.writeexts
class SshRsyncCheckExtension(morphlib.writeexts.WriteExtension):
def process_args(self, args):
@@ -33,6 +34,10 @@ class SshRsyncCheckExtension(morphlib.writeexts.WriteExtension):
'Baserock machines. It cannot be used for an initial '
'deployment.')
+ if os.environ.get('VERSION_LABEL', '') == '':
+ raise cliapp.AppException(
+ 'A VERSION_LABEL must be set when deploying an upgrade.')
+
location = args[0]
self.check_ssh_connectivity(location)
self.check_is_baserock_system(location)
diff --git a/morphlib/exts/ssh-rsync.write b/morphlib/exts/ssh-rsync.write
index c139b6c0..775619ec 100755
--- a/morphlib/exts/ssh-rsync.write
+++ b/morphlib/exts/ssh-rsync.write
@@ -54,10 +54,13 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension):
self.upgrade_remote_system(location, temp_root)
def upgrade_remote_system(self, location, temp_root):
- self.complete_fstab_for_btrfs_layout(temp_root)
-
root_disk = self.find_root_disk(location)
- version_label = os.environ.get('VERSION_LABEL')
+ uuid = cliapp.ssh_runcmd(location, ['blkid', '-s', 'UUID', '-o',
+ 'value', root_disk]).strip()
+
+ self.complete_fstab_for_btrfs_layout(temp_root, uuid)
+
+ version_label = os.environ['VERSION_LABEL']
autostart = self.get_environment_boolean('AUTOSTART')
self.status(msg='Creating remote mount point')
diff --git a/morphlib/gitversion.py b/morphlib/gitversion.py
index b1f82da6..c593c330 100644
--- a/morphlib/gitversion.py
+++ b/morphlib/gitversion.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2013 Codethink Limited
+# Copyright (C) 2013 - 2014 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -49,7 +49,9 @@ except IOError, e:
return o[0].strip()
try:
- version = run_git('describe', '--always', '--dirty=-unreproducible')
+ version = run_git('describe', '--abbrev=40', '--always',
+ '--dirty=-unreproducible',
+ '--match=DO-NOT-MATCH-ANY-TAGS')
commit = run_git('rev-parse', 'HEAD^{commit}')
tree = run_git('rev-parse', 'HEAD^{tree}')
ref = run_git('rev-parse', '--symbolic-full-name', 'HEAD')
diff --git a/morphlib/plugins/artifact_inspection_plugin.py b/morphlib/plugins/artifact_inspection_plugin.py
index 6eeece77..74645f41 100644
--- a/morphlib/plugins/artifact_inspection_plugin.py
+++ b/morphlib/plugins/artifact_inspection_plugin.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2013 Codethink Limited
+# Copyright (C) 2012-2014 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -257,9 +257,6 @@ class ManifestGenerator(object):
class ArtifactInspectionPlugin(cliapp.Plugin):
def enable(self):
- self.app.add_subcommand('run-in-artifact',
- self.run_in_artifact,
- arg_synopsis='ARTIFACT CMD')
self.app.add_subcommand('generate-manifest',
self.generate_manifest,
arg_synopsis='SYSTEM-ARTIFACT')
@@ -267,35 +264,6 @@ class ArtifactInspectionPlugin(cliapp.Plugin):
def disable(self):
pass
- def run_in_artifact(self, args):
- '''Run a command inside an extracted/mounted chunk or system.
-
- Command line arguments:
-
- * `ARTIFACT` is a filename for the artifact.
- * `CMD` is the command to run.
-
- run-in-artifact unpacks an artifact, and runs a command in
- the temporary directory it was unpacked to.
-
- The command must be given to Morph as a single argument, so
- use shell quoting appropriately.
-
- '''
-
- if len(args) < 2:
- raise cliapp.AppException(
- 'run-in-artifact requires arguments: a chunk or '
- 'system artifact and a a shell command')
-
- artifact, cmd = args[0], args[1:]
-
- def run_command_in_dir(dirname):
- output = self.app.runcmd(cmd, cwd=dirname)
- self.app.output.write(output)
-
- call_in_artifact_directory(self.app, artifact, run_command_in_dir)
-
def generate_manifest(self, args):
'''Generate a content manifest for a system image.
diff --git a/morphlib/plugins/distbuild_plugin.py b/morphlib/plugins/distbuild_plugin.py
index 50ab7eeb..653eeae8 100644
--- a/morphlib/plugins/distbuild_plugin.py
+++ b/morphlib/plugins/distbuild_plugin.py
@@ -103,7 +103,7 @@ class WorkerBuild(cliapp.Plugin):
self.app.subcommands['gc']([])
arch = artifact.arch
- bc.build_artifact(artifact, bc.new_build_env(arch))
+ bc.build_source(artifact.source, bc.new_build_env(arch))
def is_system_artifact(self, filename):
return re.match(r'^[0-9a-fA-F]{64}\.system\.', filename)
@@ -121,6 +121,11 @@ class WorkerDaemon(cliapp.Plugin):
'listen for connections on PORT',
default=3434,
group=group_distbuild)
+ self.app.settings.string(
+ ['worker-daemon-port-file'],
+ 'write port used by worker-daemon to FILE',
+ default='',
+ group=group_distbuild)
self.app.add_subcommand(
'worker-daemon',
self.worker_daemon,
@@ -136,7 +141,9 @@ class WorkerDaemon(cliapp.Plugin):
address = self.app.settings['worker-daemon-address']
port = self.app.settings['worker-daemon-port']
- router = distbuild.ListenServer(address, port, distbuild.JsonRouter)
+ port_file = self.app.settings['worker-daemon-port-file']
+ router = distbuild.ListenServer(address, port, distbuild.JsonRouter,
+ port_file=port_file)
loop = distbuild.MainLoop()
loop.add_state_machine(router)
loop.run()
@@ -156,6 +163,16 @@ class ControllerDaemon(cliapp.Plugin):
'listen for initiator connections on PORT',
default=7878,
group=group_distbuild)
+ self.app.settings.string(
+ ['controller-initiator-port-file'],
+ 'write the port to listen for initiator connections to FILE',
+ default='',
+ group=group_distbuild)
+ self.app.settings.string(
+ ['initiator-step-output-dir'],
+ 'write build output to files in DIR',
+ default='.',
+ group=group_distbuild)
self.app.settings.string(
['controller-helper-address'],
@@ -167,6 +184,11 @@ class ControllerDaemon(cliapp.Plugin):
'listen for helper connections on PORT',
default=5656,
group=group_distbuild)
+ self.app.settings.string(
+ ['controller-helper-port-file'],
+ 'write the port to listen for helper connections to FILE',
+ default='',
+ group=group_distbuild)
self.app.settings.string_list(
['worker'],
@@ -218,8 +240,10 @@ class ControllerDaemon(cliapp.Plugin):
listener_specs = [
# address, port, class to initiate on connection, class init args
('controller-helper-address', 'controller-helper-port',
+ 'controller-helper-port-file',
distbuild.HelperRouter, []),
('controller-initiator-address', 'controller-initiator-port',
+ 'controller-initiator-port-file',
distbuild.InitiatorConnection,
[artifact_cache_server, morph_instance]),
]
@@ -229,11 +253,12 @@ class ControllerDaemon(cliapp.Plugin):
queuer = distbuild.WorkerBuildQueuer()
loop.add_state_machine(queuer)
- for addr, port, sm, extra_args in listener_specs:
+ for addr, port, port_file, sm, extra_args in listener_specs:
addr = self.app.settings[addr]
port = self.app.settings[port]
+ port_file = self.app.settings[port_file]
listener = distbuild.ListenServer(
- addr, port, sm, extra_args=extra_args)
+ addr, port, sm, extra_args=extra_args, port_file=port_file)
loop.add_state_machine(listener)
for worker in self.app.settings['worker']:
diff --git a/morphlib/plugins/list_artifacts_plugin.py b/morphlib/plugins/list_artifacts_plugin.py
index 8074206b..61c8d160 100644
--- a/morphlib/plugins/list_artifacts_plugin.py
+++ b/morphlib/plugins/list_artifacts_plugin.py
@@ -105,11 +105,13 @@ class ListArtifactsPlugin(cliapp.Plugin):
self.app.settings, system_artifact.source.morphology['arch'])
ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env)
- artifact_files = set()
- for artifact in system_artifact.walk():
+ for source in set(a.source for a in system_artifact.walk()):
artifact.cache_key = ckc.compute_key(artifact)
artifact.cache_id = ckc.get_cache_id(artifact)
+ artifact_files = set()
+ for artifact in system_artifact.walk():
+
artifact_files.add(artifact.basename())
if artifact.source.morphology.needs_artifact_metadata_cached:
diff --git a/morphlib/util.py b/morphlib/util.py
index dc3dd474..cc8ce88d 100644
--- a/morphlib/util.py
+++ b/morphlib/util.py
@@ -142,7 +142,18 @@ def new_artifact_caches(settings): # pragma: no cover
def combine_aliases(app): # pragma: no cover
- '''Create a full repo-alias set from the app's settings.'''
+ '''Create a full repo-alias set from the app's settings.
+
+ The standard 'baserock:' and 'upstream:' keyed URLs use whatever trove was
+ set as 'trove-host'.
+
+ Every trove listed in 'trove-ids' has its own repo-alias created in
+ addition to the defaults. We assume these require authenticated access, so
+ the keyed URL expansions for these troves are ssh:// URLs for both read and
+ write access. This can be overridden by the user if they calculate the full
+ repo-alias string and set it in their config manually.
+
+ '''
trove_host = app.settings['trove-host']
trove_ids = app.settings['trove-id']
repo_aliases = app.settings['repo-alias']
@@ -173,7 +184,7 @@ def combine_aliases(app): # pragma: no cover
m.group('prefix'),
_expand(m.group('pull'), m.group('path')),
_expand(m.group('push'), m.group('path')))
- elif '=' not in trove_id:
+ elif '=' not in trove_id and trove_id not in alias_map:
alias_map[trove_id] = "%s=%s#%s" % (
trove_id,
_expand('ssh', trove_id),
diff --git a/morphlib/xfer-hole b/morphlib/xfer-hole
index 0d4cee7a..22ee06bf 100755
--- a/morphlib/xfer-hole
+++ b/morphlib/xfer-hole
@@ -120,8 +120,14 @@ def make_xfer_instructions(fd):
def copy_slice_from_file(to, fd, start, end):
safe_lseek(fd, start, os.SEEK_SET)
- data = os.read(fd, end - start)
- to.write(data)
+ nbytes = end - start
+ max_at_a_time = 1024**2
+ while nbytes > 0:
+ data = os.read(fd, min(nbytes, max_at_a_time))
+ if not data:
+ break
+ to.write(data)
+ nbytes -= len(data)
for kind, start, end in make_xfer_instructions(fd):