diff options
Diffstat (limited to 'morphlib')
-rw-r--r-- | morphlib/buildcommand.py | 102 | ||||
-rwxr-xr-x | morphlib/exts/initramfs.write | 4 | ||||
-rwxr-xr-x | morphlib/exts/kvm.check | 2 | ||||
-rwxr-xr-x | morphlib/exts/openstack.check | 2 | ||||
-rwxr-xr-x | morphlib/exts/rawdisk.check | 31 | ||||
-rwxr-xr-x | morphlib/exts/virtualbox-ssh.check | 2 | ||||
-rw-r--r-- | morphlib/plugins/cross-bootstrap_plugin.py | 14 | ||||
-rw-r--r-- | morphlib/plugins/deploy_plugin.py | 1 | ||||
-rw-r--r-- | morphlib/remoteartifactcache.py | 11 | ||||
-rw-r--r-- | morphlib/writeexts.py | 13 |
10 files changed, 127 insertions, 55 deletions
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 7ad7909d..f68046e3 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -274,31 +274,31 @@ class BuildCommand(object): 'name': a.name, }) - self.app.status(msg='Checking if %(kind)s needs ' - 'building %(sha1)s', - kind=a.source.morphology['kind'], - sha1=a.source.sha1[:7]) - - if self.is_built(a): - self.cache_artifacts_locally([a]) - self.app.status( - msg='The %(kind)s is cached at %(cache)s', - kind=a.source.morphology['kind'], - cache=os.path.basename(self.lac.artifact_filename(a))[:7]) - else: - self.app.status(msg='Building %(kind)s %(name)s', - name=a.name, kind=a.source.morphology['kind']) - self.build_artifact(a, build_env) + self.cache_or_build_artifact(a, build_env) self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s', kind=a.source.morphology['kind'], name=a.name, cachepath=self.lac.artifact_filename(a), chatty=(a.source.morphology['kind'] != "system")) + self.app.status_prefix = old_prefix - def is_built(self, artifact): - '''Does either cache already have the artifact?''' - return self.lac.has(artifact) or (self.rac and self.rac.has(artifact)) + def cache_or_build_artifact(self, artifact, build_env): + '''Make the built artifact available in the local cache. + + This can be done by retrieving from a remote artifact cache, or if + that doesn't work for some reason, by building the artifact locally. + + ''' + if self.rac is not None: + try: + self.cache_artifacts_locally([artifact]) + except morphlib.remoteartifactcache.GetError: + # Error is logged by the RemoteArtifactCache object. + pass + + if not self.lac.has(artifact): + self.build_artifact(artifact, build_env) def build_artifact(self, artifact, build_env): '''Build one artifact. @@ -307,6 +307,10 @@ class BuildCommand(object): in either the local or remote cache already. ''' + self.app.status(msg='Building %(kind)s %(name)s', + name=artifact.name, + kind=artifact.source.morphology['kind']) + self.get_sources(artifact) deps = self.get_recursive_deps(artifact) self.cache_artifacts_locally(deps) @@ -389,27 +393,46 @@ class BuildCommand(object): def cache_artifacts_locally(self, artifacts): '''Get artifacts missing from local cache from remote cache.''' - def copy(remote, local): - shutil.copyfileobj(remote, local) - remote.close() - local.close() + def fetch_files(to_fetch): + '''Fetch a set of files atomically. + + If an error occurs during the transfer of any files, all downloaded + data is deleted, to ensure integrity of the local cache. + + ''' + try: + for remote, local in to_fetch: + shutil.copyfileobj(remote, local) + except BaseException: + for remote, local in to_fetch: + local.abort() + raise + else: + for remote, local in to_fetch: + remote.close() + local.close() for artifact in artifacts: + # This block should fetch all artifact files in one go, using the + # 1.0/artifacts method of morph-cache-server. The code to do that + # needs bringing in from the distbuild.worker_build_connection + # module into morphlib.remoteartififactcache first. + to_fetch = [] if not self.lac.has(artifact): - self.app.status(msg='Fetching to local cache: ' - 'artifact %(name)s', - name=artifact.name) - rac_file = self.rac.get(artifact) - lac_file = self.lac.put(artifact) - copy(rac_file, lac_file) + to_fetch.append((self.rac.get(artifact), + self.lac.put(artifact))) if artifact.source.morphology.needs_artifact_metadata_cached: if not self.lac.has_artifact_metadata(artifact, 'meta'): - self.app.status(msg='Fetching to local cache: ' - 'artifact metadata %(name)s', - name=artifact.name) - copy(self.rac.get_artifact_metadata(artifact, 'meta'), - self.lac.put_artifact_metadata(artifact, 'meta')) + to_fetch.append(( + self.rac.get_artifact_metadata(artifact, 'meta'), + self.lac.put_artifact_metadata(artifact, 'meta'))) + + if len(to_fetch) > 0: + self.app.status( + msg='Fetching to local cache: artifact %(name)s', + name=artifact.name) + fetch_files(to_fetch) def create_staging_area(self, build_env, use_chroot=True, extra_env={}, extra_path=[]): @@ -479,6 +502,9 @@ class BuildCommand(object): class InitiatorBuildCommand(BuildCommand): + RECONNECT_INTERVAL = 30 # seconds + MAX_RETRIES = 1 + def __init__(self, app, addr, port): self.app = app self.addr = addr @@ -501,7 +527,13 @@ class InitiatorBuildCommand(BuildCommand): self.app.status(msg='Starting distributed build') loop = distbuild.MainLoop() - cm = distbuild.ConnectionMachine( - self.addr, self.port, distbuild.Initiator, [self.app] + args) + cm = distbuild.InitiatorConnectionMachine(self.app, + self.addr, + self.port, + distbuild.Initiator, + [self.app] + args, + self.RECONNECT_INTERVAL, + self.MAX_RETRIES) + loop.add_state_machine(cm) loop.run() diff --git a/morphlib/exts/initramfs.write b/morphlib/exts/initramfs.write index 815772f2..f8af6d84 100755 --- a/morphlib/exts/initramfs.write +++ b/morphlib/exts/initramfs.write @@ -23,5 +23,5 @@ INITRAMFS_PATH="$2" (cd "$ROOTDIR" && find . -print0 | - cpio -0 -H newc -o | - gzip -c) >"$INITRAMFS_PATH" + cpio -0 -H newc -o) | + gzip -c | install -D -m644 /dev/stdin "$INITRAMFS_PATH" diff --git a/morphlib/exts/kvm.check b/morphlib/exts/kvm.check index 957d0893..1bb4007a 100755 --- a/morphlib/exts/kvm.check +++ b/morphlib/exts/kvm.check @@ -31,6 +31,8 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): if len(args) != 1: raise cliapp.AppException('Wrong number of command line args') + self.require_btrfs_in_deployment_host_kernel() + upgrade = self.get_environment_boolean('UPGRADE') if upgrade: raise cliapp.AppException( diff --git a/morphlib/exts/openstack.check b/morphlib/exts/openstack.check index a9a8fe1b..b5173011 100755 --- a/morphlib/exts/openstack.check +++ b/morphlib/exts/openstack.check @@ -26,6 +26,8 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): if len(args) != 1: raise cliapp.AppException('Wrong number of command line args') + self.require_btrfs_in_deployment_host_kernel() + upgrade = self.get_environment_boolean('UPGRADE') if upgrade: raise cliapp.AppException( diff --git a/morphlib/exts/rawdisk.check b/morphlib/exts/rawdisk.check new file mode 100755 index 00000000..6a656ee7 --- /dev/null +++ b/morphlib/exts/rawdisk.check @@ -0,0 +1,31 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''Preparatory checks for Morph 'rawdisk' write extension''' + +import cliapp + +import morphlib.writeexts + + +class RawdiskCheckExtension(morphlib.writeexts.WriteExtension): + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + self.require_btrfs_in_deployment_host_kernel() + +RawdiskCheckExtension().run() diff --git a/morphlib/exts/virtualbox-ssh.check b/morphlib/exts/virtualbox-ssh.check index 1aeb8999..57d54db1 100755 --- a/morphlib/exts/virtualbox-ssh.check +++ b/morphlib/exts/virtualbox-ssh.check @@ -26,6 +26,8 @@ class VirtualBoxPlusSshCheckExtension(morphlib.writeexts.WriteExtension): if len(args) != 1: raise cliapp.AppException('Wrong number of command line args') + self.require_btrfs_in_deployment_host_kernel() + upgrade = self.get_environment_boolean('UPGRADE') if upgrade: raise cliapp.AppException( diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py index ec0cfbcb..bfd0d047 100644 --- a/morphlib/plugins/cross-bootstrap_plugin.py +++ b/morphlib/plugins/cross-bootstrap_plugin.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -299,18 +299,8 @@ class CrossBootstrapPlugin(cliapp.Plugin): 'Nothing to cross-compile. Only chunks built in \'bootstrap\' ' 'mode can be cross-compiled.') - # FIXME: merge with build-command's code for i, a in enumerate(cross_chunks): - if build_command.is_built(a): - self.app.status(msg='The %(kind)s %(name)s is already built', - kind=a.source.morphology['kind'], - name=a.name) - build_command.cache_artifacts_locally([a]) - else: - self.app.status(msg='Cross-building %(kind)s %(name)s', - kind=a.source.morphology['kind'], - name=a.name) - build_command.build_artifact(a, build_env) + build_command.cache_or_build_artifact(a, build_env) for i, a in enumerate(native_chunks): build_command.get_sources(a) diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 1d582949..3afb7b17 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -274,6 +274,7 @@ class DeployPlugin(cliapp.Plugin): self.app.settings['tempdir-min-space'], '/', 0) + self.app.settings['no-git-update'] = True cluster_name = morphlib.util.strip_morph_extension(args[0]) env_vars = args[1:] diff --git a/morphlib/remoteartifactcache.py b/morphlib/remoteartifactcache.py index 9f6bf69e..0f8edce8 100644 --- a/morphlib/remoteartifactcache.py +++ b/morphlib/remoteartifactcache.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -33,18 +33,19 @@ class GetError(cliapp.AppException): cliapp.AppException.__init__( self, 'Failed to get the artifact %s with cache key %s ' 'from the artifact cache %s' % - (artifact, artifact.cache_key, cache)) + (artifact.basename(), artifact.cache_key, cache)) -class GetArtifactMetadataError(cliapp.AppException): +class GetArtifactMetadataError(GetError): def __init__(self, cache, artifact, name): cliapp.AppException.__init__( self, 'Failed to get metadata %s for the artifact %s ' - 'from the artifact cache %s' % (name, artifact, cache)) + 'from the artifact cache %s' % + (name, artifact.basename(), cache)) -class GetSourceMetadataError(cliapp.AppException): +class GetSourceMetadataError(GetError): def __init__(self, cache, source, cache_key, name): cliapp.AppException.__init__( diff --git a/morphlib/writeexts.py b/morphlib/writeexts.py index d6f23e0d..74587bd1 100644 --- a/morphlib/writeexts.py +++ b/morphlib/writeexts.py @@ -104,7 +104,18 @@ class WriteExtension(cliapp.Application): self.output.write('%s\n' % (kwargs['msg'] % kwargs)) self.output.flush() - + + def check_for_btrfs_in_deployment_host_kernel(self): + with open('/proc/filesystems') as f: + text = f.read() + return '\tbtrfs\n' in text + + def require_btrfs_in_deployment_host_kernel(self): + if not self.check_for_btrfs_in_deployment_host_kernel(): + raise cliapp.AppException( + 'Error: Btrfs is required for this deployment, but was not ' + 'detected in the kernel of the machine that is running Morph.') + def create_local_system(self, temp_root, raw_disk): '''Create a raw system image locally.''' size = self.get_disk_size() |