summaryrefslogtreecommitdiff
path: root/morphlib
diff options
context:
space:
mode:
Diffstat (limited to 'morphlib')
-rw-r--r--morphlib/buildcommand.py102
-rwxr-xr-xmorphlib/exts/initramfs.write4
-rwxr-xr-xmorphlib/exts/kvm.check2
-rwxr-xr-xmorphlib/exts/openstack.check2
-rwxr-xr-xmorphlib/exts/rawdisk.check31
-rwxr-xr-xmorphlib/exts/virtualbox-ssh.check2
-rw-r--r--morphlib/plugins/cross-bootstrap_plugin.py14
-rw-r--r--morphlib/plugins/deploy_plugin.py1
-rw-r--r--morphlib/remoteartifactcache.py11
-rw-r--r--morphlib/writeexts.py13
10 files changed, 127 insertions, 55 deletions
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py
index 7ad7909d..f68046e3 100644
--- a/morphlib/buildcommand.py
+++ b/morphlib/buildcommand.py
@@ -274,31 +274,31 @@ class BuildCommand(object):
'name': a.name,
})
- self.app.status(msg='Checking if %(kind)s needs '
- 'building %(sha1)s',
- kind=a.source.morphology['kind'],
- sha1=a.source.sha1[:7])
-
- if self.is_built(a):
- self.cache_artifacts_locally([a])
- self.app.status(
- msg='The %(kind)s is cached at %(cache)s',
- kind=a.source.morphology['kind'],
- cache=os.path.basename(self.lac.artifact_filename(a))[:7])
- else:
- self.app.status(msg='Building %(kind)s %(name)s',
- name=a.name, kind=a.source.morphology['kind'])
- self.build_artifact(a, build_env)
+ self.cache_or_build_artifact(a, build_env)
self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s',
kind=a.source.morphology['kind'], name=a.name,
cachepath=self.lac.artifact_filename(a),
chatty=(a.source.morphology['kind'] != "system"))
+
self.app.status_prefix = old_prefix
- def is_built(self, artifact):
- '''Does either cache already have the artifact?'''
- return self.lac.has(artifact) or (self.rac and self.rac.has(artifact))
+ def cache_or_build_artifact(self, artifact, build_env):
+ '''Make the built artifact available in the local cache.
+
+ This can be done by retrieving from a remote artifact cache, or if
+ that doesn't work for some reason, by building the artifact locally.
+
+ '''
+ if self.rac is not None:
+ try:
+ self.cache_artifacts_locally([artifact])
+ except morphlib.remoteartifactcache.GetError:
+ # Error is logged by the RemoteArtifactCache object.
+ pass
+
+ if not self.lac.has(artifact):
+ self.build_artifact(artifact, build_env)
def build_artifact(self, artifact, build_env):
'''Build one artifact.
@@ -307,6 +307,10 @@ class BuildCommand(object):
in either the local or remote cache already.
'''
+ self.app.status(msg='Building %(kind)s %(name)s',
+ name=artifact.name,
+ kind=artifact.source.morphology['kind'])
+
self.get_sources(artifact)
deps = self.get_recursive_deps(artifact)
self.cache_artifacts_locally(deps)
@@ -389,27 +393,46 @@ class BuildCommand(object):
def cache_artifacts_locally(self, artifacts):
'''Get artifacts missing from local cache from remote cache.'''
- def copy(remote, local):
- shutil.copyfileobj(remote, local)
- remote.close()
- local.close()
+ def fetch_files(to_fetch):
+ '''Fetch a set of files atomically.
+
+ If an error occurs during the transfer of any files, all downloaded
+ data is deleted, to ensure integrity of the local cache.
+
+ '''
+ try:
+ for remote, local in to_fetch:
+ shutil.copyfileobj(remote, local)
+ except BaseException:
+ for remote, local in to_fetch:
+ local.abort()
+ raise
+ else:
+ for remote, local in to_fetch:
+ remote.close()
+ local.close()
for artifact in artifacts:
+ # This block should fetch all artifact files in one go, using the
+ # 1.0/artifacts method of morph-cache-server. The code to do that
+ # needs bringing in from the distbuild.worker_build_connection
+ # module into morphlib.remoteartififactcache first.
+ to_fetch = []
if not self.lac.has(artifact):
- self.app.status(msg='Fetching to local cache: '
- 'artifact %(name)s',
- name=artifact.name)
- rac_file = self.rac.get(artifact)
- lac_file = self.lac.put(artifact)
- copy(rac_file, lac_file)
+ to_fetch.append((self.rac.get(artifact),
+ self.lac.put(artifact)))
if artifact.source.morphology.needs_artifact_metadata_cached:
if not self.lac.has_artifact_metadata(artifact, 'meta'):
- self.app.status(msg='Fetching to local cache: '
- 'artifact metadata %(name)s',
- name=artifact.name)
- copy(self.rac.get_artifact_metadata(artifact, 'meta'),
- self.lac.put_artifact_metadata(artifact, 'meta'))
+ to_fetch.append((
+ self.rac.get_artifact_metadata(artifact, 'meta'),
+ self.lac.put_artifact_metadata(artifact, 'meta')))
+
+ if len(to_fetch) > 0:
+ self.app.status(
+ msg='Fetching to local cache: artifact %(name)s',
+ name=artifact.name)
+ fetch_files(to_fetch)
def create_staging_area(self, build_env, use_chroot=True, extra_env={},
extra_path=[]):
@@ -479,6 +502,9 @@ class BuildCommand(object):
class InitiatorBuildCommand(BuildCommand):
+ RECONNECT_INTERVAL = 30 # seconds
+ MAX_RETRIES = 1
+
def __init__(self, app, addr, port):
self.app = app
self.addr = addr
@@ -501,7 +527,13 @@ class InitiatorBuildCommand(BuildCommand):
self.app.status(msg='Starting distributed build')
loop = distbuild.MainLoop()
- cm = distbuild.ConnectionMachine(
- self.addr, self.port, distbuild.Initiator, [self.app] + args)
+ cm = distbuild.InitiatorConnectionMachine(self.app,
+ self.addr,
+ self.port,
+ distbuild.Initiator,
+ [self.app] + args,
+ self.RECONNECT_INTERVAL,
+ self.MAX_RETRIES)
+
loop.add_state_machine(cm)
loop.run()
diff --git a/morphlib/exts/initramfs.write b/morphlib/exts/initramfs.write
index 815772f2..f8af6d84 100755
--- a/morphlib/exts/initramfs.write
+++ b/morphlib/exts/initramfs.write
@@ -23,5 +23,5 @@ INITRAMFS_PATH="$2"
(cd "$ROOTDIR" &&
find . -print0 |
- cpio -0 -H newc -o |
- gzip -c) >"$INITRAMFS_PATH"
+ cpio -0 -H newc -o) |
+ gzip -c | install -D -m644 /dev/stdin "$INITRAMFS_PATH"
diff --git a/morphlib/exts/kvm.check b/morphlib/exts/kvm.check
index 957d0893..1bb4007a 100755
--- a/morphlib/exts/kvm.check
+++ b/morphlib/exts/kvm.check
@@ -31,6 +31,8 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension):
if len(args) != 1:
raise cliapp.AppException('Wrong number of command line args')
+ self.require_btrfs_in_deployment_host_kernel()
+
upgrade = self.get_environment_boolean('UPGRADE')
if upgrade:
raise cliapp.AppException(
diff --git a/morphlib/exts/openstack.check b/morphlib/exts/openstack.check
index a9a8fe1b..b5173011 100755
--- a/morphlib/exts/openstack.check
+++ b/morphlib/exts/openstack.check
@@ -26,6 +26,8 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension):
if len(args) != 1:
raise cliapp.AppException('Wrong number of command line args')
+ self.require_btrfs_in_deployment_host_kernel()
+
upgrade = self.get_environment_boolean('UPGRADE')
if upgrade:
raise cliapp.AppException(
diff --git a/morphlib/exts/rawdisk.check b/morphlib/exts/rawdisk.check
new file mode 100755
index 00000000..6a656ee7
--- /dev/null
+++ b/morphlib/exts/rawdisk.check
@@ -0,0 +1,31 @@
+#!/usr/bin/python
+# Copyright (C) 2014 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+'''Preparatory checks for Morph 'rawdisk' write extension'''
+
+import cliapp
+
+import morphlib.writeexts
+
+
+class RawdiskCheckExtension(morphlib.writeexts.WriteExtension):
+ def process_args(self, args):
+ if len(args) != 1:
+ raise cliapp.AppException('Wrong number of command line args')
+
+ self.require_btrfs_in_deployment_host_kernel()
+
+RawdiskCheckExtension().run()
diff --git a/morphlib/exts/virtualbox-ssh.check b/morphlib/exts/virtualbox-ssh.check
index 1aeb8999..57d54db1 100755
--- a/morphlib/exts/virtualbox-ssh.check
+++ b/morphlib/exts/virtualbox-ssh.check
@@ -26,6 +26,8 @@ class VirtualBoxPlusSshCheckExtension(morphlib.writeexts.WriteExtension):
if len(args) != 1:
raise cliapp.AppException('Wrong number of command line args')
+ self.require_btrfs_in_deployment_host_kernel()
+
upgrade = self.get_environment_boolean('UPGRADE')
if upgrade:
raise cliapp.AppException(
diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py
index ec0cfbcb..bfd0d047 100644
--- a/morphlib/plugins/cross-bootstrap_plugin.py
+++ b/morphlib/plugins/cross-bootstrap_plugin.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2013 Codethink Limited
+# Copyright (C) 2013-2014 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -299,18 +299,8 @@ class CrossBootstrapPlugin(cliapp.Plugin):
'Nothing to cross-compile. Only chunks built in \'bootstrap\' '
'mode can be cross-compiled.')
- # FIXME: merge with build-command's code
for i, a in enumerate(cross_chunks):
- if build_command.is_built(a):
- self.app.status(msg='The %(kind)s %(name)s is already built',
- kind=a.source.morphology['kind'],
- name=a.name)
- build_command.cache_artifacts_locally([a])
- else:
- self.app.status(msg='Cross-building %(kind)s %(name)s',
- kind=a.source.morphology['kind'],
- name=a.name)
- build_command.build_artifact(a, build_env)
+ build_command.cache_or_build_artifact(a, build_env)
for i, a in enumerate(native_chunks):
build_command.get_sources(a)
diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py
index 1d582949..3afb7b17 100644
--- a/morphlib/plugins/deploy_plugin.py
+++ b/morphlib/plugins/deploy_plugin.py
@@ -274,6 +274,7 @@ class DeployPlugin(cliapp.Plugin):
self.app.settings['tempdir-min-space'],
'/', 0)
+ self.app.settings['no-git-update'] = True
cluster_name = morphlib.util.strip_morph_extension(args[0])
env_vars = args[1:]
diff --git a/morphlib/remoteartifactcache.py b/morphlib/remoteartifactcache.py
index 9f6bf69e..0f8edce8 100644
--- a/morphlib/remoteartifactcache.py
+++ b/morphlib/remoteartifactcache.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2013 Codethink Limited
+# Copyright (C) 2012-2014 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -33,18 +33,19 @@ class GetError(cliapp.AppException):
cliapp.AppException.__init__(
self, 'Failed to get the artifact %s with cache key %s '
'from the artifact cache %s' %
- (artifact, artifact.cache_key, cache))
+ (artifact.basename(), artifact.cache_key, cache))
-class GetArtifactMetadataError(cliapp.AppException):
+class GetArtifactMetadataError(GetError):
def __init__(self, cache, artifact, name):
cliapp.AppException.__init__(
self, 'Failed to get metadata %s for the artifact %s '
- 'from the artifact cache %s' % (name, artifact, cache))
+ 'from the artifact cache %s' %
+ (name, artifact.basename(), cache))
-class GetSourceMetadataError(cliapp.AppException):
+class GetSourceMetadataError(GetError):
def __init__(self, cache, source, cache_key, name):
cliapp.AppException.__init__(
diff --git a/morphlib/writeexts.py b/morphlib/writeexts.py
index d6f23e0d..74587bd1 100644
--- a/morphlib/writeexts.py
+++ b/morphlib/writeexts.py
@@ -104,7 +104,18 @@ class WriteExtension(cliapp.Application):
self.output.write('%s\n' % (kwargs['msg'] % kwargs))
self.output.flush()
-
+
+ def check_for_btrfs_in_deployment_host_kernel(self):
+ with open('/proc/filesystems') as f:
+ text = f.read()
+ return '\tbtrfs\n' in text
+
+ def require_btrfs_in_deployment_host_kernel(self):
+ if not self.check_for_btrfs_in_deployment_host_kernel():
+ raise cliapp.AppException(
+ 'Error: Btrfs is required for this deployment, but was not '
+ 'detected in the kernel of the machine that is running Morph.')
+
def create_local_system(self, temp_root, raw_disk):
'''Create a raw system image locally.'''
size = self.get_disk_size()
View Lorry Controller Status