summaryrefslogtreecommitdiff
path: root/extensions
diff options
context:
space:
mode:
authorAdam Coldrick <adam.coldrick@codethink.co.uk>2015-06-19 15:33:22 +0000
committerAdam Coldrick <adam.coldrick@codethink.co.uk>2015-06-19 15:33:22 +0000
commit53a5919884b98613a99fe7a8397e9f2f01190b8f (patch)
tree1422b88cbba6ca6da5e45bf50690d152ceb529ab /extensions
parentcd9e3ba567a704af2585ad5b46d5b1eb9609fdd4 (diff)
parente4c6b8a69f0df2d0b3beac46865a66e0de527151 (diff)
downloaddefinitions-53a5919884b98613a99fe7a8397e9f2f01190b8f.tar.gz
Merge branch 'baserock/adamcoldrick/remove-dependencies-v3'
This stops the deployment extensions from depending on morphlib and cliapp, as well as setting the definitions version to 5. Reviewed-by: Sam Thursfield <sam.thursfield@codethink.co.uk> Reviewed-by: Paul Sherwood <paul.sherwood@codethink.co.uk>
Diffstat (limited to 'extensions')
-rw-r--r--extensions/ceph.configure11
-rwxr-xr-xextensions/distbuild-trove-nfsboot.check52
-rwxr-xr-xextensions/distbuild-trove-nfsboot.write40
-rwxr-xr-xextensions/fstab.configure4
-rwxr-xr-xextensions/hosts.configure12
-rw-r--r--extensions/image-package-example/README4
-rwxr-xr-xextensions/install-essential-files.configure21
-rwxr-xr-xextensions/install-files.configure30
-rw-r--r--extensions/jffs2.write18
-rwxr-xr-xextensions/kvm.check69
-rwxr-xr-xextensions/kvm.write25
-rwxr-xr-xextensions/nfsboot.check50
-rwxr-xr-xextensions/nfsboot.write83
-rwxr-xr-xextensions/openstack.check26
-rwxr-xr-xextensions/openstack.write11
-rw-r--r--extensions/pxeboot.write57
-rwxr-xr-xextensions/rawdisk.check17
-rwxr-xr-xextensions/rawdisk.write18
-rwxr-xr-xextensions/recv-hole158
-rwxr-xr-xextensions/simple-network.configure22
-rwxr-xr-xextensions/ssh-rsync.check26
-rwxr-xr-xextensions/ssh-rsync.write51
-rwxr-xr-xextensions/strip-gplv3.configure28
-rwxr-xr-xextensions/virtualbox-ssh.check10
-rwxr-xr-xextensions/virtualbox-ssh.write42
-rw-r--r--extensions/writeexts.py746
-rwxr-xr-xextensions/xfer-hole137
27 files changed, 1419 insertions, 349 deletions
diff --git a/extensions/ceph.configure b/extensions/ceph.configure
index c3cd92d1..3b8b2603 100644
--- a/extensions/ceph.configure
+++ b/extensions/ceph.configure
@@ -14,13 +14,14 @@
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-import cliapp
-import sys
import os
-import subprocess
import shutil
-import re
import stat
+import subprocess
+import sys
+import re
+
+import writeexts
systemd_monitor_template = """
[Unit]
@@ -75,7 +76,7 @@ executable_file_permissions = stat.S_IRUSR | stat.S_IXUSR | stat.S_IWUSR | \
stat.S_IXGRP | stat.S_IRGRP | \
stat.S_IXOTH | stat.S_IROTH
-class CephConfigurationExtension(cliapp.Application):
+class CephConfigurationExtension(writeexts.Extension):
"""
Set up ceph server daemons.
diff --git a/extensions/distbuild-trove-nfsboot.check b/extensions/distbuild-trove-nfsboot.check
index 38c491e5..76ba6dda 100755
--- a/extensions/distbuild-trove-nfsboot.check
+++ b/extensions/distbuild-trove-nfsboot.check
@@ -15,14 +15,15 @@
'''Preparatory checks for Morph 'distbuild-trove-nfsboot' write extension'''
-import cliapp
import logging
import os
+import subprocess
+import sys
-import morphlib.writeexts
+import writeexts
-class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension):
+class DistbuildTroveNFSBootCheckExtension(writeexts.WriteExtension):
nfsboot_root = '/srv/nfsboot'
remote_user = 'root'
@@ -45,7 +46,8 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension):
def process_args(self, args):
if len(args) != 1:
- raise cliapp.AppException('Wrong number of command line args')
+ raise writeexts.ExtensionError(
+ 'Wrong number of command line args')
nfs_host = args[0]
nfs_netloc = '%s@%s' % (self.remote_user, nfs_host)
@@ -55,17 +57,19 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension):
missing_vars = [var for var in self.required_vars
if not var in os.environ]
if missing_vars:
- raise cliapp.AppException(
+ raise writeexts.ExtensionError(
'Please set: %s' % ', '.join(missing_vars))
controllers = os.getenv('DISTBUILD_CONTROLLER').split()
workers = os.getenv('DISTBUILD_WORKERS').split()
if len(controllers) != 1:
- raise cliapp.AppException('Please specify exactly one controller.')
+ raise writeexts.ExtensionError(
+ 'Please specify exactly one controller.')
if len(workers) == 0:
- raise cliapp.AppException('Please specify at least one worker.')
+ raise writeexts.ExtensionError(
+ 'Please specify at least one worker.')
upgrade = self.get_environment_boolean('UPGRADE')
@@ -80,7 +84,7 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension):
if self.remote_directory_exists(nfs_netloc, system_path):
if self.get_environment_boolean('OVERWRITE') == False:
- raise cliapp.AppException(
+ raise writeexts.ExtensionError(
'System %s already exists at %s:%s. Try `morph '
'upgrade` instead of `morph deploy`.' % (
system_name, nfs_netloc, system_path))
@@ -91,27 +95,27 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension):
# Is an NFS server
try:
- cliapp.ssh_runcmd(
+ writeexts.ssh_runcmd(
netloc, ['test', '-e', '/etc/exports'])
- except cliapp.AppException:
- raise cliapp.AppException('server %s is not an nfs server'
- % netloc)
+ except writeexts.ExtensionError:
+ raise writeexts.ExtensionError('server %s is not an nfs server'
+ % netloc)
try:
- cliapp.ssh_runcmd(
+ writeexts.ssh_runcmd(
netloc, ['systemctl', 'is-enabled', 'nfs-server.service'])
- except cliapp.AppException:
- raise cliapp.AppException('server %s does not control its '
- 'nfs server by systemd' % netloc)
+ except writeexts.ExtensionError:
+ raise writeexts.ExtensionError('server %s does not control its '
+ 'nfs server by systemd' % netloc)
# TFTP server exports /srv/nfsboot/tftp
tftp_root = os.path.join(self.nfsboot_root, 'tftp')
try:
- cliapp.ssh_runcmd(
+ writeexts.ssh_runcmd(
netloc, ['test' , '-d', tftp_root])
- except cliapp.AppException:
- raise cliapp.AppException('server %s does not export %s' %
- (netloc, tftp_root))
+ except writeexts.ExtensionError:
+ raise writeexts.ExtensionError('server %s does not export %s' %
+ (netloc, tftp_root))
def check_upgradeable(self, nfs_netloc, system_name, version_label):
'''Check that there is already a version of the system present.
@@ -124,7 +128,7 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension):
system_version_path = self.system_path(system_name, version_label)
if not self.remote_directory_exists(nfs_netloc, system_path):
- raise cliapp.AppException(
+ raise writeexts.ExtensionError(
'System %s not found at %s:%s, cannot deploy an upgrade.' % (
system_name, nfs_netloc, system_path))
@@ -132,15 +136,15 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension):
if self.get_environment_boolean('OVERWRITE'):
pass
else:
- raise cliapp.AppException(
+ raise writeexts.ExtensionError(
'System %s version %s already exists at %s:%s.' % (
system_name, version_label, nfs_netloc,
system_version_path))
def remote_directory_exists(self, nfs_netloc, path):
try:
- cliapp.ssh_runcmd(nfs_netloc, ['test', '-d', path])
- except cliapp.AppException as e:
+ writeexts.ssh_runcmd(nfs_netloc, ['test', '-d', path])
+ except writeexts.ExtensionError as e:
logging.debug('SSH exception: %s', e)
return False
diff --git a/extensions/distbuild-trove-nfsboot.write b/extensions/distbuild-trove-nfsboot.write
index a5a5b094..86291794 100755
--- a/extensions/distbuild-trove-nfsboot.write
+++ b/extensions/distbuild-trove-nfsboot.write
@@ -20,14 +20,14 @@
import os
+import subprocess
import sys
import tempfile
-import cliapp
-import morphlib.writeexts
+import writeexts
-class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension):
+class DistbuildTroveNFSBootWriteExtension(writeexts.WriteExtension):
'''Create an NFS root and kernel on TFTP during Morph's deployment.
@@ -54,7 +54,7 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension):
def process_args(self, args):
if len(args) != 2:
- raise cliapp.AppException('Wrong number of command line args')
+ raise writeexts.ExtensionError('Wrong number of command line args')
local_system_path, nfs_host = args
@@ -111,17 +111,17 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension):
'''
pairs = host_map_string.split(' ')
- return morphlib.util.parse_environment_pairs({}, pairs)
+ return writeexts.parse_environment_pairs({}, pairs)
def transfer_system(self, nfs_netloc, local_system_path,
remote_system_path):
self.status(msg='Copying rootfs to %(nfs_netloc)s',
nfs_netloc=nfs_netloc)
- cliapp.ssh_runcmd(
+ writeexts.ssh_runcmd(
nfs_netloc, ['mkdir', '-p', remote_system_path])
# The deployed rootfs may have been created by OSTree, so definitely
# don't pass --hard-links to `rsync`.
- cliapp.runcmd(
+ subprocess.check_call(
['rsync', '--archive', '--delete', '--info=progress2',
'--protect-args', '--partial', '--sparse', '--xattrs',
local_system_path + '/',
@@ -131,13 +131,13 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension):
target_system_path):
self.status(msg='Duplicating rootfs to %(target_system_path)s',
target_system_path=target_system_path)
- cliapp.ssh_runcmd(nfs_netloc,
+ writeexts.ssh_runcmd(nfs_netloc,
['mkdir', '-p', target_system_path])
# We can't pass --info=progress2 here, because it may not be available
# in the remote 'rsync'. The --info setting was added in RSync 3.1.0,
# old versions of Baserock have RSync 3.0.9. So the user doesn't get
# any progress info on stdout for the 'duplicate' stage.
- cliapp.ssh_runcmd(nfs_netloc,
+ writeexts.ssh_runcmd(nfs_netloc,
['rsync', '--archive', '--delete', '--protect-args', '--partial',
'--sparse', '--xattrs', source_system_path + '/',
target_system_path], stdout=sys.stdout)
@@ -152,7 +152,7 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension):
kernel_path = os.path.relpath(try_path, local_system_path)
break
else:
- raise cliapp.AppException(
+ raise writeexts.ExtensionError(
'Could not find a kernel in the system: none of '
'%s found' % ', '.join(image_names))
return kernel_path
@@ -171,11 +171,11 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension):
versioned_kernel_name = "%s-%s" % (system_name, version_label)
kernel_name = system_name
- cliapp.ssh_runcmd(nfs_netloc,
+ writeexts.ssh_runcmd(nfs_netloc,
['ln', '-f', kernel_dest,
os.path.join(tftp_dir, versioned_kernel_name)])
- cliapp.ssh_runcmd(nfs_netloc,
+ writeexts.ssh_runcmd(nfs_netloc,
['ln', '-sf', versioned_kernel_name,
os.path.join(tftp_dir, kernel_name)])
@@ -183,7 +183,7 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension):
with tempfile.NamedTemporaryFile() as f:
f.write(text)
f.flush()
- cliapp.runcmd(
+ subprocess.check_call(
['scp', f.name, '%s:%s' % (nfs_netloc, path)])
def set_hostname(self, nfs_netloc, system_name, system_path):
@@ -223,9 +223,9 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension):
'# Generated by distbuild-trove-nfsboot.write\n' + \
config_text + '\n'
path = os.path.join(system_path, 'etc', 'distbuild')
- cliapp.ssh_runcmd(
+ writeexts.ssh_runcmd(
nfs_netloc, ['mkdir', '-p', path])
- cliapp.runcmd(
+ subprocess.check_call(
['scp', worker_ssh_key_path, '%s:%s' % (nfs_netloc, path)])
self.set_remote_file_contents(
nfs_netloc, os.path.join(path, 'distbuild.conf'), config_text)
@@ -244,9 +244,9 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension):
# Rather ugly SSH hackery follows to ensure each system path is
# listed in /etc/exports.
try:
- cliapp.ssh_runcmd(
+ writeexts.ssh_runcmd(
nfs_netloc, ['grep', '-q', exported_path, exports_path])
- except cliapp.AppException:
+ except writeexts.ExtensionError:
ip_mask = '*'
options = 'rw,no_subtree_check,no_root_squash,async'
exports_string = '%s %s(%s)\n' % (exported_path, ip_mask,
@@ -259,12 +259,12 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension):
cat >> "$temp"
mv "$temp" "$target"
'''
- cliapp.ssh_runcmd(
+ writeexts.ssh_runcmd(
nfs_netloc,
['sh', '-c', exports_append_sh, '--', exports_path],
feed_stdin=exports_string)
- cliapp.ssh_runcmd(nfs_netloc,
+ writeexts.ssh_runcmd(nfs_netloc,
['systemctl', 'restart', 'nfs-server.service'])
def update_default_version(self, remote_netloc, system_name,
@@ -276,7 +276,7 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension):
version_label)
default_path = os.path.join(system_path, 'systems', 'default')
- cliapp.ssh_runcmd(remote_netloc,
+ writeexts.ssh_runcmd(remote_netloc,
['ln', '-sfn', system_version_path, default_path])
diff --git a/extensions/fstab.configure b/extensions/fstab.configure
index b9154eee..3e67b585 100755
--- a/extensions/fstab.configure
+++ b/extensions/fstab.configure
@@ -20,9 +20,9 @@
import os
import sys
-import morphlib
+import writeexts
envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('FSTAB_')}
conf_file = os.path.join(sys.argv[1], 'etc/fstab')
-morphlib.util.write_from_dict(conf_file, envvars)
+writeexts.write_from_dict(conf_file, envvars)
diff --git a/extensions/hosts.configure b/extensions/hosts.configure
index 6b068d04..11fcf573 100755
--- a/extensions/hosts.configure
+++ b/extensions/hosts.configure
@@ -22,27 +22,29 @@ import os
import sys
import socket
-import morphlib
+import writeexts
def validate(var, line):
xs = line.split()
if len(xs) == 0:
- raise morphlib.Error("`%s: %s': line is empty" % (var, line))
+ raise writeexts.ExtensionError(
+ "`%s: %s': line is empty" % (var, line))
ip = xs[0]
hostnames = xs[1:]
if len(hostnames) == 0:
- raise morphlib.Error("`%s: %s': missing hostname" % (var, line))
+ raise writeexts.ExtensionError(
+ "`%s: %s': missing hostname" % (var, line))
family = socket.AF_INET6 if ':' in ip else socket.AF_INET
try:
socket.inet_pton(family, ip)
except socket.error:
- raise morphlib.Error("`%s: %s' invalid ip" % (var, ip))
+ raise writeexts.ExtensionError("`%s: %s' invalid ip" % (var, ip))
envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('HOSTS_')}
conf_file = os.path.join(sys.argv[1], 'etc/hosts')
-morphlib.util.write_from_dict(conf_file, envvars, validate)
+writeexts.write_from_dict(conf_file, envvars, validate)
diff --git a/extensions/image-package-example/README b/extensions/image-package-example/README
index c1322f25..f6b66cd9 100644
--- a/extensions/image-package-example/README
+++ b/extensions/image-package-example/README
@@ -5,5 +5,5 @@ These are scripts used to create disk images or install the system onto
an existing disk.
This is also implemented independently for the rawdisk.write write
-extension; see morphlib.writeexts.WriteExtension.create_local_system()
-for a similar, python implementation.
+extension; see writeexts.WriteExtension.create_local_system() for
+a similar, python implementation.
diff --git a/extensions/install-essential-files.configure b/extensions/install-essential-files.configure
index bed394df..3d33fe03 100755
--- a/extensions/install-essential-files.configure
+++ b/extensions/install-essential-files.configure
@@ -22,20 +22,11 @@ to install into the target system.
'''
-import subprocess
import os
+import subprocess
+import sys
-import cliapp
-
-class InstallEssentialFilesConfigureExtension(cliapp.Application):
-
- def process_args(self, args):
- target_root = args[0]
- os.environ["INSTALL_FILES"] = "install-files/essential-files/manifest"
- self.install_essential_files(target_root)
-
- def install_essential_files(self, target_root):
- command = os.path.join("extensions/install-files.configure")
- subprocess.check_call([command, target_root])
-
-InstallEssentialFilesConfigureExtension().run()
+target_root = sys.argv[1]
+os.environ["INSTALL_FILES"] = "install-files/essential-files/manifest"
+command = os.path.join("extensions/install-files.configure")
+subprocess.check_call([command, target_root])
diff --git a/extensions/install-files.configure b/extensions/install-files.configure
index 341cce61..64fcecca 100755
--- a/extensions/install-files.configure
+++ b/extensions/install-files.configure
@@ -22,9 +22,8 @@ to install into the target system.
'''
-import cliapp
-import os
import errno
+import os
import re
import sys
import shlex
@@ -37,7 +36,9 @@ try:
except ImportError:
jinja_available = False
-class InstallFilesConfigureExtension(cliapp.Application):
+import writeexts
+
+class InstallFilesConfigureExtension(writeexts.Extension):
def process_args(self, args):
if not 'INSTALL_FILES' in os.environ:
@@ -74,7 +75,8 @@ class InstallFilesConfigureExtension(cliapp.Application):
gid = int(m.group(5))
path = m.group(6)
else:
- raise cliapp.AppException('Invalid manifest entry, '
+ raise writeexts.ExtensionError(
+ 'Invalid manifest entry, '
'format: [template] [overwrite] '
'<octal mode> <uid decimal> <gid decimal> <filename>')
@@ -85,9 +87,9 @@ class InstallFilesConfigureExtension(cliapp.Application):
if (mode != dest_stat.st_mode
or uid != dest_stat.st_uid
or gid != dest_stat.st_gid):
- raise cliapp.AppException('"%s" exists and is not '
- 'identical to directory '
- '"%s"' % (dest_path, entry))
+ raise writeexts.ExtensionError(
+ '"%s" exists and is not identical to directory '
+ '"%s"' % (dest_path, entry))
else:
os.mkdir(dest_path, mode)
os.chown(dest_path, uid, gid)
@@ -95,8 +97,8 @@ class InstallFilesConfigureExtension(cliapp.Application):
elif stat.S_ISLNK(mode):
if os.path.lexists(dest_path) and not overwrite:
- raise cliapp.AppException('Symlink already exists at %s'
- % dest_path)
+ raise writeexts.ExtensionError('Symlink already exists at %s'
+ % dest_path)
else:
linkdest = os.readlink(os.path.join(manifest_root,
'./' + path))
@@ -105,12 +107,12 @@ class InstallFilesConfigureExtension(cliapp.Application):
elif stat.S_ISREG(mode):
if os.path.lexists(dest_path) and not overwrite:
- raise cliapp.AppException('File already exists at %s'
- % dest_path)
+ raise writeexts.ExtensionError('File already exists at %s'
+ % dest_path)
else:
if template:
if not jinja_available:
- raise cliapp.AppException(
+ raise writeexts.ExtensionError(
"Failed to install template file `%s': "
'install-files templates require jinja2'
% path)
@@ -128,7 +130,7 @@ class InstallFilesConfigureExtension(cliapp.Application):
os.chmod(dest_path, mode)
else:
- raise cliapp.AppException('Mode given in "%s" is not a file,'
- ' symlink or directory' % entry)
+ raise writeexts.ExtensionError('Mode given in "%s" is not a file,'
+ ' symlink or directory' % entry)
InstallFilesConfigureExtension().run()
diff --git a/extensions/jffs2.write b/extensions/jffs2.write
index 46b69a53..ad68204d 100644
--- a/extensions/jffs2.write
+++ b/extensions/jffs2.write
@@ -19,34 +19,34 @@
as the root filesystem.'''
-import cliapp
import os
+import subprocess
-import morphlib.writeexts
+import writeexts
-class Jffs2WriteExtension(morphlib.writeexts.WriteExtension):
+class Jffs2WriteExtension(writeexts.WriteExtension):
'''See jffs2.write.help for documentation.'''
def process_args(self, args):
if len(args) != 2:
- raise cliapp.AppException('Wrong number of command line args')
+ raise writeexts.ExtensionError('Wrong number of command line args')
temp_root, location = args
try:
self.create_jffs2_system(temp_root, location)
self.status(msg='Disk image has been created at %(location)s',
- location = location)
+ location=location)
except Exception:
self.status(msg='Failure to deploy system to %(location)s',
- location = location)
+ location=location)
raise
def create_jffs2_system(self, temp_root, location):
erase_block = self.get_erase_block_size()
- cliapp.runcmd(
+ subprocess.check_call(
['mkfs.jffs2', '--pad', '--no-cleanmarkers',
'--eraseblock='+erase_block, '-d', temp_root, '-o', location])
@@ -54,10 +54,10 @@ class Jffs2WriteExtension(morphlib.writeexts.WriteExtension):
erase_block = os.environ.get('ERASE_BLOCK', '')
if erase_block == '':
- raise cliapp.AppException('ERASE_BLOCK was not given')
+ raise writeexts.ExtensionError('ERASE_BLOCK was not given')
if not erase_block.isdigit():
- raise cliapp.AppException('ERASE_BLOCK must be a whole number')
+ raise writeexts.ExtensionError('ERASE_BLOCK must be a whole number')
return erase_block
diff --git a/extensions/kvm.check b/extensions/kvm.check
index 67cb3d38..3c277156 100755
--- a/extensions/kvm.check
+++ b/extensions/kvm.check
@@ -15,27 +15,28 @@
'''Preparatory checks for Morph 'kvm' write extension'''
-import cliapp
import os
import re
+import subprocess
import urlparse
-import morphlib.writeexts
+import writeexts
-class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension):
+class KvmPlusSshCheckExtension(writeexts.WriteExtension):
location_pattern = '^/(?P<guest>[^/]+)(?P<path>/.+)$'
def process_args(self, args):
if len(args) != 1:
- raise cliapp.AppException('Wrong number of command line args')
+ raise writeexts.ExtensionError(
+ 'Wrong number of command line args')
self.require_btrfs_in_deployment_host_kernel()
upgrade = self.get_environment_boolean('UPGRADE')
if upgrade:
- raise cliapp.AppException(
+ raise writeexts.ExtensionError(
'Use the `ssh-rsync` write extension to deploy upgrades to an '
'existing remote system.')
@@ -55,23 +56,24 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension):
x = urlparse.urlparse(location)
if x.scheme != 'kvm+ssh':
- raise cliapp.AppException(
+ raise writeexts.ExtensionError(
'URL schema must be kvm+ssh in %s' % location)
m = re.match(self.location_pattern, x.path)
if not m:
- raise cliapp.AppException('Cannot parse location %s' % location)
+ raise writeexts.ExtensionError(
+ 'Cannot parse location %s' % location)
return x.netloc, m.group('guest'), m.group('path')
def check_no_existing_libvirt_vm(self, ssh_host, vm_name):
try:
- cliapp.ssh_runcmd(ssh_host,
+ writeexts.ssh_runcmd(ssh_host,
['virsh', '--connect', 'qemu:///system', 'domstate', vm_name])
- except cliapp.AppException as e:
+ except CalledProcessError as e:
pass
else:
- raise cliapp.AppException(
+ raise writeexts.ExtensionError(
'Host %s already has a VM named %s. You can use the ssh-rsync '
'write extension to deploy upgrades to existing machines.' %
(ssh_host, vm_name))
@@ -80,35 +82,35 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension):
def check_can_write_to_given_path():
try:
- cliapp.ssh_runcmd(ssh_host, ['touch', vm_path])
- except cliapp.AppException as e:
- raise cliapp.AppException("Can't write to location %s on %s"
- % (vm_path, ssh_host))
+ writeexts.ssh_runcmd(ssh_host, ['touch', vm_path])
+ except writeexts.ExtensionError as e:
+ raise writeexts.ExtensionError(
+ "Can't write to location %s on %s" % (vm_path, ssh_host))
else:
- cliapp.ssh_runcmd(ssh_host, ['rm', vm_path])
+ writeexts.ssh_runcmd(ssh_host, ['rm', vm_path])
try:
- cliapp.ssh_runcmd(ssh_host, ['test', '-e', vm_path])
- except cliapp.AppException as e:
+ writeexts.ssh_runcmd(ssh_host, ['test', '-e', vm_path])
+ except writeexts.ExtensionError as e:
# vm_path doesn't already exist, so let's test we can write
check_can_write_to_given_path()
else:
- raise cliapp.AppException('%s already exists on %s'
- % (vm_path, ssh_host))
+ raise writeexts.ExtensionError('%s already exists on %s'
+ % (vm_path, ssh_host))
def check_extra_disks_exist(self, ssh_host, filename_list):
for filename in filename_list:
try:
- cliapp.ssh_runcmd(ssh_host, ['ls', filename])
- except cliapp.AppException as e:
- raise cliapp.AppException('Did not find file %s on host %s' %
- (filename, ssh_host))
+ writeexts.ssh_runcmd(ssh_host, ['ls', filename])
+ except writeexts.ExtensionError as e:
+ raise writeexts.ExtensionError(
+ 'Did not find file %s on host %s' % (filename, ssh_host))
def check_virtual_networks_are_started(self, ssh_host):
def check_virtual_network_is_started(network_name):
cmd = ['virsh', '-c', 'qemu:///system', 'net-info', network_name]
- net_info = cliapp.ssh_runcmd(ssh_host, cmd).split('\n')
+ net_info = writeexts.ssh_runcmd(ssh_host, cmd).split('\n')
def pretty_concat(lines):
return '\n'.join(['\t%s' % line for line in lines])
@@ -118,15 +120,15 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension):
if m:
break
else:
- raise cliapp.AppException(
+ raise writeexts.ExtensionError(
"Got unexpected output parsing output of `%s':\n%s"
% (' '.join(cmd), pretty_concat(net_info)))
network_active = m.group(1) == 'yes'
if not network_active:
- raise cliapp.AppException("Network '%s' is not started"
- % network_name)
+ raise writeexts.ExtensionError("Network '%s' is not started"
+ % network_name)
def name(nic_entry):
if ',' in nic_entry:
@@ -142,9 +144,10 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension):
if not (n.startswith('network=')
or n.startswith('bridge=')
or n == 'user'):
- raise cliapp.AppException('malformed NIC_CONFIG: %s\n'
- " (expected 'bridge=BRIDGE' 'network=NAME'"
- " or 'user')" % n)
+ raise writeexts.ExtensionError(
+ "malformed NIC_CONFIG: %s\n"
+ " (expected 'bridge=BRIDGE' 'network=NAME'"
+ " or 'user')" % n)
# --network bridge= is used to specify a bridge
# --network user is used to specify a form of NAT
@@ -159,9 +162,9 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension):
def check_host_has_virtinstall(self, ssh_host):
try:
- cliapp.ssh_runcmd(ssh_host, ['which', 'virt-install'])
- except cliapp.AppException:
- raise cliapp.AppException(
+ writeexts.ssh_runcmd(ssh_host, ['which', 'virt-install'])
+ except writeexts.ExtensionError:
+ raise writeexts.ExtensionError(
'virt-install does not seem to be installed on host %s'
% ssh_host)
diff --git a/extensions/kvm.write b/extensions/kvm.write
index 0d0c095b..2290725e 100755
--- a/extensions/kvm.write
+++ b/extensions/kvm.write
@@ -21,23 +21,23 @@ See file kvm.write.help for documentation
'''
-import cliapp
import os
import re
import sys
import tempfile
import urlparse
-import morphlib.writeexts
+import writeexts
-class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension):
+class KvmPlusSshWriteExtension(writeexts.WriteExtension):
location_pattern = '^/(?P<guest>[^/]+)(?P<path>/.+)$'
def process_args(self, args):
if len(args) != 2:
- raise cliapp.AppException('Wrong number of command line args')
+ raise writeexts.ExtensionError(
+ 'Wrong number of command line args')
temp_root, location = args
ssh_host, vm_name, vm_path = self.parse_location(location)
@@ -53,7 +53,7 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension):
except BaseException:
sys.stderr.write('Error deploying to libvirt')
os.remove(raw_disk)
- cliapp.ssh_runcmd(ssh_host, ['rm', '-f', vm_path])
+ writeexts.ssh_runcmd(ssh_host, ['rm', '-f', vm_path])
raise
else:
os.remove(raw_disk)
@@ -74,16 +74,16 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension):
self.status(msg='Transferring disk image')
- xfer_hole_path = morphlib.util.get_data_path('xfer-hole')
- recv_hole = morphlib.util.get_data('recv-hole')
+ xfer_hole_path = writeexts.get_data_path('xfer-hole')
+ recv_hole = writeexts.get_data('recv-hole')
ssh_remote_cmd = [
'sh', '-c', recv_hole, 'dummy-argv0', 'file', vm_path
]
- cliapp.runcmd(
+ subprocess.check_call(
['python', xfer_hole_path, raw_disk],
- ['ssh', ssh_host] + map(cliapp.shell_quote, ssh_remote_cmd),
+ ['ssh', ssh_host] + map(writeexts.shell_quote, ssh_remote_cmd),
stdout=None, stderr=None)
def create_libvirt_guest(self, ssh_host, vm_name, vm_path, autostart):
@@ -111,10 +111,11 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension):
'--disk', 'path=%s,bus=ide' % vm_path] + attach_opts
if not autostart:
cmdline += ['--noreboot']
- cliapp.ssh_runcmd(ssh_host, cmdline)
+ writeexts.ssh_runcmd(ssh_host, cmdline)
if autostart:
- cliapp.ssh_runcmd(ssh_host,
- ['virsh', '--connect', 'qemu:///system', 'autostart', vm_name])
+ writeexts.ssh_runcmd(ssh_host,
+ ['virsh', '--connect', 'qemu:///system',
+ 'autostart', vm_name])
KvmPlusSshWriteExtension().run()
diff --git a/extensions/nfsboot.check b/extensions/nfsboot.check
index e273f61c..499fb537 100755
--- a/extensions/nfsboot.check
+++ b/extensions/nfsboot.check
@@ -15,33 +15,35 @@
'''Preparatory checks for Morph 'nfsboot' write extension'''
-import cliapp
import os
+import subprocess
-import morphlib.writeexts
+import writeexts
-class NFSBootCheckExtension(morphlib.writeexts.WriteExtension):
+class NFSBootCheckExtension(writeexts.WriteExtension):
_nfsboot_root = '/srv/nfsboot'
def process_args(self, args):
if len(args) != 1:
- raise cliapp.AppException('Wrong number of command line args')
+ raise writeexts.ExtensionError(
+ 'Wrong number of command line args')
location = args[0]
upgrade = self.get_environment_boolean('UPGRADE')
if upgrade:
- raise cliapp.AppException(
+ raise writeexts.ExtensionError(
'Upgrading is not currently supported for NFS deployments.')
hostname = os.environ.get('HOSTNAME', None)
if hostname is None:
- raise cliapp.AppException('You must specify a HOSTNAME.')
+ raise writeexts.ExtensionError('You must specify a HOSTNAME.')
if hostname == 'baserock':
- raise cliapp.AppException('It is forbidden to nfsboot a system '
- 'with hostname "%s"' % hostname)
+ raise writeexts.ExtensionError('It is forbidden to nfsboot a '
+ 'system with hostname "%s"'
+ % hostname)
self.test_good_server(location)
@@ -49,7 +51,7 @@ class NFSBootCheckExtension(morphlib.writeexts.WriteExtension):
versioned_root = os.path.join(self._nfsboot_root, hostname, 'systems',
version_label)
if self.version_exists(versioned_root, location):
- raise cliapp.AppException(
+ raise writeexts.ExtensionError(
'Root file system for host %s (version %s) already exists on '
'the NFS server %s. Deployment aborted.' % (hostname,
version_label, location))
@@ -59,34 +61,34 @@ class NFSBootCheckExtension(morphlib.writeexts.WriteExtension):
# Is an NFS server
try:
- cliapp.ssh_runcmd(
+ writeexts.ssh_runcmd(
'root@%s' % server, ['test', '-e', '/etc/exports'])
- except cliapp.AppException:
- raise cliapp.AppException('server %s is not an nfs server'
- % server)
+ except writeexts.ExtensionError:
+ raise writeexts.ExtensionError('server %s is not an nfs server'
+ % server)
try:
- cliapp.ssh_runcmd(
+ writeexts.ssh_runcmd(
'root@%s' % server, ['systemctl', 'is-enabled',
'nfs-server.service'])
- except cliapp.AppException:
- raise cliapp.AppException('server %s does not control its '
- 'nfs server by systemd' % server)
+ except writeexts.ExtensionError:
+ raise writeexts.ExtensionError('server %s does not control its '
+ 'nfs server by systemd' % server)
# TFTP server exports /srv/nfsboot/tftp
tftp_root = os.path.join(self._nfsboot_root, 'tftp')
try:
- cliapp.ssh_runcmd(
+ writeexts.ssh_runcmd(
'root@%s' % server, ['test' , '-d', tftp_root])
- except cliapp.AppException:
- raise cliapp.AppException('server %s does not export %s' %
- (tftp_root, server))
+ except writeexts.ExtensionError:
+ raise writeexts.ExtensionError('server %s does not export %s' %
+ (tftp_root, server))
def version_exists(self, versioned_root, location):
try:
- cliapp.ssh_runcmd('root@%s' % location,
- ['test', '-d', versioned_root])
- except cliapp.AppException:
+ writeexts.ssh_runcmd('root@%s' % location,
+ ['test', '-d', versioned_root])
+ except writeexts.ExtensionError:
return False
return True
diff --git a/extensions/nfsboot.write b/extensions/nfsboot.write
index d928775e..418f8eeb 100755
--- a/extensions/nfsboot.write
+++ b/extensions/nfsboot.write
@@ -34,14 +34,13 @@ in /srv/nfsboot/nfs/
'''
-import cliapp
import os
import glob
-import morphlib.writeexts
+import writeexts
-class NFSBootWriteExtension(morphlib.writeexts.WriteExtension):
+class NFSBootWriteExtension(writeexts.WriteExtension):
'''Create an NFS root and kernel on TFTP during Morph's deployment.
@@ -66,7 +65,8 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension):
def process_args(self, args):
if len(args) != 2:
- raise cliapp.AppException('Wrong number of command line args')
+ raise writeexts.ExtensionError(
+ 'Wrong number of command line args')
temp_root, location = args
@@ -86,8 +86,8 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension):
subdirs = [os.path.join(statedir, 'home'),
os.path.join(statedir, 'opt'),
os.path.join(statedir, 'srv')]
- cliapp.ssh_runcmd('root@%s' % location,
- ['mkdir', '-p'] + subdirs)
+ writeexts.ssh_runcmd('root@%s' % location,
+ ['mkdir', '-p'] + subdirs)
def copy_kernel(self, temp_root, location, versioned_root, version,
hostname):
@@ -99,14 +99,14 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension):
kernel_src = try_path
break
else:
- raise cliapp.AppException(
+ raise writeexts.ExtensionError(
'Could not find a kernel in the system: none of '
'%s found' % ', '.join(image_names))
kernel_dest = os.path.join(versioned_root, 'orig', 'kernel')
rsync_dest = 'root@%s:%s' % (location, kernel_dest)
self.status(msg='Copying kernel')
- cliapp.runcmd(
+ subprocess.check_call(
['rsync', '-s', kernel_src, rsync_dest])
# Link the kernel to the right place
@@ -115,17 +115,17 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension):
versioned_kernel_name = "%s-%s" % (hostname, version)
kernel_name = hostname
try:
- cliapp.ssh_runcmd('root@%s' % location,
+ writeexts.ssh_runcmd('root@%s' % location,
['ln', '-f', kernel_dest,
os.path.join(tftp_dir, versioned_kernel_name)])
- cliapp.ssh_runcmd('root@%s' % location,
+ writeexts.ssh_runcmd('root@%s' % location,
['ln', '-sf', versioned_kernel_name,
os.path.join(tftp_dir, kernel_name)])
- except cliapp.AppException:
- raise cliapp.AppException('Could not create symlinks to the '
- 'kernel at %s in %s on %s'
- % (kernel_dest, tftp_dir, location))
+ except writeexts.ExtensionError:
+ raise writeexts.ExtensionError('Could not create symlinks to the '
+ 'kernel at %s in %s on %s' %
+ (kernel_dest, tftp_dir, location))
def copy_rootfs(self, temp_root, location, versioned_root, hostname):
rootfs_src = temp_root + '/'
@@ -134,51 +134,54 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension):
self.status(msg='Creating destination directories')
try:
- cliapp.ssh_runcmd('root@%s' % location,
- ['mkdir', '-p', orig_path, run_path])
- except cliapp.AppException:
- raise cliapp.AppException('Could not create dirs %s and %s on %s'
- % (orig_path, run_path, location))
+ writeexts.ssh_runcmd('root@%s' % location,
+ ['mkdir', '-p', orig_path, run_path])
+ except writeexts.ExtensionError:
+ raise writexts.ExtensionError(
+ 'Could not create dirs %s and %s on %s'
+ % (orig_path, run_path, location))
self.status(msg='Creating \'orig\' rootfs')
- cliapp.runcmd(
+ subprocess.check_call(
['rsync', '-asXSPH', '--delete', rootfs_src,
'root@%s:%s' % (location, orig_path)])
self.status(msg='Creating \'run\' rootfs')
try:
- cliapp.ssh_runcmd('root@%s' % location,
- ['rm', '-rf', run_path])
- cliapp.ssh_runcmd('root@%s' % location,
- ['cp', '-al', orig_path, run_path])
- cliapp.ssh_runcmd('root@%s' % location,
- ['rm', '-rf', os.path.join(run_path, 'etc')])
- cliapp.ssh_runcmd('root@%s' % location,
- ['cp', '-a', os.path.join(orig_path, 'etc'),
- os.path.join(run_path, 'etc')])
- except cliapp.AppException:
- raise cliapp.AppException('Could not create \'run\' rootfs'
- ' from \'orig\'')
+ writeexts.ssh_runcmd('root@%s' % location,
+ ['rm', '-rf', run_path])
+ writeexts.ssh_runcmd('root@%s' % location,
+ ['cp', '-al', orig_path, run_path])
+ writeexts.ssh_runcmd('root@%s' % location,
+ ['rm', '-rf',
+ os.path.join(run_path, 'etc')])
+ writeexts.ssh_runcmd('root@%s' % location,
+ ['cp', '-a',
+ os.path.join(orig_path, 'etc'),
+ os.path.join(run_path, 'etc')])
+ except writeexts.ExtensionError:
+ raise writeexts.ExtensionError('Could not create \'run\' rootfs'
+ ' from \'orig\'')
self.status(msg='Linking \'default\' to latest system')
try:
- cliapp.ssh_runcmd('root@%s' % location,
+ writeexts.ssh_runcmd('root@%s' % location,
['ln', '-sfn', versioned_root,
os.path.join(self._nfsboot_root, hostname, 'systems',
'default')])
- except cliapp.AppException:
- raise cliapp.AppException('Could not link \'default\' to %s'
- % versioned_root)
+ except writeexts.ExtensionError:
+ raise writeexts.ExtensionError("Could not link 'default' to %s"
+ % versioned_root)
def configure_nfs(self, location, hostname):
exported_path = os.path.join(self._nfsboot_root, hostname)
exports_path = '/etc/exports'
# If that path is not already exported:
try:
- cliapp.ssh_runcmd(
+ writeexts.ssh_runcmd(
'root@%s' % location, ['grep', '-q', exported_path,
exports_path])
- except cliapp.AppException:
+ except writeexts.ExtensionError:
ip_mask = '*'
options = 'rw,no_subtree_check,no_root_squash,async'
exports_string = '%s %s(%s)\n' % (exported_path, ip_mask, options)
@@ -190,11 +193,11 @@ cat "$target" > "$temp"
cat >> "$temp"
mv "$temp" "$target"
'''
- cliapp.ssh_runcmd(
+ writeexts.ssh_runcmd(
'root@%s' % location,
['sh', '-c', exports_append_sh, '--', exports_path],
feed_stdin=exports_string)
- cliapp.ssh_runcmd(
+ writeexts.ssh_runcmd(
'root@%s' % location, ['systemctl', 'restart',
'nfs-server.service'])
diff --git a/extensions/openstack.check b/extensions/openstack.check
index a3379763..f3ad43b7 100755
--- a/extensions/openstack.check
+++ b/extensions/openstack.check
@@ -15,25 +15,26 @@
'''Preparatory checks for Morph 'openstack' write extension'''
-import cliapp
import os
import urlparse
+
import keystoneclient
-import morphlib.writeexts
+import writeexts
-class OpenStackCheckExtension(morphlib.writeexts.WriteExtension):
+class OpenStackCheckExtension(writeexts.WriteExtension):
def process_args(self, args):
if len(args) != 1:
- raise cliapp.AppException('Wrong number of command line args')
+ raise writeexts.ExtensionError(
+ 'Wrong number of command line args')
self.require_btrfs_in_deployment_host_kernel()
upgrade = self.get_environment_boolean('UPGRADE')
if upgrade:
- raise cliapp.AppException(
+ raise writeexts.ExtensionError(
'Use the `ssh-rsync` write extension to deploy upgrades to an '
'existing remote system.')
@@ -55,7 +56,7 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension):
for key in auth_keys:
if os.environ.get(key, '') == '':
- raise cliapp.AppException(key + ' was not given')
+ raise writeexts.ExtensionError(key + ' was not given')
auth_params = {auth_keys[key]: os.environ[key] for key in auth_keys}
auth_params['auth_url'] = location
@@ -63,16 +64,17 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension):
def check_imagename(self):
if os.environ.get('OPENSTACK_IMAGENAME', '') == '':
- raise cliapp.AppException('OPENSTACK_IMAGENAME was not given')
+ raise writeexts.ExtensionError(
+ 'OPENSTACK_IMAGENAME was not given')
def check_location(self, location):
x = urlparse.urlparse(location)
if x.scheme not in ['http', 'https']:
- raise cliapp.AppException('URL schema must be http or https in %s'\
- % location)
+ raise writeexts.ExtensionError(
+ 'URL schema must be http or https in %s' % location)
if (x.path != '/v2.0' and x.path != '/v2.0/'):
- raise cliapp.AppException('API version must be v2.0 in %s'\
- % location)
+ raise writeexts.ExtensionError(
+ 'API version must be v2.0 in %s' % location)
def check_openstack_parameters(self, auth_params):
''' Check that we can connect to and authenticate with openstack '''
@@ -84,7 +86,7 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension):
except keystoneclient.exceptions.Unauthorized:
errmsg = ('Failed to authenticate with OpenStack '
'(are your credentials correct?)')
- raise cliapp.AppException(errmsg)
+ raise writeexts.ExtensionError(errmsg)
OpenStackCheckExtension().run()
diff --git a/extensions/openstack.write b/extensions/openstack.write
index 67e07c18..f1233560 100755
--- a/extensions/openstack.write
+++ b/extensions/openstack.write
@@ -17,21 +17,22 @@
'''A Morph deployment write extension for deploying to OpenStack.'''
-import cliapp
import os
+import subprocess
import tempfile
import urlparse
-import morphlib.writeexts
+import writeexts
-class OpenStackWriteExtension(morphlib.writeexts.WriteExtension):
+class OpenStackWriteExtension(writeexts.WriteExtension):
'''See openstack.write.help for documentation'''
def process_args(self, args):
if len(args) != 2:
- raise cliapp.AppException('Wrong number of command line args')
+ raise writeexts.ExtensionError(
+ 'Wrong number of command line args')
temp_root, location = args
@@ -86,7 +87,7 @@ class OpenStackWriteExtension(morphlib.writeexts.WriteExtension):
'--disk-format=raw',
'--container-format', 'bare',
'--file', raw_disk]
- cliapp.runcmd(cmdline)
+ subprocess.check_call(cmdline)
self.status(msg='Image configured.')
diff --git a/extensions/pxeboot.write b/extensions/pxeboot.write
index 3a12ebcc..20e4f6bd 100644
--- a/extensions/pxeboot.write
+++ b/extensions/pxeboot.write
@@ -19,10 +19,7 @@ import tempfile
import textwrap
import urlparse
-import cliapp
-
-import morphlib
-
+import writeexts
def _int_to_quad_dot(i):
return '.'.join((
@@ -143,7 +140,7 @@ def grouper(iterable, n, fillvalue=None):
return itertools.izip_longest(*args, fillvalue=fillvalue)
-class PXEBoot(morphlib.writeexts.WriteExtension):
+class PXEBoot(writeexts.WriteExtension):
@contextlib.contextmanager
def _vlan(self, interface, vlan):
viface = '%s.%s' % (interface, vlan)
@@ -224,12 +221,13 @@ class PXEBoot(morphlib.writeexts.WriteExtension):
@contextlib.contextmanager
def _remote_tempdir(self, hostname, template):
persist = os.environ.get('PXE_INSTALLER') in ('no', 'False')
- td = cliapp.ssh_runcmd(hostname, ['mktemp', '-d', template]).strip()
+ td = writeexts.ssh_runcmd(
+ hostname, ['mktemp', '-d', template]).strip()
try:
yield td
finally:
if not persist:
- cliapp.ssh_runcmd(hostname, ['find', td, '-delete'])
+ writeexts.ssh_runcmd(hostname, ['find', td, '-delete'])
def _serve_tftpd(self, sock, host, port, interface, tftproot):
self.settings.progname = 'tftp server'
@@ -333,26 +331,27 @@ class PXEBoot(morphlib.writeexts.WriteExtension):
def _remote_copy(self, hostname, src, dst):
persist = os.environ.get('PXE_INSTALLER') in ('no', 'False')
with open(src, 'r') as f:
- cliapp.ssh_runcmd(hostname,
- ['install', '-D', '-m644', '/proc/self/fd/0',
- dst], stdin=f, stdout=None, stderr=None)
+ writeexts.ssh_runcmd(hostname,
+ ['install', '-D', '-m644',
+ '/proc/self/fd/0', dst],
+ stdin=f, stdout=None, stderr=None)
try:
yield
finally:
if not persist:
- cliapp.ssh_runcmd(hostname, ['rm', dst])
+ writeexts.ssh_runcmd(hostname, ['rm', dst])
@contextlib.contextmanager
def _remote_symlink(self, hostname, src, dst):
persist = os.environ.get('PXE_INSTALLER') in ('no', 'False')
- cliapp.ssh_runcmd(hostname,
- ['ln', '-s', '-f', src, dst],
- stdin=None, stdout=None, stderr=None)
+ writeexts.ssh_runcmd(hostname,
+ ['ln', '-s', '-f', src, dst],
+ stdin=None, stdout=None, stderr=None)
try:
yield
finally:
if not persist:
- cliapp.ssh_runcmd(hostname, ['rm', '-f', dst])
+ writeexts.ssh_runcmd(hostname, ['rm', '-f', dst])
@contextlib.contextmanager
def remote_kernel(self, rootfs, tftp_url, macaddr):
@@ -361,7 +360,7 @@ class PXEBoot(morphlib.writeexts.WriteExtension):
if os.path.exists(kernel_path):
break
else:
- raise cliapp.AppException('Failed to locate kernel')
+ raise writeexts.ExtensionError('Failed to locate kernel')
url = urlparse.urlsplit(tftp_url)
basename = '{}-kernel'.format(_normalise_macaddr(macaddr))
target_path = os.path.join(url.path, basename)
@@ -376,7 +375,8 @@ class PXEBoot(morphlib.writeexts.WriteExtension):
yield
fdt_abs_path = os.path.join(rootfs, fdt_rel_path)
if not fdt_abs_path:
- raise cliapp.AppException('Failed to locate Flattened Device Tree')
+ raise writeexts.ExtensionError(
+ 'Failed to locate Flattened Device Tree')
url = urlparse.urlsplit(tftp_url)
basename = '{}-fdt'.format(_normalise_macaddr(macaddr))
target_path = os.path.join(url.path, basename)
@@ -389,14 +389,14 @@ class PXEBoot(morphlib.writeexts.WriteExtension):
nfsroot = target_ip + ':' + rootfs
self.status(msg='Exporting %(nfsroot)s as local nfsroot',
nfsroot=nfsroot)
- cliapp.runcmd(['exportfs', '-o', 'ro,insecure,no_root_squash',
- nfsroot])
+ subprocess.check_call(['exportfs', '-o', 'ro,insecure,no_root_squash',
+ nfsroot])
try:
yield
finally:
self.status(msg='Removing %(nfsroot)s from local nfsroots',
nfsroot=nfsroot)
- cliapp.runcmd(['exportfs', '-u', nfsroot])
+ subprocess.check_call(['exportfs', '-u', nfsroot])
@contextlib.contextmanager
def remote_nfsroot(self, rootfs, rsync_url, macaddr):
@@ -407,9 +407,10 @@ class PXEBoot(morphlib.writeexts.WriteExtension):
as tempdir:
nfsroot = urlparse.urlunsplit((url.scheme, url.netloc, tempdir,
url.query, url.fragment))
- cliapp.runcmd(['rsync', '-asSPH', '--delete', rootfs, nfsroot],
- stdin=None, stdout=open(os.devnull, 'w'),
- stderr=None)
+ subprocess.check_call(['rsync', '-asSPH', '--delete',
+ rootfs, nfsroot],
+ stdin=None, stdout=open(os.devnull, 'w'),
+ stderr=None)
yield os.path.join(os.path.basename(tempdir),
os.path.basename(rootfs))
@@ -517,8 +518,8 @@ class PXEBoot(morphlib.writeexts.WriteExtension):
def get_interface_ip(self, interface):
ip_addresses = []
- info = cliapp.runcmd(['ip', '-o', '-f', 'inet',
- 'addr', 'show', interface]).rstrip('\n')
+ info = subprocess.check_output(['ip', '-o', '-f', 'inet', 'addr',
+ 'show', interface]).rstrip('\n')
if info:
tokens = collections.deque(info.split()[1:])
ifname = tokens.popleft()
@@ -535,8 +536,8 @@ class PXEBoot(morphlib.writeexts.WriteExtension):
else:
continue
if not ip_addresses:
- raise cliapp.AppException('Interface %s has no addresses'
- % interface)
+ raise writeexts.ExtensionError('Interface %s has no addresses'
+ % interface)
if len(ip_addresses) > 1:
warnings.warn('Interface %s has multiple addresses, '
'using first (%s)' % (interface, ip_addresses[0]))
@@ -750,6 +751,6 @@ class PXEBoot(morphlib.writeexts.WriteExtension):
self.wait_for_target_to_install()
self.ipmi_reboot_target()
else:
- cliapp.AppException('Invalid PXEBOOT_MODE: %s' % mode)
+ writeexts.ExtensionError('Invalid PXEBOOT_MODE: %s' % mode)
PXEBoot().run()
diff --git a/extensions/rawdisk.check b/extensions/rawdisk.check
index 9be0ce91..61619a21 100755
--- a/extensions/rawdisk.check
+++ b/extensions/rawdisk.check
@@ -15,17 +15,16 @@
'''Preparatory checks for Morph 'rawdisk' write extension'''
-import cliapp
-
-import morphlib.writeexts
-
import os
+import writeexts
+
-class RawdiskCheckExtension(morphlib.writeexts.WriteExtension):
+class RawdiskCheckExtension(writeexts.WriteExtension):
def process_args(self, args):
if len(args) != 1:
- raise cliapp.AppException('Wrong number of command line args')
+ raise writeexts.ExtensionError(
+ 'Wrong number of command line args')
self.require_btrfs_in_deployment_host_kernel()
@@ -34,19 +33,19 @@ class RawdiskCheckExtension(morphlib.writeexts.WriteExtension):
if upgrade:
if not self.is_device(location):
if not os.path.isfile(location):
- raise cliapp.AppException(
+ raise writeexts.ExtensionError(
'Cannot upgrade %s: it is not an existing disk image' %
location)
version_label = os.environ.get('VERSION_LABEL')
if version_label is None:
- raise cliapp.AppException(
+ raise writeexts.ExtensionError(
'VERSION_LABEL was not given. It is required when '
'upgrading an existing system.')
else:
if not self.is_device(location):
if os.path.exists(location):
- raise cliapp.AppException(
+ raise writeexts.ExtensionError(
'Target %s already exists. Use `morph upgrade` if you '
'want to update an existing image.' % location)
diff --git a/extensions/rawdisk.write b/extensions/rawdisk.write
index 6f2d45ba..cdeb5018 100755
--- a/extensions/rawdisk.write
+++ b/extensions/rawdisk.write
@@ -17,22 +17,22 @@
'''A Morph deployment write extension for raw disk images.'''
-import cliapp
import os
import sys
import time
import tempfile
-import morphlib.writeexts
+import writeexts
-class RawDiskWriteExtension(morphlib.writeexts.WriteExtension):
+class RawDiskWriteExtension(writeexts.WriteExtension):
'''See rawdisk.write.help for documentation'''
def process_args(self, args):
if len(args) != 2:
- raise cliapp.AppException('Wrong number of command line args')
+ raise writeexts.ExtensionError(
+ 'Wrong number of command line args')
temp_root, location = args
upgrade = self.get_environment_boolean('UPGRADE')
@@ -69,10 +69,10 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension):
old_orig = os.path.join(mp, 'systems', 'default', 'orig')
new_orig = os.path.join(version_root, 'orig')
- cliapp.runcmd(
+ subprocess.check_call(
['btrfs', 'subvolume', 'snapshot', old_orig, new_orig])
- cliapp.runcmd(
+ subprocess.check_call(
['rsync', '-a', '--checksum', '--numeric-ids', '--delete',
temp_root + os.path.sep, new_orig])
@@ -96,11 +96,11 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension):
version_label = os.environ.get('VERSION_LABEL')
if version_label is None:
- raise cliapp.AppException('VERSION_LABEL was not given')
+ raise writeexts.ExtensionError('VERSION_LABEL was not given')
if os.path.exists(os.path.join(mp, 'systems', version_label)):
- raise cliapp.AppException('VERSION_LABEL %s already exists'
- % version_label)
+ raise writeexts.ExtensionError('VERSION_LABEL %s already exists'
+ % version_label)
return version_label
diff --git a/extensions/recv-hole b/extensions/recv-hole
new file mode 100755
index 00000000..fe69f304
--- /dev/null
+++ b/extensions/recv-hole
@@ -0,0 +1,158 @@
+#!/bin/sh
+#
+# Copyright (C) 2014-2015 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-2 =*=
+
+
+# Receive a data stream describing a sparse file, and reproduce it,
+# either to a named file or stdout.
+#
+# The data stream is simple: it's a sequence of DATA or HOLE records:
+#
+# DATA
+# 123
+# <123 bytes of binary data, NOT including newline at the end>
+#
+# HOLE
+# 123
+#
+# This shell script can be executed over ssh (given to ssh as an arguemnt,
+# with suitable escaping) on a different computer. This allows a large
+# sparse file (e.g., disk image) be transferred quickly.
+#
+# This script should be called in one of the following ways:
+#
+# recv-hole file FILENAME
+# recv-hole vbox FILENAME DISKSIZE
+#
+# In both cases, FILENAME is the pathname of the disk image on the
+# receiving end. DISKSIZE is the size of the disk image in bytes. The
+# first form is used when transferring a disk image to become an
+# identical file on the receiving end.
+#
+# The second form is used when the disk image should be converted for
+# use by VirtualBox. In this case, we want to avoid writing a
+# temporary file on disk, and then calling the VirtualBox VBoxManage
+# tool to do the conversion, since that would involve large amounts of
+# unnecessary I/O and disk usage. Instead we pipe the file directly to
+# VBoxManage, avoiding those issues. The piping is done here in this
+# script, instead of in the caller, to make it easier to run things
+# over ssh.
+#
+# However, since it's not possible seek in a Unix pipe, we have to
+# explicitly write the zeroes into the pipe. This is not
+# super-efficient, but the way to avoid that would be to avoid sending
+# a sparse file, and do the conversion to a VDI on the sending end.
+# That is out of scope for xfer-hole and recv-hole.
+
+
+set -eu
+
+
+die()
+{
+ echo "$@" 1>&2
+ exit 1
+}
+
+
+recv_hole_to_file()
+{
+ local n
+
+ read n
+ truncate --size "+$n" "$1"
+}
+
+
+recv_data_to_file()
+{
+ local n
+ read n
+
+ local blocksize=1048576
+ local blocks=$(($n / $blocksize))
+ local extra=$(($n % $blocksize))
+
+ xfer_data_to_stdout "$blocksize" "$blocks" >> "$1"
+ xfer_data_to_stdout 1 "$extra" >> "$1"
+}
+
+
+recv_hole_to_stdout()
+{
+ local n
+ read n
+ (echo "$n"; cat /dev/zero) | recv_data_to_stdout
+}
+
+
+recv_data_to_stdout()
+{
+ local n
+ read n
+
+ local blocksize=1048576
+ local blocks=$(($n / $blocksize))
+ local extra=$(($n % $blocksize))
+
+ xfer_data_to_stdout "$blocksize" "$blocks"
+ xfer_data_to_stdout 1 "$extra"
+}
+
+
+xfer_data_to_stdout()
+{
+ local log="$(mktemp)"
+ if ! dd "bs=$1" count="$2" iflag=fullblock status=noxfer 2> "$log"
+ then
+ cat "$log" 1>&2
+ rm -f "$log"
+ exit 1
+ else
+ rm -f "$log"
+ fi
+}
+
+
+type="$1"
+case "$type" in
+ file)
+ output="$2"
+ truncate --size=0 "$output"
+ while read what
+ do
+ case "$what" in
+ DATA) recv_data_to_file "$output" ;;
+ HOLE) recv_hole_to_file "$output" ;;
+ *) die "Unknown instruction: $what" ;;
+ esac
+ done
+ ;;
+ vbox)
+ output="$2"
+ disk_size="$3"
+ while read what
+ do
+ case "$what" in
+ DATA) recv_data_to_stdout ;;
+ HOLE) recv_hole_to_stdout ;;
+ *) die "Unknown instruction: $what" ;;
+ esac
+ done |
+ VBoxManage convertfromraw stdin "$output" "$disk_size"
+ ;;
+esac
diff --git a/extensions/simple-network.configure b/extensions/simple-network.configure
index 4a70f311..61d5774d 100755
--- a/extensions/simple-network.configure
+++ b/extensions/simple-network.configure
@@ -25,27 +25,26 @@ for DHCP
'''
+import errno
import os
import sys
-import errno
-import cliapp
-import morphlib
+import writeexts
-class SimpleNetworkError(morphlib.Error):
+class SimpleNetworkError(writeexts.ExtensionError):
'''Errors associated with simple network setup'''
pass
-class SimpleNetworkConfigurationExtension(cliapp.Application):
+class SimpleNetworkConfigurationExtension(object):
'''Configure /etc/network/interfaces and generate networkd .network files
Reading NETWORK_CONFIG, this extension sets up /etc/network/interfaces
and .network files in /etc/systemd/network/.
'''
- def process_args(self, args):
+ def run(self, args):
network_config = os.environ.get("NETWORK_CONFIG")
self.rename_networkd_chunk_file(args)
@@ -206,7 +205,8 @@ class SimpleNetworkConfigurationExtension(cliapp.Application):
address_line = address + '/' + str(network_suffix)
lines += ["Address=%s" % address_line]
elif address or netmask:
- raise Exception('address and netmask must be specified together')
+ raise SimpleNetworkError(
+ 'address and netmask must be specified together')
if gateway:
lines += ["Gateway=%s" % gateway]
@@ -287,6 +287,10 @@ class SimpleNetworkConfigurationExtension(cliapp.Application):
'''
- self.output.write('%s\n' % (kwargs['msg'] % kwargs))
+ sys.stdout.write('%s\n' % (kwargs['msg'] % kwargs))
-SimpleNetworkConfigurationExtension().run()
+try:
+ SimpleNetworkConfigurationExtension().run(sys.argv[1:])
+except SimpleNetworkError as e:
+ sys.stdout.write('ERROR: %s\n' % e)
+ sys.exit(1)
diff --git a/extensions/ssh-rsync.check b/extensions/ssh-rsync.check
index c3bdfd29..436aaae0 100755
--- a/extensions/ssh-rsync.check
+++ b/extensions/ssh-rsync.check
@@ -15,26 +15,27 @@
'''Preparatory checks for Morph 'ssh-rsync' write extension'''
-import cliapp
import os
-import morphlib.writeexts
+import writeexts
-class SshRsyncCheckExtension(morphlib.writeexts.WriteExtension):
+
+class SshRsyncCheckExtension(writeexts.WriteExtension):
def process_args(self, args):
if len(args) != 1:
- raise cliapp.AppException('Wrong number of command line args')
+ raise writeexts.ExtensionError(
+ 'Wrong number of command line args')
upgrade = self.get_environment_boolean('UPGRADE')
if not upgrade:
- raise cliapp.AppException(
+ raise writeexts.ExtensionError(
'The ssh-rsync write is for upgrading existing remote '
'Baserock machines. It cannot be used for an initial '
'deployment.')
if os.environ.get('VERSION_LABEL', '') == '':
- raise cliapp.AppException(
+ raise writeexts.ExtensionError(
'A VERSION_LABEL must be set when deploying an upgrade.')
location = args[0]
@@ -47,17 +48,18 @@ class SshRsyncCheckExtension(morphlib.writeexts.WriteExtension):
self.check_command_exists(location, 'rsync')
def check_is_baserock_system(self, location):
- output = cliapp.ssh_runcmd(location, ['sh', '-c',
- 'test -d /baserock || echo -n dirnotfound'])
+ output = writeexts.ssh_runcmd(
+ location,
+ ['sh', '-c', 'test -d /baserock || echo -n dirnotfound'])
if output == 'dirnotfound':
- raise cliapp.AppException('%s is not a baserock system'
- % location)
+ raise writeexts.ExtensionError('%s is not a baserock system'
+ % location)
def check_command_exists(self, location, command):
test = 'type %s > /dev/null 2>&1 || echo -n cmdnotfound' % command
- output = cliapp.ssh_runcmd(location, ['sh', '-c', test])
+ output = writeexts.ssh_runcmd(location, ['sh', '-c', test])
if output == 'cmdnotfound':
- raise cliapp.AppException(
+ raise writeexts.ExtensionError(
"%s does not have %s" % (location, command))
diff --git a/extensions/ssh-rsync.write b/extensions/ssh-rsync.write
index 6d596500..46c16662 100755
--- a/extensions/ssh-rsync.write
+++ b/extensions/ssh-rsync.write
@@ -18,23 +18,23 @@
import contextlib
-import cliapp
import os
+import subprocess
import sys
-import time
import tempfile
+import time
-import morphlib.writeexts
+import writeexts
def ssh_runcmd_ignore_failure(location, command, **kwargs):
try:
- return cliapp.ssh_runcmd(location, command, **kwargs)
- except cliapp.AppException:
+ return writeexts.ssh_runcmd(location, command, **kwargs)
+ except writeexts.ExtensionError:
pass
-class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension):
+class SshRsyncWriteExtension(writeexts.WriteExtension):
'''See ssh-rsync.write.help for documentation'''
@@ -43,7 +43,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension):
'''Read /proc/mounts on location to find which device contains "/"'''
self.status(msg='Finding device that contains "/"')
- contents = cliapp.ssh_runcmd(location, ['cat', '/proc/mounts'])
+ contents = writeexts.ssh_runcmd(location,
+ ['cat', '/proc/mounts'])
for line in contents.splitlines():
line_words = line.split()
if (line_words[1] == '/' and line_words[0] != 'rootfs'):
@@ -52,28 +53,29 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension):
@contextlib.contextmanager
def _remote_mount_point(self, location):
self.status(msg='Creating remote mount point')
- remote_mnt = cliapp.ssh_runcmd(location, ['mktemp', '-d']).strip()
+ remote_mnt = writeexts.ssh_runcmd(location,
+ ['mktemp', '-d']).strip()
try:
yield remote_mnt
finally:
self.status(msg='Removing remote mount point')
- cliapp.ssh_runcmd(location, ['rmdir', remote_mnt])
+ writeexts.ssh_runcmd(location, ['rmdir', remote_mnt])
@contextlib.contextmanager
def _remote_mount(self, location, root_disk, mountpoint):
self.status(msg='Mounting root disk')
- cliapp.ssh_runcmd(location, ['mount', root_disk, mountpoint])
+ writeexts.ssh_runcmd(location, ['mount', root_disk, mountpoint])
try:
yield
finally:
self.status(msg='Unmounting root disk')
- cliapp.ssh_runcmd(location, ['umount', mountpoint])
+ writeexts.ssh_runcmd(location, ['umount', mountpoint])
@contextlib.contextmanager
def _created_version_root(self, location, remote_mnt, version_label):
version_root = os.path.join(remote_mnt, 'systems', version_label)
self.status(msg='Creating %(root)s', root=version_root)
- cliapp.ssh_runcmd(location, ['mkdir', version_root])
+ writeexts.ssh_runcmd(location, ['mkdir', version_root])
try:
yield version_root
except BaseException as e:
@@ -93,8 +95,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension):
self.status(msg='Creating "orig" subvolume')
old_orig = self.get_old_orig(location, remote_mnt)
new_orig = os.path.join(version_root, 'orig')
- cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', 'snapshot',
- old_orig, new_orig])
+ writeexts.ssh_runcmd(location, ['btrfs', 'subvolume', 'snapshot',
+ old_orig, new_orig])
try:
yield new_orig
except BaseException as e:
@@ -106,30 +108,30 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension):
'''Populate the subvolume version_root/orig on location'''
self.status(msg='Populating "orig" subvolume')
- cliapp.runcmd(['rsync', '-as', '--checksum', '--numeric-ids',
- '--delete', temp_root + os.path.sep,
- '%s:%s' % (location, new_orig)])
+ subprocess.check_call(['rsync', '-as', '--checksum', '--numeric-ids',
+ '--delete', temp_root + os.path.sep,
+ '%s:%s' % (location, new_orig)])
@contextlib.contextmanager
def _deployed_version(self, location, version_label,
system_config_sync, system_version_manager):
self.status(msg='Calling system-version-manager to deploy upgrade')
deployment = os.path.join('/systems', version_label, 'orig')
- cliapp.ssh_runcmd(location,
+ writeexts.ssh_runcmd(location,
['env', 'BASEROCK_SYSTEM_CONFIG_SYNC='+system_config_sync,
system_version_manager, 'deploy', deployment])
try:
yield deployment
except BaseException as e:
self.status(msg='Cleaning up failed version installation')
- cliapp.ssh_runcmd(location,
+ writeexts.ssh_runcmd(location,
[system_version_manager, 'remove', version_label])
raise
def upgrade_remote_system(self, location, temp_root):
root_disk = self.find_root_disk(location)
- uuid = cliapp.ssh_runcmd(location, ['blkid', '-s', 'UUID', '-o',
- 'value', root_disk]).strip()
+ uuid = writeexts.ssh_runcmd(location,
+ ['blkid', '-s', 'UUID', '-o', 'value', root_disk]).strip()
self.complete_fstab_for_btrfs_layout(temp_root, uuid)
@@ -153,8 +155,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension):
config_sync, version_manager):
self.status(msg='Setting %(v)s as the new default system',
v=version_label)
- cliapp.ssh_runcmd(location, [version_manager,
- 'set-default', version_label])
+ writeexts.ssh_runcmd(location,
+ [version_manager, 'set-default', version_label])
if autostart:
self.status(msg="Rebooting into new system ...")
@@ -162,7 +164,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension):
def process_args(self, args):
if len(args) != 2:
- raise cliapp.AppException('Wrong number of command line args')
+ raise writeexts.ExtensionError(
+ 'Wrong number of command line args')
temp_root, location = args
diff --git a/extensions/strip-gplv3.configure b/extensions/strip-gplv3.configure
index c08061ad..0c5250e4 100755
--- a/extensions/strip-gplv3.configure
+++ b/extensions/strip-gplv3.configure
@@ -21,12 +21,14 @@ to find the files created by that chunk, then remove them.
'''
-import cliapp
-import re
-import os
import json
+import os
+import re
+import sys
+
+import writeexts
-class StripGPLv3ConfigureExtension(cliapp.Application):
+class StripGPLv3ConfigureExtension(writeexts.Extension):
gplv3_chunks = [
['autoconf', ''],
['automake', ''],
@@ -57,7 +59,8 @@ class StripGPLv3ConfigureExtension(cliapp.Application):
for chunk in self.gplv3_chunks:
regex = os.path.join(meta_dir, "%s-[^-]\+\.meta" % chunk[0])
- artifacts = self.runcmd(['find', meta_dir, '-regex', regex])
+ artifacts = subprocess.check_output(['find', meta_dir,
+ '-regex', regex])
for artifact in artifacts.split():
self.remove_chunk(target_root, artifact, chunk[1])
@@ -72,8 +75,8 @@ class StripGPLv3ConfigureExtension(cliapp.Application):
chunk_meta_data = json.load(f)
if not 'contents' in chunk_meta_data:
- raise cliapp.AppError('Chunk %s does not have a "contents" list'
- % chunk)
+ raise writeexts.ExtensionError(
+ 'Chunk %s does not have a "contents" list' % chunk)
updated_contents = []
for content_entry in reversed(chunk_meta_data['contents']):
pat = re.compile(pattern)
@@ -85,8 +88,8 @@ class StripGPLv3ConfigureExtension(cliapp.Application):
def remove_content_entry(self, target_root, content_entry):
entry_path = os.path.join(target_root, './' + content_entry)
if not entry_path.startswith(target_root):
- raise cliapp.AppException('%s is not in %s'
- % (entry_path, target_root))
+ raise writeexts.ExtensionError(
+ '%s is not in %s' % (entry_path, target_root))
if os.path.exists(entry_path):
if os.path.islink(entry_path):
os.unlink(entry_path)
@@ -96,6 +99,7 @@ class StripGPLv3ConfigureExtension(cliapp.Application):
if not os.listdir(entry_path):
os.rmdir(entry_path)
else:
- raise cliapp.AppException('%s is not a link, file or directory'
- % entry_path)
-StripGPLv3ConfigureExtension().run()
+ raise writeexts.ExtensionError(
+ '%s is not a link, file or directory' % entry_path)
+
+StripGPLv3ConfigureExtension().run(sys.argv[1:])
diff --git a/extensions/virtualbox-ssh.check b/extensions/virtualbox-ssh.check
index a97f3294..e82d58a1 100755
--- a/extensions/virtualbox-ssh.check
+++ b/extensions/virtualbox-ssh.check
@@ -15,21 +15,21 @@
'''Preparatory checks for Morph 'virtualbox-ssh' write extension'''
-import cliapp
-import morphlib.writeexts
+import writeexts
-class VirtualBoxPlusSshCheckExtension(morphlib.writeexts.WriteExtension):
+class VirtualBoxPlusSshCheckExtension(writeexts.WriteExtension):
def process_args(self, args):
if len(args) != 1:
- raise cliapp.AppException('Wrong number of command line args')
+ raise writeexts.ExtensionError(
+ 'Wrong number of command line args')
self.require_btrfs_in_deployment_host_kernel()
upgrade = self.get_environment_boolean('UPGRADE')
if upgrade:
- raise cliapp.AppException(
+ raise writeexts.ExtensionError(
'Use the `ssh-rsync` write extension to deploy upgrades to an '
'existing remote system.')
diff --git a/extensions/virtualbox-ssh.write b/extensions/virtualbox-ssh.write
index 774f2b4f..95643a4a 100755
--- a/extensions/virtualbox-ssh.write
+++ b/extensions/virtualbox-ssh.write
@@ -24,22 +24,23 @@ See file virtualbox-ssh.write.help for documentation
'''
-import cliapp
import os
import re
+import subprocess
import sys
-import time
import tempfile
+import time
import urlparse
-import morphlib.writeexts
+import writeexts
-class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension):
+class VirtualBoxPlusSshWriteExtension(writeexts.WriteExtension):
def process_args(self, args):
if len(args) != 2:
- raise cliapp.AppException('Wrong number of command line args')
+ raise writeexts.ExtensionError(
+ 'Wrong number of command line args')
temp_root, location = args
ssh_host, vm_name, vdi_path = self.parse_location(location)
@@ -59,7 +60,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension):
except BaseException:
sys.stderr.write('Error deploying to VirtualBox')
os.remove(raw_disk)
- cliapp.ssh_runcmd(ssh_host, ['rm', '-f', vdi_path])
+ writeexts.ssh_runcmd(ssh_host, ['rm', '-f', vdi_path])
raise
else:
os.remove(raw_disk)
@@ -72,11 +73,12 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension):
x = urlparse.urlparse(location)
if x.scheme != 'vbox+ssh':
- raise cliapp.AppException(
+ raise writeexts.ExtensionError(
'URL schema must be vbox+ssh in %s' % location)
m = re.match('^/(?P<guest>[^/]+)(?P<path>/.+)$', x.path)
if not m:
- raise cliapp.AppException('Cannot parse location %s' % location)
+ raise writeexts.ExtensionError(
+ 'Cannot parse location %s' % location)
return x.netloc, m.group('guest'), m.group('path')
def transfer_and_convert_to_vdi(self, raw_disk, ssh_host, vdi_path):
@@ -85,17 +87,18 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension):
self.status(msg='Transfer disk and convert to VDI')
st = os.lstat(raw_disk)
- xfer_hole_path = morphlib.util.get_data_path('xfer-hole')
- recv_hole = morphlib.util.get_data('recv-hole')
+ # TODO: Something!
+ xfer_hole_path = writeexts.get_data_path('xfer-hole')
+ recv_hole = writeexts.get_data('recv-hole')
ssh_remote_cmd = [
'sh', '-c', recv_hole,
'dummy-argv0', 'vbox', vdi_path, str(st.st_size),
]
- cliapp.runcmd(
+ subprocess.check_call(
['python', xfer_hole_path, raw_disk],
- ['ssh', ssh_host] + map(cliapp.shell_quote, ssh_remote_cmd),
+ ['ssh', ssh_host] + map(writeexts.shell_quote, ssh_remote_cmd),
stdout=None, stderr=None)
def virtualbox_version(self, ssh_host):
@@ -107,7 +110,8 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension):
# tuple is more reliable than a string and more convenient than
# comparing against the major, minor and patch numbers directly
self.status(msg='Checking version of remote VirtualBox')
- build_id = cliapp.ssh_runcmd(ssh_host, ['VBoxManage', '--version'])
+ build_id = writeexts.ssh_runcmd(ssh_host,
+ ['VBoxManage', '--version'])
version_string = re.match(r"^([0-9\.]+).*$", build_id.strip()).group(1)
return tuple(int(s or '0') for s in version_string.split('.'))
@@ -163,17 +167,17 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension):
for command in commands:
argv = ['VBoxManage'] + command
- cliapp.ssh_runcmd(ssh_host, argv)
+ writeexts.ssh_runcmd(ssh_host, argv)
def get_host_interface(self, ssh_host):
host_ipaddr = os.environ.get('HOST_IPADDR')
netmask = os.environ.get('NETMASK')
if host_ipaddr is None:
- raise cliapp.AppException('HOST_IPADDR was not given')
+ raise writeexts.ExtensionError('HOST_IPADDR was not given')
if netmask is None:
- raise cliapp.AppException('NETMASK was not given')
+ raise writeexts.ExtensionError('NETMASK was not given')
# 'VBoxManage list hostonlyifs' retrieves a list with the hostonly
# interfaces on the host. For each interface, the following lines
@@ -187,7 +191,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension):
# The following command tries to retrieve the hostonly interface
# name (e.g. vboxnet0) associated with the given ip address.
iface = None
- lines = cliapp.ssh_runcmd(ssh_host,
+ lines = writeexts.ssh_runcmd(ssh_host,
['VBoxManage', 'list', 'hostonlyifs']).splitlines()
for i, v in enumerate(lines):
if host_ipaddr in v:
@@ -195,12 +199,12 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension):
break
if iface is None:
- iface = cliapp.ssh_runcmd(ssh_host,
+ iface = writeexts.ssh_runcmd(ssh_host,
['VBoxManage', 'hostonlyif', 'create'])
# 'VBoxManage hostonlyif create' shows the name of the
# created hostonly interface inside single quotes
iface = iface[iface.find("'") + 1 : iface.rfind("'")]
- cliapp.ssh_runcmd(ssh_host,
+ writeexts.ssh_runcmd(ssh_host,
['VBoxManage', 'hostonlyif',
'ipconfig', iface,
'--ip', host_ipaddr,
diff --git a/extensions/writeexts.py b/extensions/writeexts.py
new file mode 100644
index 00000000..5c579a10
--- /dev/null
+++ b/extensions/writeexts.py
@@ -0,0 +1,746 @@
+# Copyright (C) 2012-2015 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+
+
+import contextlib
+import errno
+import fcntl
+import logging
+import os
+import re
+import select
+import shutil
+import stat
+import subprocess
+import sys
+import time
+import tempfile
+
+
+if sys.version_info >= (3, 3, 0):
+ import shlex
+ shell_quote = shlex.quote
+else:
+ import pipes
+ shell_quote = pipes.quote
+
+
+def get_data_path(relative_path):
+ extensions_dir = os.path.dirname(__file__)
+ return os.path.join(extensions_dir, relative_path)
+
+
+def get_data(relative_path):
+ with open(get_data_path(relative_path)) as f:
+ return f.read()
+
+
+def ssh_runcmd(host, args, **kwargs):
+ '''Run command over ssh'''
+ command = ['ssh', host, '--'] + [shell_quote(arg) for arg in args]
+
+ feed_stdin = kwargs.get('feed_stdin')
+ stdin = kwargs.get('stdin', subprocess.PIPE)
+ stdout = kwargs.get('stdout', subprocess.PIPE)
+ stderr = kwargs.get('stderr', subprocess.PIPE)
+
+ p = subprocess.Popen(command, stdin=stdin, stdout=stdout, stderr=stderr)
+ out, err = p.communicate(input=feed_stdin)
+ if p.returncode != 0:
+ raise ExtensionError('ssh command `%s` failed' % ' '.join(command))
+ return out
+
+
+def write_from_dict(filepath, d, validate=lambda x, y: True):
+ """Takes a dictionary and appends the contents to a file
+
+ An optional validation callback can be passed to perform validation on
+ each value in the dictionary.
+
+ e.g.
+
+ def validation_callback(dictionary_key, dictionary_value):
+ if not dictionary_value.isdigit():
+ raise Exception('value contains non-digit character(s)')
+
+ Any callback supplied to this function should raise an exception
+ if validation fails.
+
+ """
+ # Sort items asciibetically
+ # the output of the deployment should not depend
+ # on the locale of the machine running the deployment
+ items = sorted(d.iteritems(), key=lambda (k, v): [ord(c) for c in v])
+
+ for (k, v) in items:
+ validate(k, v)
+
+ with open(filepath, 'a') as f:
+ for (_, v) in items:
+ f.write('%s\n' % v)
+
+ os.fchown(f.fileno(), 0, 0)
+ os.fchmod(f.fileno(), 0644)
+
+
+def parse_environment_pairs(env, pairs):
+ '''Add key=value pairs to the environment dict.
+
+ Given a dict and a list of strings of the form key=value,
+ set dict[key] = value, unless key is already set in the
+ environment, at which point raise an exception.
+
+ This does not modify the passed in dict.
+
+ Returns the extended dict.
+
+ '''
+ extra_env = dict(p.split('=', 1) for p in pairs)
+ conflicting = [k for k in extra_env if k in env]
+ if conflicting:
+ raise ExtensionError('Environment already set: %s'
+ % ', '.join(conflicting))
+
+ # Return a dict that is the union of the two
+ # This is not the most performant, since it creates
+ # 3 unnecessary lists, but I felt this was the most
+ # easy to read. Using itertools.chain may be more efficicent
+ return dict(env.items() + extra_env.items())
+
+
+class ExtensionError(Exception):
+
+ def __init__(self, msg):
+ self.msg = msg
+
+ def __str__(self):
+ return self.msg
+
+
+class Fstab(object):
+ '''Small helper class for parsing and adding lines to /etc/fstab.'''
+
+ # There is an existing Python helper library for editing of /etc/fstab.
+ # However it is unmaintained and has an incompatible license (GPL3).
+ #
+ # https://code.launchpad.net/~computer-janitor-hackers/python-fstab/trunk
+
+ def __init__(self, filepath='/etc/fstab'):
+ if os.path.exists(filepath):
+ with open(filepath, 'r') as f:
+ self.text= f.read()
+ else:
+ self.text = ''
+ self.filepath = filepath
+ self.lines_added = 0
+
+ def get_mounts(self):
+ '''Return list of mount devices and targets in /etc/fstab.
+
+ Return value is a dict of target -> device.
+ '''
+ mounts = dict()
+ for line in self.text.splitlines():
+ words = line.split()
+ if len(words) >= 2 and not words[0].startswith('#'):
+ device, target = words[0:2]
+ mounts[target] = device
+ return mounts
+
+ def add_line(self, line):
+ '''Add a new entry to /etc/fstab.
+
+ Lines are appended, and separated from any entries made by configure
+ extensions with a comment.
+
+ '''
+ if self.lines_added == 0:
+ if len(self.text) == 0 or self.text[-1] is not '\n':
+ self.text += '\n'
+ self.text += '# Morph default system layout\n'
+ self.lines_added += 1
+
+ self.text += line + '\n'
+
+ def write(self):
+ '''Rewrite the fstab file to include all new entries.'''
+ with tempfile.NamedTemporaryFile(delete=False) as f:
+ f.write(self.text)
+ tmp = f.name
+ shutil.move(os.path.abspath(tmp), os.path.abspath(self.filepath))
+
+
+class Extension(object):
+
+ '''A base class for deployment extensions.
+
+ A subclass should subclass this class, and add a
+ ``process_args`` method.
+
+ Note that it is not necessary to subclass this class for write
+ extensions. This class is here just to collect common code for
+ write extensions.
+
+ '''
+
+ def setup_logging(self):
+ '''Direct all logging output to MORPH_LOG_FD, if set.
+
+ This file descriptor is read by Morph and written into its own log
+ file.
+
+ '''
+ log_write_fd = int(os.environ.get('MORPH_LOG_FD', 0))
+
+ if log_write_fd == 0:
+ return
+
+ formatter = logging.Formatter('%(message)s')
+
+ handler = logging.StreamHandler(os.fdopen(log_write_fd, 'w'))
+ handler.setFormatter(formatter)
+
+ logger = logging.getLogger()
+ logger.addHandler(handler)
+ logger.setLevel(logging.DEBUG)
+
+ def process_args(self, args):
+ raise NotImplementedError()
+
+ def run(self, args=None):
+ if args is None:
+ args = sys.argv[1:]
+ try:
+ self.setup_logging()
+ self.process_args(args)
+ except ExtensionError as e:
+ sys.stdout.write('ERROR: %s\n' % e)
+ sys.exit(1)
+
+ def status(self, **kwargs):
+ '''Provide status output.
+
+ The ``msg`` keyword argument is the actual message,
+ the rest are values for fields in the message as interpolated
+ by %.
+
+ '''
+ sys.stdout.write('%s\n' % (kwargs['msg'] % kwargs))
+ sys.stdout.flush()
+
+
+class WriteExtension(Extension):
+
+ '''A base class for deployment write extensions.
+
+ A subclass should subclass this class, and add a
+ ``process_args`` method.
+
+ Note that it is not necessary to subclass this class for write
+ extensions. This class is here just to collect common code for
+ write extensions.
+
+ '''
+
+ def check_for_btrfs_in_deployment_host_kernel(self):
+ with open('/proc/filesystems') as f:
+ text = f.read()
+ return '\tbtrfs\n' in text
+
+ def require_btrfs_in_deployment_host_kernel(self):
+ if not self.check_for_btrfs_in_deployment_host_kernel():
+ raise ExtensionError(
+ 'Error: Btrfs is required for this deployment, but was not '
+ 'detected in the kernel of the machine that is running Morph.')
+
+ def create_local_system(self, temp_root, raw_disk):
+ '''Create a raw system image locally.'''
+
+ with self.created_disk_image(raw_disk):
+ self.format_btrfs(raw_disk)
+ self.create_system(temp_root, raw_disk)
+
+ @contextlib.contextmanager
+ def created_disk_image(self, location):
+ size = self.get_disk_size()
+ if not size:
+ raise ExtensionError('DISK_SIZE is not defined')
+ self.create_raw_disk_image(location, size)
+ try:
+ yield
+ except BaseException:
+ os.unlink(location)
+ raise
+
+ def format_btrfs(self, raw_disk):
+ try:
+ self.mkfs_btrfs(raw_disk)
+ except BaseException:
+ sys.stderr.write('Error creating disk image')
+ raise
+
+ def create_system(self, temp_root, raw_disk):
+ with self.mount(raw_disk) as mp:
+ try:
+ self.create_btrfs_system_layout(
+ temp_root, mp, version_label='factory',
+ disk_uuid=self.get_uuid(raw_disk))
+ except BaseException as e:
+ sys.stderr.write('Error creating Btrfs system layout')
+ raise
+
+ def _parse_size(self, size):
+ '''Parse a size from a string.
+
+ Return size in bytes.
+
+ '''
+
+ m = re.match('^(\d+)([kmgKMG]?)$', size)
+ if not m:
+ return None
+
+ factors = {
+ '': 1,
+ 'k': 1024,
+ 'm': 1024**2,
+ 'g': 1024**3,
+ }
+ factor = factors[m.group(2).lower()]
+
+ return int(m.group(1)) * factor
+
+ def _parse_size_from_environment(self, env_var, default):
+ '''Parse a size from an environment variable.'''
+
+ size = os.environ.get(env_var, default)
+ if size is None:
+ return None
+ bytes = self._parse_size(size)
+ if bytes is None:
+ raise ExtensionError('Cannot parse %s value %s'
+ % (env_var, size))
+ return bytes
+
+ def get_disk_size(self):
+ '''Parse disk size from environment.'''
+ return self._parse_size_from_environment('DISK_SIZE', None)
+
+ def get_ram_size(self):
+ '''Parse RAM size from environment.'''
+ return self._parse_size_from_environment('RAM_SIZE', '1G')
+
+ def get_vcpu_count(self):
+ '''Parse the virtual cpu count from environment.'''
+ return self._parse_size_from_environment('VCPUS', '1')
+
+ def create_raw_disk_image(self, filename, size):
+ '''Create a raw disk image.'''
+
+ self.status(msg='Creating empty disk image')
+ with open(filename, 'wb') as f:
+ if size > 0:
+ f.seek(size-1)
+ f.write('\0')
+
+ def mkfs_btrfs(self, location):
+ '''Create a btrfs filesystem on the disk.'''
+
+ self.status(msg='Creating btrfs filesystem')
+ try:
+ # The following command disables some new filesystem features. We
+ # need to do this because at the time of writing, SYSLINUX has not
+ # been updated to understand these new features and will fail to
+ # boot if the kernel is on a filesystem where they are enabled.
+ subprocess.check_output(
+ ['mkfs.btrfs','-f', '-L', 'baserock',
+ '--features', '^extref',
+ '--features', '^skinny-metadata',
+ '--features', '^mixed-bg',
+ '--nodesize', '4096',
+ location])
+ except subprocess.CalledProcessError as e:
+ if 'unrecognized option \'--features\'' in e.output:
+ # Old versions of mkfs.btrfs (including v0.20, present in many
+ # Baserock releases) don't support the --features option, but
+ # also don't enable the new features by default. So we can
+ # still create a bootable system in this situation.
+ logging.debug(
+ 'Assuming mkfs.btrfs failure was because the tool is too '
+ 'old to have --features flag.')
+ subprocess.check_call(['mkfs.btrfs','-f',
+ '-L', 'baserock', location])
+ else:
+ raise
+
+ def get_uuid(self, location):
+ '''Get the UUID of a block device's file system.'''
+ # Requires util-linux blkid; busybox one ignores options and
+ # lies by exiting successfully.
+ return subprocess.check_output(['blkid', '-s', 'UUID', '-o', 'value',
+ location]).strip()
+
+ @contextlib.contextmanager
+ def mount(self, location):
+ self.status(msg='Mounting filesystem')
+ try:
+ mount_point = tempfile.mkdtemp()
+ if self.is_device(location):
+ subprocess.check_call(['mount', location, mount_point])
+ else:
+ subprocess.check_call(['mount', '-o', 'loop',
+ location, mount_point])
+ except BaseException as e:
+ sys.stderr.write('Error mounting filesystem')
+ os.rmdir(mount_point)
+ raise
+ try:
+ yield mount_point
+ finally:
+ self.status(msg='Unmounting filesystem')
+ subprocess.check_call(['umount', mount_point])
+ os.rmdir(mount_point)
+
+ def create_btrfs_system_layout(self, temp_root, mountpoint, version_label,
+ disk_uuid):
+ '''Separate base OS versions from state using subvolumes.
+
+ '''
+ initramfs = self.find_initramfs(temp_root)
+ version_root = os.path.join(mountpoint, 'systems', version_label)
+ state_root = os.path.join(mountpoint, 'state')
+
+ os.makedirs(version_root)
+ os.makedirs(state_root)
+
+ self.create_orig(version_root, temp_root)
+ system_dir = os.path.join(version_root, 'orig')
+
+ state_dirs = self.complete_fstab_for_btrfs_layout(system_dir,
+ disk_uuid)
+
+ for state_dir in state_dirs:
+ self.create_state_subvolume(system_dir, mountpoint, state_dir)
+
+ self.create_run(version_root)
+
+ os.symlink(
+ version_label, os.path.join(mountpoint, 'systems', 'default'))
+
+ if self.bootloader_config_is_wanted():
+ self.install_kernel(version_root, temp_root)
+ if self.get_dtb_path() != '':
+ self.install_dtb(version_root, temp_root)
+ self.install_syslinux_menu(mountpoint, version_root)
+ if initramfs is not None:
+ self.install_initramfs(initramfs, version_root)
+ self.generate_bootloader_config(mountpoint, disk_uuid)
+ else:
+ self.generate_bootloader_config(mountpoint)
+ self.install_bootloader(mountpoint)
+
+ def create_orig(self, version_root, temp_root):
+ '''Create the default "factory" system.'''
+
+ orig = os.path.join(version_root, 'orig')
+
+ self.status(msg='Creating orig subvolume')
+ subprocess.check_call(['btrfs', 'subvolume', 'create', orig])
+ self.status(msg='Copying files to orig subvolume')
+ subprocess.check_call(['cp', '-a', temp_root + '/.', orig + '/.'])
+
+ def create_run(self, version_root):
+ '''Create the 'run' snapshot.'''
+
+ self.status(msg='Creating run subvolume')
+ orig = os.path.join(version_root, 'orig')
+ run = os.path.join(version_root, 'run')
+ subprocess.check_call(
+ ['btrfs', 'subvolume', 'snapshot', orig, run])
+
+ def create_state_subvolume(self, system_dir, mountpoint, state_subdir):
+ '''Create a shared state subvolume.
+
+ We need to move any files added to the temporary rootfs by the
+ configure extensions to their correct home. For example, they might
+ have added keys in `/root/.ssh` which we now need to transfer to
+ `/state/root/.ssh`.
+
+ '''
+ self.status(msg='Creating %s subvolume' % state_subdir)
+ subvolume = os.path.join(mountpoint, 'state', state_subdir)
+ subprocess.check_call(['btrfs', 'subvolume', 'create', subvolume])
+ os.chmod(subvolume, 0o755)
+
+ existing_state_dir = os.path.join(system_dir, state_subdir)
+ files = []
+ if os.path.exists(existing_state_dir):
+ files = os.listdir(existing_state_dir)
+ if len(files) > 0:
+ self.status(msg='Moving existing data to %s subvolume' % subvolume)
+ for filename in files:
+ filepath = os.path.join(existing_state_dir, filename)
+ subprocess.check_call(['mv', filepath, subvolume])
+
+ def complete_fstab_for_btrfs_layout(self, system_dir, rootfs_uuid=None):
+ '''Fill in /etc/fstab entries for the default Btrfs disk layout.
+
+ In the future we should move this code out of the write extension and
+ in to a configure extension. To do that, though, we need some way of
+ informing the configure extension what layout should be used. Right now
+ a configure extension doesn't know if the system is going to end up as
+ a Btrfs disk image, a tarfile or something else and so it can't come
+ up with a sensible default fstab.
+
+ Configuration extensions can already create any /etc/fstab that they
+ like. This function only fills in entries that are missing, so if for
+ example the user configured /home to be on a separate partition, that
+ decision will be honoured and /state/home will not be created.
+
+ '''
+ shared_state_dirs = {'home', 'root', 'opt', 'srv', 'var'}
+
+ fstab = Fstab(os.path.join(system_dir, 'etc', 'fstab'))
+ existing_mounts = fstab.get_mounts()
+
+ if '/' in existing_mounts:
+ root_device = existing_mounts['/']
+ else:
+ root_device = (self.get_root_device() if rootfs_uuid is None else
+ 'UUID=%s' % rootfs_uuid)
+ fstab.add_line('%s / btrfs defaults,rw,noatime 0 1' % root_device)
+
+ state_dirs_to_create = set()
+ for state_dir in shared_state_dirs:
+ if '/' + state_dir not in existing_mounts:
+ state_dirs_to_create.add(state_dir)
+ state_subvol = os.path.join('/state', state_dir)
+ fstab.add_line(
+ '%s /%s btrfs subvol=%s,defaults,rw,noatime 0 2' %
+ (root_device, state_dir, state_subvol))
+
+ fstab.write()
+ return state_dirs_to_create
+
+ def find_initramfs(self, temp_root):
+ '''Check whether the rootfs has an initramfs.
+
+ Uses the INITRAMFS_PATH option to locate it.
+ '''
+ if 'INITRAMFS_PATH' in os.environ:
+ initramfs = os.path.join(temp_root, os.environ['INITRAMFS_PATH'])
+ if not os.path.exists(initramfs):
+ raise ExtensionError('INITRAMFS_PATH specified, '
+ 'but file does not exist')
+ return initramfs
+ return None
+
+ def install_initramfs(self, initramfs_path, version_root):
+ '''Install the initramfs outside of 'orig' or 'run' subvolumes.
+
+ This is required because syslinux doesn't traverse subvolumes when
+ loading the kernel or initramfs.
+ '''
+ self.status(msg='Installing initramfs')
+ initramfs_dest = os.path.join(version_root, 'initramfs')
+ subprocess.check_call(['cp', '-a', initramfs_path, initramfs_dest])
+
+ def install_kernel(self, version_root, temp_root):
+ '''Install the kernel outside of 'orig' or 'run' subvolumes'''
+
+ self.status(msg='Installing kernel')
+ image_names = ['vmlinuz', 'zImage', 'uImage']
+ kernel_dest = os.path.join(version_root, 'kernel')
+ for name in image_names:
+ try_path = os.path.join(temp_root, 'boot', name)
+ if os.path.exists(try_path):
+ subprocess.check_call(['cp', '-a', try_path, kernel_dest])
+ break
+
+ def install_dtb(self, version_root, temp_root):
+ '''Install the device tree outside of 'orig' or 'run' subvolumes'''
+
+ self.status(msg='Installing devicetree')
+ device_tree_path = self.get_dtb_path()
+ dtb_dest = os.path.join(version_root, 'dtb')
+ try_path = os.path.join(temp_root, device_tree_path)
+ if os.path.exists(try_path):
+ subprocess.check_call(['cp', '-a', try_path, dtb_dest])
+ else:
+ logging.error("Failed to find device tree %s", device_tree_path)
+ raise ExtensionError(
+ 'Failed to find device tree %s' % device_tree_path)
+
+ def get_dtb_path(self):
+ return os.environ.get('DTB_PATH', '')
+
+ def get_bootloader_install(self):
+ # Do we actually want to install the bootloader?
+ # Set this to "none" to prevent the install
+ return os.environ.get('BOOTLOADER_INSTALL', 'extlinux')
+
+ def get_bootloader_config_format(self):
+ # The config format for the bootloader,
+ # if not set we default to extlinux for x86
+ return os.environ.get('BOOTLOADER_CONFIG_FORMAT', 'extlinux')
+
+ def get_extra_kernel_args(self):
+ return os.environ.get('KERNEL_ARGS', '')
+
+ def get_root_device(self):
+ return os.environ.get('ROOT_DEVICE', '/dev/sda')
+
+ def generate_bootloader_config(self, real_root, disk_uuid=None):
+ '''Install extlinux on the newly created disk image.'''
+ config_function_dict = {
+ 'extlinux': self.generate_extlinux_config,
+ }
+
+ config_type = self.get_bootloader_config_format()
+ if config_type in config_function_dict:
+ config_function_dict[config_type](real_root, disk_uuid)
+ else:
+ raise ExtensionError(
+ 'Invalid BOOTLOADER_CONFIG_FORMAT %s' % config_type)
+
+ def generate_extlinux_config(self, real_root, disk_uuid=None):
+ '''Install extlinux on the newly created disk image.'''
+
+ self.status(msg='Creating extlinux.conf')
+ config = os.path.join(real_root, 'extlinux.conf')
+
+ ''' Please also update the documentation in the following files
+ if you change these default kernel args:
+ - kvm.write.help
+ - rawdisk.write.help
+ - virtualbox-ssh.write.help '''
+ kernel_args = (
+ 'rw ' # ro ought to work, but we don't test that regularly
+ 'init=/sbin/init ' # default, but it doesn't hurt to be explicit
+ 'rootfstype=btrfs ' # required when using initramfs, also boots
+ # faster when specified without initramfs
+ 'rootflags=subvol=systems/default/run ') # boot runtime subvol
+ kernel_args += 'root=%s ' % (self.get_root_device()
+ if disk_uuid is None
+ else 'UUID=%s' % disk_uuid)
+ kernel_args += self.get_extra_kernel_args()
+ with open(config, 'w') as f:
+ f.write('default linux\n')
+ f.write('timeout 1\n')
+ f.write('label linux\n')
+ f.write('kernel /systems/default/kernel\n')
+ if disk_uuid is not None:
+ f.write('initrd /systems/default/initramfs\n')
+ if self.get_dtb_path() != '':
+ f.write('devicetree /systems/default/dtb\n')
+ f.write('append %s\n' % kernel_args)
+
+ def install_bootloader(self, real_root):
+ install_function_dict = {
+ 'extlinux': self.install_bootloader_extlinux,
+ }
+
+ install_type = self.get_bootloader_install()
+ if install_type in install_function_dict:
+ install_function_dict[install_type](real_root)
+ elif install_type != 'none':
+ raise ExtensionError(
+ 'Invalid BOOTLOADER_INSTALL %s' % install_type)
+
+ def install_bootloader_extlinux(self, real_root):
+ self.status(msg='Installing extlinux')
+ subprocess.check_call(['extlinux', '--install', real_root])
+
+ # FIXME this hack seems to be necessary to let extlinux finish
+ subprocess.check_call(['sync'])
+ time.sleep(2)
+
+ def install_syslinux_menu(self, real_root, version_root):
+ '''Make syslinux/extlinux menu binary available.
+
+ The syslinux boot menu is compiled to a file named menu.c32. Extlinux
+ searches a few places for this file but it does not know to look inside
+ our subvolume, so we copy it to the filesystem root.
+
+ If the file is not available, the bootloader will still work but will
+ not be able to show a menu.
+
+ '''
+ menu_file = os.path.join(version_root, 'orig',
+ 'usr', 'share', 'syslinux', 'menu.c32')
+ if os.path.isfile(menu_file):
+ self.status(msg='Copying menu.c32')
+ shutil.copy(menu_file, real_root)
+
+ def parse_attach_disks(self):
+ '''Parse $ATTACH_DISKS into list of disks to attach.'''
+
+ if 'ATTACH_DISKS' in os.environ:
+ s = os.environ['ATTACH_DISKS']
+ return s.split(':')
+ else:
+ return []
+
+ def bootloader_config_is_wanted(self):
+ '''Does the user want to generate a bootloader config?
+
+ The user may set $BOOTLOADER_CONFIG_FORMAT to the desired
+ format. 'extlinux' is the only allowed value, and is the default
+ value for x86-32 and x86-64.
+
+ '''
+
+ def is_x86(arch):
+ return (arch == 'x86_64' or
+ (arch.startswith('i') and arch.endswith('86')))
+
+ value = os.environ.get('BOOTLOADER_CONFIG_FORMAT', '')
+ if value == '':
+ if not is_x86(os.uname()[-1]):
+ return False
+
+ return True
+
+ def get_environment_boolean(self, variable):
+ '''Parse a yes/no boolean passed through the environment.'''
+
+ value = os.environ.get(variable, 'no').lower()
+ if value in ['no', '0', 'false']:
+ return False
+ elif value in ['yes', '1', 'true']:
+ return True
+ else:
+ raise ExtensionError('Unexpected value for %s: %s' %
+ (variable, value))
+
+ def check_ssh_connectivity(self, ssh_host):
+ try:
+ output = ssh_runcmd(ssh_host, ['echo', 'test'])
+ except ExtensionError as e:
+ logging.error("Error checking SSH connectivity: %s", str(e))
+ raise ExtensionError(
+ 'Unable to SSH to %s: %s' % (ssh_host, e))
+
+ if output.strip() != 'test':
+ raise ExtensionError(
+ 'Unexpected output from remote machine: %s' % output.strip())
+
+ def is_device(self, location):
+ try:
+ st = os.stat(location)
+ return stat.S_ISBLK(st.st_mode)
+ except OSError as e:
+ if e.errno == errno.ENOENT:
+ return False
+ raise
diff --git a/extensions/xfer-hole b/extensions/xfer-hole
new file mode 100755
index 00000000..91f1be01
--- /dev/null
+++ b/extensions/xfer-hole
@@ -0,0 +1,137 @@
+#!/usr/bin/env python
+#
+# Send a sparse file more space-efficiently.
+# See recv-hole for a description of the protocol.
+#
+# Note that xfer-hole requires a version of Linux with support for
+# SEEK_DATA and SEEK_HOLE.
+#
+#
+# Copyright (C) 2014-2015 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-2 =*=
+
+
+
+import errno
+import os
+import sys
+
+
+SEEK_DATA = 3
+SEEK_HOLE = 4
+
+
+filename = sys.argv[1]
+fd = os.open(filename, os.O_RDONLY)
+pos = 0
+
+
+DATA = 'data'
+HOLE = 'hole'
+EOF = 'eof'
+
+
+def safe_lseek(fd, pos, whence):
+ try:
+ return os.lseek(fd, pos, whence)
+ except OSError as e:
+ if e.errno == errno.ENXIO:
+ return -1
+ raise
+
+
+def current_data_or_pos(fd, pos):
+ length = safe_lseek(fd, 0, os.SEEK_END)
+ next_data = safe_lseek(fd, pos, SEEK_DATA)
+ next_hole = safe_lseek(fd, pos, SEEK_HOLE)
+
+ if pos == length:
+ return EOF, pos
+ elif pos == next_data:
+ return DATA, pos
+ elif pos == next_hole:
+ return HOLE, pos
+ else:
+ assert False, \
+ ("Do not understand: pos=%d next_data=%d next_hole=%d" %
+ (pos, next_data, next_hole))
+
+
+def next_data_or_hole(fd, pos):
+ length = safe_lseek(fd, 0, os.SEEK_END)
+ next_data = safe_lseek(fd, pos, SEEK_DATA)
+ next_hole = safe_lseek(fd, pos, SEEK_HOLE)
+
+ if pos == length:
+ return EOF, pos
+ elif pos == next_data:
+ # We are at data.
+ if next_hole == -1 or next_hole == length:
+ return EOF, length
+ else:
+ return HOLE, next_hole
+ elif pos == next_hole:
+ # We are at a hole.
+ if next_data == -1 or next_data == length:
+ return EOF, length
+ else:
+ return DATA, next_data
+ else:
+ assert False, \
+ ("Do not understand: pos=%d next_data=%d next_hole=%d" %
+ (pos, next_data, next_hole))
+
+
+def find_data_and_holes(fd):
+ pos = safe_lseek(fd, 0, os.SEEK_CUR)
+
+ kind, pos = current_data_or_pos(fd, pos)
+ while kind != EOF:
+ yield kind, pos
+ kind, pos = next_data_or_hole(fd, pos)
+ yield kind, pos
+
+
+def make_xfer_instructions(fd):
+ prev_kind = None
+ prev_pos = None
+ for kind, pos in find_data_and_holes(fd):
+ if prev_kind == DATA:
+ yield (DATA, prev_pos, pos)
+ elif prev_kind == HOLE:
+ yield (HOLE, prev_pos, pos)
+ prev_kind = kind
+ prev_pos = pos
+
+
+def copy_slice_from_file(to, fd, start, end):
+ safe_lseek(fd, start, os.SEEK_SET)
+ nbytes = end - start
+ max_at_a_time = 1024**2
+ while nbytes > 0:
+ data = os.read(fd, min(nbytes, max_at_a_time))
+ if not data:
+ break
+ to.write(data)
+ nbytes -= len(data)
+
+
+for kind, start, end in make_xfer_instructions(fd):
+ if kind == HOLE:
+ sys.stdout.write('HOLE\n%d\n' % (end - start))
+ elif kind == DATA:
+ sys.stdout.write('DATA\n%d\n' % (end - start))
+ copy_slice_from_file(sys.stdout, fd, start, end)