From 370a33a33f503bba02125e13a1c8f300a656d441 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Thu, 4 Jun 2015 15:17:44 +0000 Subject: Remove dependencies on morphlib and cliapp from deployment extensions This is done by either copying some utility functions from morph into writeexts.py, and using the `subprocess` module rather than cliapp's runcmd and ssh_runcmd. Note that this means that these extensions will require "$definitions_checkout/extensions" in PYTHONPATH when they are run. This commit also updates VERSION to 5, since the PYTHONPATH requirement means that this change is incompatible with old versions of morph. Change-Id: Iec6fa7e3c7219619ce55e18493e5c37c36e97816 --- VERSION | 2 +- extensions/ceph.configure | 11 +- extensions/distbuild-trove-nfsboot.check | 52 ++++--- extensions/distbuild-trove-nfsboot.write | 40 ++--- extensions/fstab.configure | 4 +- extensions/hosts.configure | 12 +- extensions/image-package-example/README | 4 +- extensions/install-essential-files.configure | 21 +-- extensions/install-files.configure | 30 ++-- extensions/jffs2.write | 18 +-- extensions/kvm.check | 69 +++++---- extensions/kvm.write | 25 ++-- extensions/nfsboot.check | 50 ++++--- extensions/nfsboot.write | 83 +++++----- extensions/openstack.check | 26 ++-- extensions/openstack.write | 11 +- extensions/pxeboot.write | 57 +++---- extensions/rawdisk.check | 17 +-- extensions/rawdisk.write | 18 +-- extensions/simple-network.configure | 22 +-- extensions/ssh-rsync.check | 26 ++-- extensions/ssh-rsync.write | 51 ++++--- extensions/strip-gplv3.configure | 31 ++-- extensions/virtualbox-ssh.check | 10 +- extensions/virtualbox-ssh.write | 42 +++--- extensions/writeexts.py | 216 ++++++++++++++++++++------- 26 files changed, 542 insertions(+), 406 deletions(-) diff --git a/VERSION b/VERSION index 0a70affa..0a94cf8b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -version: 3 +version: 5 diff --git a/extensions/ceph.configure b/extensions/ceph.configure index c3cd92d1..3b8b2603 100644 --- a/extensions/ceph.configure +++ b/extensions/ceph.configure @@ -14,13 +14,14 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -import cliapp -import sys import os -import subprocess import shutil -import re import stat +import subprocess +import sys +import re + +import writeexts systemd_monitor_template = """ [Unit] @@ -75,7 +76,7 @@ executable_file_permissions = stat.S_IRUSR | stat.S_IXUSR | stat.S_IWUSR | \ stat.S_IXGRP | stat.S_IRGRP | \ stat.S_IXOTH | stat.S_IROTH -class CephConfigurationExtension(cliapp.Application): +class CephConfigurationExtension(writeexts.Extension): """ Set up ceph server daemons. diff --git a/extensions/distbuild-trove-nfsboot.check b/extensions/distbuild-trove-nfsboot.check index 38c491e5..76ba6dda 100755 --- a/extensions/distbuild-trove-nfsboot.check +++ b/extensions/distbuild-trove-nfsboot.check @@ -15,14 +15,15 @@ '''Preparatory checks for Morph 'distbuild-trove-nfsboot' write extension''' -import cliapp import logging import os +import subprocess +import sys -import morphlib.writeexts +import writeexts -class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): +class DistbuildTroveNFSBootCheckExtension(writeexts.WriteExtension): nfsboot_root = '/srv/nfsboot' remote_user = 'root' @@ -45,7 +46,8 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') nfs_host = args[0] nfs_netloc = '%s@%s' % (self.remote_user, nfs_host) @@ -55,17 +57,19 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): missing_vars = [var for var in self.required_vars if not var in os.environ] if missing_vars: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Please set: %s' % ', '.join(missing_vars)) controllers = os.getenv('DISTBUILD_CONTROLLER').split() workers = os.getenv('DISTBUILD_WORKERS').split() if len(controllers) != 1: - raise cliapp.AppException('Please specify exactly one controller.') + raise writeexts.ExtensionError( + 'Please specify exactly one controller.') if len(workers) == 0: - raise cliapp.AppException('Please specify at least one worker.') + raise writeexts.ExtensionError( + 'Please specify at least one worker.') upgrade = self.get_environment_boolean('UPGRADE') @@ -80,7 +84,7 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): if self.remote_directory_exists(nfs_netloc, system_path): if self.get_environment_boolean('OVERWRITE') == False: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'System %s already exists at %s:%s. Try `morph ' 'upgrade` instead of `morph deploy`.' % ( system_name, nfs_netloc, system_path)) @@ -91,27 +95,27 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): # Is an NFS server try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( netloc, ['test', '-e', '/etc/exports']) - except cliapp.AppException: - raise cliapp.AppException('server %s is not an nfs server' - % netloc) + except writeexts.ExtensionError: + raise writeexts.ExtensionError('server %s is not an nfs server' + % netloc) try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( netloc, ['systemctl', 'is-enabled', 'nfs-server.service']) - except cliapp.AppException: - raise cliapp.AppException('server %s does not control its ' - 'nfs server by systemd' % netloc) + except writeexts.ExtensionError: + raise writeexts.ExtensionError('server %s does not control its ' + 'nfs server by systemd' % netloc) # TFTP server exports /srv/nfsboot/tftp tftp_root = os.path.join(self.nfsboot_root, 'tftp') try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( netloc, ['test' , '-d', tftp_root]) - except cliapp.AppException: - raise cliapp.AppException('server %s does not export %s' % - (netloc, tftp_root)) + except writeexts.ExtensionError: + raise writeexts.ExtensionError('server %s does not export %s' % + (netloc, tftp_root)) def check_upgradeable(self, nfs_netloc, system_name, version_label): '''Check that there is already a version of the system present. @@ -124,7 +128,7 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): system_version_path = self.system_path(system_name, version_label) if not self.remote_directory_exists(nfs_netloc, system_path): - raise cliapp.AppException( + raise writeexts.ExtensionError( 'System %s not found at %s:%s, cannot deploy an upgrade.' % ( system_name, nfs_netloc, system_path)) @@ -132,15 +136,15 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): if self.get_environment_boolean('OVERWRITE'): pass else: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'System %s version %s already exists at %s:%s.' % ( system_name, version_label, nfs_netloc, system_version_path)) def remote_directory_exists(self, nfs_netloc, path): try: - cliapp.ssh_runcmd(nfs_netloc, ['test', '-d', path]) - except cliapp.AppException as e: + writeexts.ssh_runcmd(nfs_netloc, ['test', '-d', path]) + except writeexts.ExtensionError as e: logging.debug('SSH exception: %s', e) return False diff --git a/extensions/distbuild-trove-nfsboot.write b/extensions/distbuild-trove-nfsboot.write index a5a5b094..86291794 100755 --- a/extensions/distbuild-trove-nfsboot.write +++ b/extensions/distbuild-trove-nfsboot.write @@ -20,14 +20,14 @@ import os +import subprocess import sys import tempfile -import cliapp -import morphlib.writeexts +import writeexts -class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): +class DistbuildTroveNFSBootWriteExtension(writeexts.WriteExtension): '''Create an NFS root and kernel on TFTP during Morph's deployment. @@ -54,7 +54,7 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError('Wrong number of command line args') local_system_path, nfs_host = args @@ -111,17 +111,17 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): ''' pairs = host_map_string.split(' ') - return morphlib.util.parse_environment_pairs({}, pairs) + return writeexts.parse_environment_pairs({}, pairs) def transfer_system(self, nfs_netloc, local_system_path, remote_system_path): self.status(msg='Copying rootfs to %(nfs_netloc)s', nfs_netloc=nfs_netloc) - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( nfs_netloc, ['mkdir', '-p', remote_system_path]) # The deployed rootfs may have been created by OSTree, so definitely # don't pass --hard-links to `rsync`. - cliapp.runcmd( + subprocess.check_call( ['rsync', '--archive', '--delete', '--info=progress2', '--protect-args', '--partial', '--sparse', '--xattrs', local_system_path + '/', @@ -131,13 +131,13 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): target_system_path): self.status(msg='Duplicating rootfs to %(target_system_path)s', target_system_path=target_system_path) - cliapp.ssh_runcmd(nfs_netloc, + writeexts.ssh_runcmd(nfs_netloc, ['mkdir', '-p', target_system_path]) # We can't pass --info=progress2 here, because it may not be available # in the remote 'rsync'. The --info setting was added in RSync 3.1.0, # old versions of Baserock have RSync 3.0.9. So the user doesn't get # any progress info on stdout for the 'duplicate' stage. - cliapp.ssh_runcmd(nfs_netloc, + writeexts.ssh_runcmd(nfs_netloc, ['rsync', '--archive', '--delete', '--protect-args', '--partial', '--sparse', '--xattrs', source_system_path + '/', target_system_path], stdout=sys.stdout) @@ -152,7 +152,7 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): kernel_path = os.path.relpath(try_path, local_system_path) break else: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Could not find a kernel in the system: none of ' '%s found' % ', '.join(image_names)) return kernel_path @@ -171,11 +171,11 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): versioned_kernel_name = "%s-%s" % (system_name, version_label) kernel_name = system_name - cliapp.ssh_runcmd(nfs_netloc, + writeexts.ssh_runcmd(nfs_netloc, ['ln', '-f', kernel_dest, os.path.join(tftp_dir, versioned_kernel_name)]) - cliapp.ssh_runcmd(nfs_netloc, + writeexts.ssh_runcmd(nfs_netloc, ['ln', '-sf', versioned_kernel_name, os.path.join(tftp_dir, kernel_name)]) @@ -183,7 +183,7 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): with tempfile.NamedTemporaryFile() as f: f.write(text) f.flush() - cliapp.runcmd( + subprocess.check_call( ['scp', f.name, '%s:%s' % (nfs_netloc, path)]) def set_hostname(self, nfs_netloc, system_name, system_path): @@ -223,9 +223,9 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): '# Generated by distbuild-trove-nfsboot.write\n' + \ config_text + '\n' path = os.path.join(system_path, 'etc', 'distbuild') - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( nfs_netloc, ['mkdir', '-p', path]) - cliapp.runcmd( + subprocess.check_call( ['scp', worker_ssh_key_path, '%s:%s' % (nfs_netloc, path)]) self.set_remote_file_contents( nfs_netloc, os.path.join(path, 'distbuild.conf'), config_text) @@ -244,9 +244,9 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): # Rather ugly SSH hackery follows to ensure each system path is # listed in /etc/exports. try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( nfs_netloc, ['grep', '-q', exported_path, exports_path]) - except cliapp.AppException: + except writeexts.ExtensionError: ip_mask = '*' options = 'rw,no_subtree_check,no_root_squash,async' exports_string = '%s %s(%s)\n' % (exported_path, ip_mask, @@ -259,12 +259,12 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): cat >> "$temp" mv "$temp" "$target" ''' - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( nfs_netloc, ['sh', '-c', exports_append_sh, '--', exports_path], feed_stdin=exports_string) - cliapp.ssh_runcmd(nfs_netloc, + writeexts.ssh_runcmd(nfs_netloc, ['systemctl', 'restart', 'nfs-server.service']) def update_default_version(self, remote_netloc, system_name, @@ -276,7 +276,7 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): version_label) default_path = os.path.join(system_path, 'systems', 'default') - cliapp.ssh_runcmd(remote_netloc, + writeexts.ssh_runcmd(remote_netloc, ['ln', '-sfn', system_version_path, default_path]) diff --git a/extensions/fstab.configure b/extensions/fstab.configure index b9154eee..3e67b585 100755 --- a/extensions/fstab.configure +++ b/extensions/fstab.configure @@ -20,9 +20,9 @@ import os import sys -import morphlib +import writeexts envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('FSTAB_')} conf_file = os.path.join(sys.argv[1], 'etc/fstab') -morphlib.util.write_from_dict(conf_file, envvars) +writeexts.write_from_dict(conf_file, envvars) diff --git a/extensions/hosts.configure b/extensions/hosts.configure index 6b068d04..11fcf573 100755 --- a/extensions/hosts.configure +++ b/extensions/hosts.configure @@ -22,27 +22,29 @@ import os import sys import socket -import morphlib +import writeexts def validate(var, line): xs = line.split() if len(xs) == 0: - raise morphlib.Error("`%s: %s': line is empty" % (var, line)) + raise writeexts.ExtensionError( + "`%s: %s': line is empty" % (var, line)) ip = xs[0] hostnames = xs[1:] if len(hostnames) == 0: - raise morphlib.Error("`%s: %s': missing hostname" % (var, line)) + raise writeexts.ExtensionError( + "`%s: %s': missing hostname" % (var, line)) family = socket.AF_INET6 if ':' in ip else socket.AF_INET try: socket.inet_pton(family, ip) except socket.error: - raise morphlib.Error("`%s: %s' invalid ip" % (var, ip)) + raise writeexts.ExtensionError("`%s: %s' invalid ip" % (var, ip)) envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('HOSTS_')} conf_file = os.path.join(sys.argv[1], 'etc/hosts') -morphlib.util.write_from_dict(conf_file, envvars, validate) +writeexts.write_from_dict(conf_file, envvars, validate) diff --git a/extensions/image-package-example/README b/extensions/image-package-example/README index c1322f25..f6b66cd9 100644 --- a/extensions/image-package-example/README +++ b/extensions/image-package-example/README @@ -5,5 +5,5 @@ These are scripts used to create disk images or install the system onto an existing disk. This is also implemented independently for the rawdisk.write write -extension; see morphlib.writeexts.WriteExtension.create_local_system() -for a similar, python implementation. +extension; see writeexts.WriteExtension.create_local_system() for +a similar, python implementation. diff --git a/extensions/install-essential-files.configure b/extensions/install-essential-files.configure index bed394df..3d33fe03 100755 --- a/extensions/install-essential-files.configure +++ b/extensions/install-essential-files.configure @@ -22,20 +22,11 @@ to install into the target system. ''' -import subprocess import os +import subprocess +import sys -import cliapp - -class InstallEssentialFilesConfigureExtension(cliapp.Application): - - def process_args(self, args): - target_root = args[0] - os.environ["INSTALL_FILES"] = "install-files/essential-files/manifest" - self.install_essential_files(target_root) - - def install_essential_files(self, target_root): - command = os.path.join("extensions/install-files.configure") - subprocess.check_call([command, target_root]) - -InstallEssentialFilesConfigureExtension().run() +target_root = sys.argv[1] +os.environ["INSTALL_FILES"] = "install-files/essential-files/manifest" +command = os.path.join("extensions/install-files.configure") +subprocess.check_call([command, target_root]) diff --git a/extensions/install-files.configure b/extensions/install-files.configure index 341cce61..64fcecca 100755 --- a/extensions/install-files.configure +++ b/extensions/install-files.configure @@ -22,9 +22,8 @@ to install into the target system. ''' -import cliapp -import os import errno +import os import re import sys import shlex @@ -37,7 +36,9 @@ try: except ImportError: jinja_available = False -class InstallFilesConfigureExtension(cliapp.Application): +import writeexts + +class InstallFilesConfigureExtension(writeexts.Extension): def process_args(self, args): if not 'INSTALL_FILES' in os.environ: @@ -74,7 +75,8 @@ class InstallFilesConfigureExtension(cliapp.Application): gid = int(m.group(5)) path = m.group(6) else: - raise cliapp.AppException('Invalid manifest entry, ' + raise writeexts.ExtensionError( + 'Invalid manifest entry, ' 'format: [template] [overwrite] ' ' ') @@ -85,9 +87,9 @@ class InstallFilesConfigureExtension(cliapp.Application): if (mode != dest_stat.st_mode or uid != dest_stat.st_uid or gid != dest_stat.st_gid): - raise cliapp.AppException('"%s" exists and is not ' - 'identical to directory ' - '"%s"' % (dest_path, entry)) + raise writeexts.ExtensionError( + '"%s" exists and is not identical to directory ' + '"%s"' % (dest_path, entry)) else: os.mkdir(dest_path, mode) os.chown(dest_path, uid, gid) @@ -95,8 +97,8 @@ class InstallFilesConfigureExtension(cliapp.Application): elif stat.S_ISLNK(mode): if os.path.lexists(dest_path) and not overwrite: - raise cliapp.AppException('Symlink already exists at %s' - % dest_path) + raise writeexts.ExtensionError('Symlink already exists at %s' + % dest_path) else: linkdest = os.readlink(os.path.join(manifest_root, './' + path)) @@ -105,12 +107,12 @@ class InstallFilesConfigureExtension(cliapp.Application): elif stat.S_ISREG(mode): if os.path.lexists(dest_path) and not overwrite: - raise cliapp.AppException('File already exists at %s' - % dest_path) + raise writeexts.ExtensionError('File already exists at %s' + % dest_path) else: if template: if not jinja_available: - raise cliapp.AppException( + raise writeexts.ExtensionError( "Failed to install template file `%s': " 'install-files templates require jinja2' % path) @@ -128,7 +130,7 @@ class InstallFilesConfigureExtension(cliapp.Application): os.chmod(dest_path, mode) else: - raise cliapp.AppException('Mode given in "%s" is not a file,' - ' symlink or directory' % entry) + raise writeexts.ExtensionError('Mode given in "%s" is not a file,' + ' symlink or directory' % entry) InstallFilesConfigureExtension().run() diff --git a/extensions/jffs2.write b/extensions/jffs2.write index 46b69a53..ad68204d 100644 --- a/extensions/jffs2.write +++ b/extensions/jffs2.write @@ -19,34 +19,34 @@ as the root filesystem.''' -import cliapp import os +import subprocess -import morphlib.writeexts +import writeexts -class Jffs2WriteExtension(morphlib.writeexts.WriteExtension): +class Jffs2WriteExtension(writeexts.WriteExtension): '''See jffs2.write.help for documentation.''' def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError('Wrong number of command line args') temp_root, location = args try: self.create_jffs2_system(temp_root, location) self.status(msg='Disk image has been created at %(location)s', - location = location) + location=location) except Exception: self.status(msg='Failure to deploy system to %(location)s', - location = location) + location=location) raise def create_jffs2_system(self, temp_root, location): erase_block = self.get_erase_block_size() - cliapp.runcmd( + subprocess.check_call( ['mkfs.jffs2', '--pad', '--no-cleanmarkers', '--eraseblock='+erase_block, '-d', temp_root, '-o', location]) @@ -54,10 +54,10 @@ class Jffs2WriteExtension(morphlib.writeexts.WriteExtension): erase_block = os.environ.get('ERASE_BLOCK', '') if erase_block == '': - raise cliapp.AppException('ERASE_BLOCK was not given') + raise writeexts.ExtensionError('ERASE_BLOCK was not given') if not erase_block.isdigit(): - raise cliapp.AppException('ERASE_BLOCK must be a whole number') + raise writeexts.ExtensionError('ERASE_BLOCK must be a whole number') return erase_block diff --git a/extensions/kvm.check b/extensions/kvm.check index 67cb3d38..3c277156 100755 --- a/extensions/kvm.check +++ b/extensions/kvm.check @@ -15,27 +15,28 @@ '''Preparatory checks for Morph 'kvm' write extension''' -import cliapp import os import re +import subprocess import urlparse -import morphlib.writeexts +import writeexts -class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): +class KvmPlusSshCheckExtension(writeexts.WriteExtension): location_pattern = '^/(?P[^/]+)(?P/.+)$' def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') self.require_btrfs_in_deployment_host_kernel() upgrade = self.get_environment_boolean('UPGRADE') if upgrade: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Use the `ssh-rsync` write extension to deploy upgrades to an ' 'existing remote system.') @@ -55,23 +56,24 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): x = urlparse.urlparse(location) if x.scheme != 'kvm+ssh': - raise cliapp.AppException( + raise writeexts.ExtensionError( 'URL schema must be kvm+ssh in %s' % location) m = re.match(self.location_pattern, x.path) if not m: - raise cliapp.AppException('Cannot parse location %s' % location) + raise writeexts.ExtensionError( + 'Cannot parse location %s' % location) return x.netloc, m.group('guest'), m.group('path') def check_no_existing_libvirt_vm(self, ssh_host, vm_name): try: - cliapp.ssh_runcmd(ssh_host, + writeexts.ssh_runcmd(ssh_host, ['virsh', '--connect', 'qemu:///system', 'domstate', vm_name]) - except cliapp.AppException as e: + except CalledProcessError as e: pass else: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Host %s already has a VM named %s. You can use the ssh-rsync ' 'write extension to deploy upgrades to existing machines.' % (ssh_host, vm_name)) @@ -80,35 +82,35 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): def check_can_write_to_given_path(): try: - cliapp.ssh_runcmd(ssh_host, ['touch', vm_path]) - except cliapp.AppException as e: - raise cliapp.AppException("Can't write to location %s on %s" - % (vm_path, ssh_host)) + writeexts.ssh_runcmd(ssh_host, ['touch', vm_path]) + except writeexts.ExtensionError as e: + raise writeexts.ExtensionError( + "Can't write to location %s on %s" % (vm_path, ssh_host)) else: - cliapp.ssh_runcmd(ssh_host, ['rm', vm_path]) + writeexts.ssh_runcmd(ssh_host, ['rm', vm_path]) try: - cliapp.ssh_runcmd(ssh_host, ['test', '-e', vm_path]) - except cliapp.AppException as e: + writeexts.ssh_runcmd(ssh_host, ['test', '-e', vm_path]) + except writeexts.ExtensionError as e: # vm_path doesn't already exist, so let's test we can write check_can_write_to_given_path() else: - raise cliapp.AppException('%s already exists on %s' - % (vm_path, ssh_host)) + raise writeexts.ExtensionError('%s already exists on %s' + % (vm_path, ssh_host)) def check_extra_disks_exist(self, ssh_host, filename_list): for filename in filename_list: try: - cliapp.ssh_runcmd(ssh_host, ['ls', filename]) - except cliapp.AppException as e: - raise cliapp.AppException('Did not find file %s on host %s' % - (filename, ssh_host)) + writeexts.ssh_runcmd(ssh_host, ['ls', filename]) + except writeexts.ExtensionError as e: + raise writeexts.ExtensionError( + 'Did not find file %s on host %s' % (filename, ssh_host)) def check_virtual_networks_are_started(self, ssh_host): def check_virtual_network_is_started(network_name): cmd = ['virsh', '-c', 'qemu:///system', 'net-info', network_name] - net_info = cliapp.ssh_runcmd(ssh_host, cmd).split('\n') + net_info = writeexts.ssh_runcmd(ssh_host, cmd).split('\n') def pretty_concat(lines): return '\n'.join(['\t%s' % line for line in lines]) @@ -118,15 +120,15 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): if m: break else: - raise cliapp.AppException( + raise writeexts.ExtensionError( "Got unexpected output parsing output of `%s':\n%s" % (' '.join(cmd), pretty_concat(net_info))) network_active = m.group(1) == 'yes' if not network_active: - raise cliapp.AppException("Network '%s' is not started" - % network_name) + raise writeexts.ExtensionError("Network '%s' is not started" + % network_name) def name(nic_entry): if ',' in nic_entry: @@ -142,9 +144,10 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): if not (n.startswith('network=') or n.startswith('bridge=') or n == 'user'): - raise cliapp.AppException('malformed NIC_CONFIG: %s\n' - " (expected 'bridge=BRIDGE' 'network=NAME'" - " or 'user')" % n) + raise writeexts.ExtensionError( + "malformed NIC_CONFIG: %s\n" + " (expected 'bridge=BRIDGE' 'network=NAME'" + " or 'user')" % n) # --network bridge= is used to specify a bridge # --network user is used to specify a form of NAT @@ -159,9 +162,9 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): def check_host_has_virtinstall(self, ssh_host): try: - cliapp.ssh_runcmd(ssh_host, ['which', 'virt-install']) - except cliapp.AppException: - raise cliapp.AppException( + writeexts.ssh_runcmd(ssh_host, ['which', 'virt-install']) + except writeexts.ExtensionError: + raise writeexts.ExtensionError( 'virt-install does not seem to be installed on host %s' % ssh_host) diff --git a/extensions/kvm.write b/extensions/kvm.write index 0d0c095b..2290725e 100755 --- a/extensions/kvm.write +++ b/extensions/kvm.write @@ -21,23 +21,23 @@ See file kvm.write.help for documentation ''' -import cliapp import os import re import sys import tempfile import urlparse -import morphlib.writeexts +import writeexts -class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): +class KvmPlusSshWriteExtension(writeexts.WriteExtension): location_pattern = '^/(?P[^/]+)(?P/.+)$' def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') temp_root, location = args ssh_host, vm_name, vm_path = self.parse_location(location) @@ -53,7 +53,7 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): except BaseException: sys.stderr.write('Error deploying to libvirt') os.remove(raw_disk) - cliapp.ssh_runcmd(ssh_host, ['rm', '-f', vm_path]) + writeexts.ssh_runcmd(ssh_host, ['rm', '-f', vm_path]) raise else: os.remove(raw_disk) @@ -74,16 +74,16 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Transferring disk image') - xfer_hole_path = morphlib.util.get_data_path('xfer-hole') - recv_hole = morphlib.util.get_data('recv-hole') + xfer_hole_path = writeexts.get_data_path('xfer-hole') + recv_hole = writeexts.get_data('recv-hole') ssh_remote_cmd = [ 'sh', '-c', recv_hole, 'dummy-argv0', 'file', vm_path ] - cliapp.runcmd( + subprocess.check_call( ['python', xfer_hole_path, raw_disk], - ['ssh', ssh_host] + map(cliapp.shell_quote, ssh_remote_cmd), + ['ssh', ssh_host] + map(writeexts.shell_quote, ssh_remote_cmd), stdout=None, stderr=None) def create_libvirt_guest(self, ssh_host, vm_name, vm_path, autostart): @@ -111,10 +111,11 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): '--disk', 'path=%s,bus=ide' % vm_path] + attach_opts if not autostart: cmdline += ['--noreboot'] - cliapp.ssh_runcmd(ssh_host, cmdline) + writeexts.ssh_runcmd(ssh_host, cmdline) if autostart: - cliapp.ssh_runcmd(ssh_host, - ['virsh', '--connect', 'qemu:///system', 'autostart', vm_name]) + writeexts.ssh_runcmd(ssh_host, + ['virsh', '--connect', 'qemu:///system', + 'autostart', vm_name]) KvmPlusSshWriteExtension().run() diff --git a/extensions/nfsboot.check b/extensions/nfsboot.check index e273f61c..499fb537 100755 --- a/extensions/nfsboot.check +++ b/extensions/nfsboot.check @@ -15,33 +15,35 @@ '''Preparatory checks for Morph 'nfsboot' write extension''' -import cliapp import os +import subprocess -import morphlib.writeexts +import writeexts -class NFSBootCheckExtension(morphlib.writeexts.WriteExtension): +class NFSBootCheckExtension(writeexts.WriteExtension): _nfsboot_root = '/srv/nfsboot' def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') location = args[0] upgrade = self.get_environment_boolean('UPGRADE') if upgrade: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Upgrading is not currently supported for NFS deployments.') hostname = os.environ.get('HOSTNAME', None) if hostname is None: - raise cliapp.AppException('You must specify a HOSTNAME.') + raise writeexts.ExtensionError('You must specify a HOSTNAME.') if hostname == 'baserock': - raise cliapp.AppException('It is forbidden to nfsboot a system ' - 'with hostname "%s"' % hostname) + raise writeexts.ExtensionError('It is forbidden to nfsboot a ' + 'system with hostname "%s"' + % hostname) self.test_good_server(location) @@ -49,7 +51,7 @@ class NFSBootCheckExtension(morphlib.writeexts.WriteExtension): versioned_root = os.path.join(self._nfsboot_root, hostname, 'systems', version_label) if self.version_exists(versioned_root, location): - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Root file system for host %s (version %s) already exists on ' 'the NFS server %s. Deployment aborted.' % (hostname, version_label, location)) @@ -59,34 +61,34 @@ class NFSBootCheckExtension(morphlib.writeexts.WriteExtension): # Is an NFS server try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( 'root@%s' % server, ['test', '-e', '/etc/exports']) - except cliapp.AppException: - raise cliapp.AppException('server %s is not an nfs server' - % server) + except writeexts.ExtensionError: + raise writeexts.ExtensionError('server %s is not an nfs server' + % server) try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( 'root@%s' % server, ['systemctl', 'is-enabled', 'nfs-server.service']) - except cliapp.AppException: - raise cliapp.AppException('server %s does not control its ' - 'nfs server by systemd' % server) + except writeexts.ExtensionError: + raise writeexts.ExtensionError('server %s does not control its ' + 'nfs server by systemd' % server) # TFTP server exports /srv/nfsboot/tftp tftp_root = os.path.join(self._nfsboot_root, 'tftp') try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( 'root@%s' % server, ['test' , '-d', tftp_root]) - except cliapp.AppException: - raise cliapp.AppException('server %s does not export %s' % - (tftp_root, server)) + except writeexts.ExtensionError: + raise writeexts.ExtensionError('server %s does not export %s' % + (tftp_root, server)) def version_exists(self, versioned_root, location): try: - cliapp.ssh_runcmd('root@%s' % location, - ['test', '-d', versioned_root]) - except cliapp.AppException: + writeexts.ssh_runcmd('root@%s' % location, + ['test', '-d', versioned_root]) + except writeexts.ExtensionError: return False return True diff --git a/extensions/nfsboot.write b/extensions/nfsboot.write index d928775e..418f8eeb 100755 --- a/extensions/nfsboot.write +++ b/extensions/nfsboot.write @@ -34,14 +34,13 @@ in /srv/nfsboot/nfs/ ''' -import cliapp import os import glob -import morphlib.writeexts +import writeexts -class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): +class NFSBootWriteExtension(writeexts.WriteExtension): '''Create an NFS root and kernel on TFTP during Morph's deployment. @@ -66,7 +65,8 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') temp_root, location = args @@ -86,8 +86,8 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): subdirs = [os.path.join(statedir, 'home'), os.path.join(statedir, 'opt'), os.path.join(statedir, 'srv')] - cliapp.ssh_runcmd('root@%s' % location, - ['mkdir', '-p'] + subdirs) + writeexts.ssh_runcmd('root@%s' % location, + ['mkdir', '-p'] + subdirs) def copy_kernel(self, temp_root, location, versioned_root, version, hostname): @@ -99,14 +99,14 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): kernel_src = try_path break else: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Could not find a kernel in the system: none of ' '%s found' % ', '.join(image_names)) kernel_dest = os.path.join(versioned_root, 'orig', 'kernel') rsync_dest = 'root@%s:%s' % (location, kernel_dest) self.status(msg='Copying kernel') - cliapp.runcmd( + subprocess.check_call( ['rsync', '-s', kernel_src, rsync_dest]) # Link the kernel to the right place @@ -115,17 +115,17 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): versioned_kernel_name = "%s-%s" % (hostname, version) kernel_name = hostname try: - cliapp.ssh_runcmd('root@%s' % location, + writeexts.ssh_runcmd('root@%s' % location, ['ln', '-f', kernel_dest, os.path.join(tftp_dir, versioned_kernel_name)]) - cliapp.ssh_runcmd('root@%s' % location, + writeexts.ssh_runcmd('root@%s' % location, ['ln', '-sf', versioned_kernel_name, os.path.join(tftp_dir, kernel_name)]) - except cliapp.AppException: - raise cliapp.AppException('Could not create symlinks to the ' - 'kernel at %s in %s on %s' - % (kernel_dest, tftp_dir, location)) + except writeexts.ExtensionError: + raise writeexts.ExtensionError('Could not create symlinks to the ' + 'kernel at %s in %s on %s' % + (kernel_dest, tftp_dir, location)) def copy_rootfs(self, temp_root, location, versioned_root, hostname): rootfs_src = temp_root + '/' @@ -134,51 +134,54 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Creating destination directories') try: - cliapp.ssh_runcmd('root@%s' % location, - ['mkdir', '-p', orig_path, run_path]) - except cliapp.AppException: - raise cliapp.AppException('Could not create dirs %s and %s on %s' - % (orig_path, run_path, location)) + writeexts.ssh_runcmd('root@%s' % location, + ['mkdir', '-p', orig_path, run_path]) + except writeexts.ExtensionError: + raise writexts.ExtensionError( + 'Could not create dirs %s and %s on %s' + % (orig_path, run_path, location)) self.status(msg='Creating \'orig\' rootfs') - cliapp.runcmd( + subprocess.check_call( ['rsync', '-asXSPH', '--delete', rootfs_src, 'root@%s:%s' % (location, orig_path)]) self.status(msg='Creating \'run\' rootfs') try: - cliapp.ssh_runcmd('root@%s' % location, - ['rm', '-rf', run_path]) - cliapp.ssh_runcmd('root@%s' % location, - ['cp', '-al', orig_path, run_path]) - cliapp.ssh_runcmd('root@%s' % location, - ['rm', '-rf', os.path.join(run_path, 'etc')]) - cliapp.ssh_runcmd('root@%s' % location, - ['cp', '-a', os.path.join(orig_path, 'etc'), - os.path.join(run_path, 'etc')]) - except cliapp.AppException: - raise cliapp.AppException('Could not create \'run\' rootfs' - ' from \'orig\'') + writeexts.ssh_runcmd('root@%s' % location, + ['rm', '-rf', run_path]) + writeexts.ssh_runcmd('root@%s' % location, + ['cp', '-al', orig_path, run_path]) + writeexts.ssh_runcmd('root@%s' % location, + ['rm', '-rf', + os.path.join(run_path, 'etc')]) + writeexts.ssh_runcmd('root@%s' % location, + ['cp', '-a', + os.path.join(orig_path, 'etc'), + os.path.join(run_path, 'etc')]) + except writeexts.ExtensionError: + raise writeexts.ExtensionError('Could not create \'run\' rootfs' + ' from \'orig\'') self.status(msg='Linking \'default\' to latest system') try: - cliapp.ssh_runcmd('root@%s' % location, + writeexts.ssh_runcmd('root@%s' % location, ['ln', '-sfn', versioned_root, os.path.join(self._nfsboot_root, hostname, 'systems', 'default')]) - except cliapp.AppException: - raise cliapp.AppException('Could not link \'default\' to %s' - % versioned_root) + except writeexts.ExtensionError: + raise writeexts.ExtensionError("Could not link 'default' to %s" + % versioned_root) def configure_nfs(self, location, hostname): exported_path = os.path.join(self._nfsboot_root, hostname) exports_path = '/etc/exports' # If that path is not already exported: try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( 'root@%s' % location, ['grep', '-q', exported_path, exports_path]) - except cliapp.AppException: + except writeexts.ExtensionError: ip_mask = '*' options = 'rw,no_subtree_check,no_root_squash,async' exports_string = '%s %s(%s)\n' % (exported_path, ip_mask, options) @@ -190,11 +193,11 @@ cat "$target" > "$temp" cat >> "$temp" mv "$temp" "$target" ''' - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( 'root@%s' % location, ['sh', '-c', exports_append_sh, '--', exports_path], feed_stdin=exports_string) - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( 'root@%s' % location, ['systemctl', 'restart', 'nfs-server.service']) diff --git a/extensions/openstack.check b/extensions/openstack.check index a3379763..f3ad43b7 100755 --- a/extensions/openstack.check +++ b/extensions/openstack.check @@ -15,25 +15,26 @@ '''Preparatory checks for Morph 'openstack' write extension''' -import cliapp import os import urlparse + import keystoneclient -import morphlib.writeexts +import writeexts -class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): +class OpenStackCheckExtension(writeexts.WriteExtension): def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') self.require_btrfs_in_deployment_host_kernel() upgrade = self.get_environment_boolean('UPGRADE') if upgrade: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Use the `ssh-rsync` write extension to deploy upgrades to an ' 'existing remote system.') @@ -55,7 +56,7 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): for key in auth_keys: if os.environ.get(key, '') == '': - raise cliapp.AppException(key + ' was not given') + raise writeexts.ExtensionError(key + ' was not given') auth_params = {auth_keys[key]: os.environ[key] for key in auth_keys} auth_params['auth_url'] = location @@ -63,16 +64,17 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): def check_imagename(self): if os.environ.get('OPENSTACK_IMAGENAME', '') == '': - raise cliapp.AppException('OPENSTACK_IMAGENAME was not given') + raise writeexts.ExtensionError( + 'OPENSTACK_IMAGENAME was not given') def check_location(self, location): x = urlparse.urlparse(location) if x.scheme not in ['http', 'https']: - raise cliapp.AppException('URL schema must be http or https in %s'\ - % location) + raise writeexts.ExtensionError( + 'URL schema must be http or https in %s' % location) if (x.path != '/v2.0' and x.path != '/v2.0/'): - raise cliapp.AppException('API version must be v2.0 in %s'\ - % location) + raise writeexts.ExtensionError( + 'API version must be v2.0 in %s' % location) def check_openstack_parameters(self, auth_params): ''' Check that we can connect to and authenticate with openstack ''' @@ -84,7 +86,7 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): except keystoneclient.exceptions.Unauthorized: errmsg = ('Failed to authenticate with OpenStack ' '(are your credentials correct?)') - raise cliapp.AppException(errmsg) + raise writeexts.ExtensionError(errmsg) OpenStackCheckExtension().run() diff --git a/extensions/openstack.write b/extensions/openstack.write index 67e07c18..f1233560 100755 --- a/extensions/openstack.write +++ b/extensions/openstack.write @@ -17,21 +17,22 @@ '''A Morph deployment write extension for deploying to OpenStack.''' -import cliapp import os +import subprocess import tempfile import urlparse -import morphlib.writeexts +import writeexts -class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): +class OpenStackWriteExtension(writeexts.WriteExtension): '''See openstack.write.help for documentation''' def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') temp_root, location = args @@ -86,7 +87,7 @@ class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): '--disk-format=raw', '--container-format', 'bare', '--file', raw_disk] - cliapp.runcmd(cmdline) + subprocess.check_call(cmdline) self.status(msg='Image configured.') diff --git a/extensions/pxeboot.write b/extensions/pxeboot.write index 3a12ebcc..20e4f6bd 100644 --- a/extensions/pxeboot.write +++ b/extensions/pxeboot.write @@ -19,10 +19,7 @@ import tempfile import textwrap import urlparse -import cliapp - -import morphlib - +import writeexts def _int_to_quad_dot(i): return '.'.join(( @@ -143,7 +140,7 @@ def grouper(iterable, n, fillvalue=None): return itertools.izip_longest(*args, fillvalue=fillvalue) -class PXEBoot(morphlib.writeexts.WriteExtension): +class PXEBoot(writeexts.WriteExtension): @contextlib.contextmanager def _vlan(self, interface, vlan): viface = '%s.%s' % (interface, vlan) @@ -224,12 +221,13 @@ class PXEBoot(morphlib.writeexts.WriteExtension): @contextlib.contextmanager def _remote_tempdir(self, hostname, template): persist = os.environ.get('PXE_INSTALLER') in ('no', 'False') - td = cliapp.ssh_runcmd(hostname, ['mktemp', '-d', template]).strip() + td = writeexts.ssh_runcmd( + hostname, ['mktemp', '-d', template]).strip() try: yield td finally: if not persist: - cliapp.ssh_runcmd(hostname, ['find', td, '-delete']) + writeexts.ssh_runcmd(hostname, ['find', td, '-delete']) def _serve_tftpd(self, sock, host, port, interface, tftproot): self.settings.progname = 'tftp server' @@ -333,26 +331,27 @@ class PXEBoot(morphlib.writeexts.WriteExtension): def _remote_copy(self, hostname, src, dst): persist = os.environ.get('PXE_INSTALLER') in ('no', 'False') with open(src, 'r') as f: - cliapp.ssh_runcmd(hostname, - ['install', '-D', '-m644', '/proc/self/fd/0', - dst], stdin=f, stdout=None, stderr=None) + writeexts.ssh_runcmd(hostname, + ['install', '-D', '-m644', + '/proc/self/fd/0', dst], + stdin=f, stdout=None, stderr=None) try: yield finally: if not persist: - cliapp.ssh_runcmd(hostname, ['rm', dst]) + writeexts.ssh_runcmd(hostname, ['rm', dst]) @contextlib.contextmanager def _remote_symlink(self, hostname, src, dst): persist = os.environ.get('PXE_INSTALLER') in ('no', 'False') - cliapp.ssh_runcmd(hostname, - ['ln', '-s', '-f', src, dst], - stdin=None, stdout=None, stderr=None) + writeexts.ssh_runcmd(hostname, + ['ln', '-s', '-f', src, dst], + stdin=None, stdout=None, stderr=None) try: yield finally: if not persist: - cliapp.ssh_runcmd(hostname, ['rm', '-f', dst]) + writeexts.ssh_runcmd(hostname, ['rm', '-f', dst]) @contextlib.contextmanager def remote_kernel(self, rootfs, tftp_url, macaddr): @@ -361,7 +360,7 @@ class PXEBoot(morphlib.writeexts.WriteExtension): if os.path.exists(kernel_path): break else: - raise cliapp.AppException('Failed to locate kernel') + raise writeexts.ExtensionError('Failed to locate kernel') url = urlparse.urlsplit(tftp_url) basename = '{}-kernel'.format(_normalise_macaddr(macaddr)) target_path = os.path.join(url.path, basename) @@ -376,7 +375,8 @@ class PXEBoot(morphlib.writeexts.WriteExtension): yield fdt_abs_path = os.path.join(rootfs, fdt_rel_path) if not fdt_abs_path: - raise cliapp.AppException('Failed to locate Flattened Device Tree') + raise writeexts.ExtensionError( + 'Failed to locate Flattened Device Tree') url = urlparse.urlsplit(tftp_url) basename = '{}-fdt'.format(_normalise_macaddr(macaddr)) target_path = os.path.join(url.path, basename) @@ -389,14 +389,14 @@ class PXEBoot(morphlib.writeexts.WriteExtension): nfsroot = target_ip + ':' + rootfs self.status(msg='Exporting %(nfsroot)s as local nfsroot', nfsroot=nfsroot) - cliapp.runcmd(['exportfs', '-o', 'ro,insecure,no_root_squash', - nfsroot]) + subprocess.check_call(['exportfs', '-o', 'ro,insecure,no_root_squash', + nfsroot]) try: yield finally: self.status(msg='Removing %(nfsroot)s from local nfsroots', nfsroot=nfsroot) - cliapp.runcmd(['exportfs', '-u', nfsroot]) + subprocess.check_call(['exportfs', '-u', nfsroot]) @contextlib.contextmanager def remote_nfsroot(self, rootfs, rsync_url, macaddr): @@ -407,9 +407,10 @@ class PXEBoot(morphlib.writeexts.WriteExtension): as tempdir: nfsroot = urlparse.urlunsplit((url.scheme, url.netloc, tempdir, url.query, url.fragment)) - cliapp.runcmd(['rsync', '-asSPH', '--delete', rootfs, nfsroot], - stdin=None, stdout=open(os.devnull, 'w'), - stderr=None) + subprocess.check_call(['rsync', '-asSPH', '--delete', + rootfs, nfsroot], + stdin=None, stdout=open(os.devnull, 'w'), + stderr=None) yield os.path.join(os.path.basename(tempdir), os.path.basename(rootfs)) @@ -517,8 +518,8 @@ class PXEBoot(morphlib.writeexts.WriteExtension): def get_interface_ip(self, interface): ip_addresses = [] - info = cliapp.runcmd(['ip', '-o', '-f', 'inet', - 'addr', 'show', interface]).rstrip('\n') + info = subprocess.check_output(['ip', '-o', '-f', 'inet', 'addr', + 'show', interface]).rstrip('\n') if info: tokens = collections.deque(info.split()[1:]) ifname = tokens.popleft() @@ -535,8 +536,8 @@ class PXEBoot(morphlib.writeexts.WriteExtension): else: continue if not ip_addresses: - raise cliapp.AppException('Interface %s has no addresses' - % interface) + raise writeexts.ExtensionError('Interface %s has no addresses' + % interface) if len(ip_addresses) > 1: warnings.warn('Interface %s has multiple addresses, ' 'using first (%s)' % (interface, ip_addresses[0])) @@ -750,6 +751,6 @@ class PXEBoot(morphlib.writeexts.WriteExtension): self.wait_for_target_to_install() self.ipmi_reboot_target() else: - cliapp.AppException('Invalid PXEBOOT_MODE: %s' % mode) + writeexts.ExtensionError('Invalid PXEBOOT_MODE: %s' % mode) PXEBoot().run() diff --git a/extensions/rawdisk.check b/extensions/rawdisk.check index 9be0ce91..61619a21 100755 --- a/extensions/rawdisk.check +++ b/extensions/rawdisk.check @@ -15,17 +15,16 @@ '''Preparatory checks for Morph 'rawdisk' write extension''' -import cliapp - -import morphlib.writeexts - import os +import writeexts + -class RawdiskCheckExtension(morphlib.writeexts.WriteExtension): +class RawdiskCheckExtension(writeexts.WriteExtension): def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') self.require_btrfs_in_deployment_host_kernel() @@ -34,19 +33,19 @@ class RawdiskCheckExtension(morphlib.writeexts.WriteExtension): if upgrade: if not self.is_device(location): if not os.path.isfile(location): - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Cannot upgrade %s: it is not an existing disk image' % location) version_label = os.environ.get('VERSION_LABEL') if version_label is None: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'VERSION_LABEL was not given. It is required when ' 'upgrading an existing system.') else: if not self.is_device(location): if os.path.exists(location): - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Target %s already exists. Use `morph upgrade` if you ' 'want to update an existing image.' % location) diff --git a/extensions/rawdisk.write b/extensions/rawdisk.write index 6f2d45ba..cdeb5018 100755 --- a/extensions/rawdisk.write +++ b/extensions/rawdisk.write @@ -17,22 +17,22 @@ '''A Morph deployment write extension for raw disk images.''' -import cliapp import os import sys import time import tempfile -import morphlib.writeexts +import writeexts -class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): +class RawDiskWriteExtension(writeexts.WriteExtension): '''See rawdisk.write.help for documentation''' def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') temp_root, location = args upgrade = self.get_environment_boolean('UPGRADE') @@ -69,10 +69,10 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): old_orig = os.path.join(mp, 'systems', 'default', 'orig') new_orig = os.path.join(version_root, 'orig') - cliapp.runcmd( + subprocess.check_call( ['btrfs', 'subvolume', 'snapshot', old_orig, new_orig]) - cliapp.runcmd( + subprocess.check_call( ['rsync', '-a', '--checksum', '--numeric-ids', '--delete', temp_root + os.path.sep, new_orig]) @@ -96,11 +96,11 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): version_label = os.environ.get('VERSION_LABEL') if version_label is None: - raise cliapp.AppException('VERSION_LABEL was not given') + raise writeexts.ExtensionError('VERSION_LABEL was not given') if os.path.exists(os.path.join(mp, 'systems', version_label)): - raise cliapp.AppException('VERSION_LABEL %s already exists' - % version_label) + raise writeexts.ExtensionError('VERSION_LABEL %s already exists' + % version_label) return version_label diff --git a/extensions/simple-network.configure b/extensions/simple-network.configure index 4a70f311..61d5774d 100755 --- a/extensions/simple-network.configure +++ b/extensions/simple-network.configure @@ -25,27 +25,26 @@ for DHCP ''' +import errno import os import sys -import errno -import cliapp -import morphlib +import writeexts -class SimpleNetworkError(morphlib.Error): +class SimpleNetworkError(writeexts.ExtensionError): '''Errors associated with simple network setup''' pass -class SimpleNetworkConfigurationExtension(cliapp.Application): +class SimpleNetworkConfigurationExtension(object): '''Configure /etc/network/interfaces and generate networkd .network files Reading NETWORK_CONFIG, this extension sets up /etc/network/interfaces and .network files in /etc/systemd/network/. ''' - def process_args(self, args): + def run(self, args): network_config = os.environ.get("NETWORK_CONFIG") self.rename_networkd_chunk_file(args) @@ -206,7 +205,8 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): address_line = address + '/' + str(network_suffix) lines += ["Address=%s" % address_line] elif address or netmask: - raise Exception('address and netmask must be specified together') + raise SimpleNetworkError( + 'address and netmask must be specified together') if gateway: lines += ["Gateway=%s" % gateway] @@ -287,6 +287,10 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): ''' - self.output.write('%s\n' % (kwargs['msg'] % kwargs)) + sys.stdout.write('%s\n' % (kwargs['msg'] % kwargs)) -SimpleNetworkConfigurationExtension().run() +try: + SimpleNetworkConfigurationExtension().run(sys.argv[1:]) +except SimpleNetworkError as e: + sys.stdout.write('ERROR: %s\n' % e) + sys.exit(1) diff --git a/extensions/ssh-rsync.check b/extensions/ssh-rsync.check index c3bdfd29..436aaae0 100755 --- a/extensions/ssh-rsync.check +++ b/extensions/ssh-rsync.check @@ -15,26 +15,27 @@ '''Preparatory checks for Morph 'ssh-rsync' write extension''' -import cliapp import os -import morphlib.writeexts +import writeexts -class SshRsyncCheckExtension(morphlib.writeexts.WriteExtension): + +class SshRsyncCheckExtension(writeexts.WriteExtension): def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') upgrade = self.get_environment_boolean('UPGRADE') if not upgrade: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'The ssh-rsync write is for upgrading existing remote ' 'Baserock machines. It cannot be used for an initial ' 'deployment.') if os.environ.get('VERSION_LABEL', '') == '': - raise cliapp.AppException( + raise writeexts.ExtensionError( 'A VERSION_LABEL must be set when deploying an upgrade.') location = args[0] @@ -47,17 +48,18 @@ class SshRsyncCheckExtension(morphlib.writeexts.WriteExtension): self.check_command_exists(location, 'rsync') def check_is_baserock_system(self, location): - output = cliapp.ssh_runcmd(location, ['sh', '-c', - 'test -d /baserock || echo -n dirnotfound']) + output = writeexts.ssh_runcmd( + location, + ['sh', '-c', 'test -d /baserock || echo -n dirnotfound']) if output == 'dirnotfound': - raise cliapp.AppException('%s is not a baserock system' - % location) + raise writeexts.ExtensionError('%s is not a baserock system' + % location) def check_command_exists(self, location, command): test = 'type %s > /dev/null 2>&1 || echo -n cmdnotfound' % command - output = cliapp.ssh_runcmd(location, ['sh', '-c', test]) + output = writeexts.ssh_runcmd(location, ['sh', '-c', test]) if output == 'cmdnotfound': - raise cliapp.AppException( + raise writeexts.ExtensionError( "%s does not have %s" % (location, command)) diff --git a/extensions/ssh-rsync.write b/extensions/ssh-rsync.write index 6d596500..46c16662 100755 --- a/extensions/ssh-rsync.write +++ b/extensions/ssh-rsync.write @@ -18,23 +18,23 @@ import contextlib -import cliapp import os +import subprocess import sys -import time import tempfile +import time -import morphlib.writeexts +import writeexts def ssh_runcmd_ignore_failure(location, command, **kwargs): try: - return cliapp.ssh_runcmd(location, command, **kwargs) - except cliapp.AppException: + return writeexts.ssh_runcmd(location, command, **kwargs) + except writeexts.ExtensionError: pass -class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): +class SshRsyncWriteExtension(writeexts.WriteExtension): '''See ssh-rsync.write.help for documentation''' @@ -43,7 +43,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): '''Read /proc/mounts on location to find which device contains "/"''' self.status(msg='Finding device that contains "/"') - contents = cliapp.ssh_runcmd(location, ['cat', '/proc/mounts']) + contents = writeexts.ssh_runcmd(location, + ['cat', '/proc/mounts']) for line in contents.splitlines(): line_words = line.split() if (line_words[1] == '/' and line_words[0] != 'rootfs'): @@ -52,28 +53,29 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): @contextlib.contextmanager def _remote_mount_point(self, location): self.status(msg='Creating remote mount point') - remote_mnt = cliapp.ssh_runcmd(location, ['mktemp', '-d']).strip() + remote_mnt = writeexts.ssh_runcmd(location, + ['mktemp', '-d']).strip() try: yield remote_mnt finally: self.status(msg='Removing remote mount point') - cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) + writeexts.ssh_runcmd(location, ['rmdir', remote_mnt]) @contextlib.contextmanager def _remote_mount(self, location, root_disk, mountpoint): self.status(msg='Mounting root disk') - cliapp.ssh_runcmd(location, ['mount', root_disk, mountpoint]) + writeexts.ssh_runcmd(location, ['mount', root_disk, mountpoint]) try: yield finally: self.status(msg='Unmounting root disk') - cliapp.ssh_runcmd(location, ['umount', mountpoint]) + writeexts.ssh_runcmd(location, ['umount', mountpoint]) @contextlib.contextmanager def _created_version_root(self, location, remote_mnt, version_label): version_root = os.path.join(remote_mnt, 'systems', version_label) self.status(msg='Creating %(root)s', root=version_root) - cliapp.ssh_runcmd(location, ['mkdir', version_root]) + writeexts.ssh_runcmd(location, ['mkdir', version_root]) try: yield version_root except BaseException as e: @@ -93,8 +95,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Creating "orig" subvolume') old_orig = self.get_old_orig(location, remote_mnt) new_orig = os.path.join(version_root, 'orig') - cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', 'snapshot', - old_orig, new_orig]) + writeexts.ssh_runcmd(location, ['btrfs', 'subvolume', 'snapshot', + old_orig, new_orig]) try: yield new_orig except BaseException as e: @@ -106,30 +108,30 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): '''Populate the subvolume version_root/orig on location''' self.status(msg='Populating "orig" subvolume') - cliapp.runcmd(['rsync', '-as', '--checksum', '--numeric-ids', - '--delete', temp_root + os.path.sep, - '%s:%s' % (location, new_orig)]) + subprocess.check_call(['rsync', '-as', '--checksum', '--numeric-ids', + '--delete', temp_root + os.path.sep, + '%s:%s' % (location, new_orig)]) @contextlib.contextmanager def _deployed_version(self, location, version_label, system_config_sync, system_version_manager): self.status(msg='Calling system-version-manager to deploy upgrade') deployment = os.path.join('/systems', version_label, 'orig') - cliapp.ssh_runcmd(location, + writeexts.ssh_runcmd(location, ['env', 'BASEROCK_SYSTEM_CONFIG_SYNC='+system_config_sync, system_version_manager, 'deploy', deployment]) try: yield deployment except BaseException as e: self.status(msg='Cleaning up failed version installation') - cliapp.ssh_runcmd(location, + writeexts.ssh_runcmd(location, [system_version_manager, 'remove', version_label]) raise def upgrade_remote_system(self, location, temp_root): root_disk = self.find_root_disk(location) - uuid = cliapp.ssh_runcmd(location, ['blkid', '-s', 'UUID', '-o', - 'value', root_disk]).strip() + uuid = writeexts.ssh_runcmd(location, + ['blkid', '-s', 'UUID', '-o', 'value', root_disk]).strip() self.complete_fstab_for_btrfs_layout(temp_root, uuid) @@ -153,8 +155,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): config_sync, version_manager): self.status(msg='Setting %(v)s as the new default system', v=version_label) - cliapp.ssh_runcmd(location, [version_manager, - 'set-default', version_label]) + writeexts.ssh_runcmd(location, + [version_manager, 'set-default', version_label]) if autostart: self.status(msg="Rebooting into new system ...") @@ -162,7 +164,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') temp_root, location = args diff --git a/extensions/strip-gplv3.configure b/extensions/strip-gplv3.configure index c08061ad..0585fe5d 100755 --- a/extensions/strip-gplv3.configure +++ b/extensions/strip-gplv3.configure @@ -21,12 +21,14 @@ to find the files created by that chunk, then remove them. ''' -import cliapp -import re -import os import json +import os +import re +import sys + +import writeexts -class StripGPLv3ConfigureExtension(cliapp.Application): +class StripGPLv3ConfigureExtension(object): gplv3_chunks = [ ['autoconf', ''], ['automake', ''], @@ -51,7 +53,7 @@ class StripGPLv3ConfigureExtension(cliapp.Application): ['texinfo-tarball', ''], ] - def process_args(self, args): + def run(self, args): target_root = args[0] meta_dir = os.path.join(target_root, 'baserock') @@ -72,8 +74,8 @@ class StripGPLv3ConfigureExtension(cliapp.Application): chunk_meta_data = json.load(f) if not 'contents' in chunk_meta_data: - raise cliapp.AppError('Chunk %s does not have a "contents" list' - % chunk) + raise writeexts.ExtensionError( + 'Chunk %s does not have a "contents" list' % chunk) updated_contents = [] for content_entry in reversed(chunk_meta_data['contents']): pat = re.compile(pattern) @@ -85,8 +87,8 @@ class StripGPLv3ConfigureExtension(cliapp.Application): def remove_content_entry(self, target_root, content_entry): entry_path = os.path.join(target_root, './' + content_entry) if not entry_path.startswith(target_root): - raise cliapp.AppException('%s is not in %s' - % (entry_path, target_root)) + raise writeexts.ExtensionError( + '%s is not in %s' % (entry_path, target_root)) if os.path.exists(entry_path): if os.path.islink(entry_path): os.unlink(entry_path) @@ -96,6 +98,11 @@ class StripGPLv3ConfigureExtension(cliapp.Application): if not os.listdir(entry_path): os.rmdir(entry_path) else: - raise cliapp.AppException('%s is not a link, file or directory' - % entry_path) -StripGPLv3ConfigureExtension().run() + raise writeexts.ExtensionError( + '%s is not a link, file or directory' % entry_path) + +try: + StripGPLv3ConfigureExtension().run(sys.argv[1:]) +except writeexts.ExtensionError as e: + sys.stdout.write('ERROR: %s' % e) + sys.exit(1) diff --git a/extensions/virtualbox-ssh.check b/extensions/virtualbox-ssh.check index a97f3294..e82d58a1 100755 --- a/extensions/virtualbox-ssh.check +++ b/extensions/virtualbox-ssh.check @@ -15,21 +15,21 @@ '''Preparatory checks for Morph 'virtualbox-ssh' write extension''' -import cliapp -import morphlib.writeexts +import writeexts -class VirtualBoxPlusSshCheckExtension(morphlib.writeexts.WriteExtension): +class VirtualBoxPlusSshCheckExtension(writeexts.WriteExtension): def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') self.require_btrfs_in_deployment_host_kernel() upgrade = self.get_environment_boolean('UPGRADE') if upgrade: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Use the `ssh-rsync` write extension to deploy upgrades to an ' 'existing remote system.') diff --git a/extensions/virtualbox-ssh.write b/extensions/virtualbox-ssh.write index 774f2b4f..95643a4a 100755 --- a/extensions/virtualbox-ssh.write +++ b/extensions/virtualbox-ssh.write @@ -24,22 +24,23 @@ See file virtualbox-ssh.write.help for documentation ''' -import cliapp import os import re +import subprocess import sys -import time import tempfile +import time import urlparse -import morphlib.writeexts +import writeexts -class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): +class VirtualBoxPlusSshWriteExtension(writeexts.WriteExtension): def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') temp_root, location = args ssh_host, vm_name, vdi_path = self.parse_location(location) @@ -59,7 +60,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): except BaseException: sys.stderr.write('Error deploying to VirtualBox') os.remove(raw_disk) - cliapp.ssh_runcmd(ssh_host, ['rm', '-f', vdi_path]) + writeexts.ssh_runcmd(ssh_host, ['rm', '-f', vdi_path]) raise else: os.remove(raw_disk) @@ -72,11 +73,12 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): x = urlparse.urlparse(location) if x.scheme != 'vbox+ssh': - raise cliapp.AppException( + raise writeexts.ExtensionError( 'URL schema must be vbox+ssh in %s' % location) m = re.match('^/(?P[^/]+)(?P/.+)$', x.path) if not m: - raise cliapp.AppException('Cannot parse location %s' % location) + raise writeexts.ExtensionError( + 'Cannot parse location %s' % location) return x.netloc, m.group('guest'), m.group('path') def transfer_and_convert_to_vdi(self, raw_disk, ssh_host, vdi_path): @@ -85,17 +87,18 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Transfer disk and convert to VDI') st = os.lstat(raw_disk) - xfer_hole_path = morphlib.util.get_data_path('xfer-hole') - recv_hole = morphlib.util.get_data('recv-hole') + # TODO: Something! + xfer_hole_path = writeexts.get_data_path('xfer-hole') + recv_hole = writeexts.get_data('recv-hole') ssh_remote_cmd = [ 'sh', '-c', recv_hole, 'dummy-argv0', 'vbox', vdi_path, str(st.st_size), ] - cliapp.runcmd( + subprocess.check_call( ['python', xfer_hole_path, raw_disk], - ['ssh', ssh_host] + map(cliapp.shell_quote, ssh_remote_cmd), + ['ssh', ssh_host] + map(writeexts.shell_quote, ssh_remote_cmd), stdout=None, stderr=None) def virtualbox_version(self, ssh_host): @@ -107,7 +110,8 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): # tuple is more reliable than a string and more convenient than # comparing against the major, minor and patch numbers directly self.status(msg='Checking version of remote VirtualBox') - build_id = cliapp.ssh_runcmd(ssh_host, ['VBoxManage', '--version']) + build_id = writeexts.ssh_runcmd(ssh_host, + ['VBoxManage', '--version']) version_string = re.match(r"^([0-9\.]+).*$", build_id.strip()).group(1) return tuple(int(s or '0') for s in version_string.split('.')) @@ -163,17 +167,17 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): for command in commands: argv = ['VBoxManage'] + command - cliapp.ssh_runcmd(ssh_host, argv) + writeexts.ssh_runcmd(ssh_host, argv) def get_host_interface(self, ssh_host): host_ipaddr = os.environ.get('HOST_IPADDR') netmask = os.environ.get('NETMASK') if host_ipaddr is None: - raise cliapp.AppException('HOST_IPADDR was not given') + raise writeexts.ExtensionError('HOST_IPADDR was not given') if netmask is None: - raise cliapp.AppException('NETMASK was not given') + raise writeexts.ExtensionError('NETMASK was not given') # 'VBoxManage list hostonlyifs' retrieves a list with the hostonly # interfaces on the host. For each interface, the following lines @@ -187,7 +191,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): # The following command tries to retrieve the hostonly interface # name (e.g. vboxnet0) associated with the given ip address. iface = None - lines = cliapp.ssh_runcmd(ssh_host, + lines = writeexts.ssh_runcmd(ssh_host, ['VBoxManage', 'list', 'hostonlyifs']).splitlines() for i, v in enumerate(lines): if host_ipaddr in v: @@ -195,12 +199,12 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): break if iface is None: - iface = cliapp.ssh_runcmd(ssh_host, + iface = writeexts.ssh_runcmd(ssh_host, ['VBoxManage', 'hostonlyif', 'create']) # 'VBoxManage hostonlyif create' shows the name of the # created hostonly interface inside single quotes iface = iface[iface.find("'") + 1 : iface.rfind("'")] - cliapp.ssh_runcmd(ssh_host, + writeexts.ssh_runcmd(ssh_host, ['VBoxManage', 'hostonlyif', 'ipconfig', iface, '--ip', host_ipaddr, diff --git a/extensions/writeexts.py b/extensions/writeexts.py index 9357648f..2f2c896b 100644 --- a/extensions/writeexts.py +++ b/extensions/writeexts.py @@ -13,27 +13,112 @@ # with this program. If not, see . -import cliapp +import contextlib +import errno +import fcntl import logging import os +import pipes import re +import select import shutil +import stat +import subprocess import sys import time import tempfile -import errno -import stat -import contextlib -@contextlib.contextmanager -def hide_password_environment_variables(env): # pragma: no cover - password_env = { k:v for k,v in env.iteritems() if 'PASSWORD' in k } - for k in password_env: - env[k] = '(value hidden)' - yield - for k, v in password_env.iteritems(): - env[k] = v +def get_data_path(relative_path): + extensions_dir = os.path.dirname(__file__) + return os.path.join(extensions_dir, relative_path) + + +def get_data(relative_path): + with open(get_data_path(relative_path)) as f: + return f.read() + + +def ssh_runcmd(host, args, **kwargs): + '''Run command over ssh''' + command = ['ssh', host, '--'] + [pipes.quote(arg) for arg in args] + + feed_stdin = kwargs.get('feed_stdin') + stdin = kwargs.get('stdin', subprocess.PIPE) + stdout = kwargs.get('stdout', subprocess.PIPE) + stderr = kwargs.get('stderr', subprocess.PIPE) + + p = subprocess.Popen(command, stdin=stdin, stdout=stdout, stderr=stderr) + out, err = p.communicate(input=feed_stdin) + if p.returncode != 0: + raise ExtensionError('ssh command `%s` failed' % ' '.join(command)) + return out + + +def write_from_dict(filepath, d, validate=lambda x, y: True): + """Takes a dictionary and appends the contents to a file + + An optional validation callback can be passed to perform validation on + each value in the dictionary. + + e.g. + + def validation_callback(dictionary_key, dictionary_value): + if not dictionary_value.isdigit(): + raise Exception('value contains non-digit character(s)') + + Any callback supplied to this function should raise an exception + if validation fails. + + """ + # Sort items asciibetically + # the output of the deployment should not depend + # on the locale of the machine running the deployment + items = sorted(d.iteritems(), key=lambda (k, v): [ord(c) for c in v]) + + for (k, v) in items: + validate(k, v) + + with open(filepath, 'a') as f: + for (_, v) in items: + f.write('%s\n' % v) + + os.fchown(f.fileno(), 0, 0) + os.fchmod(f.fileno(), 0644) + + +def parse_environment_pairs(env, pairs): + '''Add key=value pairs to the environment dict. + + Given a dict and a list of strings of the form key=value, + set dict[key] = value, unless key is already set in the + environment, at which point raise an exception. + + This does not modify the passed in dict. + + Returns the extended dict. + + ''' + extra_env = dict(p.split('=', 1) for p in pairs) + conflicting = [k for k in extra_env if k in env] + if conflicting: + raise ExtensionError('Environment already set: %s' + % ', '.join(conflicting)) + + # Return a dict that is the union of the two + # This is not the most performant, since it creates + # 3 unnecessary lists, but I felt this was the most + # easy to read. Using itertools.chain may be more efficicent + return dict(env.items() + extra_env.items()) + + +class ExtensionError(Exception): + + def __init__(self, msg): + self.msg = msg + + def __str__(self): + return self.msg class Fstab(object): @@ -89,9 +174,9 @@ class Fstab(object): shutil.move(os.path.abspath(tmp), os.path.abspath(self.filepath)) -class WriteExtension(cliapp.Application): +class Extension(object): - '''A base class for deployment write extensions. + '''A base class for deployment extensions. A subclass should subclass this class, and add a ``process_args`` method. @@ -108,8 +193,6 @@ class WriteExtension(cliapp.Application): This file descriptor is read by Morph and written into its own log file. - This overrides cliapp's usual configurable logging setup. - ''' log_write_fd = int(os.environ.get('MORPH_LOG_FD', 0)) @@ -125,13 +208,19 @@ class WriteExtension(cliapp.Application): logger.addHandler(handler) logger.setLevel(logging.DEBUG) - def log_config(self): - with hide_password_environment_variables(os.environ): - cliapp.Application.log_config(self) - def process_args(self, args): raise NotImplementedError() + def run(self, args=None): + if args is None: + args = sys.argv[1:] + try: + self.setup_logging() + self.process_args(args) + except ExtensionError as e: + sys.stdout.write('ERROR: %s\n' % e) + sys.exit(1) + def status(self, **kwargs): '''Provide status output. @@ -140,9 +229,22 @@ class WriteExtension(cliapp.Application): by %. ''' + sys.stdout.write('%s\n' % (kwargs['msg'] % kwargs)) + sys.stdout.flush() - self.output.write('%s\n' % (kwargs['msg'] % kwargs)) - self.output.flush() + +class WriteExtension(Extension): + + '''A base class for deployment write extensions. + + A subclass should subclass this class, and add a + ``process_args`` method. + + Note that it is not necessary to subclass this class for write + extensions. This class is here just to collect common code for + write extensions. + + ''' def check_for_btrfs_in_deployment_host_kernel(self): with open('/proc/filesystems') as f: @@ -151,7 +253,7 @@ class WriteExtension(cliapp.Application): def require_btrfs_in_deployment_host_kernel(self): if not self.check_for_btrfs_in_deployment_host_kernel(): - raise cliapp.AppException( + raise ExtensionError( 'Error: Btrfs is required for this deployment, but was not ' 'detected in the kernel of the machine that is running Morph.') @@ -166,7 +268,7 @@ class WriteExtension(cliapp.Application): def created_disk_image(self, location): size = self.get_disk_size() if not size: - raise cliapp.AppException('DISK_SIZE is not defined') + raise ExtensionError('DISK_SIZE is not defined') self.create_raw_disk_image(location, size) try: yield @@ -220,8 +322,8 @@ class WriteExtension(cliapp.Application): return None bytes = self._parse_size(size) if bytes is None: - raise cliapp.AppException('Cannot parse %s value %s' - % (env_var, size)) + raise ExtensionError('Cannot parse %s value %s' + % (env_var, size)) return bytes def get_disk_size(self): @@ -254,15 +356,15 @@ class WriteExtension(cliapp.Application): # need to do this because at the time of writing, SYSLINUX has not # been updated to understand these new features and will fail to # boot if the kernel is on a filesystem where they are enabled. - cliapp.runcmd( + subprocess.check_output( ['mkfs.btrfs','-f', '-L', 'baserock', '--features', '^extref', '--features', '^skinny-metadata', '--features', '^mixed-bg', '--nodesize', '4096', location]) - except cliapp.AppException as e: - if 'unrecognized option \'--features\'' in e.msg: + except subprocess.CalledProcessError as e: + if 'unrecognized option \'--features\'' in e.output: # Old versions of mkfs.btrfs (including v0.20, present in many # Baserock releases) don't support the --features option, but # also don't enable the new features by default. So we can @@ -270,7 +372,8 @@ class WriteExtension(cliapp.Application): logging.debug( 'Assuming mkfs.btrfs failure was because the tool is too ' 'old to have --features flag.') - cliapp.runcmd(['mkfs.btrfs','-f', '-L', 'baserock', location]) + subprocess.check_call(['mkfs.btrfs','-f', + '-L', 'baserock', location]) else: raise @@ -278,8 +381,8 @@ class WriteExtension(cliapp.Application): '''Get the UUID of a block device's file system.''' # Requires util-linux blkid; busybox one ignores options and # lies by exiting successfully. - return cliapp.runcmd(['blkid', '-s', 'UUID', '-o', 'value', - location]).strip() + return subprocess.check_output(['blkid', '-s', 'UUID', '-o', 'value', + location]).strip() @contextlib.contextmanager def mount(self, location): @@ -287,9 +390,10 @@ class WriteExtension(cliapp.Application): try: mount_point = tempfile.mkdtemp() if self.is_device(location): - cliapp.runcmd(['mount', location, mount_point]) + subprocess.check_call(['mount', location, mount_point]) else: - cliapp.runcmd(['mount', '-o', 'loop', location, mount_point]) + subprocess.check_call(['mount', '-o', 'loop', + location, mount_point]) except BaseException as e: sys.stderr.write('Error mounting filesystem') os.rmdir(mount_point) @@ -298,7 +402,7 @@ class WriteExtension(cliapp.Application): yield mount_point finally: self.status(msg='Unmounting filesystem') - cliapp.runcmd(['umount', mount_point]) + subprocess.check_call(['umount', mount_point]) os.rmdir(mount_point) def create_btrfs_system_layout(self, temp_root, mountpoint, version_label, @@ -345,9 +449,9 @@ class WriteExtension(cliapp.Application): orig = os.path.join(version_root, 'orig') self.status(msg='Creating orig subvolume') - cliapp.runcmd(['btrfs', 'subvolume', 'create', orig]) + subprocess.check_call(['btrfs', 'subvolume', 'create', orig]) self.status(msg='Copying files to orig subvolume') - cliapp.runcmd(['cp', '-a', temp_root + '/.', orig + '/.']) + subprocess.check_call(['cp', '-a', temp_root + '/.', orig + '/.']) def create_run(self, version_root): '''Create the 'run' snapshot.''' @@ -355,7 +459,7 @@ class WriteExtension(cliapp.Application): self.status(msg='Creating run subvolume') orig = os.path.join(version_root, 'orig') run = os.path.join(version_root, 'run') - cliapp.runcmd( + subprocess.check_call( ['btrfs', 'subvolume', 'snapshot', orig, run]) def create_state_subvolume(self, system_dir, mountpoint, state_subdir): @@ -369,7 +473,7 @@ class WriteExtension(cliapp.Application): ''' self.status(msg='Creating %s subvolume' % state_subdir) subvolume = os.path.join(mountpoint, 'state', state_subdir) - cliapp.runcmd(['btrfs', 'subvolume', 'create', subvolume]) + subprocess.check_call(['btrfs', 'subvolume', 'create', subvolume]) os.chmod(subvolume, 0o755) existing_state_dir = os.path.join(system_dir, state_subdir) @@ -380,7 +484,7 @@ class WriteExtension(cliapp.Application): self.status(msg='Moving existing data to %s subvolume' % subvolume) for filename in files: filepath = os.path.join(existing_state_dir, filename) - cliapp.runcmd(['mv', filepath, subvolume]) + subprocess.check_call(['mv', filepath, subvolume]) def complete_fstab_for_btrfs_layout(self, system_dir, rootfs_uuid=None): '''Fill in /etc/fstab entries for the default Btrfs disk layout. @@ -430,8 +534,8 @@ class WriteExtension(cliapp.Application): if 'INITRAMFS_PATH' in os.environ: initramfs = os.path.join(temp_root, os.environ['INITRAMFS_PATH']) if not os.path.exists(initramfs): - raise cliapp.AppException('INITRAMFS_PATH specified, ' - 'but file does not exist') + raise ExtensionError('INITRAMFS_PATH specified, ' + 'but file does not exist') return initramfs return None @@ -443,7 +547,7 @@ class WriteExtension(cliapp.Application): ''' self.status(msg='Installing initramfs') initramfs_dest = os.path.join(version_root, 'initramfs') - cliapp.runcmd(['cp', '-a', initramfs_path, initramfs_dest]) + subprocess.check_call(['cp', '-a', initramfs_path, initramfs_dest]) def install_kernel(self, version_root, temp_root): '''Install the kernel outside of 'orig' or 'run' subvolumes''' @@ -454,7 +558,7 @@ class WriteExtension(cliapp.Application): for name in image_names: try_path = os.path.join(temp_root, 'boot', name) if os.path.exists(try_path): - cliapp.runcmd(['cp', '-a', try_path, kernel_dest]) + subprocess.check_call(['cp', '-a', try_path, kernel_dest]) break def install_dtb(self, version_root, temp_root): @@ -465,10 +569,10 @@ class WriteExtension(cliapp.Application): dtb_dest = os.path.join(version_root, 'dtb') try_path = os.path.join(temp_root, device_tree_path) if os.path.exists(try_path): - cliapp.runcmd(['cp', '-a', try_path, dtb_dest]) + subprocess.check_call(['cp', '-a', try_path, dtb_dest]) else: logging.error("Failed to find device tree %s", device_tree_path) - raise cliapp.AppException( + raise ExtensionError( 'Failed to find device tree %s' % device_tree_path) def get_dtb_path(self): @@ -500,7 +604,7 @@ class WriteExtension(cliapp.Application): if config_type in config_function_dict: config_function_dict[config_type](real_root, disk_uuid) else: - raise cliapp.AppException( + raise ExtensionError( 'Invalid BOOTLOADER_CONFIG_FORMAT %s' % config_type) def generate_extlinux_config(self, real_root, disk_uuid=None): @@ -544,15 +648,15 @@ class WriteExtension(cliapp.Application): if install_type in install_function_dict: install_function_dict[install_type](real_root) elif install_type != 'none': - raise cliapp.AppException( + raise ExtensionError( 'Invalid BOOTLOADER_INSTALL %s' % install_type) def install_bootloader_extlinux(self, real_root): self.status(msg='Installing extlinux') - cliapp.runcmd(['extlinux', '--install', real_root]) + subprocess.check_call(['extlinux', '--install', real_root]) # FIXME this hack seems to be necessary to let extlinux finish - cliapp.runcmd(['sync']) + subprocess.check_call(['sync']) time.sleep(2) def install_syslinux_menu(self, real_root, version_root): @@ -610,19 +714,19 @@ class WriteExtension(cliapp.Application): elif value in ['yes', '1', 'true']: return True else: - raise cliapp.AppException('Unexpected value for %s: %s' % - (variable, value)) + raise ExtensionError('Unexpected value for %s: %s' % + (variable, value)) def check_ssh_connectivity(self, ssh_host): try: - output = cliapp.ssh_runcmd(ssh_host, ['echo', 'test']) - except cliapp.AppException as e: + output = ssh_runcmd(ssh_host, ['echo', 'test']) + except ExtensionError as e: logging.error("Error checking SSH connectivity: %s", str(e)) - raise cliapp.AppException( + raise ExtensionError( 'Unable to SSH to %s: %s' % (ssh_host, e)) if output.strip() != 'test': - raise cliapp.AppException( + raise ExtensionError( 'Unexpected output from remote machine: %s' % output.strip()) def is_device(self, location): -- cgit v1.2.1