diff options
| author | Adam Coldrick <adam.coldrick@codethink.co.uk> | 2015-06-04 15:17:44 +0000 |
|---|---|---|
| committer | Adam Coldrick <adam.coldrick@codethink.co.uk> | 2015-06-10 08:04:53 +0000 |
| commit | ab90762d9a695118e7a89690c2f3b8c15e247a76 (patch) | |
| tree | b4227a6977d6e85a1830d1d97fa8087d9140dc94 | |
| parent | 20dfa465dd7a06c41947002b82ebb318168c18b2 (diff) | |
| download | definitions-baserock/adamcoldrick/remove-dependencies.tar.gz | |
Remove dependencies on morphlib and cliapp from deployment extensionsbaserock/adamcoldrick/remove-dependencies
This is done by either copying or reimplementing relevant parts
of morphlib/cliapp in extensions/writeexts.py, or by using
functionality from the Python standard library instead where
appropriate.
Note that this means that these extensions will require
"$definitions_checkout/extensions" in PYTHONPATH when they are
run.
This commit also updates VERSION to 6, since the PYTHONPATH
requirement means that this change is incompatible with old
versions of morph.
Change-Id: Iec6fa7e3c7219619ce55e18493e5c37c36e97816
26 files changed, 695 insertions, 367 deletions
@@ -1 +1 @@ -version: 3 +version: 5 diff --git a/extensions/ceph.configure b/extensions/ceph.configure index c3cd92d1..3b8b2603 100644 --- a/extensions/ceph.configure +++ b/extensions/ceph.configure @@ -14,13 +14,14 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -import cliapp -import sys import os -import subprocess import shutil -import re import stat +import subprocess +import sys +import re + +import writeexts systemd_monitor_template = """ [Unit] @@ -75,7 +76,7 @@ executable_file_permissions = stat.S_IRUSR | stat.S_IXUSR | stat.S_IWUSR | \ stat.S_IXGRP | stat.S_IRGRP | \ stat.S_IXOTH | stat.S_IROTH -class CephConfigurationExtension(cliapp.Application): +class CephConfigurationExtension(writeexts.Extension): """ Set up ceph server daemons. diff --git a/extensions/distbuild-trove-nfsboot.check b/extensions/distbuild-trove-nfsboot.check index 38c491e5..6dc5efc8 100755 --- a/extensions/distbuild-trove-nfsboot.check +++ b/extensions/distbuild-trove-nfsboot.check @@ -15,14 +15,15 @@ '''Preparatory checks for Morph 'distbuild-trove-nfsboot' write extension''' -import cliapp import logging import os +import subprocess +import sys -import morphlib.writeexts +import writeexts -class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): +class DistbuildTroveNFSBootCheckExtension(writeexts.WriteExtension): nfsboot_root = '/srv/nfsboot' remote_user = 'root' @@ -45,7 +46,8 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') nfs_host = args[0] nfs_netloc = '%s@%s' % (self.remote_user, nfs_host) @@ -55,17 +57,19 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): missing_vars = [var for var in self.required_vars if not var in os.environ] if missing_vars: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Please set: %s' % ', '.join(missing_vars)) controllers = os.getenv('DISTBUILD_CONTROLLER').split() workers = os.getenv('DISTBUILD_WORKERS').split() if len(controllers) != 1: - raise cliapp.AppException('Please specify exactly one controller.') + raise writeexts.ExtensionError( + 'Please specify exactly one controller.') if len(workers) == 0: - raise cliapp.AppException('Please specify at least one worker.') + raise writeexts.ExtensionError( + 'Please specify at least one worker.') upgrade = self.get_environment_boolean('UPGRADE') @@ -80,7 +84,7 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): if self.remote_directory_exists(nfs_netloc, system_path): if self.get_environment_boolean('OVERWRITE') == False: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'System %s already exists at %s:%s. Try `morph ' 'upgrade` instead of `morph deploy`.' % ( system_name, nfs_netloc, system_path)) @@ -91,27 +95,27 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): # Is an NFS server try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( netloc, ['test', '-e', '/etc/exports']) - except cliapp.AppException: - raise cliapp.AppException('server %s is not an nfs server' - % netloc) + except subprocess.CalledProcessError: + raise writeexts.ExtensionError('server %s is not an nfs server' + % netloc) try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( netloc, ['systemctl', 'is-enabled', 'nfs-server.service']) - except cliapp.AppException: - raise cliapp.AppException('server %s does not control its ' - 'nfs server by systemd' % netloc) + except subprocess.CalledProcessError: + raise writeexts.ExtensionError('server %s does not control its ' + 'nfs server by systemd' % netloc) # TFTP server exports /srv/nfsboot/tftp tftp_root = os.path.join(self.nfsboot_root, 'tftp') try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( netloc, ['test' , '-d', tftp_root]) - except cliapp.AppException: - raise cliapp.AppException('server %s does not export %s' % - (netloc, tftp_root)) + except subprocess.CalledProcessError: + raise writeexts.ExtensionError('server %s does not export %s' % + (netloc, tftp_root)) def check_upgradeable(self, nfs_netloc, system_name, version_label): '''Check that there is already a version of the system present. @@ -124,7 +128,7 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): system_version_path = self.system_path(system_name, version_label) if not self.remote_directory_exists(nfs_netloc, system_path): - raise cliapp.AppException( + raise writeexts.ExtensionError( 'System %s not found at %s:%s, cannot deploy an upgrade.' % ( system_name, nfs_netloc, system_path)) @@ -132,15 +136,15 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): if self.get_environment_boolean('OVERWRITE'): pass else: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'System %s version %s already exists at %s:%s.' % ( system_name, version_label, nfs_netloc, system_version_path)) def remote_directory_exists(self, nfs_netloc, path): try: - cliapp.ssh_runcmd(nfs_netloc, ['test', '-d', path]) - except cliapp.AppException as e: + writeexts.ssh_runcmd(nfs_netloc, ['test', '-d', path]) + except subprocess.CalledProcessError as e: logging.debug('SSH exception: %s', e) return False diff --git a/extensions/distbuild-trove-nfsboot.write b/extensions/distbuild-trove-nfsboot.write index a5a5b094..e7d90c6c 100755 --- a/extensions/distbuild-trove-nfsboot.write +++ b/extensions/distbuild-trove-nfsboot.write @@ -20,14 +20,14 @@ import os +import subprocess import sys import tempfile -import cliapp -import morphlib.writeexts +import writeexts -class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): +class DistbuildTroveNFSBootWriteExtension(writeexts.WriteExtension): '''Create an NFS root and kernel on TFTP during Morph's deployment. @@ -54,7 +54,7 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError('Wrong number of command line args') local_system_path, nfs_host = args @@ -111,17 +111,17 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): ''' pairs = host_map_string.split(' ') - return morphlib.util.parse_environment_pairs({}, pairs) + return writeexts.parse_environment_pairs({}, pairs) def transfer_system(self, nfs_netloc, local_system_path, remote_system_path): self.status(msg='Copying rootfs to %(nfs_netloc)s', nfs_netloc=nfs_netloc) - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( nfs_netloc, ['mkdir', '-p', remote_system_path]) # The deployed rootfs may have been created by OSTree, so definitely # don't pass --hard-links to `rsync`. - cliapp.runcmd( + subprocess.check_call( ['rsync', '--archive', '--delete', '--info=progress2', '--protect-args', '--partial', '--sparse', '--xattrs', local_system_path + '/', @@ -131,13 +131,13 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): target_system_path): self.status(msg='Duplicating rootfs to %(target_system_path)s', target_system_path=target_system_path) - cliapp.ssh_runcmd(nfs_netloc, + writeexts.ssh_runcmd(nfs_netloc, ['mkdir', '-p', target_system_path]) # We can't pass --info=progress2 here, because it may not be available # in the remote 'rsync'. The --info setting was added in RSync 3.1.0, # old versions of Baserock have RSync 3.0.9. So the user doesn't get # any progress info on stdout for the 'duplicate' stage. - cliapp.ssh_runcmd(nfs_netloc, + writeexts.ssh_runcmd(nfs_netloc, ['rsync', '--archive', '--delete', '--protect-args', '--partial', '--sparse', '--xattrs', source_system_path + '/', target_system_path], stdout=sys.stdout) @@ -152,7 +152,7 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): kernel_path = os.path.relpath(try_path, local_system_path) break else: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Could not find a kernel in the system: none of ' '%s found' % ', '.join(image_names)) return kernel_path @@ -171,11 +171,11 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): versioned_kernel_name = "%s-%s" % (system_name, version_label) kernel_name = system_name - cliapp.ssh_runcmd(nfs_netloc, + writeexts.ssh_runcmd(nfs_netloc, ['ln', '-f', kernel_dest, os.path.join(tftp_dir, versioned_kernel_name)]) - cliapp.ssh_runcmd(nfs_netloc, + writeexts.ssh_runcmd(nfs_netloc, ['ln', '-sf', versioned_kernel_name, os.path.join(tftp_dir, kernel_name)]) @@ -183,7 +183,7 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): with tempfile.NamedTemporaryFile() as f: f.write(text) f.flush() - cliapp.runcmd( + subprocess.check_call( ['scp', f.name, '%s:%s' % (nfs_netloc, path)]) def set_hostname(self, nfs_netloc, system_name, system_path): @@ -223,9 +223,9 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): '# Generated by distbuild-trove-nfsboot.write\n' + \ config_text + '\n' path = os.path.join(system_path, 'etc', 'distbuild') - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( nfs_netloc, ['mkdir', '-p', path]) - cliapp.runcmd( + subprocess.check_call( ['scp', worker_ssh_key_path, '%s:%s' % (nfs_netloc, path)]) self.set_remote_file_contents( nfs_netloc, os.path.join(path, 'distbuild.conf'), config_text) @@ -244,9 +244,9 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): # Rather ugly SSH hackery follows to ensure each system path is # listed in /etc/exports. try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( nfs_netloc, ['grep', '-q', exported_path, exports_path]) - except cliapp.AppException: + except subprocess.CalledProcessError: ip_mask = '*' options = 'rw,no_subtree_check,no_root_squash,async' exports_string = '%s %s(%s)\n' % (exported_path, ip_mask, @@ -259,12 +259,12 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): cat >> "$temp" mv "$temp" "$target" ''' - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( nfs_netloc, ['sh', '-c', exports_append_sh, '--', exports_path], feed_stdin=exports_string) - cliapp.ssh_runcmd(nfs_netloc, + writeexts.ssh_runcmd(nfs_netloc, ['systemctl', 'restart', 'nfs-server.service']) def update_default_version(self, remote_netloc, system_name, @@ -276,7 +276,7 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): version_label) default_path = os.path.join(system_path, 'systems', 'default') - cliapp.ssh_runcmd(remote_netloc, + writeexts.ssh_runcmd(remote_netloc, ['ln', '-sfn', system_version_path, default_path]) diff --git a/extensions/fstab.configure b/extensions/fstab.configure index b9154eee..3e67b585 100755 --- a/extensions/fstab.configure +++ b/extensions/fstab.configure @@ -20,9 +20,9 @@ import os import sys -import morphlib +import writeexts envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('FSTAB_')} conf_file = os.path.join(sys.argv[1], 'etc/fstab') -morphlib.util.write_from_dict(conf_file, envvars) +writeexts.write_from_dict(conf_file, envvars) diff --git a/extensions/hosts.configure b/extensions/hosts.configure index 6b068d04..11fcf573 100755 --- a/extensions/hosts.configure +++ b/extensions/hosts.configure @@ -22,27 +22,29 @@ import os import sys import socket -import morphlib +import writeexts def validate(var, line): xs = line.split() if len(xs) == 0: - raise morphlib.Error("`%s: %s': line is empty" % (var, line)) + raise writeexts.ExtensionError( + "`%s: %s': line is empty" % (var, line)) ip = xs[0] hostnames = xs[1:] if len(hostnames) == 0: - raise morphlib.Error("`%s: %s': missing hostname" % (var, line)) + raise writeexts.ExtensionError( + "`%s: %s': missing hostname" % (var, line)) family = socket.AF_INET6 if ':' in ip else socket.AF_INET try: socket.inet_pton(family, ip) except socket.error: - raise morphlib.Error("`%s: %s' invalid ip" % (var, ip)) + raise writeexts.ExtensionError("`%s: %s' invalid ip" % (var, ip)) envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('HOSTS_')} conf_file = os.path.join(sys.argv[1], 'etc/hosts') -morphlib.util.write_from_dict(conf_file, envvars, validate) +writeexts.write_from_dict(conf_file, envvars, validate) diff --git a/extensions/image-package-example/README b/extensions/image-package-example/README index c1322f25..f6b66cd9 100644 --- a/extensions/image-package-example/README +++ b/extensions/image-package-example/README @@ -5,5 +5,5 @@ These are scripts used to create disk images or install the system onto an existing disk. This is also implemented independently for the rawdisk.write write -extension; see morphlib.writeexts.WriteExtension.create_local_system() -for a similar, python implementation. +extension; see writeexts.WriteExtension.create_local_system() for +a similar, python implementation. diff --git a/extensions/install-essential-files.configure b/extensions/install-essential-files.configure index bed394df..3d33fe03 100755 --- a/extensions/install-essential-files.configure +++ b/extensions/install-essential-files.configure @@ -22,20 +22,11 @@ to install into the target system. ''' -import subprocess import os +import subprocess +import sys -import cliapp - -class InstallEssentialFilesConfigureExtension(cliapp.Application): - - def process_args(self, args): - target_root = args[0] - os.environ["INSTALL_FILES"] = "install-files/essential-files/manifest" - self.install_essential_files(target_root) - - def install_essential_files(self, target_root): - command = os.path.join("extensions/install-files.configure") - subprocess.check_call([command, target_root]) - -InstallEssentialFilesConfigureExtension().run() +target_root = sys.argv[1] +os.environ["INSTALL_FILES"] = "install-files/essential-files/manifest" +command = os.path.join("extensions/install-files.configure") +subprocess.check_call([command, target_root]) diff --git a/extensions/install-files.configure b/extensions/install-files.configure index 341cce61..64fcecca 100755 --- a/extensions/install-files.configure +++ b/extensions/install-files.configure @@ -22,9 +22,8 @@ to install into the target system. ''' -import cliapp -import os import errno +import os import re import sys import shlex @@ -37,7 +36,9 @@ try: except ImportError: jinja_available = False -class InstallFilesConfigureExtension(cliapp.Application): +import writeexts + +class InstallFilesConfigureExtension(writeexts.Extension): def process_args(self, args): if not 'INSTALL_FILES' in os.environ: @@ -74,7 +75,8 @@ class InstallFilesConfigureExtension(cliapp.Application): gid = int(m.group(5)) path = m.group(6) else: - raise cliapp.AppException('Invalid manifest entry, ' + raise writeexts.ExtensionError( + 'Invalid manifest entry, ' 'format: [template] [overwrite] ' '<octal mode> <uid decimal> <gid decimal> <filename>') @@ -85,9 +87,9 @@ class InstallFilesConfigureExtension(cliapp.Application): if (mode != dest_stat.st_mode or uid != dest_stat.st_uid or gid != dest_stat.st_gid): - raise cliapp.AppException('"%s" exists and is not ' - 'identical to directory ' - '"%s"' % (dest_path, entry)) + raise writeexts.ExtensionError( + '"%s" exists and is not identical to directory ' + '"%s"' % (dest_path, entry)) else: os.mkdir(dest_path, mode) os.chown(dest_path, uid, gid) @@ -95,8 +97,8 @@ class InstallFilesConfigureExtension(cliapp.Application): elif stat.S_ISLNK(mode): if os.path.lexists(dest_path) and not overwrite: - raise cliapp.AppException('Symlink already exists at %s' - % dest_path) + raise writeexts.ExtensionError('Symlink already exists at %s' + % dest_path) else: linkdest = os.readlink(os.path.join(manifest_root, './' + path)) @@ -105,12 +107,12 @@ class InstallFilesConfigureExtension(cliapp.Application): elif stat.S_ISREG(mode): if os.path.lexists(dest_path) and not overwrite: - raise cliapp.AppException('File already exists at %s' - % dest_path) + raise writeexts.ExtensionError('File already exists at %s' + % dest_path) else: if template: if not jinja_available: - raise cliapp.AppException( + raise writeexts.ExtensionError( "Failed to install template file `%s': " 'install-files templates require jinja2' % path) @@ -128,7 +130,7 @@ class InstallFilesConfigureExtension(cliapp.Application): os.chmod(dest_path, mode) else: - raise cliapp.AppException('Mode given in "%s" is not a file,' - ' symlink or directory' % entry) + raise writeexts.ExtensionError('Mode given in "%s" is not a file,' + ' symlink or directory' % entry) InstallFilesConfigureExtension().run() diff --git a/extensions/jffs2.write b/extensions/jffs2.write index 46b69a53..ad68204d 100644 --- a/extensions/jffs2.write +++ b/extensions/jffs2.write @@ -19,34 +19,34 @@ as the root filesystem.''' -import cliapp import os +import subprocess -import morphlib.writeexts +import writeexts -class Jffs2WriteExtension(morphlib.writeexts.WriteExtension): +class Jffs2WriteExtension(writeexts.WriteExtension): '''See jffs2.write.help for documentation.''' def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError('Wrong number of command line args') temp_root, location = args try: self.create_jffs2_system(temp_root, location) self.status(msg='Disk image has been created at %(location)s', - location = location) + location=location) except Exception: self.status(msg='Failure to deploy system to %(location)s', - location = location) + location=location) raise def create_jffs2_system(self, temp_root, location): erase_block = self.get_erase_block_size() - cliapp.runcmd( + subprocess.check_call( ['mkfs.jffs2', '--pad', '--no-cleanmarkers', '--eraseblock='+erase_block, '-d', temp_root, '-o', location]) @@ -54,10 +54,10 @@ class Jffs2WriteExtension(morphlib.writeexts.WriteExtension): erase_block = os.environ.get('ERASE_BLOCK', '') if erase_block == '': - raise cliapp.AppException('ERASE_BLOCK was not given') + raise writeexts.ExtensionError('ERASE_BLOCK was not given') if not erase_block.isdigit(): - raise cliapp.AppException('ERASE_BLOCK must be a whole number') + raise writeexts.ExtensionError('ERASE_BLOCK must be a whole number') return erase_block diff --git a/extensions/kvm.check b/extensions/kvm.check index 67cb3d38..4c0c9324 100755 --- a/extensions/kvm.check +++ b/extensions/kvm.check @@ -15,27 +15,28 @@ '''Preparatory checks for Morph 'kvm' write extension''' -import cliapp import os import re +import subprocess import urlparse -import morphlib.writeexts +import writeexts -class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): +class KvmPlusSshCheckExtension(writeexts.WriteExtension): location_pattern = '^/(?P<guest>[^/]+)(?P<path>/.+)$' def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') self.require_btrfs_in_deployment_host_kernel() upgrade = self.get_environment_boolean('UPGRADE') if upgrade: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Use the `ssh-rsync` write extension to deploy upgrades to an ' 'existing remote system.') @@ -55,23 +56,24 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): x = urlparse.urlparse(location) if x.scheme != 'kvm+ssh': - raise cliapp.AppException( + raise writeexts.ExtensionError( 'URL schema must be kvm+ssh in %s' % location) m = re.match(self.location_pattern, x.path) if not m: - raise cliapp.AppException('Cannot parse location %s' % location) + raise writeexts.ExtensionError( + 'Cannot parse location %s' % location) return x.netloc, m.group('guest'), m.group('path') def check_no_existing_libvirt_vm(self, ssh_host, vm_name): try: - cliapp.ssh_runcmd(ssh_host, + writeexts.ssh_runcmd(ssh_host, ['virsh', '--connect', 'qemu:///system', 'domstate', vm_name]) - except cliapp.AppException as e: + except CalledProcessError as e: pass else: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Host %s already has a VM named %s. You can use the ssh-rsync ' 'write extension to deploy upgrades to existing machines.' % (ssh_host, vm_name)) @@ -80,35 +82,35 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): def check_can_write_to_given_path(): try: - cliapp.ssh_runcmd(ssh_host, ['touch', vm_path]) - except cliapp.AppException as e: - raise cliapp.AppException("Can't write to location %s on %s" - % (vm_path, ssh_host)) + writeexts.ssh_runcmd(ssh_host, ['touch', vm_path]) + except subprocess.CalledProcessError as e: + raise writeexts.ExtensionError( + "Can't write to location %s on %s" % (vm_path, ssh_host)) else: - cliapp.ssh_runcmd(ssh_host, ['rm', vm_path]) + writeexts.ssh_runcmd(ssh_host, ['rm', vm_path]) try: - cliapp.ssh_runcmd(ssh_host, ['test', '-e', vm_path]) - except cliapp.AppException as e: + writeexts.ssh_runcmd(ssh_host, ['test', '-e', vm_path]) + except subprocess.CalledProcessError as e: # vm_path doesn't already exist, so let's test we can write check_can_write_to_given_path() else: - raise cliapp.AppException('%s already exists on %s' - % (vm_path, ssh_host)) + raise writeexts.ExtensionError('%s already exists on %s' + % (vm_path, ssh_host)) def check_extra_disks_exist(self, ssh_host, filename_list): for filename in filename_list: try: - cliapp.ssh_runcmd(ssh_host, ['ls', filename]) - except cliapp.AppException as e: - raise cliapp.AppException('Did not find file %s on host %s' % - (filename, ssh_host)) + writeexts.ssh_runcmd(ssh_host, ['ls', filename]) + except subprocess.CalledProcessError as e: + raise writeexts.ExtensionError( + 'Did not find file %s on host %s' % (filename, ssh_host)) def check_virtual_networks_are_started(self, ssh_host): def check_virtual_network_is_started(network_name): cmd = ['virsh', '-c', 'qemu:///system', 'net-info', network_name] - net_info = cliapp.ssh_runcmd(ssh_host, cmd).split('\n') + net_info = writeexts.ssh_runcmd(ssh_host, cmd).split('\n') def pretty_concat(lines): return '\n'.join(['\t%s' % line for line in lines]) @@ -118,15 +120,15 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): if m: break else: - raise cliapp.AppException( + raise writeexts.ExtensionError( "Got unexpected output parsing output of `%s':\n%s" % (' '.join(cmd), pretty_concat(net_info))) network_active = m.group(1) == 'yes' if not network_active: - raise cliapp.AppException("Network '%s' is not started" - % network_name) + raise writeexts.ExtensionError("Network '%s' is not started" + % network_name) def name(nic_entry): if ',' in nic_entry: @@ -142,9 +144,10 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): if not (n.startswith('network=') or n.startswith('bridge=') or n == 'user'): - raise cliapp.AppException('malformed NIC_CONFIG: %s\n' - " (expected 'bridge=BRIDGE' 'network=NAME'" - " or 'user')" % n) + raise writeexts.ExtensionError( + "malformed NIC_CONFIG: %s\n" + " (expected 'bridge=BRIDGE' 'network=NAME'" + " or 'user')" % n) # --network bridge= is used to specify a bridge # --network user is used to specify a form of NAT @@ -159,9 +162,9 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): def check_host_has_virtinstall(self, ssh_host): try: - cliapp.ssh_runcmd(ssh_host, ['which', 'virt-install']) - except cliapp.AppException: - raise cliapp.AppException( + writeexts.ssh_runcmd(ssh_host, ['which', 'virt-install']) + except subprocess.CalledProcessError: + raise writeexts.ExtensionError( 'virt-install does not seem to be installed on host %s' % ssh_host) diff --git a/extensions/kvm.write b/extensions/kvm.write index 0d0c095b..2290725e 100755 --- a/extensions/kvm.write +++ b/extensions/kvm.write @@ -21,23 +21,23 @@ See file kvm.write.help for documentation ''' -import cliapp import os import re import sys import tempfile import urlparse -import morphlib.writeexts +import writeexts -class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): +class KvmPlusSshWriteExtension(writeexts.WriteExtension): location_pattern = '^/(?P<guest>[^/]+)(?P<path>/.+)$' def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') temp_root, location = args ssh_host, vm_name, vm_path = self.parse_location(location) @@ -53,7 +53,7 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): except BaseException: sys.stderr.write('Error deploying to libvirt') os.remove(raw_disk) - cliapp.ssh_runcmd(ssh_host, ['rm', '-f', vm_path]) + writeexts.ssh_runcmd(ssh_host, ['rm', '-f', vm_path]) raise else: os.remove(raw_disk) @@ -74,16 +74,16 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Transferring disk image') - xfer_hole_path = morphlib.util.get_data_path('xfer-hole') - recv_hole = morphlib.util.get_data('recv-hole') + xfer_hole_path = writeexts.get_data_path('xfer-hole') + recv_hole = writeexts.get_data('recv-hole') ssh_remote_cmd = [ 'sh', '-c', recv_hole, 'dummy-argv0', 'file', vm_path ] - cliapp.runcmd( + subprocess.check_call( ['python', xfer_hole_path, raw_disk], - ['ssh', ssh_host] + map(cliapp.shell_quote, ssh_remote_cmd), + ['ssh', ssh_host] + map(writeexts.shell_quote, ssh_remote_cmd), stdout=None, stderr=None) def create_libvirt_guest(self, ssh_host, vm_name, vm_path, autostart): @@ -111,10 +111,11 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): '--disk', 'path=%s,bus=ide' % vm_path] + attach_opts if not autostart: cmdline += ['--noreboot'] - cliapp.ssh_runcmd(ssh_host, cmdline) + writeexts.ssh_runcmd(ssh_host, cmdline) if autostart: - cliapp.ssh_runcmd(ssh_host, - ['virsh', '--connect', 'qemu:///system', 'autostart', vm_name]) + writeexts.ssh_runcmd(ssh_host, + ['virsh', '--connect', 'qemu:///system', + 'autostart', vm_name]) KvmPlusSshWriteExtension().run() diff --git a/extensions/nfsboot.check b/extensions/nfsboot.check index e273f61c..cc138b0d 100755 --- a/extensions/nfsboot.check +++ b/extensions/nfsboot.check @@ -15,33 +15,35 @@ '''Preparatory checks for Morph 'nfsboot' write extension''' -import cliapp import os +import subprocess -import morphlib.writeexts +import writeexts -class NFSBootCheckExtension(morphlib.writeexts.WriteExtension): +class NFSBootCheckExtension(writeexts.WriteExtension): _nfsboot_root = '/srv/nfsboot' def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') location = args[0] upgrade = self.get_environment_boolean('UPGRADE') if upgrade: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Upgrading is not currently supported for NFS deployments.') hostname = os.environ.get('HOSTNAME', None) if hostname is None: - raise cliapp.AppException('You must specify a HOSTNAME.') + raise writeexts.ExtensionError('You must specify a HOSTNAME.') if hostname == 'baserock': - raise cliapp.AppException('It is forbidden to nfsboot a system ' - 'with hostname "%s"' % hostname) + raise writeexts.ExtensionError('It is forbidden to nfsboot a ' + 'system with hostname "%s"' + % hostname) self.test_good_server(location) @@ -49,7 +51,7 @@ class NFSBootCheckExtension(morphlib.writeexts.WriteExtension): versioned_root = os.path.join(self._nfsboot_root, hostname, 'systems', version_label) if self.version_exists(versioned_root, location): - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Root file system for host %s (version %s) already exists on ' 'the NFS server %s. Deployment aborted.' % (hostname, version_label, location)) @@ -59,34 +61,34 @@ class NFSBootCheckExtension(morphlib.writeexts.WriteExtension): # Is an NFS server try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( 'root@%s' % server, ['test', '-e', '/etc/exports']) - except cliapp.AppException: - raise cliapp.AppException('server %s is not an nfs server' - % server) + except subprocess.CalledProcessError: + raise writeexts.ExtensionError('server %s is not an nfs server' + % server) try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( 'root@%s' % server, ['systemctl', 'is-enabled', 'nfs-server.service']) - except cliapp.AppException: - raise cliapp.AppException('server %s does not control its ' - 'nfs server by systemd' % server) + except subprocess.CalledProcessError: + raise writeexts.ExtensionError('server %s does not control its ' + 'nfs server by systemd' % server) # TFTP server exports /srv/nfsboot/tftp tftp_root = os.path.join(self._nfsboot_root, 'tftp') try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( 'root@%s' % server, ['test' , '-d', tftp_root]) - except cliapp.AppException: - raise cliapp.AppException('server %s does not export %s' % - (tftp_root, server)) + except subprocess.CalledProcessError: + raise writeexts.ExtensionError('server %s does not export %s' % + (tftp_root, server)) def version_exists(self, versioned_root, location): try: - cliapp.ssh_runcmd('root@%s' % location, - ['test', '-d', versioned_root]) - except cliapp.AppException: + writeexts.ssh_runcmd('root@%s' % location, + ['test', '-d', versioned_root]) + except subprocess.CalledProcessError: return False return True diff --git a/extensions/nfsboot.write b/extensions/nfsboot.write index d928775e..b590ad70 100755 --- a/extensions/nfsboot.write +++ b/extensions/nfsboot.write @@ -34,14 +34,13 @@ in /srv/nfsboot/nfs/ ''' -import cliapp import os import glob -import morphlib.writeexts +import writeexts -class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): +class NFSBootWriteExtension(writeexts.WriteExtension): '''Create an NFS root and kernel on TFTP during Morph's deployment. @@ -66,7 +65,8 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') temp_root, location = args @@ -86,8 +86,8 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): subdirs = [os.path.join(statedir, 'home'), os.path.join(statedir, 'opt'), os.path.join(statedir, 'srv')] - cliapp.ssh_runcmd('root@%s' % location, - ['mkdir', '-p'] + subdirs) + writeexts.ssh_runcmd('root@%s' % location, + ['mkdir', '-p'] + subdirs) def copy_kernel(self, temp_root, location, versioned_root, version, hostname): @@ -99,14 +99,14 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): kernel_src = try_path break else: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Could not find a kernel in the system: none of ' '%s found' % ', '.join(image_names)) kernel_dest = os.path.join(versioned_root, 'orig', 'kernel') rsync_dest = 'root@%s:%s' % (location, kernel_dest) self.status(msg='Copying kernel') - cliapp.runcmd( + subprocess.check_call( ['rsync', '-s', kernel_src, rsync_dest]) # Link the kernel to the right place @@ -115,17 +115,17 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): versioned_kernel_name = "%s-%s" % (hostname, version) kernel_name = hostname try: - cliapp.ssh_runcmd('root@%s' % location, + writeexts.ssh_runcmd('root@%s' % location, ['ln', '-f', kernel_dest, os.path.join(tftp_dir, versioned_kernel_name)]) - cliapp.ssh_runcmd('root@%s' % location, + writeexts.ssh_runcmd('root@%s' % location, ['ln', '-sf', versioned_kernel_name, os.path.join(tftp_dir, kernel_name)]) - except cliapp.AppException: - raise cliapp.AppException('Could not create symlinks to the ' - 'kernel at %s in %s on %s' - % (kernel_dest, tftp_dir, location)) + except subprocess.CalledProcessError: + raise writeexts.ExtensionError('Could not create symlinks to the ' + 'kernel at %s in %s on %s' % + (kernel_dest, tftp_dir, location)) def copy_rootfs(self, temp_root, location, versioned_root, hostname): rootfs_src = temp_root + '/' @@ -134,51 +134,54 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Creating destination directories') try: - cliapp.ssh_runcmd('root@%s' % location, - ['mkdir', '-p', orig_path, run_path]) - except cliapp.AppException: - raise cliapp.AppException('Could not create dirs %s and %s on %s' - % (orig_path, run_path, location)) + writeexts.ssh_runcmd('root@%s' % location, + ['mkdir', '-p', orig_path, run_path]) + except subprocess.CalledProcessError: + raise writexts.ExtensionError( + 'Could not create dirs %s and %s on %s' + % (orig_path, run_path, location)) self.status(msg='Creating \'orig\' rootfs') - cliapp.runcmd( + subprocess.check_call( ['rsync', '-asXSPH', '--delete', rootfs_src, 'root@%s:%s' % (location, orig_path)]) self.status(msg='Creating \'run\' rootfs') try: - cliapp.ssh_runcmd('root@%s' % location, - ['rm', '-rf', run_path]) - cliapp.ssh_runcmd('root@%s' % location, - ['cp', '-al', orig_path, run_path]) - cliapp.ssh_runcmd('root@%s' % location, - ['rm', '-rf', os.path.join(run_path, 'etc')]) - cliapp.ssh_runcmd('root@%s' % location, - ['cp', '-a', os.path.join(orig_path, 'etc'), - os.path.join(run_path, 'etc')]) - except cliapp.AppException: - raise cliapp.AppException('Could not create \'run\' rootfs' - ' from \'orig\'') + writeexts.ssh_runcmd('root@%s' % location, + ['rm', '-rf', run_path]) + writeexts.ssh_runcmd('root@%s' % location, + ['cp', '-al', orig_path, run_path]) + writeexts.ssh_runcmd('root@%s' % location, + ['rm', '-rf', + os.path.join(run_path, 'etc')]) + writeexts.ssh_runcmd('root@%s' % location, + ['cp', '-a', + os.path.join(orig_path, 'etc'), + os.path.join(run_path, 'etc')]) + except subprocess.CalledProcessError: + raise writeexts.ExtensionError('Could not create \'run\' rootfs' + ' from \'orig\'') self.status(msg='Linking \'default\' to latest system') try: - cliapp.ssh_runcmd('root@%s' % location, + writeexts.ssh_runcmd('root@%s' % location, ['ln', '-sfn', versioned_root, os.path.join(self._nfsboot_root, hostname, 'systems', 'default')]) - except cliapp.AppException: - raise cliapp.AppException('Could not link \'default\' to %s' - % versioned_root) + except subprocess.CalledProcessError: + raise writeexts.ExtensionError("Could not link 'default' to %s" + % versioned_root) def configure_nfs(self, location, hostname): exported_path = os.path.join(self._nfsboot_root, hostname) exports_path = '/etc/exports' # If that path is not already exported: try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( 'root@%s' % location, ['grep', '-q', exported_path, exports_path]) - except cliapp.AppException: + except subprocess.CalledProcessError: ip_mask = '*' options = 'rw,no_subtree_check,no_root_squash,async' exports_string = '%s %s(%s)\n' % (exported_path, ip_mask, options) @@ -190,11 +193,11 @@ cat "$target" > "$temp" cat >> "$temp" mv "$temp" "$target" ''' - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( 'root@%s' % location, ['sh', '-c', exports_append_sh, '--', exports_path], feed_stdin=exports_string) - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( 'root@%s' % location, ['systemctl', 'restart', 'nfs-server.service']) diff --git a/extensions/openstack.check b/extensions/openstack.check index a3379763..f3ad43b7 100755 --- a/extensions/openstack.check +++ b/extensions/openstack.check @@ -15,25 +15,26 @@ '''Preparatory checks for Morph 'openstack' write extension''' -import cliapp import os import urlparse + import keystoneclient -import morphlib.writeexts +import writeexts -class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): +class OpenStackCheckExtension(writeexts.WriteExtension): def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') self.require_btrfs_in_deployment_host_kernel() upgrade = self.get_environment_boolean('UPGRADE') if upgrade: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Use the `ssh-rsync` write extension to deploy upgrades to an ' 'existing remote system.') @@ -55,7 +56,7 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): for key in auth_keys: if os.environ.get(key, '') == '': - raise cliapp.AppException(key + ' was not given') + raise writeexts.ExtensionError(key + ' was not given') auth_params = {auth_keys[key]: os.environ[key] for key in auth_keys} auth_params['auth_url'] = location @@ -63,16 +64,17 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): def check_imagename(self): if os.environ.get('OPENSTACK_IMAGENAME', '') == '': - raise cliapp.AppException('OPENSTACK_IMAGENAME was not given') + raise writeexts.ExtensionError( + 'OPENSTACK_IMAGENAME was not given') def check_location(self, location): x = urlparse.urlparse(location) if x.scheme not in ['http', 'https']: - raise cliapp.AppException('URL schema must be http or https in %s'\ - % location) + raise writeexts.ExtensionError( + 'URL schema must be http or https in %s' % location) if (x.path != '/v2.0' and x.path != '/v2.0/'): - raise cliapp.AppException('API version must be v2.0 in %s'\ - % location) + raise writeexts.ExtensionError( + 'API version must be v2.0 in %s' % location) def check_openstack_parameters(self, auth_params): ''' Check that we can connect to and authenticate with openstack ''' @@ -84,7 +86,7 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): except keystoneclient.exceptions.Unauthorized: errmsg = ('Failed to authenticate with OpenStack ' '(are your credentials correct?)') - raise cliapp.AppException(errmsg) + raise writeexts.ExtensionError(errmsg) OpenStackCheckExtension().run() diff --git a/extensions/openstack.write b/extensions/openstack.write index 67e07c18..f1233560 100755 --- a/extensions/openstack.write +++ b/extensions/openstack.write @@ -17,21 +17,22 @@ '''A Morph deployment write extension for deploying to OpenStack.''' -import cliapp import os +import subprocess import tempfile import urlparse -import morphlib.writeexts +import writeexts -class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): +class OpenStackWriteExtension(writeexts.WriteExtension): '''See openstack.write.help for documentation''' def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') temp_root, location = args @@ -86,7 +87,7 @@ class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): '--disk-format=raw', '--container-format', 'bare', '--file', raw_disk] - cliapp.runcmd(cmdline) + subprocess.check_call(cmdline) self.status(msg='Image configured.') diff --git a/extensions/pxeboot.write b/extensions/pxeboot.write index 3a12ebcc..20e4f6bd 100644 --- a/extensions/pxeboot.write +++ b/extensions/pxeboot.write @@ -19,10 +19,7 @@ import tempfile import textwrap import urlparse -import cliapp - -import morphlib - +import writeexts def _int_to_quad_dot(i): return '.'.join(( @@ -143,7 +140,7 @@ def grouper(iterable, n, fillvalue=None): return itertools.izip_longest(*args, fillvalue=fillvalue) -class PXEBoot(morphlib.writeexts.WriteExtension): +class PXEBoot(writeexts.WriteExtension): @contextlib.contextmanager def _vlan(self, interface, vlan): viface = '%s.%s' % (interface, vlan) @@ -224,12 +221,13 @@ class PXEBoot(morphlib.writeexts.WriteExtension): @contextlib.contextmanager def _remote_tempdir(self, hostname, template): persist = os.environ.get('PXE_INSTALLER') in ('no', 'False') - td = cliapp.ssh_runcmd(hostname, ['mktemp', '-d', template]).strip() + td = writeexts.ssh_runcmd( + hostname, ['mktemp', '-d', template]).strip() try: yield td finally: if not persist: - cliapp.ssh_runcmd(hostname, ['find', td, '-delete']) + writeexts.ssh_runcmd(hostname, ['find', td, '-delete']) def _serve_tftpd(self, sock, host, port, interface, tftproot): self.settings.progname = 'tftp server' @@ -333,26 +331,27 @@ class PXEBoot(morphlib.writeexts.WriteExtension): def _remote_copy(self, hostname, src, dst): persist = os.environ.get('PXE_INSTALLER') in ('no', 'False') with open(src, 'r') as f: - cliapp.ssh_runcmd(hostname, - ['install', '-D', '-m644', '/proc/self/fd/0', - dst], stdin=f, stdout=None, stderr=None) + writeexts.ssh_runcmd(hostname, + ['install', '-D', '-m644', + '/proc/self/fd/0', dst], + stdin=f, stdout=None, stderr=None) try: yield finally: if not persist: - cliapp.ssh_runcmd(hostname, ['rm', dst]) + writeexts.ssh_runcmd(hostname, ['rm', dst]) @contextlib.contextmanager def _remote_symlink(self, hostname, src, dst): persist = os.environ.get('PXE_INSTALLER') in ('no', 'False') - cliapp.ssh_runcmd(hostname, - ['ln', '-s', '-f', src, dst], - stdin=None, stdout=None, stderr=None) + writeexts.ssh_runcmd(hostname, + ['ln', '-s', '-f', src, dst], + stdin=None, stdout=None, stderr=None) try: yield finally: if not persist: - cliapp.ssh_runcmd(hostname, ['rm', '-f', dst]) + writeexts.ssh_runcmd(hostname, ['rm', '-f', dst]) @contextlib.contextmanager def remote_kernel(self, rootfs, tftp_url, macaddr): @@ -361,7 +360,7 @@ class PXEBoot(morphlib.writeexts.WriteExtension): if os.path.exists(kernel_path): break else: - raise cliapp.AppException('Failed to locate kernel') + raise writeexts.ExtensionError('Failed to locate kernel') url = urlparse.urlsplit(tftp_url) basename = '{}-kernel'.format(_normalise_macaddr(macaddr)) target_path = os.path.join(url.path, basename) @@ -376,7 +375,8 @@ class PXEBoot(morphlib.writeexts.WriteExtension): yield fdt_abs_path = os.path.join(rootfs, fdt_rel_path) if not fdt_abs_path: - raise cliapp.AppException('Failed to locate Flattened Device Tree') + raise writeexts.ExtensionError( + 'Failed to locate Flattened Device Tree') url = urlparse.urlsplit(tftp_url) basename = '{}-fdt'.format(_normalise_macaddr(macaddr)) target_path = os.path.join(url.path, basename) @@ -389,14 +389,14 @@ class PXEBoot(morphlib.writeexts.WriteExtension): nfsroot = target_ip + ':' + rootfs self.status(msg='Exporting %(nfsroot)s as local nfsroot', nfsroot=nfsroot) - cliapp.runcmd(['exportfs', '-o', 'ro,insecure,no_root_squash', - nfsroot]) + subprocess.check_call(['exportfs', '-o', 'ro,insecure,no_root_squash', + nfsroot]) try: yield finally: self.status(msg='Removing %(nfsroot)s from local nfsroots', nfsroot=nfsroot) - cliapp.runcmd(['exportfs', '-u', nfsroot]) + subprocess.check_call(['exportfs', '-u', nfsroot]) @contextlib.contextmanager def remote_nfsroot(self, rootfs, rsync_url, macaddr): @@ -407,9 +407,10 @@ class PXEBoot(morphlib.writeexts.WriteExtension): as tempdir: nfsroot = urlparse.urlunsplit((url.scheme, url.netloc, tempdir, url.query, url.fragment)) - cliapp.runcmd(['rsync', '-asSPH', '--delete', rootfs, nfsroot], - stdin=None, stdout=open(os.devnull, 'w'), - stderr=None) + subprocess.check_call(['rsync', '-asSPH', '--delete', + rootfs, nfsroot], + stdin=None, stdout=open(os.devnull, 'w'), + stderr=None) yield os.path.join(os.path.basename(tempdir), os.path.basename(rootfs)) @@ -517,8 +518,8 @@ class PXEBoot(morphlib.writeexts.WriteExtension): def get_interface_ip(self, interface): ip_addresses = [] - info = cliapp.runcmd(['ip', '-o', '-f', 'inet', - 'addr', 'show', interface]).rstrip('\n') + info = subprocess.check_output(['ip', '-o', '-f', 'inet', 'addr', + 'show', interface]).rstrip('\n') if info: tokens = collections.deque(info.split()[1:]) ifname = tokens.popleft() @@ -535,8 +536,8 @@ class PXEBoot(morphlib.writeexts.WriteExtension): else: continue if not ip_addresses: - raise cliapp.AppException('Interface %s has no addresses' - % interface) + raise writeexts.ExtensionError('Interface %s has no addresses' + % interface) if len(ip_addresses) > 1: warnings.warn('Interface %s has multiple addresses, ' 'using first (%s)' % (interface, ip_addresses[0])) @@ -750,6 +751,6 @@ class PXEBoot(morphlib.writeexts.WriteExtension): self.wait_for_target_to_install() self.ipmi_reboot_target() else: - cliapp.AppException('Invalid PXEBOOT_MODE: %s' % mode) + writeexts.ExtensionError('Invalid PXEBOOT_MODE: %s' % mode) PXEBoot().run() diff --git a/extensions/rawdisk.check b/extensions/rawdisk.check index 9be0ce91..61619a21 100755 --- a/extensions/rawdisk.check +++ b/extensions/rawdisk.check @@ -15,17 +15,16 @@ '''Preparatory checks for Morph 'rawdisk' write extension''' -import cliapp - -import morphlib.writeexts - import os +import writeexts + -class RawdiskCheckExtension(morphlib.writeexts.WriteExtension): +class RawdiskCheckExtension(writeexts.WriteExtension): def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') self.require_btrfs_in_deployment_host_kernel() @@ -34,19 +33,19 @@ class RawdiskCheckExtension(morphlib.writeexts.WriteExtension): if upgrade: if not self.is_device(location): if not os.path.isfile(location): - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Cannot upgrade %s: it is not an existing disk image' % location) version_label = os.environ.get('VERSION_LABEL') if version_label is None: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'VERSION_LABEL was not given. It is required when ' 'upgrading an existing system.') else: if not self.is_device(location): if os.path.exists(location): - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Target %s already exists. Use `morph upgrade` if you ' 'want to update an existing image.' % location) diff --git a/extensions/rawdisk.write b/extensions/rawdisk.write index 6f2d45ba..cdeb5018 100755 --- a/extensions/rawdisk.write +++ b/extensions/rawdisk.write @@ -17,22 +17,22 @@ '''A Morph deployment write extension for raw disk images.''' -import cliapp import os import sys import time import tempfile -import morphlib.writeexts +import writeexts -class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): +class RawDiskWriteExtension(writeexts.WriteExtension): '''See rawdisk.write.help for documentation''' def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') temp_root, location = args upgrade = self.get_environment_boolean('UPGRADE') @@ -69,10 +69,10 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): old_orig = os.path.join(mp, 'systems', 'default', 'orig') new_orig = os.path.join(version_root, 'orig') - cliapp.runcmd( + subprocess.check_call( ['btrfs', 'subvolume', 'snapshot', old_orig, new_orig]) - cliapp.runcmd( + subprocess.check_call( ['rsync', '-a', '--checksum', '--numeric-ids', '--delete', temp_root + os.path.sep, new_orig]) @@ -96,11 +96,11 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): version_label = os.environ.get('VERSION_LABEL') if version_label is None: - raise cliapp.AppException('VERSION_LABEL was not given') + raise writeexts.ExtensionError('VERSION_LABEL was not given') if os.path.exists(os.path.join(mp, 'systems', version_label)): - raise cliapp.AppException('VERSION_LABEL %s already exists' - % version_label) + raise writeexts.ExtensionError('VERSION_LABEL %s already exists' + % version_label) return version_label diff --git a/extensions/simple-network.configure b/extensions/simple-network.configure index 4a70f311..61d5774d 100755 --- a/extensions/simple-network.configure +++ b/extensions/simple-network.configure @@ -25,27 +25,26 @@ for DHCP ''' +import errno import os import sys -import errno -import cliapp -import morphlib +import writeexts -class SimpleNetworkError(morphlib.Error): +class SimpleNetworkError(writeexts.ExtensionError): '''Errors associated with simple network setup''' pass -class SimpleNetworkConfigurationExtension(cliapp.Application): +class SimpleNetworkConfigurationExtension(object): '''Configure /etc/network/interfaces and generate networkd .network files Reading NETWORK_CONFIG, this extension sets up /etc/network/interfaces and .network files in /etc/systemd/network/. ''' - def process_args(self, args): + def run(self, args): network_config = os.environ.get("NETWORK_CONFIG") self.rename_networkd_chunk_file(args) @@ -206,7 +205,8 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): address_line = address + '/' + str(network_suffix) lines += ["Address=%s" % address_line] elif address or netmask: - raise Exception('address and netmask must be specified together') + raise SimpleNetworkError( + 'address and netmask must be specified together') if gateway: lines += ["Gateway=%s" % gateway] @@ -287,6 +287,10 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): ''' - self.output.write('%s\n' % (kwargs['msg'] % kwargs)) + sys.stdout.write('%s\n' % (kwargs['msg'] % kwargs)) -SimpleNetworkConfigurationExtension().run() +try: + SimpleNetworkConfigurationExtension().run(sys.argv[1:]) +except SimpleNetworkError as e: + sys.stdout.write('ERROR: %s\n' % e) + sys.exit(1) diff --git a/extensions/ssh-rsync.check b/extensions/ssh-rsync.check index c3bdfd29..436aaae0 100755 --- a/extensions/ssh-rsync.check +++ b/extensions/ssh-rsync.check @@ -15,26 +15,27 @@ '''Preparatory checks for Morph 'ssh-rsync' write extension''' -import cliapp import os -import morphlib.writeexts +import writeexts -class SshRsyncCheckExtension(morphlib.writeexts.WriteExtension): + +class SshRsyncCheckExtension(writeexts.WriteExtension): def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') upgrade = self.get_environment_boolean('UPGRADE') if not upgrade: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'The ssh-rsync write is for upgrading existing remote ' 'Baserock machines. It cannot be used for an initial ' 'deployment.') if os.environ.get('VERSION_LABEL', '') == '': - raise cliapp.AppException( + raise writeexts.ExtensionError( 'A VERSION_LABEL must be set when deploying an upgrade.') location = args[0] @@ -47,17 +48,18 @@ class SshRsyncCheckExtension(morphlib.writeexts.WriteExtension): self.check_command_exists(location, 'rsync') def check_is_baserock_system(self, location): - output = cliapp.ssh_runcmd(location, ['sh', '-c', - 'test -d /baserock || echo -n dirnotfound']) + output = writeexts.ssh_runcmd( + location, + ['sh', '-c', 'test -d /baserock || echo -n dirnotfound']) if output == 'dirnotfound': - raise cliapp.AppException('%s is not a baserock system' - % location) + raise writeexts.ExtensionError('%s is not a baserock system' + % location) def check_command_exists(self, location, command): test = 'type %s > /dev/null 2>&1 || echo -n cmdnotfound' % command - output = cliapp.ssh_runcmd(location, ['sh', '-c', test]) + output = writeexts.ssh_runcmd(location, ['sh', '-c', test]) if output == 'cmdnotfound': - raise cliapp.AppException( + raise writeexts.ExtensionError( "%s does not have %s" % (location, command)) diff --git a/extensions/ssh-rsync.write b/extensions/ssh-rsync.write index 6d596500..7b58facc 100755 --- a/extensions/ssh-rsync.write +++ b/extensions/ssh-rsync.write @@ -18,23 +18,23 @@ import contextlib -import cliapp import os +import subprocess import sys -import time import tempfile +import time -import morphlib.writeexts +import writeexts def ssh_runcmd_ignore_failure(location, command, **kwargs): try: - return cliapp.ssh_runcmd(location, command, **kwargs) - except cliapp.AppException: + return writeexts.ssh_runcmd(location, command, **kwargs) + except subprocess.CalledProcessError: pass -class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): +class SshRsyncWriteExtension(writeexts.WriteExtension): '''See ssh-rsync.write.help for documentation''' @@ -43,7 +43,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): '''Read /proc/mounts on location to find which device contains "/"''' self.status(msg='Finding device that contains "/"') - contents = cliapp.ssh_runcmd(location, ['cat', '/proc/mounts']) + contents = writeexts.ssh_runcmd(location, + ['cat', '/proc/mounts']) for line in contents.splitlines(): line_words = line.split() if (line_words[1] == '/' and line_words[0] != 'rootfs'): @@ -52,28 +53,29 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): @contextlib.contextmanager def _remote_mount_point(self, location): self.status(msg='Creating remote mount point') - remote_mnt = cliapp.ssh_runcmd(location, ['mktemp', '-d']).strip() + remote_mnt = writeexts.ssh_runcmd(location, + ['mktemp', '-d']).strip() try: yield remote_mnt finally: self.status(msg='Removing remote mount point') - cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) + writeexts.ssh_runcmd(location, ['rmdir', remote_mnt]) @contextlib.contextmanager def _remote_mount(self, location, root_disk, mountpoint): self.status(msg='Mounting root disk') - cliapp.ssh_runcmd(location, ['mount', root_disk, mountpoint]) + writeexts.ssh_runcmd(location, ['mount', root_disk, mountpoint]) try: yield finally: self.status(msg='Unmounting root disk') - cliapp.ssh_runcmd(location, ['umount', mountpoint]) + writeexts.ssh_runcmd(location, ['umount', mountpoint]) @contextlib.contextmanager def _created_version_root(self, location, remote_mnt, version_label): version_root = os.path.join(remote_mnt, 'systems', version_label) self.status(msg='Creating %(root)s', root=version_root) - cliapp.ssh_runcmd(location, ['mkdir', version_root]) + writeexts.ssh_runcmd(location, ['mkdir', version_root]) try: yield version_root except BaseException as e: @@ -93,8 +95,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Creating "orig" subvolume') old_orig = self.get_old_orig(location, remote_mnt) new_orig = os.path.join(version_root, 'orig') - cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', 'snapshot', - old_orig, new_orig]) + writeexts.ssh_runcmd(location, ['btrfs', 'subvolume', 'snapshot', + old_orig, new_orig]) try: yield new_orig except BaseException as e: @@ -106,30 +108,30 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): '''Populate the subvolume version_root/orig on location''' self.status(msg='Populating "orig" subvolume') - cliapp.runcmd(['rsync', '-as', '--checksum', '--numeric-ids', - '--delete', temp_root + os.path.sep, - '%s:%s' % (location, new_orig)]) + subprocess.check_call(['rsync', '-as', '--checksum', '--numeric-ids', + '--delete', temp_root + os.path.sep, + '%s:%s' % (location, new_orig)]) @contextlib.contextmanager def _deployed_version(self, location, version_label, system_config_sync, system_version_manager): self.status(msg='Calling system-version-manager to deploy upgrade') deployment = os.path.join('/systems', version_label, 'orig') - cliapp.ssh_runcmd(location, + writeexts.ssh_runcmd(location, ['env', 'BASEROCK_SYSTEM_CONFIG_SYNC='+system_config_sync, system_version_manager, 'deploy', deployment]) try: yield deployment except BaseException as e: self.status(msg='Cleaning up failed version installation') - cliapp.ssh_runcmd(location, + writeexts.ssh_runcmd(location, [system_version_manager, 'remove', version_label]) raise def upgrade_remote_system(self, location, temp_root): root_disk = self.find_root_disk(location) - uuid = cliapp.ssh_runcmd(location, ['blkid', '-s', 'UUID', '-o', - 'value', root_disk]).strip() + uuid = writeexts.ssh_runcmd(location, + ['blkid', '-s', 'UUID', '-o', 'value', root_disk]).strip() self.complete_fstab_for_btrfs_layout(temp_root, uuid) @@ -153,8 +155,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): config_sync, version_manager): self.status(msg='Setting %(v)s as the new default system', v=version_label) - cliapp.ssh_runcmd(location, [version_manager, - 'set-default', version_label]) + writeexts.ssh_runcmd(location, + [version_manager, 'set-default', version_label]) if autostart: self.status(msg="Rebooting into new system ...") @@ -162,7 +164,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') temp_root, location = args diff --git a/extensions/strip-gplv3.configure b/extensions/strip-gplv3.configure index c08061ad..0585fe5d 100755 --- a/extensions/strip-gplv3.configure +++ b/extensions/strip-gplv3.configure @@ -21,12 +21,14 @@ to find the files created by that chunk, then remove them. ''' -import cliapp -import re -import os import json +import os +import re +import sys + +import writeexts -class StripGPLv3ConfigureExtension(cliapp.Application): +class StripGPLv3ConfigureExtension(object): gplv3_chunks = [ ['autoconf', ''], ['automake', ''], @@ -51,7 +53,7 @@ class StripGPLv3ConfigureExtension(cliapp.Application): ['texinfo-tarball', ''], ] - def process_args(self, args): + def run(self, args): target_root = args[0] meta_dir = os.path.join(target_root, 'baserock') @@ -72,8 +74,8 @@ class StripGPLv3ConfigureExtension(cliapp.Application): chunk_meta_data = json.load(f) if not 'contents' in chunk_meta_data: - raise cliapp.AppError('Chunk %s does not have a "contents" list' - % chunk) + raise writeexts.ExtensionError( + 'Chunk %s does not have a "contents" list' % chunk) updated_contents = [] for content_entry in reversed(chunk_meta_data['contents']): pat = re.compile(pattern) @@ -85,8 +87,8 @@ class StripGPLv3ConfigureExtension(cliapp.Application): def remove_content_entry(self, target_root, content_entry): entry_path = os.path.join(target_root, './' + content_entry) if not entry_path.startswith(target_root): - raise cliapp.AppException('%s is not in %s' - % (entry_path, target_root)) + raise writeexts.ExtensionError( + '%s is not in %s' % (entry_path, target_root)) if os.path.exists(entry_path): if os.path.islink(entry_path): os.unlink(entry_path) @@ -96,6 +98,11 @@ class StripGPLv3ConfigureExtension(cliapp.Application): if not os.listdir(entry_path): os.rmdir(entry_path) else: - raise cliapp.AppException('%s is not a link, file or directory' - % entry_path) -StripGPLv3ConfigureExtension().run() + raise writeexts.ExtensionError( + '%s is not a link, file or directory' % entry_path) + +try: + StripGPLv3ConfigureExtension().run(sys.argv[1:]) +except writeexts.ExtensionError as e: + sys.stdout.write('ERROR: %s' % e) + sys.exit(1) diff --git a/extensions/virtualbox-ssh.check b/extensions/virtualbox-ssh.check index a97f3294..e82d58a1 100755 --- a/extensions/virtualbox-ssh.check +++ b/extensions/virtualbox-ssh.check @@ -15,21 +15,21 @@ '''Preparatory checks for Morph 'virtualbox-ssh' write extension''' -import cliapp -import morphlib.writeexts +import writeexts -class VirtualBoxPlusSshCheckExtension(morphlib.writeexts.WriteExtension): +class VirtualBoxPlusSshCheckExtension(writeexts.WriteExtension): def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') self.require_btrfs_in_deployment_host_kernel() upgrade = self.get_environment_boolean('UPGRADE') if upgrade: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Use the `ssh-rsync` write extension to deploy upgrades to an ' 'existing remote system.') diff --git a/extensions/virtualbox-ssh.write b/extensions/virtualbox-ssh.write index 774f2b4f..95643a4a 100755 --- a/extensions/virtualbox-ssh.write +++ b/extensions/virtualbox-ssh.write @@ -24,22 +24,23 @@ See file virtualbox-ssh.write.help for documentation ''' -import cliapp import os import re +import subprocess import sys -import time import tempfile +import time import urlparse -import morphlib.writeexts +import writeexts -class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): +class VirtualBoxPlusSshWriteExtension(writeexts.WriteExtension): def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') temp_root, location = args ssh_host, vm_name, vdi_path = self.parse_location(location) @@ -59,7 +60,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): except BaseException: sys.stderr.write('Error deploying to VirtualBox') os.remove(raw_disk) - cliapp.ssh_runcmd(ssh_host, ['rm', '-f', vdi_path]) + writeexts.ssh_runcmd(ssh_host, ['rm', '-f', vdi_path]) raise else: os.remove(raw_disk) @@ -72,11 +73,12 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): x = urlparse.urlparse(location) if x.scheme != 'vbox+ssh': - raise cliapp.AppException( + raise writeexts.ExtensionError( 'URL schema must be vbox+ssh in %s' % location) m = re.match('^/(?P<guest>[^/]+)(?P<path>/.+)$', x.path) if not m: - raise cliapp.AppException('Cannot parse location %s' % location) + raise writeexts.ExtensionError( + 'Cannot parse location %s' % location) return x.netloc, m.group('guest'), m.group('path') def transfer_and_convert_to_vdi(self, raw_disk, ssh_host, vdi_path): @@ -85,17 +87,18 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Transfer disk and convert to VDI') st = os.lstat(raw_disk) - xfer_hole_path = morphlib.util.get_data_path('xfer-hole') - recv_hole = morphlib.util.get_data('recv-hole') + # TODO: Something! + xfer_hole_path = writeexts.get_data_path('xfer-hole') + recv_hole = writeexts.get_data('recv-hole') ssh_remote_cmd = [ 'sh', '-c', recv_hole, 'dummy-argv0', 'vbox', vdi_path, str(st.st_size), ] - cliapp.runcmd( + subprocess.check_call( ['python', xfer_hole_path, raw_disk], - ['ssh', ssh_host] + map(cliapp.shell_quote, ssh_remote_cmd), + ['ssh', ssh_host] + map(writeexts.shell_quote, ssh_remote_cmd), stdout=None, stderr=None) def virtualbox_version(self, ssh_host): @@ -107,7 +110,8 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): # tuple is more reliable than a string and more convenient than # comparing against the major, minor and patch numbers directly self.status(msg='Checking version of remote VirtualBox') - build_id = cliapp.ssh_runcmd(ssh_host, ['VBoxManage', '--version']) + build_id = writeexts.ssh_runcmd(ssh_host, + ['VBoxManage', '--version']) version_string = re.match(r"^([0-9\.]+).*$", build_id.strip()).group(1) return tuple(int(s or '0') for s in version_string.split('.')) @@ -163,17 +167,17 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): for command in commands: argv = ['VBoxManage'] + command - cliapp.ssh_runcmd(ssh_host, argv) + writeexts.ssh_runcmd(ssh_host, argv) def get_host_interface(self, ssh_host): host_ipaddr = os.environ.get('HOST_IPADDR') netmask = os.environ.get('NETMASK') if host_ipaddr is None: - raise cliapp.AppException('HOST_IPADDR was not given') + raise writeexts.ExtensionError('HOST_IPADDR was not given') if netmask is None: - raise cliapp.AppException('NETMASK was not given') + raise writeexts.ExtensionError('NETMASK was not given') # 'VBoxManage list hostonlyifs' retrieves a list with the hostonly # interfaces on the host. For each interface, the following lines @@ -187,7 +191,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): # The following command tries to retrieve the hostonly interface # name (e.g. vboxnet0) associated with the given ip address. iface = None - lines = cliapp.ssh_runcmd(ssh_host, + lines = writeexts.ssh_runcmd(ssh_host, ['VBoxManage', 'list', 'hostonlyifs']).splitlines() for i, v in enumerate(lines): if host_ipaddr in v: @@ -195,12 +199,12 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): break if iface is None: - iface = cliapp.ssh_runcmd(ssh_host, + iface = writeexts.ssh_runcmd(ssh_host, ['VBoxManage', 'hostonlyif', 'create']) # 'VBoxManage hostonlyif create' shows the name of the # created hostonly interface inside single quotes iface = iface[iface.find("'") + 1 : iface.rfind("'")] - cliapp.ssh_runcmd(ssh_host, + writeexts.ssh_runcmd(ssh_host, ['VBoxManage', 'hostonlyif', 'ipconfig', iface, '--ip', host_ipaddr, diff --git a/extensions/writeexts.py b/extensions/writeexts.py index 61d40789..f84f2288 100644 --- a/extensions/writeexts.py +++ b/extensions/writeexts.py @@ -1,4 +1,5 @@ # Copyright (C) 2012-2015 Codethink Limited +# Copyright (C) 2011, 2012 Lars Wirzenius # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,9 +16,11 @@ import contextlib import errno +import fcntl import logging import os import re +import select import shutil import stat import subprocess @@ -26,8 +29,221 @@ import tempfile import time -def shell_quote(string): - '''Return a shell-quoted version of `string`.''' +def get_data_path(relative_path): + '''Return path to a data file in the morphlib Python package. + + ``relative_path`` is the name of the data file, relative to the + `extensions/` directory. + + ''' + + extensions_dir = os.path.dirname(__file__) + return os.path.join(extensions_dir, relative_path) + + +def get_data(relative_path): # pragma: no cover + '''Return contents of a data file from the morphlib Python package. + + ``relative_path`` is the name of the data file, relative to the + `extensions/` directory. + + ''' + + with open(get_data_path(relative_path)) as f: + return f.read() + + +def runcmd(argv, *args, **kwargs): + '''Run external command or pipeline. + + Example: ``runcmd(['grep', 'foo'], ['wc', '-l'], + feed_stdin='foo\nbar\n')`` + + Return the standard output of the command. + + Raise ``ExtensionError`` if external command returns + non-zero exit code. ``*args`` and ``**kwargs`` are passed + onto ``subprocess.Popen``. + + ''' + + our_options = ( + ('ignore_fail', False), + ('log_error', True), + ) + opts = {} + for name, default in our_options: + opts[name] = default + if name in kwargs: + opts[name] = kwargs[name] + del kwargs[name] + + exit, out, err = runcmd_unchecked(argv, *args, **kwargs) + if exit != 0: + msg = 'Command failed: %s\n%s' % (' '.join(argv), err) + if opts['ignore_fail']: + if opts['log_error']: + logging.info(msg) + else: + if opts['log_error']: + logging.error(msg) + raise ExtensionError(msg) + return out + +def runcmd_unchecked(argv, *argvs, **kwargs): + '''Run external command or pipeline. + + Return the exit code, and contents of standard output and error + of the command. + + See also ``runcmd``. + + ''' + + argvs = [argv] + list(argvs) + logging.debug('run external command: %s' % repr(argvs)) + + def pop_kwarg(name, default): + if name in kwargs: + value = kwargs[name] + del kwargs[name] + return value + else: + return default + + feed_stdin = pop_kwarg('feed_stdin', '') + pipe_stdin = pop_kwarg('stdin', subprocess.PIPE) + pipe_stdout = pop_kwarg('stdout', subprocess.PIPE) + pipe_stderr = pop_kwarg('stderr', subprocess.PIPE) + + try: + pipeline = _build_pipeline(argvs, + pipe_stdin, + pipe_stdout, + pipe_stderr, + kwargs) + return _run_pipeline(pipeline, feed_stdin, pipe_stdin, + pipe_stdout, pipe_stderr) + except OSError, e: # pragma: no cover + if e.errno == errno.ENOENT and e.filename is None: + e.filename = argv[0] + raise e + else: + raise + +def _build_pipeline(argvs, pipe_stdin, pipe_stdout, pipe_stderr, kwargs): + procs = [] + for i, argv in enumerate(argvs): + if i == 0 and i == len(argvs) - 1: + stdin = pipe_stdin + stdout = pipe_stdout + stderr = pipe_stderr + elif i == 0: + stdin = pipe_stdin + stdout = subprocess.PIPE + stderr = pipe_stderr + elif i == len(argvs) - 1: + stdin = procs[-1].stdout + stdout = pipe_stdout + stderr = pipe_stderr + else: + stdin = procs[-1].stdout + stdout = subprocess.PIPE + stderr = pipe_stderr + p = subprocess.Popen(argv, stdin=stdin, stdout=stdout, + stderr=stderr, close_fds=True, **kwargs) + procs.append(p) + + return procs + +def _run_pipeline(procs, feed_stdin, pipe_stdin, pipe_stdout, pipe_stderr): + + stdout_eof = False + stderr_eof = False + out = [] + err = [] + pos = 0 + io_size = 1024 + + def set_nonblocking(fd): + flags = fcntl.fcntl(fd, fcntl.F_GETFL, 0) + flags = flags | os.O_NONBLOCK + fcntl.fcntl(fd, fcntl.F_SETFL, flags) + + if feed_stdin and pipe_stdin == subprocess.PIPE: + set_nonblocking(procs[0].stdin.fileno()) + if pipe_stdout == subprocess.PIPE: + set_nonblocking(procs[-1].stdout.fileno()) + if pipe_stderr == subprocess.PIPE: + set_nonblocking(procs[-1].stderr.fileno()) + + def still_running(): + for p in procs: + p.poll() + for p in procs: + if p.returncode is None: + return True + if pipe_stdout == subprocess.PIPE and not stdout_eof: + return True + if pipe_stderr == subprocess.PIPE and not stderr_eof: + return True # pragma: no cover + return False + + while still_running(): + rlist = [] + if not stdout_eof and pipe_stdout == subprocess.PIPE: + rlist.append(procs[-1].stdout) + if not stderr_eof and pipe_stderr == subprocess.PIPE: + rlist.append(procs[-1].stderr) + + wlist = [] + if pipe_stdin == subprocess.PIPE and pos < len(feed_stdin): + wlist.append(procs[0].stdin) + + if rlist or wlist: + try: + r, w, x = select.select(rlist, wlist, []) + except select.error, e: + err, msg = e.args + if err == errno.EINTR: + break + raise + else: + break # Let's not busywait waiting for processes to die. + + if procs[0].stdin in w and pos < len(feed_stdin): + data = feed_stdin[pos : pos+io_size] + procs[0].stdin.write(data) + pos += len(data) + if pos >= len(feed_stdin): + procs[0].stdin.close() + + if procs[-1].stdout in r: + data = procs[-1].stdout.read(io_size) + if data: + out.append(data) + else: + stdout_eof = True + + if procs[-1].stderr in r: + data = procs[-1].stderr.read(io_size) + if data: + err.append(data) + else: + stderr_eof = True + + while still_running(): + for p in procs: + if p.returncode is None: + p.wait() + + errorcodes = [p.returncode for p in procs if p.returncode != 0] or [0] + return errorcodes[-1], ''.join(out), ''.join(err) + + +def shell_quote(s): + '''Return a shell-quoted version of s.''' + lower_ascii = 'abcdefghijklmnopqrstuvwxyz' upper_ascii = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ' digits = '0123456789' @@ -35,25 +251,65 @@ def shell_quote(string): safe = set(lower_ascii + upper_ascii + digits + punctuation) quoted = [] - for character in string: - if character in safe: - quoted.append(character) - elif character == "'": + for c in s: + if c in safe: + quoted.append(c) + elif c == "'": quoted.append('"\'"') else: - quoted.append("'%c'" % character) + quoted.append("'%c'" % c) return ''.join(quoted) -def run_ssh_command(host, command): - '''Run `command` over SSH on `host`.''' - ssh_cmd = ['ssh', host, '--'] + [shell_quote(arg) for arg in command] - return subprocess.check_output(ssh_cmd) +def ssh_runcmd(target, argv, **kwargs): + '''Run command in argv on remote host target. + + This is similar to runcmd, but the command is run on the remote + machine. The command is given as an argv array; elements in the + array are automatically quoted so they get passed to the other + side correctly. + An optional ``tty=`` parameter can be passed to ``ssh_runcmd`` in + order to force or disable pseudo-tty allocation. This is often + required to run ``sudo`` on another machine and might be useful + in other situations as well. Supported values are ``tty=True`` for + forcing tty allocation, ``tty=False`` for disabling it and + ``tty=None`` for not passing anything tty related to ssh. + + With the ``tty`` option, + ``cliapp.runcmd(['ssh', '-tt', 'user@host', '--', 'sudo', 'ls'])`` + can be written as + ``cliapp.ssh_runcmd('user@host', ['sudo', 'ls'], tty=True)`` + which is more intuitive. + + The target is given as-is to ssh, and may use any syntax ssh + accepts. + + Environment variables may or may not be passed to the remote + machine: this is dependent on the ssh and sshd configurations. + Invoke env(1) explicitly to pass in the variables you need to + exist on the other end. + + Pipelines are not supported. + + ''' + + tty = kwargs.get('tty', None) + if tty: + ssh_cmd = ['ssh', '-tt', target, '--'] + elif tty is False: + ssh_cmd = ['ssh', '-T', target, '--'] + else: + ssh_cmd = ['ssh', target, '--'] + if 'tty' in kwargs: + del kwargs['tty'] + + local_argv = ssh_cmd + map(shell_quote, argv) + return runcmd(local_argv, **kwargs) def write_from_dict(filepath, d, validate=lambda x, y: True): - '''Takes a dictionary and appends the contents to a file + """Takes a dictionary and appends the contents to a file An optional validation callback can be passed to perform validation on each value in the dictionary. @@ -66,7 +322,8 @@ def write_from_dict(filepath, d, validate=lambda x, y: True): Any callback supplied to this function should raise an exception if validation fails. - ''' + + """ # Sort items asciibetically # the output of the deployment should not depend @@ -84,6 +341,31 @@ def write_from_dict(filepath, d, validate=lambda x, y: True): os.fchmod(f.fileno(), 0644) +def parse_environment_pairs(env, pairs): + '''Add key=value pairs to the environment dict. + + Given a dict and a list of strings of the form key=value, + set dict[key] = value, unless key is already set in the + environment, at which point raise an exception. + + This does not modify the passed in dict. + + Returns the extended dict. + + ''' + + extra_env = dict(p.split('=', 1) for p in pairs) + conflicting = [k for k in extra_env if k in env] + if conflicting: + raise EnvironmentAlreadySetError(conflicting) + + # Return a dict that is the union of the two + # This is not the most performant, since it creates + # 3 unnecessary lists, but I felt this was the most + # easy to read. Using itertools.chain may be more efficicent + return dict(env.items() + extra_env.items()) + + class ExtensionError(Exception): def __init__(self, msg): @@ -146,9 +428,9 @@ class Fstab(object): shutil.move(os.path.abspath(tmp), os.path.abspath(self.filepath)) -class WriteExtension(object): +class Extension(object): - '''A base class for deployment write extensions. + '''A base class for deployment extensions. A subclass should subclass this class, and add a ``process_args`` method. @@ -190,7 +472,7 @@ class WriteExtension(object): self.setup_logging() self.process_args(args) except ExtensionError as e: - sys.stdout.write('ERROR: %s' % e) + sys.stdout.write('ERROR: %s\n' % e) sys.exit(1) def status(self, **kwargs): @@ -204,6 +486,20 @@ class WriteExtension(object): sys.stdout.write('%s\n' % (kwargs['msg'] % kwargs)) sys.stdout.flush() + +class WriteExtension(Extension): + + '''A base class for deployment write extensions. + + A subclass should subclass this class, and add a + ``process_args`` method. + + Note that it is not necessary to subclass this class for write + extensions. This class is here just to collect common code for + write extensions. + + ''' + def check_for_btrfs_in_deployment_host_kernel(self): with open('/proc/filesystems') as f: text = f.read() @@ -676,7 +972,7 @@ class WriteExtension(object): def check_ssh_connectivity(self, ssh_host): try: - output = run_ssh_command(ssh_host, ['echo', 'test']) + output = ssh_runcmd(ssh_host, ['echo', 'test']) except subprocess.CalledProcessError as e: logging.error("Error checking SSH connectivity: %s", str(e)) raise ExtensionError( |