From 0a922a4813cb862d840c02a123b6da558af69aa7 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Wed, 3 Jun 2015 11:52:48 +0000 Subject: Put writeexts.py in extensions subdirectory Change-Id: I6eb8fe69416bbf483ffa8c1c317c8f8cea56ef0e --- extensions/writeexts.py | 624 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 624 insertions(+) create mode 100644 extensions/writeexts.py (limited to 'extensions') diff --git a/extensions/writeexts.py b/extensions/writeexts.py new file mode 100644 index 00000000..aa185a2b --- /dev/null +++ b/extensions/writeexts.py @@ -0,0 +1,624 @@ +# Copyright (C) 2012-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + + +import cliapp +import logging +import os +import re +import shutil +import sys +import time +import tempfile +import errno +import stat +import contextlib + +import morphlib + + +class Fstab(object): + '''Small helper class for parsing and adding lines to /etc/fstab.''' + + # There is an existing Python helper library for editing of /etc/fstab. + # However it is unmaintained and has an incompatible license (GPL3). + # + # https://code.launchpad.net/~computer-janitor-hackers/python-fstab/trunk + + def __init__(self, filepath='/etc/fstab'): + if os.path.exists(filepath): + with open(filepath, 'r') as f: + self.text= f.read() + else: + self.text = '' + self.filepath = filepath + self.lines_added = 0 + + def get_mounts(self): + '''Return list of mount devices and targets in /etc/fstab. + + Return value is a dict of target -> device. + ''' + mounts = dict() + for line in self.text.splitlines(): + words = line.split() + if len(words) >= 2 and not words[0].startswith('#'): + device, target = words[0:2] + mounts[target] = device + return mounts + + def add_line(self, line): + '''Add a new entry to /etc/fstab. + + Lines are appended, and separated from any entries made by configure + extensions with a comment. + + ''' + if self.lines_added == 0: + if len(self.text) == 0 or self.text[-1] is not '\n': + self.text += '\n' + self.text += '# Morph default system layout\n' + self.lines_added += 1 + + self.text += line + '\n' + + def write(self): + '''Rewrite the fstab file to include all new entries.''' + with morphlib.savefile.SaveFile(self.filepath, 'w') as f: + f.write(self.text) + + +class WriteExtension(cliapp.Application): + + '''A base class for deployment write extensions. + + A subclass should subclass this class, and add a + ``process_args`` method. + + Note that it is not necessary to subclass this class for write + extensions. This class is here just to collect common code for + write extensions. + + ''' + + def setup_logging(self): + '''Direct all logging output to MORPH_LOG_FD, if set. + + This file descriptor is read by Morph and written into its own log + file. + + This overrides cliapp's usual configurable logging setup. + + ''' + log_write_fd = int(os.environ.get('MORPH_LOG_FD', 0)) + + if log_write_fd == 0: + return + + formatter = logging.Formatter('%(message)s') + + handler = logging.StreamHandler(os.fdopen(log_write_fd, 'w')) + handler.setFormatter(formatter) + + logger = logging.getLogger() + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) + + def log_config(self): + with morphlib.util.hide_password_environment_variables(os.environ): + cliapp.Application.log_config(self) + + def process_args(self, args): + raise NotImplementedError() + + def status(self, **kwargs): + '''Provide status output. + + The ``msg`` keyword argument is the actual message, + the rest are values for fields in the message as interpolated + by %. + + ''' + + self.output.write('%s\n' % (kwargs['msg'] % kwargs)) + self.output.flush() + + def check_for_btrfs_in_deployment_host_kernel(self): + with open('/proc/filesystems') as f: + text = f.read() + return '\tbtrfs\n' in text + + def require_btrfs_in_deployment_host_kernel(self): + if not self.check_for_btrfs_in_deployment_host_kernel(): + raise cliapp.AppException( + 'Error: Btrfs is required for this deployment, but was not ' + 'detected in the kernel of the machine that is running Morph.') + + def create_local_system(self, temp_root, raw_disk): + '''Create a raw system image locally.''' + + with self.created_disk_image(raw_disk): + self.format_btrfs(raw_disk) + self.create_system(temp_root, raw_disk) + + @contextlib.contextmanager + def created_disk_image(self, location): + size = self.get_disk_size() + if not size: + raise cliapp.AppException('DISK_SIZE is not defined') + self.create_raw_disk_image(location, size) + try: + yield + except BaseException: + os.unlink(location) + raise + + def format_btrfs(self, raw_disk): + try: + self.mkfs_btrfs(raw_disk) + except BaseException: + sys.stderr.write('Error creating disk image') + raise + + def create_system(self, temp_root, raw_disk): + with self.mount(raw_disk) as mp: + try: + self.create_btrfs_system_layout( + temp_root, mp, version_label='factory', + disk_uuid=self.get_uuid(raw_disk)) + except BaseException as e: + sys.stderr.write('Error creating Btrfs system layout') + raise + + def _parse_size(self, size): + '''Parse a size from a string. + + Return size in bytes. + + ''' + + m = re.match('^(\d+)([kmgKMG]?)$', size) + if not m: + return None + + factors = { + '': 1, + 'k': 1024, + 'm': 1024**2, + 'g': 1024**3, + } + factor = factors[m.group(2).lower()] + + return int(m.group(1)) * factor + + def _parse_size_from_environment(self, env_var, default): + '''Parse a size from an environment variable.''' + + size = os.environ.get(env_var, default) + if size is None: + return None + bytes = self._parse_size(size) + if bytes is None: + raise morphlib.Error('Cannot parse %s value %s' % (env_var, size)) + return bytes + + def get_disk_size(self): + '''Parse disk size from environment.''' + return self._parse_size_from_environment('DISK_SIZE', None) + + def get_ram_size(self): + '''Parse RAM size from environment.''' + return self._parse_size_from_environment('RAM_SIZE', '1G') + + def get_vcpu_count(self): + '''Parse the virtual cpu count from environment.''' + return self._parse_size_from_environment('VCPUS', '1') + + def create_raw_disk_image(self, filename, size): + '''Create a raw disk image.''' + + self.status(msg='Creating empty disk image') + with open(filename, 'wb') as f: + if size > 0: + f.seek(size-1) + f.write('\0') + + def mkfs_btrfs(self, location): + '''Create a btrfs filesystem on the disk.''' + + self.status(msg='Creating btrfs filesystem') + try: + # The following command disables some new filesystem features. We + # need to do this because at the time of writing, SYSLINUX has not + # been updated to understand these new features and will fail to + # boot if the kernel is on a filesystem where they are enabled. + cliapp.runcmd( + ['mkfs.btrfs','-f', '-L', 'baserock', + '--features', '^extref', + '--features', '^skinny-metadata', + '--features', '^mixed-bg', + '--nodesize', '4096', + location]) + except cliapp.AppException as e: + if 'unrecognized option \'--features\'' in e.msg: + # Old versions of mkfs.btrfs (including v0.20, present in many + # Baserock releases) don't support the --features option, but + # also don't enable the new features by default. So we can + # still create a bootable system in this situation. + logging.debug( + 'Assuming mkfs.btrfs failure was because the tool is too ' + 'old to have --features flag.') + cliapp.runcmd(['mkfs.btrfs','-f', '-L', 'baserock', location]) + else: + raise + + def get_uuid(self, location): + '''Get the UUID of a block device's file system.''' + # Requires util-linux blkid; busybox one ignores options and + # lies by exiting successfully. + return cliapp.runcmd(['blkid', '-s', 'UUID', '-o', 'value', + location]).strip() + + @contextlib.contextmanager + def mount(self, location): + self.status(msg='Mounting filesystem') + try: + mount_point = tempfile.mkdtemp() + if self.is_device(location): + cliapp.runcmd(['mount', location, mount_point]) + else: + cliapp.runcmd(['mount', '-o', 'loop', location, mount_point]) + except BaseException as e: + sys.stderr.write('Error mounting filesystem') + os.rmdir(mount_point) + raise + try: + yield mount_point + finally: + self.status(msg='Unmounting filesystem') + cliapp.runcmd(['umount', mount_point]) + os.rmdir(mount_point) + + def create_btrfs_system_layout(self, temp_root, mountpoint, version_label, + disk_uuid): + '''Separate base OS versions from state using subvolumes. + + ''' + initramfs = self.find_initramfs(temp_root) + version_root = os.path.join(mountpoint, 'systems', version_label) + state_root = os.path.join(mountpoint, 'state') + + os.makedirs(version_root) + os.makedirs(state_root) + + self.create_orig(version_root, temp_root) + system_dir = os.path.join(version_root, 'orig') + + state_dirs = self.complete_fstab_for_btrfs_layout(system_dir, + disk_uuid) + + for state_dir in state_dirs: + self.create_state_subvolume(system_dir, mountpoint, state_dir) + + self.create_run(version_root) + + os.symlink( + version_label, os.path.join(mountpoint, 'systems', 'default')) + + if self.bootloader_config_is_wanted(): + self.install_kernel(version_root, temp_root) + if self.get_dtb_path() != '': + self.install_dtb(version_root, temp_root) + self.install_syslinux_menu(mountpoint, version_root) + if initramfs is not None: + self.install_initramfs(initramfs, version_root) + self.generate_bootloader_config(mountpoint, disk_uuid) + else: + self.generate_bootloader_config(mountpoint) + self.install_bootloader(mountpoint) + + def create_orig(self, version_root, temp_root): + '''Create the default "factory" system.''' + + orig = os.path.join(version_root, 'orig') + + self.status(msg='Creating orig subvolume') + cliapp.runcmd(['btrfs', 'subvolume', 'create', orig]) + self.status(msg='Copying files to orig subvolume') + cliapp.runcmd(['cp', '-a', temp_root + '/.', orig + '/.']) + + def create_run(self, version_root): + '''Create the 'run' snapshot.''' + + self.status(msg='Creating run subvolume') + orig = os.path.join(version_root, 'orig') + run = os.path.join(version_root, 'run') + cliapp.runcmd( + ['btrfs', 'subvolume', 'snapshot', orig, run]) + + def create_state_subvolume(self, system_dir, mountpoint, state_subdir): + '''Create a shared state subvolume. + + We need to move any files added to the temporary rootfs by the + configure extensions to their correct home. For example, they might + have added keys in `/root/.ssh` which we now need to transfer to + `/state/root/.ssh`. + + ''' + self.status(msg='Creating %s subvolume' % state_subdir) + subvolume = os.path.join(mountpoint, 'state', state_subdir) + cliapp.runcmd(['btrfs', 'subvolume', 'create', subvolume]) + os.chmod(subvolume, 0o755) + + existing_state_dir = os.path.join(system_dir, state_subdir) + files = [] + if os.path.exists(existing_state_dir): + files = os.listdir(existing_state_dir) + if len(files) > 0: + self.status(msg='Moving existing data to %s subvolume' % subvolume) + for filename in files: + filepath = os.path.join(existing_state_dir, filename) + cliapp.runcmd(['mv', filepath, subvolume]) + + def complete_fstab_for_btrfs_layout(self, system_dir, rootfs_uuid=None): + '''Fill in /etc/fstab entries for the default Btrfs disk layout. + + In the future we should move this code out of the write extension and + in to a configure extension. To do that, though, we need some way of + informing the configure extension what layout should be used. Right now + a configure extension doesn't know if the system is going to end up as + a Btrfs disk image, a tarfile or something else and so it can't come + up with a sensible default fstab. + + Configuration extensions can already create any /etc/fstab that they + like. This function only fills in entries that are missing, so if for + example the user configured /home to be on a separate partition, that + decision will be honoured and /state/home will not be created. + + ''' + shared_state_dirs = {'home', 'root', 'opt', 'srv', 'var'} + + fstab = Fstab(os.path.join(system_dir, 'etc', 'fstab')) + existing_mounts = fstab.get_mounts() + + if '/' in existing_mounts: + root_device = existing_mounts['/'] + else: + root_device = (self.get_root_device() if rootfs_uuid is None else + 'UUID=%s' % rootfs_uuid) + fstab.add_line('%s / btrfs defaults,rw,noatime 0 1' % root_device) + + state_dirs_to_create = set() + for state_dir in shared_state_dirs: + if '/' + state_dir not in existing_mounts: + state_dirs_to_create.add(state_dir) + state_subvol = os.path.join('/state', state_dir) + fstab.add_line( + '%s /%s btrfs subvol=%s,defaults,rw,noatime 0 2' % + (root_device, state_dir, state_subvol)) + + fstab.write() + return state_dirs_to_create + + def find_initramfs(self, temp_root): + '''Check whether the rootfs has an initramfs. + + Uses the INITRAMFS_PATH option to locate it. + ''' + if 'INITRAMFS_PATH' in os.environ: + initramfs = os.path.join(temp_root, os.environ['INITRAMFS_PATH']) + if not os.path.exists(initramfs): + raise morphlib.Error('INITRAMFS_PATH specified, ' + 'but file does not exist') + return initramfs + return None + + def install_initramfs(self, initramfs_path, version_root): + '''Install the initramfs outside of 'orig' or 'run' subvolumes. + + This is required because syslinux doesn't traverse subvolumes when + loading the kernel or initramfs. + ''' + self.status(msg='Installing initramfs') + initramfs_dest = os.path.join(version_root, 'initramfs') + cliapp.runcmd(['cp', '-a', initramfs_path, initramfs_dest]) + + def install_kernel(self, version_root, temp_root): + '''Install the kernel outside of 'orig' or 'run' subvolumes''' + + self.status(msg='Installing kernel') + image_names = ['vmlinuz', 'zImage', 'uImage'] + kernel_dest = os.path.join(version_root, 'kernel') + for name in image_names: + try_path = os.path.join(temp_root, 'boot', name) + if os.path.exists(try_path): + cliapp.runcmd(['cp', '-a', try_path, kernel_dest]) + break + + def install_dtb(self, version_root, temp_root): + '''Install the device tree outside of 'orig' or 'run' subvolumes''' + + self.status(msg='Installing devicetree') + device_tree_path = self.get_dtb_path() + dtb_dest = os.path.join(version_root, 'dtb') + try_path = os.path.join(temp_root, device_tree_path) + if os.path.exists(try_path): + cliapp.runcmd(['cp', '-a', try_path, dtb_dest]) + else: + logging.error("Failed to find device tree %s", device_tree_path) + raise cliapp.AppException( + 'Failed to find device tree %s' % device_tree_path) + + def get_dtb_path(self): + return os.environ.get('DTB_PATH', '') + + def get_bootloader_install(self): + # Do we actually want to install the bootloader? + # Set this to "none" to prevent the install + return os.environ.get('BOOTLOADER_INSTALL', 'extlinux') + + def get_bootloader_config_format(self): + # The config format for the bootloader, + # if not set we default to extlinux for x86 + return os.environ.get('BOOTLOADER_CONFIG_FORMAT', 'extlinux') + + def get_extra_kernel_args(self): + return os.environ.get('KERNEL_ARGS', '') + + def get_root_device(self): + return os.environ.get('ROOT_DEVICE', '/dev/sda') + + def generate_bootloader_config(self, real_root, disk_uuid=None): + '''Install extlinux on the newly created disk image.''' + config_function_dict = { + 'extlinux': self.generate_extlinux_config, + } + + config_type = self.get_bootloader_config_format() + if config_type in config_function_dict: + config_function_dict[config_type](real_root, disk_uuid) + else: + raise cliapp.AppException( + 'Invalid BOOTLOADER_CONFIG_FORMAT %s' % config_type) + + def generate_extlinux_config(self, real_root, disk_uuid=None): + '''Install extlinux on the newly created disk image.''' + + self.status(msg='Creating extlinux.conf') + config = os.path.join(real_root, 'extlinux.conf') + + ''' Please also update the documentation in the following files + if you change these default kernel args: + - kvm.write.help + - rawdisk.write.help + - virtualbox-ssh.write.help ''' + kernel_args = ( + 'rw ' # ro ought to work, but we don't test that regularly + 'init=/sbin/init ' # default, but it doesn't hurt to be explicit + 'rootfstype=btrfs ' # required when using initramfs, also boots + # faster when specified without initramfs + 'rootflags=subvol=systems/default/run ') # boot runtime subvol + kernel_args += 'root=%s ' % (self.get_root_device() + if disk_uuid is None + else 'UUID=%s' % disk_uuid) + kernel_args += self.get_extra_kernel_args() + with open(config, 'w') as f: + f.write('default linux\n') + f.write('timeout 1\n') + f.write('label linux\n') + f.write('kernel /systems/default/kernel\n') + if disk_uuid is not None: + f.write('initrd /systems/default/initramfs\n') + if self.get_dtb_path() != '': + f.write('devicetree /systems/default/dtb\n') + f.write('append %s\n' % kernel_args) + + def install_bootloader(self, real_root): + install_function_dict = { + 'extlinux': self.install_bootloader_extlinux, + } + + install_type = self.get_bootloader_install() + if install_type in install_function_dict: + install_function_dict[install_type](real_root) + elif install_type != 'none': + raise cliapp.AppException( + 'Invalid BOOTLOADER_INSTALL %s' % install_type) + + def install_bootloader_extlinux(self, real_root): + self.status(msg='Installing extlinux') + cliapp.runcmd(['extlinux', '--install', real_root]) + + # FIXME this hack seems to be necessary to let extlinux finish + cliapp.runcmd(['sync']) + time.sleep(2) + + def install_syslinux_menu(self, real_root, version_root): + '''Make syslinux/extlinux menu binary available. + + The syslinux boot menu is compiled to a file named menu.c32. Extlinux + searches a few places for this file but it does not know to look inside + our subvolume, so we copy it to the filesystem root. + + If the file is not available, the bootloader will still work but will + not be able to show a menu. + + ''' + menu_file = os.path.join(version_root, 'orig', + 'usr', 'share', 'syslinux', 'menu.c32') + if os.path.isfile(menu_file): + self.status(msg='Copying menu.c32') + shutil.copy(menu_file, real_root) + + def parse_attach_disks(self): + '''Parse $ATTACH_DISKS into list of disks to attach.''' + + if 'ATTACH_DISKS' in os.environ: + s = os.environ['ATTACH_DISKS'] + return s.split(':') + else: + return [] + + def bootloader_config_is_wanted(self): + '''Does the user want to generate a bootloader config? + + The user may set $BOOTLOADER_CONFIG_FORMAT to the desired + format. 'extlinux' is the only allowed value, and is the default + value for x86-32 and x86-64. + + ''' + + def is_x86(arch): + return (arch == 'x86_64' or + (arch.startswith('i') and arch.endswith('86'))) + + value = os.environ.get('BOOTLOADER_CONFIG_FORMAT', '') + if value == '': + if not is_x86(os.uname()[-1]): + return False + + return True + + def get_environment_boolean(self, variable): + '''Parse a yes/no boolean passed through the environment.''' + + value = os.environ.get(variable, 'no').lower() + if value in ['no', '0', 'false']: + return False + elif value in ['yes', '1', 'true']: + return True + else: + raise cliapp.AppException('Unexpected value for %s: %s' % + (variable, value)) + + def check_ssh_connectivity(self, ssh_host): + try: + output = cliapp.ssh_runcmd(ssh_host, ['echo', 'test']) + except cliapp.AppException as e: + logging.error("Error checking SSH connectivity: %s", str(e)) + raise cliapp.AppException( + 'Unable to SSH to %s: %s' % (ssh_host, e)) + + if output.strip() != 'test': + raise cliapp.AppException( + 'Unexpected output from remote machine: %s' % output.strip()) + + def is_device(self, location): + try: + st = os.stat(location) + return stat.S_ISBLK(st.st_mode) + except OSError as e: + if e.errno == errno.ENOENT: + return False + raise -- cgit v1.2.1 From d3c9b459d837f62532818be7f64c3b277a3e5885 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Wed, 10 Jun 2015 09:59:07 +0000 Subject: Stop writeexts.py depending on morphlib Change-Id: I7f3702e80678aeee89dd22116510a6d8d7e04841 --- extensions/writeexts.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) (limited to 'extensions') diff --git a/extensions/writeexts.py b/extensions/writeexts.py index aa185a2b..9357648f 100644 --- a/extensions/writeexts.py +++ b/extensions/writeexts.py @@ -25,7 +25,15 @@ import errno import stat import contextlib -import morphlib + +@contextlib.contextmanager +def hide_password_environment_variables(env): # pragma: no cover + password_env = { k:v for k,v in env.iteritems() if 'PASSWORD' in k } + for k in password_env: + env[k] = '(value hidden)' + yield + for k, v in password_env.iteritems(): + env[k] = v class Fstab(object): @@ -75,8 +83,10 @@ class Fstab(object): def write(self): '''Rewrite the fstab file to include all new entries.''' - with morphlib.savefile.SaveFile(self.filepath, 'w') as f: + with tempfile.NamedTemporaryFile(delete=False) as f: f.write(self.text) + tmp = f.name + shutil.move(os.path.abspath(tmp), os.path.abspath(self.filepath)) class WriteExtension(cliapp.Application): @@ -116,7 +126,7 @@ class WriteExtension(cliapp.Application): logger.setLevel(logging.DEBUG) def log_config(self): - with morphlib.util.hide_password_environment_variables(os.environ): + with hide_password_environment_variables(os.environ): cliapp.Application.log_config(self) def process_args(self, args): @@ -210,7 +220,8 @@ class WriteExtension(cliapp.Application): return None bytes = self._parse_size(size) if bytes is None: - raise morphlib.Error('Cannot parse %s value %s' % (env_var, size)) + raise cliapp.AppException('Cannot parse %s value %s' + % (env_var, size)) return bytes def get_disk_size(self): @@ -419,8 +430,8 @@ class WriteExtension(cliapp.Application): if 'INITRAMFS_PATH' in os.environ: initramfs = os.path.join(temp_root, os.environ['INITRAMFS_PATH']) if not os.path.exists(initramfs): - raise morphlib.Error('INITRAMFS_PATH specified, ' - 'but file does not exist') + raise cliapp.AppException('INITRAMFS_PATH specified, ' + 'but file does not exist') return initramfs return None -- cgit v1.2.1 From 840292841f4495a79a037f81a26d6b3f51e7cb8c Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Thu, 4 Jun 2015 15:25:33 +0000 Subject: Move xfer-hole and recv-hole into extensions/ Change-Id: I46ed5f3ec85f9662bebac592eff7a6eb6d628f28 --- extensions/recv-hole | 158 +++++++++++++++++++++++++++++++++++++++++++++++++++ extensions/xfer-hole | 137 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 295 insertions(+) create mode 100755 extensions/recv-hole create mode 100755 extensions/xfer-hole (limited to 'extensions') diff --git a/extensions/recv-hole b/extensions/recv-hole new file mode 100755 index 00000000..fe69f304 --- /dev/null +++ b/extensions/recv-hole @@ -0,0 +1,158 @@ +#!/bin/sh +# +# Copyright (C) 2014-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# =*= License: GPL-2 =*= + + +# Receive a data stream describing a sparse file, and reproduce it, +# either to a named file or stdout. +# +# The data stream is simple: it's a sequence of DATA or HOLE records: +# +# DATA +# 123 +# <123 bytes of binary data, NOT including newline at the end> +# +# HOLE +# 123 +# +# This shell script can be executed over ssh (given to ssh as an arguemnt, +# with suitable escaping) on a different computer. This allows a large +# sparse file (e.g., disk image) be transferred quickly. +# +# This script should be called in one of the following ways: +# +# recv-hole file FILENAME +# recv-hole vbox FILENAME DISKSIZE +# +# In both cases, FILENAME is the pathname of the disk image on the +# receiving end. DISKSIZE is the size of the disk image in bytes. The +# first form is used when transferring a disk image to become an +# identical file on the receiving end. +# +# The second form is used when the disk image should be converted for +# use by VirtualBox. In this case, we want to avoid writing a +# temporary file on disk, and then calling the VirtualBox VBoxManage +# tool to do the conversion, since that would involve large amounts of +# unnecessary I/O and disk usage. Instead we pipe the file directly to +# VBoxManage, avoiding those issues. The piping is done here in this +# script, instead of in the caller, to make it easier to run things +# over ssh. +# +# However, since it's not possible seek in a Unix pipe, we have to +# explicitly write the zeroes into the pipe. This is not +# super-efficient, but the way to avoid that would be to avoid sending +# a sparse file, and do the conversion to a VDI on the sending end. +# That is out of scope for xfer-hole and recv-hole. + + +set -eu + + +die() +{ + echo "$@" 1>&2 + exit 1 +} + + +recv_hole_to_file() +{ + local n + + read n + truncate --size "+$n" "$1" +} + + +recv_data_to_file() +{ + local n + read n + + local blocksize=1048576 + local blocks=$(($n / $blocksize)) + local extra=$(($n % $blocksize)) + + xfer_data_to_stdout "$blocksize" "$blocks" >> "$1" + xfer_data_to_stdout 1 "$extra" >> "$1" +} + + +recv_hole_to_stdout() +{ + local n + read n + (echo "$n"; cat /dev/zero) | recv_data_to_stdout +} + + +recv_data_to_stdout() +{ + local n + read n + + local blocksize=1048576 + local blocks=$(($n / $blocksize)) + local extra=$(($n % $blocksize)) + + xfer_data_to_stdout "$blocksize" "$blocks" + xfer_data_to_stdout 1 "$extra" +} + + +xfer_data_to_stdout() +{ + local log="$(mktemp)" + if ! dd "bs=$1" count="$2" iflag=fullblock status=noxfer 2> "$log" + then + cat "$log" 1>&2 + rm -f "$log" + exit 1 + else + rm -f "$log" + fi +} + + +type="$1" +case "$type" in + file) + output="$2" + truncate --size=0 "$output" + while read what + do + case "$what" in + DATA) recv_data_to_file "$output" ;; + HOLE) recv_hole_to_file "$output" ;; + *) die "Unknown instruction: $what" ;; + esac + done + ;; + vbox) + output="$2" + disk_size="$3" + while read what + do + case "$what" in + DATA) recv_data_to_stdout ;; + HOLE) recv_hole_to_stdout ;; + *) die "Unknown instruction: $what" ;; + esac + done | + VBoxManage convertfromraw stdin "$output" "$disk_size" + ;; +esac diff --git a/extensions/xfer-hole b/extensions/xfer-hole new file mode 100755 index 00000000..91f1be01 --- /dev/null +++ b/extensions/xfer-hole @@ -0,0 +1,137 @@ +#!/usr/bin/env python +# +# Send a sparse file more space-efficiently. +# See recv-hole for a description of the protocol. +# +# Note that xfer-hole requires a version of Linux with support for +# SEEK_DATA and SEEK_HOLE. +# +# +# Copyright (C) 2014-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# =*= License: GPL-2 =*= + + + +import errno +import os +import sys + + +SEEK_DATA = 3 +SEEK_HOLE = 4 + + +filename = sys.argv[1] +fd = os.open(filename, os.O_RDONLY) +pos = 0 + + +DATA = 'data' +HOLE = 'hole' +EOF = 'eof' + + +def safe_lseek(fd, pos, whence): + try: + return os.lseek(fd, pos, whence) + except OSError as e: + if e.errno == errno.ENXIO: + return -1 + raise + + +def current_data_or_pos(fd, pos): + length = safe_lseek(fd, 0, os.SEEK_END) + next_data = safe_lseek(fd, pos, SEEK_DATA) + next_hole = safe_lseek(fd, pos, SEEK_HOLE) + + if pos == length: + return EOF, pos + elif pos == next_data: + return DATA, pos + elif pos == next_hole: + return HOLE, pos + else: + assert False, \ + ("Do not understand: pos=%d next_data=%d next_hole=%d" % + (pos, next_data, next_hole)) + + +def next_data_or_hole(fd, pos): + length = safe_lseek(fd, 0, os.SEEK_END) + next_data = safe_lseek(fd, pos, SEEK_DATA) + next_hole = safe_lseek(fd, pos, SEEK_HOLE) + + if pos == length: + return EOF, pos + elif pos == next_data: + # We are at data. + if next_hole == -1 or next_hole == length: + return EOF, length + else: + return HOLE, next_hole + elif pos == next_hole: + # We are at a hole. + if next_data == -1 or next_data == length: + return EOF, length + else: + return DATA, next_data + else: + assert False, \ + ("Do not understand: pos=%d next_data=%d next_hole=%d" % + (pos, next_data, next_hole)) + + +def find_data_and_holes(fd): + pos = safe_lseek(fd, 0, os.SEEK_CUR) + + kind, pos = current_data_or_pos(fd, pos) + while kind != EOF: + yield kind, pos + kind, pos = next_data_or_hole(fd, pos) + yield kind, pos + + +def make_xfer_instructions(fd): + prev_kind = None + prev_pos = None + for kind, pos in find_data_and_holes(fd): + if prev_kind == DATA: + yield (DATA, prev_pos, pos) + elif prev_kind == HOLE: + yield (HOLE, prev_pos, pos) + prev_kind = kind + prev_pos = pos + + +def copy_slice_from_file(to, fd, start, end): + safe_lseek(fd, start, os.SEEK_SET) + nbytes = end - start + max_at_a_time = 1024**2 + while nbytes > 0: + data = os.read(fd, min(nbytes, max_at_a_time)) + if not data: + break + to.write(data) + nbytes -= len(data) + + +for kind, start, end in make_xfer_instructions(fd): + if kind == HOLE: + sys.stdout.write('HOLE\n%d\n' % (end - start)) + elif kind == DATA: + sys.stdout.write('DATA\n%d\n' % (end - start)) + copy_slice_from_file(sys.stdout, fd, start, end) -- cgit v1.2.1 From e4c6b8a69f0df2d0b3beac46865a66e0de527151 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Thu, 4 Jun 2015 15:17:44 +0000 Subject: Remove dependencies on morphlib and cliapp from deployment extensions This is done by either copying some utility functions from morph into writeexts.py, and using the `subprocess` module rather than cliapp's runcmd and ssh_runcmd. Note that this means that these extensions will require "$definitions_checkout/extensions" in PYTHONPATH when they are run. This commit also updates VERSION to 5, since the PYTHONPATH requirement means that this change is incompatible with old versions of morph. Change-Id: Iec6fa7e3c7219619ce55e18493e5c37c36e97816 --- extensions/ceph.configure | 11 +- extensions/distbuild-trove-nfsboot.check | 52 ++++--- extensions/distbuild-trove-nfsboot.write | 40 ++--- extensions/fstab.configure | 4 +- extensions/hosts.configure | 12 +- extensions/image-package-example/README | 4 +- extensions/install-essential-files.configure | 21 +-- extensions/install-files.configure | 30 ++-- extensions/jffs2.write | 18 +-- extensions/kvm.check | 69 +++++---- extensions/kvm.write | 25 +-- extensions/nfsboot.check | 50 +++--- extensions/nfsboot.write | 83 +++++----- extensions/openstack.check | 26 ++-- extensions/openstack.write | 11 +- extensions/pxeboot.write | 57 +++---- extensions/rawdisk.check | 17 +- extensions/rawdisk.write | 18 +-- extensions/simple-network.configure | 22 +-- extensions/ssh-rsync.check | 26 ++-- extensions/ssh-rsync.write | 51 +++--- extensions/strip-gplv3.configure | 28 ++-- extensions/virtualbox-ssh.check | 10 +- extensions/virtualbox-ssh.write | 42 ++--- extensions/writeexts.py | 223 ++++++++++++++++++++------- 25 files changed, 545 insertions(+), 405 deletions(-) (limited to 'extensions') diff --git a/extensions/ceph.configure b/extensions/ceph.configure index c3cd92d1..3b8b2603 100644 --- a/extensions/ceph.configure +++ b/extensions/ceph.configure @@ -14,13 +14,14 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -import cliapp -import sys import os -import subprocess import shutil -import re import stat +import subprocess +import sys +import re + +import writeexts systemd_monitor_template = """ [Unit] @@ -75,7 +76,7 @@ executable_file_permissions = stat.S_IRUSR | stat.S_IXUSR | stat.S_IWUSR | \ stat.S_IXGRP | stat.S_IRGRP | \ stat.S_IXOTH | stat.S_IROTH -class CephConfigurationExtension(cliapp.Application): +class CephConfigurationExtension(writeexts.Extension): """ Set up ceph server daemons. diff --git a/extensions/distbuild-trove-nfsboot.check b/extensions/distbuild-trove-nfsboot.check index 38c491e5..76ba6dda 100755 --- a/extensions/distbuild-trove-nfsboot.check +++ b/extensions/distbuild-trove-nfsboot.check @@ -15,14 +15,15 @@ '''Preparatory checks for Morph 'distbuild-trove-nfsboot' write extension''' -import cliapp import logging import os +import subprocess +import sys -import morphlib.writeexts +import writeexts -class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): +class DistbuildTroveNFSBootCheckExtension(writeexts.WriteExtension): nfsboot_root = '/srv/nfsboot' remote_user = 'root' @@ -45,7 +46,8 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') nfs_host = args[0] nfs_netloc = '%s@%s' % (self.remote_user, nfs_host) @@ -55,17 +57,19 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): missing_vars = [var for var in self.required_vars if not var in os.environ] if missing_vars: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Please set: %s' % ', '.join(missing_vars)) controllers = os.getenv('DISTBUILD_CONTROLLER').split() workers = os.getenv('DISTBUILD_WORKERS').split() if len(controllers) != 1: - raise cliapp.AppException('Please specify exactly one controller.') + raise writeexts.ExtensionError( + 'Please specify exactly one controller.') if len(workers) == 0: - raise cliapp.AppException('Please specify at least one worker.') + raise writeexts.ExtensionError( + 'Please specify at least one worker.') upgrade = self.get_environment_boolean('UPGRADE') @@ -80,7 +84,7 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): if self.remote_directory_exists(nfs_netloc, system_path): if self.get_environment_boolean('OVERWRITE') == False: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'System %s already exists at %s:%s. Try `morph ' 'upgrade` instead of `morph deploy`.' % ( system_name, nfs_netloc, system_path)) @@ -91,27 +95,27 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): # Is an NFS server try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( netloc, ['test', '-e', '/etc/exports']) - except cliapp.AppException: - raise cliapp.AppException('server %s is not an nfs server' - % netloc) + except writeexts.ExtensionError: + raise writeexts.ExtensionError('server %s is not an nfs server' + % netloc) try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( netloc, ['systemctl', 'is-enabled', 'nfs-server.service']) - except cliapp.AppException: - raise cliapp.AppException('server %s does not control its ' - 'nfs server by systemd' % netloc) + except writeexts.ExtensionError: + raise writeexts.ExtensionError('server %s does not control its ' + 'nfs server by systemd' % netloc) # TFTP server exports /srv/nfsboot/tftp tftp_root = os.path.join(self.nfsboot_root, 'tftp') try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( netloc, ['test' , '-d', tftp_root]) - except cliapp.AppException: - raise cliapp.AppException('server %s does not export %s' % - (netloc, tftp_root)) + except writeexts.ExtensionError: + raise writeexts.ExtensionError('server %s does not export %s' % + (netloc, tftp_root)) def check_upgradeable(self, nfs_netloc, system_name, version_label): '''Check that there is already a version of the system present. @@ -124,7 +128,7 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): system_version_path = self.system_path(system_name, version_label) if not self.remote_directory_exists(nfs_netloc, system_path): - raise cliapp.AppException( + raise writeexts.ExtensionError( 'System %s not found at %s:%s, cannot deploy an upgrade.' % ( system_name, nfs_netloc, system_path)) @@ -132,15 +136,15 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): if self.get_environment_boolean('OVERWRITE'): pass else: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'System %s version %s already exists at %s:%s.' % ( system_name, version_label, nfs_netloc, system_version_path)) def remote_directory_exists(self, nfs_netloc, path): try: - cliapp.ssh_runcmd(nfs_netloc, ['test', '-d', path]) - except cliapp.AppException as e: + writeexts.ssh_runcmd(nfs_netloc, ['test', '-d', path]) + except writeexts.ExtensionError as e: logging.debug('SSH exception: %s', e) return False diff --git a/extensions/distbuild-trove-nfsboot.write b/extensions/distbuild-trove-nfsboot.write index a5a5b094..86291794 100755 --- a/extensions/distbuild-trove-nfsboot.write +++ b/extensions/distbuild-trove-nfsboot.write @@ -20,14 +20,14 @@ import os +import subprocess import sys import tempfile -import cliapp -import morphlib.writeexts +import writeexts -class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): +class DistbuildTroveNFSBootWriteExtension(writeexts.WriteExtension): '''Create an NFS root and kernel on TFTP during Morph's deployment. @@ -54,7 +54,7 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError('Wrong number of command line args') local_system_path, nfs_host = args @@ -111,17 +111,17 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): ''' pairs = host_map_string.split(' ') - return morphlib.util.parse_environment_pairs({}, pairs) + return writeexts.parse_environment_pairs({}, pairs) def transfer_system(self, nfs_netloc, local_system_path, remote_system_path): self.status(msg='Copying rootfs to %(nfs_netloc)s', nfs_netloc=nfs_netloc) - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( nfs_netloc, ['mkdir', '-p', remote_system_path]) # The deployed rootfs may have been created by OSTree, so definitely # don't pass --hard-links to `rsync`. - cliapp.runcmd( + subprocess.check_call( ['rsync', '--archive', '--delete', '--info=progress2', '--protect-args', '--partial', '--sparse', '--xattrs', local_system_path + '/', @@ -131,13 +131,13 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): target_system_path): self.status(msg='Duplicating rootfs to %(target_system_path)s', target_system_path=target_system_path) - cliapp.ssh_runcmd(nfs_netloc, + writeexts.ssh_runcmd(nfs_netloc, ['mkdir', '-p', target_system_path]) # We can't pass --info=progress2 here, because it may not be available # in the remote 'rsync'. The --info setting was added in RSync 3.1.0, # old versions of Baserock have RSync 3.0.9. So the user doesn't get # any progress info on stdout for the 'duplicate' stage. - cliapp.ssh_runcmd(nfs_netloc, + writeexts.ssh_runcmd(nfs_netloc, ['rsync', '--archive', '--delete', '--protect-args', '--partial', '--sparse', '--xattrs', source_system_path + '/', target_system_path], stdout=sys.stdout) @@ -152,7 +152,7 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): kernel_path = os.path.relpath(try_path, local_system_path) break else: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Could not find a kernel in the system: none of ' '%s found' % ', '.join(image_names)) return kernel_path @@ -171,11 +171,11 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): versioned_kernel_name = "%s-%s" % (system_name, version_label) kernel_name = system_name - cliapp.ssh_runcmd(nfs_netloc, + writeexts.ssh_runcmd(nfs_netloc, ['ln', '-f', kernel_dest, os.path.join(tftp_dir, versioned_kernel_name)]) - cliapp.ssh_runcmd(nfs_netloc, + writeexts.ssh_runcmd(nfs_netloc, ['ln', '-sf', versioned_kernel_name, os.path.join(tftp_dir, kernel_name)]) @@ -183,7 +183,7 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): with tempfile.NamedTemporaryFile() as f: f.write(text) f.flush() - cliapp.runcmd( + subprocess.check_call( ['scp', f.name, '%s:%s' % (nfs_netloc, path)]) def set_hostname(self, nfs_netloc, system_name, system_path): @@ -223,9 +223,9 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): '# Generated by distbuild-trove-nfsboot.write\n' + \ config_text + '\n' path = os.path.join(system_path, 'etc', 'distbuild') - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( nfs_netloc, ['mkdir', '-p', path]) - cliapp.runcmd( + subprocess.check_call( ['scp', worker_ssh_key_path, '%s:%s' % (nfs_netloc, path)]) self.set_remote_file_contents( nfs_netloc, os.path.join(path, 'distbuild.conf'), config_text) @@ -244,9 +244,9 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): # Rather ugly SSH hackery follows to ensure each system path is # listed in /etc/exports. try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( nfs_netloc, ['grep', '-q', exported_path, exports_path]) - except cliapp.AppException: + except writeexts.ExtensionError: ip_mask = '*' options = 'rw,no_subtree_check,no_root_squash,async' exports_string = '%s %s(%s)\n' % (exported_path, ip_mask, @@ -259,12 +259,12 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): cat >> "$temp" mv "$temp" "$target" ''' - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( nfs_netloc, ['sh', '-c', exports_append_sh, '--', exports_path], feed_stdin=exports_string) - cliapp.ssh_runcmd(nfs_netloc, + writeexts.ssh_runcmd(nfs_netloc, ['systemctl', 'restart', 'nfs-server.service']) def update_default_version(self, remote_netloc, system_name, @@ -276,7 +276,7 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): version_label) default_path = os.path.join(system_path, 'systems', 'default') - cliapp.ssh_runcmd(remote_netloc, + writeexts.ssh_runcmd(remote_netloc, ['ln', '-sfn', system_version_path, default_path]) diff --git a/extensions/fstab.configure b/extensions/fstab.configure index b9154eee..3e67b585 100755 --- a/extensions/fstab.configure +++ b/extensions/fstab.configure @@ -20,9 +20,9 @@ import os import sys -import morphlib +import writeexts envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('FSTAB_')} conf_file = os.path.join(sys.argv[1], 'etc/fstab') -morphlib.util.write_from_dict(conf_file, envvars) +writeexts.write_from_dict(conf_file, envvars) diff --git a/extensions/hosts.configure b/extensions/hosts.configure index 6b068d04..11fcf573 100755 --- a/extensions/hosts.configure +++ b/extensions/hosts.configure @@ -22,27 +22,29 @@ import os import sys import socket -import morphlib +import writeexts def validate(var, line): xs = line.split() if len(xs) == 0: - raise morphlib.Error("`%s: %s': line is empty" % (var, line)) + raise writeexts.ExtensionError( + "`%s: %s': line is empty" % (var, line)) ip = xs[0] hostnames = xs[1:] if len(hostnames) == 0: - raise morphlib.Error("`%s: %s': missing hostname" % (var, line)) + raise writeexts.ExtensionError( + "`%s: %s': missing hostname" % (var, line)) family = socket.AF_INET6 if ':' in ip else socket.AF_INET try: socket.inet_pton(family, ip) except socket.error: - raise morphlib.Error("`%s: %s' invalid ip" % (var, ip)) + raise writeexts.ExtensionError("`%s: %s' invalid ip" % (var, ip)) envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('HOSTS_')} conf_file = os.path.join(sys.argv[1], 'etc/hosts') -morphlib.util.write_from_dict(conf_file, envvars, validate) +writeexts.write_from_dict(conf_file, envvars, validate) diff --git a/extensions/image-package-example/README b/extensions/image-package-example/README index c1322f25..f6b66cd9 100644 --- a/extensions/image-package-example/README +++ b/extensions/image-package-example/README @@ -5,5 +5,5 @@ These are scripts used to create disk images or install the system onto an existing disk. This is also implemented independently for the rawdisk.write write -extension; see morphlib.writeexts.WriteExtension.create_local_system() -for a similar, python implementation. +extension; see writeexts.WriteExtension.create_local_system() for +a similar, python implementation. diff --git a/extensions/install-essential-files.configure b/extensions/install-essential-files.configure index bed394df..3d33fe03 100755 --- a/extensions/install-essential-files.configure +++ b/extensions/install-essential-files.configure @@ -22,20 +22,11 @@ to install into the target system. ''' -import subprocess import os +import subprocess +import sys -import cliapp - -class InstallEssentialFilesConfigureExtension(cliapp.Application): - - def process_args(self, args): - target_root = args[0] - os.environ["INSTALL_FILES"] = "install-files/essential-files/manifest" - self.install_essential_files(target_root) - - def install_essential_files(self, target_root): - command = os.path.join("extensions/install-files.configure") - subprocess.check_call([command, target_root]) - -InstallEssentialFilesConfigureExtension().run() +target_root = sys.argv[1] +os.environ["INSTALL_FILES"] = "install-files/essential-files/manifest" +command = os.path.join("extensions/install-files.configure") +subprocess.check_call([command, target_root]) diff --git a/extensions/install-files.configure b/extensions/install-files.configure index 341cce61..64fcecca 100755 --- a/extensions/install-files.configure +++ b/extensions/install-files.configure @@ -22,9 +22,8 @@ to install into the target system. ''' -import cliapp -import os import errno +import os import re import sys import shlex @@ -37,7 +36,9 @@ try: except ImportError: jinja_available = False -class InstallFilesConfigureExtension(cliapp.Application): +import writeexts + +class InstallFilesConfigureExtension(writeexts.Extension): def process_args(self, args): if not 'INSTALL_FILES' in os.environ: @@ -74,7 +75,8 @@ class InstallFilesConfigureExtension(cliapp.Application): gid = int(m.group(5)) path = m.group(6) else: - raise cliapp.AppException('Invalid manifest entry, ' + raise writeexts.ExtensionError( + 'Invalid manifest entry, ' 'format: [template] [overwrite] ' ' ') @@ -85,9 +87,9 @@ class InstallFilesConfigureExtension(cliapp.Application): if (mode != dest_stat.st_mode or uid != dest_stat.st_uid or gid != dest_stat.st_gid): - raise cliapp.AppException('"%s" exists and is not ' - 'identical to directory ' - '"%s"' % (dest_path, entry)) + raise writeexts.ExtensionError( + '"%s" exists and is not identical to directory ' + '"%s"' % (dest_path, entry)) else: os.mkdir(dest_path, mode) os.chown(dest_path, uid, gid) @@ -95,8 +97,8 @@ class InstallFilesConfigureExtension(cliapp.Application): elif stat.S_ISLNK(mode): if os.path.lexists(dest_path) and not overwrite: - raise cliapp.AppException('Symlink already exists at %s' - % dest_path) + raise writeexts.ExtensionError('Symlink already exists at %s' + % dest_path) else: linkdest = os.readlink(os.path.join(manifest_root, './' + path)) @@ -105,12 +107,12 @@ class InstallFilesConfigureExtension(cliapp.Application): elif stat.S_ISREG(mode): if os.path.lexists(dest_path) and not overwrite: - raise cliapp.AppException('File already exists at %s' - % dest_path) + raise writeexts.ExtensionError('File already exists at %s' + % dest_path) else: if template: if not jinja_available: - raise cliapp.AppException( + raise writeexts.ExtensionError( "Failed to install template file `%s': " 'install-files templates require jinja2' % path) @@ -128,7 +130,7 @@ class InstallFilesConfigureExtension(cliapp.Application): os.chmod(dest_path, mode) else: - raise cliapp.AppException('Mode given in "%s" is not a file,' - ' symlink or directory' % entry) + raise writeexts.ExtensionError('Mode given in "%s" is not a file,' + ' symlink or directory' % entry) InstallFilesConfigureExtension().run() diff --git a/extensions/jffs2.write b/extensions/jffs2.write index 46b69a53..ad68204d 100644 --- a/extensions/jffs2.write +++ b/extensions/jffs2.write @@ -19,34 +19,34 @@ as the root filesystem.''' -import cliapp import os +import subprocess -import morphlib.writeexts +import writeexts -class Jffs2WriteExtension(morphlib.writeexts.WriteExtension): +class Jffs2WriteExtension(writeexts.WriteExtension): '''See jffs2.write.help for documentation.''' def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError('Wrong number of command line args') temp_root, location = args try: self.create_jffs2_system(temp_root, location) self.status(msg='Disk image has been created at %(location)s', - location = location) + location=location) except Exception: self.status(msg='Failure to deploy system to %(location)s', - location = location) + location=location) raise def create_jffs2_system(self, temp_root, location): erase_block = self.get_erase_block_size() - cliapp.runcmd( + subprocess.check_call( ['mkfs.jffs2', '--pad', '--no-cleanmarkers', '--eraseblock='+erase_block, '-d', temp_root, '-o', location]) @@ -54,10 +54,10 @@ class Jffs2WriteExtension(morphlib.writeexts.WriteExtension): erase_block = os.environ.get('ERASE_BLOCK', '') if erase_block == '': - raise cliapp.AppException('ERASE_BLOCK was not given') + raise writeexts.ExtensionError('ERASE_BLOCK was not given') if not erase_block.isdigit(): - raise cliapp.AppException('ERASE_BLOCK must be a whole number') + raise writeexts.ExtensionError('ERASE_BLOCK must be a whole number') return erase_block diff --git a/extensions/kvm.check b/extensions/kvm.check index 67cb3d38..3c277156 100755 --- a/extensions/kvm.check +++ b/extensions/kvm.check @@ -15,27 +15,28 @@ '''Preparatory checks for Morph 'kvm' write extension''' -import cliapp import os import re +import subprocess import urlparse -import morphlib.writeexts +import writeexts -class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): +class KvmPlusSshCheckExtension(writeexts.WriteExtension): location_pattern = '^/(?P[^/]+)(?P/.+)$' def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') self.require_btrfs_in_deployment_host_kernel() upgrade = self.get_environment_boolean('UPGRADE') if upgrade: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Use the `ssh-rsync` write extension to deploy upgrades to an ' 'existing remote system.') @@ -55,23 +56,24 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): x = urlparse.urlparse(location) if x.scheme != 'kvm+ssh': - raise cliapp.AppException( + raise writeexts.ExtensionError( 'URL schema must be kvm+ssh in %s' % location) m = re.match(self.location_pattern, x.path) if not m: - raise cliapp.AppException('Cannot parse location %s' % location) + raise writeexts.ExtensionError( + 'Cannot parse location %s' % location) return x.netloc, m.group('guest'), m.group('path') def check_no_existing_libvirt_vm(self, ssh_host, vm_name): try: - cliapp.ssh_runcmd(ssh_host, + writeexts.ssh_runcmd(ssh_host, ['virsh', '--connect', 'qemu:///system', 'domstate', vm_name]) - except cliapp.AppException as e: + except CalledProcessError as e: pass else: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Host %s already has a VM named %s. You can use the ssh-rsync ' 'write extension to deploy upgrades to existing machines.' % (ssh_host, vm_name)) @@ -80,35 +82,35 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): def check_can_write_to_given_path(): try: - cliapp.ssh_runcmd(ssh_host, ['touch', vm_path]) - except cliapp.AppException as e: - raise cliapp.AppException("Can't write to location %s on %s" - % (vm_path, ssh_host)) + writeexts.ssh_runcmd(ssh_host, ['touch', vm_path]) + except writeexts.ExtensionError as e: + raise writeexts.ExtensionError( + "Can't write to location %s on %s" % (vm_path, ssh_host)) else: - cliapp.ssh_runcmd(ssh_host, ['rm', vm_path]) + writeexts.ssh_runcmd(ssh_host, ['rm', vm_path]) try: - cliapp.ssh_runcmd(ssh_host, ['test', '-e', vm_path]) - except cliapp.AppException as e: + writeexts.ssh_runcmd(ssh_host, ['test', '-e', vm_path]) + except writeexts.ExtensionError as e: # vm_path doesn't already exist, so let's test we can write check_can_write_to_given_path() else: - raise cliapp.AppException('%s already exists on %s' - % (vm_path, ssh_host)) + raise writeexts.ExtensionError('%s already exists on %s' + % (vm_path, ssh_host)) def check_extra_disks_exist(self, ssh_host, filename_list): for filename in filename_list: try: - cliapp.ssh_runcmd(ssh_host, ['ls', filename]) - except cliapp.AppException as e: - raise cliapp.AppException('Did not find file %s on host %s' % - (filename, ssh_host)) + writeexts.ssh_runcmd(ssh_host, ['ls', filename]) + except writeexts.ExtensionError as e: + raise writeexts.ExtensionError( + 'Did not find file %s on host %s' % (filename, ssh_host)) def check_virtual_networks_are_started(self, ssh_host): def check_virtual_network_is_started(network_name): cmd = ['virsh', '-c', 'qemu:///system', 'net-info', network_name] - net_info = cliapp.ssh_runcmd(ssh_host, cmd).split('\n') + net_info = writeexts.ssh_runcmd(ssh_host, cmd).split('\n') def pretty_concat(lines): return '\n'.join(['\t%s' % line for line in lines]) @@ -118,15 +120,15 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): if m: break else: - raise cliapp.AppException( + raise writeexts.ExtensionError( "Got unexpected output parsing output of `%s':\n%s" % (' '.join(cmd), pretty_concat(net_info))) network_active = m.group(1) == 'yes' if not network_active: - raise cliapp.AppException("Network '%s' is not started" - % network_name) + raise writeexts.ExtensionError("Network '%s' is not started" + % network_name) def name(nic_entry): if ',' in nic_entry: @@ -142,9 +144,10 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): if not (n.startswith('network=') or n.startswith('bridge=') or n == 'user'): - raise cliapp.AppException('malformed NIC_CONFIG: %s\n' - " (expected 'bridge=BRIDGE' 'network=NAME'" - " or 'user')" % n) + raise writeexts.ExtensionError( + "malformed NIC_CONFIG: %s\n" + " (expected 'bridge=BRIDGE' 'network=NAME'" + " or 'user')" % n) # --network bridge= is used to specify a bridge # --network user is used to specify a form of NAT @@ -159,9 +162,9 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): def check_host_has_virtinstall(self, ssh_host): try: - cliapp.ssh_runcmd(ssh_host, ['which', 'virt-install']) - except cliapp.AppException: - raise cliapp.AppException( + writeexts.ssh_runcmd(ssh_host, ['which', 'virt-install']) + except writeexts.ExtensionError: + raise writeexts.ExtensionError( 'virt-install does not seem to be installed on host %s' % ssh_host) diff --git a/extensions/kvm.write b/extensions/kvm.write index 0d0c095b..2290725e 100755 --- a/extensions/kvm.write +++ b/extensions/kvm.write @@ -21,23 +21,23 @@ See file kvm.write.help for documentation ''' -import cliapp import os import re import sys import tempfile import urlparse -import morphlib.writeexts +import writeexts -class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): +class KvmPlusSshWriteExtension(writeexts.WriteExtension): location_pattern = '^/(?P[^/]+)(?P/.+)$' def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') temp_root, location = args ssh_host, vm_name, vm_path = self.parse_location(location) @@ -53,7 +53,7 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): except BaseException: sys.stderr.write('Error deploying to libvirt') os.remove(raw_disk) - cliapp.ssh_runcmd(ssh_host, ['rm', '-f', vm_path]) + writeexts.ssh_runcmd(ssh_host, ['rm', '-f', vm_path]) raise else: os.remove(raw_disk) @@ -74,16 +74,16 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Transferring disk image') - xfer_hole_path = morphlib.util.get_data_path('xfer-hole') - recv_hole = morphlib.util.get_data('recv-hole') + xfer_hole_path = writeexts.get_data_path('xfer-hole') + recv_hole = writeexts.get_data('recv-hole') ssh_remote_cmd = [ 'sh', '-c', recv_hole, 'dummy-argv0', 'file', vm_path ] - cliapp.runcmd( + subprocess.check_call( ['python', xfer_hole_path, raw_disk], - ['ssh', ssh_host] + map(cliapp.shell_quote, ssh_remote_cmd), + ['ssh', ssh_host] + map(writeexts.shell_quote, ssh_remote_cmd), stdout=None, stderr=None) def create_libvirt_guest(self, ssh_host, vm_name, vm_path, autostart): @@ -111,10 +111,11 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): '--disk', 'path=%s,bus=ide' % vm_path] + attach_opts if not autostart: cmdline += ['--noreboot'] - cliapp.ssh_runcmd(ssh_host, cmdline) + writeexts.ssh_runcmd(ssh_host, cmdline) if autostart: - cliapp.ssh_runcmd(ssh_host, - ['virsh', '--connect', 'qemu:///system', 'autostart', vm_name]) + writeexts.ssh_runcmd(ssh_host, + ['virsh', '--connect', 'qemu:///system', + 'autostart', vm_name]) KvmPlusSshWriteExtension().run() diff --git a/extensions/nfsboot.check b/extensions/nfsboot.check index e273f61c..499fb537 100755 --- a/extensions/nfsboot.check +++ b/extensions/nfsboot.check @@ -15,33 +15,35 @@ '''Preparatory checks for Morph 'nfsboot' write extension''' -import cliapp import os +import subprocess -import morphlib.writeexts +import writeexts -class NFSBootCheckExtension(morphlib.writeexts.WriteExtension): +class NFSBootCheckExtension(writeexts.WriteExtension): _nfsboot_root = '/srv/nfsboot' def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') location = args[0] upgrade = self.get_environment_boolean('UPGRADE') if upgrade: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Upgrading is not currently supported for NFS deployments.') hostname = os.environ.get('HOSTNAME', None) if hostname is None: - raise cliapp.AppException('You must specify a HOSTNAME.') + raise writeexts.ExtensionError('You must specify a HOSTNAME.') if hostname == 'baserock': - raise cliapp.AppException('It is forbidden to nfsboot a system ' - 'with hostname "%s"' % hostname) + raise writeexts.ExtensionError('It is forbidden to nfsboot a ' + 'system with hostname "%s"' + % hostname) self.test_good_server(location) @@ -49,7 +51,7 @@ class NFSBootCheckExtension(morphlib.writeexts.WriteExtension): versioned_root = os.path.join(self._nfsboot_root, hostname, 'systems', version_label) if self.version_exists(versioned_root, location): - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Root file system for host %s (version %s) already exists on ' 'the NFS server %s. Deployment aborted.' % (hostname, version_label, location)) @@ -59,34 +61,34 @@ class NFSBootCheckExtension(morphlib.writeexts.WriteExtension): # Is an NFS server try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( 'root@%s' % server, ['test', '-e', '/etc/exports']) - except cliapp.AppException: - raise cliapp.AppException('server %s is not an nfs server' - % server) + except writeexts.ExtensionError: + raise writeexts.ExtensionError('server %s is not an nfs server' + % server) try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( 'root@%s' % server, ['systemctl', 'is-enabled', 'nfs-server.service']) - except cliapp.AppException: - raise cliapp.AppException('server %s does not control its ' - 'nfs server by systemd' % server) + except writeexts.ExtensionError: + raise writeexts.ExtensionError('server %s does not control its ' + 'nfs server by systemd' % server) # TFTP server exports /srv/nfsboot/tftp tftp_root = os.path.join(self._nfsboot_root, 'tftp') try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( 'root@%s' % server, ['test' , '-d', tftp_root]) - except cliapp.AppException: - raise cliapp.AppException('server %s does not export %s' % - (tftp_root, server)) + except writeexts.ExtensionError: + raise writeexts.ExtensionError('server %s does not export %s' % + (tftp_root, server)) def version_exists(self, versioned_root, location): try: - cliapp.ssh_runcmd('root@%s' % location, - ['test', '-d', versioned_root]) - except cliapp.AppException: + writeexts.ssh_runcmd('root@%s' % location, + ['test', '-d', versioned_root]) + except writeexts.ExtensionError: return False return True diff --git a/extensions/nfsboot.write b/extensions/nfsboot.write index d928775e..418f8eeb 100755 --- a/extensions/nfsboot.write +++ b/extensions/nfsboot.write @@ -34,14 +34,13 @@ in /srv/nfsboot/nfs/ ''' -import cliapp import os import glob -import morphlib.writeexts +import writeexts -class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): +class NFSBootWriteExtension(writeexts.WriteExtension): '''Create an NFS root and kernel on TFTP during Morph's deployment. @@ -66,7 +65,8 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') temp_root, location = args @@ -86,8 +86,8 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): subdirs = [os.path.join(statedir, 'home'), os.path.join(statedir, 'opt'), os.path.join(statedir, 'srv')] - cliapp.ssh_runcmd('root@%s' % location, - ['mkdir', '-p'] + subdirs) + writeexts.ssh_runcmd('root@%s' % location, + ['mkdir', '-p'] + subdirs) def copy_kernel(self, temp_root, location, versioned_root, version, hostname): @@ -99,14 +99,14 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): kernel_src = try_path break else: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Could not find a kernel in the system: none of ' '%s found' % ', '.join(image_names)) kernel_dest = os.path.join(versioned_root, 'orig', 'kernel') rsync_dest = 'root@%s:%s' % (location, kernel_dest) self.status(msg='Copying kernel') - cliapp.runcmd( + subprocess.check_call( ['rsync', '-s', kernel_src, rsync_dest]) # Link the kernel to the right place @@ -115,17 +115,17 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): versioned_kernel_name = "%s-%s" % (hostname, version) kernel_name = hostname try: - cliapp.ssh_runcmd('root@%s' % location, + writeexts.ssh_runcmd('root@%s' % location, ['ln', '-f', kernel_dest, os.path.join(tftp_dir, versioned_kernel_name)]) - cliapp.ssh_runcmd('root@%s' % location, + writeexts.ssh_runcmd('root@%s' % location, ['ln', '-sf', versioned_kernel_name, os.path.join(tftp_dir, kernel_name)]) - except cliapp.AppException: - raise cliapp.AppException('Could not create symlinks to the ' - 'kernel at %s in %s on %s' - % (kernel_dest, tftp_dir, location)) + except writeexts.ExtensionError: + raise writeexts.ExtensionError('Could not create symlinks to the ' + 'kernel at %s in %s on %s' % + (kernel_dest, tftp_dir, location)) def copy_rootfs(self, temp_root, location, versioned_root, hostname): rootfs_src = temp_root + '/' @@ -134,51 +134,54 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Creating destination directories') try: - cliapp.ssh_runcmd('root@%s' % location, - ['mkdir', '-p', orig_path, run_path]) - except cliapp.AppException: - raise cliapp.AppException('Could not create dirs %s and %s on %s' - % (orig_path, run_path, location)) + writeexts.ssh_runcmd('root@%s' % location, + ['mkdir', '-p', orig_path, run_path]) + except writeexts.ExtensionError: + raise writexts.ExtensionError( + 'Could not create dirs %s and %s on %s' + % (orig_path, run_path, location)) self.status(msg='Creating \'orig\' rootfs') - cliapp.runcmd( + subprocess.check_call( ['rsync', '-asXSPH', '--delete', rootfs_src, 'root@%s:%s' % (location, orig_path)]) self.status(msg='Creating \'run\' rootfs') try: - cliapp.ssh_runcmd('root@%s' % location, - ['rm', '-rf', run_path]) - cliapp.ssh_runcmd('root@%s' % location, - ['cp', '-al', orig_path, run_path]) - cliapp.ssh_runcmd('root@%s' % location, - ['rm', '-rf', os.path.join(run_path, 'etc')]) - cliapp.ssh_runcmd('root@%s' % location, - ['cp', '-a', os.path.join(orig_path, 'etc'), - os.path.join(run_path, 'etc')]) - except cliapp.AppException: - raise cliapp.AppException('Could not create \'run\' rootfs' - ' from \'orig\'') + writeexts.ssh_runcmd('root@%s' % location, + ['rm', '-rf', run_path]) + writeexts.ssh_runcmd('root@%s' % location, + ['cp', '-al', orig_path, run_path]) + writeexts.ssh_runcmd('root@%s' % location, + ['rm', '-rf', + os.path.join(run_path, 'etc')]) + writeexts.ssh_runcmd('root@%s' % location, + ['cp', '-a', + os.path.join(orig_path, 'etc'), + os.path.join(run_path, 'etc')]) + except writeexts.ExtensionError: + raise writeexts.ExtensionError('Could not create \'run\' rootfs' + ' from \'orig\'') self.status(msg='Linking \'default\' to latest system') try: - cliapp.ssh_runcmd('root@%s' % location, + writeexts.ssh_runcmd('root@%s' % location, ['ln', '-sfn', versioned_root, os.path.join(self._nfsboot_root, hostname, 'systems', 'default')]) - except cliapp.AppException: - raise cliapp.AppException('Could not link \'default\' to %s' - % versioned_root) + except writeexts.ExtensionError: + raise writeexts.ExtensionError("Could not link 'default' to %s" + % versioned_root) def configure_nfs(self, location, hostname): exported_path = os.path.join(self._nfsboot_root, hostname) exports_path = '/etc/exports' # If that path is not already exported: try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( 'root@%s' % location, ['grep', '-q', exported_path, exports_path]) - except cliapp.AppException: + except writeexts.ExtensionError: ip_mask = '*' options = 'rw,no_subtree_check,no_root_squash,async' exports_string = '%s %s(%s)\n' % (exported_path, ip_mask, options) @@ -190,11 +193,11 @@ cat "$target" > "$temp" cat >> "$temp" mv "$temp" "$target" ''' - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( 'root@%s' % location, ['sh', '-c', exports_append_sh, '--', exports_path], feed_stdin=exports_string) - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( 'root@%s' % location, ['systemctl', 'restart', 'nfs-server.service']) diff --git a/extensions/openstack.check b/extensions/openstack.check index a3379763..f3ad43b7 100755 --- a/extensions/openstack.check +++ b/extensions/openstack.check @@ -15,25 +15,26 @@ '''Preparatory checks for Morph 'openstack' write extension''' -import cliapp import os import urlparse + import keystoneclient -import morphlib.writeexts +import writeexts -class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): +class OpenStackCheckExtension(writeexts.WriteExtension): def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') self.require_btrfs_in_deployment_host_kernel() upgrade = self.get_environment_boolean('UPGRADE') if upgrade: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Use the `ssh-rsync` write extension to deploy upgrades to an ' 'existing remote system.') @@ -55,7 +56,7 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): for key in auth_keys: if os.environ.get(key, '') == '': - raise cliapp.AppException(key + ' was not given') + raise writeexts.ExtensionError(key + ' was not given') auth_params = {auth_keys[key]: os.environ[key] for key in auth_keys} auth_params['auth_url'] = location @@ -63,16 +64,17 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): def check_imagename(self): if os.environ.get('OPENSTACK_IMAGENAME', '') == '': - raise cliapp.AppException('OPENSTACK_IMAGENAME was not given') + raise writeexts.ExtensionError( + 'OPENSTACK_IMAGENAME was not given') def check_location(self, location): x = urlparse.urlparse(location) if x.scheme not in ['http', 'https']: - raise cliapp.AppException('URL schema must be http or https in %s'\ - % location) + raise writeexts.ExtensionError( + 'URL schema must be http or https in %s' % location) if (x.path != '/v2.0' and x.path != '/v2.0/'): - raise cliapp.AppException('API version must be v2.0 in %s'\ - % location) + raise writeexts.ExtensionError( + 'API version must be v2.0 in %s' % location) def check_openstack_parameters(self, auth_params): ''' Check that we can connect to and authenticate with openstack ''' @@ -84,7 +86,7 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): except keystoneclient.exceptions.Unauthorized: errmsg = ('Failed to authenticate with OpenStack ' '(are your credentials correct?)') - raise cliapp.AppException(errmsg) + raise writeexts.ExtensionError(errmsg) OpenStackCheckExtension().run() diff --git a/extensions/openstack.write b/extensions/openstack.write index 67e07c18..f1233560 100755 --- a/extensions/openstack.write +++ b/extensions/openstack.write @@ -17,21 +17,22 @@ '''A Morph deployment write extension for deploying to OpenStack.''' -import cliapp import os +import subprocess import tempfile import urlparse -import morphlib.writeexts +import writeexts -class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): +class OpenStackWriteExtension(writeexts.WriteExtension): '''See openstack.write.help for documentation''' def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') temp_root, location = args @@ -86,7 +87,7 @@ class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): '--disk-format=raw', '--container-format', 'bare', '--file', raw_disk] - cliapp.runcmd(cmdline) + subprocess.check_call(cmdline) self.status(msg='Image configured.') diff --git a/extensions/pxeboot.write b/extensions/pxeboot.write index 3a12ebcc..20e4f6bd 100644 --- a/extensions/pxeboot.write +++ b/extensions/pxeboot.write @@ -19,10 +19,7 @@ import tempfile import textwrap import urlparse -import cliapp - -import morphlib - +import writeexts def _int_to_quad_dot(i): return '.'.join(( @@ -143,7 +140,7 @@ def grouper(iterable, n, fillvalue=None): return itertools.izip_longest(*args, fillvalue=fillvalue) -class PXEBoot(morphlib.writeexts.WriteExtension): +class PXEBoot(writeexts.WriteExtension): @contextlib.contextmanager def _vlan(self, interface, vlan): viface = '%s.%s' % (interface, vlan) @@ -224,12 +221,13 @@ class PXEBoot(morphlib.writeexts.WriteExtension): @contextlib.contextmanager def _remote_tempdir(self, hostname, template): persist = os.environ.get('PXE_INSTALLER') in ('no', 'False') - td = cliapp.ssh_runcmd(hostname, ['mktemp', '-d', template]).strip() + td = writeexts.ssh_runcmd( + hostname, ['mktemp', '-d', template]).strip() try: yield td finally: if not persist: - cliapp.ssh_runcmd(hostname, ['find', td, '-delete']) + writeexts.ssh_runcmd(hostname, ['find', td, '-delete']) def _serve_tftpd(self, sock, host, port, interface, tftproot): self.settings.progname = 'tftp server' @@ -333,26 +331,27 @@ class PXEBoot(morphlib.writeexts.WriteExtension): def _remote_copy(self, hostname, src, dst): persist = os.environ.get('PXE_INSTALLER') in ('no', 'False') with open(src, 'r') as f: - cliapp.ssh_runcmd(hostname, - ['install', '-D', '-m644', '/proc/self/fd/0', - dst], stdin=f, stdout=None, stderr=None) + writeexts.ssh_runcmd(hostname, + ['install', '-D', '-m644', + '/proc/self/fd/0', dst], + stdin=f, stdout=None, stderr=None) try: yield finally: if not persist: - cliapp.ssh_runcmd(hostname, ['rm', dst]) + writeexts.ssh_runcmd(hostname, ['rm', dst]) @contextlib.contextmanager def _remote_symlink(self, hostname, src, dst): persist = os.environ.get('PXE_INSTALLER') in ('no', 'False') - cliapp.ssh_runcmd(hostname, - ['ln', '-s', '-f', src, dst], - stdin=None, stdout=None, stderr=None) + writeexts.ssh_runcmd(hostname, + ['ln', '-s', '-f', src, dst], + stdin=None, stdout=None, stderr=None) try: yield finally: if not persist: - cliapp.ssh_runcmd(hostname, ['rm', '-f', dst]) + writeexts.ssh_runcmd(hostname, ['rm', '-f', dst]) @contextlib.contextmanager def remote_kernel(self, rootfs, tftp_url, macaddr): @@ -361,7 +360,7 @@ class PXEBoot(morphlib.writeexts.WriteExtension): if os.path.exists(kernel_path): break else: - raise cliapp.AppException('Failed to locate kernel') + raise writeexts.ExtensionError('Failed to locate kernel') url = urlparse.urlsplit(tftp_url) basename = '{}-kernel'.format(_normalise_macaddr(macaddr)) target_path = os.path.join(url.path, basename) @@ -376,7 +375,8 @@ class PXEBoot(morphlib.writeexts.WriteExtension): yield fdt_abs_path = os.path.join(rootfs, fdt_rel_path) if not fdt_abs_path: - raise cliapp.AppException('Failed to locate Flattened Device Tree') + raise writeexts.ExtensionError( + 'Failed to locate Flattened Device Tree') url = urlparse.urlsplit(tftp_url) basename = '{}-fdt'.format(_normalise_macaddr(macaddr)) target_path = os.path.join(url.path, basename) @@ -389,14 +389,14 @@ class PXEBoot(morphlib.writeexts.WriteExtension): nfsroot = target_ip + ':' + rootfs self.status(msg='Exporting %(nfsroot)s as local nfsroot', nfsroot=nfsroot) - cliapp.runcmd(['exportfs', '-o', 'ro,insecure,no_root_squash', - nfsroot]) + subprocess.check_call(['exportfs', '-o', 'ro,insecure,no_root_squash', + nfsroot]) try: yield finally: self.status(msg='Removing %(nfsroot)s from local nfsroots', nfsroot=nfsroot) - cliapp.runcmd(['exportfs', '-u', nfsroot]) + subprocess.check_call(['exportfs', '-u', nfsroot]) @contextlib.contextmanager def remote_nfsroot(self, rootfs, rsync_url, macaddr): @@ -407,9 +407,10 @@ class PXEBoot(morphlib.writeexts.WriteExtension): as tempdir: nfsroot = urlparse.urlunsplit((url.scheme, url.netloc, tempdir, url.query, url.fragment)) - cliapp.runcmd(['rsync', '-asSPH', '--delete', rootfs, nfsroot], - stdin=None, stdout=open(os.devnull, 'w'), - stderr=None) + subprocess.check_call(['rsync', '-asSPH', '--delete', + rootfs, nfsroot], + stdin=None, stdout=open(os.devnull, 'w'), + stderr=None) yield os.path.join(os.path.basename(tempdir), os.path.basename(rootfs)) @@ -517,8 +518,8 @@ class PXEBoot(morphlib.writeexts.WriteExtension): def get_interface_ip(self, interface): ip_addresses = [] - info = cliapp.runcmd(['ip', '-o', '-f', 'inet', - 'addr', 'show', interface]).rstrip('\n') + info = subprocess.check_output(['ip', '-o', '-f', 'inet', 'addr', + 'show', interface]).rstrip('\n') if info: tokens = collections.deque(info.split()[1:]) ifname = tokens.popleft() @@ -535,8 +536,8 @@ class PXEBoot(morphlib.writeexts.WriteExtension): else: continue if not ip_addresses: - raise cliapp.AppException('Interface %s has no addresses' - % interface) + raise writeexts.ExtensionError('Interface %s has no addresses' + % interface) if len(ip_addresses) > 1: warnings.warn('Interface %s has multiple addresses, ' 'using first (%s)' % (interface, ip_addresses[0])) @@ -750,6 +751,6 @@ class PXEBoot(morphlib.writeexts.WriteExtension): self.wait_for_target_to_install() self.ipmi_reboot_target() else: - cliapp.AppException('Invalid PXEBOOT_MODE: %s' % mode) + writeexts.ExtensionError('Invalid PXEBOOT_MODE: %s' % mode) PXEBoot().run() diff --git a/extensions/rawdisk.check b/extensions/rawdisk.check index 9be0ce91..61619a21 100755 --- a/extensions/rawdisk.check +++ b/extensions/rawdisk.check @@ -15,17 +15,16 @@ '''Preparatory checks for Morph 'rawdisk' write extension''' -import cliapp - -import morphlib.writeexts - import os +import writeexts + -class RawdiskCheckExtension(morphlib.writeexts.WriteExtension): +class RawdiskCheckExtension(writeexts.WriteExtension): def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') self.require_btrfs_in_deployment_host_kernel() @@ -34,19 +33,19 @@ class RawdiskCheckExtension(morphlib.writeexts.WriteExtension): if upgrade: if not self.is_device(location): if not os.path.isfile(location): - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Cannot upgrade %s: it is not an existing disk image' % location) version_label = os.environ.get('VERSION_LABEL') if version_label is None: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'VERSION_LABEL was not given. It is required when ' 'upgrading an existing system.') else: if not self.is_device(location): if os.path.exists(location): - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Target %s already exists. Use `morph upgrade` if you ' 'want to update an existing image.' % location) diff --git a/extensions/rawdisk.write b/extensions/rawdisk.write index 6f2d45ba..cdeb5018 100755 --- a/extensions/rawdisk.write +++ b/extensions/rawdisk.write @@ -17,22 +17,22 @@ '''A Morph deployment write extension for raw disk images.''' -import cliapp import os import sys import time import tempfile -import morphlib.writeexts +import writeexts -class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): +class RawDiskWriteExtension(writeexts.WriteExtension): '''See rawdisk.write.help for documentation''' def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') temp_root, location = args upgrade = self.get_environment_boolean('UPGRADE') @@ -69,10 +69,10 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): old_orig = os.path.join(mp, 'systems', 'default', 'orig') new_orig = os.path.join(version_root, 'orig') - cliapp.runcmd( + subprocess.check_call( ['btrfs', 'subvolume', 'snapshot', old_orig, new_orig]) - cliapp.runcmd( + subprocess.check_call( ['rsync', '-a', '--checksum', '--numeric-ids', '--delete', temp_root + os.path.sep, new_orig]) @@ -96,11 +96,11 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): version_label = os.environ.get('VERSION_LABEL') if version_label is None: - raise cliapp.AppException('VERSION_LABEL was not given') + raise writeexts.ExtensionError('VERSION_LABEL was not given') if os.path.exists(os.path.join(mp, 'systems', version_label)): - raise cliapp.AppException('VERSION_LABEL %s already exists' - % version_label) + raise writeexts.ExtensionError('VERSION_LABEL %s already exists' + % version_label) return version_label diff --git a/extensions/simple-network.configure b/extensions/simple-network.configure index 4a70f311..61d5774d 100755 --- a/extensions/simple-network.configure +++ b/extensions/simple-network.configure @@ -25,27 +25,26 @@ for DHCP ''' +import errno import os import sys -import errno -import cliapp -import morphlib +import writeexts -class SimpleNetworkError(morphlib.Error): +class SimpleNetworkError(writeexts.ExtensionError): '''Errors associated with simple network setup''' pass -class SimpleNetworkConfigurationExtension(cliapp.Application): +class SimpleNetworkConfigurationExtension(object): '''Configure /etc/network/interfaces and generate networkd .network files Reading NETWORK_CONFIG, this extension sets up /etc/network/interfaces and .network files in /etc/systemd/network/. ''' - def process_args(self, args): + def run(self, args): network_config = os.environ.get("NETWORK_CONFIG") self.rename_networkd_chunk_file(args) @@ -206,7 +205,8 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): address_line = address + '/' + str(network_suffix) lines += ["Address=%s" % address_line] elif address or netmask: - raise Exception('address and netmask must be specified together') + raise SimpleNetworkError( + 'address and netmask must be specified together') if gateway: lines += ["Gateway=%s" % gateway] @@ -287,6 +287,10 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): ''' - self.output.write('%s\n' % (kwargs['msg'] % kwargs)) + sys.stdout.write('%s\n' % (kwargs['msg'] % kwargs)) -SimpleNetworkConfigurationExtension().run() +try: + SimpleNetworkConfigurationExtension().run(sys.argv[1:]) +except SimpleNetworkError as e: + sys.stdout.write('ERROR: %s\n' % e) + sys.exit(1) diff --git a/extensions/ssh-rsync.check b/extensions/ssh-rsync.check index c3bdfd29..436aaae0 100755 --- a/extensions/ssh-rsync.check +++ b/extensions/ssh-rsync.check @@ -15,26 +15,27 @@ '''Preparatory checks for Morph 'ssh-rsync' write extension''' -import cliapp import os -import morphlib.writeexts +import writeexts -class SshRsyncCheckExtension(morphlib.writeexts.WriteExtension): + +class SshRsyncCheckExtension(writeexts.WriteExtension): def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') upgrade = self.get_environment_boolean('UPGRADE') if not upgrade: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'The ssh-rsync write is for upgrading existing remote ' 'Baserock machines. It cannot be used for an initial ' 'deployment.') if os.environ.get('VERSION_LABEL', '') == '': - raise cliapp.AppException( + raise writeexts.ExtensionError( 'A VERSION_LABEL must be set when deploying an upgrade.') location = args[0] @@ -47,17 +48,18 @@ class SshRsyncCheckExtension(morphlib.writeexts.WriteExtension): self.check_command_exists(location, 'rsync') def check_is_baserock_system(self, location): - output = cliapp.ssh_runcmd(location, ['sh', '-c', - 'test -d /baserock || echo -n dirnotfound']) + output = writeexts.ssh_runcmd( + location, + ['sh', '-c', 'test -d /baserock || echo -n dirnotfound']) if output == 'dirnotfound': - raise cliapp.AppException('%s is not a baserock system' - % location) + raise writeexts.ExtensionError('%s is not a baserock system' + % location) def check_command_exists(self, location, command): test = 'type %s > /dev/null 2>&1 || echo -n cmdnotfound' % command - output = cliapp.ssh_runcmd(location, ['sh', '-c', test]) + output = writeexts.ssh_runcmd(location, ['sh', '-c', test]) if output == 'cmdnotfound': - raise cliapp.AppException( + raise writeexts.ExtensionError( "%s does not have %s" % (location, command)) diff --git a/extensions/ssh-rsync.write b/extensions/ssh-rsync.write index 6d596500..46c16662 100755 --- a/extensions/ssh-rsync.write +++ b/extensions/ssh-rsync.write @@ -18,23 +18,23 @@ import contextlib -import cliapp import os +import subprocess import sys -import time import tempfile +import time -import morphlib.writeexts +import writeexts def ssh_runcmd_ignore_failure(location, command, **kwargs): try: - return cliapp.ssh_runcmd(location, command, **kwargs) - except cliapp.AppException: + return writeexts.ssh_runcmd(location, command, **kwargs) + except writeexts.ExtensionError: pass -class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): +class SshRsyncWriteExtension(writeexts.WriteExtension): '''See ssh-rsync.write.help for documentation''' @@ -43,7 +43,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): '''Read /proc/mounts on location to find which device contains "/"''' self.status(msg='Finding device that contains "/"') - contents = cliapp.ssh_runcmd(location, ['cat', '/proc/mounts']) + contents = writeexts.ssh_runcmd(location, + ['cat', '/proc/mounts']) for line in contents.splitlines(): line_words = line.split() if (line_words[1] == '/' and line_words[0] != 'rootfs'): @@ -52,28 +53,29 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): @contextlib.contextmanager def _remote_mount_point(self, location): self.status(msg='Creating remote mount point') - remote_mnt = cliapp.ssh_runcmd(location, ['mktemp', '-d']).strip() + remote_mnt = writeexts.ssh_runcmd(location, + ['mktemp', '-d']).strip() try: yield remote_mnt finally: self.status(msg='Removing remote mount point') - cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) + writeexts.ssh_runcmd(location, ['rmdir', remote_mnt]) @contextlib.contextmanager def _remote_mount(self, location, root_disk, mountpoint): self.status(msg='Mounting root disk') - cliapp.ssh_runcmd(location, ['mount', root_disk, mountpoint]) + writeexts.ssh_runcmd(location, ['mount', root_disk, mountpoint]) try: yield finally: self.status(msg='Unmounting root disk') - cliapp.ssh_runcmd(location, ['umount', mountpoint]) + writeexts.ssh_runcmd(location, ['umount', mountpoint]) @contextlib.contextmanager def _created_version_root(self, location, remote_mnt, version_label): version_root = os.path.join(remote_mnt, 'systems', version_label) self.status(msg='Creating %(root)s', root=version_root) - cliapp.ssh_runcmd(location, ['mkdir', version_root]) + writeexts.ssh_runcmd(location, ['mkdir', version_root]) try: yield version_root except BaseException as e: @@ -93,8 +95,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Creating "orig" subvolume') old_orig = self.get_old_orig(location, remote_mnt) new_orig = os.path.join(version_root, 'orig') - cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', 'snapshot', - old_orig, new_orig]) + writeexts.ssh_runcmd(location, ['btrfs', 'subvolume', 'snapshot', + old_orig, new_orig]) try: yield new_orig except BaseException as e: @@ -106,30 +108,30 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): '''Populate the subvolume version_root/orig on location''' self.status(msg='Populating "orig" subvolume') - cliapp.runcmd(['rsync', '-as', '--checksum', '--numeric-ids', - '--delete', temp_root + os.path.sep, - '%s:%s' % (location, new_orig)]) + subprocess.check_call(['rsync', '-as', '--checksum', '--numeric-ids', + '--delete', temp_root + os.path.sep, + '%s:%s' % (location, new_orig)]) @contextlib.contextmanager def _deployed_version(self, location, version_label, system_config_sync, system_version_manager): self.status(msg='Calling system-version-manager to deploy upgrade') deployment = os.path.join('/systems', version_label, 'orig') - cliapp.ssh_runcmd(location, + writeexts.ssh_runcmd(location, ['env', 'BASEROCK_SYSTEM_CONFIG_SYNC='+system_config_sync, system_version_manager, 'deploy', deployment]) try: yield deployment except BaseException as e: self.status(msg='Cleaning up failed version installation') - cliapp.ssh_runcmd(location, + writeexts.ssh_runcmd(location, [system_version_manager, 'remove', version_label]) raise def upgrade_remote_system(self, location, temp_root): root_disk = self.find_root_disk(location) - uuid = cliapp.ssh_runcmd(location, ['blkid', '-s', 'UUID', '-o', - 'value', root_disk]).strip() + uuid = writeexts.ssh_runcmd(location, + ['blkid', '-s', 'UUID', '-o', 'value', root_disk]).strip() self.complete_fstab_for_btrfs_layout(temp_root, uuid) @@ -153,8 +155,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): config_sync, version_manager): self.status(msg='Setting %(v)s as the new default system', v=version_label) - cliapp.ssh_runcmd(location, [version_manager, - 'set-default', version_label]) + writeexts.ssh_runcmd(location, + [version_manager, 'set-default', version_label]) if autostart: self.status(msg="Rebooting into new system ...") @@ -162,7 +164,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') temp_root, location = args diff --git a/extensions/strip-gplv3.configure b/extensions/strip-gplv3.configure index c08061ad..0c5250e4 100755 --- a/extensions/strip-gplv3.configure +++ b/extensions/strip-gplv3.configure @@ -21,12 +21,14 @@ to find the files created by that chunk, then remove them. ''' -import cliapp -import re -import os import json +import os +import re +import sys + +import writeexts -class StripGPLv3ConfigureExtension(cliapp.Application): +class StripGPLv3ConfigureExtension(writeexts.Extension): gplv3_chunks = [ ['autoconf', ''], ['automake', ''], @@ -57,7 +59,8 @@ class StripGPLv3ConfigureExtension(cliapp.Application): for chunk in self.gplv3_chunks: regex = os.path.join(meta_dir, "%s-[^-]\+\.meta" % chunk[0]) - artifacts = self.runcmd(['find', meta_dir, '-regex', regex]) + artifacts = subprocess.check_output(['find', meta_dir, + '-regex', regex]) for artifact in artifacts.split(): self.remove_chunk(target_root, artifact, chunk[1]) @@ -72,8 +75,8 @@ class StripGPLv3ConfigureExtension(cliapp.Application): chunk_meta_data = json.load(f) if not 'contents' in chunk_meta_data: - raise cliapp.AppError('Chunk %s does not have a "contents" list' - % chunk) + raise writeexts.ExtensionError( + 'Chunk %s does not have a "contents" list' % chunk) updated_contents = [] for content_entry in reversed(chunk_meta_data['contents']): pat = re.compile(pattern) @@ -85,8 +88,8 @@ class StripGPLv3ConfigureExtension(cliapp.Application): def remove_content_entry(self, target_root, content_entry): entry_path = os.path.join(target_root, './' + content_entry) if not entry_path.startswith(target_root): - raise cliapp.AppException('%s is not in %s' - % (entry_path, target_root)) + raise writeexts.ExtensionError( + '%s is not in %s' % (entry_path, target_root)) if os.path.exists(entry_path): if os.path.islink(entry_path): os.unlink(entry_path) @@ -96,6 +99,7 @@ class StripGPLv3ConfigureExtension(cliapp.Application): if not os.listdir(entry_path): os.rmdir(entry_path) else: - raise cliapp.AppException('%s is not a link, file or directory' - % entry_path) -StripGPLv3ConfigureExtension().run() + raise writeexts.ExtensionError( + '%s is not a link, file or directory' % entry_path) + +StripGPLv3ConfigureExtension().run(sys.argv[1:]) diff --git a/extensions/virtualbox-ssh.check b/extensions/virtualbox-ssh.check index a97f3294..e82d58a1 100755 --- a/extensions/virtualbox-ssh.check +++ b/extensions/virtualbox-ssh.check @@ -15,21 +15,21 @@ '''Preparatory checks for Morph 'virtualbox-ssh' write extension''' -import cliapp -import morphlib.writeexts +import writeexts -class VirtualBoxPlusSshCheckExtension(morphlib.writeexts.WriteExtension): +class VirtualBoxPlusSshCheckExtension(writeexts.WriteExtension): def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') self.require_btrfs_in_deployment_host_kernel() upgrade = self.get_environment_boolean('UPGRADE') if upgrade: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Use the `ssh-rsync` write extension to deploy upgrades to an ' 'existing remote system.') diff --git a/extensions/virtualbox-ssh.write b/extensions/virtualbox-ssh.write index 774f2b4f..95643a4a 100755 --- a/extensions/virtualbox-ssh.write +++ b/extensions/virtualbox-ssh.write @@ -24,22 +24,23 @@ See file virtualbox-ssh.write.help for documentation ''' -import cliapp import os import re +import subprocess import sys -import time import tempfile +import time import urlparse -import morphlib.writeexts +import writeexts -class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): +class VirtualBoxPlusSshWriteExtension(writeexts.WriteExtension): def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') temp_root, location = args ssh_host, vm_name, vdi_path = self.parse_location(location) @@ -59,7 +60,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): except BaseException: sys.stderr.write('Error deploying to VirtualBox') os.remove(raw_disk) - cliapp.ssh_runcmd(ssh_host, ['rm', '-f', vdi_path]) + writeexts.ssh_runcmd(ssh_host, ['rm', '-f', vdi_path]) raise else: os.remove(raw_disk) @@ -72,11 +73,12 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): x = urlparse.urlparse(location) if x.scheme != 'vbox+ssh': - raise cliapp.AppException( + raise writeexts.ExtensionError( 'URL schema must be vbox+ssh in %s' % location) m = re.match('^/(?P[^/]+)(?P/.+)$', x.path) if not m: - raise cliapp.AppException('Cannot parse location %s' % location) + raise writeexts.ExtensionError( + 'Cannot parse location %s' % location) return x.netloc, m.group('guest'), m.group('path') def transfer_and_convert_to_vdi(self, raw_disk, ssh_host, vdi_path): @@ -85,17 +87,18 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Transfer disk and convert to VDI') st = os.lstat(raw_disk) - xfer_hole_path = morphlib.util.get_data_path('xfer-hole') - recv_hole = morphlib.util.get_data('recv-hole') + # TODO: Something! + xfer_hole_path = writeexts.get_data_path('xfer-hole') + recv_hole = writeexts.get_data('recv-hole') ssh_remote_cmd = [ 'sh', '-c', recv_hole, 'dummy-argv0', 'vbox', vdi_path, str(st.st_size), ] - cliapp.runcmd( + subprocess.check_call( ['python', xfer_hole_path, raw_disk], - ['ssh', ssh_host] + map(cliapp.shell_quote, ssh_remote_cmd), + ['ssh', ssh_host] + map(writeexts.shell_quote, ssh_remote_cmd), stdout=None, stderr=None) def virtualbox_version(self, ssh_host): @@ -107,7 +110,8 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): # tuple is more reliable than a string and more convenient than # comparing against the major, minor and patch numbers directly self.status(msg='Checking version of remote VirtualBox') - build_id = cliapp.ssh_runcmd(ssh_host, ['VBoxManage', '--version']) + build_id = writeexts.ssh_runcmd(ssh_host, + ['VBoxManage', '--version']) version_string = re.match(r"^([0-9\.]+).*$", build_id.strip()).group(1) return tuple(int(s or '0') for s in version_string.split('.')) @@ -163,17 +167,17 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): for command in commands: argv = ['VBoxManage'] + command - cliapp.ssh_runcmd(ssh_host, argv) + writeexts.ssh_runcmd(ssh_host, argv) def get_host_interface(self, ssh_host): host_ipaddr = os.environ.get('HOST_IPADDR') netmask = os.environ.get('NETMASK') if host_ipaddr is None: - raise cliapp.AppException('HOST_IPADDR was not given') + raise writeexts.ExtensionError('HOST_IPADDR was not given') if netmask is None: - raise cliapp.AppException('NETMASK was not given') + raise writeexts.ExtensionError('NETMASK was not given') # 'VBoxManage list hostonlyifs' retrieves a list with the hostonly # interfaces on the host. For each interface, the following lines @@ -187,7 +191,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): # The following command tries to retrieve the hostonly interface # name (e.g. vboxnet0) associated with the given ip address. iface = None - lines = cliapp.ssh_runcmd(ssh_host, + lines = writeexts.ssh_runcmd(ssh_host, ['VBoxManage', 'list', 'hostonlyifs']).splitlines() for i, v in enumerate(lines): if host_ipaddr in v: @@ -195,12 +199,12 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): break if iface is None: - iface = cliapp.ssh_runcmd(ssh_host, + iface = writeexts.ssh_runcmd(ssh_host, ['VBoxManage', 'hostonlyif', 'create']) # 'VBoxManage hostonlyif create' shows the name of the # created hostonly interface inside single quotes iface = iface[iface.find("'") + 1 : iface.rfind("'")] - cliapp.ssh_runcmd(ssh_host, + writeexts.ssh_runcmd(ssh_host, ['VBoxManage', 'hostonlyif', 'ipconfig', iface, '--ip', host_ipaddr, diff --git a/extensions/writeexts.py b/extensions/writeexts.py index 9357648f..5c579a10 100644 --- a/extensions/writeexts.py +++ b/extensions/writeexts.py @@ -13,27 +13,119 @@ # with this program. If not, see . -import cliapp +import contextlib +import errno +import fcntl import logging import os import re +import select import shutil +import stat +import subprocess import sys import time import tempfile -import errno -import stat -import contextlib -@contextlib.contextmanager -def hide_password_environment_variables(env): # pragma: no cover - password_env = { k:v for k,v in env.iteritems() if 'PASSWORD' in k } - for k in password_env: - env[k] = '(value hidden)' - yield - for k, v in password_env.iteritems(): - env[k] = v +if sys.version_info >= (3, 3, 0): + import shlex + shell_quote = shlex.quote +else: + import pipes + shell_quote = pipes.quote + + +def get_data_path(relative_path): + extensions_dir = os.path.dirname(__file__) + return os.path.join(extensions_dir, relative_path) + + +def get_data(relative_path): + with open(get_data_path(relative_path)) as f: + return f.read() + + +def ssh_runcmd(host, args, **kwargs): + '''Run command over ssh''' + command = ['ssh', host, '--'] + [shell_quote(arg) for arg in args] + + feed_stdin = kwargs.get('feed_stdin') + stdin = kwargs.get('stdin', subprocess.PIPE) + stdout = kwargs.get('stdout', subprocess.PIPE) + stderr = kwargs.get('stderr', subprocess.PIPE) + + p = subprocess.Popen(command, stdin=stdin, stdout=stdout, stderr=stderr) + out, err = p.communicate(input=feed_stdin) + if p.returncode != 0: + raise ExtensionError('ssh command `%s` failed' % ' '.join(command)) + return out + + +def write_from_dict(filepath, d, validate=lambda x, y: True): + """Takes a dictionary and appends the contents to a file + + An optional validation callback can be passed to perform validation on + each value in the dictionary. + + e.g. + + def validation_callback(dictionary_key, dictionary_value): + if not dictionary_value.isdigit(): + raise Exception('value contains non-digit character(s)') + + Any callback supplied to this function should raise an exception + if validation fails. + + """ + # Sort items asciibetically + # the output of the deployment should not depend + # on the locale of the machine running the deployment + items = sorted(d.iteritems(), key=lambda (k, v): [ord(c) for c in v]) + + for (k, v) in items: + validate(k, v) + + with open(filepath, 'a') as f: + for (_, v) in items: + f.write('%s\n' % v) + + os.fchown(f.fileno(), 0, 0) + os.fchmod(f.fileno(), 0644) + + +def parse_environment_pairs(env, pairs): + '''Add key=value pairs to the environment dict. + + Given a dict and a list of strings of the form key=value, + set dict[key] = value, unless key is already set in the + environment, at which point raise an exception. + + This does not modify the passed in dict. + + Returns the extended dict. + + ''' + extra_env = dict(p.split('=', 1) for p in pairs) + conflicting = [k for k in extra_env if k in env] + if conflicting: + raise ExtensionError('Environment already set: %s' + % ', '.join(conflicting)) + + # Return a dict that is the union of the two + # This is not the most performant, since it creates + # 3 unnecessary lists, but I felt this was the most + # easy to read. Using itertools.chain may be more efficicent + return dict(env.items() + extra_env.items()) + + +class ExtensionError(Exception): + + def __init__(self, msg): + self.msg = msg + + def __str__(self): + return self.msg class Fstab(object): @@ -89,9 +181,9 @@ class Fstab(object): shutil.move(os.path.abspath(tmp), os.path.abspath(self.filepath)) -class WriteExtension(cliapp.Application): +class Extension(object): - '''A base class for deployment write extensions. + '''A base class for deployment extensions. A subclass should subclass this class, and add a ``process_args`` method. @@ -108,8 +200,6 @@ class WriteExtension(cliapp.Application): This file descriptor is read by Morph and written into its own log file. - This overrides cliapp's usual configurable logging setup. - ''' log_write_fd = int(os.environ.get('MORPH_LOG_FD', 0)) @@ -125,13 +215,19 @@ class WriteExtension(cliapp.Application): logger.addHandler(handler) logger.setLevel(logging.DEBUG) - def log_config(self): - with hide_password_environment_variables(os.environ): - cliapp.Application.log_config(self) - def process_args(self, args): raise NotImplementedError() + def run(self, args=None): + if args is None: + args = sys.argv[1:] + try: + self.setup_logging() + self.process_args(args) + except ExtensionError as e: + sys.stdout.write('ERROR: %s\n' % e) + sys.exit(1) + def status(self, **kwargs): '''Provide status output. @@ -140,9 +236,22 @@ class WriteExtension(cliapp.Application): by %. ''' + sys.stdout.write('%s\n' % (kwargs['msg'] % kwargs)) + sys.stdout.flush() + - self.output.write('%s\n' % (kwargs['msg'] % kwargs)) - self.output.flush() +class WriteExtension(Extension): + + '''A base class for deployment write extensions. + + A subclass should subclass this class, and add a + ``process_args`` method. + + Note that it is not necessary to subclass this class for write + extensions. This class is here just to collect common code for + write extensions. + + ''' def check_for_btrfs_in_deployment_host_kernel(self): with open('/proc/filesystems') as f: @@ -151,7 +260,7 @@ class WriteExtension(cliapp.Application): def require_btrfs_in_deployment_host_kernel(self): if not self.check_for_btrfs_in_deployment_host_kernel(): - raise cliapp.AppException( + raise ExtensionError( 'Error: Btrfs is required for this deployment, but was not ' 'detected in the kernel of the machine that is running Morph.') @@ -166,7 +275,7 @@ class WriteExtension(cliapp.Application): def created_disk_image(self, location): size = self.get_disk_size() if not size: - raise cliapp.AppException('DISK_SIZE is not defined') + raise ExtensionError('DISK_SIZE is not defined') self.create_raw_disk_image(location, size) try: yield @@ -220,8 +329,8 @@ class WriteExtension(cliapp.Application): return None bytes = self._parse_size(size) if bytes is None: - raise cliapp.AppException('Cannot parse %s value %s' - % (env_var, size)) + raise ExtensionError('Cannot parse %s value %s' + % (env_var, size)) return bytes def get_disk_size(self): @@ -254,15 +363,15 @@ class WriteExtension(cliapp.Application): # need to do this because at the time of writing, SYSLINUX has not # been updated to understand these new features and will fail to # boot if the kernel is on a filesystem where they are enabled. - cliapp.runcmd( + subprocess.check_output( ['mkfs.btrfs','-f', '-L', 'baserock', '--features', '^extref', '--features', '^skinny-metadata', '--features', '^mixed-bg', '--nodesize', '4096', location]) - except cliapp.AppException as e: - if 'unrecognized option \'--features\'' in e.msg: + except subprocess.CalledProcessError as e: + if 'unrecognized option \'--features\'' in e.output: # Old versions of mkfs.btrfs (including v0.20, present in many # Baserock releases) don't support the --features option, but # also don't enable the new features by default. So we can @@ -270,7 +379,8 @@ class WriteExtension(cliapp.Application): logging.debug( 'Assuming mkfs.btrfs failure was because the tool is too ' 'old to have --features flag.') - cliapp.runcmd(['mkfs.btrfs','-f', '-L', 'baserock', location]) + subprocess.check_call(['mkfs.btrfs','-f', + '-L', 'baserock', location]) else: raise @@ -278,8 +388,8 @@ class WriteExtension(cliapp.Application): '''Get the UUID of a block device's file system.''' # Requires util-linux blkid; busybox one ignores options and # lies by exiting successfully. - return cliapp.runcmd(['blkid', '-s', 'UUID', '-o', 'value', - location]).strip() + return subprocess.check_output(['blkid', '-s', 'UUID', '-o', 'value', + location]).strip() @contextlib.contextmanager def mount(self, location): @@ -287,9 +397,10 @@ class WriteExtension(cliapp.Application): try: mount_point = tempfile.mkdtemp() if self.is_device(location): - cliapp.runcmd(['mount', location, mount_point]) + subprocess.check_call(['mount', location, mount_point]) else: - cliapp.runcmd(['mount', '-o', 'loop', location, mount_point]) + subprocess.check_call(['mount', '-o', 'loop', + location, mount_point]) except BaseException as e: sys.stderr.write('Error mounting filesystem') os.rmdir(mount_point) @@ -298,7 +409,7 @@ class WriteExtension(cliapp.Application): yield mount_point finally: self.status(msg='Unmounting filesystem') - cliapp.runcmd(['umount', mount_point]) + subprocess.check_call(['umount', mount_point]) os.rmdir(mount_point) def create_btrfs_system_layout(self, temp_root, mountpoint, version_label, @@ -345,9 +456,9 @@ class WriteExtension(cliapp.Application): orig = os.path.join(version_root, 'orig') self.status(msg='Creating orig subvolume') - cliapp.runcmd(['btrfs', 'subvolume', 'create', orig]) + subprocess.check_call(['btrfs', 'subvolume', 'create', orig]) self.status(msg='Copying files to orig subvolume') - cliapp.runcmd(['cp', '-a', temp_root + '/.', orig + '/.']) + subprocess.check_call(['cp', '-a', temp_root + '/.', orig + '/.']) def create_run(self, version_root): '''Create the 'run' snapshot.''' @@ -355,7 +466,7 @@ class WriteExtension(cliapp.Application): self.status(msg='Creating run subvolume') orig = os.path.join(version_root, 'orig') run = os.path.join(version_root, 'run') - cliapp.runcmd( + subprocess.check_call( ['btrfs', 'subvolume', 'snapshot', orig, run]) def create_state_subvolume(self, system_dir, mountpoint, state_subdir): @@ -369,7 +480,7 @@ class WriteExtension(cliapp.Application): ''' self.status(msg='Creating %s subvolume' % state_subdir) subvolume = os.path.join(mountpoint, 'state', state_subdir) - cliapp.runcmd(['btrfs', 'subvolume', 'create', subvolume]) + subprocess.check_call(['btrfs', 'subvolume', 'create', subvolume]) os.chmod(subvolume, 0o755) existing_state_dir = os.path.join(system_dir, state_subdir) @@ -380,7 +491,7 @@ class WriteExtension(cliapp.Application): self.status(msg='Moving existing data to %s subvolume' % subvolume) for filename in files: filepath = os.path.join(existing_state_dir, filename) - cliapp.runcmd(['mv', filepath, subvolume]) + subprocess.check_call(['mv', filepath, subvolume]) def complete_fstab_for_btrfs_layout(self, system_dir, rootfs_uuid=None): '''Fill in /etc/fstab entries for the default Btrfs disk layout. @@ -430,8 +541,8 @@ class WriteExtension(cliapp.Application): if 'INITRAMFS_PATH' in os.environ: initramfs = os.path.join(temp_root, os.environ['INITRAMFS_PATH']) if not os.path.exists(initramfs): - raise cliapp.AppException('INITRAMFS_PATH specified, ' - 'but file does not exist') + raise ExtensionError('INITRAMFS_PATH specified, ' + 'but file does not exist') return initramfs return None @@ -443,7 +554,7 @@ class WriteExtension(cliapp.Application): ''' self.status(msg='Installing initramfs') initramfs_dest = os.path.join(version_root, 'initramfs') - cliapp.runcmd(['cp', '-a', initramfs_path, initramfs_dest]) + subprocess.check_call(['cp', '-a', initramfs_path, initramfs_dest]) def install_kernel(self, version_root, temp_root): '''Install the kernel outside of 'orig' or 'run' subvolumes''' @@ -454,7 +565,7 @@ class WriteExtension(cliapp.Application): for name in image_names: try_path = os.path.join(temp_root, 'boot', name) if os.path.exists(try_path): - cliapp.runcmd(['cp', '-a', try_path, kernel_dest]) + subprocess.check_call(['cp', '-a', try_path, kernel_dest]) break def install_dtb(self, version_root, temp_root): @@ -465,10 +576,10 @@ class WriteExtension(cliapp.Application): dtb_dest = os.path.join(version_root, 'dtb') try_path = os.path.join(temp_root, device_tree_path) if os.path.exists(try_path): - cliapp.runcmd(['cp', '-a', try_path, dtb_dest]) + subprocess.check_call(['cp', '-a', try_path, dtb_dest]) else: logging.error("Failed to find device tree %s", device_tree_path) - raise cliapp.AppException( + raise ExtensionError( 'Failed to find device tree %s' % device_tree_path) def get_dtb_path(self): @@ -500,7 +611,7 @@ class WriteExtension(cliapp.Application): if config_type in config_function_dict: config_function_dict[config_type](real_root, disk_uuid) else: - raise cliapp.AppException( + raise ExtensionError( 'Invalid BOOTLOADER_CONFIG_FORMAT %s' % config_type) def generate_extlinux_config(self, real_root, disk_uuid=None): @@ -544,15 +655,15 @@ class WriteExtension(cliapp.Application): if install_type in install_function_dict: install_function_dict[install_type](real_root) elif install_type != 'none': - raise cliapp.AppException( + raise ExtensionError( 'Invalid BOOTLOADER_INSTALL %s' % install_type) def install_bootloader_extlinux(self, real_root): self.status(msg='Installing extlinux') - cliapp.runcmd(['extlinux', '--install', real_root]) + subprocess.check_call(['extlinux', '--install', real_root]) # FIXME this hack seems to be necessary to let extlinux finish - cliapp.runcmd(['sync']) + subprocess.check_call(['sync']) time.sleep(2) def install_syslinux_menu(self, real_root, version_root): @@ -610,19 +721,19 @@ class WriteExtension(cliapp.Application): elif value in ['yes', '1', 'true']: return True else: - raise cliapp.AppException('Unexpected value for %s: %s' % - (variable, value)) + raise ExtensionError('Unexpected value for %s: %s' % + (variable, value)) def check_ssh_connectivity(self, ssh_host): try: - output = cliapp.ssh_runcmd(ssh_host, ['echo', 'test']) - except cliapp.AppException as e: + output = ssh_runcmd(ssh_host, ['echo', 'test']) + except ExtensionError as e: logging.error("Error checking SSH connectivity: %s", str(e)) - raise cliapp.AppException( + raise ExtensionError( 'Unable to SSH to %s: %s' % (ssh_host, e)) if output.strip() != 'test': - raise cliapp.AppException( + raise ExtensionError( 'Unexpected output from remote machine: %s' % output.strip()) def is_device(self, location): -- cgit v1.2.1