summaryrefslogtreecommitdiff
path: root/extensions/writeexts.py
diff options
context:
space:
mode:
authorAdam Coldrick <adam.coldrick@codethink.co.uk>2015-06-04 15:17:44 +0000
committerAdam Coldrick <adam.coldrick@codethink.co.uk>2015-06-11 08:55:05 +0000
commite4c6b8a69f0df2d0b3beac46865a66e0de527151 (patch)
treed837a9768fe2a15557d400301ed8b6588504eb55 /extensions/writeexts.py
parent840292841f4495a79a037f81a26d6b3f51e7cb8c (diff)
downloaddefinitions-e4c6b8a69f0df2d0b3beac46865a66e0de527151.tar.gz
Remove dependencies on morphlib and cliapp from deployment extensionsbaserock/adamcoldrick/remove-dependencies-v3
This is done by either copying some utility functions from morph into writeexts.py, and using the `subprocess` module rather than cliapp's runcmd and ssh_runcmd. Note that this means that these extensions will require "$definitions_checkout/extensions" in PYTHONPATH when they are run. This commit also updates VERSION to 5, since the PYTHONPATH requirement means that this change is incompatible with old versions of morph. Change-Id: Iec6fa7e3c7219619ce55e18493e5c37c36e97816
Diffstat (limited to 'extensions/writeexts.py')
-rw-r--r--extensions/writeexts.py223
1 files changed, 167 insertions, 56 deletions
diff --git a/extensions/writeexts.py b/extensions/writeexts.py
index 9357648f..5c579a10 100644
--- a/extensions/writeexts.py
+++ b/extensions/writeexts.py
@@ -13,27 +13,119 @@
# with this program. If not, see <http://www.gnu.org/licenses/>.
-import cliapp
+import contextlib
+import errno
+import fcntl
import logging
import os
import re
+import select
import shutil
+import stat
+import subprocess
import sys
import time
import tempfile
-import errno
-import stat
-import contextlib
-@contextlib.contextmanager
-def hide_password_environment_variables(env): # pragma: no cover
- password_env = { k:v for k,v in env.iteritems() if 'PASSWORD' in k }
- for k in password_env:
- env[k] = '(value hidden)'
- yield
- for k, v in password_env.iteritems():
- env[k] = v
+if sys.version_info >= (3, 3, 0):
+ import shlex
+ shell_quote = shlex.quote
+else:
+ import pipes
+ shell_quote = pipes.quote
+
+
+def get_data_path(relative_path):
+ extensions_dir = os.path.dirname(__file__)
+ return os.path.join(extensions_dir, relative_path)
+
+
+def get_data(relative_path):
+ with open(get_data_path(relative_path)) as f:
+ return f.read()
+
+
+def ssh_runcmd(host, args, **kwargs):
+ '''Run command over ssh'''
+ command = ['ssh', host, '--'] + [shell_quote(arg) for arg in args]
+
+ feed_stdin = kwargs.get('feed_stdin')
+ stdin = kwargs.get('stdin', subprocess.PIPE)
+ stdout = kwargs.get('stdout', subprocess.PIPE)
+ stderr = kwargs.get('stderr', subprocess.PIPE)
+
+ p = subprocess.Popen(command, stdin=stdin, stdout=stdout, stderr=stderr)
+ out, err = p.communicate(input=feed_stdin)
+ if p.returncode != 0:
+ raise ExtensionError('ssh command `%s` failed' % ' '.join(command))
+ return out
+
+
+def write_from_dict(filepath, d, validate=lambda x, y: True):
+ """Takes a dictionary and appends the contents to a file
+
+ An optional validation callback can be passed to perform validation on
+ each value in the dictionary.
+
+ e.g.
+
+ def validation_callback(dictionary_key, dictionary_value):
+ if not dictionary_value.isdigit():
+ raise Exception('value contains non-digit character(s)')
+
+ Any callback supplied to this function should raise an exception
+ if validation fails.
+
+ """
+ # Sort items asciibetically
+ # the output of the deployment should not depend
+ # on the locale of the machine running the deployment
+ items = sorted(d.iteritems(), key=lambda (k, v): [ord(c) for c in v])
+
+ for (k, v) in items:
+ validate(k, v)
+
+ with open(filepath, 'a') as f:
+ for (_, v) in items:
+ f.write('%s\n' % v)
+
+ os.fchown(f.fileno(), 0, 0)
+ os.fchmod(f.fileno(), 0644)
+
+
+def parse_environment_pairs(env, pairs):
+ '''Add key=value pairs to the environment dict.
+
+ Given a dict and a list of strings of the form key=value,
+ set dict[key] = value, unless key is already set in the
+ environment, at which point raise an exception.
+
+ This does not modify the passed in dict.
+
+ Returns the extended dict.
+
+ '''
+ extra_env = dict(p.split('=', 1) for p in pairs)
+ conflicting = [k for k in extra_env if k in env]
+ if conflicting:
+ raise ExtensionError('Environment already set: %s'
+ % ', '.join(conflicting))
+
+ # Return a dict that is the union of the two
+ # This is not the most performant, since it creates
+ # 3 unnecessary lists, but I felt this was the most
+ # easy to read. Using itertools.chain may be more efficicent
+ return dict(env.items() + extra_env.items())
+
+
+class ExtensionError(Exception):
+
+ def __init__(self, msg):
+ self.msg = msg
+
+ def __str__(self):
+ return self.msg
class Fstab(object):
@@ -89,9 +181,9 @@ class Fstab(object):
shutil.move(os.path.abspath(tmp), os.path.abspath(self.filepath))
-class WriteExtension(cliapp.Application):
+class Extension(object):
- '''A base class for deployment write extensions.
+ '''A base class for deployment extensions.
A subclass should subclass this class, and add a
``process_args`` method.
@@ -108,8 +200,6 @@ class WriteExtension(cliapp.Application):
This file descriptor is read by Morph and written into its own log
file.
- This overrides cliapp's usual configurable logging setup.
-
'''
log_write_fd = int(os.environ.get('MORPH_LOG_FD', 0))
@@ -125,13 +215,19 @@ class WriteExtension(cliapp.Application):
logger.addHandler(handler)
logger.setLevel(logging.DEBUG)
- def log_config(self):
- with hide_password_environment_variables(os.environ):
- cliapp.Application.log_config(self)
-
def process_args(self, args):
raise NotImplementedError()
+ def run(self, args=None):
+ if args is None:
+ args = sys.argv[1:]
+ try:
+ self.setup_logging()
+ self.process_args(args)
+ except ExtensionError as e:
+ sys.stdout.write('ERROR: %s\n' % e)
+ sys.exit(1)
+
def status(self, **kwargs):
'''Provide status output.
@@ -140,9 +236,22 @@ class WriteExtension(cliapp.Application):
by %.
'''
+ sys.stdout.write('%s\n' % (kwargs['msg'] % kwargs))
+ sys.stdout.flush()
+
- self.output.write('%s\n' % (kwargs['msg'] % kwargs))
- self.output.flush()
+class WriteExtension(Extension):
+
+ '''A base class for deployment write extensions.
+
+ A subclass should subclass this class, and add a
+ ``process_args`` method.
+
+ Note that it is not necessary to subclass this class for write
+ extensions. This class is here just to collect common code for
+ write extensions.
+
+ '''
def check_for_btrfs_in_deployment_host_kernel(self):
with open('/proc/filesystems') as f:
@@ -151,7 +260,7 @@ class WriteExtension(cliapp.Application):
def require_btrfs_in_deployment_host_kernel(self):
if not self.check_for_btrfs_in_deployment_host_kernel():
- raise cliapp.AppException(
+ raise ExtensionError(
'Error: Btrfs is required for this deployment, but was not '
'detected in the kernel of the machine that is running Morph.')
@@ -166,7 +275,7 @@ class WriteExtension(cliapp.Application):
def created_disk_image(self, location):
size = self.get_disk_size()
if not size:
- raise cliapp.AppException('DISK_SIZE is not defined')
+ raise ExtensionError('DISK_SIZE is not defined')
self.create_raw_disk_image(location, size)
try:
yield
@@ -220,8 +329,8 @@ class WriteExtension(cliapp.Application):
return None
bytes = self._parse_size(size)
if bytes is None:
- raise cliapp.AppException('Cannot parse %s value %s'
- % (env_var, size))
+ raise ExtensionError('Cannot parse %s value %s'
+ % (env_var, size))
return bytes
def get_disk_size(self):
@@ -254,15 +363,15 @@ class WriteExtension(cliapp.Application):
# need to do this because at the time of writing, SYSLINUX has not
# been updated to understand these new features and will fail to
# boot if the kernel is on a filesystem where they are enabled.
- cliapp.runcmd(
+ subprocess.check_output(
['mkfs.btrfs','-f', '-L', 'baserock',
'--features', '^extref',
'--features', '^skinny-metadata',
'--features', '^mixed-bg',
'--nodesize', '4096',
location])
- except cliapp.AppException as e:
- if 'unrecognized option \'--features\'' in e.msg:
+ except subprocess.CalledProcessError as e:
+ if 'unrecognized option \'--features\'' in e.output:
# Old versions of mkfs.btrfs (including v0.20, present in many
# Baserock releases) don't support the --features option, but
# also don't enable the new features by default. So we can
@@ -270,7 +379,8 @@ class WriteExtension(cliapp.Application):
logging.debug(
'Assuming mkfs.btrfs failure was because the tool is too '
'old to have --features flag.')
- cliapp.runcmd(['mkfs.btrfs','-f', '-L', 'baserock', location])
+ subprocess.check_call(['mkfs.btrfs','-f',
+ '-L', 'baserock', location])
else:
raise
@@ -278,8 +388,8 @@ class WriteExtension(cliapp.Application):
'''Get the UUID of a block device's file system.'''
# Requires util-linux blkid; busybox one ignores options and
# lies by exiting successfully.
- return cliapp.runcmd(['blkid', '-s', 'UUID', '-o', 'value',
- location]).strip()
+ return subprocess.check_output(['blkid', '-s', 'UUID', '-o', 'value',
+ location]).strip()
@contextlib.contextmanager
def mount(self, location):
@@ -287,9 +397,10 @@ class WriteExtension(cliapp.Application):
try:
mount_point = tempfile.mkdtemp()
if self.is_device(location):
- cliapp.runcmd(['mount', location, mount_point])
+ subprocess.check_call(['mount', location, mount_point])
else:
- cliapp.runcmd(['mount', '-o', 'loop', location, mount_point])
+ subprocess.check_call(['mount', '-o', 'loop',
+ location, mount_point])
except BaseException as e:
sys.stderr.write('Error mounting filesystem')
os.rmdir(mount_point)
@@ -298,7 +409,7 @@ class WriteExtension(cliapp.Application):
yield mount_point
finally:
self.status(msg='Unmounting filesystem')
- cliapp.runcmd(['umount', mount_point])
+ subprocess.check_call(['umount', mount_point])
os.rmdir(mount_point)
def create_btrfs_system_layout(self, temp_root, mountpoint, version_label,
@@ -345,9 +456,9 @@ class WriteExtension(cliapp.Application):
orig = os.path.join(version_root, 'orig')
self.status(msg='Creating orig subvolume')
- cliapp.runcmd(['btrfs', 'subvolume', 'create', orig])
+ subprocess.check_call(['btrfs', 'subvolume', 'create', orig])
self.status(msg='Copying files to orig subvolume')
- cliapp.runcmd(['cp', '-a', temp_root + '/.', orig + '/.'])
+ subprocess.check_call(['cp', '-a', temp_root + '/.', orig + '/.'])
def create_run(self, version_root):
'''Create the 'run' snapshot.'''
@@ -355,7 +466,7 @@ class WriteExtension(cliapp.Application):
self.status(msg='Creating run subvolume')
orig = os.path.join(version_root, 'orig')
run = os.path.join(version_root, 'run')
- cliapp.runcmd(
+ subprocess.check_call(
['btrfs', 'subvolume', 'snapshot', orig, run])
def create_state_subvolume(self, system_dir, mountpoint, state_subdir):
@@ -369,7 +480,7 @@ class WriteExtension(cliapp.Application):
'''
self.status(msg='Creating %s subvolume' % state_subdir)
subvolume = os.path.join(mountpoint, 'state', state_subdir)
- cliapp.runcmd(['btrfs', 'subvolume', 'create', subvolume])
+ subprocess.check_call(['btrfs', 'subvolume', 'create', subvolume])
os.chmod(subvolume, 0o755)
existing_state_dir = os.path.join(system_dir, state_subdir)
@@ -380,7 +491,7 @@ class WriteExtension(cliapp.Application):
self.status(msg='Moving existing data to %s subvolume' % subvolume)
for filename in files:
filepath = os.path.join(existing_state_dir, filename)
- cliapp.runcmd(['mv', filepath, subvolume])
+ subprocess.check_call(['mv', filepath, subvolume])
def complete_fstab_for_btrfs_layout(self, system_dir, rootfs_uuid=None):
'''Fill in /etc/fstab entries for the default Btrfs disk layout.
@@ -430,8 +541,8 @@ class WriteExtension(cliapp.Application):
if 'INITRAMFS_PATH' in os.environ:
initramfs = os.path.join(temp_root, os.environ['INITRAMFS_PATH'])
if not os.path.exists(initramfs):
- raise cliapp.AppException('INITRAMFS_PATH specified, '
- 'but file does not exist')
+ raise ExtensionError('INITRAMFS_PATH specified, '
+ 'but file does not exist')
return initramfs
return None
@@ -443,7 +554,7 @@ class WriteExtension(cliapp.Application):
'''
self.status(msg='Installing initramfs')
initramfs_dest = os.path.join(version_root, 'initramfs')
- cliapp.runcmd(['cp', '-a', initramfs_path, initramfs_dest])
+ subprocess.check_call(['cp', '-a', initramfs_path, initramfs_dest])
def install_kernel(self, version_root, temp_root):
'''Install the kernel outside of 'orig' or 'run' subvolumes'''
@@ -454,7 +565,7 @@ class WriteExtension(cliapp.Application):
for name in image_names:
try_path = os.path.join(temp_root, 'boot', name)
if os.path.exists(try_path):
- cliapp.runcmd(['cp', '-a', try_path, kernel_dest])
+ subprocess.check_call(['cp', '-a', try_path, kernel_dest])
break
def install_dtb(self, version_root, temp_root):
@@ -465,10 +576,10 @@ class WriteExtension(cliapp.Application):
dtb_dest = os.path.join(version_root, 'dtb')
try_path = os.path.join(temp_root, device_tree_path)
if os.path.exists(try_path):
- cliapp.runcmd(['cp', '-a', try_path, dtb_dest])
+ subprocess.check_call(['cp', '-a', try_path, dtb_dest])
else:
logging.error("Failed to find device tree %s", device_tree_path)
- raise cliapp.AppException(
+ raise ExtensionError(
'Failed to find device tree %s' % device_tree_path)
def get_dtb_path(self):
@@ -500,7 +611,7 @@ class WriteExtension(cliapp.Application):
if config_type in config_function_dict:
config_function_dict[config_type](real_root, disk_uuid)
else:
- raise cliapp.AppException(
+ raise ExtensionError(
'Invalid BOOTLOADER_CONFIG_FORMAT %s' % config_type)
def generate_extlinux_config(self, real_root, disk_uuid=None):
@@ -544,15 +655,15 @@ class WriteExtension(cliapp.Application):
if install_type in install_function_dict:
install_function_dict[install_type](real_root)
elif install_type != 'none':
- raise cliapp.AppException(
+ raise ExtensionError(
'Invalid BOOTLOADER_INSTALL %s' % install_type)
def install_bootloader_extlinux(self, real_root):
self.status(msg='Installing extlinux')
- cliapp.runcmd(['extlinux', '--install', real_root])
+ subprocess.check_call(['extlinux', '--install', real_root])
# FIXME this hack seems to be necessary to let extlinux finish
- cliapp.runcmd(['sync'])
+ subprocess.check_call(['sync'])
time.sleep(2)
def install_syslinux_menu(self, real_root, version_root):
@@ -610,19 +721,19 @@ class WriteExtension(cliapp.Application):
elif value in ['yes', '1', 'true']:
return True
else:
- raise cliapp.AppException('Unexpected value for %s: %s' %
- (variable, value))
+ raise ExtensionError('Unexpected value for %s: %s' %
+ (variable, value))
def check_ssh_connectivity(self, ssh_host):
try:
- output = cliapp.ssh_runcmd(ssh_host, ['echo', 'test'])
- except cliapp.AppException as e:
+ output = ssh_runcmd(ssh_host, ['echo', 'test'])
+ except ExtensionError as e:
logging.error("Error checking SSH connectivity: %s", str(e))
- raise cliapp.AppException(
+ raise ExtensionError(
'Unable to SSH to %s: %s' % (ssh_host, e))
if output.strip() != 'test':
- raise cliapp.AppException(
+ raise ExtensionError(
'Unexpected output from remote machine: %s' % output.strip())
def is_device(self, location):