diff options
| -rw-r--r-- | morphlib/app.py | 27 | ||||
| -rw-r--r-- | morphlib/builder2.py | 17 | ||||
| -rw-r--r-- | morphlib/exts/kvm.write.help | 31 | ||||
| -rwxr-xr-x | morphlib/exts/nfsboot.write | 13 | ||||
| -rw-r--r-- | morphlib/exts/nfsboot.write.help | 9 | ||||
| -rwxr-xr-x | morphlib/exts/openstack.check | 2 | ||||
| -rw-r--r-- | morphlib/exts/rawdisk.write.help | 30 | ||||
| -rw-r--r-- | morphlib/exts/virtualbox-ssh.write.help | 30 | ||||
| -rw-r--r-- | morphlib/stagingarea.py | 61 | ||||
| -rw-r--r-- | morphlib/util.py | 17 | ||||
| -rw-r--r-- | morphlib/writeexts.py | 27 |
11 files changed, 208 insertions, 56 deletions
diff --git a/morphlib/app.py b/morphlib/app.py index 930e023d..eb0ff3b7 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -17,6 +17,7 @@ import cliapp import logging import os +import pipes import sys import time import urlparse @@ -348,7 +349,7 @@ class Morph(cliapp.Application): self.output.write('%s %s\n' % (timestamp, text)) self.output.flush() - def runcmd(self, argv, *args, **kwargs): + def _prepare_for_runcmd(self, argv, args, kwargs): if 'env' not in kwargs: kwargs['env'] = dict(os.environ) @@ -358,16 +359,15 @@ class Morph(cliapp.Application): else: print_command = True - # convert the command line arguments into a string - commands = [argv] + list(args) - for command in commands: - if isinstance(command, list): - for i in xrange(0, len(command)): - command[i] = str(command[i]) - commands = [' '.join(command) for command in commands] - - # print the command line if print_command: + # Print the command line + commands = [argv] + list(args) + for command in commands: + if isinstance(command, list): + for i in xrange(0, len(command)): + command[i] = str(command[i]) + commands = ' '.join(map(pipes.quote, command)) + self.status(msg='# %(cmdline)s', cmdline=' | '.join(commands), chatty=True) @@ -377,9 +377,14 @@ class Morph(cliapp.Application): morphlib.util.log_environment_changes(self, kwargs['env'], prev) self.prev_env = kwargs['env'] - # run the command line + def runcmd(self, argv, *args, **kwargs): + self._prepare_for_runcmd(argv, args, kwargs) return cliapp.Application.runcmd(self, argv, *args, **kwargs) + def runcmd_unchecked(self, argv, *args, **kwargs): + self._prepare_for_runcmd(argv, args, kwargs) + return cliapp.Application.runcmd_unchecked(self, argv, *args, **kwargs) + def parse_args(self, args, configs_only=False): return self.settings.parse_args(args, configs_only=configs_only, diff --git a/morphlib/builder2.py b/morphlib/builder2.py index f71f21db..9e158ea1 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -665,11 +665,18 @@ class SystemBuilder(BuilderBase): # pragma: no cover ) try: for bin in sorted(os.listdir(sys_integration_dir)): - self.app.runcmd( - morphlib.util.containerised_cmdline( - [os.path.join(SYSTEM_INTEGRATION_PATH, bin)], - root=rootdir, mounts=to_mount, mount_proc=True), - env=env) + argv = [os.path.join(SYSTEM_INTEGRATION_PATH, bin)] + container_config = dict( + root=rootdir, mounts=to_mount, mount_proc=True) + cmdline = morphlib.util.containerised_cmdline( + argv, **container_config) + exit, out, err = self.app.runcmd_unchecked( + cmdline, env=env) + if exit != 0: + logging.debug('Command returned code %i', exit) + msg = error_message_for_containerised_commandline( + argv, err, container_config) + raise cliapp.AppException(msg) except BaseException, e: self.app.status( msg='Error while running system integration commands', diff --git a/morphlib/exts/kvm.write.help b/morphlib/exts/kvm.write.help index 26a54d9c..04393b8a 100644 --- a/morphlib/exts/kvm.write.help +++ b/morphlib/exts/kvm.write.help @@ -42,6 +42,35 @@ help: | * AUTOSTART=<VALUE>` - boolean. If it is set, the VM will be started when it has been deployed. + * DTB_PATH=path: **(MANDATORY)** for systems that require a device tree + binary - Give the full path (without a leading /) to the location of the + DTB in the built system image . The deployment will fail if `path` does + not exist. + + * BOOTLOADER_INSTALL=value: the bootloader to be installed + **(MANDATORY)** for non-x86 systems + + allowed values = + - 'extlinux' (default) - the extlinux bootloader will + be installed + - 'none' - no bootloader will be installed by `morph deploy`. A + bootloader must be installed manually. This value must be used when + deploying non-x86 systems such as ARM. + + * BOOTLOADER_CONFIG_FORMAT=value: the bootloader format to be used. + If not specified for x86-32 and x86-64 systems, 'extlinux' will be used + + allowed values = + - 'extlinux' + + * KERNEL_ARGS=args: optional additional kernel command-line parameters to + be appended to the default set. The default set is: + + 'rw init=/sbin/init rootfstype=btrfs \ + rootflags=subvol=systems/default/run \ + root=[name or UUID of root filesystem]' + + (See https://www.kernel.org/doc/Documentation/kernel-parameters.txt) + (See `morph help deploy` for details of how to pass parameters to write extensions) - diff --git a/morphlib/exts/nfsboot.write b/morphlib/exts/nfsboot.write index 8d3d6df7..49d71174 100755 --- a/morphlib/exts/nfsboot.write +++ b/morphlib/exts/nfsboot.write @@ -17,6 +17,16 @@ '''A Morph deployment write extension for deploying to an nfsboot server +*** DO NOT USE *** +- This was written before 'proper' deployment mechanisms were in place +It is unlikely to work at all and will not work correctly + +Use the pxeboot write extension instead + +*** + + + An nfsboot server is defined as a baserock system that has tftp and nfs servers running, the tftp server is exporting the contents of /srv/nfsboot/tftp/ and the user has sufficient permissions to create nfs roots @@ -125,7 +135,7 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Creating destination directories') try: - cliapp.ssh_runcmd('root@%s' % location, + cliapp.ssh_runcmd('root@%s' % location, ['mkdir', '-p', orig_path, run_path]) except cliapp.AppException: raise cliapp.AppException('Could not create dirs %s and %s on %s' @@ -191,4 +201,3 @@ mv "$temp" "$target" NFSBootWriteExtension().run() - diff --git a/morphlib/exts/nfsboot.write.help b/morphlib/exts/nfsboot.write.help index 598b1b23..310fd7a4 100644 --- a/morphlib/exts/nfsboot.write.help +++ b/morphlib/exts/nfsboot.write.help @@ -1,6 +1,13 @@ help: | + *** DO NOT USE *** + - This was written before 'proper' deployment mechanisms were in place. + It is unlikely to work at all, and will not work correctly. + + Use the pxeboot write extension instead + + *** Deploy a system image and kernel to an nfsboot server. - + An nfsboot server is defined as a baserock system that has tftp and nfs servers running, the tftp server is exporting the contents of /srv/nfsboot/tftp/ and the user has sufficient diff --git a/morphlib/exts/openstack.check b/morphlib/exts/openstack.check index a6856c31..3850d481 100755 --- a/morphlib/exts/openstack.check +++ b/morphlib/exts/openstack.check @@ -81,7 +81,7 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): exit, out, err = cliapp.runcmd_unchecked(cmdline) if exit != 0: - if err.startswith('The request you have made requires ' \ + if err.startswith('The request you have made requires ' 'authentication. (HTTP 401)'): raise cliapp.AppException('Invalid OpenStack credentials.') else: diff --git a/morphlib/exts/rawdisk.write.help b/morphlib/exts/rawdisk.write.help index 81f35024..54af81c4 100644 --- a/morphlib/exts/rawdisk.write.help +++ b/morphlib/exts/rawdisk.write.help @@ -34,5 +34,35 @@ help: | * INITRAMFS_PATH=path: the location of an initramfs for the bootloader to tell Linux to use, rather than booting the rootfs directly. + * DTB_PATH=path: **(MANDATORY)** for systems that require a device tree + binary - Give the full path (without a leading /) to the location of the + DTB in the built system image . The deployment will fail if `path` does + not exist. + + * BOOTLOADER_INSTALL=value: the bootloader to be installed + **(MANDATORY)** for non-x86 systems + + allowed values = + - 'extlinux' (default) - the extlinux bootloader will + be installed + - 'none' - no bootloader will be installed by `morph deploy`. A + bootloader must be installed manually. This value must be used when + deploying non-x86 systems such as ARM. + + * BOOTLOADER_CONFIG_FORMAT=value: the bootloader format to be used. + If not specified for x86-32 and x86-64 systems, 'extlinux' will be used + + allowed values = + - 'extlinux' + + * KERNEL_ARGS=args: optional additional kernel command-line parameters to + be appended to the default set. The default set is: + + 'rw init=/sbin/init rootfstype=btrfs \ + rootflags=subvol=systems/default/run \ + root=[name or UUID of root filesystem]' + + (See https://www.kernel.org/doc/Documentation/kernel-parameters.txt) + (See `morph help deploy` for details of how to pass parameters to write extensions) diff --git a/morphlib/exts/virtualbox-ssh.write.help b/morphlib/exts/virtualbox-ssh.write.help index b4c59553..cb50acc0 100644 --- a/morphlib/exts/virtualbox-ssh.write.help +++ b/morphlib/exts/virtualbox-ssh.write.help @@ -41,6 +41,36 @@ help: | * INITRAMFS_PATH=path: the location of an initramfs for the bootloader to tell Linux to use, rather than booting the rootfs directly. + * DTB_PATH=path: **(MANDATORY)** for systems that require a device tree + binary - Give the full path (without a leading /) to the location of the + DTB in the built system image . The deployment will fail if `path` does + not exist. + + * BOOTLOADER_INSTALL=value: the bootloader to be installed + **(MANDATORY)** for non-x86 systems + + allowed values = + - 'extlinux' (default) - the extlinux bootloader will + be installed + - 'none' - no bootloader will be installed by `morph deploy`. A + bootloader must be installed manually. This value must be used when + deploying non-x86 systems such as ARM. + + * BOOTLOADER_CONFIG_FORMAT=value: the bootloader format to be used. + If not specified for x86-32 and x86-64 systems, 'extlinux' will be used + + allowed values = + - 'extlinux' + + * KERNEL_ARGS=args: optional additional kernel command-line parameters to + be appended to the default set. The default set is: + + 'rw init=/sbin/init rootfstype=btrfs \ + rootflags=subvol=systems/default/run \ + root=[name or UUID of root filesystem]' + + (See https://www.kernel.org/doc/Documentation/kernel-parameters.txt) + * AUTOSTART=<VALUE> - boolean. If it is set, the VM will be started when it has been deployed. diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index 42fdd076..fd3f6881 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -267,7 +267,13 @@ class StagingArea(object): do_not_mount_dirs += [temp_dir] logging.debug("Not mounting dirs %r" % do_not_mount_dirs) + if self.use_chroot: + mounts = self.to_mount_in_staging + else: + mounts = [(os.path.join(self.dirname, target), type, source) + for target, type, source in self.to_mount_in_bootstrap] mount_proc = self.use_chroot + if ccache_dir and not self._app.settings['no-ccache']: ccache_target = os.path.join( self.dirname, kwargs['env']['CCACHE_DIR'].lstrip('/')) @@ -275,27 +281,38 @@ class StagingArea(object): else: binds = () - if self.use_chroot: - mounts = self.to_mount_in_staging - else: - mounts = [(os.path.join(self.dirname, target), type, source) - for target, type, source in self.to_mount_in_bootstrap] + container_config=dict( + cwd=kwargs.pop('cwd', '/'), + root=chroot_dir, + mounts=mounts, + mount_proc=mount_proc, + binds=binds, + writable_paths=do_not_mount_dirs) + cmdline = morphlib.util.containerised_cmdline( - argv, cwd=kwargs.pop('cwd', '/'), - root=chroot_dir, mounts=mounts, - binds=binds, mount_proc=mount_proc, - writable_paths=do_not_mount_dirs) - try: - if kwargs.get('logfile') != None: - logfile = kwargs.pop('logfile') - teecmd = ['tee', '-a', logfile] - return self._app.runcmd(cmdline, teecmd, **kwargs) - else: - return self._app.runcmd(cmdline, **kwargs) - except cliapp.AppException as e: - raise cliapp.AppException('In staging area %s: running ' - 'command \'%s\' failed.' % - (self.dirname, ' '.join(argv))) + argv, **container_config) + + if kwargs.get('logfile') != None: + logfile = kwargs.pop('logfile') + teecmd = ['tee', '-a', logfile] + exit, out, err = self._app.runcmd_unchecked( + cmdline, teecmd, **kwargs) + else: + exit, out, err = self._app.runcmd_unchecked(cmdline, **kwargs) + + if exit == 0: + return out + else: + logging.debug('Command returned code %i', exit) + msg = morphlib.util.error_message_for_containerised_commandline( + argv, err, container_config) + raise cliapp.AppException( + 'In staging area %s: %s' % (self._failed_location(), msg)) + + def _failed_location(self): # pragma: no cover + '''Path this staging area will be moved to if an error occurs.''' + return os.path.join(self._app.settings['tempdir'], 'failed', + os.path.basename(self.dirname)) def abort(self): # pragma: no cover '''Handle what to do with a staging area in the case of failure. @@ -304,9 +321,7 @@ class StagingArea(object): # TODO: when we add the option to throw away failed builds, # hook it up here - - dest_dir = os.path.join(self._app.settings['tempdir'], - 'failed', os.path.basename(self.dirname)) + dest_dir = self._failed_location() os.rename(self.dirname, dest_dir) self.dirname = dest_dir diff --git a/morphlib/util.py b/morphlib/util.py index 6f735387..e7a8a50e 100644 --- a/morphlib/util.py +++ b/morphlib/util.py @@ -16,6 +16,7 @@ import contextlib import itertools import os +import pipes import re import subprocess import textwrap @@ -41,7 +42,6 @@ try: from multiprocessing import cpu_count except NotImplementedError: # pragma: no cover cpu_count = lambda: 1 -import os def indent(string, spaces=4): @@ -626,3 +626,18 @@ def containerised_cmdline(args, cwd='.', root='/', binds=(), cmdargs.append(root) cmdargs.extend(args) return unshared_cmdline(cmdargs, root=root, **kwargs) + + +def error_message_for_containerised_commandline( + argv, err, container_kwargs): # pragma: no cover + '''Return a semi-readable error message for a containerised command.''' + + # This function should do some formatting of the container_kwargs dict, + # rather than just dumping it in the error message, but that is better than + # nothing. + + argv_string = ' '.join(map(pipes.quote, argv)) + return 'Command failed: %s:\n' \ + 'Containerisation settings: %s\n' \ + 'Error output:\n%s' \ + % (argv_string, container_kwargs, err) diff --git a/morphlib/writeexts.py b/morphlib/writeexts.py index 91936f64..6ab2dd55 100644 --- a/morphlib/writeexts.py +++ b/morphlib/writeexts.py @@ -83,14 +83,14 @@ class Fstab(object): class WriteExtension(cliapp.Application): '''A base class for deployment write extensions. - + A subclass should subclass this class, and add a ``process_args`` method. - + Note that it is not necessary to subclass this class for write extensions. This class is here just to collect common code for write extensions. - + ''' def setup_logging(self): @@ -125,13 +125,13 @@ class WriteExtension(cliapp.Application): def status(self, **kwargs): '''Provide status output. - + The ``msg`` keyword argument is the actual message, the rest are values for fields in the message as interpolated by %. - + ''' - + self.output.write('%s\n' % (kwargs['msg'] % kwargs)) self.output.flush() @@ -184,9 +184,9 @@ class WriteExtension(cliapp.Application): def _parse_size(self, size): '''Parse a size from a string. - + Return size in bytes. - + ''' m = re.match('^(\d+)([kmgKMG]?)$', size) @@ -474,6 +474,12 @@ class WriteExtension(cliapp.Application): self.status(msg='Creating extlinux.conf') config = os.path.join(real_root, 'extlinux.conf') + + ''' Please also update the documentation in the following files + if you change these default kernel args: + - kvm.write.help + - rawdisk.write.help + - virtualbox-ssh.write.help ''' kernel_args = ( 'rw ' # ro ought to work, but we don't test that regularly 'init=/sbin/init ' # default, but it doesn't hurt to be explicit @@ -545,9 +551,8 @@ class WriteExtension(cliapp.Application): '''Does the user want to generate a bootloader config? The user may set $BOOTLOADER_CONFIG_FORMAT to the desired - format (u-boot or extlinux). If not set, extlinux is the - default but will be generated on x86-32 and x86-64, but not - otherwise. + format. 'extlinux' is the only allowed value, and is the default + value for x86-32 and x86-64. ''' |