diff options
-rw-r--r-- | morphlib/builder2.py | 25 | ||||
-rw-r--r-- | morphlib/cachekeycomputer.py | 2 | ||||
-rwxr-xr-x | morphlib/exts/kvm.check | 35 | ||||
-rwxr-xr-x | morphlib/exts/kvm.write | 4 | ||||
-rwxr-xr-x | morphlib/exts/nfsboot.check | 34 | ||||
-rwxr-xr-x | morphlib/exts/openstack.check | 35 | ||||
-rwxr-xr-x | morphlib/exts/rawdisk.write | 7 | ||||
-rwxr-xr-x | morphlib/exts/ssh-rsync.check | 36 | ||||
-rwxr-xr-x | morphlib/exts/ssh-rsync.write | 142 | ||||
-rwxr-xr-x | morphlib/exts/tar.check | 24 | ||||
-rwxr-xr-x | morphlib/exts/virtualbox-ssh.check | 35 | ||||
-rwxr-xr-x | morphlib/exts/virtualbox-ssh.write | 2 | ||||
-rw-r--r-- | morphlib/gitdir.py | 7 | ||||
-rw-r--r-- | morphlib/gitdir_tests.py | 11 | ||||
-rw-r--r-- | morphlib/plugins/deploy_plugin.py | 79 | ||||
-rw-r--r-- | morphlib/writeexts.py | 215 | ||||
-rw-r--r-- | tests.as-root/tarball-image-is-sensible.stdout | 1 | ||||
-rw-r--r-- | tests.build/bootstrap-mode.stdout | 1 | ||||
-rw-r--r-- | tests.build/build-stratum-with-submodules.stdout | 1 | ||||
-rw-r--r-- | tests.build/build-system.stdout | 1 | ||||
-rwxr-xr-x | tests.deploy/deploy-cluster.script | 17 | ||||
-rw-r--r-- | yarns/deployment.yarn | 30 | ||||
-rw-r--r-- | yarns/implementations.yarn | 10 | ||||
-rw-r--r-- | yarns/morph.shell-lib | 8 |
24 files changed, 577 insertions, 185 deletions
diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 2dca738c..2c99c6f6 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -549,7 +549,6 @@ class SystemBuilder(BuilderBase): # pragma: no cover fs_root = self.staging_area.destdir(self.artifact.source) self.unpack_strata(fs_root) self.write_metadata(fs_root, rootfs_name) - self.create_fstab(fs_root) self.copy_kernel_into_artifact_cache(fs_root) unslashy_root = fs_root[1:] def uproot_info(info): @@ -649,30 +648,6 @@ class SystemBuilder(BuilderBase): # pragma: no cover os.chmod(os_release_file, 0644) - def create_fstab(self, path): - '''Create an /etc/fstab inside a system tree. - - The fstab is created using assumptions of the disk layout. - If the assumptions are wrong, extend this code so it can deal - with other cases. - - ''' - - self.app.status(msg='Creating fstab in %(path)s', - path=path, chatty=True) - with self.build_watch('create-fstab'): - fstab = os.path.join(path, 'etc', 'fstab') - if not os.path.exists(fstab): - # FIXME: should exist - if not os.path.exists(os.path.dirname(fstab)): - os.makedirs(os.path.dirname(fstab)) - # We create an empty fstab: systemd does not require - # /sys and /proc entries, and we can't know what the - # right entry for / is. The fstab gets built during - # deployment instead, when that information is available. - with open(fstab, 'w'): - pass - def copy_kernel_into_artifact_cache(self, path): '''Copy the installed kernel image into the local artifact cache. diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index bb536f82..3efe1cbb 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -114,6 +114,6 @@ class CacheKeyComputer(object): if kind == 'stratum': keys['stratum-format-version'] = 1 elif kind == 'system': - keys['system-compatibility-version'] = "1~ (temporary, root rw)" + keys['system-compatibility-version'] = "2~ (upgradable, root rw)" return keys diff --git a/morphlib/exts/kvm.check b/morphlib/exts/kvm.check new file mode 100755 index 00000000..be7c51c2 --- /dev/null +++ b/morphlib/exts/kvm.check @@ -0,0 +1,35 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''Preparatory checks for Morph 'kvm' write extension''' + +import cliapp + +import morphlib.writeexts + + +class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + upgrade = self.get_environment_boolean('UPGRADE') + if upgrade: + raise cliapp.AppException( + 'Use the `ssh-rsync` write extension to deploy upgrades to an ' + 'existing remote system.') + +KvmPlusSshCheckExtension().run() diff --git a/morphlib/exts/kvm.write b/morphlib/exts/kvm.write index 4f877c22..94560972 100755 --- a/morphlib/exts/kvm.write +++ b/morphlib/exts/kvm.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -56,7 +56,7 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): temp_root, location = args ssh_host, vm_name, vm_path = self.parse_location(location) - autostart = self.parse_autostart() + autostart = self.get_environment_boolean('AUTOSTART') fd, raw_disk = tempfile.mkstemp() os.close(fd) diff --git a/morphlib/exts/nfsboot.check b/morphlib/exts/nfsboot.check new file mode 100755 index 00000000..092a1df7 --- /dev/null +++ b/morphlib/exts/nfsboot.check @@ -0,0 +1,34 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''Preparatory checks for Morph 'nfsboot' write extension''' + +import cliapp + +import morphlib.writeexts + + +class NFSBootCheckExtension(morphlib.writeexts.WriteExtension): + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + upgrade = self.get_environment_boolean('UPGRADE') + if upgrade: + raise cliapp.AppException( + 'Upgrading is not currently supported for NFS deployments.') + +NFSBootCheckExtension().run() diff --git a/morphlib/exts/openstack.check b/morphlib/exts/openstack.check new file mode 100755 index 00000000..a9a8fe1b --- /dev/null +++ b/morphlib/exts/openstack.check @@ -0,0 +1,35 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''Preparatory checks for Morph 'openstack' write extension''' + +import cliapp + +import morphlib.writeexts + + +class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + upgrade = self.get_environment_boolean('UPGRADE') + if upgrade: + raise cliapp.AppException( + 'Use the `ssh-rsync` write extension to deploy upgrades to an ' + 'existing remote system.') + +OpenStackCheckExtension().run() diff --git a/morphlib/exts/rawdisk.write b/morphlib/exts/rawdisk.write index 8723ac0c..87edf7bf 100755 --- a/morphlib/exts/rawdisk.write +++ b/morphlib/exts/rawdisk.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -50,12 +50,15 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): self.create_local_system(temp_root, location) self.status(msg='Disk image has been created at %s' % location) except Exception: - os.remove(location) self.status(msg='Failure to create disk image at %s' % location) + if os.path.exists(location): + os.remove(location) raise def upgrade_local_system(self, raw_disk, temp_root): + self.complete_fstab_for_btrfs_layout(temp_root) + mp = self.mount(raw_disk) version_label = self.get_version_label(mp) diff --git a/morphlib/exts/ssh-rsync.check b/morphlib/exts/ssh-rsync.check new file mode 100755 index 00000000..90029cb4 --- /dev/null +++ b/morphlib/exts/ssh-rsync.check @@ -0,0 +1,36 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''Preparatory checks for Morph 'ssh-rsync' write extension''' + +import cliapp + +import morphlib.writeexts + + +class SshRsyncCheckExtension(morphlib.writeexts.WriteExtension): + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + upgrade = self.get_environment_boolean('UPGRADE') + if not upgrade: + raise cliapp.AppException( + 'The ssh-rsync write is for upgrading existing remote ' + 'Baserock machines. It cannot be used for an initial ' + 'deployment.') + +SshRsyncCheckExtension().run() diff --git a/morphlib/exts/ssh-rsync.write b/morphlib/exts/ssh-rsync.write index 211dbe5e..509520ae 100755 --- a/morphlib/exts/ssh-rsync.write +++ b/morphlib/exts/ssh-rsync.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,6 +26,14 @@ import tempfile import morphlib.writeexts + +def ssh_runcmd_ignore_failure(location, command, **kwargs): + try: + return cliapp.ssh_runcmd(location, command, **kwargs) + except cliapp.AppException: + pass + + class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): '''Upgrade a running baserock system with ssh and rsync. @@ -47,8 +55,11 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.upgrade_remote_system(location, temp_root) def upgrade_remote_system(self, location, temp_root): + self.complete_fstab_for_btrfs_layout(temp_root) + root_disk = self.find_root_disk(location) version_label = os.environ.get('VERSION_LABEL') + autostart = self.get_environment_boolean('AUTOSTART') self.status(msg='Creating remote mount point') remote_mnt = cliapp.ssh_runcmd(location, ['mktemp', '-d']).strip() @@ -56,15 +67,11 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Mounting root disk') cliapp.ssh_runcmd(location, ['mount', root_disk, remote_mnt]) except Exception as e: - try: - cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) - except: - pass + ssh_runcmd_ignore_failure(location, ['rmdir', remote_mnt]) raise e try: version_root = os.path.join(remote_mnt, 'systems', version_label) - run_dir = os.path.join(version_root, 'run') orig_dir = os.path.join(version_root, 'orig') self.status(msg='Creating %s' % version_root) @@ -73,80 +80,40 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.create_remote_orig(location, version_root, remote_mnt, temp_root) - self.status(msg='Creating "run" subvolume') - cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', - 'snapshot', orig_dir, run_dir]) - - self.status(msg='Updating system configuration') - bscs_loc = os.path.join(run_dir, 'usr', 'bin', - 'baserock-system-config-sync') - - output = cliapp.ssh_runcmd(location, ['sh', '-c', - '"$1" merge "$2" &> /dev/null || echo -n cmdfailed', - '-', bscs_loc, version_label]) - if output == "cmdfailed": - self.status(msg='Updating system configuration failed') - - self.install_remote_kernel(location, version_root, temp_root) - default_path = os.path.join(remote_mnt, 'systems', 'default') - if self.bootloader_is_wanted(): - output = cliapp.ssh_runcmd(location, ['sh', '-c', - 'test -e "$1" && stat -c %F "$1" ' - '|| echo missing file', - '-', default_path]) - if output != "symbolic link": - # we are upgrading and old system that does - # not have an updated extlinux config file - self.update_remote_extlinux(location, remote_mnt, - version_label) - cliapp.ssh_runcmd(location, ['ln', '-sfn', version_label, - default_path]) + # Use the system-version-manager from the new system we just + # installed, so that we can upgrade from systems that don't have + # it installed. + self.status(msg='Calling system-version-manager to deploy upgrade') + deployment = os.path.join('/systems', version_label, 'orig') + system_config_sync = os.path.join( + remote_mnt, 'systems', version_label, 'orig', 'usr', 'bin', + 'baserock-system-config-sync') + system_version_manager = os.path.join( + remote_mnt, 'systems', version_label, 'orig', 'usr', 'bin', + 'system-version-manager') + cliapp.ssh_runcmd(location, + ['env', 'BASEROCK_SYSTEM_CONFIG_SYNC='+system_config_sync, + system_version_manager, 'deploy', deployment]) + + self.status(msg='Setting %s as the new default system' % + version_label) + cliapp.ssh_runcmd(location, + [system_version_manager, 'set-default', version_label]) except Exception as e: - try: - cliapp.ssh_runcmd(location, - ['btrfs', 'subvolume', 'delete', run_dir]) - except: - pass - try: - cliapp.ssh_runcmd(location, - ['btrfs', 'subvolume', 'delete', orig_dir]) - except: - pass - try: - cliapp.ssh_runcmd(location, ['rm', '-rf', version_root]) - except: - pass + self.status(msg='Deployment failed') + ssh_runcmd_ignore_failure( + location, ['btrfs', 'subvolume', 'delete', orig_dir]) + ssh_runcmd_ignore_failure( + location, ['rm', '-rf', version_root]) raise e finally: self.status(msg='Removing temporary mounts') cliapp.ssh_runcmd(location, ['umount', remote_mnt]) cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) - def update_remote_extlinux(self, location, remote_mnt, version_label): - '''Install/reconfigure extlinux on location''' - - self.status(msg='Creating extlinux.conf') - config = os.path.join(remote_mnt, 'extlinux.conf') - temp_fd, temp_path = tempfile.mkstemp() - with os.fdopen(temp_fd, 'w') as f: - f.write('default linux\n') - f.write('timeout 1\n') - f.write('label linux\n') - f.write('kernel /systems/default/kernel\n') - f.write('append root=/dev/sda ' - 'rootflags=subvol=systems/default/run ' - 'init=/sbin/init rw\n') - - try: - cliapp.runcmd(['rsync', '-as', temp_path, - '%s:%s~' % (location, config)]) - cliapp.ssh_runcmd(location, ['mv', config+'~', config]) - except Exception as e: - try: - cliapp.ssh_runcmd(location, ['rm', '-f', config+'~']) - except: - pass - raise e + if autostart: + self.status(msg="Rebooting into new system ...") + ssh_runcmd_ignore_failure(location, ['reboot']) def create_remote_orig(self, location, version_root, remote_mnt, temp_root): @@ -178,18 +145,6 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): if (line_words[1] == '/' and line_words[0] != 'rootfs'): return line_words[0] - def install_remote_kernel(self, location, version_root, temp_root): - '''Install the kernel in temp_root inside version_root on location''' - - self.status(msg='Installing kernel') - image_names = ('vmlinuz', 'zImage', 'uImage') - kernel_dest = os.path.join(version_root, 'kernel') - for name in image_names: - try_path = os.path.join(temp_root, 'boot', name) - if os.path.exists(try_path): - cliapp.runcmd(['rsync', '-as', try_path, - '%s:%s' % (location, kernel_dest)]) - def check_valid_target(self, location): try: cliapp.ssh_runcmd(location, ['true']) @@ -203,10 +158,17 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): raise cliapp.AppException('%s is not a baserock system' % location) - output = cliapp.ssh_runcmd(location, ['sh', '-c', - 'type rsync &> /dev/null || echo -n cmdnotfound']) - if output == 'cmdnotfound': - raise cliapp.AppException('%s does not have rsync' - % location) + def check_command_exists(command): + test = 'type %s > /dev/null 2>&1 || echo -n cmdnotfound' % command + output = cliapp.ssh_runcmd(location, ['sh', '-c', test]) + if output == 'cmdnotfound': + raise cliapp.AppException( + "%s does not have %s" % (location, command)) + + # The deploy requires baserock-system-config-sync and + # system-version-manager in the new system only. The old system doesn't + # need to have them at all. + check_command_exists('rsync') + SshRsyncWriteExtension().run() diff --git a/morphlib/exts/tar.check b/morphlib/exts/tar.check new file mode 100755 index 00000000..cbeaf163 --- /dev/null +++ b/morphlib/exts/tar.check @@ -0,0 +1,24 @@ +#!/bin/sh +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# Preparatory checks for Morph 'tar' write extension + +set -eu + +if [ "$UPGRADE" == "yes" ]; then + echo >&2 "ERROR: Cannot upgrade a tar file deployment." + exit 1 +fi diff --git a/morphlib/exts/virtualbox-ssh.check b/morphlib/exts/virtualbox-ssh.check new file mode 100755 index 00000000..1aeb8999 --- /dev/null +++ b/morphlib/exts/virtualbox-ssh.check @@ -0,0 +1,35 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''Preparatory checks for Morph 'virtualbox-ssh' write extension''' + +import cliapp + +import morphlib.writeexts + + +class VirtualBoxPlusSshCheckExtension(morphlib.writeexts.WriteExtension): + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + upgrade = self.get_environment_boolean('UPGRADE') + if upgrade: + raise cliapp.AppException( + 'Use the `ssh-rsync` write extension to deploy upgrades to an ' + 'existing remote system.') + +VirtualBoxPlusSshCheckExtension().run() diff --git a/morphlib/exts/virtualbox-ssh.write b/morphlib/exts/virtualbox-ssh.write index 204b2447..2a2f3f7b 100755 --- a/morphlib/exts/virtualbox-ssh.write +++ b/morphlib/exts/virtualbox-ssh.write @@ -62,7 +62,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): temp_root, location = args ssh_host, vm_name, vdi_path = self.parse_location(location) - autostart = self.parse_autostart() + autostart = self.get_environment_boolean('AUTOSTART') fd, raw_disk = tempfile.mkstemp() os.close(fd) diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py index be2137b2..f5ef0061 100644 --- a/morphlib/gitdir.py +++ b/morphlib/gitdir.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -610,6 +610,11 @@ class GitDirectory(object): except Exception, e: raise RefDeleteError(self, ref, old_sha1, e) + def describe(self): + version = self._runcmd( + ['git', 'describe', '--always', '--dirty=-unreproducible']) + return version.strip() + def init(dirname): '''Initialise a new git repository.''' diff --git a/morphlib/gitdir_tests.py b/morphlib/gitdir_tests.py index 21a6b5b8..8c312c1b 100644 --- a/morphlib/gitdir_tests.py +++ b/morphlib/gitdir_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -202,6 +202,15 @@ class GitDirectoryContentsTests(unittest.TestCase): ) self.assertEqual(expected, gd.get_commit_contents(commit).split('\n')) + def test_describe(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + + gd._runcmd(['git', 'tag', '-a', '-m', 'Example', 'example', 'HEAD']) + self.assertEqual(gd.describe(), 'example-unreproducible') + + gd._runcmd(['git', 'reset', '--hard']) + self.assertEqual(gd.describe(), 'example') + class GitDirectoryRefTwiddlingTests(unittest.TestCase): diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index e8f1d217..61e1b5a4 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -16,6 +16,7 @@ import cliapp import contextlib +import json import os import shutil import stat @@ -33,9 +34,20 @@ import morphlib import morphlib.plugins.branch_and_merge_plugin +class ExtensionNotFoundError(morphlib.Error): + pass + + class DeployPlugin(cliapp.Plugin): def enable(self): + group_deploy = 'Deploy Options' + self.app.settings.boolean(['upgrade'], + 'specify that you want to upgrade an ' + 'existing cluster of systems rather than do ' + 'an initial deployment', + group=group_deploy) + self.app.add_subcommand( 'deploy', self.deploy, arg_synopsis='CLUSTER [SYSTEM.KEY=VALUE]') @@ -250,6 +262,13 @@ class DeployPlugin(cliapp.Plugin): are set as environment variables when either the configuration or the write extension runs (except `type` and `location`). + Deployment configuration is stored in the deployed system as + /baserock/deployment.meta. THIS CONTAINS ALL ENVIRONMENT VARIABLES SET + DURINGR DEPLOYMENT, so make sure you have no sensitive information in + your environment that is being leaked. As a special case, any + environment/deployment variable that contains 'PASSWORD' in its name is + stripped out and not stored in the final system. + ''' if not args: @@ -348,6 +367,9 @@ class DeployPlugin(cliapp.Plugin): deploy_params.items() + user_env.items()) + is_upgrade = 'yes' if self.app.settings['upgrade'] else 'no' + final_env['UPGRADE'] = is_upgrade + deployment_type = final_env.pop('type', None) if not deployment_type: raise morphlib.Error('"type" is undefined ' @@ -364,6 +386,15 @@ class DeployPlugin(cliapp.Plugin): def do_deploy(self, build_command, root_repo_dir, ref, artifact, deployment_type, location, env): + # Run optional write check extension. These are separate from the write + # extension because it may be several minutes before the write + # extension itself has the chance to raise an error. + try: + self._run_extension( + root_repo_dir, ref, deployment_type, '.check', + [location], env) + except ExtensionNotFoundError: + pass # Create a tempdir for this deployment to work in deploy_tempdir = tempfile.mkdtemp( @@ -396,6 +427,14 @@ class DeployPlugin(cliapp.Plugin): msg='System unpacked at %(system_tree)s', system_tree=system_tree) + self.app.status( + msg='Writing deployment metadata file') + metadata = self.create_metadata( + artifact, root_repo_dir, deployment_type, location, env) + metadata_path = os.path.join( + system_tree, 'baserock', 'deployment.meta') + with morphlib.savefile.SaveFile(metadata_path, 'w') as f: + f.write(json.dumps(metadata, indent=4, sort_keys=True)) # Run configuration extensions. self.app.status(msg='Configure system') @@ -445,7 +484,7 @@ class DeployPlugin(cliapp.Plugin): code_dir = os.path.dirname(morphlib.__file__) ext_filename = os.path.join(code_dir, 'exts', name + kind) if not os.path.exists(ext_filename): - raise morphlib.Error( + raise ExtensionNotFoundError( 'Could not find extension %s%s' % (name, kind)) if not self._is_executable(ext_filename): raise morphlib.Error( @@ -473,3 +512,41 @@ class DeployPlugin(cliapp.Plugin): st = os.stat(filename) mask = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH return (stat.S_IMODE(st.st_mode) & mask) != 0 + + def create_metadata(self, system_artifact, root_repo_dir, deployment_type, + location, env): + '''Deployment-specific metadata. + + The `build` and `deploy` operations must be from the same ref, so full + info on the root repo that the system came from is in + /baserock/${system_artifact}.meta and is not duplicated here. We do + store a `git describe` of the definitions.git repo as a convenience for + post-upgrade hooks that we may need to implement at a future date: + the `git describe` output lists the last tag, which will hopefully help + us to identify which release of a system was deployed without having to + keep a list of SHA1s somewhere or query a Trove. + + ''' + + def remove_passwords(env): + def is_password(key): + return 'PASSWORD' in key + return { k:v for k, v in env.iteritems() if not is_password(k) } + + meta = { + 'system-artifact-name': system_artifact.name, + 'configuration': remove_passwords(env), + 'deployment-type': deployment_type, + 'location': location, + 'definitions-version': { + 'describe': root_repo_dir.describe(), + }, + 'morph-version': { + 'ref': morphlib.gitversion.ref, + 'tree': morphlib.gitversion.tree, + 'commit': morphlib.gitversion.commit, + 'version': morphlib.gitversion.version, + }, + } + + return meta diff --git a/morphlib/writeexts.py b/morphlib/writeexts.py index 9dbc77e6..1849f406 100644 --- a/morphlib/writeexts.py +++ b/morphlib/writeexts.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,12 +17,65 @@ import cliapp import os import re +import shutil import sys import time import tempfile import morphlib + +class Fstab(object): + '''Small helper class for parsing and adding lines to /etc/fstab.''' + + # There is an existing Python helper library for editing of /etc/fstab. + # However it is unmaintained and has an incompatible license (GPL3). + # + # https://code.launchpad.net/~computer-janitor-hackers/python-fstab/trunk + + def __init__(self, filepath='/etc/fstab'): + if os.path.exists(filepath): + with open(filepath, 'r') as f: + self.text= f.read() + else: + self.text = '' + self.filepath = filepath + self.lines_added = 0 + + def get_mounts(self): + '''Return list of mount devices and targets in /etc/fstab. + + Return value is a dict of target -> device. + ''' + mounts = dict() + for line in self.text.splitlines(): + words = line.split() + if len(words) >= 2 and not words[0].startswith('#'): + device, target = words[0:2] + mounts[target] = device + return mounts + + def add_line(self, line): + '''Add a new entry to /etc/fstab. + + Lines are appended, and separated from any entries made by configure + extensions with a comment. + + ''' + if self.lines_added == 0: + if len(self.text) == 0 or self.text[-1] is not '\n': + self.text += '\n' + self.text += '# Morph default system layout\n' + self.lines_added += 1 + + self.text += line + '\n' + + def write(self): + '''Rewrite the fstab file to include all new entries.''' + with morphlib.savefile.SaveFile(self.filepath, 'w') as f: + f.write(self.text) + + class WriteExtension(cliapp.Application): '''A base class for deployment write extensions. @@ -53,7 +106,6 @@ class WriteExtension(cliapp.Application): def create_local_system(self, temp_root, raw_disk): '''Create a raw system image locally.''' - size = self.get_disk_size() if not size: raise cliapp.AppException('DISK_SIZE is not defined') @@ -66,19 +118,10 @@ class WriteExtension(cliapp.Application): os.remove(raw_disk) raise try: - version_label = 'factory' - version_root = os.path.join(mp, 'systems', version_label) - os.makedirs(version_root) - self.create_state(mp) - self.create_orig(version_root, temp_root) - self.create_fstab(version_root) - self.create_run(version_root) - os.symlink(version_label, os.path.join(mp, 'systems', 'default')) - if self.bootloader_is_wanted(): - self.install_kernel(version_root, temp_root) - self.install_extlinux(mp) + self.create_btrfs_system_layout( + temp_root, mp, version_label='factory') except BaseException, e: - sys.stderr.write('Error creating disk image') + sys.stderr.write('Error creating Btrfs system layout') self.unmount(mp) os.remove(raw_disk) raise @@ -129,16 +172,6 @@ class WriteExtension(cliapp.Application): '''Parse the virtual cpu count from environment.''' return self._parse_size_from_environment('VCPUS', '1') - def create_state(self, real_root): - '''Create the state subvolumes that are shared between versions''' - - self.status(msg='Creating state subvolumes') - os.mkdir(os.path.join(real_root, 'state')) - statedirs = ['home', 'opt', 'srv'] - for statedir in statedirs: - dirpath = os.path.join(real_root, 'state', statedir) - cliapp.runcmd(['btrfs', 'subvolume', 'create', dirpath]) - def create_raw_disk_image(self, filename, size): '''Create a raw disk image.''' @@ -178,6 +211,34 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['umount', mount_point]) os.rmdir(mount_point) + def create_btrfs_system_layout(self, temp_root, mountpoint, version_label): + '''Separate base OS versions from state using subvolumes. + + ''' + version_root = os.path.join(mountpoint, 'systems', version_label) + state_root = os.path.join(mountpoint, 'state') + + os.makedirs(version_root) + os.makedirs(state_root) + + self.create_orig(version_root, temp_root) + system_dir = os.path.join(version_root, 'orig') + + state_dirs = self.complete_fstab_for_btrfs_layout(system_dir) + + for state_dir in state_dirs: + self.create_state_subvolume(system_dir, mountpoint, state_dir) + + self.create_run(version_root) + + os.symlink( + version_label, os.path.join(mountpoint, 'systems', 'default')) + + if self.bootloader_is_wanted(): + self.install_kernel(version_root, temp_root) + self.install_syslinux_menu(mountpoint, version_root) + self.install_extlinux(mountpoint) + def create_orig(self, version_root, temp_root): '''Create the default "factory" system.''' @@ -197,29 +258,68 @@ class WriteExtension(cliapp.Application): cliapp.runcmd( ['btrfs', 'subvolume', 'snapshot', orig, run]) - def create_fstab(self, version_root): - '''Create an fstab.''' + def create_state_subvolume(self, system_dir, mountpoint, state_subdir): + '''Create a shared state subvolume. - self.status(msg='Creating fstab') - fstab = os.path.join(version_root, 'orig', 'etc', 'fstab') + We need to move any files added to the temporary rootfs by the + configure extensions to their correct home. For example, they might + have added keys in `/root/.ssh` which we now need to transfer to + `/state/root/.ssh`. - if os.path.exists(fstab): - with open(fstab, 'r') as f: - contents = f.read() - else: - contents = '' + ''' + self.status(msg='Creating %s subvolume' % state_subdir) + subvolume = os.path.join(mountpoint, 'state', state_subdir) + cliapp.runcmd(['btrfs', 'subvolume', 'create', subvolume]) + os.chmod(subvolume, 0755) + + existing_state_dir = os.path.join(system_dir, state_subdir) + files = [] + if os.path.exists(existing_state_dir): + files = os.listdir(existing_state_dir) + if len(files) > 0: + self.status(msg='Moving existing data to %s subvolume' % subvolume) + for filename in files: + filepath = os.path.join(existing_state_dir, filename) + shutil.move(filepath, subvolume) + + def complete_fstab_for_btrfs_layout(self, system_dir): + '''Fill in /etc/fstab entries for the default Btrfs disk layout. + + In the future we should move this code out of the write extension and + in to a configure extension. To do that, though, we need some way of + informing the configure extension what layout should be used. Right now + a configure extension doesn't know if the system is going to end up as + a Btrfs disk image, a tarfile or something else and so it can't come + up with a sensible default fstab. + + Configuration extensions can already create any /etc/fstab that they + like. This function only fills in entries that are missing, so if for + example the user configured /home to be on a separate partition, that + decision will be honoured and /state/home will not be created. - got_root = False - for line in contents.splitlines(): - words = line.split() - if len(words) >= 2 and not words[0].startswith('#'): - got_root = got_root or words[1] == '/' + ''' + shared_state_dirs = {'home', 'root', 'opt', 'srv', 'var'} + + fstab = Fstab(os.path.join(system_dir, 'etc', 'fstab')) + existing_mounts = fstab.get_mounts() + + if '/' in existing_mounts: + root_device = existing_mounts['/'] + else: + root_device = '/dev/sda' + fstab.add_line('/dev/sda / btrfs defaults,rw,noatime 0 1') - if not got_root: - contents += '\n/dev/sda / btrfs defaults,rw,noatime 0 1\n' + state_dirs_to_create = set() + for state_dir in shared_state_dirs: + if '/' + state_dir not in existing_mounts: + state_dirs_to_create.add(state_dir) + state_subvol = os.path.join('/state', state_dir) + fstab.add_line( + '%s /%s btrfs subvol=%s,defaults,rw,noatime 0 2' % + (root_device, state_dir, state_subvol)) - with open(fstab, 'w') as f: - f.write(contents) + fstab.write() + return state_dirs_to_create def install_kernel(self, version_root, temp_root): '''Install the kernel outside of 'orig' or 'run' subvolumes''' @@ -254,6 +354,23 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['sync']) time.sleep(2) + def install_syslinux_menu(self, real_root, version_root): + '''Make syslinux/extlinux menu binary available. + + The syslinux boot menu is compiled to a file named menu.c32. Extlinux + searches a few places for this file but it does not know to look inside + our subvolume, so we copy it to the filesystem root. + + If the file is not available, the bootloader will still work but will + not be able to show a menu. + + ''' + menu_file = os.path.join(version_root, 'orig', + 'usr', 'share', 'syslinux', 'menu.c32') + if os.path.isfile(menu_file): + self.status(msg='Copying menu.c32') + shutil.copy(menu_file, real_root) + def parse_attach_disks(self): '''Parse $ATTACH_DISKS into list of disks to attach.''' @@ -285,14 +402,14 @@ class WriteExtension(cliapp.Application): return value == 'yes' - def parse_autostart(self): - '''Parse $AUTOSTART to determine if VMs should be started.''' + def get_environment_boolean(self, variable): + '''Parse a yes/no boolean passed through the environment.''' - autostart = os.environ.get('AUTOSTART', 'no') - if autostart == 'no': + value = os.environ.get(variable, 'no').lower() + if value in ['no', '0', 'false']: return False - elif autostart == 'yes': + elif value in ['yes', '1', 'true']: return True else: - raise cliapp.AppException('Unexpected value for AUTOSTART: %s' % - autostart) + raise cliapp.AppException('Unexpected value for %s: %s' % + (variable, value)) diff --git a/tests.as-root/tarball-image-is-sensible.stdout b/tests.as-root/tarball-image-is-sensible.stdout index 4141dee8..cf74a1ec 100644 --- a/tests.as-root/tarball-image-is-sensible.stdout +++ b/tests.as-root/tarball-image-is-sensible.stdout @@ -33,7 +33,6 @@ ./boot/System.map ./boot/vmlinuz ./etc/ -./etc/fstab ./etc/os-release ./extlinux.conf NAME="Baserock" diff --git a/tests.build/bootstrap-mode.stdout b/tests.build/bootstrap-mode.stdout index e1747b15..26544a75 100644 --- a/tests.build/bootstrap-mode.stdout +++ b/tests.build/bootstrap-mode.stdout @@ -7,7 +7,6 @@ build-essential strata: hello-system: ./ etc/ -etc/fstab etc/os-release usr/ usr/bin/ diff --git a/tests.build/build-stratum-with-submodules.stdout b/tests.build/build-stratum-with-submodules.stdout index 6dda5049..d4d03e13 100644 --- a/tests.build/build-stratum-with-submodules.stdout +++ b/tests.build/build-stratum-with-submodules.stdout @@ -1,4 +1,3 @@ ./ etc/ -etc/fstab etc/os-release diff --git a/tests.build/build-system.stdout b/tests.build/build-system.stdout index 2e8270dc..4d0fac2f 100644 --- a/tests.build/build-system.stdout +++ b/tests.build/build-system.stdout @@ -2,5 +2,4 @@ bin/ bin/hello etc/ -etc/fstab etc/os-release diff --git a/tests.deploy/deploy-cluster.script b/tests.deploy/deploy-cluster.script index 0efc8d3c..3ef60479 100755 --- a/tests.deploy/deploy-cluster.script +++ b/tests.deploy/deploy-cluster.script @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -29,8 +29,11 @@ cd "$DATADIR/workspace/branch1" "$SRCDIR/scripts/test-morph" build linux-system +GIT_DIR=test:morphs/.git git tag -a my-test-tag -m "Example tag" HEAD + "$SRCDIR/scripts/test-morph" --log "$DATADIR/deploy.log" \ deploy test_cluster \ + linux-system-2.EXAMPLE_PASSWORD="secret" \ linux-system-2.HOSTNAME="baserock-rocks-even-more" \ > /dev/null @@ -44,3 +47,15 @@ hostname2=$(tar -xf $outputdir/linux-system-2.tar ./etc/hostname -O) [ "$hostname1" = baserock-rocks ] [ "$hostname2" = baserock-rocks-even-more ] + +tar -xf $outputdir/linux-system-2.tar ./baserock/deployment.meta +metadata=baserock/deployment.meta + +# Check that 'git describe' of definitions repo was stored correctly +echo -n "definitions-version: " +"$SRCDIR/scripts/yaml-extract" $metadata definitions-version + +echo -n "configuration.HOSTNAME: " +"$SRCDIR/scripts/yaml-extract" $metadata configuration HOSTNAME + +! (grep -q "EXAMPLE_PASSWORD" $metadata) diff --git a/yarns/deployment.yarn b/yarns/deployment.yarn index 855ecc52..f98d2751 100644 --- a/yarns/deployment.yarn +++ b/yarns/deployment.yarn @@ -9,7 +9,7 @@ Morph Deployment Tests THEN morph failed AND the deploy error message includes the string "morph deploy is only supported for cluster morphologies" - SCENARIO deploying a cluster morphology + SCENARIO deploying a cluster morphology as a tarfile GIVEN a workspace AND a git server WHEN the user checks out the system branch called master @@ -17,3 +17,31 @@ Morph Deployment Tests WHEN the user builds the system test-system in branch master AND the user attempts to deploy the cluster test-cluster in branch master with options system.location=test.tar THEN morph succeeded + +Some deployment types support upgrades, but some do not and Morph needs to make +this clear. + + SCENARIO attempting to upgrade a tarfile deployment + GIVEN a workspace + AND a git server + WHEN the user checks out the system branch called master + GIVEN a cluster called test-cluster for deploying only the test-system system as type tar in system branch master + WHEN the user builds the system test-system in branch master + AND the user attempts to upgrade the cluster test-cluster in branch master with options system.location=test.tar + THEN morph failed + +The rawdisk write extension supports both initial deployment and subsequent +upgrades. Note that the rawdisk upgrade code needs bringing up to date to use +the new Baserock OS version manager tool. Also, the test deploys an identical +base OS as an upgrade. While pointless, this is permitted and does exercise +the same code paths as a real upgrade. + + SCENARIO deploying a cluster morphology as rawdisk and then upgrading it + GIVEN a workspace + AND a git server + WHEN the user checks out the system branch called master + GIVEN a cluster called test-cluster for deploying only the test-system system as type rawdisk in system branch master + WHEN the user builds the system test-system in branch master + AND the user attempts to deploy the cluster test-cluster in branch master with options system.location=test.img system.DISK_SIZE=10M system.VERSION_LABEL=test1 + AND the user attempts to upgrade the cluster test-cluster in branch master with options system.location=test.img system.VERSION_LABEL=test2 + THEN morph succeeded diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index ccebabca..1e1b2fd5 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -663,6 +663,16 @@ them, so they can be added to the end of the implements section. if [ $MATCH_1 == "deploys" ]; then run_morph "$@" else attempt_morph "$@"; fi + IMPLEMENTS WHEN the user (attempts to upgrade|upgrades) the (system|cluster) (\S+) in branch (\S+)( with options (.*))? + cd "$DATADIR/workspace/$MATCH_4" + set -- deploy --upgrade "$MATCH_3" + if [ "$MATCH_5" != '' ]; then + # eval used so word splitting in the text is preserved + eval set -- '"$@"' $MATCH_6 + fi + if [ $MATCH_1 == "upgrades" ]; then run_morph "$@" + else attempt_morph "$@"; fi + To successfully deploy systems, we need a cluster morphology. Since the common case is to just have one system, we generate a stub morphology with only the minimal information. diff --git a/yarns/morph.shell-lib b/yarns/morph.shell-lib index 31dcc7af..e025f8f5 100644 --- a/yarns/morph.shell-lib +++ b/yarns/morph.shell-lib @@ -22,17 +22,13 @@ # Run Morph from the source tree, ignoring any configuration files. # This way the test suite is not affected by any configuration the user # or system may have. Instead, we'll use the `$DATADIR/morph.conf` file, -# which tests can create, if they want to. Unfortunately, currently yarn -# does not set a $SRCDIR that points at the source tree, so if the test -# needs to cd away from there, things can break. We work around this -# by allowing the caller to set $SRCDIR if they want to, and if it isn't -# set, we default to . (current working directory). +# which tests can create, if they want to. run_morph() { { set +e - "${SRCDIR:-.}"/morph \ + PYTHONPATH="$SRCDIR" "$SRCDIR"/morph \ --cachedir-min-space=0 --tempdir-min-space=0 \ --no-default-config --config "$DATADIR/morph.conf" "$@" \ 2> "$DATADIR/result-$1" |