From c7e6642d855283b85182097b24226e78aa78e199 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 6 Feb 2013 20:44:47 +0000 Subject: Add morphlib module for common write extension code --- writeexts.py | 157 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100755 writeexts.py diff --git a/writeexts.py b/writeexts.py new file mode 100755 index 00000000..23473021 --- /dev/null +++ b/writeexts.py @@ -0,0 +1,157 @@ +# Copyright (C) 2012-2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import os +import re +import time +import tempfile + + +class WriteExtension(cliapp.Application): + + '''A base class for deployment write extensions. + + A subclass should subclass this class, and add a + ``process_args`` method. + + Note that it is not necessary to subclass this class for write + extensions. This class is here just to collect common code for + write extensions. + + ''' + + def process_args(self, args): + raise NotImplementedError() + + def status(self, **kwargs): + '''Provide status output. + + The ``msg`` keyword argument is the actual message, + the rest are values for fields in the message as interpolated + by %. + + ''' + + self.output.write('%s\n' % (kwargs['msg'] % kwargs)) + + def get_disk_size(self): + '''Parse disk size from environment.''' + + size = os.environ.get('DISK_SIZE', '1G') + m = re.match('^(\d+)([kmgKMG]?)$', size) + if not m: + raise morphlib.Error('Cannot parse disk size %s' % size) + + factors = { + '': 1, + 'k': 1024, + 'm': 1024**2, + 'g': 1024**3, + } + factor = factors[m.group(2).lower()] + + return int(m.group(1)) * factor + + def create_raw_disk_image(self, filename, size): + '''Create a raw disk image.''' + + self.status(msg='Creating empty disk image') + cliapp.runcmd( + ['dd', + 'if=/dev/zero', + 'of=' + filename, + 'bs=1', + 'seek=%d' % size, + 'count=0']) + + def mkfs_btrfs(self, location): + '''Create a btrfs filesystem on the disk.''' + self.status(msg='Creating btrfs filesystem') + cliapp.runcmd(['mkfs.btrfs', '-L', 'baserock', location]) + + def mount(self, location): + '''Mount the filesystem so it can be tweaked. + + Return path to the mount point. + The mount point is a newly created temporary directory. + The caller must call self.unmount to unmount on the return value. + + ''' + + self.status(msg='Mounting filesystem') + tempdir = tempfile.mkdtemp() + # FIXME: This hardcodes the loop device. + cliapp.runcmd(['mount', '-o', 'loop=loop0', location, tempdir]) + return tempdir + + def unmount(self, mount_point): + '''Unmount the filesystem mounted by self.mount. + + Also, remove the temporary directory. + + ''' + + self.status(msg='Unmounting filesystem') + cliapp.runcmd(['umount', mount_point]) + os.rmdir(mount_point) + + def create_factory(self, real_root, temp_root): + '''Create the default "factory" system.''' + + factory = os.path.join(real_root, 'factory') + + self.status(msg='Creating factory subvolume') + cliapp.runcmd(['btrfs', 'subvolume', 'create', factory]) + self.status(msg='Copying files to factory subvolume') + cliapp.runcmd(['cp', '-a', temp_root + '/.', factory + '/.']) + + # The kernel needs to be on the root volume. + self.status(msg='Copying boot directory to root subvolume') + factory_boot = os.path.join(factory, 'boot') + root_boot = os.path.join(real_root, 'boot') + cliapp.runcmd(['cp', '-a', factory_boot, root_boot]) + + def create_fstab(self, real_root): + '''Create an fstab.''' + + self.status(msg='Creating fstab') + fstab = os.path.join(real_root, 'factory', 'etc', 'fstab') + with open(fstab, 'w') as f: + f.write('proc /proc proc defaults 0 0\n') + f.write('sysfs /sys sysfs defaults 0 0\n') + f.write('/dev/sda / btrfs defaults,rw,noatime 0 1\n') + + def install_extlinux(self, real_root): + '''Install extlinux on the newly created disk image.''' + + self.status(msg='Creating extlinux.conf') + config = os.path.join(real_root, 'extlinux.conf') + with open(config, 'w') as f: + f.write('default linux\n') + f.write('timeout 1\n') + f.write('label linux\n') + f.write('kernel /boot/vmlinuz\n') + f.write('append root=/dev/sda rootflags=subvol=factory ' + 'init=/sbin/init rw\n') + + self.status(msg='Installing extlinux') + cliapp.runcmd(['extlinux', '--install', real_root]) + + # FIXME this hack seems to be necessary to let extlinux finish + cliapp.runcmd(['sync']) + time.sleep(2) + -- cgit v1.2.1 From 439623c9ffbd56fb1241265321e9a2dd8e1437f5 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 7 Feb 2013 11:27:52 +0000 Subject: Create hole in-process without executing dd(1) Suggested-By: Richard Maw --- writeexts.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/writeexts.py b/writeexts.py index 23473021..676a9d22 100755 --- a/writeexts.py +++ b/writeexts.py @@ -70,13 +70,10 @@ class WriteExtension(cliapp.Application): '''Create a raw disk image.''' self.status(msg='Creating empty disk image') - cliapp.runcmd( - ['dd', - 'if=/dev/zero', - 'of=' + filename, - 'bs=1', - 'seek=%d' % size, - 'count=0']) + with open(filename, 'wb') as f: + if size > 0: + f.seek(size-1) + f.write('\0') def mkfs_btrfs(self, location): '''Create a btrfs filesystem on the disk.''' -- cgit v1.2.1 From b333b5d15a15b8221f2433467a3765364a2b1e67 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 7 Feb 2013 11:29:57 +0000 Subject: Let mount choose loop device Suggested-By: Richard Maw --- writeexts.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/writeexts.py b/writeexts.py index 676a9d22..a407c059 100755 --- a/writeexts.py +++ b/writeexts.py @@ -91,8 +91,7 @@ class WriteExtension(cliapp.Application): self.status(msg='Mounting filesystem') tempdir = tempfile.mkdtemp() - # FIXME: This hardcodes the loop device. - cliapp.runcmd(['mount', '-o', 'loop=loop0', location, tempdir]) + cliapp.runcmd(['mount', '-o', 'loop', location, tempdir]) return tempdir def unmount(self, mount_point): -- cgit v1.2.1 From 65d7afb64c344a219b831eeb0e4bb0ebcbbf4ab6 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 7 Feb 2013 11:45:19 +0000 Subject: Do away with unnecessary fstab entries for proc, sys Suggested-By: Richard Maw --- writeexts.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/writeexts.py b/writeexts.py index a407c059..fae0d5d9 100755 --- a/writeexts.py +++ b/writeexts.py @@ -127,8 +127,6 @@ class WriteExtension(cliapp.Application): self.status(msg='Creating fstab') fstab = os.path.join(real_root, 'factory', 'etc', 'fstab') with open(fstab, 'w') as f: - f.write('proc /proc proc defaults 0 0\n') - f.write('sysfs /sys sysfs defaults 0 0\n') f.write('/dev/sda / btrfs defaults,rw,noatime 0 1\n') def install_extlinux(self, real_root): -- cgit v1.2.1 From 4aeffa2de48737a481cd20688f8b1b874cf7e46e Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 7 Feb 2013 11:41:04 +0000 Subject: Refactor: Add WriteExtension.create_local_system method This allows code sharing amongst all the places that create a system in a raw disk image. This also adds the creation of a factory-run subvolume, and fixes error messages for errors that happen during a disk image creation. Suggested-By: Richard Maw Suggested-By: Sam Thursfield --- writeexts.py | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index fae0d5d9..60848345 100755 --- a/writeexts.py +++ b/writeexts.py @@ -47,6 +47,31 @@ class WriteExtension(cliapp.Application): ''' self.output.write('%s\n' % (kwargs['msg'] % kwargs)) + + def create_local_system(self, temp_root, raw_disk): + '''Create a raw system image locally.''' + + size = self.get_disk_size() + self.create_raw_disk_image(raw_disk, size) + try: + self.mkfs_btrfs(raw_disk) + mp = self.mount(raw_disk) + except BaseException: + sys.stderr.write('Error creating disk image') + os.remove(raw_disk) + raise + try: + self.create_factory(mp, temp_root) + self.create_fstab(mp) + self.create_factory_run(mp) + self.install_extlinux(mp) + except BaseException, e: + sys.stderr.write('Error creating disk image') + self.unmount(mp) + os.remove(raw_disk) + raise + else: + self.unmount(mp) def get_disk_size(self): '''Parse disk size from environment.''' @@ -120,6 +145,15 @@ class WriteExtension(cliapp.Application): factory_boot = os.path.join(factory, 'boot') root_boot = os.path.join(real_root, 'boot') cliapp.runcmd(['cp', '-a', factory_boot, root_boot]) + + def create_factory_run(self, real_root): + '''Create the 'factory-run' snapshot.''' + + self.status(msg='Creating factory-run subvolume') + factory = os.path.join(real_root, 'factory') + factory_run = factory + '-run' + cliapp.runcmd( + ['btrfs', 'subvolume', 'snapshot', factory, factory_run]) def create_fstab(self, real_root): '''Create an fstab.''' @@ -139,7 +173,7 @@ class WriteExtension(cliapp.Application): f.write('timeout 1\n') f.write('label linux\n') f.write('kernel /boot/vmlinuz\n') - f.write('append root=/dev/sda rootflags=subvol=factory ' + f.write('append root=/dev/sda rootflags=subvol=factory-run ' 'init=/sbin/init rw\n') self.status(msg='Installing extlinux') -- cgit v1.2.1 From bbe187ecbef21c29c237d76b13d5b3beb2379d9e Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Tue, 12 Feb 2013 16:47:05 +0000 Subject: Add missing "import sys" to fix error messages Reported-By: Richard Maw --- writeexts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/writeexts.py b/writeexts.py index 60848345..a479f3cb 100755 --- a/writeexts.py +++ b/writeexts.py @@ -17,6 +17,7 @@ import cliapp import os import re +import sys import time import tempfile -- cgit v1.2.1 From 7b9fb69291729cb00b8073875f17d8b0e66252df Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 13 Mar 2013 13:52:52 +0000 Subject: Add ATTACH_DISKS support to kvm --- writeexts.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/writeexts.py b/writeexts.py index a479f3cb..469b8557 100755 --- a/writeexts.py +++ b/writeexts.py @@ -184,3 +184,11 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['sync']) time.sleep(2) + def parse_attach_disks(self): + '''Parse $ATTACH_DISKS into list of disks to attach.''' + + if 'ATTACH_DISKS' in os.environ: + s = os.environ['ATTACH_DISKS'] + return s.split(':') + else: + return [] -- cgit v1.2.1 From 2fa5a589912f660ddc2a08024eb511479042e29e Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 28 Mar 2013 13:28:19 +0000 Subject: Add method to parse $RAM_SIZE --- writeexts.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/writeexts.py b/writeexts.py index 469b8557..af48b375 100755 --- a/writeexts.py +++ b/writeexts.py @@ -74,13 +74,16 @@ class WriteExtension(cliapp.Application): else: self.unmount(mp) - def get_disk_size(self): - '''Parse disk size from environment.''' + def _parse_size(self, size): + '''Parse a size from a string. + + Return size in bytes. - size = os.environ.get('DISK_SIZE', '1G') + ''' + m = re.match('^(\d+)([kmgKMG]?)$', size) if not m: - raise morphlib.Error('Cannot parse disk size %s' % size) + return None factors = { '': 1, @@ -92,6 +95,23 @@ class WriteExtension(cliapp.Application): return int(m.group(1)) * factor + def _parse_size_from_environment(self, env_var, default): + '''Parse a size from an environment variable.''' + + size = os.environ.get(env_var, default) + bytes = self._parse_size(size) + if bytes is None: + raise morphlib.Error('Cannot parse %s value %s' % (env_var, size)) + return bytes + + def get_disk_size(self): + '''Parse disk size from environment.''' + return self._parse_size_from_environment('DISK_SIZE', '1G') + + def get_ram_size(self): + '''Parse RAM size from environment.''' + return self._parse_size_from_environment('RAM_SIZE', '1G') + def create_raw_disk_image(self, filename, size): '''Create a raw disk image.''' -- cgit v1.2.1 From 4cd3a4060146a37cdef216ee4dc346c0f7229f8c Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 2 May 2013 16:36:57 +0100 Subject: Add entry for / in fstab but only if there isn't one --- writeexts.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/writeexts.py b/writeexts.py index af48b375..2cdba86f 100755 --- a/writeexts.py +++ b/writeexts.py @@ -179,10 +179,26 @@ class WriteExtension(cliapp.Application): def create_fstab(self, real_root): '''Create an fstab.''' - self.status(msg='Creating fstab') + self.status(msg='Creating fstab') fstab = os.path.join(real_root, 'factory', 'etc', 'fstab') + + if os.path.exists(fstab): + with open(fstab, 'r') as f: + contents = f.read() + else: + contents = '' + + got_root = False + for line in contents.splitlines(): + words = line.split() + if len(words) >= 2 and not words[0].startswith('#'): + got_root = got_root or words[1] == '/' + + if not got_root: + contents += '\n/dev/sda / btrfs defaults,rw,noatime 0 1\n' + with open(fstab, 'w') as f: - f.write('/dev/sda / btrfs defaults,rw,noatime 0 1\n') + f.write(contents) def install_extlinux(self, real_root): '''Install extlinux on the newly created disk image.''' -- cgit v1.2.1 From 81cbbef81e44cc4bd1f3440c40a524667be698c3 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 8 May 2013 11:57:10 +0100 Subject: Make bootloader installation for disk images optional --- writeexts.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index af48b375..23b876ae 100755 --- a/writeexts.py +++ b/writeexts.py @@ -65,7 +65,8 @@ class WriteExtension(cliapp.Application): self.create_factory(mp, temp_root) self.create_fstab(mp) self.create_factory_run(mp) - self.install_extlinux(mp) + if self.bootloader_is_wanted(): + self.install_extlinux(mp) except BaseException, e: sys.stderr.write('Error creating disk image') self.unmount(mp) @@ -212,3 +213,21 @@ class WriteExtension(cliapp.Application): return s.split(':') else: return [] + + def bootloader_is_wanted(self): + '''Does the user request a bootloader? + + The user may set $BOOTLOADER to yes, no, or auto. If not + set, auto is the default and means that the bootloader will + be installed on x86-32 and x86-64, but not otherwise. + + ''' + + value = os.environ.get('BOOTLOADER', 'auto') + if value == 'auto': + if os.uname()[-1] in ['x86_32', 'x86_64']: + value = 'yes' + else: + value = 'no' + + return value == 'yes' -- cgit v1.2.1 From fbfcdcedf7b7e69f3d340b92785b3b4355e3f543 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 8 May 2013 13:27:21 +0100 Subject: Fix recognition of x86-32 Reported-by: Richard Maw --- writeexts.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index 23b876ae..6c28f7fb 100755 --- a/writeexts.py +++ b/writeexts.py @@ -223,9 +223,13 @@ class WriteExtension(cliapp.Application): ''' + def is_x86(arch): + return (arch == 'x86_64' or + (arch.startswith('i') and arch.endswith('86'))) + value = os.environ.get('BOOTLOADER', 'auto') if value == 'auto': - if os.uname()[-1] in ['x86_32', 'x86_64']: + if ix_x86(os.uname()[-1]): value = 'yes' else: value = 'no' -- cgit v1.2.1 From 2287b2a81ca8a19e41cc9edf3cb8c2c361bfa4e4 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 8 May 2013 17:04:45 +0100 Subject: Add AUTOSTART to kvm and libvirt write extensions If AUTOSTART is 'yes' then the VM will be started once it is created. If it is 'no' or undefined, then it will need to be manually started. If it is any other value, then an exception is raised. --- writeexts.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/writeexts.py b/writeexts.py index 2cdba86f..9caab839 100755 --- a/writeexts.py +++ b/writeexts.py @@ -228,3 +228,15 @@ class WriteExtension(cliapp.Application): return s.split(':') else: return [] + + def parse_autostart(self): + '''Parse $AUTOSTART to determine if VMs should be started.''' + + autostart = os.environ.get('AUTOSTART', 'no') + if autostart == 'no': + return False + elif autostart == 'yes': + return True + else: + raise cliapp.AppException('Unexpected value for AUTOSTART: %s' % + autostart) -- cgit v1.2.1 From 0dd6bf15bfb6fba31c2ce238ce352d2185ac1ba8 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 9 May 2013 13:46:10 +0000 Subject: Fix typo --- writeexts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index ed8f7ba6..48847d56 100755 --- a/writeexts.py +++ b/writeexts.py @@ -245,7 +245,7 @@ class WriteExtension(cliapp.Application): value = os.environ.get('BOOTLOADER', 'auto') if value == 'auto': - if ix_x86(os.uname()[-1]): + if is_x86(os.uname()[-1]): value = 'yes' else: value = 'no' -- cgit v1.2.1 From 302f6ca1eb5e90704b645b186d97a22d44d9f938 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Fri, 10 May 2013 14:31:39 +0000 Subject: Use a different disk system layout for rawdisk This is in preparation for making deployments able to upgrade existing baserock systems. --- writeexts.py | 61 +++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/writeexts.py b/writeexts.py index 48847d56..cb47f63d 100755 --- a/writeexts.py +++ b/writeexts.py @@ -62,10 +62,14 @@ class WriteExtension(cliapp.Application): os.remove(raw_disk) raise try: - self.create_factory(mp, temp_root) - self.create_fstab(mp) - self.create_factory_run(mp) + version_label = 'version1' + version_root = os.path.join(mp, 'systems', version_label) + os.makedirs(version_root) + self.create_orig(version_root, temp_root) + self.create_fstab(version_root) + self.create_run(version_root) if self.bootloader_is_wanted(): + self.install_kernel(version_root, temp_root) self.install_extlinux(mp) except BaseException, e: sys.stderr.write('Error creating disk image') @@ -152,36 +156,30 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['umount', mount_point]) os.rmdir(mount_point) - def create_factory(self, real_root, temp_root): + def create_orig(self, version_root, temp_root): '''Create the default "factory" system.''' - factory = os.path.join(real_root, 'factory') + orig = os.path.join(version_root, 'orig') - self.status(msg='Creating factory subvolume') - cliapp.runcmd(['btrfs', 'subvolume', 'create', factory]) - self.status(msg='Copying files to factory subvolume') - cliapp.runcmd(['cp', '-a', temp_root + '/.', factory + '/.']) + self.status(msg='Creating orig subvolume') + cliapp.runcmd(['btrfs', 'subvolume', 'create', orig]) + self.status(msg='Copying files to orig subvolume') + cliapp.runcmd(['cp', '-a', temp_root + '/.', orig + '/.']) - # The kernel needs to be on the root volume. - self.status(msg='Copying boot directory to root subvolume') - factory_boot = os.path.join(factory, 'boot') - root_boot = os.path.join(real_root, 'boot') - cliapp.runcmd(['cp', '-a', factory_boot, root_boot]) - - def create_factory_run(self, real_root): - '''Create the 'factory-run' snapshot.''' + def create_run(self, version_root): + '''Create the 'run' snapshot.''' - self.status(msg='Creating factory-run subvolume') - factory = os.path.join(real_root, 'factory') - factory_run = factory + '-run' + self.status(msg='Creating run subvolume') + orig = os.path.join(version_root, 'orig') + run = os.path.join(version_root, 'run') cliapp.runcmd( - ['btrfs', 'subvolume', 'snapshot', factory, factory_run]) + ['btrfs', 'subvolume', 'snapshot', orig, run]) - def create_fstab(self, real_root): + def create_fstab(self, version_root): '''Create an fstab.''' self.status(msg='Creating fstab') - fstab = os.path.join(real_root, 'factory', 'etc', 'fstab') + fstab = os.path.join(version_root, 'orig', 'etc', 'fstab') if os.path.exists(fstab): with open(fstab, 'r') as f: @@ -201,6 +199,18 @@ class WriteExtension(cliapp.Application): with open(fstab, 'w') as f: f.write(contents) + def install_kernel(self, version_root, temp_root): + '''Install the kernel outside of 'orig' or 'run' subvolumes''' + + self.status(msg='Installing kernel') + image_names = ['vmlinuz', 'zImage', 'uImage'] + kernel_dest = os.path.join(version_root, 'linux') + for name in image_names: + try_path = os.path.join(temp_root, 'boot', name) + if os.path.exists(try_path): + cliapp.runcmd(['cp', '-a', try_path, kernel_dest]) + break + def install_extlinux(self, real_root): '''Install extlinux on the newly created disk image.''' @@ -210,8 +220,9 @@ class WriteExtension(cliapp.Application): f.write('default linux\n') f.write('timeout 1\n') f.write('label linux\n') - f.write('kernel /boot/vmlinuz\n') - f.write('append root=/dev/sda rootflags=subvol=factory-run ' + f.write('kernel /systems/version1/linux\n') + f.write('append root=/dev/sda ' + 'rootflags=subvol=systems/version1/run ' 'init=/sbin/init rw\n') self.status(msg='Installing extlinux') -- cgit v1.2.1 From 4025d614bbb9faac17a53672c4bd19e69c74cb4b Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Fri, 10 May 2013 16:44:51 +0000 Subject: Add 'state' dirs as btrfs subvolumes These subvolumes exist in state/{home,opt,srv} of the disk's root. They are not mounted by default. --- writeexts.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/writeexts.py b/writeexts.py index cb47f63d..49cd89dd 100755 --- a/writeexts.py +++ b/writeexts.py @@ -65,6 +65,7 @@ class WriteExtension(cliapp.Application): version_label = 'version1' version_root = os.path.join(mp, 'systems', version_label) os.makedirs(version_root) + self.create_state(mp) self.create_orig(version_root, temp_root) self.create_fstab(version_root) self.create_run(version_root) @@ -117,6 +118,16 @@ class WriteExtension(cliapp.Application): '''Parse RAM size from environment.''' return self._parse_size_from_environment('RAM_SIZE', '1G') + def create_state(self, real_root): + '''Create the state subvolumes that are shared between versions''' + + self.status(msg='Creating state subvolumes') + os.mkdir(os.path.join(real_root, 'state')) + statedirs = ['home', 'opt', 'srv'] + for statedir in statedirs: + dirpath = os.path.join(real_root, 'state', statedir) + cliapp.runcmd(['btrfs', 'subvolume', 'create', dirpath]) + def create_raw_disk_image(self, filename, size): '''Create a raw disk image.''' -- cgit v1.2.1 From fb623be664e18d06359b21a9b420a24bd22e23f8 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Mon, 13 May 2013 15:53:51 +0000 Subject: Use a kernel named 'kernel' instead of 'linux' --- writeexts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/writeexts.py b/writeexts.py index c74a4b76..f6465886 100755 --- a/writeexts.py +++ b/writeexts.py @@ -215,7 +215,7 @@ class WriteExtension(cliapp.Application): self.status(msg='Installing kernel') image_names = ['vmlinuz', 'zImage', 'uImage'] - kernel_dest = os.path.join(version_root, 'linux') + kernel_dest = os.path.join(version_root, 'kernel') for name in image_names: try_path = os.path.join(temp_root, 'boot', name) if os.path.exists(try_path): @@ -231,7 +231,7 @@ class WriteExtension(cliapp.Application): f.write('default linux\n') f.write('timeout 1\n') f.write('label linux\n') - f.write('kernel /systems/version1/linux\n') + f.write('kernel /systems/version1/kernel\n') f.write('append root=/dev/sda ' 'rootflags=subvol=systems/version1/run ' 'init=/sbin/init rw\n') -- cgit v1.2.1 From 764983c38c8dbd331ecdfbe4d8fa1e60a7aa30be Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Sun, 19 May 2013 18:59:11 +0000 Subject: Allow installing extlinux to other system versions. --- writeexts.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/writeexts.py b/writeexts.py index f6465886..10b17e76 100755 --- a/writeexts.py +++ b/writeexts.py @@ -71,7 +71,7 @@ class WriteExtension(cliapp.Application): self.create_run(version_root) if self.bootloader_is_wanted(): self.install_kernel(version_root, temp_root) - self.install_extlinux(mp) + self.install_extlinux(mp, version_label) except BaseException, e: sys.stderr.write('Error creating disk image') self.unmount(mp) @@ -222,7 +222,7 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['cp', '-a', try_path, kernel_dest]) break - def install_extlinux(self, real_root): + def install_extlinux(self, real_root, version_label): '''Install extlinux on the newly created disk image.''' self.status(msg='Creating extlinux.conf') @@ -231,9 +231,9 @@ class WriteExtension(cliapp.Application): f.write('default linux\n') f.write('timeout 1\n') f.write('label linux\n') - f.write('kernel /systems/version1/kernel\n') + f.write('kernel /systems/' + version_label + '/kernel\n') f.write('append root=/dev/sda ' - 'rootflags=subvol=systems/version1/run ' + 'rootflags=subvol=systems/' + version_label + '/run ' 'init=/sbin/init rw\n') self.status(msg='Installing extlinux') -- cgit v1.2.1 From a1b219eb53f132d716e41238dd8239c885192bcb Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Thu, 23 May 2013 15:20:52 +0000 Subject: Use the name factory for the first system version. --- writeexts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index 10b17e76..9e98747c 100755 --- a/writeexts.py +++ b/writeexts.py @@ -62,7 +62,7 @@ class WriteExtension(cliapp.Application): os.remove(raw_disk) raise try: - version_label = 'version1' + version_label = 'factory' version_root = os.path.join(mp, 'systems', version_label) os.makedirs(version_root) self.create_state(mp) -- cgit v1.2.1 From 4f405d42db8ed828b95df98a3f69e5a66537e8e9 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Tue, 28 May 2013 16:09:49 +0000 Subject: Remove executable permissions from *.py files that have them --- writeexts.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 writeexts.py diff --git a/writeexts.py b/writeexts.py old mode 100755 new mode 100644 -- cgit v1.2.1 From 861ee2270f257d3d5e2a840315b2b5a6f664ae20 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Wed, 29 May 2013 11:04:27 +0000 Subject: Make create_local_system fail if DISK_SIZE isn't defined --- writeexts.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index 9e98747c..df4cec33 100644 --- a/writeexts.py +++ b/writeexts.py @@ -53,6 +53,8 @@ class WriteExtension(cliapp.Application): '''Create a raw system image locally.''' size = self.get_disk_size() + if not size: + raise cliapp.AppException('DISK_SIZE is not defined') self.create_raw_disk_image(raw_disk, size) try: self.mkfs_btrfs(raw_disk) @@ -105,6 +107,8 @@ class WriteExtension(cliapp.Application): '''Parse a size from an environment variable.''' size = os.environ.get(env_var, default) + if size is None: + return None bytes = self._parse_size(size) if bytes is None: raise morphlib.Error('Cannot parse %s value %s' % (env_var, size)) @@ -112,7 +116,7 @@ class WriteExtension(cliapp.Application): def get_disk_size(self): '''Parse disk size from environment.''' - return self._parse_size_from_environment('DISK_SIZE', '1G') + return self._parse_size_from_environment('DISK_SIZE', None) def get_ram_size(self): '''Parse RAM size from environment.''' -- cgit v1.2.1 From 71bc1d10f601dab9023c73e207bc787923b4f55c Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Thu, 13 Jun 2013 10:40:09 +0000 Subject: Set up a symlink to the default system version in rawdisk/kvm/vbox deployments Also Change them to use the "default" symlink in the extlinux.conf they create, instead of hardcoding the current system version name --- writeexts.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/writeexts.py b/writeexts.py index df4cec33..de4189f8 100644 --- a/writeexts.py +++ b/writeexts.py @@ -71,9 +71,10 @@ class WriteExtension(cliapp.Application): self.create_orig(version_root, temp_root) self.create_fstab(version_root) self.create_run(version_root) + os.symlink(version_label, os.path.join(mp, 'systems', 'default')) if self.bootloader_is_wanted(): self.install_kernel(version_root, temp_root) - self.install_extlinux(mp, version_label) + self.install_extlinux(mp) except BaseException, e: sys.stderr.write('Error creating disk image') self.unmount(mp) @@ -226,7 +227,7 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['cp', '-a', try_path, kernel_dest]) break - def install_extlinux(self, real_root, version_label): + def install_extlinux(self, real_root): '''Install extlinux on the newly created disk image.''' self.status(msg='Creating extlinux.conf') @@ -235,9 +236,9 @@ class WriteExtension(cliapp.Application): f.write('default linux\n') f.write('timeout 1\n') f.write('label linux\n') - f.write('kernel /systems/' + version_label + '/kernel\n') + f.write('kernel /systems/default/kernel\n') f.write('append root=/dev/sda ' - 'rootflags=subvol=systems/' + version_label + '/run ' + 'rootflags=subvol=systems/default/run ' 'init=/sbin/init rw\n') self.status(msg='Installing extlinux') -- cgit v1.2.1 From 2a52a1b530e2a430a34a28255ec9a2dff4f34534 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Mon, 10 Jun 2013 22:32:09 +0000 Subject: Write extensions: Flush output when using status() This is required to get real-time output and the timestamps meaning anything useful. --- writeexts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/writeexts.py b/writeexts.py index de4189f8..410b8c9f 100644 --- a/writeexts.py +++ b/writeexts.py @@ -48,6 +48,7 @@ class WriteExtension(cliapp.Application): ''' self.output.write('%s\n' % (kwargs['msg'] % kwargs)) + self.output.flush() def create_local_system(self, temp_root, raw_disk): '''Create a raw system image locally.''' -- cgit v1.2.1 From 21fa4e7da4c566ecaba3cc70dedb74c982674832 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Tue, 2 Jul 2013 07:24:30 +0000 Subject: Allow to set the number of cpus for virtualbox and kvm deployments. --- writeexts.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/writeexts.py b/writeexts.py index 410b8c9f..2a54d757 100644 --- a/writeexts.py +++ b/writeexts.py @@ -124,6 +124,10 @@ class WriteExtension(cliapp.Application): '''Parse RAM size from environment.''' return self._parse_size_from_environment('RAM_SIZE', '1G') + def get_vcpu_count(self): + '''Parse the virtual cpu count from environment.''' + return self._parse_size_from_environment('VCPUS', '1') + def create_state(self, real_root): '''Create the state subvolumes that are shared between versions''' -- cgit v1.2.1 From cc0ceb1a96e685574067fffeb8e9f1f7f23ec8d4 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Wed, 17 Jul 2013 12:48:43 +0000 Subject: Import morphlib as we are using morphlib.Error --- writeexts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/writeexts.py b/writeexts.py index 2a54d757..9dbc77e6 100644 --- a/writeexts.py +++ b/writeexts.py @@ -21,6 +21,7 @@ import sys import time import tempfile +import morphlib class WriteExtension(cliapp.Application): -- cgit v1.2.1 From 307466b3ee56be7978d202219a6b4666176825d2 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Wed, 12 Feb 2014 11:40:07 +0000 Subject: Adding syslinux 'menu.c32' file during the deployment. We will need this file to enable a bootloader menu to choose between OS after an upgrade. --- writeexts.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index 9dbc77e6..dd0e8b79 100644 --- a/writeexts.py +++ b/writeexts.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -76,6 +76,7 @@ class WriteExtension(cliapp.Application): os.symlink(version_label, os.path.join(mp, 'systems', 'default')) if self.bootloader_is_wanted(): self.install_kernel(version_root, temp_root) + self.install_syslinux_menu(mp, version_root) self.install_extlinux(mp) except BaseException, e: sys.stderr.write('Error creating disk image') @@ -254,6 +255,23 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['sync']) time.sleep(2) + def install_syslinux_menu(self, real_root, version_root): + '''Make syslinux/extlinux menu binary available. + + The syslinux boot menu is compiled to a file named menu.c32. Extlinux + searches a few places for this file but it does not know to look inside + our subvolume, so we copy it to the filesystem root. + + If the file is not available, the bootloader will still work but will + not be able to show a menu. + + ''' + menu_file = os.path.join(version_root, 'orig', + 'usr', 'share', 'syslinux', 'menu.c32') + if os.path.isfile(menu_file): + self.status(msg='Copying menu.c32') + shutil.copy(menu_file, real_root) + def parse_attach_disks(self): '''Parse $ATTACH_DISKS into list of disks to attach.''' -- cgit v1.2.1 From 206e9f65972b8d6abb8a0f7efb460fdfe8b722c0 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Wed, 12 Feb 2014 19:03:05 +0000 Subject: deploy: Finish off the Btrfs system layout implementation The shared state directories defined in writeexts.py (/var, /home etc.) are now separate Btrfs subvolumes that are mounted in place using fstab. There are some warnings on mounting /var and /srv about the mountpoint not being empty. Not yet investigated. If a configure extension has already added / to the fstab, use the device it chose rather than assuming /dev/sda. This is required for the vdaboot.configure extension that we use for OpenStack deployments. Similarly, if a configure extension has added an entry for a state directory in /etc/fstab already, we don't replace it with a /state/xxx directory. That's only done as a default behaviour. --- writeexts.py | 183 +++++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 141 insertions(+), 42 deletions(-) diff --git a/writeexts.py b/writeexts.py index dd0e8b79..a07c697f 100644 --- a/writeexts.py +++ b/writeexts.py @@ -17,12 +17,65 @@ import cliapp import os import re +import shutil import sys import time import tempfile import morphlib + +class Fstab(object): + '''Small helper class for parsing and adding lines to /etc/fstab.''' + + # There is an existing Python helper library for editing of /etc/fstab. + # However it is unmaintained and has an incompatible license (GPL3). + # + # https://code.launchpad.net/~computer-janitor-hackers/python-fstab/trunk + + def __init__(self, filepath='/etc/fstab'): + if os.path.exists(filepath): + with open(filepath, 'r') as f: + self.text= f.read() + else: + self.text = '' + self.filepath = filepath + self.lines_added = 0 + + def get_mounts(self): + '''Return list of mount devices and targets in /etc/fstab. + + Return value is a dict of target -> device. + ''' + mounts = dict() + for line in self.text.splitlines(): + words = line.split() + if len(words) >= 2 and not words[0].startswith('#'): + device, target = words[0:2] + mounts[target] = device + return mounts + + def add_line(self, line): + '''Add a new entry to /etc/fstab. + + Lines are appended, and separated from any entries made by configure + extensions with a comment. + + ''' + if self.lines_added == 0: + if len(self.text) == 0 or self.text[-1] is not '\n': + self.text += '\n' + self.text += '# Morph default system layout\n' + self.lines_added += 1 + + self.text += line + '\n' + + def write(self): + '''Rewrite the fstab file to include all new entries.''' + with morphlib.savefile.SaveFile(self.filepath, 'w') as f: + f.write(self.text) + + class WriteExtension(cliapp.Application): '''A base class for deployment write extensions. @@ -53,7 +106,6 @@ class WriteExtension(cliapp.Application): def create_local_system(self, temp_root, raw_disk): '''Create a raw system image locally.''' - size = self.get_disk_size() if not size: raise cliapp.AppException('DISK_SIZE is not defined') @@ -66,20 +118,10 @@ class WriteExtension(cliapp.Application): os.remove(raw_disk) raise try: - version_label = 'factory' - version_root = os.path.join(mp, 'systems', version_label) - os.makedirs(version_root) - self.create_state(mp) - self.create_orig(version_root, temp_root) - self.create_fstab(version_root) - self.create_run(version_root) - os.symlink(version_label, os.path.join(mp, 'systems', 'default')) - if self.bootloader_is_wanted(): - self.install_kernel(version_root, temp_root) - self.install_syslinux_menu(mp, version_root) - self.install_extlinux(mp) + self.create_btrfs_system_layout( + temp_root, mp, version_label='factory') except BaseException, e: - sys.stderr.write('Error creating disk image') + sys.stderr.write('Error creating Btrfs system layout') self.unmount(mp) os.remove(raw_disk) raise @@ -130,16 +172,6 @@ class WriteExtension(cliapp.Application): '''Parse the virtual cpu count from environment.''' return self._parse_size_from_environment('VCPUS', '1') - def create_state(self, real_root): - '''Create the state subvolumes that are shared between versions''' - - self.status(msg='Creating state subvolumes') - os.mkdir(os.path.join(real_root, 'state')) - statedirs = ['home', 'opt', 'srv'] - for statedir in statedirs: - dirpath = os.path.join(real_root, 'state', statedir) - cliapp.runcmd(['btrfs', 'subvolume', 'create', dirpath]) - def create_raw_disk_image(self, filename, size): '''Create a raw disk image.''' @@ -179,6 +211,34 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['umount', mount_point]) os.rmdir(mount_point) + def create_btrfs_system_layout(self, temp_root, mountpoint, version_label): + '''Separate base OS versions from state using subvolumes. + + ''' + version_root = os.path.join(mountpoint, 'systems', version_label) + state_root = os.path.join(mountpoint, 'state') + + os.makedirs(version_root) + os.makedirs(state_root) + + self.create_orig(version_root, temp_root) + system_dir = os.path.join(version_root, 'orig') + + state_dirs = self.complete_fstab_for_btrfs_layout(system_dir) + + for state_dir in state_dirs: + self.create_state_subvolume(system_dir, mountpoint, state_dir) + + self.create_run(version_root) + + os.symlink( + version_label, os.path.join(mountpoint, 'systems', 'default')) + + if self.bootloader_is_wanted(): + self.install_kernel(version_root, temp_root) + self.install_syslinux_menu(mountpoint, version_root) + self.install_extlinux(mountpoint) + def create_orig(self, version_root, temp_root): '''Create the default "factory" system.''' @@ -198,29 +258,68 @@ class WriteExtension(cliapp.Application): cliapp.runcmd( ['btrfs', 'subvolume', 'snapshot', orig, run]) - def create_fstab(self, version_root): - '''Create an fstab.''' + def create_state_subvolume(self, system_dir, mountpoint, state_subdir): + '''Create a shared state subvolume. - self.status(msg='Creating fstab') - fstab = os.path.join(version_root, 'orig', 'etc', 'fstab') + We need to move any files added to the temporary rootfs by the + configure extensions to their correct home. For example, they might + have added keys in `/root/.ssh` which we now need to transfer to + `/state/root/.ssh`. - if os.path.exists(fstab): - with open(fstab, 'r') as f: - contents = f.read() - else: - contents = '' + ''' + self.status(msg='Creating %s subvolume' % state_subdir) + subvolume = os.path.join(mountpoint, 'state', state_subdir) + cliapp.runcmd(['btrfs', 'subvolume', 'create', subvolume]) + os.chmod(subvolume, 0755) + + existing_state_dir = os.path.join(system_dir, state_subdir) + files = [] + if os.path.exists(existing_state_dir): + files = os.listdir(existing_state_dir) + if len(files) > 0: + self.status(msg='Moving existing data to %s subvolume' % subvolume) + for filename in files: + filepath = os.path.join(existing_state_dir, filename) + shutil.move(filepath, subvolume) + + def complete_fstab_for_btrfs_layout(self, system_dir): + '''Fill in /etc/fstab entries for the default Btrfs disk layout. + + In the future we should move this code out of the write extension and + in to a configure extension. To do that, though, we need some way of + informing the configure extension what layout should be used. Right now + a configure extension doesn't know if the system is going to end up as + a Btrfs disk image, a tarfile or something else and so it can't come + up with a sensible default fstab. + + Configuration extensions can already create any /etc/fstab that they + like. This function only fills in entries that are missing, so if for + example the user configured /home to be on a separate partition, that + decision will be honoured and /state/home will not be created. - got_root = False - for line in contents.splitlines(): - words = line.split() - if len(words) >= 2 and not words[0].startswith('#'): - got_root = got_root or words[1] == '/' + ''' + shared_state_dirs = {'home', 'root', 'opt', 'srv', 'var'} - if not got_root: - contents += '\n/dev/sda / btrfs defaults,rw,noatime 0 1\n' + fstab = Fstab(os.path.join(system_dir, 'etc', 'fstab')) + existing_mounts = fstab.get_mounts() - with open(fstab, 'w') as f: - f.write(contents) + if '/' in existing_mounts: + root_device = existing_mounts['/'] + else: + root_device = '/dev/sda' + fstab.add_line('/dev/sda / btrfs defaults,rw,noatime 0 1') + + state_dirs_to_create = set() + for state_dir in shared_state_dirs: + if '/' + state_dir not in existing_mounts: + state_dirs_to_create.add(state_dir) + state_subvol = os.path.join('/state', state_dir) + fstab.add_line( + '%s /%s btrfs subvol=%s,defaults,rw,noatime 0 2' % + (root_device, state_dir, state_subvol)) + + fstab.write() + return state_dirs_to_create def install_kernel(self, version_root, temp_root): '''Install the kernel outside of 'orig' or 'run' subvolumes''' -- cgit v1.2.1 From ccaed5352ce9413cee3d7ab4b60514d0bc4fdbaa Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Mon, 17 Feb 2014 15:36:56 +0000 Subject: Make parse_autostart() into more general get_environment_boolean() Also, be more flexible when parsing environment booleans -- convert to lower case and match 0/1 and true/false as well as yes/no. --- writeexts.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/writeexts.py b/writeexts.py index a07c697f..1849f406 100644 --- a/writeexts.py +++ b/writeexts.py @@ -402,14 +402,14 @@ class WriteExtension(cliapp.Application): return value == 'yes' - def parse_autostart(self): - '''Parse $AUTOSTART to determine if VMs should be started.''' + def get_environment_boolean(self, variable): + '''Parse a yes/no boolean passed through the environment.''' - autostart = os.environ.get('AUTOSTART', 'no') - if autostart == 'no': + value = os.environ.get(variable, 'no').lower() + if value in ['no', '0', 'false']: return False - elif autostart == 'yes': + elif value in ['yes', '1', 'true']: return True else: - raise cliapp.AppException('Unexpected value for AUTOSTART: %s' % - autostart) + raise cliapp.AppException('Unexpected value for %s: %s' % + (variable, value)) -- cgit v1.2.1 From c1197bd420c9a9b97b7263def4215d93cfc2ddbd Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Fri, 4 Apr 2014 11:40:38 +0000 Subject: Adding support to add extra kernel args in extlinux.conf --- writeexts.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index 1849f406..bff21e8d 100644 --- a/writeexts.py +++ b/writeexts.py @@ -333,11 +333,15 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['cp', '-a', try_path, kernel_dest]) break + def get_extra_kernel_args(self): + return os.environ.get('KERNEL_ARGS', '') + def install_extlinux(self, real_root): '''Install extlinux on the newly created disk image.''' self.status(msg='Creating extlinux.conf') config = os.path.join(real_root, 'extlinux.conf') + kernel_args = self.get_extra_kernel_args() with open(config, 'w') as f: f.write('default linux\n') f.write('timeout 1\n') @@ -345,7 +349,7 @@ class WriteExtension(cliapp.Application): f.write('kernel /systems/default/kernel\n') f.write('append root=/dev/sda ' 'rootflags=subvol=systems/default/run ' - 'init=/sbin/init rw\n') + '%s init=/sbin/init rw\n' % (kernel_args)) self.status(msg='Installing extlinux') cliapp.runcmd(['extlinux', '--install', real_root]) -- cgit v1.2.1 From f7b4039973398400e355c5ea2a068dac4bbc994b Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Mon, 14 Apr 2014 12:35:26 +0000 Subject: deploy: Share SSH connectivity check in the common writeexts.py code Also, change it to log the real error message in morph.log before raising a more general exception to the user. --- writeexts.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/writeexts.py b/writeexts.py index bff21e8d..3f9c33d5 100644 --- a/writeexts.py +++ b/writeexts.py @@ -15,6 +15,7 @@ import cliapp +import logging import os import re import shutil @@ -417,3 +418,11 @@ class WriteExtension(cliapp.Application): else: raise cliapp.AppException('Unexpected value for %s: %s' % (variable, value)) + + def check_ssh_connectivity(self, ssh_host): + try: + cliapp.ssh_runcmd(ssh_host, ['true']) + except cliapp.AppException as e: + logging.error("Error checking SSH connectivity: %s", str(e)) + raise cliapp.AppException( + 'Unable to SSH to %s: %s' % (ssh_host, e)) -- cgit v1.2.1 From 20d6402959869913cb3cd9b78348325e7098bf53 Mon Sep 17 00:00:00 2001 From: Daniel Silverstone Date: Fri, 16 May 2014 09:50:24 +0000 Subject: Fix state subvolume generator to preserve permissions shutil.move() does not preserve permissions, file modes, ownerships etc, resulting in much confusion when prepopulating a non-root user during deployment. This change to `mv` fixes that. --- writeexts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index 3f9c33d5..b4912db1 100644 --- a/writeexts.py +++ b/writeexts.py @@ -281,7 +281,7 @@ class WriteExtension(cliapp.Application): self.status(msg='Moving existing data to %s subvolume' % subvolume) for filename in files: filepath = os.path.join(existing_state_dir, filename) - shutil.move(filepath, subvolume) + cliapp.runcmd(['mv', filepath, subvolume]) def complete_fstab_for_btrfs_layout(self, system_dir): '''Fill in /etc/fstab entries for the default Btrfs disk layout. -- cgit v1.2.1 From 0ad75f58510350b460b5456b137eed1cae006bd9 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Mon, 2 Jun 2014 11:09:29 +0000 Subject: Add initramfs support to write extensions that produce disks If INITRAMFS_PATH is specified and the file exists, then the produced kernel command line will use root=UUID=$uuid_of_created_disk rather than root=/dev/sda, which may be incorrect. Help files have been updated to mention the new option. This leads to an unfortunate duplication of the path to the initramfs, in both the location field of the nested deployment and the INITRAMFS_PATH of the disk image creation. However, an initramfs could be produced by a chunk and put in the same place, so it doesn't make sense to couple the rawdisk and initramfs write extensions to remove this duplication. Similarly, there may be multiple valid initramfs in the rootfs e.g. extlinux loads a hypervisor, which is Linux + initramfs, and the initramfs then boots a guest Linux system, which uses a different initramfs. This makes it important to explicitly let the rootfs write extensions know which to use, or not as the case may be. util-linux's blkid is required, since the busybox version ignores the options to filter its output, and parsing the output is undesirable. Because syslinux's btrfs subvolume support is limited to being able to use a non-0 default subvolume, the initramfs has to be copied out of the run-time rootfs subvolume and into the boot subvolume. This pushed the required disk space of a minimal system over the 512M threshold because we do not have the userland tooling support to be able to do a btrfs file contents clone. --- writeexts.py | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 8 deletions(-) diff --git a/writeexts.py b/writeexts.py index b4912db1..334dc15c 100644 --- a/writeexts.py +++ b/writeexts.py @@ -120,7 +120,8 @@ class WriteExtension(cliapp.Application): raise try: self.create_btrfs_system_layout( - temp_root, mp, version_label='factory') + temp_root, mp, version_label='factory', + disk_uuid=self.get_uuid(raw_disk)) except BaseException, e: sys.stderr.write('Error creating Btrfs system layout') self.unmount(mp) @@ -186,6 +187,13 @@ class WriteExtension(cliapp.Application): '''Create a btrfs filesystem on the disk.''' self.status(msg='Creating btrfs filesystem') cliapp.runcmd(['mkfs.btrfs', '-L', 'baserock', location]) + + def get_uuid(self, location): + '''Get the UUID of a block device's file system.''' + # Requires util-linux blkid; busybox one ignores options and + # lies by exiting successfully. + return cliapp.runcmd(['blkid', '-s', 'UUID', '-o', 'value', + location]).strip() def mount(self, location): '''Mount the filesystem so it can be tweaked. @@ -212,10 +220,12 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['umount', mount_point]) os.rmdir(mount_point) - def create_btrfs_system_layout(self, temp_root, mountpoint, version_label): + def create_btrfs_system_layout(self, temp_root, mountpoint, version_label, + disk_uuid=None): '''Separate base OS versions from state using subvolumes. ''' + initramfs = self.find_initramfs(temp_root) version_root = os.path.join(mountpoint, 'systems', version_label) state_root = os.path.join(mountpoint, 'state') @@ -238,7 +248,12 @@ class WriteExtension(cliapp.Application): if self.bootloader_is_wanted(): self.install_kernel(version_root, temp_root) self.install_syslinux_menu(mountpoint, version_root) - self.install_extlinux(mountpoint) + if initramfs is not None: + self.install_initramfs(initramfs, version_root) + self.install_extlinux(mountpoint, disk_uuid) + else: + self.install_extlinux(mountpoint) + def create_orig(self, version_root, temp_root): '''Create the default "factory" system.''' @@ -322,6 +337,29 @@ class WriteExtension(cliapp.Application): fstab.write() return state_dirs_to_create + def find_initramfs(self, temp_root): + '''Check whether the rootfs has an initramfs. + + Uses the INITRAMFS_PATH option to locate it. + ''' + if 'INITRAMFS_PATH' in os.environ: + initramfs = os.path.join(temp_root, os.environ['INITRAMFS_PATH']) + if not os.path.exists(initramfs): + raise morphlib.Error('INITRAMFS_PATH specified, ' + 'but file does not exist') + return initramfs + return None + + def install_initramfs(self, initramfs_path, version_root): + '''Install the initramfs outside of 'orig' or 'run' subvolumes. + + This is required because syslinux doesn't traverse subvolumes when + loading the kernel or initramfs. + ''' + self.status(msg='Installing initramfs') + initramfs_dest = os.path.join(version_root, 'initramfs') + cliapp.runcmd(['cp', '-a', initramfs_path, initramfs_dest]) + def install_kernel(self, version_root, temp_root): '''Install the kernel outside of 'orig' or 'run' subvolumes''' @@ -337,20 +375,28 @@ class WriteExtension(cliapp.Application): def get_extra_kernel_args(self): return os.environ.get('KERNEL_ARGS', '') - def install_extlinux(self, real_root): + def install_extlinux(self, real_root, disk_uuid=None): '''Install extlinux on the newly created disk image.''' self.status(msg='Creating extlinux.conf') config = os.path.join(real_root, 'extlinux.conf') - kernel_args = self.get_extra_kernel_args() + kernel_args = ( + 'rw ' # ro ought to work, but we don't test that regularly + 'init=/sbin/init ' # default, but it doesn't hurt to be explicit + 'rootfstype=btrfs ' # required when using initramfs, also boots + # faster when specified without initramfs + 'rootflags=subvol=systems/default/run ') # boot runtime subvol + kernel_args += 'root=%s ' % ('/dev/sda' if disk_uuid is None + else 'UUID=%s' % disk_uuid) + kernel_args += self.get_extra_kernel_args() with open(config, 'w') as f: f.write('default linux\n') f.write('timeout 1\n') f.write('label linux\n') f.write('kernel /systems/default/kernel\n') - f.write('append root=/dev/sda ' - 'rootflags=subvol=systems/default/run ' - '%s init=/sbin/init rw\n' % (kernel_args)) + if disk_uuid is not None: + f.write('initrd /systems/default/initramfs\n') + f.write('append %s\n' % kernel_args) self.status(msg='Installing extlinux') cliapp.runcmd(['extlinux', '--install', real_root]) -- cgit v1.2.1 From 1d9b7fa268c971f363b4139a536bb8facb8d023e Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 4 Jun 2014 11:03:06 +0000 Subject: Make uuid mandatory when calling create_btrfs_system_layout It's only ever called from the core of the write extensions library, so we can change it to be mandatory in all cases. install_extlinux is left with the uuid being optional and defaulting to /dev/sda, since it is called directly from the rawdisk write extension currently, and upgrades of systems that have an initramfs will be part of a later patch series. --- writeexts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index 334dc15c..fb7261f6 100644 --- a/writeexts.py +++ b/writeexts.py @@ -221,7 +221,7 @@ class WriteExtension(cliapp.Application): os.rmdir(mount_point) def create_btrfs_system_layout(self, temp_root, mountpoint, version_label, - disk_uuid=None): + disk_uuid): '''Separate base OS versions from state using subvolumes. ''' -- cgit v1.2.1 From ba6aa8eb7cb8bd249e810972de339b2832f7a665 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 4 Jun 2014 11:10:43 +0000 Subject: Use UUID in fstab entries if provided This makes systems use the UUID of the disk in the fstab when there is no pre-existing fstab entry for /. This happens whether the system has an initramfs or not, since it should be harmless, and by this point we're in userland, so can know what the UUIDs are. Passing the UUID to `complete_fstab_for_btrfs_layout` is optional, and defaults to the old behaviour of using /dev/sda, since it is called directly by some write extensions for doing upgrades, and upgrading systems that use an initramfs will be part of a later patch. --- writeexts.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/writeexts.py b/writeexts.py index fb7261f6..d6f23e0d 100644 --- a/writeexts.py +++ b/writeexts.py @@ -235,7 +235,8 @@ class WriteExtension(cliapp.Application): self.create_orig(version_root, temp_root) system_dir = os.path.join(version_root, 'orig') - state_dirs = self.complete_fstab_for_btrfs_layout(system_dir) + state_dirs = self.complete_fstab_for_btrfs_layout(system_dir, + disk_uuid) for state_dir in state_dirs: self.create_state_subvolume(system_dir, mountpoint, state_dir) @@ -298,7 +299,7 @@ class WriteExtension(cliapp.Application): filepath = os.path.join(existing_state_dir, filename) cliapp.runcmd(['mv', filepath, subvolume]) - def complete_fstab_for_btrfs_layout(self, system_dir): + def complete_fstab_for_btrfs_layout(self, system_dir, rootfs_uuid=None): '''Fill in /etc/fstab entries for the default Btrfs disk layout. In the future we should move this code out of the write extension and @@ -322,8 +323,9 @@ class WriteExtension(cliapp.Application): if '/' in existing_mounts: root_device = existing_mounts['/'] else: - root_device = '/dev/sda' - fstab.add_line('/dev/sda / btrfs defaults,rw,noatime 0 1') + root_device = ('/dev/sda' if rootfs_uuid is None else + 'UUID=%s' % rootfs_uuid) + fstab.add_line('%s / btrfs defaults,rw,noatime 0 1' % root_device) state_dirs_to_create = set() for state_dir in shared_state_dirs: -- cgit v1.2.1 From 21c9683278cb01a4e9425923740124217ce47723 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 10 Jun 2014 17:18:37 +0000 Subject: Check for presence of btrfs before trying to use it If btrfs is not present in the kernel we end up with strange output like this: Error creating disk image2014-06-10 16:00:40 [devel-system-x86_64-generic][my-raw-disk-image][rawdisk.write]Failure to create disk image at /src/tmp/testdev.img ERROR: Command failed: mount -o loop /src/tmp/testdev.img /src/tmp/deployments/tmpQ7wXO1/tmp4lVDcu/tmpvHSzDE mount: mounting /dev/loop0 on /src/tmp/deployments/tmpQ7wXO1/tmp4lVDcu/tmpvHSzDE failed: Device or resource busy To avoid this confusing error, Morph should explicitly check first. --- writeexts.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index d6f23e0d..74587bd1 100644 --- a/writeexts.py +++ b/writeexts.py @@ -104,7 +104,18 @@ class WriteExtension(cliapp.Application): self.output.write('%s\n' % (kwargs['msg'] % kwargs)) self.output.flush() - + + def check_for_btrfs_in_deployment_host_kernel(self): + with open('/proc/filesystems') as f: + text = f.read() + return '\tbtrfs\n' in text + + def require_btrfs_in_deployment_host_kernel(self): + if not self.check_for_btrfs_in_deployment_host_kernel(): + raise cliapp.AppException( + 'Error: Btrfs is required for this deployment, but was not ' + 'detected in the kernel of the machine that is running Morph.') + def create_local_system(self, temp_root, raw_disk): '''Create a raw system image locally.''' size = self.get_disk_size() -- cgit v1.2.1 From 37b7e39aa7c2216c385b87f8c986821ad2cc1d52 Mon Sep 17 00:00:00 2001 From: James Thomas Date: Wed, 30 Jul 2014 17:46:58 +0100 Subject: Support setting a different root device using ROOT_DEVICE --- writeexts.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/writeexts.py b/writeexts.py index 74587bd1..1b8770e2 100644 --- a/writeexts.py +++ b/writeexts.py @@ -334,7 +334,7 @@ class WriteExtension(cliapp.Application): if '/' in existing_mounts: root_device = existing_mounts['/'] else: - root_device = ('/dev/sda' if rootfs_uuid is None else + root_device = (self.get_root_device() if rootfs_uuid is None else 'UUID=%s' % rootfs_uuid) fstab.add_line('%s / btrfs defaults,rw,noatime 0 1' % root_device) @@ -388,6 +388,9 @@ class WriteExtension(cliapp.Application): def get_extra_kernel_args(self): return os.environ.get('KERNEL_ARGS', '') + def get_root_device(self): + return os.environ.get('ROOT_DEVICE', '/dev/sda') + def install_extlinux(self, real_root, disk_uuid=None): '''Install extlinux on the newly created disk image.''' @@ -399,7 +402,7 @@ class WriteExtension(cliapp.Application): 'rootfstype=btrfs ' # required when using initramfs, also boots # faster when specified without initramfs 'rootflags=subvol=systems/default/run ') # boot runtime subvol - kernel_args += 'root=%s ' % ('/dev/sda' if disk_uuid is None + kernel_args += 'root=%s ' % (self.get_root_device() if disk_uuid is None else 'UUID=%s' % disk_uuid) kernel_args += self.get_extra_kernel_args() with open(config, 'w') as f: -- cgit v1.2.1 From d071207ed1a4a895219cb5ce4731eba52319b8b3 Mon Sep 17 00:00:00 2001 From: James Thomas Date: Wed, 30 Jul 2014 18:15:49 +0100 Subject: Make bootloader config/install more generic Remove the BOOTLOADER environment variable and instead favour BOOTLOADER_CONFIG_FORMAT to set the desired bootloader format, and BOOTLOADER_INSTALL to set the type of bootloader to install. For example, since u-boot can boot using extlinux.conf files, it's conceivable that someone might want to do CONFIG_FORMAT=extlinux.conf, INSTALL=u-boot. However, for most platforms you would want to set INSTALL to "none" --- writeexts.py | 69 +++++++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 52 insertions(+), 17 deletions(-) diff --git a/writeexts.py b/writeexts.py index 1b8770e2..dec318d9 100644 --- a/writeexts.py +++ b/writeexts.py @@ -257,15 +257,15 @@ class WriteExtension(cliapp.Application): os.symlink( version_label, os.path.join(mountpoint, 'systems', 'default')) - if self.bootloader_is_wanted(): + if self.bootloader_config_is_wanted(): self.install_kernel(version_root, temp_root) self.install_syslinux_menu(mountpoint, version_root) if initramfs is not None: self.install_initramfs(initramfs, version_root) - self.install_extlinux(mountpoint, disk_uuid) + self.generate_bootloader_config(mountpoint, disk_uuid) else: - self.install_extlinux(mountpoint) - + self.generate_bootloader_config(mountpoint) + self.install_bootloader(mountpoint) def create_orig(self, version_root, temp_root): '''Create the default "factory" system.''' @@ -385,13 +385,36 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['cp', '-a', try_path, kernel_dest]) break + def get_bootloader_install(self): + # Do we actually want to install the bootloader? + # Set this to "none" to prevent the install + return os.environ.get('BOOTLOADER_INSTALL', 'extlinux') + + def get_bootloader_config_format(self): + # The config format for the bootloader, + # if not set we default to extlinux for x86 + return os.environ.get('BOOTLOADER_CONFIG_FORMAT', 'extlinux') + def get_extra_kernel_args(self): return os.environ.get('KERNEL_ARGS', '') def get_root_device(self): return os.environ.get('ROOT_DEVICE', '/dev/sda') - def install_extlinux(self, real_root, disk_uuid=None): + def generate_bootloader_config(self, real_root, disk_uuid=None): + '''Install extlinux on the newly created disk image.''' + config_function_dict = { + 'extlinux': self.generate_extlinux_config, + } + + config_type = self.get_bootloader_config_format() + if config_type in config_function_dict: + config_function_dict[config_type](real_root, disk_uuid) + else: + raise cliapp.AppException( + 'Invalid BOOTLOADER_CONFIG_FORMAT %s' % config_type) + + def generate_extlinux_config(self, real_root, disk_uuid=None): '''Install extlinux on the newly created disk image.''' self.status(msg='Creating extlinux.conf') @@ -414,6 +437,19 @@ class WriteExtension(cliapp.Application): f.write('initrd /systems/default/initramfs\n') f.write('append %s\n' % kernel_args) + def install_bootloader(self, real_root): + install_function_dict = { + 'extlinux': self.install_bootloader_extlinux, + } + + install_type = self.get_bootloader_install() + if install_type in install_function_dict: + install_function_dict[install_type](real_root) + elif install_type != 'none': + raise cliapp.AppException( + 'Invalid BOOTLOADER_INSTALL %s' % install_type) + + def install_bootloader_extlinux(self, real_root): self.status(msg='Installing extlinux') cliapp.runcmd(['extlinux', '--install', real_root]) @@ -447,12 +483,13 @@ class WriteExtension(cliapp.Application): else: return [] - def bootloader_is_wanted(self): - '''Does the user request a bootloader? + def bootloader_config_is_wanted(self): + '''Does the user want to generate a bootloader config? - The user may set $BOOTLOADER to yes, no, or auto. If not - set, auto is the default and means that the bootloader will - be installed on x86-32 and x86-64, but not otherwise. + The user may set $BOOTLOADER_CONFIG_FORMAT to the desired + format (u-boot or extlinux). If not set, extlinux is the + default but will be generated on x86-32 and x86-64, but not + otherwise. ''' @@ -460,14 +497,12 @@ class WriteExtension(cliapp.Application): return (arch == 'x86_64' or (arch.startswith('i') and arch.endswith('86'))) - value = os.environ.get('BOOTLOADER', 'auto') - if value == 'auto': - if is_x86(os.uname()[-1]): - value = 'yes' - else: - value = 'no' + value = os.environ.get('BOOTLOADER_CONFIG_FORMAT', '') + if value == '': + if not is_x86(os.uname()[-1]): + return False - return value == 'yes' + return True def get_environment_boolean(self, variable): '''Parse a yes/no boolean passed through the environment.''' -- cgit v1.2.1 From 6d8b1e2d8f6b9ddc9f94ef306737a17d25ee1a48 Mon Sep 17 00:00:00 2001 From: James Thomas Date: Tue, 22 Jul 2014 11:11:27 +0000 Subject: Add support for a device tree to be set using DTB_PATH --- writeexts.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/writeexts.py b/writeexts.py index dec318d9..863a9351 100644 --- a/writeexts.py +++ b/writeexts.py @@ -259,6 +259,8 @@ class WriteExtension(cliapp.Application): if self.bootloader_config_is_wanted(): self.install_kernel(version_root, temp_root) + if self.get_dtb_path() != '': + self.install_dtb(version_root, temp_root) self.install_syslinux_menu(mountpoint, version_root) if initramfs is not None: self.install_initramfs(initramfs, version_root) @@ -385,6 +387,23 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['cp', '-a', try_path, kernel_dest]) break + def install_dtb(self, version_root, temp_root): + '''Install the device tree outside of 'orig' or 'run' subvolumes''' + + self.status(msg='Installing devicetree') + device_tree_path = self.get_dtb_path() + dtb_dest = os.path.join(version_root, 'dtb') + try_path = os.path.join(temp_root, device_tree_path) + if os.path.exists(try_path): + cliapp.runcmd(['cp', '-a', try_path, dtb_dest]) + else: + logging.error("Failed to find device tree %s", device_tree_path) + raise cliapp.AppException( + 'Failed to find device tree %s' % device_tree_path) + + def get_dtb_path(self): + return os.environ.get('DTB_PATH', '') + def get_bootloader_install(self): # Do we actually want to install the bootloader? # Set this to "none" to prevent the install @@ -435,6 +454,8 @@ class WriteExtension(cliapp.Application): f.write('kernel /systems/default/kernel\n') if disk_uuid is not None: f.write('initrd /systems/default/initramfs\n') + if self.get_dtb_path() != '': + f.write('devicetree /systems/default/dtb\n') f.write('append %s\n' % kernel_args) def install_bootloader(self, real_root): -- cgit v1.2.1 From d2dc2bb2585c97fc54847950a8c2f74a77ef9526 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Tue, 12 Aug 2014 13:59:09 +0000 Subject: Merge remote-tracking branch 'origin/baserock/james/writeexts_support_jetson' --- writeexts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index 863a9351..c3605b1c 100644 --- a/writeexts.py +++ b/writeexts.py @@ -444,7 +444,8 @@ class WriteExtension(cliapp.Application): 'rootfstype=btrfs ' # required when using initramfs, also boots # faster when specified without initramfs 'rootflags=subvol=systems/default/run ') # boot runtime subvol - kernel_args += 'root=%s ' % (self.get_root_device() if disk_uuid is None + kernel_args += 'root=%s ' % (self.get_root_device() + if disk_uuid is None else 'UUID=%s' % disk_uuid) kernel_args += self.get_extra_kernel_args() with open(config, 'w') as f: -- cgit v1.2.1 From 39991810f14b836e99d184f554d0354a96eaa0c7 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Fri, 29 Aug 2014 23:18:47 +0100 Subject: deploy: Make Python extensions log debug messages to MORPH_LOG_FD by default Previously logging was disabled for Python deploy extensions. We get a lot of useful information for free in the log file by doing this: the environment will be written when the subprocess starts, and if it crashes the full backtrace will be written there too. Subcommand execution with cliapp.runcmd() will also be logged. --- writeexts.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index c3605b1c..5102bfdc 100644 --- a/writeexts.py +++ b/writeexts.py @@ -89,7 +89,30 @@ class WriteExtension(cliapp.Application): write extensions. ''' - + + def setup_logging(self): + '''Direct all logging output to MORPH_LOG_FD, if set. + + This file descriptor is read by Morph and written into its own log + file. + + This overrides cliapp's usual configurable logging setup. + + ''' + log_write_fd = int(os.environ.get('MORPH_LOG_FD', 0)) + + if log_write_fd == 0: + return + + formatter = logging.Formatter('%(message)s') + + handler = logging.StreamHandler(os.fdopen(log_write_fd, 'w')) + handler.setFormatter(formatter) + + logger = logging.getLogger() + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) + def process_args(self, args): raise NotImplementedError() -- cgit v1.2.1 From 23ead0ffc536d39dda148c6bbf3be6cce3561208 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 8 Aug 2014 13:17:01 +0000 Subject: Transfer sparse files faster for kvm, vbox deployment The KVM and VirtualBox deployments use sparse files for raw disk images. This means they can store a large disk (say, tens or hundreds of gigabytes) without using more disk space than is required for the actual content (e.g., a gigabyte or so for the files in the root filesystem). The kernel and filesystem make the unwritten parts of the disk image look as if they are filled with zero bytes. This is good. However, during deployment those sparse files get transferred as if there really are a lot of zeroes. Those zeroes take a lot of time to transfer. rsync, for example, does not handle large holes efficiently. This change introduces a couple of helper tools (morphlib/xfer-hole and morphlib/recv-hole), which transfer the holes more efficiently. The xfer-hole program reads a file and outputs records like these: DATA 123 binary data (exaclyt 123 bytes and no newline at the end) HOLE 3245 xfer-hole can do this efficiently, without having to read through all the zeroes in the holes, using the SEEK_DATA and SEEK_HOLE arguments to lseek. Using this, the holes take only take a few bytes each, making it possible to transfer a disk image faster. In my benchmarks, transferring a 100G byte disk image took about 100 seconds for KVM, and 220 seconds for VirtualBox (which needs to more work at the receiver to convert the raw disk to a VDI). Both benchmarks were from a VM on my laptop to the laptop itself. The interesting bit here is that the receiver (recv-hole) is simple enough that it can be implemented in a bit of shell script, and the text of the shell script can be run on the remote end by giving it to ssh as a command line argument. This means there is no need to install any special tools on the receiver, which makes using this improvement much simpler. --- recv-hole | 134 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ xfer-hole | 132 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 266 insertions(+) create mode 100755 recv-hole create mode 100755 xfer-hole diff --git a/recv-hole b/recv-hole new file mode 100755 index 00000000..75f80a6a --- /dev/null +++ b/recv-hole @@ -0,0 +1,134 @@ +#!/bin/sh +# +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + + +# Receive a data stream describing a sparse file, and reproduce it, +# either to a named file or stdout. +# +# The data stream is simple: it's a sequence of DATA or HOLE records: +# +# DATA +# 123 +# <123 bytes of binary data, NOT including newline at the end> +# +# HOLE +# 123 +# +# This shell script can be executed over ssh (given to ssh as an arguemnt, +# with suitable escaping) on a different computer. This allows a large +# sparse file (e.g., disk image) be transferred quickly. + + +set -eu + + +die() +{ + echo "$@" 1>&2 + exit 1 +} + + +recv_hole_to_file() +{ + local n + + read n + truncate --size "+$n" "$1" +} + + +recv_data_to_file() +{ + local n + read n + + local blocksize=1048576 + local blocks="$(echo "$n" / "$blocksize" | bc)" + local extra="$(echo "$n" % "$blocksize" | bc)" + + xfer_data_to_stdout "$blocksize" "$blocks" >> "$1" + xfer_data_to_stdout 1 "$extra" >> "$1" +} + + +recv_hole_to_stdout() +{ + local n + read n + (echo "$n"; cat /dev/zero) | recv_data_to_stdout +} + + +recv_data_to_stdout() +{ + local n + read n + + local blocksize=1048576 + local blocks="$(echo "$n" / "$blocksize" | bc)" + local extra="$(echo "$n" % "$blocksize" | bc)" + + xfer_data_to_stdout "$blocksize" "$blocks" + xfer_data_to_stdout 1 "$extra" +} + + +xfer_data_to_stdout() +{ + local log="$(mktemp)" + if ! dd "bs=$1" count="$2" iflag=fullblock status=noxfer 2> "$log" + then + cat "$log" 1>&2 + rm -f "$log" + exit 1 + else + rm -f "$log" + fi +} + + +type="$1" +case "$type" in + file) + output="$2" + truncate --size=0 "$output" + while read what + do + case "$what" in + DATA) recv_data_to_file "$output" ;; + HOLE) recv_hole_to_file "$output" ;; + *) die "Unknown instruction: $what" ;; + esac + done + ;; + vbox) + output="$2" + disk_size="$3" + while read what + do + case "$what" in + DATA) recv_data_to_stdout ;; + HOLE) recv_hole_to_stdout ;; + *) die "Unknown instruction: $what" ;; + esac + done | + VBoxManage convertfromraw stdin "$output" "$disk_size" + ;; +esac diff --git a/xfer-hole b/xfer-hole new file mode 100755 index 00000000..0d4cee7a --- /dev/null +++ b/xfer-hole @@ -0,0 +1,132 @@ +#!/usr/bin/env python +# +# Send a sparse file more space-efficiently. +# See recv-hole for a description of the protocol. +# +# Note that xfer-hole requires a version of Linux with support for +# SEEK_DATA and SEEK_HOLE. +# +# +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + + + +import errno +import os +import sys + + +SEEK_DATA = 3 +SEEK_HOLE = 4 + + +filename = sys.argv[1] +fd = os.open(filename, os.O_RDONLY) +pos = 0 + + +DATA = 'data' +HOLE = 'hole' +EOF = 'eof' + + +def safe_lseek(fd, pos, whence): + try: + return os.lseek(fd, pos, whence) + except OSError as e: + if e.errno == errno.ENXIO: + return -1 + raise + + +def current_data_or_pos(fd, pos): + length = safe_lseek(fd, 0, os.SEEK_END) + next_data = safe_lseek(fd, pos, SEEK_DATA) + next_hole = safe_lseek(fd, pos, SEEK_HOLE) + + if pos == length: + return EOF, pos + elif pos == next_data: + return DATA, pos + elif pos == next_hole: + return HOLE, pos + else: + assert False, \ + ("Do not understand: pos=%d next_data=%d next_hole=%d" % + (pos, next_data, next_hole)) + + +def next_data_or_hole(fd, pos): + length = safe_lseek(fd, 0, os.SEEK_END) + next_data = safe_lseek(fd, pos, SEEK_DATA) + next_hole = safe_lseek(fd, pos, SEEK_HOLE) + + if pos == length: + return EOF, pos + elif pos == next_data: + # We are at data. + if next_hole == -1 or next_hole == length: + return EOF, length + else: + return HOLE, next_hole + elif pos == next_hole: + # We are at a hole. + if next_data == -1 or next_data == length: + return EOF, length + else: + return DATA, next_data + else: + assert False, \ + ("Do not understand: pos=%d next_data=%d next_hole=%d" % + (pos, next_data, next_hole)) + + +def find_data_and_holes(fd): + pos = safe_lseek(fd, 0, os.SEEK_CUR) + + kind, pos = current_data_or_pos(fd, pos) + while kind != EOF: + yield kind, pos + kind, pos = next_data_or_hole(fd, pos) + yield kind, pos + + +def make_xfer_instructions(fd): + prev_kind = None + prev_pos = None + for kind, pos in find_data_and_holes(fd): + if prev_kind == DATA: + yield (DATA, prev_pos, pos) + elif prev_kind == HOLE: + yield (HOLE, prev_pos, pos) + prev_kind = kind + prev_pos = pos + + +def copy_slice_from_file(to, fd, start, end): + safe_lseek(fd, start, os.SEEK_SET) + data = os.read(fd, end - start) + to.write(data) + + +for kind, start, end in make_xfer_instructions(fd): + if kind == HOLE: + sys.stdout.write('HOLE\n%d\n' % (end - start)) + elif kind == DATA: + sys.stdout.write('DATA\n%d\n' % (end - start)) + copy_slice_from_file(sys.stdout, fd, start, end) -- cgit v1.2.1 From c136fffbdffa81f03eeb84ce491f987af53009a5 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 5 Sep 2014 13:18:05 +0000 Subject: Document why VBoxManage is run in recv-hole instead of caller --- recv-hole | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/recv-hole b/recv-hole index 75f80a6a..a406cc98 100755 --- a/recv-hole +++ b/recv-hole @@ -33,6 +33,31 @@ # This shell script can be executed over ssh (given to ssh as an arguemnt, # with suitable escaping) on a different computer. This allows a large # sparse file (e.g., disk image) be transferred quickly. +# +# This script should be called in one of the following ways: +# +# recv-hole file FILENAME +# recv-hole vbox FILENAME DISKSIZE +# +# In both cases, FILENAME is the pathname of the disk image on the +# receiving end. DISKSIZE is the size of the disk image in bytes. The +# first form is used when transferring a disk image to become an +# identical file on the receiving end. +# +# The second form is used when the disk image should be converted for +# use by VirtualBox. In this case, we want to avoid writing a +# temporary file on disk, and then calling the VirtualBox VBoxManage +# tool to do the conversion, since that would involve large amounts of +# unnecessary I/O and disk usage. Instead we pipe the file directly to +# VBoxManage, avoiding those issues. The piping is done here in this +# script, instead of in the caller, to make it easier to run things +# over ssh. +# +# However, since it's not possible seek in a Unix pipe, we have to +# explicitly write the zeroes into the pipe. This is not +# super-efficient, but the way to avoid that would be to avoid sending +# a sparse file, and do the conversion to a VDI on the sending end. +# That is out of scope for xfer-hole and recv-hole. set -eu -- cgit v1.2.1 From 626adf0eb70e059de498a01ab66864a45128c935 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 5 Sep 2014 13:27:36 +0000 Subject: Use $((...))) instead of $(... | bc) --- recv-hole | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/recv-hole b/recv-hole index a406cc98..d6504bf6 100755 --- a/recv-hole +++ b/recv-hole @@ -85,8 +85,8 @@ recv_data_to_file() read n local blocksize=1048576 - local blocks="$(echo "$n" / "$blocksize" | bc)" - local extra="$(echo "$n" % "$blocksize" | bc)" + local blocks=$(($n / $blocksize)) + local extra=$(($n % $blocksize)) xfer_data_to_stdout "$blocksize" "$blocks" >> "$1" xfer_data_to_stdout 1 "$extra" >> "$1" @@ -107,8 +107,8 @@ recv_data_to_stdout() read n local blocksize=1048576 - local blocks="$(echo "$n" / "$blocksize" | bc)" - local extra="$(echo "$n" % "$blocksize" | bc)" + local blocks=$(($n / $blocksize)) + local extra=$(($n % $blocksize)) xfer_data_to_stdout "$blocksize" "$blocks" xfer_data_to_stdout 1 "$extra" -- cgit v1.2.1 From f05a161053f68244894fb0db73041ed5909dcbc0 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Fri, 19 Sep 2014 15:26:36 +0000 Subject: Prevent cliapp from logging env. variables with 'PASSWORD' in their name The upstream cliapp project is not interested in this functionality right now. --- writeexts.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/writeexts.py b/writeexts.py index 5102bfdc..0fd0ad7b 100644 --- a/writeexts.py +++ b/writeexts.py @@ -113,6 +113,10 @@ class WriteExtension(cliapp.Application): logger.addHandler(handler) logger.setLevel(logging.DEBUG) + def log_config(self): + with morphlib.util.hide_password_environment_variables(os.environ): + cliapp.Application.log_config(self) + def process_args(self, args): raise NotImplementedError() -- cgit v1.2.1 From 9b8eca5826cf32c790bbb1e1f8836dc1754a4ef9 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Tue, 30 Sep 2014 10:17:05 +0100 Subject: xfer-hole: Fix bug in copy_slice_from_file. os.read is limited to an int in size. copy_slice_from_file was trying to os.read more than that causing an OverflowError. --- xfer-hole | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/xfer-hole b/xfer-hole index 0d4cee7a..22ee06bf 100755 --- a/xfer-hole +++ b/xfer-hole @@ -120,8 +120,14 @@ def make_xfer_instructions(fd): def copy_slice_from_file(to, fd, start, end): safe_lseek(fd, start, os.SEEK_SET) - data = os.read(fd, end - start) - to.write(data) + nbytes = end - start + max_at_a_time = 1024**2 + while nbytes > 0: + data = os.read(fd, min(nbytes, max_at_a_time)) + if not data: + break + to.write(data) + nbytes -= len(data) for kind, start, end in make_xfer_instructions(fd): -- cgit v1.2.1 From 10e6486de6f6f45a19bcd6099d7830718a9d12a8 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Fri, 10 Oct 2014 10:56:52 +0000 Subject: Add 'is_device' function to check if we are deploying to a device --- writeexts.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/writeexts.py b/writeexts.py index 0fd0ad7b..48c9434e 100644 --- a/writeexts.py +++ b/writeexts.py @@ -22,6 +22,8 @@ import shutil import sys import time import tempfile +import errno +import stat import morphlib @@ -572,3 +574,12 @@ class WriteExtension(cliapp.Application): logging.error("Error checking SSH connectivity: %s", str(e)) raise cliapp.AppException( 'Unable to SSH to %s: %s' % (ssh_host, e)) + + def is_device(self, location): + try: + st = os.stat(location) + return stat.S_ISBLK(st.st_mode) + except OSError as e: + if e.errno == errno.ENOENT: + return False + raise -- cgit v1.2.1 From 27af6089593df7d7a422a45f19f2456870832c93 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Thu, 16 Oct 2014 17:25:03 +0000 Subject: Add force flag to 'mkfs.btrfs' This way when deploying to a device it will format it without asking the user if the device already has format. --- writeexts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index 48c9434e..4b9e4fcd 100644 --- a/writeexts.py +++ b/writeexts.py @@ -226,7 +226,7 @@ class WriteExtension(cliapp.Application): def mkfs_btrfs(self, location): '''Create a btrfs filesystem on the disk.''' self.status(msg='Creating btrfs filesystem') - cliapp.runcmd(['mkfs.btrfs', '-L', 'baserock', location]) + cliapp.runcmd(['mkfs.btrfs', '-f', '-L', 'baserock', location]) def get_uuid(self, location): '''Get the UUID of a block device's file system.''' -- cgit v1.2.1 From ca385c659297749b37af13be4fec76c8d68db0b7 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Thu, 16 Oct 2014 17:25:37 +0000 Subject: Don't loop mount when mounting a device --- writeexts.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index 4b9e4fcd..31e7a0b6 100644 --- a/writeexts.py +++ b/writeexts.py @@ -246,7 +246,10 @@ class WriteExtension(cliapp.Application): self.status(msg='Mounting filesystem') tempdir = tempfile.mkdtemp() - cliapp.runcmd(['mount', '-o', 'loop', location, tempdir]) + if self.is_device(location): + cliapp.runcmd(['mount', location, tempdir]) + else: + cliapp.runcmd(['mount', '-o', 'loop', location, tempdir]) return tempdir def unmount(self, mount_point): -- cgit v1.2.1 From 5aa74fd32200d84e81e1473eed9e155ebd5a379b Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Tue, 11 Nov 2014 15:42:07 +0000 Subject: Split create_local_system on various methods This way we can still use create_local_system to create a raw disk, but also reuse bits of it to be able e.g. to deploy to devices. --- writeexts.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/writeexts.py b/writeexts.py index 31e7a0b6..7030c0b5 100644 --- a/writeexts.py +++ b/writeexts.py @@ -24,6 +24,7 @@ import time import tempfile import errno import stat +import contextlib import morphlib @@ -147,25 +148,39 @@ class WriteExtension(cliapp.Application): def create_local_system(self, temp_root, raw_disk): '''Create a raw system image locally.''' + + with self.created_disk_image(raw_disk): + self.format_btrfs(raw_disk) + self.create_system(temp_root, raw_disk) + + @contextlib.contextmanager + def created_disk_image(self, location): size = self.get_disk_size() if not size: raise cliapp.AppException('DISK_SIZE is not defined') - self.create_raw_disk_image(raw_disk, size) + self.create_raw_disk_image(location, size) + try: + yield + except BaseException: + os.unlink(location) + raise + + def format_btrfs(self, raw_disk): try: self.mkfs_btrfs(raw_disk) - mp = self.mount(raw_disk) except BaseException: sys.stderr.write('Error creating disk image') - os.remove(raw_disk) raise + + def create_system(self, temp_root, raw_disk): try: + mp = self.mount(raw_disk) self.create_btrfs_system_layout( temp_root, mp, version_label='factory', disk_uuid=self.get_uuid(raw_disk)) except BaseException, e: sys.stderr.write('Error creating Btrfs system layout') self.unmount(mp) - os.remove(raw_disk) raise else: self.unmount(mp) -- cgit v1.2.1 From b03e6f1c4342d48c3fb9e530a49807e08a0162e7 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Mon, 1 Dec 2014 15:25:21 +0000 Subject: writeexts.py: convert 'mount' to context manager --- writeexts.py | 65 +++++++++++++++++++++++++----------------------------------- 1 file changed, 27 insertions(+), 38 deletions(-) diff --git a/writeexts.py b/writeexts.py index 7030c0b5..91936f64 100644 --- a/writeexts.py +++ b/writeexts.py @@ -173,17 +173,14 @@ class WriteExtension(cliapp.Application): raise def create_system(self, temp_root, raw_disk): - try: - mp = self.mount(raw_disk) - self.create_btrfs_system_layout( - temp_root, mp, version_label='factory', - disk_uuid=self.get_uuid(raw_disk)) - except BaseException, e: - sys.stderr.write('Error creating Btrfs system layout') - self.unmount(mp) - raise - else: - self.unmount(mp) + with self.mount(raw_disk) as mp: + try: + self.create_btrfs_system_layout( + temp_root, mp, version_label='factory', + disk_uuid=self.get_uuid(raw_disk)) + except BaseException, e: + sys.stderr.write('Error creating Btrfs system layout') + raise def _parse_size(self, size): '''Parse a size from a string. @@ -249,34 +246,26 @@ class WriteExtension(cliapp.Application): # lies by exiting successfully. return cliapp.runcmd(['blkid', '-s', 'UUID', '-o', 'value', location]).strip() - - def mount(self, location): - '''Mount the filesystem so it can be tweaked. - - Return path to the mount point. - The mount point is a newly created temporary directory. - The caller must call self.unmount to unmount on the return value. - - ''' - self.status(msg='Mounting filesystem') - tempdir = tempfile.mkdtemp() - if self.is_device(location): - cliapp.runcmd(['mount', location, tempdir]) - else: - cliapp.runcmd(['mount', '-o', 'loop', location, tempdir]) - return tempdir - - def unmount(self, mount_point): - '''Unmount the filesystem mounted by self.mount. - - Also, remove the temporary directory. - - ''' - - self.status(msg='Unmounting filesystem') - cliapp.runcmd(['umount', mount_point]) - os.rmdir(mount_point) + @contextlib.contextmanager + def mount(self, location): + self.status(msg='Mounting filesystem') + try: + mount_point = tempfile.mkdtemp() + if self.is_device(location): + cliapp.runcmd(['mount', location, mount_point]) + else: + cliapp.runcmd(['mount', '-o', 'loop', location, mount_point]) + except BaseException, e: + sys.stderr.write('Error mounting filesystem') + os.rmdir(mount_point) + raise + try: + yield mount_point + finally: + self.status(msg='Unmounting filesystem') + cliapp.runcmd(['umount', mount_point]) + os.rmdir(mount_point) def create_btrfs_system_layout(self, temp_root, mountpoint, version_label, disk_uuid): -- cgit v1.2.1 From c39f26ac3af32a4b45ea9e48e0c6542346ef1fb5 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Fri, 12 Dec 2014 11:30:13 +0000 Subject: Document BOOTLOADER_INSTALL and BOOTLOADER_CONFIG_FORMAT write extension parameters --- writeexts.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/writeexts.py b/writeexts.py index 91936f64..068f0741 100644 --- a/writeexts.py +++ b/writeexts.py @@ -545,9 +545,8 @@ class WriteExtension(cliapp.Application): '''Does the user want to generate a bootloader config? The user may set $BOOTLOADER_CONFIG_FORMAT to the desired - format (u-boot or extlinux). If not set, extlinux is the - default but will be generated on x86-32 and x86-64, but not - otherwise. + format. 'extlinux' is the only allowed value, and is the default + value for x86-32 and x86-64. ''' -- cgit v1.2.1 From 85281cd3faa5e563f7e3efe6225ae6d988d23925 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Fri, 12 Dec 2014 11:31:09 +0000 Subject: Whitespace removal --- writeexts.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/writeexts.py b/writeexts.py index 068f0741..fd2f5529 100644 --- a/writeexts.py +++ b/writeexts.py @@ -83,14 +83,14 @@ class Fstab(object): class WriteExtension(cliapp.Application): '''A base class for deployment write extensions. - + A subclass should subclass this class, and add a ``process_args`` method. - + Note that it is not necessary to subclass this class for write extensions. This class is here just to collect common code for write extensions. - + ''' def setup_logging(self): @@ -125,13 +125,13 @@ class WriteExtension(cliapp.Application): def status(self, **kwargs): '''Provide status output. - + The ``msg`` keyword argument is the actual message, the rest are values for fields in the message as interpolated by %. - + ''' - + self.output.write('%s\n' % (kwargs['msg'] % kwargs)) self.output.flush() @@ -184,9 +184,9 @@ class WriteExtension(cliapp.Application): def _parse_size(self, size): '''Parse a size from a string. - + Return size in bytes. - + ''' m = re.match('^(\d+)([kmgKMG]?)$', size) -- cgit v1.2.1 From c5d5f960449b881b323a0353c332ef5d36d4490d Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Fri, 12 Dec 2014 11:36:04 +0000 Subject: Document KERNEL_ARGS write extension parameter --- writeexts.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/writeexts.py b/writeexts.py index fd2f5529..6ab2dd55 100644 --- a/writeexts.py +++ b/writeexts.py @@ -474,6 +474,12 @@ class WriteExtension(cliapp.Application): self.status(msg='Creating extlinux.conf') config = os.path.join(real_root, 'extlinux.conf') + + ''' Please also update the documentation in the following files + if you change these default kernel args: + - kvm.write.help + - rawdisk.write.help + - virtualbox-ssh.write.help ''' kernel_args = ( 'rw ' # ro ought to work, but we don't test that regularly 'init=/sbin/init ' # default, but it doesn't hurt to be explicit -- cgit v1.2.1 From 9fe18e52c08117dd2705b0c19f40d2096d02dea7 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Thu, 26 Feb 2015 16:42:28 +0000 Subject: Fix Morph producing unbootable systems Since the version of btrfs-progs in the Baserock reference system definitions was updated to v3.18.2, Morph has produced unbootable x86 systems. This is down to lack of support for new Btrfs features in SYSLINUX. This patch disables the new features so that deployed systems will boot. Although I generally don't want to have compatibility code in Morph, this patch keeps support for the old mkfs.btrfs (which doesn't support the commandline options that we need to pass to new mkfs.btrfs). This will hopefully save people from suffering 'I need to use new Morph to upgrade my devel system, but new Morph doesn't run on my devel system'. --- writeexts.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index 6ab2dd55..ad4fabe9 100644 --- a/writeexts.py +++ b/writeexts.py @@ -237,8 +237,32 @@ class WriteExtension(cliapp.Application): def mkfs_btrfs(self, location): '''Create a btrfs filesystem on the disk.''' + self.status(msg='Creating btrfs filesystem') - cliapp.runcmd(['mkfs.btrfs', '-f', '-L', 'baserock', location]) + try: + # The following command disables some new filesystem features. We + # need to do this because at the time of writing, SYSLINUX has not + # been updated to understand these new features and will fail to + # boot if the kernel is on a filesystem where they are enabled. + cliapp.runcmd( + ['mkfs.btrfs','-f', '-L', 'baserock', + '--features', '^extref', + '--features', '^skinny-metadata', + '--features', '^mixed-bg', + '--nodesize', '4096', + location]) + except cliapp.AppException as e: + if 'unrecognized option \'--features\'' in e.msg: + # Old versions of mkfs.btrfs (including v0.20, present in many + # Baserock releases) don't support the --features option, but + # also don't enable the new features by default. So we can + # still create a bootable system in this situation. + logging.debug( + 'Assuming mkfs.btrfs failure was because the tool is too ' + 'old to have --features flag.') + cliapp.runcmd(['mkfs.btrfs','-f', '-L', 'baserock', location]) + else: + raise def get_uuid(self, location): '''Get the UUID of a block device's file system.''' -- cgit v1.2.1 From 61a0d3d5475ebb082c5e757850153b1a134d6c23 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Fri, 27 Feb 2015 16:20:21 +0000 Subject: Fix copyright years --- writeexts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index ad4fabe9..ab451d14 100644 --- a/writeexts.py +++ b/writeexts.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -- cgit v1.2.1 From 2a5de41bdf592c4a104b481db0261009870efd24 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Fri, 27 Feb 2015 16:20:21 +0000 Subject: Fix copyright years --- writeexts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index ad4fabe9..ab451d14 100644 --- a/writeexts.py +++ b/writeexts.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -- cgit v1.2.1 From 8b1d5d732404abd70fe1ef91be84a3f2ce4c1c31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 13 Mar 2015 14:25:00 +0000 Subject: Use python3 compatible notation for octal constants Change-Id: I771c3de9cecda7a503f4d36ae5d9fabc040892e4 --- writeexts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index ab451d14..9f166d7f 100644 --- a/writeexts.py +++ b/writeexts.py @@ -360,7 +360,7 @@ class WriteExtension(cliapp.Application): self.status(msg='Creating %s subvolume' % state_subdir) subvolume = os.path.join(mountpoint, 'state', state_subdir) cliapp.runcmd(['btrfs', 'subvolume', 'create', subvolume]) - os.chmod(subvolume, 0755) + os.chmod(subvolume, 0o755) existing_state_dir = os.path.join(system_dir, state_subdir) files = [] -- cgit v1.2.1 From e9461f5add5f701017ed74d5ca1e0d6b0c812bbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 13 Mar 2015 18:18:55 +0000 Subject: Use the modern way of the GPL copyright header: URL instead real address Change-Id: I992dc0c1d40f563ade56a833162d409b02be90a0 --- writeexts.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/writeexts.py b/writeexts.py index ab451d14..6eec4465 100644 --- a/writeexts.py +++ b/writeexts.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . import cliapp -- cgit v1.2.1 From 37dd50f95b35096b2f9c926b684764503126aa7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 13 Mar 2015 18:18:55 +0000 Subject: Use the modern way of the GPL copyright header: URL instead real address Change-Id: I992dc0c1d40f563ade56a833162d409b02be90a0 --- recv-hole | 5 ++--- xfer-hole | 5 ++--- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/recv-hole b/recv-hole index d6504bf6..fe69f304 100755 --- a/recv-hole +++ b/recv-hole @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -12,8 +12,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . # # =*= License: GPL-2 =*= diff --git a/xfer-hole b/xfer-hole index 22ee06bf..91f1be01 100755 --- a/xfer-hole +++ b/xfer-hole @@ -7,7 +7,7 @@ # SEEK_DATA and SEEK_HOLE. # # -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,8 +19,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . # # =*= License: GPL-2 =*= -- cgit v1.2.1 From 705845ad37abae8fe89f2d698e58e41ee863d807 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 13 Mar 2015 13:02:24 +0000 Subject: Use python3 compatible notation for catching exceptions Change-Id: Ibda7a938cd16e35517a531140f39ef4664d85c72 --- writeexts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/writeexts.py b/writeexts.py index 3382767c..129b2bc4 100644 --- a/writeexts.py +++ b/writeexts.py @@ -177,7 +177,7 @@ class WriteExtension(cliapp.Application): self.create_btrfs_system_layout( temp_root, mp, version_label='factory', disk_uuid=self.get_uuid(raw_disk)) - except BaseException, e: + except BaseException as e: sys.stderr.write('Error creating Btrfs system layout') raise @@ -279,7 +279,7 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['mount', location, mount_point]) else: cliapp.runcmd(['mount', '-o', 'loop', location, mount_point]) - except BaseException, e: + except BaseException as e: sys.stderr.write('Error mounting filesystem') os.rmdir(mount_point) raise -- cgit v1.2.1 From 5c6fe9af9a1f2a226020a7a701cf31347fbb3fc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 13 Mar 2015 18:18:55 +0000 Subject: Use the modern way of the GPL copyright header: URL instead real address Change-Id: I992dc0c1d40f563ade56a833162d409b02be90a0 --- writeexts.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/writeexts.py b/writeexts.py index 9f166d7f..3382767c 100644 --- a/writeexts.py +++ b/writeexts.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . import cliapp -- cgit v1.2.1 From aaef5cfe335ebbe742b7eb1593ec51bbeeb4dc7c Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Thu, 19 Mar 2015 13:04:00 +0000 Subject: deploy: Tighten SSH connectivity check I accidentally tried to deploy a Baserock upgrade to a Fedora cloud machine. Every SSH command that Morph ran got the following output: Please login as the user "fedora" rather than the user "root". The existing implementation of check_ssh_connectivity() didn't raise an exception, leading to confusing errors further down. The new implementation produces this error: ERROR: Unexpected output from remote machine: Please login as the user "fedora" rather than the user "root". Change-Id: Ida5a82b25d759167aa842194b0d833d0565b4acf --- writeexts.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index ab451d14..270cade9 100644 --- a/writeexts.py +++ b/writeexts.py @@ -605,12 +605,16 @@ class WriteExtension(cliapp.Application): def check_ssh_connectivity(self, ssh_host): try: - cliapp.ssh_runcmd(ssh_host, ['true']) + output = cliapp.ssh_runcmd(ssh_host, ['echo', 'test']) except cliapp.AppException as e: logging.error("Error checking SSH connectivity: %s", str(e)) raise cliapp.AppException( 'Unable to SSH to %s: %s' % (ssh_host, e)) + if output.strip() != 'test': + raise cliapp.AppException( + 'Unexpected output from remote machine: %s' % output.strip()) + def is_device(self, location): try: st = os.stat(location) -- cgit v1.2.1 From 0a922a4813cb862d840c02a123b6da558af69aa7 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Wed, 3 Jun 2015 11:52:48 +0000 Subject: Put writeexts.py in extensions subdirectory Change-Id: I6eb8fe69416bbf483ffa8c1c317c8f8cea56ef0e --- extensions/writeexts.py | 624 ++++++++++++++++++++++++++++++++++++++++++++++++ writeexts.py | 624 ------------------------------------------------ 2 files changed, 624 insertions(+), 624 deletions(-) create mode 100644 extensions/writeexts.py delete mode 100644 writeexts.py diff --git a/extensions/writeexts.py b/extensions/writeexts.py new file mode 100644 index 00000000..aa185a2b --- /dev/null +++ b/extensions/writeexts.py @@ -0,0 +1,624 @@ +# Copyright (C) 2012-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + + +import cliapp +import logging +import os +import re +import shutil +import sys +import time +import tempfile +import errno +import stat +import contextlib + +import morphlib + + +class Fstab(object): + '''Small helper class for parsing and adding lines to /etc/fstab.''' + + # There is an existing Python helper library for editing of /etc/fstab. + # However it is unmaintained and has an incompatible license (GPL3). + # + # https://code.launchpad.net/~computer-janitor-hackers/python-fstab/trunk + + def __init__(self, filepath='/etc/fstab'): + if os.path.exists(filepath): + with open(filepath, 'r') as f: + self.text= f.read() + else: + self.text = '' + self.filepath = filepath + self.lines_added = 0 + + def get_mounts(self): + '''Return list of mount devices and targets in /etc/fstab. + + Return value is a dict of target -> device. + ''' + mounts = dict() + for line in self.text.splitlines(): + words = line.split() + if len(words) >= 2 and not words[0].startswith('#'): + device, target = words[0:2] + mounts[target] = device + return mounts + + def add_line(self, line): + '''Add a new entry to /etc/fstab. + + Lines are appended, and separated from any entries made by configure + extensions with a comment. + + ''' + if self.lines_added == 0: + if len(self.text) == 0 or self.text[-1] is not '\n': + self.text += '\n' + self.text += '# Morph default system layout\n' + self.lines_added += 1 + + self.text += line + '\n' + + def write(self): + '''Rewrite the fstab file to include all new entries.''' + with morphlib.savefile.SaveFile(self.filepath, 'w') as f: + f.write(self.text) + + +class WriteExtension(cliapp.Application): + + '''A base class for deployment write extensions. + + A subclass should subclass this class, and add a + ``process_args`` method. + + Note that it is not necessary to subclass this class for write + extensions. This class is here just to collect common code for + write extensions. + + ''' + + def setup_logging(self): + '''Direct all logging output to MORPH_LOG_FD, if set. + + This file descriptor is read by Morph and written into its own log + file. + + This overrides cliapp's usual configurable logging setup. + + ''' + log_write_fd = int(os.environ.get('MORPH_LOG_FD', 0)) + + if log_write_fd == 0: + return + + formatter = logging.Formatter('%(message)s') + + handler = logging.StreamHandler(os.fdopen(log_write_fd, 'w')) + handler.setFormatter(formatter) + + logger = logging.getLogger() + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) + + def log_config(self): + with morphlib.util.hide_password_environment_variables(os.environ): + cliapp.Application.log_config(self) + + def process_args(self, args): + raise NotImplementedError() + + def status(self, **kwargs): + '''Provide status output. + + The ``msg`` keyword argument is the actual message, + the rest are values for fields in the message as interpolated + by %. + + ''' + + self.output.write('%s\n' % (kwargs['msg'] % kwargs)) + self.output.flush() + + def check_for_btrfs_in_deployment_host_kernel(self): + with open('/proc/filesystems') as f: + text = f.read() + return '\tbtrfs\n' in text + + def require_btrfs_in_deployment_host_kernel(self): + if not self.check_for_btrfs_in_deployment_host_kernel(): + raise cliapp.AppException( + 'Error: Btrfs is required for this deployment, but was not ' + 'detected in the kernel of the machine that is running Morph.') + + def create_local_system(self, temp_root, raw_disk): + '''Create a raw system image locally.''' + + with self.created_disk_image(raw_disk): + self.format_btrfs(raw_disk) + self.create_system(temp_root, raw_disk) + + @contextlib.contextmanager + def created_disk_image(self, location): + size = self.get_disk_size() + if not size: + raise cliapp.AppException('DISK_SIZE is not defined') + self.create_raw_disk_image(location, size) + try: + yield + except BaseException: + os.unlink(location) + raise + + def format_btrfs(self, raw_disk): + try: + self.mkfs_btrfs(raw_disk) + except BaseException: + sys.stderr.write('Error creating disk image') + raise + + def create_system(self, temp_root, raw_disk): + with self.mount(raw_disk) as mp: + try: + self.create_btrfs_system_layout( + temp_root, mp, version_label='factory', + disk_uuid=self.get_uuid(raw_disk)) + except BaseException as e: + sys.stderr.write('Error creating Btrfs system layout') + raise + + def _parse_size(self, size): + '''Parse a size from a string. + + Return size in bytes. + + ''' + + m = re.match('^(\d+)([kmgKMG]?)$', size) + if not m: + return None + + factors = { + '': 1, + 'k': 1024, + 'm': 1024**2, + 'g': 1024**3, + } + factor = factors[m.group(2).lower()] + + return int(m.group(1)) * factor + + def _parse_size_from_environment(self, env_var, default): + '''Parse a size from an environment variable.''' + + size = os.environ.get(env_var, default) + if size is None: + return None + bytes = self._parse_size(size) + if bytes is None: + raise morphlib.Error('Cannot parse %s value %s' % (env_var, size)) + return bytes + + def get_disk_size(self): + '''Parse disk size from environment.''' + return self._parse_size_from_environment('DISK_SIZE', None) + + def get_ram_size(self): + '''Parse RAM size from environment.''' + return self._parse_size_from_environment('RAM_SIZE', '1G') + + def get_vcpu_count(self): + '''Parse the virtual cpu count from environment.''' + return self._parse_size_from_environment('VCPUS', '1') + + def create_raw_disk_image(self, filename, size): + '''Create a raw disk image.''' + + self.status(msg='Creating empty disk image') + with open(filename, 'wb') as f: + if size > 0: + f.seek(size-1) + f.write('\0') + + def mkfs_btrfs(self, location): + '''Create a btrfs filesystem on the disk.''' + + self.status(msg='Creating btrfs filesystem') + try: + # The following command disables some new filesystem features. We + # need to do this because at the time of writing, SYSLINUX has not + # been updated to understand these new features and will fail to + # boot if the kernel is on a filesystem where they are enabled. + cliapp.runcmd( + ['mkfs.btrfs','-f', '-L', 'baserock', + '--features', '^extref', + '--features', '^skinny-metadata', + '--features', '^mixed-bg', + '--nodesize', '4096', + location]) + except cliapp.AppException as e: + if 'unrecognized option \'--features\'' in e.msg: + # Old versions of mkfs.btrfs (including v0.20, present in many + # Baserock releases) don't support the --features option, but + # also don't enable the new features by default. So we can + # still create a bootable system in this situation. + logging.debug( + 'Assuming mkfs.btrfs failure was because the tool is too ' + 'old to have --features flag.') + cliapp.runcmd(['mkfs.btrfs','-f', '-L', 'baserock', location]) + else: + raise + + def get_uuid(self, location): + '''Get the UUID of a block device's file system.''' + # Requires util-linux blkid; busybox one ignores options and + # lies by exiting successfully. + return cliapp.runcmd(['blkid', '-s', 'UUID', '-o', 'value', + location]).strip() + + @contextlib.contextmanager + def mount(self, location): + self.status(msg='Mounting filesystem') + try: + mount_point = tempfile.mkdtemp() + if self.is_device(location): + cliapp.runcmd(['mount', location, mount_point]) + else: + cliapp.runcmd(['mount', '-o', 'loop', location, mount_point]) + except BaseException as e: + sys.stderr.write('Error mounting filesystem') + os.rmdir(mount_point) + raise + try: + yield mount_point + finally: + self.status(msg='Unmounting filesystem') + cliapp.runcmd(['umount', mount_point]) + os.rmdir(mount_point) + + def create_btrfs_system_layout(self, temp_root, mountpoint, version_label, + disk_uuid): + '''Separate base OS versions from state using subvolumes. + + ''' + initramfs = self.find_initramfs(temp_root) + version_root = os.path.join(mountpoint, 'systems', version_label) + state_root = os.path.join(mountpoint, 'state') + + os.makedirs(version_root) + os.makedirs(state_root) + + self.create_orig(version_root, temp_root) + system_dir = os.path.join(version_root, 'orig') + + state_dirs = self.complete_fstab_for_btrfs_layout(system_dir, + disk_uuid) + + for state_dir in state_dirs: + self.create_state_subvolume(system_dir, mountpoint, state_dir) + + self.create_run(version_root) + + os.symlink( + version_label, os.path.join(mountpoint, 'systems', 'default')) + + if self.bootloader_config_is_wanted(): + self.install_kernel(version_root, temp_root) + if self.get_dtb_path() != '': + self.install_dtb(version_root, temp_root) + self.install_syslinux_menu(mountpoint, version_root) + if initramfs is not None: + self.install_initramfs(initramfs, version_root) + self.generate_bootloader_config(mountpoint, disk_uuid) + else: + self.generate_bootloader_config(mountpoint) + self.install_bootloader(mountpoint) + + def create_orig(self, version_root, temp_root): + '''Create the default "factory" system.''' + + orig = os.path.join(version_root, 'orig') + + self.status(msg='Creating orig subvolume') + cliapp.runcmd(['btrfs', 'subvolume', 'create', orig]) + self.status(msg='Copying files to orig subvolume') + cliapp.runcmd(['cp', '-a', temp_root + '/.', orig + '/.']) + + def create_run(self, version_root): + '''Create the 'run' snapshot.''' + + self.status(msg='Creating run subvolume') + orig = os.path.join(version_root, 'orig') + run = os.path.join(version_root, 'run') + cliapp.runcmd( + ['btrfs', 'subvolume', 'snapshot', orig, run]) + + def create_state_subvolume(self, system_dir, mountpoint, state_subdir): + '''Create a shared state subvolume. + + We need to move any files added to the temporary rootfs by the + configure extensions to their correct home. For example, they might + have added keys in `/root/.ssh` which we now need to transfer to + `/state/root/.ssh`. + + ''' + self.status(msg='Creating %s subvolume' % state_subdir) + subvolume = os.path.join(mountpoint, 'state', state_subdir) + cliapp.runcmd(['btrfs', 'subvolume', 'create', subvolume]) + os.chmod(subvolume, 0o755) + + existing_state_dir = os.path.join(system_dir, state_subdir) + files = [] + if os.path.exists(existing_state_dir): + files = os.listdir(existing_state_dir) + if len(files) > 0: + self.status(msg='Moving existing data to %s subvolume' % subvolume) + for filename in files: + filepath = os.path.join(existing_state_dir, filename) + cliapp.runcmd(['mv', filepath, subvolume]) + + def complete_fstab_for_btrfs_layout(self, system_dir, rootfs_uuid=None): + '''Fill in /etc/fstab entries for the default Btrfs disk layout. + + In the future we should move this code out of the write extension and + in to a configure extension. To do that, though, we need some way of + informing the configure extension what layout should be used. Right now + a configure extension doesn't know if the system is going to end up as + a Btrfs disk image, a tarfile or something else and so it can't come + up with a sensible default fstab. + + Configuration extensions can already create any /etc/fstab that they + like. This function only fills in entries that are missing, so if for + example the user configured /home to be on a separate partition, that + decision will be honoured and /state/home will not be created. + + ''' + shared_state_dirs = {'home', 'root', 'opt', 'srv', 'var'} + + fstab = Fstab(os.path.join(system_dir, 'etc', 'fstab')) + existing_mounts = fstab.get_mounts() + + if '/' in existing_mounts: + root_device = existing_mounts['/'] + else: + root_device = (self.get_root_device() if rootfs_uuid is None else + 'UUID=%s' % rootfs_uuid) + fstab.add_line('%s / btrfs defaults,rw,noatime 0 1' % root_device) + + state_dirs_to_create = set() + for state_dir in shared_state_dirs: + if '/' + state_dir not in existing_mounts: + state_dirs_to_create.add(state_dir) + state_subvol = os.path.join('/state', state_dir) + fstab.add_line( + '%s /%s btrfs subvol=%s,defaults,rw,noatime 0 2' % + (root_device, state_dir, state_subvol)) + + fstab.write() + return state_dirs_to_create + + def find_initramfs(self, temp_root): + '''Check whether the rootfs has an initramfs. + + Uses the INITRAMFS_PATH option to locate it. + ''' + if 'INITRAMFS_PATH' in os.environ: + initramfs = os.path.join(temp_root, os.environ['INITRAMFS_PATH']) + if not os.path.exists(initramfs): + raise morphlib.Error('INITRAMFS_PATH specified, ' + 'but file does not exist') + return initramfs + return None + + def install_initramfs(self, initramfs_path, version_root): + '''Install the initramfs outside of 'orig' or 'run' subvolumes. + + This is required because syslinux doesn't traverse subvolumes when + loading the kernel or initramfs. + ''' + self.status(msg='Installing initramfs') + initramfs_dest = os.path.join(version_root, 'initramfs') + cliapp.runcmd(['cp', '-a', initramfs_path, initramfs_dest]) + + def install_kernel(self, version_root, temp_root): + '''Install the kernel outside of 'orig' or 'run' subvolumes''' + + self.status(msg='Installing kernel') + image_names = ['vmlinuz', 'zImage', 'uImage'] + kernel_dest = os.path.join(version_root, 'kernel') + for name in image_names: + try_path = os.path.join(temp_root, 'boot', name) + if os.path.exists(try_path): + cliapp.runcmd(['cp', '-a', try_path, kernel_dest]) + break + + def install_dtb(self, version_root, temp_root): + '''Install the device tree outside of 'orig' or 'run' subvolumes''' + + self.status(msg='Installing devicetree') + device_tree_path = self.get_dtb_path() + dtb_dest = os.path.join(version_root, 'dtb') + try_path = os.path.join(temp_root, device_tree_path) + if os.path.exists(try_path): + cliapp.runcmd(['cp', '-a', try_path, dtb_dest]) + else: + logging.error("Failed to find device tree %s", device_tree_path) + raise cliapp.AppException( + 'Failed to find device tree %s' % device_tree_path) + + def get_dtb_path(self): + return os.environ.get('DTB_PATH', '') + + def get_bootloader_install(self): + # Do we actually want to install the bootloader? + # Set this to "none" to prevent the install + return os.environ.get('BOOTLOADER_INSTALL', 'extlinux') + + def get_bootloader_config_format(self): + # The config format for the bootloader, + # if not set we default to extlinux for x86 + return os.environ.get('BOOTLOADER_CONFIG_FORMAT', 'extlinux') + + def get_extra_kernel_args(self): + return os.environ.get('KERNEL_ARGS', '') + + def get_root_device(self): + return os.environ.get('ROOT_DEVICE', '/dev/sda') + + def generate_bootloader_config(self, real_root, disk_uuid=None): + '''Install extlinux on the newly created disk image.''' + config_function_dict = { + 'extlinux': self.generate_extlinux_config, + } + + config_type = self.get_bootloader_config_format() + if config_type in config_function_dict: + config_function_dict[config_type](real_root, disk_uuid) + else: + raise cliapp.AppException( + 'Invalid BOOTLOADER_CONFIG_FORMAT %s' % config_type) + + def generate_extlinux_config(self, real_root, disk_uuid=None): + '''Install extlinux on the newly created disk image.''' + + self.status(msg='Creating extlinux.conf') + config = os.path.join(real_root, 'extlinux.conf') + + ''' Please also update the documentation in the following files + if you change these default kernel args: + - kvm.write.help + - rawdisk.write.help + - virtualbox-ssh.write.help ''' + kernel_args = ( + 'rw ' # ro ought to work, but we don't test that regularly + 'init=/sbin/init ' # default, but it doesn't hurt to be explicit + 'rootfstype=btrfs ' # required when using initramfs, also boots + # faster when specified without initramfs + 'rootflags=subvol=systems/default/run ') # boot runtime subvol + kernel_args += 'root=%s ' % (self.get_root_device() + if disk_uuid is None + else 'UUID=%s' % disk_uuid) + kernel_args += self.get_extra_kernel_args() + with open(config, 'w') as f: + f.write('default linux\n') + f.write('timeout 1\n') + f.write('label linux\n') + f.write('kernel /systems/default/kernel\n') + if disk_uuid is not None: + f.write('initrd /systems/default/initramfs\n') + if self.get_dtb_path() != '': + f.write('devicetree /systems/default/dtb\n') + f.write('append %s\n' % kernel_args) + + def install_bootloader(self, real_root): + install_function_dict = { + 'extlinux': self.install_bootloader_extlinux, + } + + install_type = self.get_bootloader_install() + if install_type in install_function_dict: + install_function_dict[install_type](real_root) + elif install_type != 'none': + raise cliapp.AppException( + 'Invalid BOOTLOADER_INSTALL %s' % install_type) + + def install_bootloader_extlinux(self, real_root): + self.status(msg='Installing extlinux') + cliapp.runcmd(['extlinux', '--install', real_root]) + + # FIXME this hack seems to be necessary to let extlinux finish + cliapp.runcmd(['sync']) + time.sleep(2) + + def install_syslinux_menu(self, real_root, version_root): + '''Make syslinux/extlinux menu binary available. + + The syslinux boot menu is compiled to a file named menu.c32. Extlinux + searches a few places for this file but it does not know to look inside + our subvolume, so we copy it to the filesystem root. + + If the file is not available, the bootloader will still work but will + not be able to show a menu. + + ''' + menu_file = os.path.join(version_root, 'orig', + 'usr', 'share', 'syslinux', 'menu.c32') + if os.path.isfile(menu_file): + self.status(msg='Copying menu.c32') + shutil.copy(menu_file, real_root) + + def parse_attach_disks(self): + '''Parse $ATTACH_DISKS into list of disks to attach.''' + + if 'ATTACH_DISKS' in os.environ: + s = os.environ['ATTACH_DISKS'] + return s.split(':') + else: + return [] + + def bootloader_config_is_wanted(self): + '''Does the user want to generate a bootloader config? + + The user may set $BOOTLOADER_CONFIG_FORMAT to the desired + format. 'extlinux' is the only allowed value, and is the default + value for x86-32 and x86-64. + + ''' + + def is_x86(arch): + return (arch == 'x86_64' or + (arch.startswith('i') and arch.endswith('86'))) + + value = os.environ.get('BOOTLOADER_CONFIG_FORMAT', '') + if value == '': + if not is_x86(os.uname()[-1]): + return False + + return True + + def get_environment_boolean(self, variable): + '''Parse a yes/no boolean passed through the environment.''' + + value = os.environ.get(variable, 'no').lower() + if value in ['no', '0', 'false']: + return False + elif value in ['yes', '1', 'true']: + return True + else: + raise cliapp.AppException('Unexpected value for %s: %s' % + (variable, value)) + + def check_ssh_connectivity(self, ssh_host): + try: + output = cliapp.ssh_runcmd(ssh_host, ['echo', 'test']) + except cliapp.AppException as e: + logging.error("Error checking SSH connectivity: %s", str(e)) + raise cliapp.AppException( + 'Unable to SSH to %s: %s' % (ssh_host, e)) + + if output.strip() != 'test': + raise cliapp.AppException( + 'Unexpected output from remote machine: %s' % output.strip()) + + def is_device(self, location): + try: + st = os.stat(location) + return stat.S_ISBLK(st.st_mode) + except OSError as e: + if e.errno == errno.ENOENT: + return False + raise diff --git a/writeexts.py b/writeexts.py deleted file mode 100644 index aa185a2b..00000000 --- a/writeexts.py +++ /dev/null @@ -1,624 +0,0 @@ -# Copyright (C) 2012-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . - - -import cliapp -import logging -import os -import re -import shutil -import sys -import time -import tempfile -import errno -import stat -import contextlib - -import morphlib - - -class Fstab(object): - '''Small helper class for parsing and adding lines to /etc/fstab.''' - - # There is an existing Python helper library for editing of /etc/fstab. - # However it is unmaintained and has an incompatible license (GPL3). - # - # https://code.launchpad.net/~computer-janitor-hackers/python-fstab/trunk - - def __init__(self, filepath='/etc/fstab'): - if os.path.exists(filepath): - with open(filepath, 'r') as f: - self.text= f.read() - else: - self.text = '' - self.filepath = filepath - self.lines_added = 0 - - def get_mounts(self): - '''Return list of mount devices and targets in /etc/fstab. - - Return value is a dict of target -> device. - ''' - mounts = dict() - for line in self.text.splitlines(): - words = line.split() - if len(words) >= 2 and not words[0].startswith('#'): - device, target = words[0:2] - mounts[target] = device - return mounts - - def add_line(self, line): - '''Add a new entry to /etc/fstab. - - Lines are appended, and separated from any entries made by configure - extensions with a comment. - - ''' - if self.lines_added == 0: - if len(self.text) == 0 or self.text[-1] is not '\n': - self.text += '\n' - self.text += '# Morph default system layout\n' - self.lines_added += 1 - - self.text += line + '\n' - - def write(self): - '''Rewrite the fstab file to include all new entries.''' - with morphlib.savefile.SaveFile(self.filepath, 'w') as f: - f.write(self.text) - - -class WriteExtension(cliapp.Application): - - '''A base class for deployment write extensions. - - A subclass should subclass this class, and add a - ``process_args`` method. - - Note that it is not necessary to subclass this class for write - extensions. This class is here just to collect common code for - write extensions. - - ''' - - def setup_logging(self): - '''Direct all logging output to MORPH_LOG_FD, if set. - - This file descriptor is read by Morph and written into its own log - file. - - This overrides cliapp's usual configurable logging setup. - - ''' - log_write_fd = int(os.environ.get('MORPH_LOG_FD', 0)) - - if log_write_fd == 0: - return - - formatter = logging.Formatter('%(message)s') - - handler = logging.StreamHandler(os.fdopen(log_write_fd, 'w')) - handler.setFormatter(formatter) - - logger = logging.getLogger() - logger.addHandler(handler) - logger.setLevel(logging.DEBUG) - - def log_config(self): - with morphlib.util.hide_password_environment_variables(os.environ): - cliapp.Application.log_config(self) - - def process_args(self, args): - raise NotImplementedError() - - def status(self, **kwargs): - '''Provide status output. - - The ``msg`` keyword argument is the actual message, - the rest are values for fields in the message as interpolated - by %. - - ''' - - self.output.write('%s\n' % (kwargs['msg'] % kwargs)) - self.output.flush() - - def check_for_btrfs_in_deployment_host_kernel(self): - with open('/proc/filesystems') as f: - text = f.read() - return '\tbtrfs\n' in text - - def require_btrfs_in_deployment_host_kernel(self): - if not self.check_for_btrfs_in_deployment_host_kernel(): - raise cliapp.AppException( - 'Error: Btrfs is required for this deployment, but was not ' - 'detected in the kernel of the machine that is running Morph.') - - def create_local_system(self, temp_root, raw_disk): - '''Create a raw system image locally.''' - - with self.created_disk_image(raw_disk): - self.format_btrfs(raw_disk) - self.create_system(temp_root, raw_disk) - - @contextlib.contextmanager - def created_disk_image(self, location): - size = self.get_disk_size() - if not size: - raise cliapp.AppException('DISK_SIZE is not defined') - self.create_raw_disk_image(location, size) - try: - yield - except BaseException: - os.unlink(location) - raise - - def format_btrfs(self, raw_disk): - try: - self.mkfs_btrfs(raw_disk) - except BaseException: - sys.stderr.write('Error creating disk image') - raise - - def create_system(self, temp_root, raw_disk): - with self.mount(raw_disk) as mp: - try: - self.create_btrfs_system_layout( - temp_root, mp, version_label='factory', - disk_uuid=self.get_uuid(raw_disk)) - except BaseException as e: - sys.stderr.write('Error creating Btrfs system layout') - raise - - def _parse_size(self, size): - '''Parse a size from a string. - - Return size in bytes. - - ''' - - m = re.match('^(\d+)([kmgKMG]?)$', size) - if not m: - return None - - factors = { - '': 1, - 'k': 1024, - 'm': 1024**2, - 'g': 1024**3, - } - factor = factors[m.group(2).lower()] - - return int(m.group(1)) * factor - - def _parse_size_from_environment(self, env_var, default): - '''Parse a size from an environment variable.''' - - size = os.environ.get(env_var, default) - if size is None: - return None - bytes = self._parse_size(size) - if bytes is None: - raise morphlib.Error('Cannot parse %s value %s' % (env_var, size)) - return bytes - - def get_disk_size(self): - '''Parse disk size from environment.''' - return self._parse_size_from_environment('DISK_SIZE', None) - - def get_ram_size(self): - '''Parse RAM size from environment.''' - return self._parse_size_from_environment('RAM_SIZE', '1G') - - def get_vcpu_count(self): - '''Parse the virtual cpu count from environment.''' - return self._parse_size_from_environment('VCPUS', '1') - - def create_raw_disk_image(self, filename, size): - '''Create a raw disk image.''' - - self.status(msg='Creating empty disk image') - with open(filename, 'wb') as f: - if size > 0: - f.seek(size-1) - f.write('\0') - - def mkfs_btrfs(self, location): - '''Create a btrfs filesystem on the disk.''' - - self.status(msg='Creating btrfs filesystem') - try: - # The following command disables some new filesystem features. We - # need to do this because at the time of writing, SYSLINUX has not - # been updated to understand these new features and will fail to - # boot if the kernel is on a filesystem where they are enabled. - cliapp.runcmd( - ['mkfs.btrfs','-f', '-L', 'baserock', - '--features', '^extref', - '--features', '^skinny-metadata', - '--features', '^mixed-bg', - '--nodesize', '4096', - location]) - except cliapp.AppException as e: - if 'unrecognized option \'--features\'' in e.msg: - # Old versions of mkfs.btrfs (including v0.20, present in many - # Baserock releases) don't support the --features option, but - # also don't enable the new features by default. So we can - # still create a bootable system in this situation. - logging.debug( - 'Assuming mkfs.btrfs failure was because the tool is too ' - 'old to have --features flag.') - cliapp.runcmd(['mkfs.btrfs','-f', '-L', 'baserock', location]) - else: - raise - - def get_uuid(self, location): - '''Get the UUID of a block device's file system.''' - # Requires util-linux blkid; busybox one ignores options and - # lies by exiting successfully. - return cliapp.runcmd(['blkid', '-s', 'UUID', '-o', 'value', - location]).strip() - - @contextlib.contextmanager - def mount(self, location): - self.status(msg='Mounting filesystem') - try: - mount_point = tempfile.mkdtemp() - if self.is_device(location): - cliapp.runcmd(['mount', location, mount_point]) - else: - cliapp.runcmd(['mount', '-o', 'loop', location, mount_point]) - except BaseException as e: - sys.stderr.write('Error mounting filesystem') - os.rmdir(mount_point) - raise - try: - yield mount_point - finally: - self.status(msg='Unmounting filesystem') - cliapp.runcmd(['umount', mount_point]) - os.rmdir(mount_point) - - def create_btrfs_system_layout(self, temp_root, mountpoint, version_label, - disk_uuid): - '''Separate base OS versions from state using subvolumes. - - ''' - initramfs = self.find_initramfs(temp_root) - version_root = os.path.join(mountpoint, 'systems', version_label) - state_root = os.path.join(mountpoint, 'state') - - os.makedirs(version_root) - os.makedirs(state_root) - - self.create_orig(version_root, temp_root) - system_dir = os.path.join(version_root, 'orig') - - state_dirs = self.complete_fstab_for_btrfs_layout(system_dir, - disk_uuid) - - for state_dir in state_dirs: - self.create_state_subvolume(system_dir, mountpoint, state_dir) - - self.create_run(version_root) - - os.symlink( - version_label, os.path.join(mountpoint, 'systems', 'default')) - - if self.bootloader_config_is_wanted(): - self.install_kernel(version_root, temp_root) - if self.get_dtb_path() != '': - self.install_dtb(version_root, temp_root) - self.install_syslinux_menu(mountpoint, version_root) - if initramfs is not None: - self.install_initramfs(initramfs, version_root) - self.generate_bootloader_config(mountpoint, disk_uuid) - else: - self.generate_bootloader_config(mountpoint) - self.install_bootloader(mountpoint) - - def create_orig(self, version_root, temp_root): - '''Create the default "factory" system.''' - - orig = os.path.join(version_root, 'orig') - - self.status(msg='Creating orig subvolume') - cliapp.runcmd(['btrfs', 'subvolume', 'create', orig]) - self.status(msg='Copying files to orig subvolume') - cliapp.runcmd(['cp', '-a', temp_root + '/.', orig + '/.']) - - def create_run(self, version_root): - '''Create the 'run' snapshot.''' - - self.status(msg='Creating run subvolume') - orig = os.path.join(version_root, 'orig') - run = os.path.join(version_root, 'run') - cliapp.runcmd( - ['btrfs', 'subvolume', 'snapshot', orig, run]) - - def create_state_subvolume(self, system_dir, mountpoint, state_subdir): - '''Create a shared state subvolume. - - We need to move any files added to the temporary rootfs by the - configure extensions to their correct home. For example, they might - have added keys in `/root/.ssh` which we now need to transfer to - `/state/root/.ssh`. - - ''' - self.status(msg='Creating %s subvolume' % state_subdir) - subvolume = os.path.join(mountpoint, 'state', state_subdir) - cliapp.runcmd(['btrfs', 'subvolume', 'create', subvolume]) - os.chmod(subvolume, 0o755) - - existing_state_dir = os.path.join(system_dir, state_subdir) - files = [] - if os.path.exists(existing_state_dir): - files = os.listdir(existing_state_dir) - if len(files) > 0: - self.status(msg='Moving existing data to %s subvolume' % subvolume) - for filename in files: - filepath = os.path.join(existing_state_dir, filename) - cliapp.runcmd(['mv', filepath, subvolume]) - - def complete_fstab_for_btrfs_layout(self, system_dir, rootfs_uuid=None): - '''Fill in /etc/fstab entries for the default Btrfs disk layout. - - In the future we should move this code out of the write extension and - in to a configure extension. To do that, though, we need some way of - informing the configure extension what layout should be used. Right now - a configure extension doesn't know if the system is going to end up as - a Btrfs disk image, a tarfile or something else and so it can't come - up with a sensible default fstab. - - Configuration extensions can already create any /etc/fstab that they - like. This function only fills in entries that are missing, so if for - example the user configured /home to be on a separate partition, that - decision will be honoured and /state/home will not be created. - - ''' - shared_state_dirs = {'home', 'root', 'opt', 'srv', 'var'} - - fstab = Fstab(os.path.join(system_dir, 'etc', 'fstab')) - existing_mounts = fstab.get_mounts() - - if '/' in existing_mounts: - root_device = existing_mounts['/'] - else: - root_device = (self.get_root_device() if rootfs_uuid is None else - 'UUID=%s' % rootfs_uuid) - fstab.add_line('%s / btrfs defaults,rw,noatime 0 1' % root_device) - - state_dirs_to_create = set() - for state_dir in shared_state_dirs: - if '/' + state_dir not in existing_mounts: - state_dirs_to_create.add(state_dir) - state_subvol = os.path.join('/state', state_dir) - fstab.add_line( - '%s /%s btrfs subvol=%s,defaults,rw,noatime 0 2' % - (root_device, state_dir, state_subvol)) - - fstab.write() - return state_dirs_to_create - - def find_initramfs(self, temp_root): - '''Check whether the rootfs has an initramfs. - - Uses the INITRAMFS_PATH option to locate it. - ''' - if 'INITRAMFS_PATH' in os.environ: - initramfs = os.path.join(temp_root, os.environ['INITRAMFS_PATH']) - if not os.path.exists(initramfs): - raise morphlib.Error('INITRAMFS_PATH specified, ' - 'but file does not exist') - return initramfs - return None - - def install_initramfs(self, initramfs_path, version_root): - '''Install the initramfs outside of 'orig' or 'run' subvolumes. - - This is required because syslinux doesn't traverse subvolumes when - loading the kernel or initramfs. - ''' - self.status(msg='Installing initramfs') - initramfs_dest = os.path.join(version_root, 'initramfs') - cliapp.runcmd(['cp', '-a', initramfs_path, initramfs_dest]) - - def install_kernel(self, version_root, temp_root): - '''Install the kernel outside of 'orig' or 'run' subvolumes''' - - self.status(msg='Installing kernel') - image_names = ['vmlinuz', 'zImage', 'uImage'] - kernel_dest = os.path.join(version_root, 'kernel') - for name in image_names: - try_path = os.path.join(temp_root, 'boot', name) - if os.path.exists(try_path): - cliapp.runcmd(['cp', '-a', try_path, kernel_dest]) - break - - def install_dtb(self, version_root, temp_root): - '''Install the device tree outside of 'orig' or 'run' subvolumes''' - - self.status(msg='Installing devicetree') - device_tree_path = self.get_dtb_path() - dtb_dest = os.path.join(version_root, 'dtb') - try_path = os.path.join(temp_root, device_tree_path) - if os.path.exists(try_path): - cliapp.runcmd(['cp', '-a', try_path, dtb_dest]) - else: - logging.error("Failed to find device tree %s", device_tree_path) - raise cliapp.AppException( - 'Failed to find device tree %s' % device_tree_path) - - def get_dtb_path(self): - return os.environ.get('DTB_PATH', '') - - def get_bootloader_install(self): - # Do we actually want to install the bootloader? - # Set this to "none" to prevent the install - return os.environ.get('BOOTLOADER_INSTALL', 'extlinux') - - def get_bootloader_config_format(self): - # The config format for the bootloader, - # if not set we default to extlinux for x86 - return os.environ.get('BOOTLOADER_CONFIG_FORMAT', 'extlinux') - - def get_extra_kernel_args(self): - return os.environ.get('KERNEL_ARGS', '') - - def get_root_device(self): - return os.environ.get('ROOT_DEVICE', '/dev/sda') - - def generate_bootloader_config(self, real_root, disk_uuid=None): - '''Install extlinux on the newly created disk image.''' - config_function_dict = { - 'extlinux': self.generate_extlinux_config, - } - - config_type = self.get_bootloader_config_format() - if config_type in config_function_dict: - config_function_dict[config_type](real_root, disk_uuid) - else: - raise cliapp.AppException( - 'Invalid BOOTLOADER_CONFIG_FORMAT %s' % config_type) - - def generate_extlinux_config(self, real_root, disk_uuid=None): - '''Install extlinux on the newly created disk image.''' - - self.status(msg='Creating extlinux.conf') - config = os.path.join(real_root, 'extlinux.conf') - - ''' Please also update the documentation in the following files - if you change these default kernel args: - - kvm.write.help - - rawdisk.write.help - - virtualbox-ssh.write.help ''' - kernel_args = ( - 'rw ' # ro ought to work, but we don't test that regularly - 'init=/sbin/init ' # default, but it doesn't hurt to be explicit - 'rootfstype=btrfs ' # required when using initramfs, also boots - # faster when specified without initramfs - 'rootflags=subvol=systems/default/run ') # boot runtime subvol - kernel_args += 'root=%s ' % (self.get_root_device() - if disk_uuid is None - else 'UUID=%s' % disk_uuid) - kernel_args += self.get_extra_kernel_args() - with open(config, 'w') as f: - f.write('default linux\n') - f.write('timeout 1\n') - f.write('label linux\n') - f.write('kernel /systems/default/kernel\n') - if disk_uuid is not None: - f.write('initrd /systems/default/initramfs\n') - if self.get_dtb_path() != '': - f.write('devicetree /systems/default/dtb\n') - f.write('append %s\n' % kernel_args) - - def install_bootloader(self, real_root): - install_function_dict = { - 'extlinux': self.install_bootloader_extlinux, - } - - install_type = self.get_bootloader_install() - if install_type in install_function_dict: - install_function_dict[install_type](real_root) - elif install_type != 'none': - raise cliapp.AppException( - 'Invalid BOOTLOADER_INSTALL %s' % install_type) - - def install_bootloader_extlinux(self, real_root): - self.status(msg='Installing extlinux') - cliapp.runcmd(['extlinux', '--install', real_root]) - - # FIXME this hack seems to be necessary to let extlinux finish - cliapp.runcmd(['sync']) - time.sleep(2) - - def install_syslinux_menu(self, real_root, version_root): - '''Make syslinux/extlinux menu binary available. - - The syslinux boot menu is compiled to a file named menu.c32. Extlinux - searches a few places for this file but it does not know to look inside - our subvolume, so we copy it to the filesystem root. - - If the file is not available, the bootloader will still work but will - not be able to show a menu. - - ''' - menu_file = os.path.join(version_root, 'orig', - 'usr', 'share', 'syslinux', 'menu.c32') - if os.path.isfile(menu_file): - self.status(msg='Copying menu.c32') - shutil.copy(menu_file, real_root) - - def parse_attach_disks(self): - '''Parse $ATTACH_DISKS into list of disks to attach.''' - - if 'ATTACH_DISKS' in os.environ: - s = os.environ['ATTACH_DISKS'] - return s.split(':') - else: - return [] - - def bootloader_config_is_wanted(self): - '''Does the user want to generate a bootloader config? - - The user may set $BOOTLOADER_CONFIG_FORMAT to the desired - format. 'extlinux' is the only allowed value, and is the default - value for x86-32 and x86-64. - - ''' - - def is_x86(arch): - return (arch == 'x86_64' or - (arch.startswith('i') and arch.endswith('86'))) - - value = os.environ.get('BOOTLOADER_CONFIG_FORMAT', '') - if value == '': - if not is_x86(os.uname()[-1]): - return False - - return True - - def get_environment_boolean(self, variable): - '''Parse a yes/no boolean passed through the environment.''' - - value = os.environ.get(variable, 'no').lower() - if value in ['no', '0', 'false']: - return False - elif value in ['yes', '1', 'true']: - return True - else: - raise cliapp.AppException('Unexpected value for %s: %s' % - (variable, value)) - - def check_ssh_connectivity(self, ssh_host): - try: - output = cliapp.ssh_runcmd(ssh_host, ['echo', 'test']) - except cliapp.AppException as e: - logging.error("Error checking SSH connectivity: %s", str(e)) - raise cliapp.AppException( - 'Unable to SSH to %s: %s' % (ssh_host, e)) - - if output.strip() != 'test': - raise cliapp.AppException( - 'Unexpected output from remote machine: %s' % output.strip()) - - def is_device(self, location): - try: - st = os.stat(location) - return stat.S_ISBLK(st.st_mode) - except OSError as e: - if e.errno == errno.ENOENT: - return False - raise -- cgit v1.2.1 From d3c9b459d837f62532818be7f64c3b277a3e5885 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Wed, 10 Jun 2015 09:59:07 +0000 Subject: Stop writeexts.py depending on morphlib Change-Id: I7f3702e80678aeee89dd22116510a6d8d7e04841 --- extensions/writeexts.py | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/extensions/writeexts.py b/extensions/writeexts.py index aa185a2b..9357648f 100644 --- a/extensions/writeexts.py +++ b/extensions/writeexts.py @@ -25,7 +25,15 @@ import errno import stat import contextlib -import morphlib + +@contextlib.contextmanager +def hide_password_environment_variables(env): # pragma: no cover + password_env = { k:v for k,v in env.iteritems() if 'PASSWORD' in k } + for k in password_env: + env[k] = '(value hidden)' + yield + for k, v in password_env.iteritems(): + env[k] = v class Fstab(object): @@ -75,8 +83,10 @@ class Fstab(object): def write(self): '''Rewrite the fstab file to include all new entries.''' - with morphlib.savefile.SaveFile(self.filepath, 'w') as f: + with tempfile.NamedTemporaryFile(delete=False) as f: f.write(self.text) + tmp = f.name + shutil.move(os.path.abspath(tmp), os.path.abspath(self.filepath)) class WriteExtension(cliapp.Application): @@ -116,7 +126,7 @@ class WriteExtension(cliapp.Application): logger.setLevel(logging.DEBUG) def log_config(self): - with morphlib.util.hide_password_environment_variables(os.environ): + with hide_password_environment_variables(os.environ): cliapp.Application.log_config(self) def process_args(self, args): @@ -210,7 +220,8 @@ class WriteExtension(cliapp.Application): return None bytes = self._parse_size(size) if bytes is None: - raise morphlib.Error('Cannot parse %s value %s' % (env_var, size)) + raise cliapp.AppException('Cannot parse %s value %s' + % (env_var, size)) return bytes def get_disk_size(self): @@ -419,8 +430,8 @@ class WriteExtension(cliapp.Application): if 'INITRAMFS_PATH' in os.environ: initramfs = os.path.join(temp_root, os.environ['INITRAMFS_PATH']) if not os.path.exists(initramfs): - raise morphlib.Error('INITRAMFS_PATH specified, ' - 'but file does not exist') + raise cliapp.AppException('INITRAMFS_PATH specified, ' + 'but file does not exist') return initramfs return None -- cgit v1.2.1 From 840292841f4495a79a037f81a26d6b3f51e7cb8c Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Thu, 4 Jun 2015 15:25:33 +0000 Subject: Move xfer-hole and recv-hole into extensions/ Change-Id: I46ed5f3ec85f9662bebac592eff7a6eb6d628f28 --- extensions/recv-hole | 158 +++++++++++++++++++++++++++++++++++++++++++++++++++ extensions/xfer-hole | 137 ++++++++++++++++++++++++++++++++++++++++++++ recv-hole | 158 --------------------------------------------------- xfer-hole | 137 -------------------------------------------- 4 files changed, 295 insertions(+), 295 deletions(-) create mode 100755 extensions/recv-hole create mode 100755 extensions/xfer-hole delete mode 100755 recv-hole delete mode 100755 xfer-hole diff --git a/extensions/recv-hole b/extensions/recv-hole new file mode 100755 index 00000000..fe69f304 --- /dev/null +++ b/extensions/recv-hole @@ -0,0 +1,158 @@ +#!/bin/sh +# +# Copyright (C) 2014-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# =*= License: GPL-2 =*= + + +# Receive a data stream describing a sparse file, and reproduce it, +# either to a named file or stdout. +# +# The data stream is simple: it's a sequence of DATA or HOLE records: +# +# DATA +# 123 +# <123 bytes of binary data, NOT including newline at the end> +# +# HOLE +# 123 +# +# This shell script can be executed over ssh (given to ssh as an arguemnt, +# with suitable escaping) on a different computer. This allows a large +# sparse file (e.g., disk image) be transferred quickly. +# +# This script should be called in one of the following ways: +# +# recv-hole file FILENAME +# recv-hole vbox FILENAME DISKSIZE +# +# In both cases, FILENAME is the pathname of the disk image on the +# receiving end. DISKSIZE is the size of the disk image in bytes. The +# first form is used when transferring a disk image to become an +# identical file on the receiving end. +# +# The second form is used when the disk image should be converted for +# use by VirtualBox. In this case, we want to avoid writing a +# temporary file on disk, and then calling the VirtualBox VBoxManage +# tool to do the conversion, since that would involve large amounts of +# unnecessary I/O and disk usage. Instead we pipe the file directly to +# VBoxManage, avoiding those issues. The piping is done here in this +# script, instead of in the caller, to make it easier to run things +# over ssh. +# +# However, since it's not possible seek in a Unix pipe, we have to +# explicitly write the zeroes into the pipe. This is not +# super-efficient, but the way to avoid that would be to avoid sending +# a sparse file, and do the conversion to a VDI on the sending end. +# That is out of scope for xfer-hole and recv-hole. + + +set -eu + + +die() +{ + echo "$@" 1>&2 + exit 1 +} + + +recv_hole_to_file() +{ + local n + + read n + truncate --size "+$n" "$1" +} + + +recv_data_to_file() +{ + local n + read n + + local blocksize=1048576 + local blocks=$(($n / $blocksize)) + local extra=$(($n % $blocksize)) + + xfer_data_to_stdout "$blocksize" "$blocks" >> "$1" + xfer_data_to_stdout 1 "$extra" >> "$1" +} + + +recv_hole_to_stdout() +{ + local n + read n + (echo "$n"; cat /dev/zero) | recv_data_to_stdout +} + + +recv_data_to_stdout() +{ + local n + read n + + local blocksize=1048576 + local blocks=$(($n / $blocksize)) + local extra=$(($n % $blocksize)) + + xfer_data_to_stdout "$blocksize" "$blocks" + xfer_data_to_stdout 1 "$extra" +} + + +xfer_data_to_stdout() +{ + local log="$(mktemp)" + if ! dd "bs=$1" count="$2" iflag=fullblock status=noxfer 2> "$log" + then + cat "$log" 1>&2 + rm -f "$log" + exit 1 + else + rm -f "$log" + fi +} + + +type="$1" +case "$type" in + file) + output="$2" + truncate --size=0 "$output" + while read what + do + case "$what" in + DATA) recv_data_to_file "$output" ;; + HOLE) recv_hole_to_file "$output" ;; + *) die "Unknown instruction: $what" ;; + esac + done + ;; + vbox) + output="$2" + disk_size="$3" + while read what + do + case "$what" in + DATA) recv_data_to_stdout ;; + HOLE) recv_hole_to_stdout ;; + *) die "Unknown instruction: $what" ;; + esac + done | + VBoxManage convertfromraw stdin "$output" "$disk_size" + ;; +esac diff --git a/extensions/xfer-hole b/extensions/xfer-hole new file mode 100755 index 00000000..91f1be01 --- /dev/null +++ b/extensions/xfer-hole @@ -0,0 +1,137 @@ +#!/usr/bin/env python +# +# Send a sparse file more space-efficiently. +# See recv-hole for a description of the protocol. +# +# Note that xfer-hole requires a version of Linux with support for +# SEEK_DATA and SEEK_HOLE. +# +# +# Copyright (C) 2014-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# =*= License: GPL-2 =*= + + + +import errno +import os +import sys + + +SEEK_DATA = 3 +SEEK_HOLE = 4 + + +filename = sys.argv[1] +fd = os.open(filename, os.O_RDONLY) +pos = 0 + + +DATA = 'data' +HOLE = 'hole' +EOF = 'eof' + + +def safe_lseek(fd, pos, whence): + try: + return os.lseek(fd, pos, whence) + except OSError as e: + if e.errno == errno.ENXIO: + return -1 + raise + + +def current_data_or_pos(fd, pos): + length = safe_lseek(fd, 0, os.SEEK_END) + next_data = safe_lseek(fd, pos, SEEK_DATA) + next_hole = safe_lseek(fd, pos, SEEK_HOLE) + + if pos == length: + return EOF, pos + elif pos == next_data: + return DATA, pos + elif pos == next_hole: + return HOLE, pos + else: + assert False, \ + ("Do not understand: pos=%d next_data=%d next_hole=%d" % + (pos, next_data, next_hole)) + + +def next_data_or_hole(fd, pos): + length = safe_lseek(fd, 0, os.SEEK_END) + next_data = safe_lseek(fd, pos, SEEK_DATA) + next_hole = safe_lseek(fd, pos, SEEK_HOLE) + + if pos == length: + return EOF, pos + elif pos == next_data: + # We are at data. + if next_hole == -1 or next_hole == length: + return EOF, length + else: + return HOLE, next_hole + elif pos == next_hole: + # We are at a hole. + if next_data == -1 or next_data == length: + return EOF, length + else: + return DATA, next_data + else: + assert False, \ + ("Do not understand: pos=%d next_data=%d next_hole=%d" % + (pos, next_data, next_hole)) + + +def find_data_and_holes(fd): + pos = safe_lseek(fd, 0, os.SEEK_CUR) + + kind, pos = current_data_or_pos(fd, pos) + while kind != EOF: + yield kind, pos + kind, pos = next_data_or_hole(fd, pos) + yield kind, pos + + +def make_xfer_instructions(fd): + prev_kind = None + prev_pos = None + for kind, pos in find_data_and_holes(fd): + if prev_kind == DATA: + yield (DATA, prev_pos, pos) + elif prev_kind == HOLE: + yield (HOLE, prev_pos, pos) + prev_kind = kind + prev_pos = pos + + +def copy_slice_from_file(to, fd, start, end): + safe_lseek(fd, start, os.SEEK_SET) + nbytes = end - start + max_at_a_time = 1024**2 + while nbytes > 0: + data = os.read(fd, min(nbytes, max_at_a_time)) + if not data: + break + to.write(data) + nbytes -= len(data) + + +for kind, start, end in make_xfer_instructions(fd): + if kind == HOLE: + sys.stdout.write('HOLE\n%d\n' % (end - start)) + elif kind == DATA: + sys.stdout.write('DATA\n%d\n' % (end - start)) + copy_slice_from_file(sys.stdout, fd, start, end) diff --git a/recv-hole b/recv-hole deleted file mode 100755 index fe69f304..00000000 --- a/recv-hole +++ /dev/null @@ -1,158 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . -# -# =*= License: GPL-2 =*= - - -# Receive a data stream describing a sparse file, and reproduce it, -# either to a named file or stdout. -# -# The data stream is simple: it's a sequence of DATA or HOLE records: -# -# DATA -# 123 -# <123 bytes of binary data, NOT including newline at the end> -# -# HOLE -# 123 -# -# This shell script can be executed over ssh (given to ssh as an arguemnt, -# with suitable escaping) on a different computer. This allows a large -# sparse file (e.g., disk image) be transferred quickly. -# -# This script should be called in one of the following ways: -# -# recv-hole file FILENAME -# recv-hole vbox FILENAME DISKSIZE -# -# In both cases, FILENAME is the pathname of the disk image on the -# receiving end. DISKSIZE is the size of the disk image in bytes. The -# first form is used when transferring a disk image to become an -# identical file on the receiving end. -# -# The second form is used when the disk image should be converted for -# use by VirtualBox. In this case, we want to avoid writing a -# temporary file on disk, and then calling the VirtualBox VBoxManage -# tool to do the conversion, since that would involve large amounts of -# unnecessary I/O and disk usage. Instead we pipe the file directly to -# VBoxManage, avoiding those issues. The piping is done here in this -# script, instead of in the caller, to make it easier to run things -# over ssh. -# -# However, since it's not possible seek in a Unix pipe, we have to -# explicitly write the zeroes into the pipe. This is not -# super-efficient, but the way to avoid that would be to avoid sending -# a sparse file, and do the conversion to a VDI on the sending end. -# That is out of scope for xfer-hole and recv-hole. - - -set -eu - - -die() -{ - echo "$@" 1>&2 - exit 1 -} - - -recv_hole_to_file() -{ - local n - - read n - truncate --size "+$n" "$1" -} - - -recv_data_to_file() -{ - local n - read n - - local blocksize=1048576 - local blocks=$(($n / $blocksize)) - local extra=$(($n % $blocksize)) - - xfer_data_to_stdout "$blocksize" "$blocks" >> "$1" - xfer_data_to_stdout 1 "$extra" >> "$1" -} - - -recv_hole_to_stdout() -{ - local n - read n - (echo "$n"; cat /dev/zero) | recv_data_to_stdout -} - - -recv_data_to_stdout() -{ - local n - read n - - local blocksize=1048576 - local blocks=$(($n / $blocksize)) - local extra=$(($n % $blocksize)) - - xfer_data_to_stdout "$blocksize" "$blocks" - xfer_data_to_stdout 1 "$extra" -} - - -xfer_data_to_stdout() -{ - local log="$(mktemp)" - if ! dd "bs=$1" count="$2" iflag=fullblock status=noxfer 2> "$log" - then - cat "$log" 1>&2 - rm -f "$log" - exit 1 - else - rm -f "$log" - fi -} - - -type="$1" -case "$type" in - file) - output="$2" - truncate --size=0 "$output" - while read what - do - case "$what" in - DATA) recv_data_to_file "$output" ;; - HOLE) recv_hole_to_file "$output" ;; - *) die "Unknown instruction: $what" ;; - esac - done - ;; - vbox) - output="$2" - disk_size="$3" - while read what - do - case "$what" in - DATA) recv_data_to_stdout ;; - HOLE) recv_hole_to_stdout ;; - *) die "Unknown instruction: $what" ;; - esac - done | - VBoxManage convertfromraw stdin "$output" "$disk_size" - ;; -esac diff --git a/xfer-hole b/xfer-hole deleted file mode 100755 index 91f1be01..00000000 --- a/xfer-hole +++ /dev/null @@ -1,137 +0,0 @@ -#!/usr/bin/env python -# -# Send a sparse file more space-efficiently. -# See recv-hole for a description of the protocol. -# -# Note that xfer-hole requires a version of Linux with support for -# SEEK_DATA and SEEK_HOLE. -# -# -# Copyright (C) 2014-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . -# -# =*= License: GPL-2 =*= - - - -import errno -import os -import sys - - -SEEK_DATA = 3 -SEEK_HOLE = 4 - - -filename = sys.argv[1] -fd = os.open(filename, os.O_RDONLY) -pos = 0 - - -DATA = 'data' -HOLE = 'hole' -EOF = 'eof' - - -def safe_lseek(fd, pos, whence): - try: - return os.lseek(fd, pos, whence) - except OSError as e: - if e.errno == errno.ENXIO: - return -1 - raise - - -def current_data_or_pos(fd, pos): - length = safe_lseek(fd, 0, os.SEEK_END) - next_data = safe_lseek(fd, pos, SEEK_DATA) - next_hole = safe_lseek(fd, pos, SEEK_HOLE) - - if pos == length: - return EOF, pos - elif pos == next_data: - return DATA, pos - elif pos == next_hole: - return HOLE, pos - else: - assert False, \ - ("Do not understand: pos=%d next_data=%d next_hole=%d" % - (pos, next_data, next_hole)) - - -def next_data_or_hole(fd, pos): - length = safe_lseek(fd, 0, os.SEEK_END) - next_data = safe_lseek(fd, pos, SEEK_DATA) - next_hole = safe_lseek(fd, pos, SEEK_HOLE) - - if pos == length: - return EOF, pos - elif pos == next_data: - # We are at data. - if next_hole == -1 or next_hole == length: - return EOF, length - else: - return HOLE, next_hole - elif pos == next_hole: - # We are at a hole. - if next_data == -1 or next_data == length: - return EOF, length - else: - return DATA, next_data - else: - assert False, \ - ("Do not understand: pos=%d next_data=%d next_hole=%d" % - (pos, next_data, next_hole)) - - -def find_data_and_holes(fd): - pos = safe_lseek(fd, 0, os.SEEK_CUR) - - kind, pos = current_data_or_pos(fd, pos) - while kind != EOF: - yield kind, pos - kind, pos = next_data_or_hole(fd, pos) - yield kind, pos - - -def make_xfer_instructions(fd): - prev_kind = None - prev_pos = None - for kind, pos in find_data_and_holes(fd): - if prev_kind == DATA: - yield (DATA, prev_pos, pos) - elif prev_kind == HOLE: - yield (HOLE, prev_pos, pos) - prev_kind = kind - prev_pos = pos - - -def copy_slice_from_file(to, fd, start, end): - safe_lseek(fd, start, os.SEEK_SET) - nbytes = end - start - max_at_a_time = 1024**2 - while nbytes > 0: - data = os.read(fd, min(nbytes, max_at_a_time)) - if not data: - break - to.write(data) - nbytes -= len(data) - - -for kind, start, end in make_xfer_instructions(fd): - if kind == HOLE: - sys.stdout.write('HOLE\n%d\n' % (end - start)) - elif kind == DATA: - sys.stdout.write('DATA\n%d\n' % (end - start)) - copy_slice_from_file(sys.stdout, fd, start, end) -- cgit v1.2.1 From e4c6b8a69f0df2d0b3beac46865a66e0de527151 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Thu, 4 Jun 2015 15:17:44 +0000 Subject: Remove dependencies on morphlib and cliapp from deployment extensions This is done by either copying some utility functions from morph into writeexts.py, and using the `subprocess` module rather than cliapp's runcmd and ssh_runcmd. Note that this means that these extensions will require "$definitions_checkout/extensions" in PYTHONPATH when they are run. This commit also updates VERSION to 5, since the PYTHONPATH requirement means that this change is incompatible with old versions of morph. Change-Id: Iec6fa7e3c7219619ce55e18493e5c37c36e97816 --- VERSION | 2 +- extensions/ceph.configure | 11 +- extensions/distbuild-trove-nfsboot.check | 52 ++++--- extensions/distbuild-trove-nfsboot.write | 40 ++--- extensions/fstab.configure | 4 +- extensions/hosts.configure | 12 +- extensions/image-package-example/README | 4 +- extensions/install-essential-files.configure | 21 +-- extensions/install-files.configure | 30 ++-- extensions/jffs2.write | 18 +-- extensions/kvm.check | 69 +++++---- extensions/kvm.write | 25 +-- extensions/nfsboot.check | 50 +++--- extensions/nfsboot.write | 83 +++++----- extensions/openstack.check | 26 ++-- extensions/openstack.write | 11 +- extensions/pxeboot.write | 57 +++---- extensions/rawdisk.check | 17 +- extensions/rawdisk.write | 18 +-- extensions/simple-network.configure | 22 +-- extensions/ssh-rsync.check | 26 ++-- extensions/ssh-rsync.write | 51 +++--- extensions/strip-gplv3.configure | 28 ++-- extensions/virtualbox-ssh.check | 10 +- extensions/virtualbox-ssh.write | 42 ++--- extensions/writeexts.py | 223 ++++++++++++++++++++------- 26 files changed, 546 insertions(+), 406 deletions(-) diff --git a/VERSION b/VERSION index 0a70affa..0a94cf8b 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -version: 3 +version: 5 diff --git a/extensions/ceph.configure b/extensions/ceph.configure index c3cd92d1..3b8b2603 100644 --- a/extensions/ceph.configure +++ b/extensions/ceph.configure @@ -14,13 +14,14 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -import cliapp -import sys import os -import subprocess import shutil -import re import stat +import subprocess +import sys +import re + +import writeexts systemd_monitor_template = """ [Unit] @@ -75,7 +76,7 @@ executable_file_permissions = stat.S_IRUSR | stat.S_IXUSR | stat.S_IWUSR | \ stat.S_IXGRP | stat.S_IRGRP | \ stat.S_IXOTH | stat.S_IROTH -class CephConfigurationExtension(cliapp.Application): +class CephConfigurationExtension(writeexts.Extension): """ Set up ceph server daemons. diff --git a/extensions/distbuild-trove-nfsboot.check b/extensions/distbuild-trove-nfsboot.check index 38c491e5..76ba6dda 100755 --- a/extensions/distbuild-trove-nfsboot.check +++ b/extensions/distbuild-trove-nfsboot.check @@ -15,14 +15,15 @@ '''Preparatory checks for Morph 'distbuild-trove-nfsboot' write extension''' -import cliapp import logging import os +import subprocess +import sys -import morphlib.writeexts +import writeexts -class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): +class DistbuildTroveNFSBootCheckExtension(writeexts.WriteExtension): nfsboot_root = '/srv/nfsboot' remote_user = 'root' @@ -45,7 +46,8 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') nfs_host = args[0] nfs_netloc = '%s@%s' % (self.remote_user, nfs_host) @@ -55,17 +57,19 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): missing_vars = [var for var in self.required_vars if not var in os.environ] if missing_vars: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Please set: %s' % ', '.join(missing_vars)) controllers = os.getenv('DISTBUILD_CONTROLLER').split() workers = os.getenv('DISTBUILD_WORKERS').split() if len(controllers) != 1: - raise cliapp.AppException('Please specify exactly one controller.') + raise writeexts.ExtensionError( + 'Please specify exactly one controller.') if len(workers) == 0: - raise cliapp.AppException('Please specify at least one worker.') + raise writeexts.ExtensionError( + 'Please specify at least one worker.') upgrade = self.get_environment_boolean('UPGRADE') @@ -80,7 +84,7 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): if self.remote_directory_exists(nfs_netloc, system_path): if self.get_environment_boolean('OVERWRITE') == False: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'System %s already exists at %s:%s. Try `morph ' 'upgrade` instead of `morph deploy`.' % ( system_name, nfs_netloc, system_path)) @@ -91,27 +95,27 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): # Is an NFS server try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( netloc, ['test', '-e', '/etc/exports']) - except cliapp.AppException: - raise cliapp.AppException('server %s is not an nfs server' - % netloc) + except writeexts.ExtensionError: + raise writeexts.ExtensionError('server %s is not an nfs server' + % netloc) try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( netloc, ['systemctl', 'is-enabled', 'nfs-server.service']) - except cliapp.AppException: - raise cliapp.AppException('server %s does not control its ' - 'nfs server by systemd' % netloc) + except writeexts.ExtensionError: + raise writeexts.ExtensionError('server %s does not control its ' + 'nfs server by systemd' % netloc) # TFTP server exports /srv/nfsboot/tftp tftp_root = os.path.join(self.nfsboot_root, 'tftp') try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( netloc, ['test' , '-d', tftp_root]) - except cliapp.AppException: - raise cliapp.AppException('server %s does not export %s' % - (netloc, tftp_root)) + except writeexts.ExtensionError: + raise writeexts.ExtensionError('server %s does not export %s' % + (netloc, tftp_root)) def check_upgradeable(self, nfs_netloc, system_name, version_label): '''Check that there is already a version of the system present. @@ -124,7 +128,7 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): system_version_path = self.system_path(system_name, version_label) if not self.remote_directory_exists(nfs_netloc, system_path): - raise cliapp.AppException( + raise writeexts.ExtensionError( 'System %s not found at %s:%s, cannot deploy an upgrade.' % ( system_name, nfs_netloc, system_path)) @@ -132,15 +136,15 @@ class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): if self.get_environment_boolean('OVERWRITE'): pass else: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'System %s version %s already exists at %s:%s.' % ( system_name, version_label, nfs_netloc, system_version_path)) def remote_directory_exists(self, nfs_netloc, path): try: - cliapp.ssh_runcmd(nfs_netloc, ['test', '-d', path]) - except cliapp.AppException as e: + writeexts.ssh_runcmd(nfs_netloc, ['test', '-d', path]) + except writeexts.ExtensionError as e: logging.debug('SSH exception: %s', e) return False diff --git a/extensions/distbuild-trove-nfsboot.write b/extensions/distbuild-trove-nfsboot.write index a5a5b094..86291794 100755 --- a/extensions/distbuild-trove-nfsboot.write +++ b/extensions/distbuild-trove-nfsboot.write @@ -20,14 +20,14 @@ import os +import subprocess import sys import tempfile -import cliapp -import morphlib.writeexts +import writeexts -class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): +class DistbuildTroveNFSBootWriteExtension(writeexts.WriteExtension): '''Create an NFS root and kernel on TFTP during Morph's deployment. @@ -54,7 +54,7 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError('Wrong number of command line args') local_system_path, nfs_host = args @@ -111,17 +111,17 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): ''' pairs = host_map_string.split(' ') - return morphlib.util.parse_environment_pairs({}, pairs) + return writeexts.parse_environment_pairs({}, pairs) def transfer_system(self, nfs_netloc, local_system_path, remote_system_path): self.status(msg='Copying rootfs to %(nfs_netloc)s', nfs_netloc=nfs_netloc) - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( nfs_netloc, ['mkdir', '-p', remote_system_path]) # The deployed rootfs may have been created by OSTree, so definitely # don't pass --hard-links to `rsync`. - cliapp.runcmd( + subprocess.check_call( ['rsync', '--archive', '--delete', '--info=progress2', '--protect-args', '--partial', '--sparse', '--xattrs', local_system_path + '/', @@ -131,13 +131,13 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): target_system_path): self.status(msg='Duplicating rootfs to %(target_system_path)s', target_system_path=target_system_path) - cliapp.ssh_runcmd(nfs_netloc, + writeexts.ssh_runcmd(nfs_netloc, ['mkdir', '-p', target_system_path]) # We can't pass --info=progress2 here, because it may not be available # in the remote 'rsync'. The --info setting was added in RSync 3.1.0, # old versions of Baserock have RSync 3.0.9. So the user doesn't get # any progress info on stdout for the 'duplicate' stage. - cliapp.ssh_runcmd(nfs_netloc, + writeexts.ssh_runcmd(nfs_netloc, ['rsync', '--archive', '--delete', '--protect-args', '--partial', '--sparse', '--xattrs', source_system_path + '/', target_system_path], stdout=sys.stdout) @@ -152,7 +152,7 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): kernel_path = os.path.relpath(try_path, local_system_path) break else: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Could not find a kernel in the system: none of ' '%s found' % ', '.join(image_names)) return kernel_path @@ -171,11 +171,11 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): versioned_kernel_name = "%s-%s" % (system_name, version_label) kernel_name = system_name - cliapp.ssh_runcmd(nfs_netloc, + writeexts.ssh_runcmd(nfs_netloc, ['ln', '-f', kernel_dest, os.path.join(tftp_dir, versioned_kernel_name)]) - cliapp.ssh_runcmd(nfs_netloc, + writeexts.ssh_runcmd(nfs_netloc, ['ln', '-sf', versioned_kernel_name, os.path.join(tftp_dir, kernel_name)]) @@ -183,7 +183,7 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): with tempfile.NamedTemporaryFile() as f: f.write(text) f.flush() - cliapp.runcmd( + subprocess.check_call( ['scp', f.name, '%s:%s' % (nfs_netloc, path)]) def set_hostname(self, nfs_netloc, system_name, system_path): @@ -223,9 +223,9 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): '# Generated by distbuild-trove-nfsboot.write\n' + \ config_text + '\n' path = os.path.join(system_path, 'etc', 'distbuild') - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( nfs_netloc, ['mkdir', '-p', path]) - cliapp.runcmd( + subprocess.check_call( ['scp', worker_ssh_key_path, '%s:%s' % (nfs_netloc, path)]) self.set_remote_file_contents( nfs_netloc, os.path.join(path, 'distbuild.conf'), config_text) @@ -244,9 +244,9 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): # Rather ugly SSH hackery follows to ensure each system path is # listed in /etc/exports. try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( nfs_netloc, ['grep', '-q', exported_path, exports_path]) - except cliapp.AppException: + except writeexts.ExtensionError: ip_mask = '*' options = 'rw,no_subtree_check,no_root_squash,async' exports_string = '%s %s(%s)\n' % (exported_path, ip_mask, @@ -259,12 +259,12 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): cat >> "$temp" mv "$temp" "$target" ''' - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( nfs_netloc, ['sh', '-c', exports_append_sh, '--', exports_path], feed_stdin=exports_string) - cliapp.ssh_runcmd(nfs_netloc, + writeexts.ssh_runcmd(nfs_netloc, ['systemctl', 'restart', 'nfs-server.service']) def update_default_version(self, remote_netloc, system_name, @@ -276,7 +276,7 @@ class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): version_label) default_path = os.path.join(system_path, 'systems', 'default') - cliapp.ssh_runcmd(remote_netloc, + writeexts.ssh_runcmd(remote_netloc, ['ln', '-sfn', system_version_path, default_path]) diff --git a/extensions/fstab.configure b/extensions/fstab.configure index b9154eee..3e67b585 100755 --- a/extensions/fstab.configure +++ b/extensions/fstab.configure @@ -20,9 +20,9 @@ import os import sys -import morphlib +import writeexts envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('FSTAB_')} conf_file = os.path.join(sys.argv[1], 'etc/fstab') -morphlib.util.write_from_dict(conf_file, envvars) +writeexts.write_from_dict(conf_file, envvars) diff --git a/extensions/hosts.configure b/extensions/hosts.configure index 6b068d04..11fcf573 100755 --- a/extensions/hosts.configure +++ b/extensions/hosts.configure @@ -22,27 +22,29 @@ import os import sys import socket -import morphlib +import writeexts def validate(var, line): xs = line.split() if len(xs) == 0: - raise morphlib.Error("`%s: %s': line is empty" % (var, line)) + raise writeexts.ExtensionError( + "`%s: %s': line is empty" % (var, line)) ip = xs[0] hostnames = xs[1:] if len(hostnames) == 0: - raise morphlib.Error("`%s: %s': missing hostname" % (var, line)) + raise writeexts.ExtensionError( + "`%s: %s': missing hostname" % (var, line)) family = socket.AF_INET6 if ':' in ip else socket.AF_INET try: socket.inet_pton(family, ip) except socket.error: - raise morphlib.Error("`%s: %s' invalid ip" % (var, ip)) + raise writeexts.ExtensionError("`%s: %s' invalid ip" % (var, ip)) envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('HOSTS_')} conf_file = os.path.join(sys.argv[1], 'etc/hosts') -morphlib.util.write_from_dict(conf_file, envvars, validate) +writeexts.write_from_dict(conf_file, envvars, validate) diff --git a/extensions/image-package-example/README b/extensions/image-package-example/README index c1322f25..f6b66cd9 100644 --- a/extensions/image-package-example/README +++ b/extensions/image-package-example/README @@ -5,5 +5,5 @@ These are scripts used to create disk images or install the system onto an existing disk. This is also implemented independently for the rawdisk.write write -extension; see morphlib.writeexts.WriteExtension.create_local_system() -for a similar, python implementation. +extension; see writeexts.WriteExtension.create_local_system() for +a similar, python implementation. diff --git a/extensions/install-essential-files.configure b/extensions/install-essential-files.configure index bed394df..3d33fe03 100755 --- a/extensions/install-essential-files.configure +++ b/extensions/install-essential-files.configure @@ -22,20 +22,11 @@ to install into the target system. ''' -import subprocess import os +import subprocess +import sys -import cliapp - -class InstallEssentialFilesConfigureExtension(cliapp.Application): - - def process_args(self, args): - target_root = args[0] - os.environ["INSTALL_FILES"] = "install-files/essential-files/manifest" - self.install_essential_files(target_root) - - def install_essential_files(self, target_root): - command = os.path.join("extensions/install-files.configure") - subprocess.check_call([command, target_root]) - -InstallEssentialFilesConfigureExtension().run() +target_root = sys.argv[1] +os.environ["INSTALL_FILES"] = "install-files/essential-files/manifest" +command = os.path.join("extensions/install-files.configure") +subprocess.check_call([command, target_root]) diff --git a/extensions/install-files.configure b/extensions/install-files.configure index 341cce61..64fcecca 100755 --- a/extensions/install-files.configure +++ b/extensions/install-files.configure @@ -22,9 +22,8 @@ to install into the target system. ''' -import cliapp -import os import errno +import os import re import sys import shlex @@ -37,7 +36,9 @@ try: except ImportError: jinja_available = False -class InstallFilesConfigureExtension(cliapp.Application): +import writeexts + +class InstallFilesConfigureExtension(writeexts.Extension): def process_args(self, args): if not 'INSTALL_FILES' in os.environ: @@ -74,7 +75,8 @@ class InstallFilesConfigureExtension(cliapp.Application): gid = int(m.group(5)) path = m.group(6) else: - raise cliapp.AppException('Invalid manifest entry, ' + raise writeexts.ExtensionError( + 'Invalid manifest entry, ' 'format: [template] [overwrite] ' ' ') @@ -85,9 +87,9 @@ class InstallFilesConfigureExtension(cliapp.Application): if (mode != dest_stat.st_mode or uid != dest_stat.st_uid or gid != dest_stat.st_gid): - raise cliapp.AppException('"%s" exists and is not ' - 'identical to directory ' - '"%s"' % (dest_path, entry)) + raise writeexts.ExtensionError( + '"%s" exists and is not identical to directory ' + '"%s"' % (dest_path, entry)) else: os.mkdir(dest_path, mode) os.chown(dest_path, uid, gid) @@ -95,8 +97,8 @@ class InstallFilesConfigureExtension(cliapp.Application): elif stat.S_ISLNK(mode): if os.path.lexists(dest_path) and not overwrite: - raise cliapp.AppException('Symlink already exists at %s' - % dest_path) + raise writeexts.ExtensionError('Symlink already exists at %s' + % dest_path) else: linkdest = os.readlink(os.path.join(manifest_root, './' + path)) @@ -105,12 +107,12 @@ class InstallFilesConfigureExtension(cliapp.Application): elif stat.S_ISREG(mode): if os.path.lexists(dest_path) and not overwrite: - raise cliapp.AppException('File already exists at %s' - % dest_path) + raise writeexts.ExtensionError('File already exists at %s' + % dest_path) else: if template: if not jinja_available: - raise cliapp.AppException( + raise writeexts.ExtensionError( "Failed to install template file `%s': " 'install-files templates require jinja2' % path) @@ -128,7 +130,7 @@ class InstallFilesConfigureExtension(cliapp.Application): os.chmod(dest_path, mode) else: - raise cliapp.AppException('Mode given in "%s" is not a file,' - ' symlink or directory' % entry) + raise writeexts.ExtensionError('Mode given in "%s" is not a file,' + ' symlink or directory' % entry) InstallFilesConfigureExtension().run() diff --git a/extensions/jffs2.write b/extensions/jffs2.write index 46b69a53..ad68204d 100644 --- a/extensions/jffs2.write +++ b/extensions/jffs2.write @@ -19,34 +19,34 @@ as the root filesystem.''' -import cliapp import os +import subprocess -import morphlib.writeexts +import writeexts -class Jffs2WriteExtension(morphlib.writeexts.WriteExtension): +class Jffs2WriteExtension(writeexts.WriteExtension): '''See jffs2.write.help for documentation.''' def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError('Wrong number of command line args') temp_root, location = args try: self.create_jffs2_system(temp_root, location) self.status(msg='Disk image has been created at %(location)s', - location = location) + location=location) except Exception: self.status(msg='Failure to deploy system to %(location)s', - location = location) + location=location) raise def create_jffs2_system(self, temp_root, location): erase_block = self.get_erase_block_size() - cliapp.runcmd( + subprocess.check_call( ['mkfs.jffs2', '--pad', '--no-cleanmarkers', '--eraseblock='+erase_block, '-d', temp_root, '-o', location]) @@ -54,10 +54,10 @@ class Jffs2WriteExtension(morphlib.writeexts.WriteExtension): erase_block = os.environ.get('ERASE_BLOCK', '') if erase_block == '': - raise cliapp.AppException('ERASE_BLOCK was not given') + raise writeexts.ExtensionError('ERASE_BLOCK was not given') if not erase_block.isdigit(): - raise cliapp.AppException('ERASE_BLOCK must be a whole number') + raise writeexts.ExtensionError('ERASE_BLOCK must be a whole number') return erase_block diff --git a/extensions/kvm.check b/extensions/kvm.check index 67cb3d38..3c277156 100755 --- a/extensions/kvm.check +++ b/extensions/kvm.check @@ -15,27 +15,28 @@ '''Preparatory checks for Morph 'kvm' write extension''' -import cliapp import os import re +import subprocess import urlparse -import morphlib.writeexts +import writeexts -class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): +class KvmPlusSshCheckExtension(writeexts.WriteExtension): location_pattern = '^/(?P[^/]+)(?P/.+)$' def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') self.require_btrfs_in_deployment_host_kernel() upgrade = self.get_environment_boolean('UPGRADE') if upgrade: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Use the `ssh-rsync` write extension to deploy upgrades to an ' 'existing remote system.') @@ -55,23 +56,24 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): x = urlparse.urlparse(location) if x.scheme != 'kvm+ssh': - raise cliapp.AppException( + raise writeexts.ExtensionError( 'URL schema must be kvm+ssh in %s' % location) m = re.match(self.location_pattern, x.path) if not m: - raise cliapp.AppException('Cannot parse location %s' % location) + raise writeexts.ExtensionError( + 'Cannot parse location %s' % location) return x.netloc, m.group('guest'), m.group('path') def check_no_existing_libvirt_vm(self, ssh_host, vm_name): try: - cliapp.ssh_runcmd(ssh_host, + writeexts.ssh_runcmd(ssh_host, ['virsh', '--connect', 'qemu:///system', 'domstate', vm_name]) - except cliapp.AppException as e: + except CalledProcessError as e: pass else: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Host %s already has a VM named %s. You can use the ssh-rsync ' 'write extension to deploy upgrades to existing machines.' % (ssh_host, vm_name)) @@ -80,35 +82,35 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): def check_can_write_to_given_path(): try: - cliapp.ssh_runcmd(ssh_host, ['touch', vm_path]) - except cliapp.AppException as e: - raise cliapp.AppException("Can't write to location %s on %s" - % (vm_path, ssh_host)) + writeexts.ssh_runcmd(ssh_host, ['touch', vm_path]) + except writeexts.ExtensionError as e: + raise writeexts.ExtensionError( + "Can't write to location %s on %s" % (vm_path, ssh_host)) else: - cliapp.ssh_runcmd(ssh_host, ['rm', vm_path]) + writeexts.ssh_runcmd(ssh_host, ['rm', vm_path]) try: - cliapp.ssh_runcmd(ssh_host, ['test', '-e', vm_path]) - except cliapp.AppException as e: + writeexts.ssh_runcmd(ssh_host, ['test', '-e', vm_path]) + except writeexts.ExtensionError as e: # vm_path doesn't already exist, so let's test we can write check_can_write_to_given_path() else: - raise cliapp.AppException('%s already exists on %s' - % (vm_path, ssh_host)) + raise writeexts.ExtensionError('%s already exists on %s' + % (vm_path, ssh_host)) def check_extra_disks_exist(self, ssh_host, filename_list): for filename in filename_list: try: - cliapp.ssh_runcmd(ssh_host, ['ls', filename]) - except cliapp.AppException as e: - raise cliapp.AppException('Did not find file %s on host %s' % - (filename, ssh_host)) + writeexts.ssh_runcmd(ssh_host, ['ls', filename]) + except writeexts.ExtensionError as e: + raise writeexts.ExtensionError( + 'Did not find file %s on host %s' % (filename, ssh_host)) def check_virtual_networks_are_started(self, ssh_host): def check_virtual_network_is_started(network_name): cmd = ['virsh', '-c', 'qemu:///system', 'net-info', network_name] - net_info = cliapp.ssh_runcmd(ssh_host, cmd).split('\n') + net_info = writeexts.ssh_runcmd(ssh_host, cmd).split('\n') def pretty_concat(lines): return '\n'.join(['\t%s' % line for line in lines]) @@ -118,15 +120,15 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): if m: break else: - raise cliapp.AppException( + raise writeexts.ExtensionError( "Got unexpected output parsing output of `%s':\n%s" % (' '.join(cmd), pretty_concat(net_info))) network_active = m.group(1) == 'yes' if not network_active: - raise cliapp.AppException("Network '%s' is not started" - % network_name) + raise writeexts.ExtensionError("Network '%s' is not started" + % network_name) def name(nic_entry): if ',' in nic_entry: @@ -142,9 +144,10 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): if not (n.startswith('network=') or n.startswith('bridge=') or n == 'user'): - raise cliapp.AppException('malformed NIC_CONFIG: %s\n' - " (expected 'bridge=BRIDGE' 'network=NAME'" - " or 'user')" % n) + raise writeexts.ExtensionError( + "malformed NIC_CONFIG: %s\n" + " (expected 'bridge=BRIDGE' 'network=NAME'" + " or 'user')" % n) # --network bridge= is used to specify a bridge # --network user is used to specify a form of NAT @@ -159,9 +162,9 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): def check_host_has_virtinstall(self, ssh_host): try: - cliapp.ssh_runcmd(ssh_host, ['which', 'virt-install']) - except cliapp.AppException: - raise cliapp.AppException( + writeexts.ssh_runcmd(ssh_host, ['which', 'virt-install']) + except writeexts.ExtensionError: + raise writeexts.ExtensionError( 'virt-install does not seem to be installed on host %s' % ssh_host) diff --git a/extensions/kvm.write b/extensions/kvm.write index 0d0c095b..2290725e 100755 --- a/extensions/kvm.write +++ b/extensions/kvm.write @@ -21,23 +21,23 @@ See file kvm.write.help for documentation ''' -import cliapp import os import re import sys import tempfile import urlparse -import morphlib.writeexts +import writeexts -class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): +class KvmPlusSshWriteExtension(writeexts.WriteExtension): location_pattern = '^/(?P[^/]+)(?P/.+)$' def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') temp_root, location = args ssh_host, vm_name, vm_path = self.parse_location(location) @@ -53,7 +53,7 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): except BaseException: sys.stderr.write('Error deploying to libvirt') os.remove(raw_disk) - cliapp.ssh_runcmd(ssh_host, ['rm', '-f', vm_path]) + writeexts.ssh_runcmd(ssh_host, ['rm', '-f', vm_path]) raise else: os.remove(raw_disk) @@ -74,16 +74,16 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Transferring disk image') - xfer_hole_path = morphlib.util.get_data_path('xfer-hole') - recv_hole = morphlib.util.get_data('recv-hole') + xfer_hole_path = writeexts.get_data_path('xfer-hole') + recv_hole = writeexts.get_data('recv-hole') ssh_remote_cmd = [ 'sh', '-c', recv_hole, 'dummy-argv0', 'file', vm_path ] - cliapp.runcmd( + subprocess.check_call( ['python', xfer_hole_path, raw_disk], - ['ssh', ssh_host] + map(cliapp.shell_quote, ssh_remote_cmd), + ['ssh', ssh_host] + map(writeexts.shell_quote, ssh_remote_cmd), stdout=None, stderr=None) def create_libvirt_guest(self, ssh_host, vm_name, vm_path, autostart): @@ -111,10 +111,11 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): '--disk', 'path=%s,bus=ide' % vm_path] + attach_opts if not autostart: cmdline += ['--noreboot'] - cliapp.ssh_runcmd(ssh_host, cmdline) + writeexts.ssh_runcmd(ssh_host, cmdline) if autostart: - cliapp.ssh_runcmd(ssh_host, - ['virsh', '--connect', 'qemu:///system', 'autostart', vm_name]) + writeexts.ssh_runcmd(ssh_host, + ['virsh', '--connect', 'qemu:///system', + 'autostart', vm_name]) KvmPlusSshWriteExtension().run() diff --git a/extensions/nfsboot.check b/extensions/nfsboot.check index e273f61c..499fb537 100755 --- a/extensions/nfsboot.check +++ b/extensions/nfsboot.check @@ -15,33 +15,35 @@ '''Preparatory checks for Morph 'nfsboot' write extension''' -import cliapp import os +import subprocess -import morphlib.writeexts +import writeexts -class NFSBootCheckExtension(morphlib.writeexts.WriteExtension): +class NFSBootCheckExtension(writeexts.WriteExtension): _nfsboot_root = '/srv/nfsboot' def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') location = args[0] upgrade = self.get_environment_boolean('UPGRADE') if upgrade: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Upgrading is not currently supported for NFS deployments.') hostname = os.environ.get('HOSTNAME', None) if hostname is None: - raise cliapp.AppException('You must specify a HOSTNAME.') + raise writeexts.ExtensionError('You must specify a HOSTNAME.') if hostname == 'baserock': - raise cliapp.AppException('It is forbidden to nfsboot a system ' - 'with hostname "%s"' % hostname) + raise writeexts.ExtensionError('It is forbidden to nfsboot a ' + 'system with hostname "%s"' + % hostname) self.test_good_server(location) @@ -49,7 +51,7 @@ class NFSBootCheckExtension(morphlib.writeexts.WriteExtension): versioned_root = os.path.join(self._nfsboot_root, hostname, 'systems', version_label) if self.version_exists(versioned_root, location): - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Root file system for host %s (version %s) already exists on ' 'the NFS server %s. Deployment aborted.' % (hostname, version_label, location)) @@ -59,34 +61,34 @@ class NFSBootCheckExtension(morphlib.writeexts.WriteExtension): # Is an NFS server try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( 'root@%s' % server, ['test', '-e', '/etc/exports']) - except cliapp.AppException: - raise cliapp.AppException('server %s is not an nfs server' - % server) + except writeexts.ExtensionError: + raise writeexts.ExtensionError('server %s is not an nfs server' + % server) try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( 'root@%s' % server, ['systemctl', 'is-enabled', 'nfs-server.service']) - except cliapp.AppException: - raise cliapp.AppException('server %s does not control its ' - 'nfs server by systemd' % server) + except writeexts.ExtensionError: + raise writeexts.ExtensionError('server %s does not control its ' + 'nfs server by systemd' % server) # TFTP server exports /srv/nfsboot/tftp tftp_root = os.path.join(self._nfsboot_root, 'tftp') try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( 'root@%s' % server, ['test' , '-d', tftp_root]) - except cliapp.AppException: - raise cliapp.AppException('server %s does not export %s' % - (tftp_root, server)) + except writeexts.ExtensionError: + raise writeexts.ExtensionError('server %s does not export %s' % + (tftp_root, server)) def version_exists(self, versioned_root, location): try: - cliapp.ssh_runcmd('root@%s' % location, - ['test', '-d', versioned_root]) - except cliapp.AppException: + writeexts.ssh_runcmd('root@%s' % location, + ['test', '-d', versioned_root]) + except writeexts.ExtensionError: return False return True diff --git a/extensions/nfsboot.write b/extensions/nfsboot.write index d928775e..418f8eeb 100755 --- a/extensions/nfsboot.write +++ b/extensions/nfsboot.write @@ -34,14 +34,13 @@ in /srv/nfsboot/nfs/ ''' -import cliapp import os import glob -import morphlib.writeexts +import writeexts -class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): +class NFSBootWriteExtension(writeexts.WriteExtension): '''Create an NFS root and kernel on TFTP during Morph's deployment. @@ -66,7 +65,8 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') temp_root, location = args @@ -86,8 +86,8 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): subdirs = [os.path.join(statedir, 'home'), os.path.join(statedir, 'opt'), os.path.join(statedir, 'srv')] - cliapp.ssh_runcmd('root@%s' % location, - ['mkdir', '-p'] + subdirs) + writeexts.ssh_runcmd('root@%s' % location, + ['mkdir', '-p'] + subdirs) def copy_kernel(self, temp_root, location, versioned_root, version, hostname): @@ -99,14 +99,14 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): kernel_src = try_path break else: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Could not find a kernel in the system: none of ' '%s found' % ', '.join(image_names)) kernel_dest = os.path.join(versioned_root, 'orig', 'kernel') rsync_dest = 'root@%s:%s' % (location, kernel_dest) self.status(msg='Copying kernel') - cliapp.runcmd( + subprocess.check_call( ['rsync', '-s', kernel_src, rsync_dest]) # Link the kernel to the right place @@ -115,17 +115,17 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): versioned_kernel_name = "%s-%s" % (hostname, version) kernel_name = hostname try: - cliapp.ssh_runcmd('root@%s' % location, + writeexts.ssh_runcmd('root@%s' % location, ['ln', '-f', kernel_dest, os.path.join(tftp_dir, versioned_kernel_name)]) - cliapp.ssh_runcmd('root@%s' % location, + writeexts.ssh_runcmd('root@%s' % location, ['ln', '-sf', versioned_kernel_name, os.path.join(tftp_dir, kernel_name)]) - except cliapp.AppException: - raise cliapp.AppException('Could not create symlinks to the ' - 'kernel at %s in %s on %s' - % (kernel_dest, tftp_dir, location)) + except writeexts.ExtensionError: + raise writeexts.ExtensionError('Could not create symlinks to the ' + 'kernel at %s in %s on %s' % + (kernel_dest, tftp_dir, location)) def copy_rootfs(self, temp_root, location, versioned_root, hostname): rootfs_src = temp_root + '/' @@ -134,51 +134,54 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Creating destination directories') try: - cliapp.ssh_runcmd('root@%s' % location, - ['mkdir', '-p', orig_path, run_path]) - except cliapp.AppException: - raise cliapp.AppException('Could not create dirs %s and %s on %s' - % (orig_path, run_path, location)) + writeexts.ssh_runcmd('root@%s' % location, + ['mkdir', '-p', orig_path, run_path]) + except writeexts.ExtensionError: + raise writexts.ExtensionError( + 'Could not create dirs %s and %s on %s' + % (orig_path, run_path, location)) self.status(msg='Creating \'orig\' rootfs') - cliapp.runcmd( + subprocess.check_call( ['rsync', '-asXSPH', '--delete', rootfs_src, 'root@%s:%s' % (location, orig_path)]) self.status(msg='Creating \'run\' rootfs') try: - cliapp.ssh_runcmd('root@%s' % location, - ['rm', '-rf', run_path]) - cliapp.ssh_runcmd('root@%s' % location, - ['cp', '-al', orig_path, run_path]) - cliapp.ssh_runcmd('root@%s' % location, - ['rm', '-rf', os.path.join(run_path, 'etc')]) - cliapp.ssh_runcmd('root@%s' % location, - ['cp', '-a', os.path.join(orig_path, 'etc'), - os.path.join(run_path, 'etc')]) - except cliapp.AppException: - raise cliapp.AppException('Could not create \'run\' rootfs' - ' from \'orig\'') + writeexts.ssh_runcmd('root@%s' % location, + ['rm', '-rf', run_path]) + writeexts.ssh_runcmd('root@%s' % location, + ['cp', '-al', orig_path, run_path]) + writeexts.ssh_runcmd('root@%s' % location, + ['rm', '-rf', + os.path.join(run_path, 'etc')]) + writeexts.ssh_runcmd('root@%s' % location, + ['cp', '-a', + os.path.join(orig_path, 'etc'), + os.path.join(run_path, 'etc')]) + except writeexts.ExtensionError: + raise writeexts.ExtensionError('Could not create \'run\' rootfs' + ' from \'orig\'') self.status(msg='Linking \'default\' to latest system') try: - cliapp.ssh_runcmd('root@%s' % location, + writeexts.ssh_runcmd('root@%s' % location, ['ln', '-sfn', versioned_root, os.path.join(self._nfsboot_root, hostname, 'systems', 'default')]) - except cliapp.AppException: - raise cliapp.AppException('Could not link \'default\' to %s' - % versioned_root) + except writeexts.ExtensionError: + raise writeexts.ExtensionError("Could not link 'default' to %s" + % versioned_root) def configure_nfs(self, location, hostname): exported_path = os.path.join(self._nfsboot_root, hostname) exports_path = '/etc/exports' # If that path is not already exported: try: - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( 'root@%s' % location, ['grep', '-q', exported_path, exports_path]) - except cliapp.AppException: + except writeexts.ExtensionError: ip_mask = '*' options = 'rw,no_subtree_check,no_root_squash,async' exports_string = '%s %s(%s)\n' % (exported_path, ip_mask, options) @@ -190,11 +193,11 @@ cat "$target" > "$temp" cat >> "$temp" mv "$temp" "$target" ''' - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( 'root@%s' % location, ['sh', '-c', exports_append_sh, '--', exports_path], feed_stdin=exports_string) - cliapp.ssh_runcmd( + writeexts.ssh_runcmd( 'root@%s' % location, ['systemctl', 'restart', 'nfs-server.service']) diff --git a/extensions/openstack.check b/extensions/openstack.check index a3379763..f3ad43b7 100755 --- a/extensions/openstack.check +++ b/extensions/openstack.check @@ -15,25 +15,26 @@ '''Preparatory checks for Morph 'openstack' write extension''' -import cliapp import os import urlparse + import keystoneclient -import morphlib.writeexts +import writeexts -class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): +class OpenStackCheckExtension(writeexts.WriteExtension): def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') self.require_btrfs_in_deployment_host_kernel() upgrade = self.get_environment_boolean('UPGRADE') if upgrade: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Use the `ssh-rsync` write extension to deploy upgrades to an ' 'existing remote system.') @@ -55,7 +56,7 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): for key in auth_keys: if os.environ.get(key, '') == '': - raise cliapp.AppException(key + ' was not given') + raise writeexts.ExtensionError(key + ' was not given') auth_params = {auth_keys[key]: os.environ[key] for key in auth_keys} auth_params['auth_url'] = location @@ -63,16 +64,17 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): def check_imagename(self): if os.environ.get('OPENSTACK_IMAGENAME', '') == '': - raise cliapp.AppException('OPENSTACK_IMAGENAME was not given') + raise writeexts.ExtensionError( + 'OPENSTACK_IMAGENAME was not given') def check_location(self, location): x = urlparse.urlparse(location) if x.scheme not in ['http', 'https']: - raise cliapp.AppException('URL schema must be http or https in %s'\ - % location) + raise writeexts.ExtensionError( + 'URL schema must be http or https in %s' % location) if (x.path != '/v2.0' and x.path != '/v2.0/'): - raise cliapp.AppException('API version must be v2.0 in %s'\ - % location) + raise writeexts.ExtensionError( + 'API version must be v2.0 in %s' % location) def check_openstack_parameters(self, auth_params): ''' Check that we can connect to and authenticate with openstack ''' @@ -84,7 +86,7 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): except keystoneclient.exceptions.Unauthorized: errmsg = ('Failed to authenticate with OpenStack ' '(are your credentials correct?)') - raise cliapp.AppException(errmsg) + raise writeexts.ExtensionError(errmsg) OpenStackCheckExtension().run() diff --git a/extensions/openstack.write b/extensions/openstack.write index 67e07c18..f1233560 100755 --- a/extensions/openstack.write +++ b/extensions/openstack.write @@ -17,21 +17,22 @@ '''A Morph deployment write extension for deploying to OpenStack.''' -import cliapp import os +import subprocess import tempfile import urlparse -import morphlib.writeexts +import writeexts -class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): +class OpenStackWriteExtension(writeexts.WriteExtension): '''See openstack.write.help for documentation''' def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') temp_root, location = args @@ -86,7 +87,7 @@ class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): '--disk-format=raw', '--container-format', 'bare', '--file', raw_disk] - cliapp.runcmd(cmdline) + subprocess.check_call(cmdline) self.status(msg='Image configured.') diff --git a/extensions/pxeboot.write b/extensions/pxeboot.write index 3a12ebcc..20e4f6bd 100644 --- a/extensions/pxeboot.write +++ b/extensions/pxeboot.write @@ -19,10 +19,7 @@ import tempfile import textwrap import urlparse -import cliapp - -import morphlib - +import writeexts def _int_to_quad_dot(i): return '.'.join(( @@ -143,7 +140,7 @@ def grouper(iterable, n, fillvalue=None): return itertools.izip_longest(*args, fillvalue=fillvalue) -class PXEBoot(morphlib.writeexts.WriteExtension): +class PXEBoot(writeexts.WriteExtension): @contextlib.contextmanager def _vlan(self, interface, vlan): viface = '%s.%s' % (interface, vlan) @@ -224,12 +221,13 @@ class PXEBoot(morphlib.writeexts.WriteExtension): @contextlib.contextmanager def _remote_tempdir(self, hostname, template): persist = os.environ.get('PXE_INSTALLER') in ('no', 'False') - td = cliapp.ssh_runcmd(hostname, ['mktemp', '-d', template]).strip() + td = writeexts.ssh_runcmd( + hostname, ['mktemp', '-d', template]).strip() try: yield td finally: if not persist: - cliapp.ssh_runcmd(hostname, ['find', td, '-delete']) + writeexts.ssh_runcmd(hostname, ['find', td, '-delete']) def _serve_tftpd(self, sock, host, port, interface, tftproot): self.settings.progname = 'tftp server' @@ -333,26 +331,27 @@ class PXEBoot(morphlib.writeexts.WriteExtension): def _remote_copy(self, hostname, src, dst): persist = os.environ.get('PXE_INSTALLER') in ('no', 'False') with open(src, 'r') as f: - cliapp.ssh_runcmd(hostname, - ['install', '-D', '-m644', '/proc/self/fd/0', - dst], stdin=f, stdout=None, stderr=None) + writeexts.ssh_runcmd(hostname, + ['install', '-D', '-m644', + '/proc/self/fd/0', dst], + stdin=f, stdout=None, stderr=None) try: yield finally: if not persist: - cliapp.ssh_runcmd(hostname, ['rm', dst]) + writeexts.ssh_runcmd(hostname, ['rm', dst]) @contextlib.contextmanager def _remote_symlink(self, hostname, src, dst): persist = os.environ.get('PXE_INSTALLER') in ('no', 'False') - cliapp.ssh_runcmd(hostname, - ['ln', '-s', '-f', src, dst], - stdin=None, stdout=None, stderr=None) + writeexts.ssh_runcmd(hostname, + ['ln', '-s', '-f', src, dst], + stdin=None, stdout=None, stderr=None) try: yield finally: if not persist: - cliapp.ssh_runcmd(hostname, ['rm', '-f', dst]) + writeexts.ssh_runcmd(hostname, ['rm', '-f', dst]) @contextlib.contextmanager def remote_kernel(self, rootfs, tftp_url, macaddr): @@ -361,7 +360,7 @@ class PXEBoot(morphlib.writeexts.WriteExtension): if os.path.exists(kernel_path): break else: - raise cliapp.AppException('Failed to locate kernel') + raise writeexts.ExtensionError('Failed to locate kernel') url = urlparse.urlsplit(tftp_url) basename = '{}-kernel'.format(_normalise_macaddr(macaddr)) target_path = os.path.join(url.path, basename) @@ -376,7 +375,8 @@ class PXEBoot(morphlib.writeexts.WriteExtension): yield fdt_abs_path = os.path.join(rootfs, fdt_rel_path) if not fdt_abs_path: - raise cliapp.AppException('Failed to locate Flattened Device Tree') + raise writeexts.ExtensionError( + 'Failed to locate Flattened Device Tree') url = urlparse.urlsplit(tftp_url) basename = '{}-fdt'.format(_normalise_macaddr(macaddr)) target_path = os.path.join(url.path, basename) @@ -389,14 +389,14 @@ class PXEBoot(morphlib.writeexts.WriteExtension): nfsroot = target_ip + ':' + rootfs self.status(msg='Exporting %(nfsroot)s as local nfsroot', nfsroot=nfsroot) - cliapp.runcmd(['exportfs', '-o', 'ro,insecure,no_root_squash', - nfsroot]) + subprocess.check_call(['exportfs', '-o', 'ro,insecure,no_root_squash', + nfsroot]) try: yield finally: self.status(msg='Removing %(nfsroot)s from local nfsroots', nfsroot=nfsroot) - cliapp.runcmd(['exportfs', '-u', nfsroot]) + subprocess.check_call(['exportfs', '-u', nfsroot]) @contextlib.contextmanager def remote_nfsroot(self, rootfs, rsync_url, macaddr): @@ -407,9 +407,10 @@ class PXEBoot(morphlib.writeexts.WriteExtension): as tempdir: nfsroot = urlparse.urlunsplit((url.scheme, url.netloc, tempdir, url.query, url.fragment)) - cliapp.runcmd(['rsync', '-asSPH', '--delete', rootfs, nfsroot], - stdin=None, stdout=open(os.devnull, 'w'), - stderr=None) + subprocess.check_call(['rsync', '-asSPH', '--delete', + rootfs, nfsroot], + stdin=None, stdout=open(os.devnull, 'w'), + stderr=None) yield os.path.join(os.path.basename(tempdir), os.path.basename(rootfs)) @@ -517,8 +518,8 @@ class PXEBoot(morphlib.writeexts.WriteExtension): def get_interface_ip(self, interface): ip_addresses = [] - info = cliapp.runcmd(['ip', '-o', '-f', 'inet', - 'addr', 'show', interface]).rstrip('\n') + info = subprocess.check_output(['ip', '-o', '-f', 'inet', 'addr', + 'show', interface]).rstrip('\n') if info: tokens = collections.deque(info.split()[1:]) ifname = tokens.popleft() @@ -535,8 +536,8 @@ class PXEBoot(morphlib.writeexts.WriteExtension): else: continue if not ip_addresses: - raise cliapp.AppException('Interface %s has no addresses' - % interface) + raise writeexts.ExtensionError('Interface %s has no addresses' + % interface) if len(ip_addresses) > 1: warnings.warn('Interface %s has multiple addresses, ' 'using first (%s)' % (interface, ip_addresses[0])) @@ -750,6 +751,6 @@ class PXEBoot(morphlib.writeexts.WriteExtension): self.wait_for_target_to_install() self.ipmi_reboot_target() else: - cliapp.AppException('Invalid PXEBOOT_MODE: %s' % mode) + writeexts.ExtensionError('Invalid PXEBOOT_MODE: %s' % mode) PXEBoot().run() diff --git a/extensions/rawdisk.check b/extensions/rawdisk.check index 9be0ce91..61619a21 100755 --- a/extensions/rawdisk.check +++ b/extensions/rawdisk.check @@ -15,17 +15,16 @@ '''Preparatory checks for Morph 'rawdisk' write extension''' -import cliapp - -import morphlib.writeexts - import os +import writeexts + -class RawdiskCheckExtension(morphlib.writeexts.WriteExtension): +class RawdiskCheckExtension(writeexts.WriteExtension): def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') self.require_btrfs_in_deployment_host_kernel() @@ -34,19 +33,19 @@ class RawdiskCheckExtension(morphlib.writeexts.WriteExtension): if upgrade: if not self.is_device(location): if not os.path.isfile(location): - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Cannot upgrade %s: it is not an existing disk image' % location) version_label = os.environ.get('VERSION_LABEL') if version_label is None: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'VERSION_LABEL was not given. It is required when ' 'upgrading an existing system.') else: if not self.is_device(location): if os.path.exists(location): - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Target %s already exists. Use `morph upgrade` if you ' 'want to update an existing image.' % location) diff --git a/extensions/rawdisk.write b/extensions/rawdisk.write index 6f2d45ba..cdeb5018 100755 --- a/extensions/rawdisk.write +++ b/extensions/rawdisk.write @@ -17,22 +17,22 @@ '''A Morph deployment write extension for raw disk images.''' -import cliapp import os import sys import time import tempfile -import morphlib.writeexts +import writeexts -class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): +class RawDiskWriteExtension(writeexts.WriteExtension): '''See rawdisk.write.help for documentation''' def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') temp_root, location = args upgrade = self.get_environment_boolean('UPGRADE') @@ -69,10 +69,10 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): old_orig = os.path.join(mp, 'systems', 'default', 'orig') new_orig = os.path.join(version_root, 'orig') - cliapp.runcmd( + subprocess.check_call( ['btrfs', 'subvolume', 'snapshot', old_orig, new_orig]) - cliapp.runcmd( + subprocess.check_call( ['rsync', '-a', '--checksum', '--numeric-ids', '--delete', temp_root + os.path.sep, new_orig]) @@ -96,11 +96,11 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): version_label = os.environ.get('VERSION_LABEL') if version_label is None: - raise cliapp.AppException('VERSION_LABEL was not given') + raise writeexts.ExtensionError('VERSION_LABEL was not given') if os.path.exists(os.path.join(mp, 'systems', version_label)): - raise cliapp.AppException('VERSION_LABEL %s already exists' - % version_label) + raise writeexts.ExtensionError('VERSION_LABEL %s already exists' + % version_label) return version_label diff --git a/extensions/simple-network.configure b/extensions/simple-network.configure index 4a70f311..61d5774d 100755 --- a/extensions/simple-network.configure +++ b/extensions/simple-network.configure @@ -25,27 +25,26 @@ for DHCP ''' +import errno import os import sys -import errno -import cliapp -import morphlib +import writeexts -class SimpleNetworkError(morphlib.Error): +class SimpleNetworkError(writeexts.ExtensionError): '''Errors associated with simple network setup''' pass -class SimpleNetworkConfigurationExtension(cliapp.Application): +class SimpleNetworkConfigurationExtension(object): '''Configure /etc/network/interfaces and generate networkd .network files Reading NETWORK_CONFIG, this extension sets up /etc/network/interfaces and .network files in /etc/systemd/network/. ''' - def process_args(self, args): + def run(self, args): network_config = os.environ.get("NETWORK_CONFIG") self.rename_networkd_chunk_file(args) @@ -206,7 +205,8 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): address_line = address + '/' + str(network_suffix) lines += ["Address=%s" % address_line] elif address or netmask: - raise Exception('address and netmask must be specified together') + raise SimpleNetworkError( + 'address and netmask must be specified together') if gateway: lines += ["Gateway=%s" % gateway] @@ -287,6 +287,10 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): ''' - self.output.write('%s\n' % (kwargs['msg'] % kwargs)) + sys.stdout.write('%s\n' % (kwargs['msg'] % kwargs)) -SimpleNetworkConfigurationExtension().run() +try: + SimpleNetworkConfigurationExtension().run(sys.argv[1:]) +except SimpleNetworkError as e: + sys.stdout.write('ERROR: %s\n' % e) + sys.exit(1) diff --git a/extensions/ssh-rsync.check b/extensions/ssh-rsync.check index c3bdfd29..436aaae0 100755 --- a/extensions/ssh-rsync.check +++ b/extensions/ssh-rsync.check @@ -15,26 +15,27 @@ '''Preparatory checks for Morph 'ssh-rsync' write extension''' -import cliapp import os -import morphlib.writeexts +import writeexts -class SshRsyncCheckExtension(morphlib.writeexts.WriteExtension): + +class SshRsyncCheckExtension(writeexts.WriteExtension): def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') upgrade = self.get_environment_boolean('UPGRADE') if not upgrade: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'The ssh-rsync write is for upgrading existing remote ' 'Baserock machines. It cannot be used for an initial ' 'deployment.') if os.environ.get('VERSION_LABEL', '') == '': - raise cliapp.AppException( + raise writeexts.ExtensionError( 'A VERSION_LABEL must be set when deploying an upgrade.') location = args[0] @@ -47,17 +48,18 @@ class SshRsyncCheckExtension(morphlib.writeexts.WriteExtension): self.check_command_exists(location, 'rsync') def check_is_baserock_system(self, location): - output = cliapp.ssh_runcmd(location, ['sh', '-c', - 'test -d /baserock || echo -n dirnotfound']) + output = writeexts.ssh_runcmd( + location, + ['sh', '-c', 'test -d /baserock || echo -n dirnotfound']) if output == 'dirnotfound': - raise cliapp.AppException('%s is not a baserock system' - % location) + raise writeexts.ExtensionError('%s is not a baserock system' + % location) def check_command_exists(self, location, command): test = 'type %s > /dev/null 2>&1 || echo -n cmdnotfound' % command - output = cliapp.ssh_runcmd(location, ['sh', '-c', test]) + output = writeexts.ssh_runcmd(location, ['sh', '-c', test]) if output == 'cmdnotfound': - raise cliapp.AppException( + raise writeexts.ExtensionError( "%s does not have %s" % (location, command)) diff --git a/extensions/ssh-rsync.write b/extensions/ssh-rsync.write index 6d596500..46c16662 100755 --- a/extensions/ssh-rsync.write +++ b/extensions/ssh-rsync.write @@ -18,23 +18,23 @@ import contextlib -import cliapp import os +import subprocess import sys -import time import tempfile +import time -import morphlib.writeexts +import writeexts def ssh_runcmd_ignore_failure(location, command, **kwargs): try: - return cliapp.ssh_runcmd(location, command, **kwargs) - except cliapp.AppException: + return writeexts.ssh_runcmd(location, command, **kwargs) + except writeexts.ExtensionError: pass -class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): +class SshRsyncWriteExtension(writeexts.WriteExtension): '''See ssh-rsync.write.help for documentation''' @@ -43,7 +43,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): '''Read /proc/mounts on location to find which device contains "/"''' self.status(msg='Finding device that contains "/"') - contents = cliapp.ssh_runcmd(location, ['cat', '/proc/mounts']) + contents = writeexts.ssh_runcmd(location, + ['cat', '/proc/mounts']) for line in contents.splitlines(): line_words = line.split() if (line_words[1] == '/' and line_words[0] != 'rootfs'): @@ -52,28 +53,29 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): @contextlib.contextmanager def _remote_mount_point(self, location): self.status(msg='Creating remote mount point') - remote_mnt = cliapp.ssh_runcmd(location, ['mktemp', '-d']).strip() + remote_mnt = writeexts.ssh_runcmd(location, + ['mktemp', '-d']).strip() try: yield remote_mnt finally: self.status(msg='Removing remote mount point') - cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) + writeexts.ssh_runcmd(location, ['rmdir', remote_mnt]) @contextlib.contextmanager def _remote_mount(self, location, root_disk, mountpoint): self.status(msg='Mounting root disk') - cliapp.ssh_runcmd(location, ['mount', root_disk, mountpoint]) + writeexts.ssh_runcmd(location, ['mount', root_disk, mountpoint]) try: yield finally: self.status(msg='Unmounting root disk') - cliapp.ssh_runcmd(location, ['umount', mountpoint]) + writeexts.ssh_runcmd(location, ['umount', mountpoint]) @contextlib.contextmanager def _created_version_root(self, location, remote_mnt, version_label): version_root = os.path.join(remote_mnt, 'systems', version_label) self.status(msg='Creating %(root)s', root=version_root) - cliapp.ssh_runcmd(location, ['mkdir', version_root]) + writeexts.ssh_runcmd(location, ['mkdir', version_root]) try: yield version_root except BaseException as e: @@ -93,8 +95,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Creating "orig" subvolume') old_orig = self.get_old_orig(location, remote_mnt) new_orig = os.path.join(version_root, 'orig') - cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', 'snapshot', - old_orig, new_orig]) + writeexts.ssh_runcmd(location, ['btrfs', 'subvolume', 'snapshot', + old_orig, new_orig]) try: yield new_orig except BaseException as e: @@ -106,30 +108,30 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): '''Populate the subvolume version_root/orig on location''' self.status(msg='Populating "orig" subvolume') - cliapp.runcmd(['rsync', '-as', '--checksum', '--numeric-ids', - '--delete', temp_root + os.path.sep, - '%s:%s' % (location, new_orig)]) + subprocess.check_call(['rsync', '-as', '--checksum', '--numeric-ids', + '--delete', temp_root + os.path.sep, + '%s:%s' % (location, new_orig)]) @contextlib.contextmanager def _deployed_version(self, location, version_label, system_config_sync, system_version_manager): self.status(msg='Calling system-version-manager to deploy upgrade') deployment = os.path.join('/systems', version_label, 'orig') - cliapp.ssh_runcmd(location, + writeexts.ssh_runcmd(location, ['env', 'BASEROCK_SYSTEM_CONFIG_SYNC='+system_config_sync, system_version_manager, 'deploy', deployment]) try: yield deployment except BaseException as e: self.status(msg='Cleaning up failed version installation') - cliapp.ssh_runcmd(location, + writeexts.ssh_runcmd(location, [system_version_manager, 'remove', version_label]) raise def upgrade_remote_system(self, location, temp_root): root_disk = self.find_root_disk(location) - uuid = cliapp.ssh_runcmd(location, ['blkid', '-s', 'UUID', '-o', - 'value', root_disk]).strip() + uuid = writeexts.ssh_runcmd(location, + ['blkid', '-s', 'UUID', '-o', 'value', root_disk]).strip() self.complete_fstab_for_btrfs_layout(temp_root, uuid) @@ -153,8 +155,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): config_sync, version_manager): self.status(msg='Setting %(v)s as the new default system', v=version_label) - cliapp.ssh_runcmd(location, [version_manager, - 'set-default', version_label]) + writeexts.ssh_runcmd(location, + [version_manager, 'set-default', version_label]) if autostart: self.status(msg="Rebooting into new system ...") @@ -162,7 +164,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') temp_root, location = args diff --git a/extensions/strip-gplv3.configure b/extensions/strip-gplv3.configure index c08061ad..0c5250e4 100755 --- a/extensions/strip-gplv3.configure +++ b/extensions/strip-gplv3.configure @@ -21,12 +21,14 @@ to find the files created by that chunk, then remove them. ''' -import cliapp -import re -import os import json +import os +import re +import sys + +import writeexts -class StripGPLv3ConfigureExtension(cliapp.Application): +class StripGPLv3ConfigureExtension(writeexts.Extension): gplv3_chunks = [ ['autoconf', ''], ['automake', ''], @@ -57,7 +59,8 @@ class StripGPLv3ConfigureExtension(cliapp.Application): for chunk in self.gplv3_chunks: regex = os.path.join(meta_dir, "%s-[^-]\+\.meta" % chunk[0]) - artifacts = self.runcmd(['find', meta_dir, '-regex', regex]) + artifacts = subprocess.check_output(['find', meta_dir, + '-regex', regex]) for artifact in artifacts.split(): self.remove_chunk(target_root, artifact, chunk[1]) @@ -72,8 +75,8 @@ class StripGPLv3ConfigureExtension(cliapp.Application): chunk_meta_data = json.load(f) if not 'contents' in chunk_meta_data: - raise cliapp.AppError('Chunk %s does not have a "contents" list' - % chunk) + raise writeexts.ExtensionError( + 'Chunk %s does not have a "contents" list' % chunk) updated_contents = [] for content_entry in reversed(chunk_meta_data['contents']): pat = re.compile(pattern) @@ -85,8 +88,8 @@ class StripGPLv3ConfigureExtension(cliapp.Application): def remove_content_entry(self, target_root, content_entry): entry_path = os.path.join(target_root, './' + content_entry) if not entry_path.startswith(target_root): - raise cliapp.AppException('%s is not in %s' - % (entry_path, target_root)) + raise writeexts.ExtensionError( + '%s is not in %s' % (entry_path, target_root)) if os.path.exists(entry_path): if os.path.islink(entry_path): os.unlink(entry_path) @@ -96,6 +99,7 @@ class StripGPLv3ConfigureExtension(cliapp.Application): if not os.listdir(entry_path): os.rmdir(entry_path) else: - raise cliapp.AppException('%s is not a link, file or directory' - % entry_path) -StripGPLv3ConfigureExtension().run() + raise writeexts.ExtensionError( + '%s is not a link, file or directory' % entry_path) + +StripGPLv3ConfigureExtension().run(sys.argv[1:]) diff --git a/extensions/virtualbox-ssh.check b/extensions/virtualbox-ssh.check index a97f3294..e82d58a1 100755 --- a/extensions/virtualbox-ssh.check +++ b/extensions/virtualbox-ssh.check @@ -15,21 +15,21 @@ '''Preparatory checks for Morph 'virtualbox-ssh' write extension''' -import cliapp -import morphlib.writeexts +import writeexts -class VirtualBoxPlusSshCheckExtension(morphlib.writeexts.WriteExtension): +class VirtualBoxPlusSshCheckExtension(writeexts.WriteExtension): def process_args(self, args): if len(args) != 1: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') self.require_btrfs_in_deployment_host_kernel() upgrade = self.get_environment_boolean('UPGRADE') if upgrade: - raise cliapp.AppException( + raise writeexts.ExtensionError( 'Use the `ssh-rsync` write extension to deploy upgrades to an ' 'existing remote system.') diff --git a/extensions/virtualbox-ssh.write b/extensions/virtualbox-ssh.write index 774f2b4f..95643a4a 100755 --- a/extensions/virtualbox-ssh.write +++ b/extensions/virtualbox-ssh.write @@ -24,22 +24,23 @@ See file virtualbox-ssh.write.help for documentation ''' -import cliapp import os import re +import subprocess import sys -import time import tempfile +import time import urlparse -import morphlib.writeexts +import writeexts -class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): +class VirtualBoxPlusSshWriteExtension(writeexts.WriteExtension): def process_args(self, args): if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') + raise writeexts.ExtensionError( + 'Wrong number of command line args') temp_root, location = args ssh_host, vm_name, vdi_path = self.parse_location(location) @@ -59,7 +60,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): except BaseException: sys.stderr.write('Error deploying to VirtualBox') os.remove(raw_disk) - cliapp.ssh_runcmd(ssh_host, ['rm', '-f', vdi_path]) + writeexts.ssh_runcmd(ssh_host, ['rm', '-f', vdi_path]) raise else: os.remove(raw_disk) @@ -72,11 +73,12 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): x = urlparse.urlparse(location) if x.scheme != 'vbox+ssh': - raise cliapp.AppException( + raise writeexts.ExtensionError( 'URL schema must be vbox+ssh in %s' % location) m = re.match('^/(?P[^/]+)(?P/.+)$', x.path) if not m: - raise cliapp.AppException('Cannot parse location %s' % location) + raise writeexts.ExtensionError( + 'Cannot parse location %s' % location) return x.netloc, m.group('guest'), m.group('path') def transfer_and_convert_to_vdi(self, raw_disk, ssh_host, vdi_path): @@ -85,17 +87,18 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Transfer disk and convert to VDI') st = os.lstat(raw_disk) - xfer_hole_path = morphlib.util.get_data_path('xfer-hole') - recv_hole = morphlib.util.get_data('recv-hole') + # TODO: Something! + xfer_hole_path = writeexts.get_data_path('xfer-hole') + recv_hole = writeexts.get_data('recv-hole') ssh_remote_cmd = [ 'sh', '-c', recv_hole, 'dummy-argv0', 'vbox', vdi_path, str(st.st_size), ] - cliapp.runcmd( + subprocess.check_call( ['python', xfer_hole_path, raw_disk], - ['ssh', ssh_host] + map(cliapp.shell_quote, ssh_remote_cmd), + ['ssh', ssh_host] + map(writeexts.shell_quote, ssh_remote_cmd), stdout=None, stderr=None) def virtualbox_version(self, ssh_host): @@ -107,7 +110,8 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): # tuple is more reliable than a string and more convenient than # comparing against the major, minor and patch numbers directly self.status(msg='Checking version of remote VirtualBox') - build_id = cliapp.ssh_runcmd(ssh_host, ['VBoxManage', '--version']) + build_id = writeexts.ssh_runcmd(ssh_host, + ['VBoxManage', '--version']) version_string = re.match(r"^([0-9\.]+).*$", build_id.strip()).group(1) return tuple(int(s or '0') for s in version_string.split('.')) @@ -163,17 +167,17 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): for command in commands: argv = ['VBoxManage'] + command - cliapp.ssh_runcmd(ssh_host, argv) + writeexts.ssh_runcmd(ssh_host, argv) def get_host_interface(self, ssh_host): host_ipaddr = os.environ.get('HOST_IPADDR') netmask = os.environ.get('NETMASK') if host_ipaddr is None: - raise cliapp.AppException('HOST_IPADDR was not given') + raise writeexts.ExtensionError('HOST_IPADDR was not given') if netmask is None: - raise cliapp.AppException('NETMASK was not given') + raise writeexts.ExtensionError('NETMASK was not given') # 'VBoxManage list hostonlyifs' retrieves a list with the hostonly # interfaces on the host. For each interface, the following lines @@ -187,7 +191,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): # The following command tries to retrieve the hostonly interface # name (e.g. vboxnet0) associated with the given ip address. iface = None - lines = cliapp.ssh_runcmd(ssh_host, + lines = writeexts.ssh_runcmd(ssh_host, ['VBoxManage', 'list', 'hostonlyifs']).splitlines() for i, v in enumerate(lines): if host_ipaddr in v: @@ -195,12 +199,12 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): break if iface is None: - iface = cliapp.ssh_runcmd(ssh_host, + iface = writeexts.ssh_runcmd(ssh_host, ['VBoxManage', 'hostonlyif', 'create']) # 'VBoxManage hostonlyif create' shows the name of the # created hostonly interface inside single quotes iface = iface[iface.find("'") + 1 : iface.rfind("'")] - cliapp.ssh_runcmd(ssh_host, + writeexts.ssh_runcmd(ssh_host, ['VBoxManage', 'hostonlyif', 'ipconfig', iface, '--ip', host_ipaddr, diff --git a/extensions/writeexts.py b/extensions/writeexts.py index 9357648f..5c579a10 100644 --- a/extensions/writeexts.py +++ b/extensions/writeexts.py @@ -13,27 +13,119 @@ # with this program. If not, see . -import cliapp +import contextlib +import errno +import fcntl import logging import os import re +import select import shutil +import stat +import subprocess import sys import time import tempfile -import errno -import stat -import contextlib -@contextlib.contextmanager -def hide_password_environment_variables(env): # pragma: no cover - password_env = { k:v for k,v in env.iteritems() if 'PASSWORD' in k } - for k in password_env: - env[k] = '(value hidden)' - yield - for k, v in password_env.iteritems(): - env[k] = v +if sys.version_info >= (3, 3, 0): + import shlex + shell_quote = shlex.quote +else: + import pipes + shell_quote = pipes.quote + + +def get_data_path(relative_path): + extensions_dir = os.path.dirname(__file__) + return os.path.join(extensions_dir, relative_path) + + +def get_data(relative_path): + with open(get_data_path(relative_path)) as f: + return f.read() + + +def ssh_runcmd(host, args, **kwargs): + '''Run command over ssh''' + command = ['ssh', host, '--'] + [shell_quote(arg) for arg in args] + + feed_stdin = kwargs.get('feed_stdin') + stdin = kwargs.get('stdin', subprocess.PIPE) + stdout = kwargs.get('stdout', subprocess.PIPE) + stderr = kwargs.get('stderr', subprocess.PIPE) + + p = subprocess.Popen(command, stdin=stdin, stdout=stdout, stderr=stderr) + out, err = p.communicate(input=feed_stdin) + if p.returncode != 0: + raise ExtensionError('ssh command `%s` failed' % ' '.join(command)) + return out + + +def write_from_dict(filepath, d, validate=lambda x, y: True): + """Takes a dictionary and appends the contents to a file + + An optional validation callback can be passed to perform validation on + each value in the dictionary. + + e.g. + + def validation_callback(dictionary_key, dictionary_value): + if not dictionary_value.isdigit(): + raise Exception('value contains non-digit character(s)') + + Any callback supplied to this function should raise an exception + if validation fails. + + """ + # Sort items asciibetically + # the output of the deployment should not depend + # on the locale of the machine running the deployment + items = sorted(d.iteritems(), key=lambda (k, v): [ord(c) for c in v]) + + for (k, v) in items: + validate(k, v) + + with open(filepath, 'a') as f: + for (_, v) in items: + f.write('%s\n' % v) + + os.fchown(f.fileno(), 0, 0) + os.fchmod(f.fileno(), 0644) + + +def parse_environment_pairs(env, pairs): + '''Add key=value pairs to the environment dict. + + Given a dict and a list of strings of the form key=value, + set dict[key] = value, unless key is already set in the + environment, at which point raise an exception. + + This does not modify the passed in dict. + + Returns the extended dict. + + ''' + extra_env = dict(p.split('=', 1) for p in pairs) + conflicting = [k for k in extra_env if k in env] + if conflicting: + raise ExtensionError('Environment already set: %s' + % ', '.join(conflicting)) + + # Return a dict that is the union of the two + # This is not the most performant, since it creates + # 3 unnecessary lists, but I felt this was the most + # easy to read. Using itertools.chain may be more efficicent + return dict(env.items() + extra_env.items()) + + +class ExtensionError(Exception): + + def __init__(self, msg): + self.msg = msg + + def __str__(self): + return self.msg class Fstab(object): @@ -89,9 +181,9 @@ class Fstab(object): shutil.move(os.path.abspath(tmp), os.path.abspath(self.filepath)) -class WriteExtension(cliapp.Application): +class Extension(object): - '''A base class for deployment write extensions. + '''A base class for deployment extensions. A subclass should subclass this class, and add a ``process_args`` method. @@ -108,8 +200,6 @@ class WriteExtension(cliapp.Application): This file descriptor is read by Morph and written into its own log file. - This overrides cliapp's usual configurable logging setup. - ''' log_write_fd = int(os.environ.get('MORPH_LOG_FD', 0)) @@ -125,13 +215,19 @@ class WriteExtension(cliapp.Application): logger.addHandler(handler) logger.setLevel(logging.DEBUG) - def log_config(self): - with hide_password_environment_variables(os.environ): - cliapp.Application.log_config(self) - def process_args(self, args): raise NotImplementedError() + def run(self, args=None): + if args is None: + args = sys.argv[1:] + try: + self.setup_logging() + self.process_args(args) + except ExtensionError as e: + sys.stdout.write('ERROR: %s\n' % e) + sys.exit(1) + def status(self, **kwargs): '''Provide status output. @@ -140,9 +236,22 @@ class WriteExtension(cliapp.Application): by %. ''' + sys.stdout.write('%s\n' % (kwargs['msg'] % kwargs)) + sys.stdout.flush() + - self.output.write('%s\n' % (kwargs['msg'] % kwargs)) - self.output.flush() +class WriteExtension(Extension): + + '''A base class for deployment write extensions. + + A subclass should subclass this class, and add a + ``process_args`` method. + + Note that it is not necessary to subclass this class for write + extensions. This class is here just to collect common code for + write extensions. + + ''' def check_for_btrfs_in_deployment_host_kernel(self): with open('/proc/filesystems') as f: @@ -151,7 +260,7 @@ class WriteExtension(cliapp.Application): def require_btrfs_in_deployment_host_kernel(self): if not self.check_for_btrfs_in_deployment_host_kernel(): - raise cliapp.AppException( + raise ExtensionError( 'Error: Btrfs is required for this deployment, but was not ' 'detected in the kernel of the machine that is running Morph.') @@ -166,7 +275,7 @@ class WriteExtension(cliapp.Application): def created_disk_image(self, location): size = self.get_disk_size() if not size: - raise cliapp.AppException('DISK_SIZE is not defined') + raise ExtensionError('DISK_SIZE is not defined') self.create_raw_disk_image(location, size) try: yield @@ -220,8 +329,8 @@ class WriteExtension(cliapp.Application): return None bytes = self._parse_size(size) if bytes is None: - raise cliapp.AppException('Cannot parse %s value %s' - % (env_var, size)) + raise ExtensionError('Cannot parse %s value %s' + % (env_var, size)) return bytes def get_disk_size(self): @@ -254,15 +363,15 @@ class WriteExtension(cliapp.Application): # need to do this because at the time of writing, SYSLINUX has not # been updated to understand these new features and will fail to # boot if the kernel is on a filesystem where they are enabled. - cliapp.runcmd( + subprocess.check_output( ['mkfs.btrfs','-f', '-L', 'baserock', '--features', '^extref', '--features', '^skinny-metadata', '--features', '^mixed-bg', '--nodesize', '4096', location]) - except cliapp.AppException as e: - if 'unrecognized option \'--features\'' in e.msg: + except subprocess.CalledProcessError as e: + if 'unrecognized option \'--features\'' in e.output: # Old versions of mkfs.btrfs (including v0.20, present in many # Baserock releases) don't support the --features option, but # also don't enable the new features by default. So we can @@ -270,7 +379,8 @@ class WriteExtension(cliapp.Application): logging.debug( 'Assuming mkfs.btrfs failure was because the tool is too ' 'old to have --features flag.') - cliapp.runcmd(['mkfs.btrfs','-f', '-L', 'baserock', location]) + subprocess.check_call(['mkfs.btrfs','-f', + '-L', 'baserock', location]) else: raise @@ -278,8 +388,8 @@ class WriteExtension(cliapp.Application): '''Get the UUID of a block device's file system.''' # Requires util-linux blkid; busybox one ignores options and # lies by exiting successfully. - return cliapp.runcmd(['blkid', '-s', 'UUID', '-o', 'value', - location]).strip() + return subprocess.check_output(['blkid', '-s', 'UUID', '-o', 'value', + location]).strip() @contextlib.contextmanager def mount(self, location): @@ -287,9 +397,10 @@ class WriteExtension(cliapp.Application): try: mount_point = tempfile.mkdtemp() if self.is_device(location): - cliapp.runcmd(['mount', location, mount_point]) + subprocess.check_call(['mount', location, mount_point]) else: - cliapp.runcmd(['mount', '-o', 'loop', location, mount_point]) + subprocess.check_call(['mount', '-o', 'loop', + location, mount_point]) except BaseException as e: sys.stderr.write('Error mounting filesystem') os.rmdir(mount_point) @@ -298,7 +409,7 @@ class WriteExtension(cliapp.Application): yield mount_point finally: self.status(msg='Unmounting filesystem') - cliapp.runcmd(['umount', mount_point]) + subprocess.check_call(['umount', mount_point]) os.rmdir(mount_point) def create_btrfs_system_layout(self, temp_root, mountpoint, version_label, @@ -345,9 +456,9 @@ class WriteExtension(cliapp.Application): orig = os.path.join(version_root, 'orig') self.status(msg='Creating orig subvolume') - cliapp.runcmd(['btrfs', 'subvolume', 'create', orig]) + subprocess.check_call(['btrfs', 'subvolume', 'create', orig]) self.status(msg='Copying files to orig subvolume') - cliapp.runcmd(['cp', '-a', temp_root + '/.', orig + '/.']) + subprocess.check_call(['cp', '-a', temp_root + '/.', orig + '/.']) def create_run(self, version_root): '''Create the 'run' snapshot.''' @@ -355,7 +466,7 @@ class WriteExtension(cliapp.Application): self.status(msg='Creating run subvolume') orig = os.path.join(version_root, 'orig') run = os.path.join(version_root, 'run') - cliapp.runcmd( + subprocess.check_call( ['btrfs', 'subvolume', 'snapshot', orig, run]) def create_state_subvolume(self, system_dir, mountpoint, state_subdir): @@ -369,7 +480,7 @@ class WriteExtension(cliapp.Application): ''' self.status(msg='Creating %s subvolume' % state_subdir) subvolume = os.path.join(mountpoint, 'state', state_subdir) - cliapp.runcmd(['btrfs', 'subvolume', 'create', subvolume]) + subprocess.check_call(['btrfs', 'subvolume', 'create', subvolume]) os.chmod(subvolume, 0o755) existing_state_dir = os.path.join(system_dir, state_subdir) @@ -380,7 +491,7 @@ class WriteExtension(cliapp.Application): self.status(msg='Moving existing data to %s subvolume' % subvolume) for filename in files: filepath = os.path.join(existing_state_dir, filename) - cliapp.runcmd(['mv', filepath, subvolume]) + subprocess.check_call(['mv', filepath, subvolume]) def complete_fstab_for_btrfs_layout(self, system_dir, rootfs_uuid=None): '''Fill in /etc/fstab entries for the default Btrfs disk layout. @@ -430,8 +541,8 @@ class WriteExtension(cliapp.Application): if 'INITRAMFS_PATH' in os.environ: initramfs = os.path.join(temp_root, os.environ['INITRAMFS_PATH']) if not os.path.exists(initramfs): - raise cliapp.AppException('INITRAMFS_PATH specified, ' - 'but file does not exist') + raise ExtensionError('INITRAMFS_PATH specified, ' + 'but file does not exist') return initramfs return None @@ -443,7 +554,7 @@ class WriteExtension(cliapp.Application): ''' self.status(msg='Installing initramfs') initramfs_dest = os.path.join(version_root, 'initramfs') - cliapp.runcmd(['cp', '-a', initramfs_path, initramfs_dest]) + subprocess.check_call(['cp', '-a', initramfs_path, initramfs_dest]) def install_kernel(self, version_root, temp_root): '''Install the kernel outside of 'orig' or 'run' subvolumes''' @@ -454,7 +565,7 @@ class WriteExtension(cliapp.Application): for name in image_names: try_path = os.path.join(temp_root, 'boot', name) if os.path.exists(try_path): - cliapp.runcmd(['cp', '-a', try_path, kernel_dest]) + subprocess.check_call(['cp', '-a', try_path, kernel_dest]) break def install_dtb(self, version_root, temp_root): @@ -465,10 +576,10 @@ class WriteExtension(cliapp.Application): dtb_dest = os.path.join(version_root, 'dtb') try_path = os.path.join(temp_root, device_tree_path) if os.path.exists(try_path): - cliapp.runcmd(['cp', '-a', try_path, dtb_dest]) + subprocess.check_call(['cp', '-a', try_path, dtb_dest]) else: logging.error("Failed to find device tree %s", device_tree_path) - raise cliapp.AppException( + raise ExtensionError( 'Failed to find device tree %s' % device_tree_path) def get_dtb_path(self): @@ -500,7 +611,7 @@ class WriteExtension(cliapp.Application): if config_type in config_function_dict: config_function_dict[config_type](real_root, disk_uuid) else: - raise cliapp.AppException( + raise ExtensionError( 'Invalid BOOTLOADER_CONFIG_FORMAT %s' % config_type) def generate_extlinux_config(self, real_root, disk_uuid=None): @@ -544,15 +655,15 @@ class WriteExtension(cliapp.Application): if install_type in install_function_dict: install_function_dict[install_type](real_root) elif install_type != 'none': - raise cliapp.AppException( + raise ExtensionError( 'Invalid BOOTLOADER_INSTALL %s' % install_type) def install_bootloader_extlinux(self, real_root): self.status(msg='Installing extlinux') - cliapp.runcmd(['extlinux', '--install', real_root]) + subprocess.check_call(['extlinux', '--install', real_root]) # FIXME this hack seems to be necessary to let extlinux finish - cliapp.runcmd(['sync']) + subprocess.check_call(['sync']) time.sleep(2) def install_syslinux_menu(self, real_root, version_root): @@ -610,19 +721,19 @@ class WriteExtension(cliapp.Application): elif value in ['yes', '1', 'true']: return True else: - raise cliapp.AppException('Unexpected value for %s: %s' % - (variable, value)) + raise ExtensionError('Unexpected value for %s: %s' % + (variable, value)) def check_ssh_connectivity(self, ssh_host): try: - output = cliapp.ssh_runcmd(ssh_host, ['echo', 'test']) - except cliapp.AppException as e: + output = ssh_runcmd(ssh_host, ['echo', 'test']) + except ExtensionError as e: logging.error("Error checking SSH connectivity: %s", str(e)) - raise cliapp.AppException( + raise ExtensionError( 'Unable to SSH to %s: %s' % (ssh_host, e)) if output.strip() != 'test': - raise cliapp.AppException( + raise ExtensionError( 'Unexpected output from remote machine: %s' % output.strip()) def is_device(self, location): -- cgit v1.2.1