From c7e6642d855283b85182097b24226e78aa78e199 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 6 Feb 2013 20:44:47 +0000 Subject: Add morphlib module for common write extension code --- writeexts.py | 157 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100755 writeexts.py diff --git a/writeexts.py b/writeexts.py new file mode 100755 index 00000000..23473021 --- /dev/null +++ b/writeexts.py @@ -0,0 +1,157 @@ +# Copyright (C) 2012-2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import os +import re +import time +import tempfile + + +class WriteExtension(cliapp.Application): + + '''A base class for deployment write extensions. + + A subclass should subclass this class, and add a + ``process_args`` method. + + Note that it is not necessary to subclass this class for write + extensions. This class is here just to collect common code for + write extensions. + + ''' + + def process_args(self, args): + raise NotImplementedError() + + def status(self, **kwargs): + '''Provide status output. + + The ``msg`` keyword argument is the actual message, + the rest are values for fields in the message as interpolated + by %. + + ''' + + self.output.write('%s\n' % (kwargs['msg'] % kwargs)) + + def get_disk_size(self): + '''Parse disk size from environment.''' + + size = os.environ.get('DISK_SIZE', '1G') + m = re.match('^(\d+)([kmgKMG]?)$', size) + if not m: + raise morphlib.Error('Cannot parse disk size %s' % size) + + factors = { + '': 1, + 'k': 1024, + 'm': 1024**2, + 'g': 1024**3, + } + factor = factors[m.group(2).lower()] + + return int(m.group(1)) * factor + + def create_raw_disk_image(self, filename, size): + '''Create a raw disk image.''' + + self.status(msg='Creating empty disk image') + cliapp.runcmd( + ['dd', + 'if=/dev/zero', + 'of=' + filename, + 'bs=1', + 'seek=%d' % size, + 'count=0']) + + def mkfs_btrfs(self, location): + '''Create a btrfs filesystem on the disk.''' + self.status(msg='Creating btrfs filesystem') + cliapp.runcmd(['mkfs.btrfs', '-L', 'baserock', location]) + + def mount(self, location): + '''Mount the filesystem so it can be tweaked. + + Return path to the mount point. + The mount point is a newly created temporary directory. + The caller must call self.unmount to unmount on the return value. + + ''' + + self.status(msg='Mounting filesystem') + tempdir = tempfile.mkdtemp() + # FIXME: This hardcodes the loop device. + cliapp.runcmd(['mount', '-o', 'loop=loop0', location, tempdir]) + return tempdir + + def unmount(self, mount_point): + '''Unmount the filesystem mounted by self.mount. + + Also, remove the temporary directory. + + ''' + + self.status(msg='Unmounting filesystem') + cliapp.runcmd(['umount', mount_point]) + os.rmdir(mount_point) + + def create_factory(self, real_root, temp_root): + '''Create the default "factory" system.''' + + factory = os.path.join(real_root, 'factory') + + self.status(msg='Creating factory subvolume') + cliapp.runcmd(['btrfs', 'subvolume', 'create', factory]) + self.status(msg='Copying files to factory subvolume') + cliapp.runcmd(['cp', '-a', temp_root + '/.', factory + '/.']) + + # The kernel needs to be on the root volume. + self.status(msg='Copying boot directory to root subvolume') + factory_boot = os.path.join(factory, 'boot') + root_boot = os.path.join(real_root, 'boot') + cliapp.runcmd(['cp', '-a', factory_boot, root_boot]) + + def create_fstab(self, real_root): + '''Create an fstab.''' + + self.status(msg='Creating fstab') + fstab = os.path.join(real_root, 'factory', 'etc', 'fstab') + with open(fstab, 'w') as f: + f.write('proc /proc proc defaults 0 0\n') + f.write('sysfs /sys sysfs defaults 0 0\n') + f.write('/dev/sda / btrfs defaults,rw,noatime 0 1\n') + + def install_extlinux(self, real_root): + '''Install extlinux on the newly created disk image.''' + + self.status(msg='Creating extlinux.conf') + config = os.path.join(real_root, 'extlinux.conf') + with open(config, 'w') as f: + f.write('default linux\n') + f.write('timeout 1\n') + f.write('label linux\n') + f.write('kernel /boot/vmlinuz\n') + f.write('append root=/dev/sda rootflags=subvol=factory ' + 'init=/sbin/init rw\n') + + self.status(msg='Installing extlinux') + cliapp.runcmd(['extlinux', '--install', real_root]) + + # FIXME this hack seems to be necessary to let extlinux finish + cliapp.runcmd(['sync']) + time.sleep(2) + -- cgit v1.2.1 From 439623c9ffbd56fb1241265321e9a2dd8e1437f5 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 7 Feb 2013 11:27:52 +0000 Subject: Create hole in-process without executing dd(1) Suggested-By: Richard Maw --- writeexts.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/writeexts.py b/writeexts.py index 23473021..676a9d22 100755 --- a/writeexts.py +++ b/writeexts.py @@ -70,13 +70,10 @@ class WriteExtension(cliapp.Application): '''Create a raw disk image.''' self.status(msg='Creating empty disk image') - cliapp.runcmd( - ['dd', - 'if=/dev/zero', - 'of=' + filename, - 'bs=1', - 'seek=%d' % size, - 'count=0']) + with open(filename, 'wb') as f: + if size > 0: + f.seek(size-1) + f.write('\0') def mkfs_btrfs(self, location): '''Create a btrfs filesystem on the disk.''' -- cgit v1.2.1 From b333b5d15a15b8221f2433467a3765364a2b1e67 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 7 Feb 2013 11:29:57 +0000 Subject: Let mount choose loop device Suggested-By: Richard Maw --- writeexts.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/writeexts.py b/writeexts.py index 676a9d22..a407c059 100755 --- a/writeexts.py +++ b/writeexts.py @@ -91,8 +91,7 @@ class WriteExtension(cliapp.Application): self.status(msg='Mounting filesystem') tempdir = tempfile.mkdtemp() - # FIXME: This hardcodes the loop device. - cliapp.runcmd(['mount', '-o', 'loop=loop0', location, tempdir]) + cliapp.runcmd(['mount', '-o', 'loop', location, tempdir]) return tempdir def unmount(self, mount_point): -- cgit v1.2.1 From 65d7afb64c344a219b831eeb0e4bb0ebcbbf4ab6 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 7 Feb 2013 11:45:19 +0000 Subject: Do away with unnecessary fstab entries for proc, sys Suggested-By: Richard Maw --- writeexts.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/writeexts.py b/writeexts.py index a407c059..fae0d5d9 100755 --- a/writeexts.py +++ b/writeexts.py @@ -127,8 +127,6 @@ class WriteExtension(cliapp.Application): self.status(msg='Creating fstab') fstab = os.path.join(real_root, 'factory', 'etc', 'fstab') with open(fstab, 'w') as f: - f.write('proc /proc proc defaults 0 0\n') - f.write('sysfs /sys sysfs defaults 0 0\n') f.write('/dev/sda / btrfs defaults,rw,noatime 0 1\n') def install_extlinux(self, real_root): -- cgit v1.2.1 From 4aeffa2de48737a481cd20688f8b1b874cf7e46e Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 7 Feb 2013 11:41:04 +0000 Subject: Refactor: Add WriteExtension.create_local_system method This allows code sharing amongst all the places that create a system in a raw disk image. This also adds the creation of a factory-run subvolume, and fixes error messages for errors that happen during a disk image creation. Suggested-By: Richard Maw Suggested-By: Sam Thursfield --- writeexts.py | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index fae0d5d9..60848345 100755 --- a/writeexts.py +++ b/writeexts.py @@ -47,6 +47,31 @@ class WriteExtension(cliapp.Application): ''' self.output.write('%s\n' % (kwargs['msg'] % kwargs)) + + def create_local_system(self, temp_root, raw_disk): + '''Create a raw system image locally.''' + + size = self.get_disk_size() + self.create_raw_disk_image(raw_disk, size) + try: + self.mkfs_btrfs(raw_disk) + mp = self.mount(raw_disk) + except BaseException: + sys.stderr.write('Error creating disk image') + os.remove(raw_disk) + raise + try: + self.create_factory(mp, temp_root) + self.create_fstab(mp) + self.create_factory_run(mp) + self.install_extlinux(mp) + except BaseException, e: + sys.stderr.write('Error creating disk image') + self.unmount(mp) + os.remove(raw_disk) + raise + else: + self.unmount(mp) def get_disk_size(self): '''Parse disk size from environment.''' @@ -120,6 +145,15 @@ class WriteExtension(cliapp.Application): factory_boot = os.path.join(factory, 'boot') root_boot = os.path.join(real_root, 'boot') cliapp.runcmd(['cp', '-a', factory_boot, root_boot]) + + def create_factory_run(self, real_root): + '''Create the 'factory-run' snapshot.''' + + self.status(msg='Creating factory-run subvolume') + factory = os.path.join(real_root, 'factory') + factory_run = factory + '-run' + cliapp.runcmd( + ['btrfs', 'subvolume', 'snapshot', factory, factory_run]) def create_fstab(self, real_root): '''Create an fstab.''' @@ -139,7 +173,7 @@ class WriteExtension(cliapp.Application): f.write('timeout 1\n') f.write('label linux\n') f.write('kernel /boot/vmlinuz\n') - f.write('append root=/dev/sda rootflags=subvol=factory ' + f.write('append root=/dev/sda rootflags=subvol=factory-run ' 'init=/sbin/init rw\n') self.status(msg='Installing extlinux') -- cgit v1.2.1 From bbe187ecbef21c29c237d76b13d5b3beb2379d9e Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Tue, 12 Feb 2013 16:47:05 +0000 Subject: Add missing "import sys" to fix error messages Reported-By: Richard Maw --- writeexts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/writeexts.py b/writeexts.py index 60848345..a479f3cb 100755 --- a/writeexts.py +++ b/writeexts.py @@ -17,6 +17,7 @@ import cliapp import os import re +import sys import time import tempfile -- cgit v1.2.1 From 7b9fb69291729cb00b8073875f17d8b0e66252df Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 13 Mar 2013 13:52:52 +0000 Subject: Add ATTACH_DISKS support to kvm --- writeexts.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/writeexts.py b/writeexts.py index a479f3cb..469b8557 100755 --- a/writeexts.py +++ b/writeexts.py @@ -184,3 +184,11 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['sync']) time.sleep(2) + def parse_attach_disks(self): + '''Parse $ATTACH_DISKS into list of disks to attach.''' + + if 'ATTACH_DISKS' in os.environ: + s = os.environ['ATTACH_DISKS'] + return s.split(':') + else: + return [] -- cgit v1.2.1 From 2fa5a589912f660ddc2a08024eb511479042e29e Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 28 Mar 2013 13:28:19 +0000 Subject: Add method to parse $RAM_SIZE --- writeexts.py | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/writeexts.py b/writeexts.py index 469b8557..af48b375 100755 --- a/writeexts.py +++ b/writeexts.py @@ -74,13 +74,16 @@ class WriteExtension(cliapp.Application): else: self.unmount(mp) - def get_disk_size(self): - '''Parse disk size from environment.''' + def _parse_size(self, size): + '''Parse a size from a string. + + Return size in bytes. - size = os.environ.get('DISK_SIZE', '1G') + ''' + m = re.match('^(\d+)([kmgKMG]?)$', size) if not m: - raise morphlib.Error('Cannot parse disk size %s' % size) + return None factors = { '': 1, @@ -92,6 +95,23 @@ class WriteExtension(cliapp.Application): return int(m.group(1)) * factor + def _parse_size_from_environment(self, env_var, default): + '''Parse a size from an environment variable.''' + + size = os.environ.get(env_var, default) + bytes = self._parse_size(size) + if bytes is None: + raise morphlib.Error('Cannot parse %s value %s' % (env_var, size)) + return bytes + + def get_disk_size(self): + '''Parse disk size from environment.''' + return self._parse_size_from_environment('DISK_SIZE', '1G') + + def get_ram_size(self): + '''Parse RAM size from environment.''' + return self._parse_size_from_environment('RAM_SIZE', '1G') + def create_raw_disk_image(self, filename, size): '''Create a raw disk image.''' -- cgit v1.2.1 From 4cd3a4060146a37cdef216ee4dc346c0f7229f8c Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 2 May 2013 16:36:57 +0100 Subject: Add entry for / in fstab but only if there isn't one --- writeexts.py | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/writeexts.py b/writeexts.py index af48b375..2cdba86f 100755 --- a/writeexts.py +++ b/writeexts.py @@ -179,10 +179,26 @@ class WriteExtension(cliapp.Application): def create_fstab(self, real_root): '''Create an fstab.''' - self.status(msg='Creating fstab') + self.status(msg='Creating fstab') fstab = os.path.join(real_root, 'factory', 'etc', 'fstab') + + if os.path.exists(fstab): + with open(fstab, 'r') as f: + contents = f.read() + else: + contents = '' + + got_root = False + for line in contents.splitlines(): + words = line.split() + if len(words) >= 2 and not words[0].startswith('#'): + got_root = got_root or words[1] == '/' + + if not got_root: + contents += '\n/dev/sda / btrfs defaults,rw,noatime 0 1\n' + with open(fstab, 'w') as f: - f.write('/dev/sda / btrfs defaults,rw,noatime 0 1\n') + f.write(contents) def install_extlinux(self, real_root): '''Install extlinux on the newly created disk image.''' -- cgit v1.2.1 From 81cbbef81e44cc4bd1f3440c40a524667be698c3 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 8 May 2013 11:57:10 +0100 Subject: Make bootloader installation for disk images optional --- writeexts.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index af48b375..23b876ae 100755 --- a/writeexts.py +++ b/writeexts.py @@ -65,7 +65,8 @@ class WriteExtension(cliapp.Application): self.create_factory(mp, temp_root) self.create_fstab(mp) self.create_factory_run(mp) - self.install_extlinux(mp) + if self.bootloader_is_wanted(): + self.install_extlinux(mp) except BaseException, e: sys.stderr.write('Error creating disk image') self.unmount(mp) @@ -212,3 +213,21 @@ class WriteExtension(cliapp.Application): return s.split(':') else: return [] + + def bootloader_is_wanted(self): + '''Does the user request a bootloader? + + The user may set $BOOTLOADER to yes, no, or auto. If not + set, auto is the default and means that the bootloader will + be installed on x86-32 and x86-64, but not otherwise. + + ''' + + value = os.environ.get('BOOTLOADER', 'auto') + if value == 'auto': + if os.uname()[-1] in ['x86_32', 'x86_64']: + value = 'yes' + else: + value = 'no' + + return value == 'yes' -- cgit v1.2.1 From fbfcdcedf7b7e69f3d340b92785b3b4355e3f543 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 8 May 2013 13:27:21 +0100 Subject: Fix recognition of x86-32 Reported-by: Richard Maw --- writeexts.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index 23b876ae..6c28f7fb 100755 --- a/writeexts.py +++ b/writeexts.py @@ -223,9 +223,13 @@ class WriteExtension(cliapp.Application): ''' + def is_x86(arch): + return (arch == 'x86_64' or + (arch.startswith('i') and arch.endswith('86'))) + value = os.environ.get('BOOTLOADER', 'auto') if value == 'auto': - if os.uname()[-1] in ['x86_32', 'x86_64']: + if ix_x86(os.uname()[-1]): value = 'yes' else: value = 'no' -- cgit v1.2.1 From 2287b2a81ca8a19e41cc9edf3cb8c2c361bfa4e4 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 8 May 2013 17:04:45 +0100 Subject: Add AUTOSTART to kvm and libvirt write extensions If AUTOSTART is 'yes' then the VM will be started once it is created. If it is 'no' or undefined, then it will need to be manually started. If it is any other value, then an exception is raised. --- writeexts.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/writeexts.py b/writeexts.py index 2cdba86f..9caab839 100755 --- a/writeexts.py +++ b/writeexts.py @@ -228,3 +228,15 @@ class WriteExtension(cliapp.Application): return s.split(':') else: return [] + + def parse_autostart(self): + '''Parse $AUTOSTART to determine if VMs should be started.''' + + autostart = os.environ.get('AUTOSTART', 'no') + if autostart == 'no': + return False + elif autostart == 'yes': + return True + else: + raise cliapp.AppException('Unexpected value for AUTOSTART: %s' % + autostart) -- cgit v1.2.1 From 0dd6bf15bfb6fba31c2ce238ce352d2185ac1ba8 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 9 May 2013 13:46:10 +0000 Subject: Fix typo --- writeexts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index ed8f7ba6..48847d56 100755 --- a/writeexts.py +++ b/writeexts.py @@ -245,7 +245,7 @@ class WriteExtension(cliapp.Application): value = os.environ.get('BOOTLOADER', 'auto') if value == 'auto': - if ix_x86(os.uname()[-1]): + if is_x86(os.uname()[-1]): value = 'yes' else: value = 'no' -- cgit v1.2.1 From 302f6ca1eb5e90704b645b186d97a22d44d9f938 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Fri, 10 May 2013 14:31:39 +0000 Subject: Use a different disk system layout for rawdisk This is in preparation for making deployments able to upgrade existing baserock systems. --- writeexts.py | 61 +++++++++++++++++++++++++++++++++++------------------------- 1 file changed, 36 insertions(+), 25 deletions(-) diff --git a/writeexts.py b/writeexts.py index 48847d56..cb47f63d 100755 --- a/writeexts.py +++ b/writeexts.py @@ -62,10 +62,14 @@ class WriteExtension(cliapp.Application): os.remove(raw_disk) raise try: - self.create_factory(mp, temp_root) - self.create_fstab(mp) - self.create_factory_run(mp) + version_label = 'version1' + version_root = os.path.join(mp, 'systems', version_label) + os.makedirs(version_root) + self.create_orig(version_root, temp_root) + self.create_fstab(version_root) + self.create_run(version_root) if self.bootloader_is_wanted(): + self.install_kernel(version_root, temp_root) self.install_extlinux(mp) except BaseException, e: sys.stderr.write('Error creating disk image') @@ -152,36 +156,30 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['umount', mount_point]) os.rmdir(mount_point) - def create_factory(self, real_root, temp_root): + def create_orig(self, version_root, temp_root): '''Create the default "factory" system.''' - factory = os.path.join(real_root, 'factory') + orig = os.path.join(version_root, 'orig') - self.status(msg='Creating factory subvolume') - cliapp.runcmd(['btrfs', 'subvolume', 'create', factory]) - self.status(msg='Copying files to factory subvolume') - cliapp.runcmd(['cp', '-a', temp_root + '/.', factory + '/.']) + self.status(msg='Creating orig subvolume') + cliapp.runcmd(['btrfs', 'subvolume', 'create', orig]) + self.status(msg='Copying files to orig subvolume') + cliapp.runcmd(['cp', '-a', temp_root + '/.', orig + '/.']) - # The kernel needs to be on the root volume. - self.status(msg='Copying boot directory to root subvolume') - factory_boot = os.path.join(factory, 'boot') - root_boot = os.path.join(real_root, 'boot') - cliapp.runcmd(['cp', '-a', factory_boot, root_boot]) - - def create_factory_run(self, real_root): - '''Create the 'factory-run' snapshot.''' + def create_run(self, version_root): + '''Create the 'run' snapshot.''' - self.status(msg='Creating factory-run subvolume') - factory = os.path.join(real_root, 'factory') - factory_run = factory + '-run' + self.status(msg='Creating run subvolume') + orig = os.path.join(version_root, 'orig') + run = os.path.join(version_root, 'run') cliapp.runcmd( - ['btrfs', 'subvolume', 'snapshot', factory, factory_run]) + ['btrfs', 'subvolume', 'snapshot', orig, run]) - def create_fstab(self, real_root): + def create_fstab(self, version_root): '''Create an fstab.''' self.status(msg='Creating fstab') - fstab = os.path.join(real_root, 'factory', 'etc', 'fstab') + fstab = os.path.join(version_root, 'orig', 'etc', 'fstab') if os.path.exists(fstab): with open(fstab, 'r') as f: @@ -201,6 +199,18 @@ class WriteExtension(cliapp.Application): with open(fstab, 'w') as f: f.write(contents) + def install_kernel(self, version_root, temp_root): + '''Install the kernel outside of 'orig' or 'run' subvolumes''' + + self.status(msg='Installing kernel') + image_names = ['vmlinuz', 'zImage', 'uImage'] + kernel_dest = os.path.join(version_root, 'linux') + for name in image_names: + try_path = os.path.join(temp_root, 'boot', name) + if os.path.exists(try_path): + cliapp.runcmd(['cp', '-a', try_path, kernel_dest]) + break + def install_extlinux(self, real_root): '''Install extlinux on the newly created disk image.''' @@ -210,8 +220,9 @@ class WriteExtension(cliapp.Application): f.write('default linux\n') f.write('timeout 1\n') f.write('label linux\n') - f.write('kernel /boot/vmlinuz\n') - f.write('append root=/dev/sda rootflags=subvol=factory-run ' + f.write('kernel /systems/version1/linux\n') + f.write('append root=/dev/sda ' + 'rootflags=subvol=systems/version1/run ' 'init=/sbin/init rw\n') self.status(msg='Installing extlinux') -- cgit v1.2.1 From 4025d614bbb9faac17a53672c4bd19e69c74cb4b Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Fri, 10 May 2013 16:44:51 +0000 Subject: Add 'state' dirs as btrfs subvolumes These subvolumes exist in state/{home,opt,srv} of the disk's root. They are not mounted by default. --- writeexts.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/writeexts.py b/writeexts.py index cb47f63d..49cd89dd 100755 --- a/writeexts.py +++ b/writeexts.py @@ -65,6 +65,7 @@ class WriteExtension(cliapp.Application): version_label = 'version1' version_root = os.path.join(mp, 'systems', version_label) os.makedirs(version_root) + self.create_state(mp) self.create_orig(version_root, temp_root) self.create_fstab(version_root) self.create_run(version_root) @@ -117,6 +118,16 @@ class WriteExtension(cliapp.Application): '''Parse RAM size from environment.''' return self._parse_size_from_environment('RAM_SIZE', '1G') + def create_state(self, real_root): + '''Create the state subvolumes that are shared between versions''' + + self.status(msg='Creating state subvolumes') + os.mkdir(os.path.join(real_root, 'state')) + statedirs = ['home', 'opt', 'srv'] + for statedir in statedirs: + dirpath = os.path.join(real_root, 'state', statedir) + cliapp.runcmd(['btrfs', 'subvolume', 'create', dirpath]) + def create_raw_disk_image(self, filename, size): '''Create a raw disk image.''' -- cgit v1.2.1 From fb623be664e18d06359b21a9b420a24bd22e23f8 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Mon, 13 May 2013 15:53:51 +0000 Subject: Use a kernel named 'kernel' instead of 'linux' --- writeexts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/writeexts.py b/writeexts.py index c74a4b76..f6465886 100755 --- a/writeexts.py +++ b/writeexts.py @@ -215,7 +215,7 @@ class WriteExtension(cliapp.Application): self.status(msg='Installing kernel') image_names = ['vmlinuz', 'zImage', 'uImage'] - kernel_dest = os.path.join(version_root, 'linux') + kernel_dest = os.path.join(version_root, 'kernel') for name in image_names: try_path = os.path.join(temp_root, 'boot', name) if os.path.exists(try_path): @@ -231,7 +231,7 @@ class WriteExtension(cliapp.Application): f.write('default linux\n') f.write('timeout 1\n') f.write('label linux\n') - f.write('kernel /systems/version1/linux\n') + f.write('kernel /systems/version1/kernel\n') f.write('append root=/dev/sda ' 'rootflags=subvol=systems/version1/run ' 'init=/sbin/init rw\n') -- cgit v1.2.1 From 764983c38c8dbd331ecdfbe4d8fa1e60a7aa30be Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Sun, 19 May 2013 18:59:11 +0000 Subject: Allow installing extlinux to other system versions. --- writeexts.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/writeexts.py b/writeexts.py index f6465886..10b17e76 100755 --- a/writeexts.py +++ b/writeexts.py @@ -71,7 +71,7 @@ class WriteExtension(cliapp.Application): self.create_run(version_root) if self.bootloader_is_wanted(): self.install_kernel(version_root, temp_root) - self.install_extlinux(mp) + self.install_extlinux(mp, version_label) except BaseException, e: sys.stderr.write('Error creating disk image') self.unmount(mp) @@ -222,7 +222,7 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['cp', '-a', try_path, kernel_dest]) break - def install_extlinux(self, real_root): + def install_extlinux(self, real_root, version_label): '''Install extlinux on the newly created disk image.''' self.status(msg='Creating extlinux.conf') @@ -231,9 +231,9 @@ class WriteExtension(cliapp.Application): f.write('default linux\n') f.write('timeout 1\n') f.write('label linux\n') - f.write('kernel /systems/version1/kernel\n') + f.write('kernel /systems/' + version_label + '/kernel\n') f.write('append root=/dev/sda ' - 'rootflags=subvol=systems/version1/run ' + 'rootflags=subvol=systems/' + version_label + '/run ' 'init=/sbin/init rw\n') self.status(msg='Installing extlinux') -- cgit v1.2.1 From a1b219eb53f132d716e41238dd8239c885192bcb Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Thu, 23 May 2013 15:20:52 +0000 Subject: Use the name factory for the first system version. --- writeexts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index 10b17e76..9e98747c 100755 --- a/writeexts.py +++ b/writeexts.py @@ -62,7 +62,7 @@ class WriteExtension(cliapp.Application): os.remove(raw_disk) raise try: - version_label = 'version1' + version_label = 'factory' version_root = os.path.join(mp, 'systems', version_label) os.makedirs(version_root) self.create_state(mp) -- cgit v1.2.1 From 4f405d42db8ed828b95df98a3f69e5a66537e8e9 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Tue, 28 May 2013 16:09:49 +0000 Subject: Remove executable permissions from *.py files that have them --- writeexts.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 writeexts.py diff --git a/writeexts.py b/writeexts.py old mode 100755 new mode 100644 -- cgit v1.2.1 From 861ee2270f257d3d5e2a840315b2b5a6f664ae20 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Wed, 29 May 2013 11:04:27 +0000 Subject: Make create_local_system fail if DISK_SIZE isn't defined --- writeexts.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index 9e98747c..df4cec33 100644 --- a/writeexts.py +++ b/writeexts.py @@ -53,6 +53,8 @@ class WriteExtension(cliapp.Application): '''Create a raw system image locally.''' size = self.get_disk_size() + if not size: + raise cliapp.AppException('DISK_SIZE is not defined') self.create_raw_disk_image(raw_disk, size) try: self.mkfs_btrfs(raw_disk) @@ -105,6 +107,8 @@ class WriteExtension(cliapp.Application): '''Parse a size from an environment variable.''' size = os.environ.get(env_var, default) + if size is None: + return None bytes = self._parse_size(size) if bytes is None: raise morphlib.Error('Cannot parse %s value %s' % (env_var, size)) @@ -112,7 +116,7 @@ class WriteExtension(cliapp.Application): def get_disk_size(self): '''Parse disk size from environment.''' - return self._parse_size_from_environment('DISK_SIZE', '1G') + return self._parse_size_from_environment('DISK_SIZE', None) def get_ram_size(self): '''Parse RAM size from environment.''' -- cgit v1.2.1 From 71bc1d10f601dab9023c73e207bc787923b4f55c Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Thu, 13 Jun 2013 10:40:09 +0000 Subject: Set up a symlink to the default system version in rawdisk/kvm/vbox deployments Also Change them to use the "default" symlink in the extlinux.conf they create, instead of hardcoding the current system version name --- writeexts.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/writeexts.py b/writeexts.py index df4cec33..de4189f8 100644 --- a/writeexts.py +++ b/writeexts.py @@ -71,9 +71,10 @@ class WriteExtension(cliapp.Application): self.create_orig(version_root, temp_root) self.create_fstab(version_root) self.create_run(version_root) + os.symlink(version_label, os.path.join(mp, 'systems', 'default')) if self.bootloader_is_wanted(): self.install_kernel(version_root, temp_root) - self.install_extlinux(mp, version_label) + self.install_extlinux(mp) except BaseException, e: sys.stderr.write('Error creating disk image') self.unmount(mp) @@ -226,7 +227,7 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['cp', '-a', try_path, kernel_dest]) break - def install_extlinux(self, real_root, version_label): + def install_extlinux(self, real_root): '''Install extlinux on the newly created disk image.''' self.status(msg='Creating extlinux.conf') @@ -235,9 +236,9 @@ class WriteExtension(cliapp.Application): f.write('default linux\n') f.write('timeout 1\n') f.write('label linux\n') - f.write('kernel /systems/' + version_label + '/kernel\n') + f.write('kernel /systems/default/kernel\n') f.write('append root=/dev/sda ' - 'rootflags=subvol=systems/' + version_label + '/run ' + 'rootflags=subvol=systems/default/run ' 'init=/sbin/init rw\n') self.status(msg='Installing extlinux') -- cgit v1.2.1 From 2a52a1b530e2a430a34a28255ec9a2dff4f34534 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Mon, 10 Jun 2013 22:32:09 +0000 Subject: Write extensions: Flush output when using status() This is required to get real-time output and the timestamps meaning anything useful. --- writeexts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/writeexts.py b/writeexts.py index de4189f8..410b8c9f 100644 --- a/writeexts.py +++ b/writeexts.py @@ -48,6 +48,7 @@ class WriteExtension(cliapp.Application): ''' self.output.write('%s\n' % (kwargs['msg'] % kwargs)) + self.output.flush() def create_local_system(self, temp_root, raw_disk): '''Create a raw system image locally.''' -- cgit v1.2.1 From 21fa4e7da4c566ecaba3cc70dedb74c982674832 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Tue, 2 Jul 2013 07:24:30 +0000 Subject: Allow to set the number of cpus for virtualbox and kvm deployments. --- writeexts.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/writeexts.py b/writeexts.py index 410b8c9f..2a54d757 100644 --- a/writeexts.py +++ b/writeexts.py @@ -124,6 +124,10 @@ class WriteExtension(cliapp.Application): '''Parse RAM size from environment.''' return self._parse_size_from_environment('RAM_SIZE', '1G') + def get_vcpu_count(self): + '''Parse the virtual cpu count from environment.''' + return self._parse_size_from_environment('VCPUS', '1') + def create_state(self, real_root): '''Create the state subvolumes that are shared between versions''' -- cgit v1.2.1 From cc0ceb1a96e685574067fffeb8e9f1f7f23ec8d4 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Wed, 17 Jul 2013 12:48:43 +0000 Subject: Import morphlib as we are using morphlib.Error --- writeexts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/writeexts.py b/writeexts.py index 2a54d757..9dbc77e6 100644 --- a/writeexts.py +++ b/writeexts.py @@ -21,6 +21,7 @@ import sys import time import tempfile +import morphlib class WriteExtension(cliapp.Application): -- cgit v1.2.1 From 307466b3ee56be7978d202219a6b4666176825d2 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Wed, 12 Feb 2014 11:40:07 +0000 Subject: Adding syslinux 'menu.c32' file during the deployment. We will need this file to enable a bootloader menu to choose between OS after an upgrade. --- writeexts.py | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index 9dbc77e6..dd0e8b79 100644 --- a/writeexts.py +++ b/writeexts.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -76,6 +76,7 @@ class WriteExtension(cliapp.Application): os.symlink(version_label, os.path.join(mp, 'systems', 'default')) if self.bootloader_is_wanted(): self.install_kernel(version_root, temp_root) + self.install_syslinux_menu(mp, version_root) self.install_extlinux(mp) except BaseException, e: sys.stderr.write('Error creating disk image') @@ -254,6 +255,23 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['sync']) time.sleep(2) + def install_syslinux_menu(self, real_root, version_root): + '''Make syslinux/extlinux menu binary available. + + The syslinux boot menu is compiled to a file named menu.c32. Extlinux + searches a few places for this file but it does not know to look inside + our subvolume, so we copy it to the filesystem root. + + If the file is not available, the bootloader will still work but will + not be able to show a menu. + + ''' + menu_file = os.path.join(version_root, 'orig', + 'usr', 'share', 'syslinux', 'menu.c32') + if os.path.isfile(menu_file): + self.status(msg='Copying menu.c32') + shutil.copy(menu_file, real_root) + def parse_attach_disks(self): '''Parse $ATTACH_DISKS into list of disks to attach.''' -- cgit v1.2.1 From 206e9f65972b8d6abb8a0f7efb460fdfe8b722c0 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Wed, 12 Feb 2014 19:03:05 +0000 Subject: deploy: Finish off the Btrfs system layout implementation The shared state directories defined in writeexts.py (/var, /home etc.) are now separate Btrfs subvolumes that are mounted in place using fstab. There are some warnings on mounting /var and /srv about the mountpoint not being empty. Not yet investigated. If a configure extension has already added / to the fstab, use the device it chose rather than assuming /dev/sda. This is required for the vdaboot.configure extension that we use for OpenStack deployments. Similarly, if a configure extension has added an entry for a state directory in /etc/fstab already, we don't replace it with a /state/xxx directory. That's only done as a default behaviour. --- writeexts.py | 183 +++++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 141 insertions(+), 42 deletions(-) diff --git a/writeexts.py b/writeexts.py index dd0e8b79..a07c697f 100644 --- a/writeexts.py +++ b/writeexts.py @@ -17,12 +17,65 @@ import cliapp import os import re +import shutil import sys import time import tempfile import morphlib + +class Fstab(object): + '''Small helper class for parsing and adding lines to /etc/fstab.''' + + # There is an existing Python helper library for editing of /etc/fstab. + # However it is unmaintained and has an incompatible license (GPL3). + # + # https://code.launchpad.net/~computer-janitor-hackers/python-fstab/trunk + + def __init__(self, filepath='/etc/fstab'): + if os.path.exists(filepath): + with open(filepath, 'r') as f: + self.text= f.read() + else: + self.text = '' + self.filepath = filepath + self.lines_added = 0 + + def get_mounts(self): + '''Return list of mount devices and targets in /etc/fstab. + + Return value is a dict of target -> device. + ''' + mounts = dict() + for line in self.text.splitlines(): + words = line.split() + if len(words) >= 2 and not words[0].startswith('#'): + device, target = words[0:2] + mounts[target] = device + return mounts + + def add_line(self, line): + '''Add a new entry to /etc/fstab. + + Lines are appended, and separated from any entries made by configure + extensions with a comment. + + ''' + if self.lines_added == 0: + if len(self.text) == 0 or self.text[-1] is not '\n': + self.text += '\n' + self.text += '# Morph default system layout\n' + self.lines_added += 1 + + self.text += line + '\n' + + def write(self): + '''Rewrite the fstab file to include all new entries.''' + with morphlib.savefile.SaveFile(self.filepath, 'w') as f: + f.write(self.text) + + class WriteExtension(cliapp.Application): '''A base class for deployment write extensions. @@ -53,7 +106,6 @@ class WriteExtension(cliapp.Application): def create_local_system(self, temp_root, raw_disk): '''Create a raw system image locally.''' - size = self.get_disk_size() if not size: raise cliapp.AppException('DISK_SIZE is not defined') @@ -66,20 +118,10 @@ class WriteExtension(cliapp.Application): os.remove(raw_disk) raise try: - version_label = 'factory' - version_root = os.path.join(mp, 'systems', version_label) - os.makedirs(version_root) - self.create_state(mp) - self.create_orig(version_root, temp_root) - self.create_fstab(version_root) - self.create_run(version_root) - os.symlink(version_label, os.path.join(mp, 'systems', 'default')) - if self.bootloader_is_wanted(): - self.install_kernel(version_root, temp_root) - self.install_syslinux_menu(mp, version_root) - self.install_extlinux(mp) + self.create_btrfs_system_layout( + temp_root, mp, version_label='factory') except BaseException, e: - sys.stderr.write('Error creating disk image') + sys.stderr.write('Error creating Btrfs system layout') self.unmount(mp) os.remove(raw_disk) raise @@ -130,16 +172,6 @@ class WriteExtension(cliapp.Application): '''Parse the virtual cpu count from environment.''' return self._parse_size_from_environment('VCPUS', '1') - def create_state(self, real_root): - '''Create the state subvolumes that are shared between versions''' - - self.status(msg='Creating state subvolumes') - os.mkdir(os.path.join(real_root, 'state')) - statedirs = ['home', 'opt', 'srv'] - for statedir in statedirs: - dirpath = os.path.join(real_root, 'state', statedir) - cliapp.runcmd(['btrfs', 'subvolume', 'create', dirpath]) - def create_raw_disk_image(self, filename, size): '''Create a raw disk image.''' @@ -179,6 +211,34 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['umount', mount_point]) os.rmdir(mount_point) + def create_btrfs_system_layout(self, temp_root, mountpoint, version_label): + '''Separate base OS versions from state using subvolumes. + + ''' + version_root = os.path.join(mountpoint, 'systems', version_label) + state_root = os.path.join(mountpoint, 'state') + + os.makedirs(version_root) + os.makedirs(state_root) + + self.create_orig(version_root, temp_root) + system_dir = os.path.join(version_root, 'orig') + + state_dirs = self.complete_fstab_for_btrfs_layout(system_dir) + + for state_dir in state_dirs: + self.create_state_subvolume(system_dir, mountpoint, state_dir) + + self.create_run(version_root) + + os.symlink( + version_label, os.path.join(mountpoint, 'systems', 'default')) + + if self.bootloader_is_wanted(): + self.install_kernel(version_root, temp_root) + self.install_syslinux_menu(mountpoint, version_root) + self.install_extlinux(mountpoint) + def create_orig(self, version_root, temp_root): '''Create the default "factory" system.''' @@ -198,29 +258,68 @@ class WriteExtension(cliapp.Application): cliapp.runcmd( ['btrfs', 'subvolume', 'snapshot', orig, run]) - def create_fstab(self, version_root): - '''Create an fstab.''' + def create_state_subvolume(self, system_dir, mountpoint, state_subdir): + '''Create a shared state subvolume. - self.status(msg='Creating fstab') - fstab = os.path.join(version_root, 'orig', 'etc', 'fstab') + We need to move any files added to the temporary rootfs by the + configure extensions to their correct home. For example, they might + have added keys in `/root/.ssh` which we now need to transfer to + `/state/root/.ssh`. - if os.path.exists(fstab): - with open(fstab, 'r') as f: - contents = f.read() - else: - contents = '' + ''' + self.status(msg='Creating %s subvolume' % state_subdir) + subvolume = os.path.join(mountpoint, 'state', state_subdir) + cliapp.runcmd(['btrfs', 'subvolume', 'create', subvolume]) + os.chmod(subvolume, 0755) + + existing_state_dir = os.path.join(system_dir, state_subdir) + files = [] + if os.path.exists(existing_state_dir): + files = os.listdir(existing_state_dir) + if len(files) > 0: + self.status(msg='Moving existing data to %s subvolume' % subvolume) + for filename in files: + filepath = os.path.join(existing_state_dir, filename) + shutil.move(filepath, subvolume) + + def complete_fstab_for_btrfs_layout(self, system_dir): + '''Fill in /etc/fstab entries for the default Btrfs disk layout. + + In the future we should move this code out of the write extension and + in to a configure extension. To do that, though, we need some way of + informing the configure extension what layout should be used. Right now + a configure extension doesn't know if the system is going to end up as + a Btrfs disk image, a tarfile or something else and so it can't come + up with a sensible default fstab. + + Configuration extensions can already create any /etc/fstab that they + like. This function only fills in entries that are missing, so if for + example the user configured /home to be on a separate partition, that + decision will be honoured and /state/home will not be created. - got_root = False - for line in contents.splitlines(): - words = line.split() - if len(words) >= 2 and not words[0].startswith('#'): - got_root = got_root or words[1] == '/' + ''' + shared_state_dirs = {'home', 'root', 'opt', 'srv', 'var'} - if not got_root: - contents += '\n/dev/sda / btrfs defaults,rw,noatime 0 1\n' + fstab = Fstab(os.path.join(system_dir, 'etc', 'fstab')) + existing_mounts = fstab.get_mounts() - with open(fstab, 'w') as f: - f.write(contents) + if '/' in existing_mounts: + root_device = existing_mounts['/'] + else: + root_device = '/dev/sda' + fstab.add_line('/dev/sda / btrfs defaults,rw,noatime 0 1') + + state_dirs_to_create = set() + for state_dir in shared_state_dirs: + if '/' + state_dir not in existing_mounts: + state_dirs_to_create.add(state_dir) + state_subvol = os.path.join('/state', state_dir) + fstab.add_line( + '%s /%s btrfs subvol=%s,defaults,rw,noatime 0 2' % + (root_device, state_dir, state_subvol)) + + fstab.write() + return state_dirs_to_create def install_kernel(self, version_root, temp_root): '''Install the kernel outside of 'orig' or 'run' subvolumes''' -- cgit v1.2.1 From ccaed5352ce9413cee3d7ab4b60514d0bc4fdbaa Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Mon, 17 Feb 2014 15:36:56 +0000 Subject: Make parse_autostart() into more general get_environment_boolean() Also, be more flexible when parsing environment booleans -- convert to lower case and match 0/1 and true/false as well as yes/no. --- writeexts.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/writeexts.py b/writeexts.py index a07c697f..1849f406 100644 --- a/writeexts.py +++ b/writeexts.py @@ -402,14 +402,14 @@ class WriteExtension(cliapp.Application): return value == 'yes' - def parse_autostart(self): - '''Parse $AUTOSTART to determine if VMs should be started.''' + def get_environment_boolean(self, variable): + '''Parse a yes/no boolean passed through the environment.''' - autostart = os.environ.get('AUTOSTART', 'no') - if autostart == 'no': + value = os.environ.get(variable, 'no').lower() + if value in ['no', '0', 'false']: return False - elif autostart == 'yes': + elif value in ['yes', '1', 'true']: return True else: - raise cliapp.AppException('Unexpected value for AUTOSTART: %s' % - autostart) + raise cliapp.AppException('Unexpected value for %s: %s' % + (variable, value)) -- cgit v1.2.1 From c1197bd420c9a9b97b7263def4215d93cfc2ddbd Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Fri, 4 Apr 2014 11:40:38 +0000 Subject: Adding support to add extra kernel args in extlinux.conf --- writeexts.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index 1849f406..bff21e8d 100644 --- a/writeexts.py +++ b/writeexts.py @@ -333,11 +333,15 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['cp', '-a', try_path, kernel_dest]) break + def get_extra_kernel_args(self): + return os.environ.get('KERNEL_ARGS', '') + def install_extlinux(self, real_root): '''Install extlinux on the newly created disk image.''' self.status(msg='Creating extlinux.conf') config = os.path.join(real_root, 'extlinux.conf') + kernel_args = self.get_extra_kernel_args() with open(config, 'w') as f: f.write('default linux\n') f.write('timeout 1\n') @@ -345,7 +349,7 @@ class WriteExtension(cliapp.Application): f.write('kernel /systems/default/kernel\n') f.write('append root=/dev/sda ' 'rootflags=subvol=systems/default/run ' - 'init=/sbin/init rw\n') + '%s init=/sbin/init rw\n' % (kernel_args)) self.status(msg='Installing extlinux') cliapp.runcmd(['extlinux', '--install', real_root]) -- cgit v1.2.1 From f7b4039973398400e355c5ea2a068dac4bbc994b Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Mon, 14 Apr 2014 12:35:26 +0000 Subject: deploy: Share SSH connectivity check in the common writeexts.py code Also, change it to log the real error message in morph.log before raising a more general exception to the user. --- writeexts.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/writeexts.py b/writeexts.py index bff21e8d..3f9c33d5 100644 --- a/writeexts.py +++ b/writeexts.py @@ -15,6 +15,7 @@ import cliapp +import logging import os import re import shutil @@ -417,3 +418,11 @@ class WriteExtension(cliapp.Application): else: raise cliapp.AppException('Unexpected value for %s: %s' % (variable, value)) + + def check_ssh_connectivity(self, ssh_host): + try: + cliapp.ssh_runcmd(ssh_host, ['true']) + except cliapp.AppException as e: + logging.error("Error checking SSH connectivity: %s", str(e)) + raise cliapp.AppException( + 'Unable to SSH to %s: %s' % (ssh_host, e)) -- cgit v1.2.1 From 20d6402959869913cb3cd9b78348325e7098bf53 Mon Sep 17 00:00:00 2001 From: Daniel Silverstone Date: Fri, 16 May 2014 09:50:24 +0000 Subject: Fix state subvolume generator to preserve permissions shutil.move() does not preserve permissions, file modes, ownerships etc, resulting in much confusion when prepopulating a non-root user during deployment. This change to `mv` fixes that. --- writeexts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index 3f9c33d5..b4912db1 100644 --- a/writeexts.py +++ b/writeexts.py @@ -281,7 +281,7 @@ class WriteExtension(cliapp.Application): self.status(msg='Moving existing data to %s subvolume' % subvolume) for filename in files: filepath = os.path.join(existing_state_dir, filename) - shutil.move(filepath, subvolume) + cliapp.runcmd(['mv', filepath, subvolume]) def complete_fstab_for_btrfs_layout(self, system_dir): '''Fill in /etc/fstab entries for the default Btrfs disk layout. -- cgit v1.2.1 From 0ad75f58510350b460b5456b137eed1cae006bd9 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Mon, 2 Jun 2014 11:09:29 +0000 Subject: Add initramfs support to write extensions that produce disks If INITRAMFS_PATH is specified and the file exists, then the produced kernel command line will use root=UUID=$uuid_of_created_disk rather than root=/dev/sda, which may be incorrect. Help files have been updated to mention the new option. This leads to an unfortunate duplication of the path to the initramfs, in both the location field of the nested deployment and the INITRAMFS_PATH of the disk image creation. However, an initramfs could be produced by a chunk and put in the same place, so it doesn't make sense to couple the rawdisk and initramfs write extensions to remove this duplication. Similarly, there may be multiple valid initramfs in the rootfs e.g. extlinux loads a hypervisor, which is Linux + initramfs, and the initramfs then boots a guest Linux system, which uses a different initramfs. This makes it important to explicitly let the rootfs write extensions know which to use, or not as the case may be. util-linux's blkid is required, since the busybox version ignores the options to filter its output, and parsing the output is undesirable. Because syslinux's btrfs subvolume support is limited to being able to use a non-0 default subvolume, the initramfs has to be copied out of the run-time rootfs subvolume and into the boot subvolume. This pushed the required disk space of a minimal system over the 512M threshold because we do not have the userland tooling support to be able to do a btrfs file contents clone. --- writeexts.py | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 54 insertions(+), 8 deletions(-) diff --git a/writeexts.py b/writeexts.py index b4912db1..334dc15c 100644 --- a/writeexts.py +++ b/writeexts.py @@ -120,7 +120,8 @@ class WriteExtension(cliapp.Application): raise try: self.create_btrfs_system_layout( - temp_root, mp, version_label='factory') + temp_root, mp, version_label='factory', + disk_uuid=self.get_uuid(raw_disk)) except BaseException, e: sys.stderr.write('Error creating Btrfs system layout') self.unmount(mp) @@ -186,6 +187,13 @@ class WriteExtension(cliapp.Application): '''Create a btrfs filesystem on the disk.''' self.status(msg='Creating btrfs filesystem') cliapp.runcmd(['mkfs.btrfs', '-L', 'baserock', location]) + + def get_uuid(self, location): + '''Get the UUID of a block device's file system.''' + # Requires util-linux blkid; busybox one ignores options and + # lies by exiting successfully. + return cliapp.runcmd(['blkid', '-s', 'UUID', '-o', 'value', + location]).strip() def mount(self, location): '''Mount the filesystem so it can be tweaked. @@ -212,10 +220,12 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['umount', mount_point]) os.rmdir(mount_point) - def create_btrfs_system_layout(self, temp_root, mountpoint, version_label): + def create_btrfs_system_layout(self, temp_root, mountpoint, version_label, + disk_uuid=None): '''Separate base OS versions from state using subvolumes. ''' + initramfs = self.find_initramfs(temp_root) version_root = os.path.join(mountpoint, 'systems', version_label) state_root = os.path.join(mountpoint, 'state') @@ -238,7 +248,12 @@ class WriteExtension(cliapp.Application): if self.bootloader_is_wanted(): self.install_kernel(version_root, temp_root) self.install_syslinux_menu(mountpoint, version_root) - self.install_extlinux(mountpoint) + if initramfs is not None: + self.install_initramfs(initramfs, version_root) + self.install_extlinux(mountpoint, disk_uuid) + else: + self.install_extlinux(mountpoint) + def create_orig(self, version_root, temp_root): '''Create the default "factory" system.''' @@ -322,6 +337,29 @@ class WriteExtension(cliapp.Application): fstab.write() return state_dirs_to_create + def find_initramfs(self, temp_root): + '''Check whether the rootfs has an initramfs. + + Uses the INITRAMFS_PATH option to locate it. + ''' + if 'INITRAMFS_PATH' in os.environ: + initramfs = os.path.join(temp_root, os.environ['INITRAMFS_PATH']) + if not os.path.exists(initramfs): + raise morphlib.Error('INITRAMFS_PATH specified, ' + 'but file does not exist') + return initramfs + return None + + def install_initramfs(self, initramfs_path, version_root): + '''Install the initramfs outside of 'orig' or 'run' subvolumes. + + This is required because syslinux doesn't traverse subvolumes when + loading the kernel or initramfs. + ''' + self.status(msg='Installing initramfs') + initramfs_dest = os.path.join(version_root, 'initramfs') + cliapp.runcmd(['cp', '-a', initramfs_path, initramfs_dest]) + def install_kernel(self, version_root, temp_root): '''Install the kernel outside of 'orig' or 'run' subvolumes''' @@ -337,20 +375,28 @@ class WriteExtension(cliapp.Application): def get_extra_kernel_args(self): return os.environ.get('KERNEL_ARGS', '') - def install_extlinux(self, real_root): + def install_extlinux(self, real_root, disk_uuid=None): '''Install extlinux on the newly created disk image.''' self.status(msg='Creating extlinux.conf') config = os.path.join(real_root, 'extlinux.conf') - kernel_args = self.get_extra_kernel_args() + kernel_args = ( + 'rw ' # ro ought to work, but we don't test that regularly + 'init=/sbin/init ' # default, but it doesn't hurt to be explicit + 'rootfstype=btrfs ' # required when using initramfs, also boots + # faster when specified without initramfs + 'rootflags=subvol=systems/default/run ') # boot runtime subvol + kernel_args += 'root=%s ' % ('/dev/sda' if disk_uuid is None + else 'UUID=%s' % disk_uuid) + kernel_args += self.get_extra_kernel_args() with open(config, 'w') as f: f.write('default linux\n') f.write('timeout 1\n') f.write('label linux\n') f.write('kernel /systems/default/kernel\n') - f.write('append root=/dev/sda ' - 'rootflags=subvol=systems/default/run ' - '%s init=/sbin/init rw\n' % (kernel_args)) + if disk_uuid is not None: + f.write('initrd /systems/default/initramfs\n') + f.write('append %s\n' % kernel_args) self.status(msg='Installing extlinux') cliapp.runcmd(['extlinux', '--install', real_root]) -- cgit v1.2.1 From 1d9b7fa268c971f363b4139a536bb8facb8d023e Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 4 Jun 2014 11:03:06 +0000 Subject: Make uuid mandatory when calling create_btrfs_system_layout It's only ever called from the core of the write extensions library, so we can change it to be mandatory in all cases. install_extlinux is left with the uuid being optional and defaulting to /dev/sda, since it is called directly from the rawdisk write extension currently, and upgrades of systems that have an initramfs will be part of a later patch series. --- writeexts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index 334dc15c..fb7261f6 100644 --- a/writeexts.py +++ b/writeexts.py @@ -221,7 +221,7 @@ class WriteExtension(cliapp.Application): os.rmdir(mount_point) def create_btrfs_system_layout(self, temp_root, mountpoint, version_label, - disk_uuid=None): + disk_uuid): '''Separate base OS versions from state using subvolumes. ''' -- cgit v1.2.1 From ba6aa8eb7cb8bd249e810972de339b2832f7a665 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 4 Jun 2014 11:10:43 +0000 Subject: Use UUID in fstab entries if provided This makes systems use the UUID of the disk in the fstab when there is no pre-existing fstab entry for /. This happens whether the system has an initramfs or not, since it should be harmless, and by this point we're in userland, so can know what the UUIDs are. Passing the UUID to `complete_fstab_for_btrfs_layout` is optional, and defaults to the old behaviour of using /dev/sda, since it is called directly by some write extensions for doing upgrades, and upgrading systems that use an initramfs will be part of a later patch. --- writeexts.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/writeexts.py b/writeexts.py index fb7261f6..d6f23e0d 100644 --- a/writeexts.py +++ b/writeexts.py @@ -235,7 +235,8 @@ class WriteExtension(cliapp.Application): self.create_orig(version_root, temp_root) system_dir = os.path.join(version_root, 'orig') - state_dirs = self.complete_fstab_for_btrfs_layout(system_dir) + state_dirs = self.complete_fstab_for_btrfs_layout(system_dir, + disk_uuid) for state_dir in state_dirs: self.create_state_subvolume(system_dir, mountpoint, state_dir) @@ -298,7 +299,7 @@ class WriteExtension(cliapp.Application): filepath = os.path.join(existing_state_dir, filename) cliapp.runcmd(['mv', filepath, subvolume]) - def complete_fstab_for_btrfs_layout(self, system_dir): + def complete_fstab_for_btrfs_layout(self, system_dir, rootfs_uuid=None): '''Fill in /etc/fstab entries for the default Btrfs disk layout. In the future we should move this code out of the write extension and @@ -322,8 +323,9 @@ class WriteExtension(cliapp.Application): if '/' in existing_mounts: root_device = existing_mounts['/'] else: - root_device = '/dev/sda' - fstab.add_line('/dev/sda / btrfs defaults,rw,noatime 0 1') + root_device = ('/dev/sda' if rootfs_uuid is None else + 'UUID=%s' % rootfs_uuid) + fstab.add_line('%s / btrfs defaults,rw,noatime 0 1' % root_device) state_dirs_to_create = set() for state_dir in shared_state_dirs: -- cgit v1.2.1 From 21c9683278cb01a4e9425923740124217ce47723 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 10 Jun 2014 17:18:37 +0000 Subject: Check for presence of btrfs before trying to use it If btrfs is not present in the kernel we end up with strange output like this: Error creating disk image2014-06-10 16:00:40 [devel-system-x86_64-generic][my-raw-disk-image][rawdisk.write]Failure to create disk image at /src/tmp/testdev.img ERROR: Command failed: mount -o loop /src/tmp/testdev.img /src/tmp/deployments/tmpQ7wXO1/tmp4lVDcu/tmpvHSzDE mount: mounting /dev/loop0 on /src/tmp/deployments/tmpQ7wXO1/tmp4lVDcu/tmpvHSzDE failed: Device or resource busy To avoid this confusing error, Morph should explicitly check first. --- writeexts.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index d6f23e0d..74587bd1 100644 --- a/writeexts.py +++ b/writeexts.py @@ -104,7 +104,18 @@ class WriteExtension(cliapp.Application): self.output.write('%s\n' % (kwargs['msg'] % kwargs)) self.output.flush() - + + def check_for_btrfs_in_deployment_host_kernel(self): + with open('/proc/filesystems') as f: + text = f.read() + return '\tbtrfs\n' in text + + def require_btrfs_in_deployment_host_kernel(self): + if not self.check_for_btrfs_in_deployment_host_kernel(): + raise cliapp.AppException( + 'Error: Btrfs is required for this deployment, but was not ' + 'detected in the kernel of the machine that is running Morph.') + def create_local_system(self, temp_root, raw_disk): '''Create a raw system image locally.''' size = self.get_disk_size() -- cgit v1.2.1 From 37b7e39aa7c2216c385b87f8c986821ad2cc1d52 Mon Sep 17 00:00:00 2001 From: James Thomas Date: Wed, 30 Jul 2014 17:46:58 +0100 Subject: Support setting a different root device using ROOT_DEVICE --- writeexts.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/writeexts.py b/writeexts.py index 74587bd1..1b8770e2 100644 --- a/writeexts.py +++ b/writeexts.py @@ -334,7 +334,7 @@ class WriteExtension(cliapp.Application): if '/' in existing_mounts: root_device = existing_mounts['/'] else: - root_device = ('/dev/sda' if rootfs_uuid is None else + root_device = (self.get_root_device() if rootfs_uuid is None else 'UUID=%s' % rootfs_uuid) fstab.add_line('%s / btrfs defaults,rw,noatime 0 1' % root_device) @@ -388,6 +388,9 @@ class WriteExtension(cliapp.Application): def get_extra_kernel_args(self): return os.environ.get('KERNEL_ARGS', '') + def get_root_device(self): + return os.environ.get('ROOT_DEVICE', '/dev/sda') + def install_extlinux(self, real_root, disk_uuid=None): '''Install extlinux on the newly created disk image.''' @@ -399,7 +402,7 @@ class WriteExtension(cliapp.Application): 'rootfstype=btrfs ' # required when using initramfs, also boots # faster when specified without initramfs 'rootflags=subvol=systems/default/run ') # boot runtime subvol - kernel_args += 'root=%s ' % ('/dev/sda' if disk_uuid is None + kernel_args += 'root=%s ' % (self.get_root_device() if disk_uuid is None else 'UUID=%s' % disk_uuid) kernel_args += self.get_extra_kernel_args() with open(config, 'w') as f: -- cgit v1.2.1 From d071207ed1a4a895219cb5ce4731eba52319b8b3 Mon Sep 17 00:00:00 2001 From: James Thomas Date: Wed, 30 Jul 2014 18:15:49 +0100 Subject: Make bootloader config/install more generic Remove the BOOTLOADER environment variable and instead favour BOOTLOADER_CONFIG_FORMAT to set the desired bootloader format, and BOOTLOADER_INSTALL to set the type of bootloader to install. For example, since u-boot can boot using extlinux.conf files, it's conceivable that someone might want to do CONFIG_FORMAT=extlinux.conf, INSTALL=u-boot. However, for most platforms you would want to set INSTALL to "none" --- writeexts.py | 69 +++++++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 52 insertions(+), 17 deletions(-) diff --git a/writeexts.py b/writeexts.py index 1b8770e2..dec318d9 100644 --- a/writeexts.py +++ b/writeexts.py @@ -257,15 +257,15 @@ class WriteExtension(cliapp.Application): os.symlink( version_label, os.path.join(mountpoint, 'systems', 'default')) - if self.bootloader_is_wanted(): + if self.bootloader_config_is_wanted(): self.install_kernel(version_root, temp_root) self.install_syslinux_menu(mountpoint, version_root) if initramfs is not None: self.install_initramfs(initramfs, version_root) - self.install_extlinux(mountpoint, disk_uuid) + self.generate_bootloader_config(mountpoint, disk_uuid) else: - self.install_extlinux(mountpoint) - + self.generate_bootloader_config(mountpoint) + self.install_bootloader(mountpoint) def create_orig(self, version_root, temp_root): '''Create the default "factory" system.''' @@ -385,13 +385,36 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['cp', '-a', try_path, kernel_dest]) break + def get_bootloader_install(self): + # Do we actually want to install the bootloader? + # Set this to "none" to prevent the install + return os.environ.get('BOOTLOADER_INSTALL', 'extlinux') + + def get_bootloader_config_format(self): + # The config format for the bootloader, + # if not set we default to extlinux for x86 + return os.environ.get('BOOTLOADER_CONFIG_FORMAT', 'extlinux') + def get_extra_kernel_args(self): return os.environ.get('KERNEL_ARGS', '') def get_root_device(self): return os.environ.get('ROOT_DEVICE', '/dev/sda') - def install_extlinux(self, real_root, disk_uuid=None): + def generate_bootloader_config(self, real_root, disk_uuid=None): + '''Install extlinux on the newly created disk image.''' + config_function_dict = { + 'extlinux': self.generate_extlinux_config, + } + + config_type = self.get_bootloader_config_format() + if config_type in config_function_dict: + config_function_dict[config_type](real_root, disk_uuid) + else: + raise cliapp.AppException( + 'Invalid BOOTLOADER_CONFIG_FORMAT %s' % config_type) + + def generate_extlinux_config(self, real_root, disk_uuid=None): '''Install extlinux on the newly created disk image.''' self.status(msg='Creating extlinux.conf') @@ -414,6 +437,19 @@ class WriteExtension(cliapp.Application): f.write('initrd /systems/default/initramfs\n') f.write('append %s\n' % kernel_args) + def install_bootloader(self, real_root): + install_function_dict = { + 'extlinux': self.install_bootloader_extlinux, + } + + install_type = self.get_bootloader_install() + if install_type in install_function_dict: + install_function_dict[install_type](real_root) + elif install_type != 'none': + raise cliapp.AppException( + 'Invalid BOOTLOADER_INSTALL %s' % install_type) + + def install_bootloader_extlinux(self, real_root): self.status(msg='Installing extlinux') cliapp.runcmd(['extlinux', '--install', real_root]) @@ -447,12 +483,13 @@ class WriteExtension(cliapp.Application): else: return [] - def bootloader_is_wanted(self): - '''Does the user request a bootloader? + def bootloader_config_is_wanted(self): + '''Does the user want to generate a bootloader config? - The user may set $BOOTLOADER to yes, no, or auto. If not - set, auto is the default and means that the bootloader will - be installed on x86-32 and x86-64, but not otherwise. + The user may set $BOOTLOADER_CONFIG_FORMAT to the desired + format (u-boot or extlinux). If not set, extlinux is the + default but will be generated on x86-32 and x86-64, but not + otherwise. ''' @@ -460,14 +497,12 @@ class WriteExtension(cliapp.Application): return (arch == 'x86_64' or (arch.startswith('i') and arch.endswith('86'))) - value = os.environ.get('BOOTLOADER', 'auto') - if value == 'auto': - if is_x86(os.uname()[-1]): - value = 'yes' - else: - value = 'no' + value = os.environ.get('BOOTLOADER_CONFIG_FORMAT', '') + if value == '': + if not is_x86(os.uname()[-1]): + return False - return value == 'yes' + return True def get_environment_boolean(self, variable): '''Parse a yes/no boolean passed through the environment.''' -- cgit v1.2.1 From 6d8b1e2d8f6b9ddc9f94ef306737a17d25ee1a48 Mon Sep 17 00:00:00 2001 From: James Thomas Date: Tue, 22 Jul 2014 11:11:27 +0000 Subject: Add support for a device tree to be set using DTB_PATH --- writeexts.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/writeexts.py b/writeexts.py index dec318d9..863a9351 100644 --- a/writeexts.py +++ b/writeexts.py @@ -259,6 +259,8 @@ class WriteExtension(cliapp.Application): if self.bootloader_config_is_wanted(): self.install_kernel(version_root, temp_root) + if self.get_dtb_path() != '': + self.install_dtb(version_root, temp_root) self.install_syslinux_menu(mountpoint, version_root) if initramfs is not None: self.install_initramfs(initramfs, version_root) @@ -385,6 +387,23 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['cp', '-a', try_path, kernel_dest]) break + def install_dtb(self, version_root, temp_root): + '''Install the device tree outside of 'orig' or 'run' subvolumes''' + + self.status(msg='Installing devicetree') + device_tree_path = self.get_dtb_path() + dtb_dest = os.path.join(version_root, 'dtb') + try_path = os.path.join(temp_root, device_tree_path) + if os.path.exists(try_path): + cliapp.runcmd(['cp', '-a', try_path, dtb_dest]) + else: + logging.error("Failed to find device tree %s", device_tree_path) + raise cliapp.AppException( + 'Failed to find device tree %s' % device_tree_path) + + def get_dtb_path(self): + return os.environ.get('DTB_PATH', '') + def get_bootloader_install(self): # Do we actually want to install the bootloader? # Set this to "none" to prevent the install @@ -435,6 +454,8 @@ class WriteExtension(cliapp.Application): f.write('kernel /systems/default/kernel\n') if disk_uuid is not None: f.write('initrd /systems/default/initramfs\n') + if self.get_dtb_path() != '': + f.write('devicetree /systems/default/dtb\n') f.write('append %s\n' % kernel_args) def install_bootloader(self, real_root): -- cgit v1.2.1 From d2dc2bb2585c97fc54847950a8c2f74a77ef9526 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Tue, 12 Aug 2014 13:59:09 +0000 Subject: Merge remote-tracking branch 'origin/baserock/james/writeexts_support_jetson' --- writeexts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index 863a9351..c3605b1c 100644 --- a/writeexts.py +++ b/writeexts.py @@ -444,7 +444,8 @@ class WriteExtension(cliapp.Application): 'rootfstype=btrfs ' # required when using initramfs, also boots # faster when specified without initramfs 'rootflags=subvol=systems/default/run ') # boot runtime subvol - kernel_args += 'root=%s ' % (self.get_root_device() if disk_uuid is None + kernel_args += 'root=%s ' % (self.get_root_device() + if disk_uuid is None else 'UUID=%s' % disk_uuid) kernel_args += self.get_extra_kernel_args() with open(config, 'w') as f: -- cgit v1.2.1 From 39991810f14b836e99d184f554d0354a96eaa0c7 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Fri, 29 Aug 2014 23:18:47 +0100 Subject: deploy: Make Python extensions log debug messages to MORPH_LOG_FD by default Previously logging was disabled for Python deploy extensions. We get a lot of useful information for free in the log file by doing this: the environment will be written when the subprocess starts, and if it crashes the full backtrace will be written there too. Subcommand execution with cliapp.runcmd() will also be logged. --- writeexts.py | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index c3605b1c..5102bfdc 100644 --- a/writeexts.py +++ b/writeexts.py @@ -89,7 +89,30 @@ class WriteExtension(cliapp.Application): write extensions. ''' - + + def setup_logging(self): + '''Direct all logging output to MORPH_LOG_FD, if set. + + This file descriptor is read by Morph and written into its own log + file. + + This overrides cliapp's usual configurable logging setup. + + ''' + log_write_fd = int(os.environ.get('MORPH_LOG_FD', 0)) + + if log_write_fd == 0: + return + + formatter = logging.Formatter('%(message)s') + + handler = logging.StreamHandler(os.fdopen(log_write_fd, 'w')) + handler.setFormatter(formatter) + + logger = logging.getLogger() + logger.addHandler(handler) + logger.setLevel(logging.DEBUG) + def process_args(self, args): raise NotImplementedError() -- cgit v1.2.1 From f05a161053f68244894fb0db73041ed5909dcbc0 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Fri, 19 Sep 2014 15:26:36 +0000 Subject: Prevent cliapp from logging env. variables with 'PASSWORD' in their name The upstream cliapp project is not interested in this functionality right now. --- writeexts.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/writeexts.py b/writeexts.py index 5102bfdc..0fd0ad7b 100644 --- a/writeexts.py +++ b/writeexts.py @@ -113,6 +113,10 @@ class WriteExtension(cliapp.Application): logger.addHandler(handler) logger.setLevel(logging.DEBUG) + def log_config(self): + with morphlib.util.hide_password_environment_variables(os.environ): + cliapp.Application.log_config(self) + def process_args(self, args): raise NotImplementedError() -- cgit v1.2.1 From 10e6486de6f6f45a19bcd6099d7830718a9d12a8 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Fri, 10 Oct 2014 10:56:52 +0000 Subject: Add 'is_device' function to check if we are deploying to a device --- writeexts.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/writeexts.py b/writeexts.py index 0fd0ad7b..48c9434e 100644 --- a/writeexts.py +++ b/writeexts.py @@ -22,6 +22,8 @@ import shutil import sys import time import tempfile +import errno +import stat import morphlib @@ -572,3 +574,12 @@ class WriteExtension(cliapp.Application): logging.error("Error checking SSH connectivity: %s", str(e)) raise cliapp.AppException( 'Unable to SSH to %s: %s' % (ssh_host, e)) + + def is_device(self, location): + try: + st = os.stat(location) + return stat.S_ISBLK(st.st_mode) + except OSError as e: + if e.errno == errno.ENOENT: + return False + raise -- cgit v1.2.1 From 27af6089593df7d7a422a45f19f2456870832c93 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Thu, 16 Oct 2014 17:25:03 +0000 Subject: Add force flag to 'mkfs.btrfs' This way when deploying to a device it will format it without asking the user if the device already has format. --- writeexts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index 48c9434e..4b9e4fcd 100644 --- a/writeexts.py +++ b/writeexts.py @@ -226,7 +226,7 @@ class WriteExtension(cliapp.Application): def mkfs_btrfs(self, location): '''Create a btrfs filesystem on the disk.''' self.status(msg='Creating btrfs filesystem') - cliapp.runcmd(['mkfs.btrfs', '-L', 'baserock', location]) + cliapp.runcmd(['mkfs.btrfs', '-f', '-L', 'baserock', location]) def get_uuid(self, location): '''Get the UUID of a block device's file system.''' -- cgit v1.2.1 From ca385c659297749b37af13be4fec76c8d68db0b7 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Thu, 16 Oct 2014 17:25:37 +0000 Subject: Don't loop mount when mounting a device --- writeexts.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index 4b9e4fcd..31e7a0b6 100644 --- a/writeexts.py +++ b/writeexts.py @@ -246,7 +246,10 @@ class WriteExtension(cliapp.Application): self.status(msg='Mounting filesystem') tempdir = tempfile.mkdtemp() - cliapp.runcmd(['mount', '-o', 'loop', location, tempdir]) + if self.is_device(location): + cliapp.runcmd(['mount', location, tempdir]) + else: + cliapp.runcmd(['mount', '-o', 'loop', location, tempdir]) return tempdir def unmount(self, mount_point): -- cgit v1.2.1 From 5aa74fd32200d84e81e1473eed9e155ebd5a379b Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Tue, 11 Nov 2014 15:42:07 +0000 Subject: Split create_local_system on various methods This way we can still use create_local_system to create a raw disk, but also reuse bits of it to be able e.g. to deploy to devices. --- writeexts.py | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/writeexts.py b/writeexts.py index 31e7a0b6..7030c0b5 100644 --- a/writeexts.py +++ b/writeexts.py @@ -24,6 +24,7 @@ import time import tempfile import errno import stat +import contextlib import morphlib @@ -147,25 +148,39 @@ class WriteExtension(cliapp.Application): def create_local_system(self, temp_root, raw_disk): '''Create a raw system image locally.''' + + with self.created_disk_image(raw_disk): + self.format_btrfs(raw_disk) + self.create_system(temp_root, raw_disk) + + @contextlib.contextmanager + def created_disk_image(self, location): size = self.get_disk_size() if not size: raise cliapp.AppException('DISK_SIZE is not defined') - self.create_raw_disk_image(raw_disk, size) + self.create_raw_disk_image(location, size) + try: + yield + except BaseException: + os.unlink(location) + raise + + def format_btrfs(self, raw_disk): try: self.mkfs_btrfs(raw_disk) - mp = self.mount(raw_disk) except BaseException: sys.stderr.write('Error creating disk image') - os.remove(raw_disk) raise + + def create_system(self, temp_root, raw_disk): try: + mp = self.mount(raw_disk) self.create_btrfs_system_layout( temp_root, mp, version_label='factory', disk_uuid=self.get_uuid(raw_disk)) except BaseException, e: sys.stderr.write('Error creating Btrfs system layout') self.unmount(mp) - os.remove(raw_disk) raise else: self.unmount(mp) -- cgit v1.2.1 From b03e6f1c4342d48c3fb9e530a49807e08a0162e7 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Mon, 1 Dec 2014 15:25:21 +0000 Subject: writeexts.py: convert 'mount' to context manager --- writeexts.py | 65 +++++++++++++++++++++++++----------------------------------- 1 file changed, 27 insertions(+), 38 deletions(-) diff --git a/writeexts.py b/writeexts.py index 7030c0b5..91936f64 100644 --- a/writeexts.py +++ b/writeexts.py @@ -173,17 +173,14 @@ class WriteExtension(cliapp.Application): raise def create_system(self, temp_root, raw_disk): - try: - mp = self.mount(raw_disk) - self.create_btrfs_system_layout( - temp_root, mp, version_label='factory', - disk_uuid=self.get_uuid(raw_disk)) - except BaseException, e: - sys.stderr.write('Error creating Btrfs system layout') - self.unmount(mp) - raise - else: - self.unmount(mp) + with self.mount(raw_disk) as mp: + try: + self.create_btrfs_system_layout( + temp_root, mp, version_label='factory', + disk_uuid=self.get_uuid(raw_disk)) + except BaseException, e: + sys.stderr.write('Error creating Btrfs system layout') + raise def _parse_size(self, size): '''Parse a size from a string. @@ -249,34 +246,26 @@ class WriteExtension(cliapp.Application): # lies by exiting successfully. return cliapp.runcmd(['blkid', '-s', 'UUID', '-o', 'value', location]).strip() - - def mount(self, location): - '''Mount the filesystem so it can be tweaked. - - Return path to the mount point. - The mount point is a newly created temporary directory. - The caller must call self.unmount to unmount on the return value. - - ''' - self.status(msg='Mounting filesystem') - tempdir = tempfile.mkdtemp() - if self.is_device(location): - cliapp.runcmd(['mount', location, tempdir]) - else: - cliapp.runcmd(['mount', '-o', 'loop', location, tempdir]) - return tempdir - - def unmount(self, mount_point): - '''Unmount the filesystem mounted by self.mount. - - Also, remove the temporary directory. - - ''' - - self.status(msg='Unmounting filesystem') - cliapp.runcmd(['umount', mount_point]) - os.rmdir(mount_point) + @contextlib.contextmanager + def mount(self, location): + self.status(msg='Mounting filesystem') + try: + mount_point = tempfile.mkdtemp() + if self.is_device(location): + cliapp.runcmd(['mount', location, mount_point]) + else: + cliapp.runcmd(['mount', '-o', 'loop', location, mount_point]) + except BaseException, e: + sys.stderr.write('Error mounting filesystem') + os.rmdir(mount_point) + raise + try: + yield mount_point + finally: + self.status(msg='Unmounting filesystem') + cliapp.runcmd(['umount', mount_point]) + os.rmdir(mount_point) def create_btrfs_system_layout(self, temp_root, mountpoint, version_label, disk_uuid): -- cgit v1.2.1 From c39f26ac3af32a4b45ea9e48e0c6542346ef1fb5 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Fri, 12 Dec 2014 11:30:13 +0000 Subject: Document BOOTLOADER_INSTALL and BOOTLOADER_CONFIG_FORMAT write extension parameters --- writeexts.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/writeexts.py b/writeexts.py index 91936f64..068f0741 100644 --- a/writeexts.py +++ b/writeexts.py @@ -545,9 +545,8 @@ class WriteExtension(cliapp.Application): '''Does the user want to generate a bootloader config? The user may set $BOOTLOADER_CONFIG_FORMAT to the desired - format (u-boot or extlinux). If not set, extlinux is the - default but will be generated on x86-32 and x86-64, but not - otherwise. + format. 'extlinux' is the only allowed value, and is the default + value for x86-32 and x86-64. ''' -- cgit v1.2.1 From 85281cd3faa5e563f7e3efe6225ae6d988d23925 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Fri, 12 Dec 2014 11:31:09 +0000 Subject: Whitespace removal --- writeexts.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/writeexts.py b/writeexts.py index 068f0741..fd2f5529 100644 --- a/writeexts.py +++ b/writeexts.py @@ -83,14 +83,14 @@ class Fstab(object): class WriteExtension(cliapp.Application): '''A base class for deployment write extensions. - + A subclass should subclass this class, and add a ``process_args`` method. - + Note that it is not necessary to subclass this class for write extensions. This class is here just to collect common code for write extensions. - + ''' def setup_logging(self): @@ -125,13 +125,13 @@ class WriteExtension(cliapp.Application): def status(self, **kwargs): '''Provide status output. - + The ``msg`` keyword argument is the actual message, the rest are values for fields in the message as interpolated by %. - + ''' - + self.output.write('%s\n' % (kwargs['msg'] % kwargs)) self.output.flush() @@ -184,9 +184,9 @@ class WriteExtension(cliapp.Application): def _parse_size(self, size): '''Parse a size from a string. - + Return size in bytes. - + ''' m = re.match('^(\d+)([kmgKMG]?)$', size) -- cgit v1.2.1 From c5d5f960449b881b323a0353c332ef5d36d4490d Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Fri, 12 Dec 2014 11:36:04 +0000 Subject: Document KERNEL_ARGS write extension parameter --- writeexts.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/writeexts.py b/writeexts.py index fd2f5529..6ab2dd55 100644 --- a/writeexts.py +++ b/writeexts.py @@ -474,6 +474,12 @@ class WriteExtension(cliapp.Application): self.status(msg='Creating extlinux.conf') config = os.path.join(real_root, 'extlinux.conf') + + ''' Please also update the documentation in the following files + if you change these default kernel args: + - kvm.write.help + - rawdisk.write.help + - virtualbox-ssh.write.help ''' kernel_args = ( 'rw ' # ro ought to work, but we don't test that regularly 'init=/sbin/init ' # default, but it doesn't hurt to be explicit -- cgit v1.2.1 From 9fe18e52c08117dd2705b0c19f40d2096d02dea7 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Thu, 26 Feb 2015 16:42:28 +0000 Subject: Fix Morph producing unbootable systems Since the version of btrfs-progs in the Baserock reference system definitions was updated to v3.18.2, Morph has produced unbootable x86 systems. This is down to lack of support for new Btrfs features in SYSLINUX. This patch disables the new features so that deployed systems will boot. Although I generally don't want to have compatibility code in Morph, this patch keeps support for the old mkfs.btrfs (which doesn't support the commandline options that we need to pass to new mkfs.btrfs). This will hopefully save people from suffering 'I need to use new Morph to upgrade my devel system, but new Morph doesn't run on my devel system'. --- writeexts.py | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index 6ab2dd55..ad4fabe9 100644 --- a/writeexts.py +++ b/writeexts.py @@ -237,8 +237,32 @@ class WriteExtension(cliapp.Application): def mkfs_btrfs(self, location): '''Create a btrfs filesystem on the disk.''' + self.status(msg='Creating btrfs filesystem') - cliapp.runcmd(['mkfs.btrfs', '-f', '-L', 'baserock', location]) + try: + # The following command disables some new filesystem features. We + # need to do this because at the time of writing, SYSLINUX has not + # been updated to understand these new features and will fail to + # boot if the kernel is on a filesystem where they are enabled. + cliapp.runcmd( + ['mkfs.btrfs','-f', '-L', 'baserock', + '--features', '^extref', + '--features', '^skinny-metadata', + '--features', '^mixed-bg', + '--nodesize', '4096', + location]) + except cliapp.AppException as e: + if 'unrecognized option \'--features\'' in e.msg: + # Old versions of mkfs.btrfs (including v0.20, present in many + # Baserock releases) don't support the --features option, but + # also don't enable the new features by default. So we can + # still create a bootable system in this situation. + logging.debug( + 'Assuming mkfs.btrfs failure was because the tool is too ' + 'old to have --features flag.') + cliapp.runcmd(['mkfs.btrfs','-f', '-L', 'baserock', location]) + else: + raise def get_uuid(self, location): '''Get the UUID of a block device's file system.''' -- cgit v1.2.1 From 61a0d3d5475ebb082c5e757850153b1a134d6c23 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Fri, 27 Feb 2015 16:20:21 +0000 Subject: Fix copyright years --- writeexts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index ad4fabe9..ab451d14 100644 --- a/writeexts.py +++ b/writeexts.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -- cgit v1.2.1 From 2a5de41bdf592c4a104b481db0261009870efd24 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Fri, 27 Feb 2015 16:20:21 +0000 Subject: Fix copyright years --- writeexts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index ad4fabe9..ab451d14 100644 --- a/writeexts.py +++ b/writeexts.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -- cgit v1.2.1 From 8b1d5d732404abd70fe1ef91be84a3f2ce4c1c31 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 13 Mar 2015 14:25:00 +0000 Subject: Use python3 compatible notation for octal constants Change-Id: I771c3de9cecda7a503f4d36ae5d9fabc040892e4 --- writeexts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index ab451d14..9f166d7f 100644 --- a/writeexts.py +++ b/writeexts.py @@ -360,7 +360,7 @@ class WriteExtension(cliapp.Application): self.status(msg='Creating %s subvolume' % state_subdir) subvolume = os.path.join(mountpoint, 'state', state_subdir) cliapp.runcmd(['btrfs', 'subvolume', 'create', subvolume]) - os.chmod(subvolume, 0755) + os.chmod(subvolume, 0o755) existing_state_dir = os.path.join(system_dir, state_subdir) files = [] -- cgit v1.2.1 From e9461f5add5f701017ed74d5ca1e0d6b0c812bbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 13 Mar 2015 18:18:55 +0000 Subject: Use the modern way of the GPL copyright header: URL instead real address Change-Id: I992dc0c1d40f563ade56a833162d409b02be90a0 --- writeexts.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/writeexts.py b/writeexts.py index ab451d14..6eec4465 100644 --- a/writeexts.py +++ b/writeexts.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . import cliapp -- cgit v1.2.1 From 705845ad37abae8fe89f2d698e58e41ee863d807 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 13 Mar 2015 13:02:24 +0000 Subject: Use python3 compatible notation for catching exceptions Change-Id: Ibda7a938cd16e35517a531140f39ef4664d85c72 --- writeexts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/writeexts.py b/writeexts.py index 3382767c..129b2bc4 100644 --- a/writeexts.py +++ b/writeexts.py @@ -177,7 +177,7 @@ class WriteExtension(cliapp.Application): self.create_btrfs_system_layout( temp_root, mp, version_label='factory', disk_uuid=self.get_uuid(raw_disk)) - except BaseException, e: + except BaseException as e: sys.stderr.write('Error creating Btrfs system layout') raise @@ -279,7 +279,7 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['mount', location, mount_point]) else: cliapp.runcmd(['mount', '-o', 'loop', location, mount_point]) - except BaseException, e: + except BaseException as e: sys.stderr.write('Error mounting filesystem') os.rmdir(mount_point) raise -- cgit v1.2.1 From 5c6fe9af9a1f2a226020a7a701cf31347fbb3fc6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 13 Mar 2015 18:18:55 +0000 Subject: Use the modern way of the GPL copyright header: URL instead real address Change-Id: I992dc0c1d40f563ade56a833162d409b02be90a0 --- writeexts.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/writeexts.py b/writeexts.py index 9f166d7f..3382767c 100644 --- a/writeexts.py +++ b/writeexts.py @@ -10,8 +10,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . import cliapp -- cgit v1.2.1 From aaef5cfe335ebbe742b7eb1593ec51bbeeb4dc7c Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Thu, 19 Mar 2015 13:04:00 +0000 Subject: deploy: Tighten SSH connectivity check I accidentally tried to deploy a Baserock upgrade to a Fedora cloud machine. Every SSH command that Morph ran got the following output: Please login as the user "fedora" rather than the user "root". The existing implementation of check_ssh_connectivity() didn't raise an exception, leading to confusing errors further down. The new implementation produces this error: ERROR: Unexpected output from remote machine: Please login as the user "fedora" rather than the user "root". Change-Id: Ida5a82b25d759167aa842194b0d833d0565b4acf --- writeexts.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/writeexts.py b/writeexts.py index ab451d14..270cade9 100644 --- a/writeexts.py +++ b/writeexts.py @@ -605,12 +605,16 @@ class WriteExtension(cliapp.Application): def check_ssh_connectivity(self, ssh_host): try: - cliapp.ssh_runcmd(ssh_host, ['true']) + output = cliapp.ssh_runcmd(ssh_host, ['echo', 'test']) except cliapp.AppException as e: logging.error("Error checking SSH connectivity: %s", str(e)) raise cliapp.AppException( 'Unable to SSH to %s: %s' % (ssh_host, e)) + if output.strip() != 'test': + raise cliapp.AppException( + 'Unexpected output from remote machine: %s' % output.strip()) + def is_device(self, location): try: st = os.stat(location) -- cgit v1.2.1