From 656f30003c1cf3b3f1f70758d9d06bdd3693f994 Mon Sep 17 00:00:00 2001 From: Edward Cragg Date: Tue, 4 Aug 2015 12:21:57 +0100 Subject: Rawdisk partitioning v2: Add partitioning functions Add partitioning functions to the rawdisk.write deployment extension Change-Id: I45bcabc191951d086b8f4ae028a248f95a5e6f2e --- extensions/openstack.write | 2 +- extensions/partitioning.py | 138 ++++++++++++++++++++++++ extensions/rawdisk.write | 73 +++++++------ extensions/writeexts.py | 263 +++++++++++++++++++++++++++++++++++++-------- 4 files changed, 399 insertions(+), 77 deletions(-) create mode 100644 extensions/partitioning.py (limited to 'extensions') diff --git a/extensions/openstack.write b/extensions/openstack.write index f1233560..f0d2fc0b 100755 --- a/extensions/openstack.write +++ b/extensions/openstack.write @@ -51,7 +51,7 @@ class OpenStackWriteExtension(writeexts.WriteExtension): def set_extlinux_root_to_virtio(self, raw_disk): '''Re-configures extlinux to use virtio disks''' self.status(msg='Updating extlinux.conf') - with self.mount(raw_disk) as mp: + with self.find_and_mount_rootfs(raw_disk) as mp: path = os.path.join(mp, 'extlinux.conf') with open(path) as f: diff --git a/extensions/partitioning.py b/extensions/partitioning.py new file mode 100644 index 00000000..0b048a74 --- /dev/null +++ b/extensions/partitioning.py @@ -0,0 +1,138 @@ +#!/usr/bin/python +# Copyright (C) 2012-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + + +"""A module providing Baserock-specific partitioning functions""" + +import pyfdisk +import writeexts + +def do_partitioning(location, disk_size, temp_root, part_spec): + '''Perform partitioning + + Perform partitioning using the pyfdisk.py module. Documentation + for this, and guidance on how to create a partition specification can + be found in extensions/pyfdisk.README + + This function also validates essential parts of the partition layout + + Args: + location: Path to the target device or image + temp_root: Location of the unpacked Baserock rootfs + part_spec: Path to a YAML formatted partition specification + Returns: + A pyfdisk.py Device object + Raises: + writeexts.ExtensionError + ''' + # Create partition table and filesystems + try: + dev = pyfdisk.load_yaml(location, disk_size, part_spec) + writeexts.Extension.status(msg='Loaded partition specification: %s' % + part_spec) + + # FIXME: GPT currently not supported due to missing tools + if dev.partition_table_format.lower() == 'gpt': + raise writeexts.ExtensionError('GPT partition tables are not ' + 'currently supported') + + writeexts.Extension.status(msg=str(dev.partitionlist)) + writeexts.Extension.status(msg='Writing partition table') + dev.commit() + dev.create_filesystems(skip=['/']) + except (pyfdisk.PartitioningError, pyfdisk.FdiskError) as e: + raise writeexts.ExtensionError(e.msg) + + mountpoints = set(part.mountpoint for part in dev.partitionlist + if hasattr(part, 'mountpoint')) + if '/' not in mountpoints: + raise writeexts.ExtensionError('No partition with root ' + 'mountpoint, please specify a ' + 'partition with \'mountpoint: /\' ' + 'in the partition specification') + + mounted_partitions = set(part for part in dev.partitionlist + if hasattr(part, 'mountpoint')) + + # Create root filesystem, and copy files to partitions + for part in mounted_partitions: + if not hasattr(part, 'filesystem'): + raise writeexts.ExtensionError('Cannot mount a partition ' + 'without filesystem, please specify one ' + 'for \'%s\' partition in the partition ' + 'specification' % part.mountpoint) + if part.mountpoint == '/': + # Check that bootable flag is set for MBR devices + if (hasattr(part, 'boot') + and str(part.boot).lower() not in ('yes', 'true') + and dev.partition_table_format.lower() == 'mbr'): + writeexts.Extension.status(msg='WARNING: Boot partition ' + 'needs bootable flag set to ' + 'boot with extlinux/syslinux') + + return dev + +def write_raw_files(location, temp_root, dev_or_part, start_offset=0): + '''Write files with `dd`''' + offset = start_offset + for raw_args in dev_or_part.raw_files: + r = RawFile(temp_root, offset, **raw_args) + offset = r.next_offset + r.dd(location) + + +class RawFile(object): + '''A class to hold information about a raw file to write to a device''' + + def __init__(self, source_root, wr_offset=0, sector_size=512, **kwargs): + '''Initialisation function + + Args: + source_root: Base path for filenames + wr_offset: Offset to write to (and offset per-file offsets by) + sector_size: Device sector size (default: 512) + **kwargs: + file: A path to the file to write (combined with source_root) + offset_sectors: An offset to write to in sectors (optional) + offset_bytes: An offset to write to in bytes (optional) + ''' + if 'file' not in kwargs: + raise writeexts.ExtensionError('Missing file name or path') + self.path = os.path.join(source_root, + re.sub('^/', '', kwargs['file'])) + + if not os.path.exists(self.path): + raise writeexts.ExtensionError('File not found: %s' % self.path) + elif os.path.isdir(self.path): + raise writeexts.ExtensionError('Can only dd regular files') + else: + self.size = os.stat(self.path).st_size + + self.offset = wr_offset + if 'offset_bytes' in kwargs: + self.offset += kwargs['offset_bytes'] + elif 'offset_sectors' in kwargs: + self.offset += kwargs['offset_sectors'] * sector_size + + # Offset of the first free byte after this file + self.next_offset = self.size + self.offset + + def dd(self, location): + writeexts.Extension.status(msg='Writing %s at %d bytes' % + (self.path, self.offset)) + subprocess.check_call(['dd', 'if=%s' % self.path, + 'of=%s' % location, 'bs=1', + 'seek=%s' % self.offset, 'conv=notrunc']) + subprocess.check_call('sync') diff --git a/extensions/rawdisk.write b/extensions/rawdisk.write index 49d0a1e8..6be546a1 100755 --- a/extensions/rawdisk.write +++ b/extensions/rawdisk.write @@ -17,7 +17,10 @@ '''A Morph deployment write extension for raw disk images.''' +import contextlib import os +import pyfdisk +import re import subprocess import sys import time @@ -44,54 +47,64 @@ class RawDiskWriteExtension(writeexts.WriteExtension): try: if not self.is_device(location): with self.created_disk_image(location): - self.format_btrfs(location) - self.create_system(temp_root, location) + self.create_partitioned_system(temp_root, location) self.status(msg='Disk image has been created at %s' % location) else: - self.format_btrfs(location) - self.create_system(temp_root, location) + self.create_partitioned_system(temp_root, location) self.status(msg='System deployed to %s' % location) except Exception: self.status(msg='Failure to deploy system to %s' % location) raise - def upgrade_local_system(self, raw_disk, temp_root): + def upgrade_local_system(self, location, temp_root): self.complete_fstab_for_btrfs_layout(temp_root) - with self.mount(raw_disk) as mp: - version_label = self.get_version_label(mp) - self.status(msg='Updating image to a new version with label %s' % - version_label) + try: + with self.mount(location) as mp: + self.do_upgrade(mp, temp_root) + return + except subprocess.CalledProcessError: + pass - version_root = os.path.join(mp, 'systems', version_label) - os.mkdir(version_root) + # Failed to mount a raw image, search for a Baserock root filesystem + # in the device's partitions + with self.find_and_mount_rootfs(location) as mp: + self.do_upgrade(mp, temp_root) - old_orig = os.path.join(mp, 'systems', 'default', 'orig') - new_orig = os.path.join(version_root, 'orig') - subprocess.check_call( - ['btrfs', 'subvolume', 'snapshot', old_orig, new_orig]) + def do_upgrade(self, mp, temp_root): + version_label = self.get_version_label(mp) + self.status(msg='Updating image to a new version with label %s' % + version_label) - subprocess.check_call( - ['rsync', '-a', '--checksum', '--numeric-ids', '--delete', - temp_root + os.path.sep, new_orig]) + version_root = os.path.join(mp, 'systems', version_label) + os.mkdir(version_root) - self.create_run(version_root) + old_orig = os.path.join(mp, 'systems', 'default', 'orig') + new_orig = os.path.join(version_root, 'orig') + subprocess.check_call( + ['btrfs', 'subvolume', 'snapshot', old_orig, new_orig]) - default_path = os.path.join(mp, 'systems', 'default') - if os.path.exists(default_path): - os.remove(default_path) - else: - # we are upgrading and old system that does - # not have an updated extlinux config file - if self.bootloader_config_is_wanted(): - self.generate_bootloader_config(mp) - self.install_bootloader(mp) - os.symlink(version_label, default_path) + subprocess.check_call( + ['rsync', '-a', '--checksum', '--numeric-ids', '--delete', + temp_root + os.path.sep, new_orig]) + self.create_run(version_root) + + default_path = os.path.join(mp, 'systems', 'default') + if os.path.exists(default_path): + os.remove(default_path) + else: + # we are upgrading and old system that does + # not have an updated extlinux config file if self.bootloader_config_is_wanted(): - self.install_kernel(version_root, temp_root) + self.generate_bootloader_config(mp) + self.install_bootloader(mp) + os.symlink(version_label, default_path) + + if self.bootloader_config_is_wanted(): + self.install_kernel(version_root, temp_root) def get_version_label(self, mp): version_label = os.environ.get('VERSION_LABEL') diff --git a/extensions/writeexts.py b/extensions/writeexts.py index 000c8270..0768057d 100644 --- a/extensions/writeexts.py +++ b/extensions/writeexts.py @@ -18,6 +18,8 @@ import errno import fcntl import logging import os +import partitioning +import pyfdisk import re import select import shutil @@ -228,7 +230,8 @@ class Extension(object): sys.stdout.write('ERROR: %s\n' % e) sys.exit(1) - def status(self, **kwargs): + @staticmethod + def status(**kwargs): '''Provide status output. The ``msg`` keyword argument is the actual message, @@ -264,12 +267,11 @@ class WriteExtension(Extension): 'Error: Btrfs is required for this deployment, but was not ' 'detected in the kernel of the machine that is running Morph.') - def create_local_system(self, temp_root, raw_disk): + def create_local_system(self, temp_root, location): '''Create a raw system image locally.''' - with self.created_disk_image(raw_disk): - self.format_btrfs(raw_disk) - self.create_system(temp_root, raw_disk) + with self.created_disk_image(location): + self.create_partitioned_system(temp_root, location) @contextlib.contextmanager def created_disk_image(self, location): @@ -290,16 +292,6 @@ class WriteExtension(Extension): sys.stderr.write('Error creating disk image') raise - def create_system(self, temp_root, raw_disk): - with self.mount(raw_disk) as mp: - try: - self.create_btrfs_system_layout( - temp_root, mp, version_label='factory', - disk_uuid=self.get_uuid(raw_disk)) - except BaseException as e: - sys.stderr.write('Error creating Btrfs system layout') - raise - def _parse_size(self, size): '''Parse a size from a string. @@ -384,11 +376,26 @@ class WriteExtension(Extension): else: raise - def get_uuid(self, location): - '''Get the UUID of a block device's file system.''' - # Requires util-linux blkid; busybox one ignores options and - # lies by exiting successfully. - return subprocess.check_output(['blkid', '-s', 'UUID', '-o', 'value', + def get_uuid(self, location, offset=0, disk=False): + '''Get the UUID of a block device's file system. + + Requires util-linux blkid; the busybox version ignores options and + lies by exiting successfully. + + Args: + location: Path of device or image to inspect + offset: A byte offset - which should point to the start of a + partition containing a filesystem + disk: Boolean, if true, find the disk (partition table) UUID, + rather than a filesystem UUID. Offset has no effect. + ''' + if disk: + field = 'PTUUID' + else: + field = 'UUID' + + return subprocess.check_output(['blkid', '-s', field, '-o', + 'value', '-p', '-O', str(offset), location]).strip() @contextlib.contextmanager @@ -413,7 +420,7 @@ class WriteExtension(Extension): os.rmdir(mount_point) def create_btrfs_system_layout(self, temp_root, mountpoint, version_label, - disk_uuid): + rootfs_uuid, device): '''Separate base OS versions from state using subvolumes. ''' @@ -424,11 +431,9 @@ class WriteExtension(Extension): os.makedirs(version_root) os.makedirs(state_root) - self.create_orig(version_root, temp_root) - system_dir = os.path.join(version_root, 'orig') - + system_dir = self.create_orig(version_root, temp_root) state_dirs = self.complete_fstab_for_btrfs_layout(system_dir, - disk_uuid) + rootfs_uuid, device) for state_dir in state_dirs: self.create_state_subvolume(system_dir, mountpoint, state_dir) @@ -445,10 +450,32 @@ class WriteExtension(Extension): self.install_syslinux_menu(mountpoint, version_root) if initramfs is not None: self.install_initramfs(initramfs, version_root) - self.generate_bootloader_config(mountpoint, disk_uuid) + self.generate_bootloader_config(mountpoint, + rootfs_uuid=rootfs_uuid) else: - self.generate_bootloader_config(mountpoint) - self.install_bootloader(mountpoint) + disk_uuid = self.get_uuid(device.location, disk=True) + root_num = next(r.number for r in device.partitionlist + if hasattr(r, 'mountpoint') + and r.mountpoint == '/') + self.generate_bootloader_config(mountpoint, + disk_uuid=disk_uuid, + root_partition=root_num) + self.install_bootloader(mountpoint, system_dir, device.location) + + # Move this? + # Delete contents of partition mountpoints in the rootfs to leave an + # empty mount drectory (files are copied to the actual partition + # separately), or create an empty mount directory in the rootfs. + for part in device.partitionlist: + if hasattr(part, 'mountpoint') and part.mountpoint != '/': + part_mount_dir = os.path.join(system_dir, + re.sub('^/', '', part.mountpoint)) + if os.path.exists(part_mount_dir): + self.empty_dir(part_mount_dir) + else: + self.status(msg='Creating empty mount directory ' + 'for %s partition' % part.mountpoint) + os.mkdir(part_mount_dir) def create_orig(self, version_root, temp_root): '''Create the default "factory" system.''' @@ -460,6 +487,8 @@ class WriteExtension(Extension): self.status(msg='Copying files to orig subvolume') subprocess.check_call(['cp', '-a', temp_root + '/.', orig + '/.']) + return orig + def create_run(self, version_root): '''Create the 'run' snapshot.''' @@ -484,16 +513,37 @@ class WriteExtension(Extension): os.chmod(subvolume, 0o755) existing_state_dir = os.path.join(system_dir, state_subdir) + self.move_or_copy_dir(existing_state_dir, subvolume) + + def move_or_copy_dir(self, source_dir, target_dir, copy=False): + '''Move or copy all files source_dir, to target_dir''' + + cmd = 'mv' + act = 'Mov' + if copy: + cmd = 'cp' + act = 'Copy' + files = [] - if os.path.exists(existing_state_dir): - files = os.listdir(existing_state_dir) + if os.path.exists(source_dir): + files = os.listdir(source_dir) if len(files) > 0: - self.status(msg='Moving existing data to %s subvolume' % subvolume) + self.status(msg='%sing data to %s' % (act, target_dir)) + for filename in files: + filepath = os.path.join(source_dir, filename) + subprocess.check_call([cmd, filepath, target_dir]) + + def empty_dir(self, directory): + '''Empty the contents of a directory, but not the directory itself''' + files = [] + if os.path.exists(directory): + files = os.listdir(directory) for filename in files: - filepath = os.path.join(existing_state_dir, filename) - subprocess.check_call(['mv', filepath, subvolume]) + filepath = os.path.join(directory, filename) + subprocess.check_call(['rm', '-rf', filepath]) - def complete_fstab_for_btrfs_layout(self, system_dir, rootfs_uuid=None): + def complete_fstab_for_btrfs_layout(self, system_dir, + rootfs_uuid=None, device=None): '''Fill in /etc/fstab entries for the default Btrfs disk layout. In the future we should move this code out of the write extension and @@ -521,9 +571,33 @@ class WriteExtension(Extension): 'UUID=%s' % rootfs_uuid) fstab.add_line('%s / btrfs defaults,rw,noatime 0 1' % root_device) + # Add fstab entries for partitions + partition_mounts = set() + if device: + mount_parts = set(p for p in device.partitionlist + if hasattr(p, 'mountpoint') and p.mountpoint != '/') + part_mountpoints = set(p.mountpoint for p in mount_parts) + for part in mount_parts: + if part.mountpoint not in existing_mounts: + part_uuid = self.get_uuid(device.location, + part.extent.start * + device.sector_size) + self.status(msg='Adding fstab entry for %s ' + 'partition' % part.mountpoint) + fstab.add_line('UUID=%s %s %s defaults,rw,noatime ' + '0 2' % (part_uuid, part.mountpoint, + part.filesystem)) + else: + self.status(msg='WARNING: an entry already exists in ' + 'fstab for %s partition, skipping' % + part.mountpoint) + + # Add entries for state dirs state_dirs_to_create = set() for state_dir in shared_state_dirs: - if '/' + state_dir not in existing_mounts: + mp = '/' + state_dir + if (mp not in existing_mounts and + (device and mp not in part_mountpoints)): state_dirs_to_create.add(state_dir) state_subvol = os.path.join('/state', state_dir) fstab.add_line( @@ -601,7 +675,7 @@ class WriteExtension(Extension): def get_root_device(self): return os.environ.get('ROOT_DEVICE', '/dev/sda') - def generate_bootloader_config(self, real_root, disk_uuid=None): + def generate_bootloader_config(self, *args, **kwargs): '''Install extlinux on the newly created disk image.''' config_function_dict = { 'extlinux': self.generate_extlinux_config, @@ -609,13 +683,24 @@ class WriteExtension(Extension): config_type = self.get_bootloader_config_format() if config_type in config_function_dict: - config_function_dict[config_type](real_root, disk_uuid) + config_function_dict[config_type](*args, **kwargs) else: raise ExtensionError( 'Invalid BOOTLOADER_CONFIG_FORMAT %s' % config_type) - def generate_extlinux_config(self, real_root, disk_uuid=None): - '''Install extlinux on the newly created disk image.''' + def generate_extlinux_config(self, real_root, + rootfs_uuid=None, + disk_uuid=None, root_partition=False): + '''Generate the extlinux configuration file + + Args: + real_root: Path to the mounted top level of the root filesystem + rootfs_uuid: Specify a filesystem UUID which can be loaded using + an initramfs + disk_uuid: Disk UUID, can be used without an initramfs + root_partition: Partition number of the boot partition if using + disk_uuid + ''' self.status(msg='Creating extlinux.conf') config = os.path.join(real_root, 'extlinux.conf') @@ -631,34 +716,41 @@ class WriteExtension(Extension): 'rootfstype=btrfs ' # required when using initramfs, also boots # faster when specified without initramfs 'rootflags=subvol=systems/default/run ') # boot runtime subvol - kernel_args += 'root=%s ' % (self.get_root_device() - if disk_uuid is None - else 'UUID=%s' % disk_uuid) + + if rootfs_uuid: + root_device = 'UUID=%s' % rootfs_uuid + elif disk_uuid: + root_device = 'PARTUUID=%s-%02d' % (disk_uuid, root_partition) + else: + # Fall back to the root partition named in the cluster + root_device = '%s%d' % (self.get_root_device(), root_partition) + kernel_args += 'root=%s ' % root_device + kernel_args += self.get_extra_kernel_args() with open(config, 'w') as f: f.write('default linux\n') f.write('timeout 1\n') f.write('label linux\n') f.write('kernel /systems/default/kernel\n') - if disk_uuid is not None: + if rootfs_uuid is not None: f.write('initrd /systems/default/initramfs\n') if self.get_dtb_path() != '': f.write('devicetree /systems/default/dtb\n') f.write('append %s\n' % kernel_args) - def install_bootloader(self, real_root): + def install_bootloader(self, *args): install_function_dict = { 'extlinux': self.install_bootloader_extlinux, } install_type = self.get_bootloader_install() if install_type in install_function_dict: - install_function_dict[install_type](real_root) + install_function_dict[install_type](*args) elif install_type != 'none': raise ExtensionError( 'Invalid BOOTLOADER_INSTALL %s' % install_type) - def install_bootloader_extlinux(self, real_root): + def install_bootloader_extlinux(self, real_root, orig_root, location): self.status(msg='Installing extlinux') subprocess.check_call(['extlinux', '--install', real_root]) @@ -666,6 +758,14 @@ class WriteExtension(Extension): subprocess.check_call(['sync']) time.sleep(2) + # Install Syslinux MBR blob + self.status(msg='Installing syslinux MBR blob') + mbr_blob_location = os.path.join(orig_root, + 'usr/share/syslinux/mbr.bin') + subprocess.check_call(['dd', 'if=%s' % mbr_blob_location, + 'of=%s' % location, + 'bs=440', 'count=1', 'conv=notrunc']) + def install_syslinux_menu(self, real_root, version_root): '''Make syslinux/extlinux menu binary available. @@ -744,3 +844,74 @@ class WriteExtension(Extension): if e.errno == errno.ENOENT: return False raise + + def create_partitioned_system(self, temp_root, location): + '''Create a Baserock system in a partitioned disk image or device''' + + part_spec = os.environ.get('PARTITION_FILE', 'partitioning/default') + + disk_size = self.get_disk_size() + if not disk_size: + raise writeexts.ExtensionError('DISK_SIZE is not defined') + + dev = partitioning.do_partitioning(location, disk_size, + temp_root, part_spec) + + for part in dev.partitionlist: + if not hasattr(part, 'mountpoint'): + continue + if part.mountpoint == '/': + # Re-format the rootfs, to include needed extra features + with pyfdisk.create_loopback(location, + part.extent.start * + dev.sector_size, part.size) as l: + self.mkfs_btrfs(l) + + self.status(msg='Mounting partition %d' % part.number) + offset = part.extent.start * dev.sector_size + with self.mount_partition(location, offset) as part_mount_dir: + if part.mountpoint == '/': + # Install system + root_uuid = self.get_uuid(location, part.extent.start * + dev.sector_size) + self.create_btrfs_system_layout(temp_root, part_mount_dir, + 'factory', root_uuid, dev) + else: + # Copy files to partition from unpacked rootfs + src_dir = os.path.join(temp_root, + re.sub('^/', '', part.mountpoint)) + self.status(msg='Copying files for %s partition' % + part.mountpoint) + self.move_or_copy_dir(src_dir, part_mount_dir, copy=True) + + # Write raw files + if hasattr(dev, 'raw_files'): + partitioning.write_raw_files(location, temp_root, dev) + for part in dev.partitionlist: + if hasattr(part, 'raw_files'): + # dd seek is used, which skips n blocks before writing, + # so we must skip n-1 sectors before writing in order to + # start writing files to the first block of the partition + partitioning.write_raw_files(location, temp_root, part, + (part.extent.start - 1) * + dev.sector_size) + + @contextlib.contextmanager + def mount_partition(self, location, offset_bytes): + """Mount a partition in a partitioned device or image""" + with pyfdisk.create_loopback(location, offset=offset_bytes) as loop: + with self.mount(loop) as mountpoint: + yield mountpoint + + @contextlib.contextmanager + def find_and_mount_rootfs(self, location): + """Find a Baserock rootfs in a partitioned device or image""" + sector_size = pyfdisk.get_sector_size(location) + for offset in pyfdisk.get_disk_offsets(location): + with self.mount_partition(location, offset * sector_size) as mp: + path = os.path.join(mp, 'systems/default/orig/baserock') + if os.path.exists(path): + self.status(msg='Found a Baserock rootfs at ' + 'offset %d sectors/%d bytes' % + (offset, offset * sector_size)) + yield mp -- cgit v1.2.1