summaryrefslogtreecommitdiff
path: root/extensions/writeexts.py
diff options
context:
space:
mode:
Diffstat (limited to 'extensions/writeexts.py')
-rw-r--r--extensions/writeexts.py263
1 files changed, 217 insertions, 46 deletions
diff --git a/extensions/writeexts.py b/extensions/writeexts.py
index 000c827..0768057 100644
--- a/extensions/writeexts.py
+++ b/extensions/writeexts.py
@@ -18,6 +18,8 @@ import errno
import fcntl
import logging
import os
+import partitioning
+import pyfdisk
import re
import select
import shutil
@@ -228,7 +230,8 @@ class Extension(object):
sys.stdout.write('ERROR: %s\n' % e)
sys.exit(1)
- def status(self, **kwargs):
+ @staticmethod
+ def status(**kwargs):
'''Provide status output.
The ``msg`` keyword argument is the actual message,
@@ -264,12 +267,11 @@ class WriteExtension(Extension):
'Error: Btrfs is required for this deployment, but was not '
'detected in the kernel of the machine that is running Morph.')
- def create_local_system(self, temp_root, raw_disk):
+ def create_local_system(self, temp_root, location):
'''Create a raw system image locally.'''
- with self.created_disk_image(raw_disk):
- self.format_btrfs(raw_disk)
- self.create_system(temp_root, raw_disk)
+ with self.created_disk_image(location):
+ self.create_partitioned_system(temp_root, location)
@contextlib.contextmanager
def created_disk_image(self, location):
@@ -290,16 +292,6 @@ class WriteExtension(Extension):
sys.stderr.write('Error creating disk image')
raise
- def create_system(self, temp_root, raw_disk):
- with self.mount(raw_disk) as mp:
- try:
- self.create_btrfs_system_layout(
- temp_root, mp, version_label='factory',
- disk_uuid=self.get_uuid(raw_disk))
- except BaseException as e:
- sys.stderr.write('Error creating Btrfs system layout')
- raise
-
def _parse_size(self, size):
'''Parse a size from a string.
@@ -384,11 +376,26 @@ class WriteExtension(Extension):
else:
raise
- def get_uuid(self, location):
- '''Get the UUID of a block device's file system.'''
- # Requires util-linux blkid; busybox one ignores options and
- # lies by exiting successfully.
- return subprocess.check_output(['blkid', '-s', 'UUID', '-o', 'value',
+ def get_uuid(self, location, offset=0, disk=False):
+ '''Get the UUID of a block device's file system.
+
+ Requires util-linux blkid; the busybox version ignores options and
+ lies by exiting successfully.
+
+ Args:
+ location: Path of device or image to inspect
+ offset: A byte offset - which should point to the start of a
+ partition containing a filesystem
+ disk: Boolean, if true, find the disk (partition table) UUID,
+ rather than a filesystem UUID. Offset has no effect.
+ '''
+ if disk:
+ field = 'PTUUID'
+ else:
+ field = 'UUID'
+
+ return subprocess.check_output(['blkid', '-s', field, '-o',
+ 'value', '-p', '-O', str(offset),
location]).strip()
@contextlib.contextmanager
@@ -413,7 +420,7 @@ class WriteExtension(Extension):
os.rmdir(mount_point)
def create_btrfs_system_layout(self, temp_root, mountpoint, version_label,
- disk_uuid):
+ rootfs_uuid, device):
'''Separate base OS versions from state using subvolumes.
'''
@@ -424,11 +431,9 @@ class WriteExtension(Extension):
os.makedirs(version_root)
os.makedirs(state_root)
- self.create_orig(version_root, temp_root)
- system_dir = os.path.join(version_root, 'orig')
-
+ system_dir = self.create_orig(version_root, temp_root)
state_dirs = self.complete_fstab_for_btrfs_layout(system_dir,
- disk_uuid)
+ rootfs_uuid, device)
for state_dir in state_dirs:
self.create_state_subvolume(system_dir, mountpoint, state_dir)
@@ -445,10 +450,32 @@ class WriteExtension(Extension):
self.install_syslinux_menu(mountpoint, version_root)
if initramfs is not None:
self.install_initramfs(initramfs, version_root)
- self.generate_bootloader_config(mountpoint, disk_uuid)
+ self.generate_bootloader_config(mountpoint,
+ rootfs_uuid=rootfs_uuid)
else:
- self.generate_bootloader_config(mountpoint)
- self.install_bootloader(mountpoint)
+ disk_uuid = self.get_uuid(device.location, disk=True)
+ root_num = next(r.number for r in device.partitionlist
+ if hasattr(r, 'mountpoint')
+ and r.mountpoint == '/')
+ self.generate_bootloader_config(mountpoint,
+ disk_uuid=disk_uuid,
+ root_partition=root_num)
+ self.install_bootloader(mountpoint, system_dir, device.location)
+
+ # Move this?
+ # Delete contents of partition mountpoints in the rootfs to leave an
+ # empty mount drectory (files are copied to the actual partition
+ # separately), or create an empty mount directory in the rootfs.
+ for part in device.partitionlist:
+ if hasattr(part, 'mountpoint') and part.mountpoint != '/':
+ part_mount_dir = os.path.join(system_dir,
+ re.sub('^/', '', part.mountpoint))
+ if os.path.exists(part_mount_dir):
+ self.empty_dir(part_mount_dir)
+ else:
+ self.status(msg='Creating empty mount directory '
+ 'for %s partition' % part.mountpoint)
+ os.mkdir(part_mount_dir)
def create_orig(self, version_root, temp_root):
'''Create the default "factory" system.'''
@@ -460,6 +487,8 @@ class WriteExtension(Extension):
self.status(msg='Copying files to orig subvolume')
subprocess.check_call(['cp', '-a', temp_root + '/.', orig + '/.'])
+ return orig
+
def create_run(self, version_root):
'''Create the 'run' snapshot.'''
@@ -484,16 +513,37 @@ class WriteExtension(Extension):
os.chmod(subvolume, 0o755)
existing_state_dir = os.path.join(system_dir, state_subdir)
+ self.move_or_copy_dir(existing_state_dir, subvolume)
+
+ def move_or_copy_dir(self, source_dir, target_dir, copy=False):
+ '''Move or copy all files source_dir, to target_dir'''
+
+ cmd = 'mv'
+ act = 'Mov'
+ if copy:
+ cmd = 'cp'
+ act = 'Copy'
+
files = []
- if os.path.exists(existing_state_dir):
- files = os.listdir(existing_state_dir)
+ if os.path.exists(source_dir):
+ files = os.listdir(source_dir)
if len(files) > 0:
- self.status(msg='Moving existing data to %s subvolume' % subvolume)
+ self.status(msg='%sing data to %s' % (act, target_dir))
+ for filename in files:
+ filepath = os.path.join(source_dir, filename)
+ subprocess.check_call([cmd, filepath, target_dir])
+
+ def empty_dir(self, directory):
+ '''Empty the contents of a directory, but not the directory itself'''
+ files = []
+ if os.path.exists(directory):
+ files = os.listdir(directory)
for filename in files:
- filepath = os.path.join(existing_state_dir, filename)
- subprocess.check_call(['mv', filepath, subvolume])
+ filepath = os.path.join(directory, filename)
+ subprocess.check_call(['rm', '-rf', filepath])
- def complete_fstab_for_btrfs_layout(self, system_dir, rootfs_uuid=None):
+ def complete_fstab_for_btrfs_layout(self, system_dir,
+ rootfs_uuid=None, device=None):
'''Fill in /etc/fstab entries for the default Btrfs disk layout.
In the future we should move this code out of the write extension and
@@ -521,9 +571,33 @@ class WriteExtension(Extension):
'UUID=%s' % rootfs_uuid)
fstab.add_line('%s / btrfs defaults,rw,noatime 0 1' % root_device)
+ # Add fstab entries for partitions
+ partition_mounts = set()
+ if device:
+ mount_parts = set(p for p in device.partitionlist
+ if hasattr(p, 'mountpoint') and p.mountpoint != '/')
+ part_mountpoints = set(p.mountpoint for p in mount_parts)
+ for part in mount_parts:
+ if part.mountpoint not in existing_mounts:
+ part_uuid = self.get_uuid(device.location,
+ part.extent.start *
+ device.sector_size)
+ self.status(msg='Adding fstab entry for %s '
+ 'partition' % part.mountpoint)
+ fstab.add_line('UUID=%s %s %s defaults,rw,noatime '
+ '0 2' % (part_uuid, part.mountpoint,
+ part.filesystem))
+ else:
+ self.status(msg='WARNING: an entry already exists in '
+ 'fstab for %s partition, skipping' %
+ part.mountpoint)
+
+ # Add entries for state dirs
state_dirs_to_create = set()
for state_dir in shared_state_dirs:
- if '/' + state_dir not in existing_mounts:
+ mp = '/' + state_dir
+ if (mp not in existing_mounts and
+ (device and mp not in part_mountpoints)):
state_dirs_to_create.add(state_dir)
state_subvol = os.path.join('/state', state_dir)
fstab.add_line(
@@ -601,7 +675,7 @@ class WriteExtension(Extension):
def get_root_device(self):
return os.environ.get('ROOT_DEVICE', '/dev/sda')
- def generate_bootloader_config(self, real_root, disk_uuid=None):
+ def generate_bootloader_config(self, *args, **kwargs):
'''Install extlinux on the newly created disk image.'''
config_function_dict = {
'extlinux': self.generate_extlinux_config,
@@ -609,13 +683,24 @@ class WriteExtension(Extension):
config_type = self.get_bootloader_config_format()
if config_type in config_function_dict:
- config_function_dict[config_type](real_root, disk_uuid)
+ config_function_dict[config_type](*args, **kwargs)
else:
raise ExtensionError(
'Invalid BOOTLOADER_CONFIG_FORMAT %s' % config_type)
- def generate_extlinux_config(self, real_root, disk_uuid=None):
- '''Install extlinux on the newly created disk image.'''
+ def generate_extlinux_config(self, real_root,
+ rootfs_uuid=None,
+ disk_uuid=None, root_partition=False):
+ '''Generate the extlinux configuration file
+
+ Args:
+ real_root: Path to the mounted top level of the root filesystem
+ rootfs_uuid: Specify a filesystem UUID which can be loaded using
+ an initramfs
+ disk_uuid: Disk UUID, can be used without an initramfs
+ root_partition: Partition number of the boot partition if using
+ disk_uuid
+ '''
self.status(msg='Creating extlinux.conf')
config = os.path.join(real_root, 'extlinux.conf')
@@ -631,34 +716,41 @@ class WriteExtension(Extension):
'rootfstype=btrfs ' # required when using initramfs, also boots
# faster when specified without initramfs
'rootflags=subvol=systems/default/run ') # boot runtime subvol
- kernel_args += 'root=%s ' % (self.get_root_device()
- if disk_uuid is None
- else 'UUID=%s' % disk_uuid)
+
+ if rootfs_uuid:
+ root_device = 'UUID=%s' % rootfs_uuid
+ elif disk_uuid:
+ root_device = 'PARTUUID=%s-%02d' % (disk_uuid, root_partition)
+ else:
+ # Fall back to the root partition named in the cluster
+ root_device = '%s%d' % (self.get_root_device(), root_partition)
+ kernel_args += 'root=%s ' % root_device
+
kernel_args += self.get_extra_kernel_args()
with open(config, 'w') as f:
f.write('default linux\n')
f.write('timeout 1\n')
f.write('label linux\n')
f.write('kernel /systems/default/kernel\n')
- if disk_uuid is not None:
+ if rootfs_uuid is not None:
f.write('initrd /systems/default/initramfs\n')
if self.get_dtb_path() != '':
f.write('devicetree /systems/default/dtb\n')
f.write('append %s\n' % kernel_args)
- def install_bootloader(self, real_root):
+ def install_bootloader(self, *args):
install_function_dict = {
'extlinux': self.install_bootloader_extlinux,
}
install_type = self.get_bootloader_install()
if install_type in install_function_dict:
- install_function_dict[install_type](real_root)
+ install_function_dict[install_type](*args)
elif install_type != 'none':
raise ExtensionError(
'Invalid BOOTLOADER_INSTALL %s' % install_type)
- def install_bootloader_extlinux(self, real_root):
+ def install_bootloader_extlinux(self, real_root, orig_root, location):
self.status(msg='Installing extlinux')
subprocess.check_call(['extlinux', '--install', real_root])
@@ -666,6 +758,14 @@ class WriteExtension(Extension):
subprocess.check_call(['sync'])
time.sleep(2)
+ # Install Syslinux MBR blob
+ self.status(msg='Installing syslinux MBR blob')
+ mbr_blob_location = os.path.join(orig_root,
+ 'usr/share/syslinux/mbr.bin')
+ subprocess.check_call(['dd', 'if=%s' % mbr_blob_location,
+ 'of=%s' % location,
+ 'bs=440', 'count=1', 'conv=notrunc'])
+
def install_syslinux_menu(self, real_root, version_root):
'''Make syslinux/extlinux menu binary available.
@@ -744,3 +844,74 @@ class WriteExtension(Extension):
if e.errno == errno.ENOENT:
return False
raise
+
+ def create_partitioned_system(self, temp_root, location):
+ '''Create a Baserock system in a partitioned disk image or device'''
+
+ part_spec = os.environ.get('PARTITION_FILE', 'partitioning/default')
+
+ disk_size = self.get_disk_size()
+ if not disk_size:
+ raise writeexts.ExtensionError('DISK_SIZE is not defined')
+
+ dev = partitioning.do_partitioning(location, disk_size,
+ temp_root, part_spec)
+
+ for part in dev.partitionlist:
+ if not hasattr(part, 'mountpoint'):
+ continue
+ if part.mountpoint == '/':
+ # Re-format the rootfs, to include needed extra features
+ with pyfdisk.create_loopback(location,
+ part.extent.start *
+ dev.sector_size, part.size) as l:
+ self.mkfs_btrfs(l)
+
+ self.status(msg='Mounting partition %d' % part.number)
+ offset = part.extent.start * dev.sector_size
+ with self.mount_partition(location, offset) as part_mount_dir:
+ if part.mountpoint == '/':
+ # Install system
+ root_uuid = self.get_uuid(location, part.extent.start *
+ dev.sector_size)
+ self.create_btrfs_system_layout(temp_root, part_mount_dir,
+ 'factory', root_uuid, dev)
+ else:
+ # Copy files to partition from unpacked rootfs
+ src_dir = os.path.join(temp_root,
+ re.sub('^/', '', part.mountpoint))
+ self.status(msg='Copying files for %s partition' %
+ part.mountpoint)
+ self.move_or_copy_dir(src_dir, part_mount_dir, copy=True)
+
+ # Write raw files
+ if hasattr(dev, 'raw_files'):
+ partitioning.write_raw_files(location, temp_root, dev)
+ for part in dev.partitionlist:
+ if hasattr(part, 'raw_files'):
+ # dd seek is used, which skips n blocks before writing,
+ # so we must skip n-1 sectors before writing in order to
+ # start writing files to the first block of the partition
+ partitioning.write_raw_files(location, temp_root, part,
+ (part.extent.start - 1) *
+ dev.sector_size)
+
+ @contextlib.contextmanager
+ def mount_partition(self, location, offset_bytes):
+ """Mount a partition in a partitioned device or image"""
+ with pyfdisk.create_loopback(location, offset=offset_bytes) as loop:
+ with self.mount(loop) as mountpoint:
+ yield mountpoint
+
+ @contextlib.contextmanager
+ def find_and_mount_rootfs(self, location):
+ """Find a Baserock rootfs in a partitioned device or image"""
+ sector_size = pyfdisk.get_sector_size(location)
+ for offset in pyfdisk.get_disk_offsets(location):
+ with self.mount_partition(location, offset * sector_size) as mp:
+ path = os.path.join(mp, 'systems/default/orig/baserock')
+ if os.path.exists(path):
+ self.status(msg='Found a Baserock rootfs at '
+ 'offset %d sectors/%d bytes' %
+ (offset, offset * sector_size))
+ yield mp