summaryrefslogtreecommitdiff
path: root/extensions/writeexts.py
diff options
context:
space:
mode:
Diffstat (limited to 'extensions/writeexts.py')
-rw-r--r--extensions/writeexts.py1072
1 files changed, 0 insertions, 1072 deletions
diff --git a/extensions/writeexts.py b/extensions/writeexts.py
deleted file mode 100644
index 5b79093b..00000000
--- a/extensions/writeexts.py
+++ /dev/null
@@ -1,1072 +0,0 @@
-#!/usr/bin/python2
-# Copyright (C) 2012-2015 Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program. If not, see <http://www.gnu.org/licenses/>.
-
-
-import contextlib
-import errno
-import logging
-import os
-import re
-import shutil
-import stat
-import subprocess
-import sys
-import time
-import tempfile
-
-import partitioning
-import pyfdisk
-import writeexts
-
-
-if sys.version_info >= (3, 3, 0):
- import shlex
- shell_quote = shlex.quote
-else:
- import pipes
- shell_quote = pipes.quote
-
-
-def get_data_path(relative_path):
- extensions_dir = os.path.dirname(__file__)
- return os.path.join(extensions_dir, relative_path)
-
-
-def get_data(relative_path):
- with open(get_data_path(relative_path)) as f:
- return f.read()
-
-
-def ssh_runcmd(host, args, **kwargs):
- '''Run command over ssh'''
- command = ['ssh', host, '--'] + [shell_quote(arg) for arg in args]
-
- feed_stdin = kwargs.get('feed_stdin')
- stdin = kwargs.get('stdin', subprocess.PIPE)
- stdout = kwargs.get('stdout', subprocess.PIPE)
- stderr = kwargs.get('stderr', subprocess.PIPE)
-
- p = subprocess.Popen(command, stdin=stdin, stdout=stdout, stderr=stderr)
- out, err = p.communicate(input=feed_stdin)
- if p.returncode != 0:
- raise ExtensionError('ssh command `%s` failed' % ' '.join(command))
- return out
-
-
-def write_from_dict(filepath, d, validate=lambda x, y: True):
- """Takes a dictionary and appends the contents to a file
-
- An optional validation callback can be passed to perform validation on
- each value in the dictionary.
-
- e.g.
-
- def validation_callback(dictionary_key, dictionary_value):
- if not dictionary_value.isdigit():
- raise Exception('value contains non-digit character(s)')
-
- Any callback supplied to this function should raise an exception
- if validation fails.
-
- """
- # Sort items asciibetically
- # the output of the deployment should not depend
- # on the locale of the machine running the deployment
- items = sorted(d.iteritems(), key=lambda (k, v): [ord(c) for c in v])
-
- for (k, v) in items:
- validate(k, v)
-
- with open(filepath, 'a') as f:
- for (_, v) in items:
- f.write('%s\n' % v)
-
- os.fchown(f.fileno(), 0, 0)
- os.fchmod(f.fileno(), 0644)
-
-
-def parse_environment_pairs(env, pairs):
- '''Add key=value pairs to the environment dict.
-
- Given a dict and a list of strings of the form key=value,
- set dict[key] = value, unless key is already set in the
- environment, at which point raise an exception.
-
- This does not modify the passed in dict.
-
- Returns the extended dict.
-
- '''
- extra_env = dict(p.split('=', 1) for p in pairs)
- conflicting = [k for k in extra_env if k in env]
- if conflicting:
- raise ExtensionError('Environment already set: %s'
- % ', '.join(conflicting))
-
- # Return a dict that is the union of the two
- # This is not the most performant, since it creates
- # 3 unnecessary lists, but I felt this was the most
- # easy to read. Using itertools.chain may be more efficicent
- return dict(env.items() + extra_env.items())
-
-
-class ExtensionError(Exception):
-
- def __init__(self, msg):
- self.msg = msg
-
- def __str__(self):
- return self.msg
-
-
-class Fstab(object):
- '''Small helper class for parsing and adding lines to /etc/fstab.'''
-
- # There is an existing Python helper library for editing of /etc/fstab.
- # However it is unmaintained and has an incompatible license (GPL3).
- #
- # https://code.launchpad.net/~computer-janitor-hackers/python-fstab/trunk
-
- def __init__(self, filepath='/etc/fstab'):
- if os.path.exists(filepath):
- with open(filepath, 'r') as f:
- self.text= f.read()
- else:
- self.text = ''
- self.filepath = filepath
- self.lines_added = 0
-
- def get_mounts(self):
- '''Return list of mount devices and targets in /etc/fstab.
-
- Return value is a dict of target -> device.
- '''
- mounts = dict()
- for line in self.text.splitlines():
- words = line.split()
- if len(words) >= 2 and not words[0].startswith('#'):
- device, target = words[0:2]
- mounts[target] = device
- return mounts
-
- def add_line(self, line):
- '''Add a new entry to /etc/fstab.
-
- Lines are appended, and separated from any entries made by configure
- extensions with a comment.
-
- '''
- if self.lines_added == 0:
- if len(self.text) == 0 or self.text[-1] is not '\n':
- self.text += '\n'
- self.text += '# Morph default system layout\n'
- self.lines_added += 1
-
- self.text += line + '\n'
-
- def write(self):
- '''Rewrite the fstab file to include all new entries.'''
- with tempfile.NamedTemporaryFile(delete=False) as f:
- f.write(self.text)
- tmp = f.name
- shutil.move(os.path.abspath(tmp), os.path.abspath(self.filepath))
-
-
-class Extension(object):
-
- '''A base class for deployment extensions.
-
- A subclass should subclass this class, and add a
- ``process_args`` method.
-
- Note that it is not necessary to subclass this class for write
- extensions. This class is here just to collect common code for
- write extensions.
-
- '''
-
- def setup_logging(self):
- '''Direct all logging output to MORPH_LOG_FD, if set.
-
- This file descriptor is read by Morph and written into its own log
- file.
-
- '''
- log_write_fd = int(os.environ.get('MORPH_LOG_FD', 0))
-
- if log_write_fd == 0:
- return
-
- formatter = logging.Formatter('%(message)s')
-
- handler = logging.StreamHandler(os.fdopen(log_write_fd, 'w'))
- handler.setFormatter(formatter)
-
- logger = logging.getLogger()
- logger.addHandler(handler)
- logger.setLevel(logging.DEBUG)
-
- def process_args(self, args):
- raise NotImplementedError()
-
- def run(self, args=None):
- if args is None:
- args = sys.argv[1:]
- try:
- self.setup_logging()
- self.process_args(args)
- except ExtensionError as e:
- sys.stdout.write('ERROR: %s\n' % e)
- sys.exit(1)
-
- @staticmethod
- def status(**kwargs):
- '''Provide status output.
-
- The ``msg`` keyword argument is the actual message,
- the rest are values for fields in the message as interpolated
- by %.
-
- '''
- sys.stdout.write('%s\n' % (kwargs['msg'] % kwargs))
- sys.stdout.flush()
-
-
-class WriteExtension(Extension):
-
- '''A base class for deployment write extensions.
-
- A subclass should subclass this class, and add a
- ``process_args`` method.
-
- Note that it is not necessary to subclass this class for write
- extensions. This class is here just to collect common code for
- write extensions.
-
- '''
-
- def check_for_btrfs_in_deployment_host_kernel(self):
- with open('/proc/filesystems') as f:
- text = f.read()
- return '\tbtrfs\n' in text
-
- def require_btrfs_in_deployment_host_kernel(self):
- if not self.check_for_btrfs_in_deployment_host_kernel():
- raise ExtensionError(
- 'Error: Btrfs is required for this deployment, but was not '
- 'detected in the kernel of the machine that is running Morph.')
-
- def create_local_system(self, temp_root, location):
- '''Create a raw system image locally.'''
-
- with self.created_disk_image(location):
- self.create_baserock_system(temp_root, location)
-
- def create_baserock_system(self, temp_root, location):
- if self.get_environment_boolean('USE_PARTITIONING', 'no'):
- self.create_partitioned_system(temp_root, location)
- else:
- self.format_btrfs(location)
- self.create_unpartitioned_system(temp_root, location)
-
- @contextlib.contextmanager
- def created_disk_image(self, location):
- size = self.get_disk_size()
- if not size:
- raise ExtensionError('DISK_SIZE is not defined')
- self.create_raw_disk_image(location, size)
- try:
- yield
- except BaseException:
- os.unlink(location)
- raise
-
- def format_btrfs(self, raw_disk):
- try:
- self.mkfs_btrfs(raw_disk)
- except BaseException:
- sys.stderr.write('Error creating disk image')
- raise
-
- def create_unpartitioned_system(self, temp_root, raw_disk):
- '''Deploy a bootable Baserock system within a single Btrfs filesystem.
-
- Called if USE_PARTITIONING=no (the default) is set in the deployment
- options.
-
- '''
- with self.mount(raw_disk) as mp:
- try:
- self.create_versioned_layout(mp, version_label='factory')
- self.create_btrfs_system_rootfs(
- temp_root, mp, version_label='factory',
- rootfs_uuid=self.get_uuid(raw_disk))
- if self.bootloader_config_is_wanted():
- self.create_bootloader_config(
- temp_root, mp, version_label='factory',
- rootfs_uuid=self.get_uuid(raw_disk))
- except BaseException:
- sys.stderr.write('Error creating Btrfs system layout')
- raise
-
- def _parse_size(self, size):
- '''Parse a size from a string.
-
- Return size in bytes.
-
- '''
-
- m = re.match('^(\d+)([kmgKMG]?)$', size)
- if not m:
- return None
-
- factors = {
- '': 1,
- 'k': 1024,
- 'm': 1024**2,
- 'g': 1024**3,
- }
- factor = factors[m.group(2).lower()]
-
- return int(m.group(1)) * factor
-
- def _parse_size_from_environment(self, env_var, default):
- '''Parse a size from an environment variable.'''
-
- size = os.environ.get(env_var, default)
- if size is None:
- return None
- bytes = self._parse_size(size)
- if bytes is None:
- raise ExtensionError('Cannot parse %s value %s'
- % (env_var, size))
- return bytes
-
- def get_disk_size(self):
- '''Parse disk size from environment.'''
- return self._parse_size_from_environment('DISK_SIZE', None)
-
- def get_ram_size(self):
- '''Parse RAM size from environment.'''
- return self._parse_size_from_environment('RAM_SIZE', '1G')
-
- def get_vcpu_count(self):
- '''Parse the virtual cpu count from environment.'''
- return self._parse_size_from_environment('VCPUS', '1')
-
- def create_raw_disk_image(self, filename, size):
- '''Create a raw disk image.'''
-
- self.status(msg='Creating empty disk image')
- with open(filename, 'wb') as f:
- if size > 0:
- f.seek(size-1)
- f.write('\0')
-
- def mkfs_btrfs(self, location):
- '''Create a btrfs filesystem on the disk.'''
-
- self.status(msg='Creating btrfs filesystem')
- try:
- # The following command disables some new filesystem features. We
- # need to do this because at the time of writing, SYSLINUX has not
- # been updated to understand these new features and will fail to
- # boot if the kernel is on a filesystem where they are enabled.
- subprocess.check_output(
- ['mkfs.btrfs','-f', '-L', 'baserock',
- '--features', '^extref',
- '--features', '^skinny-metadata',
- '--features', '^mixed-bg',
- '--nodesize', '4096',
- location], stderr=subprocess.STDOUT)
- except subprocess.CalledProcessError as e:
- if 'unrecognized option \'--features\'' in e.output:
- # Old versions of mkfs.btrfs (including v0.20, present in many
- # Baserock releases) don't support the --features option, but
- # also don't enable the new features by default. So we can
- # still create a bootable system in this situation.
- logging.debug(
- 'Assuming mkfs.btrfs failure was because the tool is too '
- 'old to have --features flag.')
- subprocess.check_call(['mkfs.btrfs','-f',
- '-L', 'baserock', location])
- else:
- raise
-
- def get_uuid(self, location, offset=0):
- '''Get the filesystem UUID of a block device's file system.
-
- Requires util-linux blkid; the busybox version ignores options and
- lies by exiting successfully.
-
- Args:
- location: Path of device or image to inspect
- offset: A byte offset - which should point to the start of a
- partition containing a filesystem
- '''
-
- return subprocess.check_output(['blkid', '-s', 'UUID', '-o',
- 'value', '-p', '-O', str(offset),
- location]).strip()
-
- @contextlib.contextmanager
- def mount(self, location):
- self.status(msg='Mounting filesystem')
- try:
- mount_point = tempfile.mkdtemp()
- if self.is_device(location):
- subprocess.check_call(['mount', location, mount_point])
- else:
- subprocess.check_call(['mount', '-o', 'loop',
- location, mount_point])
- except BaseException:
- sys.stderr.write('Error mounting filesystem')
- os.rmdir(mount_point)
- raise
- try:
- yield mount_point
- finally:
- self.status(msg='Unmounting filesystem')
- subprocess.check_call(['umount', mount_point])
- os.rmdir(mount_point)
-
- def create_versioned_layout(self, mountpoint, version_label):
- '''Create a versioned directory structure within a partition.
-
- The Baserock project has defined a 'reference upgrade mechanism'. This
- mandates a specific directory layout. It consists of a toplevel
- '/systems' directory, containing subdirectories named with a 'version
- label'. These subdirectories contain the actual OS content.
-
- For the root file system, a Btrfs partition must be used. For each
- version, two subvolumes are created: 'orig' and 'run'. This is handled
- in create_btrfs_system_rootfs().
-
- Other partitions (e.g. /boot) can also follow the same layout. In the
- case of /boot, content goes directly in the version directory. That
- means there are no 'orig' and 'run' subvolumes, which avoids the
- need to use Btrfs.
-
- The `system-version-manager` tool from tbdiff.git is responsible for
- deploying live upgrades, and it understands this layout.
-
- '''
- version_root = os.path.join(mountpoint, 'systems', version_label)
-
- os.makedirs(version_root)
- os.symlink(
- version_label, os.path.join(mountpoint, 'systems', 'default'))
-
- def create_btrfs_system_rootfs(self, temp_root, mountpoint, version_label,
- rootfs_uuid, device=None):
- '''Separate base OS versions from state using subvolumes.
-
- The 'device' parameter should be a pyfdisk.Device instance,
- as returned by partitioning.do_partitioning(), that describes the
- partition layout of the target device. This is used to set up
- mountpoints in the root partition for the other partitions.
- If no 'device' instance is passed, no mountpoints are set up in the
- rootfs.
-
- '''
- version_root = os.path.join(mountpoint, 'systems', version_label)
- state_root = os.path.join(mountpoint, 'state')
- os.makedirs(state_root)
-
- system_dir = self.create_orig(version_root, temp_root)
- state_dirs = self.complete_fstab_for_btrfs_layout(system_dir,
- rootfs_uuid, device)
-
- for state_dir in state_dirs:
- self.create_state_subvolume(system_dir, mountpoint, state_dir)
-
- self.create_run(version_root)
-
- if device:
- self.create_partition_mountpoints(device, system_dir)
-
- def create_bootloader_config(self, temp_root, mountpoint, version_label,
- rootfs_uuid, device=None):
- '''Setup the bootloader.
-
- '''
- initramfs = self.find_initramfs(temp_root)
- version_root = os.path.join(mountpoint, 'systems', version_label)
-
- self.install_kernel(version_root, temp_root)
- if self.get_dtb_path() != '':
- self.install_dtb(version_root, temp_root)
- self.install_syslinux_menu(mountpoint, temp_root)
- if initramfs is not None:
- # Using initramfs - can boot a rootfs with a filesystem UUID
- self.install_initramfs(initramfs, version_root)
- self.generate_bootloader_config(mountpoint,
- rootfs_uuid=rootfs_uuid)
- else:
- if device:
- # A partitioned disk or image - boot with partition UUID
- root_part = device.get_partition_by_mountpoint('/')
- root_guid = device.get_partition_uuid(root_part)
- self.generate_bootloader_config(mountpoint,
- root_guid=root_guid)
- else:
- # Unpartitioned and no initramfs - cannot boot with a UUID
- self.generate_bootloader_config(mountpoint)
- self.install_bootloader(mountpoint)
-
- def create_partition_mountpoints(self, device, system_dir):
- '''Create (or empty) partition mountpoints in the root filesystem
-
- Delete contents of partition mountpoints in the rootfs to leave an
- empty mount drectory (files are copied to the actual partition in
- create_partitioned_system()), or create an empty mount directory in
- the rootfs if the mount path doesn't exist.
-
- Args:
- device: A pyfdisk.py Device object describing the partitioning
- system_dir: A path to the Baserock rootfs to be modified
- '''
-
- for part in device.partitionlist:
- if hasattr(part, 'mountpoint') and part.mountpoint != '/':
- part_mount_dir = os.path.join(system_dir,
- re.sub('^/', '', part.mountpoint))
- if os.path.exists(part_mount_dir):
- self.status(msg='Deleting files in mountpoint '
- 'for %s partition' % part.mountpoint)
- self.empty_dir(part_mount_dir)
- else:
- self.status(msg='Creating empty mount directory '
- 'for %s partition' % part.mountpoint)
- os.mkdir(part_mount_dir)
-
- def create_orig(self, version_root, temp_root):
- '''Create the default "factory" system.'''
-
- orig = os.path.join(version_root, 'orig')
-
- self.status(msg='Creating orig subvolume')
- subprocess.check_call(['btrfs', 'subvolume', 'create', orig])
- self.status(msg='Copying files to orig subvolume')
- subprocess.check_call(['cp', '-a', temp_root + '/.', orig + '/.'])
-
- return orig
-
- def create_run(self, version_root):
- '''Create the 'run' snapshot.'''
-
- self.status(msg='Creating run subvolume')
- orig = os.path.join(version_root, 'orig')
- run = os.path.join(version_root, 'run')
- subprocess.check_call(
- ['btrfs', 'subvolume', 'snapshot', orig, run])
-
- def create_state_subvolume(self, system_dir, mountpoint, state_subdir):
- '''Create a shared state subvolume.
-
- We need to move any files added to the temporary rootfs by the
- configure extensions to their correct home. For example, they might
- have added keys in `/root/.ssh` which we now need to transfer to
- `/state/root/.ssh`.
-
- '''
- self.status(msg='Creating %s subvolume' % state_subdir)
- subvolume = os.path.join(mountpoint, 'state', state_subdir)
- subprocess.check_call(['btrfs', 'subvolume', 'create', subvolume])
- os.chmod(subvolume, 0o755)
-
- existing_state_dir = os.path.join(system_dir, state_subdir)
- self.move_dir_contents(existing_state_dir, subvolume)
-
- def move_dir_contents(self, source_dir, target_dir):
- '''Move all files source_dir, to target_dir'''
-
- n = self.__cmd_files_in_dir(['mv'], source_dir, target_dir)
- if n:
- self.status(msg='Moved %d files to %s' % (n, target_dir))
-
- def copy_dir_contents(self, source_dir, target_dir):
- '''Copy all files source_dir, to target_dir'''
-
- n = self.__cmd_files_in_dir(['cp', '-a', '-r'], source_dir, target_dir)
- if n:
- self.status(msg='Copied %d files to %s' % (n, target_dir))
-
- def empty_dir(self, directory):
- '''Empty the contents of a directory, but not the directory itself'''
-
- n = self.__cmd_files_in_dir(['rm', '-rf'], directory)
- if n:
- self.status(msg='Deleted %d files in %s' % (n, directory))
-
- def __cmd_files_in_dir(self, cmd, source_dir, target_dir=None):
- files = []
- if os.path.exists(source_dir):
- files = os.listdir(source_dir)
- for filename in files:
- filepath = os.path.join(source_dir, filename)
- add_params = [filepath, target_dir] if target_dir else [filepath]
- subprocess.check_call(cmd + add_params)
- return len(files)
-
- def complete_fstab_for_btrfs_layout(self, system_dir,
- rootfs_uuid=None, device=None):
- '''Fill in /etc/fstab entries for the default Btrfs disk layout.
-
- In the future we should move this code out of the write extension and
- in to a configure extension. To do that, though, we need some way of
- informing the configure extension what layout should be used. Right now
- a configure extension doesn't know if the system is going to end up as
- a Btrfs disk image, a tarfile or something else and so it can't come
- up with a sensible default fstab.
-
- Configuration extensions can already create any /etc/fstab that they
- like. This function only fills in entries that are missing, so if for
- example the user configured /home to be on a separate partition, that
- decision will be honoured and /state/home will not be created.
-
- '''
- shared_state_dirs = {'home', 'root', 'opt', 'srv', 'var'}
-
- fstab = Fstab(os.path.join(system_dir, 'etc', 'fstab'))
- existing_mounts = fstab.get_mounts()
-
- if '/' in existing_mounts:
- root_device = existing_mounts['/']
- else:
- root_device = (self.get_root_device() if rootfs_uuid is None else
- 'UUID=%s' % rootfs_uuid)
- fstab.add_line('%s / btrfs defaults,rw,noatime 0 1' % root_device)
-
- # Add fstab entries for partitions
- part_mountpoints = set()
- if device:
- mount_parts = set(p for p in device.partitionlist
- if hasattr(p, 'mountpoint') and p.mountpoint != '/')
- for part in mount_parts:
- if part.mountpoint not in existing_mounts:
- # Get filesystem UUID
- part_uuid = self.get_uuid(device.location,
- part.extent.start *
- device.sector_size)
- self.status(msg='Adding fstab entry for %s '
- 'partition' % part.mountpoint)
- fstab.add_line('UUID=%s %s %s defaults,rw,noatime '
- '0 2' % (part_uuid, part.mountpoint,
- part.filesystem))
- part_mountpoints.add(part.mountpoint)
- else:
- self.status(msg='WARNING: an entry already exists in '
- 'fstab for %s partition, skipping' %
- part.mountpoint)
-
- # Add entries for state dirs
- all_mountpoints = set(existing_mounts.keys()) | part_mountpoints
- state_dirs_to_create = set()
- for state_dir in shared_state_dirs:
- mp = '/' + state_dir
- if mp not in all_mountpoints:
- state_dirs_to_create.add(state_dir)
- state_subvol = os.path.join('/state', state_dir)
- fstab.add_line(
- '%s /%s btrfs subvol=%s,defaults,rw,noatime 0 2' %
- (root_device, state_dir, state_subvol))
-
- fstab.write()
- return state_dirs_to_create
-
- def find_initramfs(self, temp_root):
- '''Check whether the rootfs has an initramfs.
-
- Uses the INITRAMFS_PATH option to locate it.
- '''
- if 'INITRAMFS_PATH' in os.environ:
- initramfs = os.path.join(temp_root, os.environ['INITRAMFS_PATH'])
- if not os.path.exists(initramfs):
- raise ExtensionError('INITRAMFS_PATH specified, '
- 'but file does not exist')
- return initramfs
- return None
-
- def install_initramfs(self, initramfs_path, version_root):
- '''Install the initramfs outside of 'orig' or 'run' subvolumes.
-
- This is required because syslinux doesn't traverse subvolumes when
- loading the kernel or initramfs.
- '''
- self.status(msg='Installing initramfs')
- initramfs_dest = os.path.join(version_root, 'initramfs')
- subprocess.check_call(['cp', '-a', initramfs_path, initramfs_dest])
-
- def install_kernel(self, version_root, temp_root):
- '''Install the kernel outside of 'orig' or 'run' subvolumes'''
-
- self.status(msg='Installing kernel')
- image_names = ['vmlinuz', 'zImage', 'uImage']
- kernel_dest = os.path.join(version_root, 'kernel')
- for name in image_names:
- try_path = os.path.join(temp_root, 'boot', name)
- if os.path.exists(try_path):
- subprocess.check_call(['cp', '-a', try_path, kernel_dest])
- break
-
- def install_dtb(self, version_root, temp_root):
- '''Install the device tree outside of 'orig' or 'run' subvolumes'''
-
- self.status(msg='Installing devicetree')
- device_tree_path = self.get_dtb_path()
- dtb_dest = os.path.join(version_root, 'dtb')
- try_path = os.path.join(temp_root, device_tree_path)
- if os.path.exists(try_path):
- subprocess.check_call(['cp', '-a', try_path, dtb_dest])
- else:
- logging.error("Failed to find device tree %s", device_tree_path)
- raise ExtensionError(
- 'Failed to find device tree %s' % device_tree_path)
-
- def get_dtb_path(self):
- return os.environ.get('DTB_PATH', '')
-
- def get_bootloader_install(self):
- # Do we actually want to install the bootloader?
- # Set this to "none" to prevent the install
- return os.environ.get('BOOTLOADER_INSTALL', 'extlinux')
-
- def get_bootloader_config_format(self):
- # The config format for the bootloader,
- # if not set we default to extlinux for x86
- return os.environ.get('BOOTLOADER_CONFIG_FORMAT', 'extlinux')
-
- def get_extra_kernel_args(self):
- return os.environ.get('KERNEL_ARGS', '')
-
- def get_root_device(self):
- return os.environ.get('ROOT_DEVICE', '/dev/sda')
-
- def generate_bootloader_config(self, *args, **kwargs):
- '''Install extlinux on the newly created disk image.'''
- config_function_dict = {
- 'extlinux': self.generate_extlinux_config,
- }
-
- config_type = self.get_bootloader_config_format()
- if config_type in config_function_dict:
- config_function_dict[config_type](*args, **kwargs)
- else:
- raise ExtensionError(
- 'Invalid BOOTLOADER_CONFIG_FORMAT %s' % config_type)
-
- def generate_extlinux_config(self, real_root,
- rootfs_uuid=None, root_guid=None):
- '''Generate the extlinux configuration file
-
- Args:
- real_root: Path to the mounted top level of the root filesystem
- rootfs_uuid: Specify a filesystem UUID which can be loaded using
- an initramfs aware of filesystems
- root_guid: Specify a partition GUID, can be used without an
- initramfs
- '''
-
- self.status(msg='Creating extlinux.conf')
- # To be compatible with u-boot, create the extlinux.conf file in
- # /extlinux/ rather than /
- # Syslinux, however, requires this to be in /, so create a symlink
- # as well
- config_path = os.path.join(real_root, 'extlinux')
- os.makedirs(config_path)
- config = os.path.join(config_path, 'extlinux.conf')
- os.symlink('extlinux/extlinux.conf', os.path.join(real_root,
- 'extlinux.conf'))
-
- ''' Please also update the documentation in the following files
- if you change these default kernel args:
- - kvm.write.help
- - rawdisk.write.help
- - virtualbox-ssh.write.help '''
- kernel_args = (
- 'rw ' # ro ought to work, but we don't test that regularly
- 'init=/sbin/init ' # default, but it doesn't hurt to be explicit
- 'rootfstype=btrfs ' # required when using initramfs, also boots
- # faster when specified without initramfs
- 'rootflags=subvol=systems/default/run ') # boot runtime subvol
-
- # See init/do_mounts.c:182 in the kernel source, in the comment above
- # function name_to_dev_t(), for an explanation of the available
- # options for the kernel parameter 'root', particularly when using
- # GUID/UUIDs
- if rootfs_uuid:
- root_device = 'UUID=%s' % rootfs_uuid
- elif root_guid:
- root_device = 'PARTUUID=%s' % root_guid
- else:
- # Fall back to the root partition named in the cluster
- root_device = self.get_root_device()
- kernel_args += 'root=%s ' % root_device
-
- kernel_args += self.get_extra_kernel_args()
- with open(config, 'w') as f:
- f.write('default linux\n')
- f.write('timeout 1\n')
- f.write('label linux\n')
- f.write('kernel /systems/default/kernel\n')
- if rootfs_uuid is not None:
- f.write('initrd /systems/default/initramfs\n')
- if self.get_dtb_path() != '':
- f.write('devicetree /systems/default/dtb\n')
- f.write('append %s\n' % kernel_args)
-
- def install_bootloader(self, *args, **kwargs):
- install_function_dict = {
- 'extlinux': self.install_bootloader_extlinux,
- }
-
- install_type = self.get_bootloader_install()
- if install_type in install_function_dict:
- install_function_dict[install_type](*args, **kwargs)
- elif install_type != 'none':
- raise ExtensionError(
- 'Invalid BOOTLOADER_INSTALL %s' % install_type)
-
- def install_bootloader_extlinux(self, real_root):
- self.status(msg='Installing extlinux')
- subprocess.check_call(['extlinux', '--install', real_root])
-
- # FIXME this hack seems to be necessary to let extlinux finish
- subprocess.check_call(['sync'])
- time.sleep(2)
-
- def install_syslinux_blob(self, device, orig_root):
- '''Install Syslinux MBR blob
-
- This is the first stage of boot (for partitioned images) on x86
- machines. It is not required where there is no partition table. The
- syslinux bootloader is written to the MBR, and is capable of loading
- extlinux. This only works when the partition is set as bootable (MBR),
- or the legacy boot flag is set (GPT). The blob is built with extlinux,
- and found in the rootfs'''
-
- pt_format = device.partition_table_format.lower()
- if pt_format in ('gpb', 'mbr'):
- blob = 'mbr.bin'
- elif pt_format == 'gpt':
- blob = 'gptmbr.bin'
- blob_name = 'usr/share/syslinux/' + blob
- self.status(msg='Installing syslinux %s blob' % pt_format.upper())
- blob_location = os.path.join(orig_root, blob_name)
- if os.path.exists(blob_location):
- subprocess.check_call(['dd', 'if=%s' % blob_location,
- 'of=%s' % device.location,
- 'bs=440', 'count=1', 'conv=notrunc'])
- else:
- raise ExtensionError('MBR blob not found. Is this the correct'
- 'architecture? The MBR blob will only be built for x86'
- 'systems. You may wish to configure BOOTLOADER_INSTALL')
-
- def install_syslinux_menu(self, real_root, temp_root):
- '''Make syslinux/extlinux menu binary available.
-
- The syslinux boot menu is compiled to a file named menu.c32. Extlinux
- searches a few places for this file but it does not know to look inside
- our subvolume, so we copy it to the filesystem root.
-
- If the file is not available, the bootloader will still work but will
- not be able to show a menu.
-
- '''
- menu_file = os.path.join(temp_root, 'usr', 'share', 'syslinux',
- 'menu.c32')
- if os.path.isfile(menu_file):
- self.status(msg='Copying menu.c32')
- shutil.copy(menu_file, real_root)
-
- def parse_attach_disks(self):
- '''Parse $ATTACH_DISKS into list of disks to attach.'''
-
- if 'ATTACH_DISKS' in os.environ:
- s = os.environ['ATTACH_DISKS']
- return s.split(':')
- else:
- return []
-
- def bootloader_config_is_wanted(self):
- '''Does the user want to generate a bootloader config?
-
- The user may set $BOOTLOADER_CONFIG_FORMAT to the desired
- format. 'extlinux' is the only allowed value, and is the default
- value for x86-32 and x86-64.
-
- '''
-
- def is_x86(arch):
- return (arch == 'x86_64' or
- (arch.startswith('i') and arch.endswith('86')))
-
- value = os.environ.get('BOOTLOADER_CONFIG_FORMAT', '')
- if value == '':
- if not is_x86(os.uname()[-1]):
- return False
-
- return True
-
- def get_environment_boolean(self, variable, default='no'):
- '''Parse a yes/no boolean passed through the environment.'''
-
- value = os.environ.get(variable, default).lower()
- if value in ('no', '0', 'false'):
- return False
- elif value in ('yes', '1', 'true'):
- return True
- else:
- raise ExtensionError('Unexpected value for %s: %s' %
- (variable, value))
-
- def check_ssh_connectivity(self, ssh_host):
- try:
- output = ssh_runcmd(ssh_host, ['echo', 'test'])
- except ExtensionError as e:
- logging.error("Error checking SSH connectivity: %s", str(e))
- raise ExtensionError(
- 'Unable to SSH to %s: %s' % (ssh_host, e))
-
- if output.strip() != 'test':
- raise ExtensionError(
- 'Unexpected output from remote machine: %s' % output.strip())
-
- def is_device(self, location):
- try:
- st = os.stat(location)
- return stat.S_ISBLK(st.st_mode)
- except OSError as e:
- if e.errno == errno.ENOENT:
- return False
- raise
-
- def create_partitioned_system(self, temp_root, location):
- '''Deploy a bootable Baserock system with a custom partition layout.
-
- Called if USE_PARTITIONING=yes is set in the deployment options.
-
- '''
- part_spec = os.environ.get('PARTITION_FILE', 'partitioning/default')
-
- disk_size = self.get_disk_size()
- if not disk_size:
- raise writeexts.ExtensionError('DISK_SIZE is not defined')
-
- dev = partitioning.do_partitioning(location, disk_size,
- temp_root, part_spec)
- boot_partition_available = dev.get_partition_by_mountpoint('/boot')
-
- for part in dev.partitionlist:
- if not hasattr(part, 'mountpoint'):
- continue
- if part.mountpoint == '/':
- # Re-format the rootfs, to include needed extra features
- with pyfdisk.create_loopback(location,
- part.extent.start *
- dev.sector_size, part.size) as l:
- self.mkfs_btrfs(l)
-
- self.status(msg='Mounting partition %d' % part.number)
- offset = part.extent.start * dev.sector_size
- with self.mount_partition(location,
- offset, part.size) as part_mount_dir:
- if part.mountpoint == '/':
- # Install root filesystem
- rfs_uuid = self.get_uuid(location, part.extent.start *
- dev.sector_size)
- self.create_versioned_layout(part_mount_dir, 'factory')
- self.create_btrfs_system_rootfs(temp_root, part_mount_dir,
- 'factory', rfs_uuid, dev)
-
- # If there's no /boot partition, but we do need to generate
- # a bootloader configuration file, then it needs to go in
- # the root partition.
- if (boot_partition_available is False
- and self.bootloader_config_is_wanted()):
- self.create_bootloader_config(
- temp_root, part_mount_dir, 'factory', rfs_uuid,
- dev)
-
- if self.get_bootloader_install() == 'extlinux':
- # The extlinux/syslinux MBR blob always needs to be
- # installed in the root partition.
- self.install_syslinux_blob(dev, temp_root)
- else:
- # Copy files to partition from unpacked rootfs
- src_dir = os.path.join(temp_root,
- re.sub('^/', '', part.mountpoint))
- self.status(msg='Copying files to %s partition' %
- part.mountpoint)
- self.copy_dir_contents(src_dir, part_mount_dir)
-
- if (part.mountpoint == '/boot' and
- self.bootloader_config_is_wanted()):
- # We need to mirror the layout of the root partition in the
- # /boot partition. Each kernel lives in its own
- # systems/$version_label/ directory within the /boot
- # partition.
- self.create_versioned_layout(part_mount_dir, 'factory')
- self.create_bootloader_config(temp_root, part_mount_dir,
- 'factory', None, dev)
-
- # Write raw files to disk with dd
- partitioning.process_raw_files(dev, temp_root)
-
- @contextlib.contextmanager
- def mount_partition(self, location, offset_bytes, size_bytes):
- '''Mount a partition in a partitioned device or image'''
-
- with pyfdisk.create_loopback(location, offset=offset_bytes,
- size=size_bytes) as loop:
- with self.mount(loop) as mountpoint:
- yield mountpoint
-
- @contextlib.contextmanager
- def find_and_mount_rootfs(self, location):
- '''
- Mount a Baserock rootfs inside a partitioned device or image
-
- This function searches a disk image or device, with unknown
- partitioning scheme, for a Baserock rootfs. This is done by finding
- offsets and sizes of partitions in the partition table, mounting each
- partition, and checking whether a known path exists in the mount.
-
- Args:
- location: the location of the disk image or device to search
- Returns:
- A path to the mount point of the mounted Baserock rootfs
- '''
-
- if pyfdisk.get_pt_type(location) == 'none':
- with self.mount(location) as mountpoint:
- yield mountpoint
-
- sector_size = pyfdisk.get_sector_size(location)
- partn_sizes = pyfdisk.get_partition_sector_sizes(location)
- for i, offset in enumerate(pyfdisk.get_partition_offsets(location)):
- try:
- with self.mount_partition(location, offset * sector_size,
- partn_sizes[i] * sector_size) as mp:
- path = os.path.join(mp, 'systems/default/orig/baserock')
- if os.path.exists(path):
- self.status(msg='Found a Baserock rootfs at '
- 'offset %d sectors/%d bytes' %
- (offset, offset * sector_size))
- yield mp
- except BaseException:
- # Probably a partition without a filesystem, carry on
- pass