diff options
40 files changed, 3302 insertions, 0 deletions
diff --git a/add-config-files.configure b/add-config-files.configure new file mode 100755 index 00000000..2cf96fd1 --- /dev/null +++ b/add-config-files.configure @@ -0,0 +1,26 @@ +#!/bin/sh +# Copyright (C) 2013,2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +# Copy all files located in $SRC_CONFIG_DIR to the image /etc. + + +set -e + +if [ "x${SRC_CONFIG_DIR}" != x ] +then + cp -r "$SRC_CONFIG_DIR"/* "$1/etc/" +fi + diff --git a/distbuild-trove-nfsboot.check b/distbuild-trove-nfsboot.check new file mode 100755 index 00000000..38c491e5 --- /dev/null +++ b/distbuild-trove-nfsboot.check @@ -0,0 +1,150 @@ +#!/usr/bin/python +# Copyright (C) 2014-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + +'''Preparatory checks for Morph 'distbuild-trove-nfsboot' write extension''' + +import cliapp +import logging +import os + +import morphlib.writeexts + + +class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): + + nfsboot_root = '/srv/nfsboot' + remote_user = 'root' + + required_vars = [ + 'DISTBUILD_CONTROLLER', + 'DISTBUILD_GIT_SERVER', + 'DISTBUILD_SHARED_ARTIFACT_CACHE', + 'DISTBUILD_TROVE_ID', + 'DISTBUILD_WORKERS', + 'DISTBUILD_WORKER_SSH_KEY', + ] + + def system_path(self, system_name, version_label=None): + if version_label: + return os.path.join(self.nfsboot_root, system_name, 'systems', + version_label, 'run') + else: + return os.path.join(self.nfsboot_root, system_name) + + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + nfs_host = args[0] + nfs_netloc = '%s@%s' % (self.remote_user, nfs_host) + + version_label = os.getenv('VERSION_LABEL', 'factory') + + missing_vars = [var for var in self.required_vars + if not var in os.environ] + if missing_vars: + raise cliapp.AppException( + 'Please set: %s' % ', '.join(missing_vars)) + + controllers = os.getenv('DISTBUILD_CONTROLLER').split() + workers = os.getenv('DISTBUILD_WORKERS').split() + + if len(controllers) != 1: + raise cliapp.AppException('Please specify exactly one controller.') + + if len(workers) == 0: + raise cliapp.AppException('Please specify at least one worker.') + + upgrade = self.get_environment_boolean('UPGRADE') + + self.check_good_server(nfs_netloc) + + system_names = set(controllers + workers) + for system_name in system_names: + if upgrade: + self.check_upgradeable(nfs_netloc, system_name, version_label) + else: + system_path = self.system_path(system_name) + + if self.remote_directory_exists(nfs_netloc, system_path): + if self.get_environment_boolean('OVERWRITE') == False: + raise cliapp.AppException( + 'System %s already exists at %s:%s. Try `morph ' + 'upgrade` instead of `morph deploy`.' % ( + system_name, nfs_netloc, system_path)) + + def check_good_server(self, netloc): + # FIXME: assumes root + self.check_ssh_connectivity(netloc.split('@')[-1]) + + # Is an NFS server + try: + cliapp.ssh_runcmd( + netloc, ['test', '-e', '/etc/exports']) + except cliapp.AppException: + raise cliapp.AppException('server %s is not an nfs server' + % netloc) + try: + cliapp.ssh_runcmd( + netloc, ['systemctl', 'is-enabled', 'nfs-server.service']) + + except cliapp.AppException: + raise cliapp.AppException('server %s does not control its ' + 'nfs server by systemd' % netloc) + + # TFTP server exports /srv/nfsboot/tftp + tftp_root = os.path.join(self.nfsboot_root, 'tftp') + try: + cliapp.ssh_runcmd( + netloc, ['test' , '-d', tftp_root]) + except cliapp.AppException: + raise cliapp.AppException('server %s does not export %s' % + (netloc, tftp_root)) + + def check_upgradeable(self, nfs_netloc, system_name, version_label): + '''Check that there is already a version of the system present. + + Distbuild nodes are stateless, so an upgrade is actually pretty much + the same as an initial deployment. This test is just a sanity check. + + ''' + system_path = self.system_path(system_name) + system_version_path = self.system_path(system_name, version_label) + + if not self.remote_directory_exists(nfs_netloc, system_path): + raise cliapp.AppException( + 'System %s not found at %s:%s, cannot deploy an upgrade.' % ( + system_name, nfs_netloc, system_path)) + + if self.remote_directory_exists(nfs_netloc, system_version_path): + if self.get_environment_boolean('OVERWRITE'): + pass + else: + raise cliapp.AppException( + 'System %s version %s already exists at %s:%s.' % ( + system_name, version_label, nfs_netloc, + system_version_path)) + + def remote_directory_exists(self, nfs_netloc, path): + try: + cliapp.ssh_runcmd(nfs_netloc, ['test', '-d', path]) + except cliapp.AppException as e: + logging.debug('SSH exception: %s', e) + return False + + return True + + +DistbuildTroveNFSBootCheckExtension().run() diff --git a/distbuild-trove-nfsboot.write b/distbuild-trove-nfsboot.write new file mode 100755 index 00000000..a5a5b094 --- /dev/null +++ b/distbuild-trove-nfsboot.write @@ -0,0 +1,283 @@ +#!/usr/bin/python +# Copyright (C) 2013-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +'''Morph .write extension for a distbuild network booting off a Trove with NFS. + +''' + + +import os +import sys +import tempfile + +import cliapp +import morphlib.writeexts + + +class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): + + '''Create an NFS root and kernel on TFTP during Morph's deployment. + + See distbuild-trove-nfsboot.help for documentation. + + ''' + + nfsboot_root = '/srv/nfsboot' + remote_user = 'root' + + def system_path(self, system_name, version_label=None): + if version_label: + # The 'run' directory is kind of a historical artifact. Baserock + # systems that have Btrfs root disks maintain an orig/ and a run/ + # subvolume, so that one can find changes that have been made at + # runtime. For distbuild systems, this isn't necessary because the + # root filesystems of the nodes are effectively stateless. However, + # existing systems have bootloaders configured to look for the + # 'run' directory, so we need to keep creating it. + return os.path.join(self.nfsboot_root, system_name, 'systems', + version_label, 'run') + else: + return os.path.join(self.nfsboot_root, system_name) + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + local_system_path, nfs_host = args + + nfs_netloc = '%s@%s' % (self.remote_user, nfs_host) + + version_label = os.getenv('VERSION_LABEL', 'factory') + + controller_name = os.getenv('DISTBUILD_CONTROLLER') + worker_names = os.getenv('DISTBUILD_WORKERS').split() + system_names = set([controller_name] + worker_names) + + git_server = os.getenv('DISTBUILD_GIT_SERVER') + shared_artifact_cache = os.getenv('DISTBUILD_SHARED_ARTIFACT_CACHE') + trove_id = os.getenv('DISTBUILD_TROVE_ID') + worker_ssh_key_path = os.getenv('DISTBUILD_WORKER_SSH_KEY') + + host_map = self.parse_host_map_string(os.getenv('HOST_MAP', '')) + + kernel_relpath = self.find_kernel(local_system_path) + + copied_rootfs = None + for system_name in system_names: + remote_system_path = self.system_path(system_name, version_label) + if copied_rootfs is None: + self.transfer_system( + nfs_netloc, local_system_path, remote_system_path) + copied_rootfs = remote_system_path + else: + self.duplicate_remote_system( + nfs_netloc, copied_rootfs, remote_system_path) + + for system_name in system_names: + remote_system_path = self.system_path(system_name, version_label) + self.link_kernel_to_tftpboot_path( + nfs_netloc, system_name, version_label, kernel_relpath) + self.set_hostname( + nfs_netloc, system_name, remote_system_path) + self.write_distbuild_config( + nfs_netloc, system_name, remote_system_path, git_server, + shared_artifact_cache, trove_id, worker_ssh_key_path, + controller_name, worker_names, host_map=host_map) + + self.configure_nfs_exports(nfs_netloc, system_names) + + for system_name in system_names: + self.update_default_version(nfs_netloc, system_name, version_label) + + def parse_host_map_string(self, host_map_string): + '''Parse the HOST_MAP variable + + Returns a dict mapping hostname to value (where value is an IP + address, a fully-qualified domain name, an alternate hostname, or + whatever). + + ''' + pairs = host_map_string.split(' ') + return morphlib.util.parse_environment_pairs({}, pairs) + + def transfer_system(self, nfs_netloc, local_system_path, + remote_system_path): + self.status(msg='Copying rootfs to %(nfs_netloc)s', + nfs_netloc=nfs_netloc) + cliapp.ssh_runcmd( + nfs_netloc, ['mkdir', '-p', remote_system_path]) + # The deployed rootfs may have been created by OSTree, so definitely + # don't pass --hard-links to `rsync`. + cliapp.runcmd( + ['rsync', '--archive', '--delete', '--info=progress2', + '--protect-args', '--partial', '--sparse', '--xattrs', + local_system_path + '/', + '%s:%s' % (nfs_netloc, remote_system_path)], stdout=sys.stdout) + + def duplicate_remote_system(self, nfs_netloc, source_system_path, + target_system_path): + self.status(msg='Duplicating rootfs to %(target_system_path)s', + target_system_path=target_system_path) + cliapp.ssh_runcmd(nfs_netloc, + ['mkdir', '-p', target_system_path]) + # We can't pass --info=progress2 here, because it may not be available + # in the remote 'rsync'. The --info setting was added in RSync 3.1.0, + # old versions of Baserock have RSync 3.0.9. So the user doesn't get + # any progress info on stdout for the 'duplicate' stage. + cliapp.ssh_runcmd(nfs_netloc, + ['rsync', '--archive', '--delete', '--protect-args', '--partial', + '--sparse', '--xattrs', source_system_path + '/', + target_system_path], stdout=sys.stdout) + + def find_kernel(self, local_system_path): + bootdir = os.path.join(local_system_path, 'boot') + image_names = ['vmlinuz', 'zImage', 'uImage'] + + for name in image_names: + try_path = os.path.join(bootdir, name) + if os.path.exists(try_path): + kernel_path = os.path.relpath(try_path, local_system_path) + break + else: + raise cliapp.AppException( + 'Could not find a kernel in the system: none of ' + '%s found' % ', '.join(image_names)) + return kernel_path + + def link_kernel_to_tftpboot_path(self, nfs_netloc, system_name, + version_label, kernel_relpath): + '''Create links for TFTP server for a system's kernel.''' + + remote_system_path = self.system_path(system_name, version_label) + kernel_dest = os.path.join(remote_system_path, kernel_relpath) + + self.status(msg='Creating links to %(name)s kernel in tftp directory', + name=system_name) + tftp_dir = os.path.join(self.nfsboot_root , 'tftp') + + versioned_kernel_name = "%s-%s" % (system_name, version_label) + kernel_name = system_name + + cliapp.ssh_runcmd(nfs_netloc, + ['ln', '-f', kernel_dest, + os.path.join(tftp_dir, versioned_kernel_name)]) + + cliapp.ssh_runcmd(nfs_netloc, + ['ln', '-sf', versioned_kernel_name, + os.path.join(tftp_dir, kernel_name)]) + + def set_remote_file_contents(self, nfs_netloc, path, text): + with tempfile.NamedTemporaryFile() as f: + f.write(text) + f.flush() + cliapp.runcmd( + ['scp', f.name, '%s:%s' % (nfs_netloc, path)]) + + def set_hostname(self, nfs_netloc, system_name, system_path): + hostname_path = os.path.join(system_path, 'etc', 'hostname') + self.set_remote_file_contents( + nfs_netloc, hostname_path, system_name + '\n') + + def write_distbuild_config(self, nfs_netloc, system_name, system_path, + git_server, shared_artifact_cache, trove_id, + worker_ssh_key_path, controller_name, + worker_names, host_map = {}): + '''Write /etc/distbuild/distbuild.conf on the node. + + This .write extension takes advantage of the 'generic' mode of + distbuild.configure. Each node is not configured until first-boot, + when distbuild-setup.service runs and configures the node based on the + contents of /etc/distbuild/distbuild.conf. + + ''' + def host(hostname): + return host_map.get(hostname, hostname) + + config = { + 'ARTIFACT_CACHE_SERVER': host(shared_artifact_cache), + 'CONTROLLERHOST': host(controller_name), + 'TROVE_HOST': host(git_server), + 'TROVE_ID': trove_id, + 'DISTBUILD_CONTROLLER': system_name == controller_name, + 'DISTBUILD_WORKER': system_name in worker_names, + 'WORKERS': ', '.join(map(host, worker_names)), + 'WORKER_SSH_KEY': '/etc/distbuild/worker.key', + } + + config_text = '\n'.join( + '%s: %s' % (key, value) for key, value in config.iteritems()) + config_text = \ + '# Generated by distbuild-trove-nfsboot.write\n' + \ + config_text + '\n' + path = os.path.join(system_path, 'etc', 'distbuild') + cliapp.ssh_runcmd( + nfs_netloc, ['mkdir', '-p', path]) + cliapp.runcmd( + ['scp', worker_ssh_key_path, '%s:%s' % (nfs_netloc, path)]) + self.set_remote_file_contents( + nfs_netloc, os.path.join(path, 'distbuild.conf'), config_text) + + def configure_nfs_exports(self, nfs_netloc, system_names): + '''Ensure the Trove is set up to export the NFS roots we need. + + This doesn't handle setting up the TFTP daemon. We assume that is + already running. + + ''' + for system_name in system_names: + exported_path = self.system_path(system_name) + exports_path = '/etc/exports' + + # Rather ugly SSH hackery follows to ensure each system path is + # listed in /etc/exports. + try: + cliapp.ssh_runcmd( + nfs_netloc, ['grep', '-q', exported_path, exports_path]) + except cliapp.AppException: + ip_mask = '*' + options = 'rw,no_subtree_check,no_root_squash,async' + exports_string = '%s %s(%s)\n' % (exported_path, ip_mask, + options) + exports_append_sh = '''\ + set -eu + target="$1" + temp=$(mktemp) + cat "$target" > "$temp" + cat >> "$temp" + mv "$temp" "$target" + ''' + cliapp.ssh_runcmd( + nfs_netloc, + ['sh', '-c', exports_append_sh, '--', exports_path], + feed_stdin=exports_string) + + cliapp.ssh_runcmd(nfs_netloc, + ['systemctl', 'restart', 'nfs-server.service']) + + def update_default_version(self, remote_netloc, system_name, + version_label): + self.status(msg='Linking \'default\' to %(version)s for %(system)s', + version=version_label, system=system_name) + system_path = self.system_path(system_name) + system_version_path = os.path.join(system_path, 'systems', + version_label) + default_path = os.path.join(system_path, 'systems', 'default') + + cliapp.ssh_runcmd(remote_netloc, + ['ln', '-sfn', system_version_path, default_path]) + + +DistbuildTroveNFSBootWriteExtension().run() diff --git a/distbuild-trove-nfsboot.write.help b/distbuild-trove-nfsboot.write.help new file mode 100644 index 00000000..62f1455c --- /dev/null +++ b/distbuild-trove-nfsboot.write.help @@ -0,0 +1,49 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see <http://www.gnu.org/licenses/>. + +help: | + Deploy a distbuild network, using a Trove to serve the kernel and rootfs. + + The `location` argument is the hostname of the Trove system. + + The following configuration values must be specified: + + - DISTBUILD_CONTROLLER: hostname of controller system + - DISTBUILD_WORKERS: hostnames of each worker system + - DISTBUILD_GIT_SERVER: Trove hostname + - DISTBUILD_SHARED_ARTIFACT_CACHE: Trove hostname + - DISTBUILD_TROVE_ID: Trove ID + - DISTBUILD_WORKER_SSH_KEY: SSH key to be used for ssh:// repos + + A note on TROVE_ID: the current distbuild-setup service requires that + a single 'Trove ID' is specified. This is used in Morph for expanding + keyed URLs. If you set TROVE_ID=foo for example, foo:bar will be expanded + to git://$GIT_SERVER/foo, in addition to the standard baserock: and + upstream: prefixes that you can use. + + The WORKER_SSH_KEY must be provided, even if you don't need it. The + distbuild-setup service could be changed to make it optional. + + The following configuration values are optional: + + - HOST_MAP: a list of key=value pairs mapping hostnames to IP addresses, + or fully-qualified domain names. Useful if you + cannot rely on hostname resolution working for your deploment. + + The extension will connect to root@location via ssh to copy the kernel and + rootfs, and configure the nfs server. It will duplicate the kernel and + rootfs once for each node in the distbuild network. + + The deployment mechanism makes assumptions about the bootloader + configuration of the target machines. diff --git a/fstab.configure b/fstab.configure new file mode 100755 index 00000000..b9154eee --- /dev/null +++ b/fstab.configure @@ -0,0 +1,28 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright © 2013-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. +# +# =*= License: GPL-2 =*= + + +import os +import sys + +import morphlib + +envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('FSTAB_')} + +conf_file = os.path.join(sys.argv[1], 'etc/fstab') +morphlib.util.write_from_dict(conf_file, envvars) diff --git a/hosts.configure b/hosts.configure new file mode 100755 index 00000000..6b068d04 --- /dev/null +++ b/hosts.configure @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright © 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + + +import os +import sys +import socket + +import morphlib + +def validate(var, line): + xs = line.split() + if len(xs) == 0: + raise morphlib.Error("`%s: %s': line is empty" % (var, line)) + + ip = xs[0] + hostnames = xs[1:] + + if len(hostnames) == 0: + raise morphlib.Error("`%s: %s': missing hostname" % (var, line)) + + family = socket.AF_INET6 if ':' in ip else socket.AF_INET + + try: + socket.inet_pton(family, ip) + except socket.error: + raise morphlib.Error("`%s: %s' invalid ip" % (var, ip)) + +envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('HOSTS_')} + +conf_file = os.path.join(sys.argv[1], 'etc/hosts') +morphlib.util.write_from_dict(conf_file, envvars, validate) diff --git a/initramfs.write b/initramfs.write new file mode 100755 index 00000000..1059defa --- /dev/null +++ b/initramfs.write @@ -0,0 +1,26 @@ +#!/bin/sh +# Copyright (C) 2014-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. +# +# =*= License: GPL-2 =*= + +set -e + +ROOTDIR="$1" +INITRAMFS_PATH="$2" + +(cd "$ROOTDIR" && + find . -print0 | + cpio -0 -H newc -o) | + gzip -c | install -D -m644 /dev/stdin "$INITRAMFS_PATH" diff --git a/initramfs.write.help b/initramfs.write.help new file mode 100644 index 00000000..54d3ae8c --- /dev/null +++ b/initramfs.write.help @@ -0,0 +1,55 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see <http://www.gnu.org/licenses/>. + +help: | + + Create an initramfs for a system by taking an existing system and + converting it to the appropriate format. + + The system must have a `/init` executable as the userland entry-point. + This can have a different path, if `rdinit=$path` is added to + the kernel command line. This can be added to the `rawdisk`, + `virtualbox-ssh` and `kvm` write extensions with the `KERNEL_CMDLINE` + option. + + It is possible to use a ramfs as the final rootfs without a `/init` + executable, by setting `root=/dev/mem`, or `rdinit=/sbin/init`, + but this is beyond the scope for the `initramfs.write` extension. + + The intended use of initramfs.write is to be part of a nested + deployment, so the parent system has an initramfs stored as + `/boot/initramfs.gz`. See the following example: + + name: initramfs-test + kind: cluster + systems: + - morph: minimal-system-x86_64-generic + deploy: + system: + type: rawdisk + location: initramfs-system-x86_64.img + DISK_SIZE: 1G + HOSTNAME: initramfs-system + INITRAMFS_PATH: boot/initramfs.gz + subsystems: + - morph: initramfs-x86_64 + deploy: + initramfs: + type: initramfs + location: boot/initramfs.gz + + Parameters: + + * location: the path where the initramfs will be installed (e.g. + `boot/initramfs.gz`) in the above example diff --git a/install-essential-files.configure b/install-essential-files.configure new file mode 100755 index 00000000..2779b0d4 --- /dev/null +++ b/install-essential-files.configure @@ -0,0 +1,42 @@ +#!/usr/bin/env python2 +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +''' A Morph configuration extension for adding essential files to a system + +It will read the manifest files located in essential-files/manifest, +then use the contens of those files to determine which files +to install into the target system. + +''' + +import subprocess +import os + +import cliapp + +class InstallEssentialFilesConfigureExtension(cliapp.Application): + + def process_args(self, args): + target_root = args[0] + os.environ["INSTALL_FILES"] = "essential-files/manifest" + self.install_essential_files(target_root) + + def install_essential_files(self, target_root): + command = os.path.join(os.path.dirname(__file__), + "install-files.configure") + subprocess.check_call([command, target_root]) + +InstallEssentialFilesConfigureExtension().run() diff --git a/install-essential-files.configure.help b/install-essential-files.configure.help new file mode 100644 index 00000000..9148aeff --- /dev/null +++ b/install-essential-files.configure.help @@ -0,0 +1,20 @@ +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see <http://www.gnu.org/licenses/>. + +help: | + This installs files from the essential-files/ folder in your + definitions.git repo, according to essential-files/manifest. + + It wraps the install-files.configure extension. Take a look to that + extension help to know more about the format of the manifest file. diff --git a/install-files.configure b/install-files.configure new file mode 100755 index 00000000..341cce61 --- /dev/null +++ b/install-files.configure @@ -0,0 +1,134 @@ +#!/usr/bin/python +# Copyright (C) 2013-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +''' A Morph configuration extension for adding arbitrary files to a system + +It will read the manifest files specified in the environment variable +INSTALL_FILES, then use the contens of those files to determine which files +to install into the target system. + +''' + +import cliapp +import os +import errno +import re +import sys +import shlex +import shutil +import stat + +try: + import jinja2 + jinja_available = True +except ImportError: + jinja_available = False + +class InstallFilesConfigureExtension(cliapp.Application): + + def process_args(self, args): + if not 'INSTALL_FILES' in os.environ: + return + target_root = args[0] + manifests = shlex.split(os.environ['INSTALL_FILES']) + for manifest in manifests: + self.install_manifest(manifest, target_root) + + def install_manifest(self, manifest, target_root): + manifest_dir = os.path.dirname(manifest) + with open(manifest) as f: + entries = f.readlines() + for entry in entries: + self.install_entry(entry, manifest_dir, target_root) + + def force_symlink(self, source, link_name): + try: + os.symlink(source, link_name) + except OSError as e: + if e.errno == errno.EEXIST: + os.remove(link_name) + os.symlink(source, link_name) + + def install_entry(self, entry, manifest_root, target_root): + m = re.match('(template )?(overwrite )?' + '([0-7]+) ([0-9]+) ([0-9]+) (\S+)', entry) + + if m: + template = m.group(1) + overwrite = m.group(2) + mode = int(m.group(3), 8) # mode is octal + uid = int(m.group(4)) + gid = int(m.group(5)) + path = m.group(6) + else: + raise cliapp.AppException('Invalid manifest entry, ' + 'format: [template] [overwrite] ' + '<octal mode> <uid decimal> <gid decimal> <filename>') + + dest_path = os.path.join(target_root, './' + path) + if stat.S_ISDIR(mode): + if os.path.exists(dest_path) and not overwrite: + dest_stat = os.stat(dest_path) + if (mode != dest_stat.st_mode + or uid != dest_stat.st_uid + or gid != dest_stat.st_gid): + raise cliapp.AppException('"%s" exists and is not ' + 'identical to directory ' + '"%s"' % (dest_path, entry)) + else: + os.mkdir(dest_path, mode) + os.chown(dest_path, uid, gid) + os.chmod(dest_path, mode) + + elif stat.S_ISLNK(mode): + if os.path.lexists(dest_path) and not overwrite: + raise cliapp.AppException('Symlink already exists at %s' + % dest_path) + else: + linkdest = os.readlink(os.path.join(manifest_root, + './' + path)) + self.force_symlink(linkdest, dest_path) + os.lchown(dest_path, uid, gid) + + elif stat.S_ISREG(mode): + if os.path.lexists(dest_path) and not overwrite: + raise cliapp.AppException('File already exists at %s' + % dest_path) + else: + if template: + if not jinja_available: + raise cliapp.AppException( + "Failed to install template file `%s': " + 'install-files templates require jinja2' + % path) + + loader = jinja2.FileSystemLoader(manifest_root) + env = jinja2.Environment(loader=loader, + keep_trailing_newline=True) + + env.get_template(path).stream(os.environ).dump(dest_path) + else: + shutil.copyfile(os.path.join(manifest_root, './' + path), + dest_path) + + os.chown(dest_path, uid, gid) + os.chmod(dest_path, mode) + + else: + raise cliapp.AppException('Mode given in "%s" is not a file,' + ' symlink or directory' % entry) + +InstallFilesConfigureExtension().run() diff --git a/install-files.configure.help b/install-files.configure.help new file mode 100644 index 00000000..991c26c8 --- /dev/null +++ b/install-files.configure.help @@ -0,0 +1,74 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see <http://www.gnu.org/licenses/>. + +help: | + Install a set of files onto a system + + To use this extension you create a directory of files you want to install + onto the target system. + + In this example we want to copy some ssh keys onto a system + + % mkdir sshkeyfiles + % mkdir -p sshkeyfiles/root/.ssh + % cp id_rsa sshkeyfiles/root/.ssh + % cp id_rsa.pub sshkeyfiles/root/.ssh + + Now we need to create a manifest file to set the file modes + and persmissions. The manifest file should be created inside the + directory that contains the files we're trying to install. + + cat << EOF > sshkeyfiles/manifest + 0040755 0 0 /root/.ssh + 0100600 0 0 /root/.ssh/id_rsa + 0100644 0 0 /root/.ssh/id_rsa.pub + EOF + + Then we add the path to our manifest to our cluster morph, + this path should be relative to the system definitions repository. + + INSTALL_FILES: sshkeysfiles/manifest + + More generally entries in the manifest are formatted as: + [overwrite] <octal mode> <uid decimal> <gid decimal> <filename> + + NOTE: Directories on the target must be created if they do not exist. + + The extension supports files, symlinks and directories. + + For example, + + 0100644 0 0 /etc/issue + + creates a regular file at /etc/issue with 644 permissions, + uid 0 and gid 0, if the file doesn't already exist. + + overwrite 0100644 0 0 /etc/issue + + creates a regular file at /etc/issue with 644 permissions, + uid 0 and gid 0, if the file already exists it is overwritten. + + 0100755 0 0 /usr/bin/foo + + creates an executable file at /usr/bin/foo + + 0040755 0 0 /etc/foodir + + creates a directory with 755 permissions + + 0120000 0 0 /usr/bin/bar + + creates a symlink at /usr/bin/bar + + NOTE: You will still need to make a symlink in the manifest directory. diff --git a/kvm.check b/kvm.check new file mode 100755 index 00000000..67cb3d38 --- /dev/null +++ b/kvm.check @@ -0,0 +1,169 @@ +#!/usr/bin/python +# Copyright (C) 2014-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + +'''Preparatory checks for Morph 'kvm' write extension''' + +import cliapp +import os +import re +import urlparse + +import morphlib.writeexts + + +class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): + + location_pattern = '^/(?P<guest>[^/]+)(?P<path>/.+)$' + + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + self.require_btrfs_in_deployment_host_kernel() + + upgrade = self.get_environment_boolean('UPGRADE') + if upgrade: + raise cliapp.AppException( + 'Use the `ssh-rsync` write extension to deploy upgrades to an ' + 'existing remote system.') + + location = args[0] + ssh_host, vm_name, vm_path = self.check_and_parse_location(location) + + self.check_ssh_connectivity(ssh_host) + self.check_can_create_file_at_given_path(ssh_host, vm_path) + self.check_no_existing_libvirt_vm(ssh_host, vm_name) + self.check_extra_disks_exist(ssh_host, self.parse_attach_disks()) + self.check_virtual_networks_are_started(ssh_host) + self.check_host_has_virtinstall(ssh_host) + + def check_and_parse_location(self, location): + '''Check and parse the location argument to get relevant data.''' + + x = urlparse.urlparse(location) + + if x.scheme != 'kvm+ssh': + raise cliapp.AppException( + 'URL schema must be kvm+ssh in %s' % location) + + m = re.match(self.location_pattern, x.path) + if not m: + raise cliapp.AppException('Cannot parse location %s' % location) + + return x.netloc, m.group('guest'), m.group('path') + + def check_no_existing_libvirt_vm(self, ssh_host, vm_name): + try: + cliapp.ssh_runcmd(ssh_host, + ['virsh', '--connect', 'qemu:///system', 'domstate', vm_name]) + except cliapp.AppException as e: + pass + else: + raise cliapp.AppException( + 'Host %s already has a VM named %s. You can use the ssh-rsync ' + 'write extension to deploy upgrades to existing machines.' % + (ssh_host, vm_name)) + + def check_can_create_file_at_given_path(self, ssh_host, vm_path): + + def check_can_write_to_given_path(): + try: + cliapp.ssh_runcmd(ssh_host, ['touch', vm_path]) + except cliapp.AppException as e: + raise cliapp.AppException("Can't write to location %s on %s" + % (vm_path, ssh_host)) + else: + cliapp.ssh_runcmd(ssh_host, ['rm', vm_path]) + + try: + cliapp.ssh_runcmd(ssh_host, ['test', '-e', vm_path]) + except cliapp.AppException as e: + # vm_path doesn't already exist, so let's test we can write + check_can_write_to_given_path() + else: + raise cliapp.AppException('%s already exists on %s' + % (vm_path, ssh_host)) + + def check_extra_disks_exist(self, ssh_host, filename_list): + for filename in filename_list: + try: + cliapp.ssh_runcmd(ssh_host, ['ls', filename]) + except cliapp.AppException as e: + raise cliapp.AppException('Did not find file %s on host %s' % + (filename, ssh_host)) + + def check_virtual_networks_are_started(self, ssh_host): + + def check_virtual_network_is_started(network_name): + cmd = ['virsh', '-c', 'qemu:///system', 'net-info', network_name] + net_info = cliapp.ssh_runcmd(ssh_host, cmd).split('\n') + + def pretty_concat(lines): + return '\n'.join(['\t%s' % line for line in lines]) + + for line in net_info: + m = re.match('^Active:\W*(\w+)\W*', line) + if m: + break + else: + raise cliapp.AppException( + "Got unexpected output parsing output of `%s':\n%s" + % (' '.join(cmd), pretty_concat(net_info))) + + network_active = m.group(1) == 'yes' + + if not network_active: + raise cliapp.AppException("Network '%s' is not started" + % network_name) + + def name(nic_entry): + if ',' in nic_entry: + # network=NETWORK_NAME,mac=12:34,model=e1000... + return nic_entry[:nic_entry.find(',')].lstrip('network=') + else: + return nic_entry.lstrip('network=') # NETWORK_NAME + + if 'NIC_CONFIG' in os.environ: + nics = os.environ['NIC_CONFIG'].split() + + for n in nics: + if not (n.startswith('network=') + or n.startswith('bridge=') + or n == 'user'): + raise cliapp.AppException('malformed NIC_CONFIG: %s\n' + " (expected 'bridge=BRIDGE' 'network=NAME'" + " or 'user')" % n) + + # --network bridge= is used to specify a bridge + # --network user is used to specify a form of NAT + # (see the virt-install(1) man page) + networks = [name(n) for n in nics if not n.startswith('bridge=') + and not n.startswith('user')] + else: + networks = ['default'] + + for network in networks: + check_virtual_network_is_started(network) + + def check_host_has_virtinstall(self, ssh_host): + try: + cliapp.ssh_runcmd(ssh_host, ['which', 'virt-install']) + except cliapp.AppException: + raise cliapp.AppException( + 'virt-install does not seem to be installed on host %s' + % ssh_host) + + +KvmPlusSshCheckExtension().run() diff --git a/kvm.write b/kvm.write new file mode 100755 index 00000000..0d0c095b --- /dev/null +++ b/kvm.write @@ -0,0 +1,120 @@ +#!/usr/bin/python +# Copyright (C) 2012-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +'''A Morph deployment write extension for deploying to KVM+libvirt. + +See file kvm.write.help for documentation + +''' + + +import cliapp +import os +import re +import sys +import tempfile +import urlparse + +import morphlib.writeexts + + +class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): + + location_pattern = '^/(?P<guest>[^/]+)(?P<path>/.+)$' + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + temp_root, location = args + ssh_host, vm_name, vm_path = self.parse_location(location) + autostart = self.get_environment_boolean('AUTOSTART') + + fd, raw_disk = tempfile.mkstemp() + os.close(fd) + self.create_local_system(temp_root, raw_disk) + + try: + self.transfer(raw_disk, ssh_host, vm_path) + self.create_libvirt_guest(ssh_host, vm_name, vm_path, autostart) + except BaseException: + sys.stderr.write('Error deploying to libvirt') + os.remove(raw_disk) + cliapp.ssh_runcmd(ssh_host, ['rm', '-f', vm_path]) + raise + else: + os.remove(raw_disk) + + self.status( + msg='Virtual machine %(vm_name)s has been created', + vm_name=vm_name) + + def parse_location(self, location): + '''Parse the location argument to get relevant data.''' + + x = urlparse.urlparse(location) + m = re.match('^/(?P<guest>[^/]+)(?P<path>/.+)$', x.path) + return x.netloc, m.group('guest'), m.group('path') + + def transfer(self, raw_disk, ssh_host, vm_path): + '''Transfer raw disk image to libvirt host.''' + + self.status(msg='Transferring disk image') + + xfer_hole_path = morphlib.util.get_data_path('xfer-hole') + recv_hole = morphlib.util.get_data('recv-hole') + + ssh_remote_cmd = [ + 'sh', '-c', recv_hole, 'dummy-argv0', 'file', vm_path + ] + + cliapp.runcmd( + ['python', xfer_hole_path, raw_disk], + ['ssh', ssh_host] + map(cliapp.shell_quote, ssh_remote_cmd), + stdout=None, stderr=None) + + def create_libvirt_guest(self, ssh_host, vm_name, vm_path, autostart): + '''Create the libvirt virtual machine.''' + + self.status(msg='Creating libvirt/kvm virtual machine') + + attach_disks = self.parse_attach_disks() + attach_opts = [] + for disk in attach_disks: + attach_opts.extend(['--disk', 'path=%s' % disk]) + + if 'NIC_CONFIG' in os.environ: + nics = os.environ['NIC_CONFIG'].split() + for nic in nics: + attach_opts.extend(['--network', nic]) + + ram_mebibytes = str(self.get_ram_size() / (1024**2)) + + vcpu_count = str(self.get_vcpu_count()) + + cmdline = ['virt-install', '--connect', 'qemu:///system', + '--import', '--name', vm_name, '--vnc', + '--ram', ram_mebibytes, '--vcpus', vcpu_count, + '--disk', 'path=%s,bus=ide' % vm_path] + attach_opts + if not autostart: + cmdline += ['--noreboot'] + cliapp.ssh_runcmd(ssh_host, cmdline) + + if autostart: + cliapp.ssh_runcmd(ssh_host, + ['virsh', '--connect', 'qemu:///system', 'autostart', vm_name]) + +KvmPlusSshWriteExtension().run() diff --git a/kvm.write.help b/kvm.write.help new file mode 100644 index 00000000..812a5309 --- /dev/null +++ b/kvm.write.help @@ -0,0 +1,90 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see <http://www.gnu.org/licenses/>. + +help: | + + Deploy a Baserock system as a *new* KVM/LibVirt virtual machine. + + Use the `ssh-rsync` write extension to deploy upgrades to an *existing* VM + + Parameters: + + * location: a custom URL scheme of the form `kvm+ssh://HOST/GUEST/PATH`, + where: + * HOST is the name of the host on which KVM/LibVirt is running + * GUEST is the name of the guest VM on that host + * PATH is the path to the disk image that should be created, + on that host. For example, + `kvm+ssh://alice@192.168.122.1/testsys/home/alice/testys.img` where + * `alice@192.168.122.1` is the target host as given to ssh, + **from within the development host** (which may be + different from the target host's normal address); + * `testsys` is the name of the new guest VM'; + * `/home/alice/testys.img` is the pathname of the disk image files + on the target host. + + * HOSTNAME=name: the hostname of the **guest** VM within the network into + which it is being deployed + + * DISK_SIZE=X: the size of the VM's primary virtual hard disk. `X` should + use a suffix of `K`, `M`, or `G` (in upper or lower case) to indicate + kilo-, mega-, or gigabytes. For example, `DISK_SIZE=100G` would create a + 100 gigabyte disk image. **This parameter is mandatory**. + + * RAM_SIZE=X: The amount of RAM that the virtual machine should allocate + for itself from the host. `X` is interpreted in the same was as for + DISK_SIZE`, and defaults to `1G` + + * VCPUS=n: the number of virtual CPUs for the VM. Allowed values 1-32. Do + not use more CPU cores than you have available physically (real cores, no + hyperthreads) + + * INITRAMFS_PATH=path: the location of an initramfs for the bootloader to + tell Linux to use, rather than booting the rootfs directly. + + * AUTOSTART=<VALUE>` - boolean. If it is set, the VM will be started when + it has been deployed. + + * DTB_PATH=path: **(MANDATORY)** for systems that require a device tree + binary - Give the full path (without a leading /) to the location of the + DTB in the built system image . The deployment will fail if `path` does + not exist. + + * BOOTLOADER_INSTALL=value: the bootloader to be installed + **(MANDATORY)** for non-x86 systems + + allowed values = + - 'extlinux' (default) - the extlinux bootloader will + be installed + - 'none' - no bootloader will be installed by `morph deploy`. A + bootloader must be installed manually. This value must be used when + deploying non-x86 systems such as ARM. + + * BOOTLOADER_CONFIG_FORMAT=value: the bootloader format to be used. + If not specified for x86-32 and x86-64 systems, 'extlinux' will be used + + allowed values = + - 'extlinux' + + * KERNEL_ARGS=args: optional additional kernel command-line parameters to + be appended to the default set. The default set is: + + 'rw init=/sbin/init rootfstype=btrfs \ + rootflags=subvol=systems/default/run \ + root=[name or UUID of root filesystem]' + + (See https://www.kernel.org/doc/Documentation/kernel-parameters.txt) + + (See `morph help deploy` for details of how to pass parameters to write + extensions) diff --git a/nfsboot.check b/nfsboot.check new file mode 100755 index 00000000..e273f61c --- /dev/null +++ b/nfsboot.check @@ -0,0 +1,95 @@ +#!/usr/bin/python +# Copyright (C) 2014-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + +'''Preparatory checks for Morph 'nfsboot' write extension''' + +import cliapp +import os + +import morphlib.writeexts + + +class NFSBootCheckExtension(morphlib.writeexts.WriteExtension): + + _nfsboot_root = '/srv/nfsboot' + + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + location = args[0] + + upgrade = self.get_environment_boolean('UPGRADE') + if upgrade: + raise cliapp.AppException( + 'Upgrading is not currently supported for NFS deployments.') + + hostname = os.environ.get('HOSTNAME', None) + if hostname is None: + raise cliapp.AppException('You must specify a HOSTNAME.') + if hostname == 'baserock': + raise cliapp.AppException('It is forbidden to nfsboot a system ' + 'with hostname "%s"' % hostname) + + self.test_good_server(location) + + version_label = os.getenv('VERSION_LABEL', 'factory') + versioned_root = os.path.join(self._nfsboot_root, hostname, 'systems', + version_label) + if self.version_exists(versioned_root, location): + raise cliapp.AppException( + 'Root file system for host %s (version %s) already exists on ' + 'the NFS server %s. Deployment aborted.' % (hostname, + version_label, location)) + + def test_good_server(self, server): + self.check_ssh_connectivity(server) + + # Is an NFS server + try: + cliapp.ssh_runcmd( + 'root@%s' % server, ['test', '-e', '/etc/exports']) + except cliapp.AppException: + raise cliapp.AppException('server %s is not an nfs server' + % server) + try: + cliapp.ssh_runcmd( + 'root@%s' % server, ['systemctl', 'is-enabled', + 'nfs-server.service']) + + except cliapp.AppException: + raise cliapp.AppException('server %s does not control its ' + 'nfs server by systemd' % server) + + # TFTP server exports /srv/nfsboot/tftp + tftp_root = os.path.join(self._nfsboot_root, 'tftp') + try: + cliapp.ssh_runcmd( + 'root@%s' % server, ['test' , '-d', tftp_root]) + except cliapp.AppException: + raise cliapp.AppException('server %s does not export %s' % + (tftp_root, server)) + + def version_exists(self, versioned_root, location): + try: + cliapp.ssh_runcmd('root@%s' % location, + ['test', '-d', versioned_root]) + except cliapp.AppException: + return False + + return True + + +NFSBootCheckExtension().run() diff --git a/nfsboot.configure b/nfsboot.configure new file mode 100755 index 00000000..6a68dc48 --- /dev/null +++ b/nfsboot.configure @@ -0,0 +1,30 @@ +#!/bin/sh +# Copyright (C) 2013-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +# Remove all networking interfaces. On nfsboot systems, eth0 is set up +# during kernel init, and the normal ifup@eth0.service systemd unit +# would break the NFS connection and cause the system to hang. + + +set -e +if [ "$NFSBOOT_CONFIGURE" ]; then + # Remove all networking interfaces but loopback + cat > "$1/etc/network/interfaces" <<EOF +auto lo +iface lo inet loopback +EOF + +fi diff --git a/nfsboot.write b/nfsboot.write new file mode 100755 index 00000000..d928775e --- /dev/null +++ b/nfsboot.write @@ -0,0 +1,202 @@ +#!/usr/bin/python +# Copyright (C) 2013-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +'''A Morph deployment write extension for deploying to an nfsboot server + +*** DO NOT USE *** +- This was written before 'proper' deployment mechanisms were in place +It is unlikely to work at all and will not work correctly + +Use the pxeboot write extension instead + +*** + + + +An nfsboot server is defined as a baserock system that has tftp and nfs +servers running, the tftp server is exporting the contents of +/srv/nfsboot/tftp/ and the user has sufficient permissions to create nfs roots +in /srv/nfsboot/nfs/ + +''' + + +import cliapp +import os +import glob + +import morphlib.writeexts + + +class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): + + '''Create an NFS root and kernel on TFTP during Morph's deployment. + + The location command line argument is the hostname of the nfsboot server. + The user is expected to provide the location argument + using the following syntax: + + HOST + + where: + + * HOST is the host of the nfsboot server + + The extension will connect to root@HOST via ssh to copy the kernel and + rootfs, and configure the nfs server. + + It requires root because it uses systemd, and reads/writes to /etc. + + ''' + + _nfsboot_root = '/srv/nfsboot' + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + temp_root, location = args + + version_label = os.getenv('VERSION_LABEL', 'factory') + hostname = os.environ['HOSTNAME'] + + versioned_root = os.path.join(self._nfsboot_root, hostname, 'systems', + version_label) + + self.copy_rootfs(temp_root, location, versioned_root, hostname) + self.copy_kernel(temp_root, location, versioned_root, version_label, + hostname) + self.configure_nfs(location, hostname) + + def create_local_state(self, location, hostname): + statedir = os.path.join(self._nfsboot_root, hostname, 'state') + subdirs = [os.path.join(statedir, 'home'), + os.path.join(statedir, 'opt'), + os.path.join(statedir, 'srv')] + cliapp.ssh_runcmd('root@%s' % location, + ['mkdir', '-p'] + subdirs) + + def copy_kernel(self, temp_root, location, versioned_root, version, + hostname): + bootdir = os.path.join(temp_root, 'boot') + image_names = ['vmlinuz', 'zImage', 'uImage'] + for name in image_names: + try_path = os.path.join(bootdir, name) + if os.path.exists(try_path): + kernel_src = try_path + break + else: + raise cliapp.AppException( + 'Could not find a kernel in the system: none of ' + '%s found' % ', '.join(image_names)) + + kernel_dest = os.path.join(versioned_root, 'orig', 'kernel') + rsync_dest = 'root@%s:%s' % (location, kernel_dest) + self.status(msg='Copying kernel') + cliapp.runcmd( + ['rsync', '-s', kernel_src, rsync_dest]) + + # Link the kernel to the right place + self.status(msg='Creating links to kernel in tftp directory') + tftp_dir = os.path.join(self._nfsboot_root , 'tftp') + versioned_kernel_name = "%s-%s" % (hostname, version) + kernel_name = hostname + try: + cliapp.ssh_runcmd('root@%s' % location, + ['ln', '-f', kernel_dest, + os.path.join(tftp_dir, versioned_kernel_name)]) + + cliapp.ssh_runcmd('root@%s' % location, + ['ln', '-sf', versioned_kernel_name, + os.path.join(tftp_dir, kernel_name)]) + except cliapp.AppException: + raise cliapp.AppException('Could not create symlinks to the ' + 'kernel at %s in %s on %s' + % (kernel_dest, tftp_dir, location)) + + def copy_rootfs(self, temp_root, location, versioned_root, hostname): + rootfs_src = temp_root + '/' + orig_path = os.path.join(versioned_root, 'orig') + run_path = os.path.join(versioned_root, 'run') + + self.status(msg='Creating destination directories') + try: + cliapp.ssh_runcmd('root@%s' % location, + ['mkdir', '-p', orig_path, run_path]) + except cliapp.AppException: + raise cliapp.AppException('Could not create dirs %s and %s on %s' + % (orig_path, run_path, location)) + + self.status(msg='Creating \'orig\' rootfs') + cliapp.runcmd( + ['rsync', '-asXSPH', '--delete', rootfs_src, + 'root@%s:%s' % (location, orig_path)]) + + self.status(msg='Creating \'run\' rootfs') + try: + cliapp.ssh_runcmd('root@%s' % location, + ['rm', '-rf', run_path]) + cliapp.ssh_runcmd('root@%s' % location, + ['cp', '-al', orig_path, run_path]) + cliapp.ssh_runcmd('root@%s' % location, + ['rm', '-rf', os.path.join(run_path, 'etc')]) + cliapp.ssh_runcmd('root@%s' % location, + ['cp', '-a', os.path.join(orig_path, 'etc'), + os.path.join(run_path, 'etc')]) + except cliapp.AppException: + raise cliapp.AppException('Could not create \'run\' rootfs' + ' from \'orig\'') + + self.status(msg='Linking \'default\' to latest system') + try: + cliapp.ssh_runcmd('root@%s' % location, + ['ln', '-sfn', versioned_root, + os.path.join(self._nfsboot_root, hostname, 'systems', + 'default')]) + except cliapp.AppException: + raise cliapp.AppException('Could not link \'default\' to %s' + % versioned_root) + + def configure_nfs(self, location, hostname): + exported_path = os.path.join(self._nfsboot_root, hostname) + exports_path = '/etc/exports' + # If that path is not already exported: + try: + cliapp.ssh_runcmd( + 'root@%s' % location, ['grep', '-q', exported_path, + exports_path]) + except cliapp.AppException: + ip_mask = '*' + options = 'rw,no_subtree_check,no_root_squash,async' + exports_string = '%s %s(%s)\n' % (exported_path, ip_mask, options) + exports_append_sh = '''\ +set -eu +target="$1" +temp=$(mktemp) +cat "$target" > "$temp" +cat >> "$temp" +mv "$temp" "$target" +''' + cliapp.ssh_runcmd( + 'root@%s' % location, + ['sh', '-c', exports_append_sh, '--', exports_path], + feed_stdin=exports_string) + cliapp.ssh_runcmd( + 'root@%s' % location, ['systemctl', 'restart', + 'nfs-server.service']) + + +NFSBootWriteExtension().run() diff --git a/nfsboot.write.help b/nfsboot.write.help new file mode 100644 index 00000000..186c479a --- /dev/null +++ b/nfsboot.write.help @@ -0,0 +1,33 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see <http://www.gnu.org/licenses/>. + +help: | + *** DO NOT USE *** + - This was written before 'proper' deployment mechanisms were in place. + It is unlikely to work at all, and will not work correctly. + + Use the pxeboot write extension instead + + *** + Deploy a system image and kernel to an nfsboot server. + + An nfsboot server is defined as a baserock system that has + tftp and nfs servers running, the tftp server is exporting + the contents of /srv/nfsboot/tftp/ and the user has sufficient + permissions to create nfs roots in /srv/nfsboot/nfs/. + + The `location` argument is the hostname of the nfsboot server. + + The extension will connect to root@HOST via ssh to copy the + kernel and rootfs, and configure the nfs server. diff --git a/openstack.check b/openstack.check new file mode 100755 index 00000000..a3379763 --- /dev/null +++ b/openstack.check @@ -0,0 +1,90 @@ +#!/usr/bin/python +# Copyright (C) 2014-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + +'''Preparatory checks for Morph 'openstack' write extension''' + +import cliapp +import os +import urlparse +import keystoneclient + +import morphlib.writeexts + + +class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): + + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + self.require_btrfs_in_deployment_host_kernel() + + upgrade = self.get_environment_boolean('UPGRADE') + if upgrade: + raise cliapp.AppException( + 'Use the `ssh-rsync` write extension to deploy upgrades to an ' + 'existing remote system.') + + location = args[0] + self.check_location(location) + + self.check_imagename() + self.check_openstack_parameters(self._get_auth_parameters(location)) + + def _get_auth_parameters(self, location): + '''Check the environment variables needed and returns all. + + The environment variables are described in the class documentation. + ''' + + auth_keys = {'OPENSTACK_USER': 'username', + 'OPENSTACK_TENANT': 'tenant_name', + 'OPENSTACK_PASSWORD': 'password'} + + for key in auth_keys: + if os.environ.get(key, '') == '': + raise cliapp.AppException(key + ' was not given') + + auth_params = {auth_keys[key]: os.environ[key] for key in auth_keys} + auth_params['auth_url'] = location + return auth_params + + def check_imagename(self): + if os.environ.get('OPENSTACK_IMAGENAME', '') == '': + raise cliapp.AppException('OPENSTACK_IMAGENAME was not given') + + def check_location(self, location): + x = urlparse.urlparse(location) + if x.scheme not in ['http', 'https']: + raise cliapp.AppException('URL schema must be http or https in %s'\ + % location) + if (x.path != '/v2.0' and x.path != '/v2.0/'): + raise cliapp.AppException('API version must be v2.0 in %s'\ + % location) + + def check_openstack_parameters(self, auth_params): + ''' Check that we can connect to and authenticate with openstack ''' + + self.status(msg='Checking OpenStack credentials...') + + try: + keystoneclient.v2_0.Client(**auth_params) + except keystoneclient.exceptions.Unauthorized: + errmsg = ('Failed to authenticate with OpenStack ' + '(are your credentials correct?)') + raise cliapp.AppException(errmsg) + + +OpenStackCheckExtension().run() diff --git a/openstack.write b/openstack.write new file mode 100755 index 00000000..67e07c18 --- /dev/null +++ b/openstack.write @@ -0,0 +1,93 @@ +#!/usr/bin/python +# Copyright (C) 2013-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +'''A Morph deployment write extension for deploying to OpenStack.''' + + +import cliapp +import os +import tempfile +import urlparse + +import morphlib.writeexts + + +class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): + + '''See openstack.write.help for documentation''' + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + temp_root, location = args + + os_params = self.get_openstack_parameters() + + fd, raw_disk = tempfile.mkstemp() + os.close(fd) + self.create_local_system(temp_root, raw_disk) + self.status(msg='Temporary disk image has been created at %s' + % raw_disk) + + self.set_extlinux_root_to_virtio(raw_disk) + + self.configure_openstack_image(raw_disk, location, os_params) + + def set_extlinux_root_to_virtio(self, raw_disk): + '''Re-configures extlinux to use virtio disks''' + self.status(msg='Updating extlinux.conf') + with self.mount(raw_disk) as mp: + path = os.path.join(mp, 'extlinux.conf') + + with open(path) as f: + extlinux_conf = f.read() + + extlinux_conf = extlinux_conf.replace('root=/dev/sda', + 'root=/dev/vda') + with open(path, "w") as f: + f.write(extlinux_conf) + + def get_openstack_parameters(self): + '''Get the environment variables needed. + + The environment variables are described in the class documentation. + ''' + + keys = ('OPENSTACK_USER', 'OPENSTACK_TENANT', + 'OPENSTACK_IMAGENAME', 'OPENSTACK_PASSWORD') + return (os.environ[key] for key in keys) + + def configure_openstack_image(self, raw_disk, auth_url, os_params): + '''Configure the image in OpenStack using glance-client''' + self.status(msg='Configuring OpenStack image...') + + username, tenant_name, image_name, password = os_params + cmdline = ['glance', + '--os-username', username, + '--os-tenant-name', tenant_name, + '--os-password', password, + '--os-auth-url', auth_url, + 'image-create', + '--name=%s' % image_name, + '--disk-format=raw', + '--container-format', 'bare', + '--file', raw_disk] + cliapp.runcmd(cmdline) + + self.status(msg='Image configured.') + +OpenStackWriteExtension().run() diff --git a/openstack.write.help b/openstack.write.help new file mode 100644 index 00000000..26983060 --- /dev/null +++ b/openstack.write.help @@ -0,0 +1,51 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see <http://www.gnu.org/licenses/>. + +help: | + + Deploy a Baserock system as a *new* OpenStack virtual machine. + (Use the `ssh-rsync` write extension to deploy upgrades to an *existing* + VM) + + Deploys the system to the OpenStack host using python-glanceclient. + + Parameters: + + * location: the authentication url of the OpenStack server using the + following syntax: + + http://HOST:PORT/VERSION + + where + + * HOST is the host running OpenStack + * PORT is the port which is using OpenStack for authentications. + * VERSION is the authentication version of OpenStack (Only v2.0 + supported) + + * OPENSTACK_USER=username: the username to use in the `--os-username` + argument to `glance`. + + * OPENSTACK_TENANT=tenant: the project name to use in the + `--os-tenant-name` argument to `glance`. + + * OPENSTACK_IMAGENAME=imagename: the name of the image to use in the + `--name` argument to `glance`. + + * OPENSTACK_PASSWORD=password: the password of the OpenStack user. (We + recommend passing this on the command-line, rather than setting an + environment variable or storing it in a cluster cluster definition file.) + + (See `morph help deploy` for details of how to pass parameters to write + extensions) diff --git a/rawdisk.check b/rawdisk.check new file mode 100755 index 00000000..9be0ce91 --- /dev/null +++ b/rawdisk.check @@ -0,0 +1,53 @@ +#!/usr/bin/python +# Copyright (C) 2014-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + +'''Preparatory checks for Morph 'rawdisk' write extension''' + +import cliapp + +import morphlib.writeexts + +import os + + +class RawdiskCheckExtension(morphlib.writeexts.WriteExtension): + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + self.require_btrfs_in_deployment_host_kernel() + + location = args[0] + upgrade = self.get_environment_boolean('UPGRADE') + if upgrade: + if not self.is_device(location): + if not os.path.isfile(location): + raise cliapp.AppException( + 'Cannot upgrade %s: it is not an existing disk image' % + location) + + version_label = os.environ.get('VERSION_LABEL') + if version_label is None: + raise cliapp.AppException( + 'VERSION_LABEL was not given. It is required when ' + 'upgrading an existing system.') + else: + if not self.is_device(location): + if os.path.exists(location): + raise cliapp.AppException( + 'Target %s already exists. Use `morph upgrade` if you ' + 'want to update an existing image.' % location) + +RawdiskCheckExtension().run() diff --git a/rawdisk.write b/rawdisk.write new file mode 100755 index 00000000..6f2d45ba --- /dev/null +++ b/rawdisk.write @@ -0,0 +1,108 @@ +#!/usr/bin/python +# Copyright (C) 2012-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +'''A Morph deployment write extension for raw disk images.''' + + +import cliapp +import os +import sys +import time +import tempfile + +import morphlib.writeexts + + +class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): + + '''See rawdisk.write.help for documentation''' + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + temp_root, location = args + upgrade = self.get_environment_boolean('UPGRADE') + + if upgrade: + self.upgrade_local_system(location, temp_root) + else: + try: + if not self.is_device(location): + with self.created_disk_image(location): + self.format_btrfs(location) + self.create_system(temp_root, location) + self.status(msg='Disk image has been created at %s' % + location) + else: + self.format_btrfs(location) + self.create_system(temp_root, location) + self.status(msg='System deployed to %s' % location) + except Exception: + self.status(msg='Failure to deploy system to %s' % + location) + raise + + def upgrade_local_system(self, raw_disk, temp_root): + self.complete_fstab_for_btrfs_layout(temp_root) + + with self.mount(raw_disk) as mp: + version_label = self.get_version_label(mp) + self.status(msg='Updating image to a new version with label %s' % + version_label) + + version_root = os.path.join(mp, 'systems', version_label) + os.mkdir(version_root) + + old_orig = os.path.join(mp, 'systems', 'default', 'orig') + new_orig = os.path.join(version_root, 'orig') + cliapp.runcmd( + ['btrfs', 'subvolume', 'snapshot', old_orig, new_orig]) + + cliapp.runcmd( + ['rsync', '-a', '--checksum', '--numeric-ids', '--delete', + temp_root + os.path.sep, new_orig]) + + self.create_run(version_root) + + default_path = os.path.join(mp, 'systems', 'default') + if os.path.exists(default_path): + os.remove(default_path) + else: + # we are upgrading and old system that does + # not have an updated extlinux config file + if self.bootloader_config_is_wanted(): + self.generate_bootloader_config(mp) + self.install_bootloader(mp) + os.symlink(version_label, default_path) + + if self.bootloader_config_is_wanted(): + self.install_kernel(version_root, temp_root) + + def get_version_label(self, mp): + version_label = os.environ.get('VERSION_LABEL') + + if version_label is None: + raise cliapp.AppException('VERSION_LABEL was not given') + + if os.path.exists(os.path.join(mp, 'systems', version_label)): + raise cliapp.AppException('VERSION_LABEL %s already exists' + % version_label) + + return version_label + + +RawDiskWriteExtension().run() diff --git a/rawdisk.write.help b/rawdisk.write.help new file mode 100644 index 00000000..52ed73fb --- /dev/null +++ b/rawdisk.write.help @@ -0,0 +1,82 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see <http://www.gnu.org/licenses/>. + +help: | + + Write a system produced by Morph to a physical disk, or to a file that can + be used as a virtual disk. The target will be formatted as a single Btrfs + partition, with the system image written to a subvolume in /systems, and + other subvolumes created for /home, /opt, /root, /srv and /var. + + When written to a physical drive, the drive can be used as the boot device + for a 'real' machine. + + When written to a file, the file can be used independently of `morph` to + create virtual machines with KVM / libvirt, OpenStack or, after converting + it to VDI format, VirtualBox. + + `morph deploy` will fail if the file specified by `location` already + exists. + + If used in `morph upgrade`, the rootfs produced by 'morph build' is added + to the existing raw disk image or device as an additional btrfs sub-volume. + `morph upgrade` will fail if the file specified by `location` does not + exist, or is not a Baserock raw disk image. (Most users are unlikely to + need or use this functionality: it is useful mainly for developers working + on the Baserock tools.) + + Parameters: + + * location: the pathname of the disk image to be created/upgraded, or the + path to the physical device. + + * VERSION_LABEL=label - should contain only alpha-numeric + characters and the '-' (hyphen) character. Mandatory if being used with + `morph update` + + * INITRAMFS_PATH=path: the location of an initramfs for the bootloader to + tell Linux to use, rather than booting the rootfs directly. + + * DTB_PATH=path: **(MANDATORY)** for systems that require a device tree + binary - Give the full path (without a leading /) to the location of the + DTB in the built system image . The deployment will fail if `path` does + not exist. + + * BOOTLOADER_INSTALL=value: the bootloader to be installed + **(MANDATORY)** for non-x86 systems + + allowed values = + - 'extlinux' (default) - the extlinux bootloader will + be installed + - 'none' - no bootloader will be installed by `morph deploy`. A + bootloader must be installed manually. This value must be used when + deploying non-x86 systems such as ARM. + + * BOOTLOADER_CONFIG_FORMAT=value: the bootloader format to be used. + If not specified for x86-32 and x86-64 systems, 'extlinux' will be used + + allowed values = + - 'extlinux' + + * KERNEL_ARGS=args: optional additional kernel command-line parameters to + be appended to the default set. The default set is: + + 'rw init=/sbin/init rootfstype=btrfs \ + rootflags=subvol=systems/default/run \ + root=[name or UUID of root filesystem]' + + (See https://www.kernel.org/doc/Documentation/kernel-parameters.txt) + + (See `morph help deploy` for details of how to pass parameters to write + extensions) diff --git a/set-hostname.configure b/set-hostname.configure new file mode 100755 index 00000000..4b2424d8 --- /dev/null +++ b/set-hostname.configure @@ -0,0 +1,26 @@ +#!/bin/sh +# Copyright (C) 2013,2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +# Set hostname on system from HOSTNAME. + + +set -e + +if [ -n "$HOSTNAME" ] +then + echo "$HOSTNAME" > "$1/etc/hostname" +fi + diff --git a/simple-network.configure b/simple-network.configure new file mode 100755 index 00000000..4a70f311 --- /dev/null +++ b/simple-network.configure @@ -0,0 +1,292 @@ +#!/usr/bin/python +# Copyright (C) 2013,2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + +'''A Morph deployment configuration extension to handle network configutation + +This extension prepares /etc/network/interfaces and networkd .network files +in /etc/systemd/network/ with the interfaces specified during deployment. + +If no network configuration is provided, eth0 will be configured for DHCP +with the hostname of the system in the case of /etc/network/interfaces. +In the case of networkd, any interface starting by e* will be configured +for DHCP +''' + + +import os +import sys +import errno +import cliapp + +import morphlib + + +class SimpleNetworkError(morphlib.Error): + '''Errors associated with simple network setup''' + pass + + +class SimpleNetworkConfigurationExtension(cliapp.Application): + '''Configure /etc/network/interfaces and generate networkd .network files + + Reading NETWORK_CONFIG, this extension sets up /etc/network/interfaces + and .network files in /etc/systemd/network/. + ''' + + def process_args(self, args): + network_config = os.environ.get("NETWORK_CONFIG") + + self.rename_networkd_chunk_file(args) + + if network_config is None: + self.generate_default_network_config(args) + else: + self.status(msg="Processing NETWORK_CONFIG=%(nc)s", + nc=network_config) + + stanzas = self.parse_network_stanzas(network_config) + + self.generate_interfaces_file(args, stanzas) + self.generate_networkd_files(args, stanzas) + + def rename_networkd_chunk_file(self, args): + """Rename the 10-dchp.network file generated in the systemd chunk + + The systemd chunk will place something in 10-dhcp.network, which will + have higher precedence than anything added in this extension (we + start at 50-*). + + We should check for that file and rename it instead remove it in + case the file is being used by the user. + + Until both the following happen, we should continue to rename that + default config file: + + 1. simple-network.configure is always run when systemd is included + 2. We've been building systems without systemd including that default + networkd config for long enough that nobody should be including + that config file. + """ + file_path = os.path.join(args[0], "etc", "systemd", "network", + "10-dhcp.network") + + if os.path.isfile(file_path): + try: + os.rename(file_path, file_path + ".morph") + self.status(msg="Renaming networkd file from systemd chunk: \ + %(f)s to %(f)s.morph", f=file_path) + except OSError: + pass + + def generate_default_network_config(self, args): + """Generate default network config: DHCP in all the interfaces""" + + default_network_config_interfaces = "lo:loopback;" \ + "eth0:dhcp,hostname=$(hostname)" + default_network_config_networkd = "e*:dhcp" + + stanzas_interfaces = self.parse_network_stanzas( + default_network_config_interfaces) + stanzas_networkd = self.parse_network_stanzas( + default_network_config_networkd) + + self.generate_interfaces_file(args, stanzas_interfaces) + self.generate_networkd_files(args, stanzas_networkd) + + def generate_interfaces_file(self, args, stanzas): + """Generate /etc/network/interfaces file""" + + iface_file = self.generate_iface_file(stanzas) + + directory_path = os.path.join(args[0], "etc", "network") + self.make_sure_path_exists(directory_path) + file_path = os.path.join(directory_path, "interfaces") + with open(file_path, "w") as f: + f.write(iface_file) + + def generate_iface_file(self, stanzas): + """Generate an interfaces file from the provided stanzas. + + The interfaces will be sorted by name, with loopback sorted first. + """ + + def cmp_iface_names(a, b): + a = a['name'] + b = b['name'] + if a == "lo": + return -1 + elif b == "lo": + return 1 + else: + return cmp(a,b) + + return "\n".join(self.generate_iface_stanza(stanza) + for stanza in sorted(stanzas, cmp=cmp_iface_names)) + + def generate_iface_stanza(self, stanza): + """Generate an interfaces stanza from the provided data.""" + + name = stanza['name'] + itype = stanza['type'] + lines = ["auto %s" % name, "iface %s inet %s" % (name, itype)] + lines += [" %s %s" % elem for elem in stanza['args'].items()] + lines += [""] + return "\n".join(lines) + + def generate_networkd_files(self, args, stanzas): + """Generate .network files""" + + for i, stanza in enumerate(stanzas, 50): + iface_file = self.generate_networkd_file(stanza) + + if iface_file is None: + continue + + directory_path = os.path.join(args[0], "etc", "systemd", "network") + self.make_sure_path_exists(directory_path) + file_path = os.path.join(directory_path, + "%s-%s.network" % (i, stanza['name'])) + + with open(file_path, "w") as f: + f.write(iface_file) + + def generate_networkd_file(self, stanza): + """Generate an .network file from the provided data.""" + + name = stanza['name'] + itype = stanza['type'] + pairs = stanza['args'].items() + + if itype == "loopback": + return + + lines = ["[Match]"] + lines += ["Name=%s\n" % name] + lines += ["[Network]"] + if itype == "dhcp": + lines += ["DHCP=yes"] + else: + lines += self.generate_networkd_entries(pairs) + + return "\n".join(lines) + + def generate_networkd_entries(self, pairs): + """Generate networkd configuration entries with the other parameters""" + + address = None + netmask = None + gateway = None + dns = None + lines = [] + + for pair in pairs: + if pair[0] == 'address': + address = pair[1] + elif pair[0] == 'netmask': + netmask = pair[1] + elif pair[0] == 'gateway': + gateway = pair[1] + elif pair[0] == 'dns': + dns = pair[1] + + if address and netmask: + network_suffix = self.convert_net_mask_to_cidr_suffix (netmask); + address_line = address + '/' + str(network_suffix) + lines += ["Address=%s" % address_line] + elif address or netmask: + raise Exception('address and netmask must be specified together') + + if gateway: + lines += ["Gateway=%s" % gateway] + + if dns: + lines += ["DNS=%s" % dns] + + return lines + + def convert_net_mask_to_cidr_suffix(self, mask): + """Convert dotted decimal form of a subnet mask to CIDR suffix notation + + For example: 255.255.255.0 -> 24 + """ + return sum(bin(int(x)).count('1') for x in mask.split('.')) + + def parse_network_stanzas(self, config): + """Parse a network config environment variable into stanzas. + + Network config stanzas are semi-colon separated. + """ + + return [self.parse_network_stanza(s) for s in config.split(";")] + + def parse_network_stanza(self, stanza): + """Parse a network config stanza into name, type and arguments. + + Each stanza is of the form name:type[,arg=value]... + + For example: + lo:loopback + eth0:dhcp + eth1:static,address=10.0.0.1,netmask=255.255.0.0 + """ + elements = stanza.split(",") + lead = elements.pop(0).split(":") + if len(lead) != 2: + raise SimpleNetworkError("Stanza '%s' is missing its type" % + stanza) + iface = lead[0] + iface_type = lead[1] + + if iface_type not in ['loopback', 'static', 'dhcp']: + raise SimpleNetworkError("Stanza '%s' has unknown interface type" + " '%s'" % (stanza, iface_type)) + + argpairs = [element.split("=", 1) for element in elements] + output_stanza = { "name": iface, + "type": iface_type, + "args": {} } + for argpair in argpairs: + if len(argpair) != 2: + raise SimpleNetworkError("Stanza '%s' has bad argument '%r'" + % (stanza, argpair.pop(0))) + if argpair[0] in output_stanza["args"]: + raise SimpleNetworkError("Stanza '%s' has repeated argument" + " %s" % (stanza, argpair[0])) + output_stanza["args"][argpair[0]] = argpair[1] + + return output_stanza + + def make_sure_path_exists(self, path): + try: + os.makedirs(path) + except OSError as e: + if e.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise SimpleNetworkError("Unable to create directory '%s'" + % path) + + def status(self, **kwargs): + '''Provide status output. + + The ``msg`` keyword argument is the actual message, + the rest are values for fields in the message as interpolated + by %. + + ''' + + self.output.write('%s\n' % (kwargs['msg'] % kwargs)) + +SimpleNetworkConfigurationExtension().run() diff --git a/ssh-rsync.check b/ssh-rsync.check new file mode 100755 index 00000000..c3bdfd29 --- /dev/null +++ b/ssh-rsync.check @@ -0,0 +1,64 @@ +#!/usr/bin/python +# Copyright (C) 2014-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + +'''Preparatory checks for Morph 'ssh-rsync' write extension''' + +import cliapp + +import os + +import morphlib.writeexts + +class SshRsyncCheckExtension(morphlib.writeexts.WriteExtension): + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + upgrade = self.get_environment_boolean('UPGRADE') + if not upgrade: + raise cliapp.AppException( + 'The ssh-rsync write is for upgrading existing remote ' + 'Baserock machines. It cannot be used for an initial ' + 'deployment.') + + if os.environ.get('VERSION_LABEL', '') == '': + raise cliapp.AppException( + 'A VERSION_LABEL must be set when deploying an upgrade.') + + location = args[0] + self.check_ssh_connectivity(location) + self.check_is_baserock_system(location) + + # The new system that being deployed as an upgrade must contain + # baserock-system-config-sync and system-version-manager. However, the + # old system simply needs to have SSH and rsync. + self.check_command_exists(location, 'rsync') + + def check_is_baserock_system(self, location): + output = cliapp.ssh_runcmd(location, ['sh', '-c', + 'test -d /baserock || echo -n dirnotfound']) + if output == 'dirnotfound': + raise cliapp.AppException('%s is not a baserock system' + % location) + + def check_command_exists(self, location, command): + test = 'type %s > /dev/null 2>&1 || echo -n cmdnotfound' % command + output = cliapp.ssh_runcmd(location, ['sh', '-c', test]) + if output == 'cmdnotfound': + raise cliapp.AppException( + "%s does not have %s" % (location, command)) + + +SshRsyncCheckExtension().run() diff --git a/ssh-rsync.write b/ssh-rsync.write new file mode 100755 index 00000000..6d596500 --- /dev/null +++ b/ssh-rsync.write @@ -0,0 +1,172 @@ +#!/usr/bin/python +# Copyright (C) 2013-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +'''A Morph deployment write extension for upgrading systems over ssh.''' + + +import contextlib +import cliapp +import os +import sys +import time +import tempfile + +import morphlib.writeexts + + +def ssh_runcmd_ignore_failure(location, command, **kwargs): + try: + return cliapp.ssh_runcmd(location, command, **kwargs) + except cliapp.AppException: + pass + + +class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): + + '''See ssh-rsync.write.help for documentation''' + + + def find_root_disk(self, location): + '''Read /proc/mounts on location to find which device contains "/"''' + + self.status(msg='Finding device that contains "/"') + contents = cliapp.ssh_runcmd(location, ['cat', '/proc/mounts']) + for line in contents.splitlines(): + line_words = line.split() + if (line_words[1] == '/' and line_words[0] != 'rootfs'): + return line_words[0] + + @contextlib.contextmanager + def _remote_mount_point(self, location): + self.status(msg='Creating remote mount point') + remote_mnt = cliapp.ssh_runcmd(location, ['mktemp', '-d']).strip() + try: + yield remote_mnt + finally: + self.status(msg='Removing remote mount point') + cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) + + @contextlib.contextmanager + def _remote_mount(self, location, root_disk, mountpoint): + self.status(msg='Mounting root disk') + cliapp.ssh_runcmd(location, ['mount', root_disk, mountpoint]) + try: + yield + finally: + self.status(msg='Unmounting root disk') + cliapp.ssh_runcmd(location, ['umount', mountpoint]) + + @contextlib.contextmanager + def _created_version_root(self, location, remote_mnt, version_label): + version_root = os.path.join(remote_mnt, 'systems', version_label) + self.status(msg='Creating %(root)s', root=version_root) + cliapp.ssh_runcmd(location, ['mkdir', version_root]) + try: + yield version_root + except BaseException as e: + # catch all, we always want to clean up + self.status(msg='Cleaning up %(root)s', root=version_root) + ssh_runcmd_ignore_failure(location, ['rmdir', version_root]) + raise + + def get_old_orig(self, location, remote_mnt): + '''Identify which subvolume to snapshot from''' + + # rawdisk upgrades use 'default' + return os.path.join(remote_mnt, 'systems', 'default', 'orig') + + @contextlib.contextmanager + def _created_orig_subvolume(self, location, remote_mnt, version_root): + self.status(msg='Creating "orig" subvolume') + old_orig = self.get_old_orig(location, remote_mnt) + new_orig = os.path.join(version_root, 'orig') + cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', 'snapshot', + old_orig, new_orig]) + try: + yield new_orig + except BaseException as e: + ssh_runcmd_ignore_failure( + location, ['btrfs', 'subvolume', 'delete', new_orig]) + raise + + def populate_remote_orig(self, location, new_orig, temp_root): + '''Populate the subvolume version_root/orig on location''' + + self.status(msg='Populating "orig" subvolume') + cliapp.runcmd(['rsync', '-as', '--checksum', '--numeric-ids', + '--delete', temp_root + os.path.sep, + '%s:%s' % (location, new_orig)]) + + @contextlib.contextmanager + def _deployed_version(self, location, version_label, + system_config_sync, system_version_manager): + self.status(msg='Calling system-version-manager to deploy upgrade') + deployment = os.path.join('/systems', version_label, 'orig') + cliapp.ssh_runcmd(location, + ['env', 'BASEROCK_SYSTEM_CONFIG_SYNC='+system_config_sync, + system_version_manager, 'deploy', deployment]) + try: + yield deployment + except BaseException as e: + self.status(msg='Cleaning up failed version installation') + cliapp.ssh_runcmd(location, + [system_version_manager, 'remove', version_label]) + raise + + def upgrade_remote_system(self, location, temp_root): + root_disk = self.find_root_disk(location) + uuid = cliapp.ssh_runcmd(location, ['blkid', '-s', 'UUID', '-o', + 'value', root_disk]).strip() + + self.complete_fstab_for_btrfs_layout(temp_root, uuid) + + version_label = os.environ['VERSION_LABEL'] + autostart = self.get_environment_boolean('AUTOSTART') + + with self._remote_mount_point(location) as remote_mnt, \ + self._remote_mount(location, root_disk, remote_mnt), \ + self._created_version_root(location, remote_mnt, + version_label) as version_root, \ + self._created_orig_subvolume(location, remote_mnt, + version_root) as orig: + self.populate_remote_orig(location, orig, temp_root) + system_root = os.path.join(remote_mnt, 'systems', + version_label, 'orig') + config_sync = os.path.join(system_root, 'usr', 'bin', + 'baserock-system-config-sync') + version_manager = os.path.join(system_root, 'usr', 'bin', + 'system-version-manager') + with self._deployed_version(location, version_label, + config_sync, version_manager): + self.status(msg='Setting %(v)s as the new default system', + v=version_label) + cliapp.ssh_runcmd(location, [version_manager, + 'set-default', version_label]) + + if autostart: + self.status(msg="Rebooting into new system ...") + ssh_runcmd_ignore_failure(location, ['reboot']) + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + temp_root, location = args + + self.upgrade_remote_system(location, temp_root) + + +SshRsyncWriteExtension().run() diff --git a/ssh-rsync.write.help b/ssh-rsync.write.help new file mode 100644 index 00000000..f3f79ed5 --- /dev/null +++ b/ssh-rsync.write.help @@ -0,0 +1,50 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see <http://www.gnu.org/licenses/>. + +help: | + + Upgrade a Baserock system which is already deployed: + - as a KVM/LibVirt, OpenStack or vbox-ssh virtual machine; + - on a Jetson board. + + Copies a binary delta over to the target system and arranges for it + to be bootable. + + The recommended way to use this extension is by calling `morph upgrade`. + Using `morph deploy --upgrade` is deprecated. + + The upgrade will fail if: + - no VM is deployed and running at `location`; + - the target system is not a Baserock system; + - the target's filesystem and its layout are not compatible with that + created by `morph deploy`." + + See also the 'Upgrading a Baserock installation' section of the 'Using + Baserock` page at wiki.baserock.org + http://wiki.baserock.org/devel-with/#index8h2 + + Parameters: + + * location: the 'user@hostname' string that will be used by ssh and rsync. + 'user' will always be `root` and `hostname` the hostname or address of + the system being upgraded. + + * VERSION_LABEL=label - **(MANDATORY)** should contain only alpha-numeric + characters and the '-' (hyphen) character. + + * AUTOSTART=<VALUE>` - boolean. If it is set, the VM will be started when + it has been deployed. + + (See `morph help deploy` for details of how to pass parameters to write + extensions) diff --git a/sshkeys.configure b/sshkeys.configure new file mode 100755 index 00000000..7a5a8379 --- /dev/null +++ b/sshkeys.configure @@ -0,0 +1,25 @@ +#!/bin/sh +# +# Copyright 2014 Codethink Ltd +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +set -e + +if [ "$SSHKEYS" ] +then + install -d -m 700 "$1/root/.ssh" + echo Adding Key in "$SSHKEYS" to authorized_keys file + cat $SSHKEYS >> "$1/root/.ssh/authorized_keys" +fi diff --git a/sysroot.check b/sysroot.check new file mode 100755 index 00000000..71b35175 --- /dev/null +++ b/sysroot.check @@ -0,0 +1,23 @@ +#!/bin/sh +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + +# Preparatory checks for Morph 'sysroot' write extension + +set -eu + +if [ "$UPGRADE" == "yes" ]; then + echo >&2 "ERROR: Cannot upgrade a sysroot deployment" + exit 1 +fi diff --git a/sysroot.write b/sysroot.write new file mode 100755 index 00000000..46f1a780 --- /dev/null +++ b/sysroot.write @@ -0,0 +1,22 @@ +#!/bin/sh +# Copyright (C) 2014,2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + +# A Morph write extension to deploy to another directory + +set -eu + +mkdir -p "$2" + +cp -a "$1"/* "$2" diff --git a/tar.check b/tar.check new file mode 100755 index 00000000..f2304d46 --- /dev/null +++ b/tar.check @@ -0,0 +1,23 @@ +#!/bin/sh +# Copyright (C) 2014-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + +# Preparatory checks for Morph 'tar' write extension + +set -eu + +if [ "$UPGRADE" == "yes" ]; then + echo >&2 "ERROR: Cannot upgrade a tar file deployment." + exit 1 +fi diff --git a/tar.write b/tar.write new file mode 100755 index 00000000..01b545b4 --- /dev/null +++ b/tar.write @@ -0,0 +1,20 @@ +#!/bin/sh +# Copyright (C) 2013,2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + +# A Morph write extension to deploy to a .tar file + +set -eu + +tar -C "$1" -cf "$2" . diff --git a/tar.write.help b/tar.write.help new file mode 100644 index 00000000..b45c61fa --- /dev/null +++ b/tar.write.help @@ -0,0 +1,19 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see <http://www.gnu.org/licenses/>. + +help: | + Create a .tar file of the deployed system. + + The `location` argument is a pathname to the .tar file to be + created. diff --git a/vdaboot.configure b/vdaboot.configure new file mode 100755 index 00000000..60de925b --- /dev/null +++ b/vdaboot.configure @@ -0,0 +1,33 @@ +#!/bin/sh +# Copyright (C) 2013,2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +# Change the "/" mount point to /dev/vda to use virtio disks. + +set -e + +if [ "$OPENSTACK_USER" ] +then + # Modifying fstab + if [ -f "$1/etc/fstab" ] + then + mv "$1/etc/fstab" "$1/etc/fstab.old" + awk 'BEGIN {print "/dev/vda / btrfs defaults,rw,noatime 0 1"}; + $2 != "/" {print $0 };' "$1/etc/fstab.old" > "$1/etc/fstab" + rm "$1/etc/fstab.old" + else + echo "/dev/vda / btrfs defaults,rw,noatime 0 1"> "$1/etc/fstab" + fi +fi diff --git a/virtualbox-ssh.check b/virtualbox-ssh.check new file mode 100755 index 00000000..a97f3294 --- /dev/null +++ b/virtualbox-ssh.check @@ -0,0 +1,36 @@ +#!/usr/bin/python +# Copyright (C) 2014-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + +'''Preparatory checks for Morph 'virtualbox-ssh' write extension''' + +import cliapp + +import morphlib.writeexts + + +class VirtualBoxPlusSshCheckExtension(morphlib.writeexts.WriteExtension): + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + self.require_btrfs_in_deployment_host_kernel() + + upgrade = self.get_environment_boolean('UPGRADE') + if upgrade: + raise cliapp.AppException( + 'Use the `ssh-rsync` write extension to deploy upgrades to an ' + 'existing remote system.') + +VirtualBoxPlusSshCheckExtension().run() diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write new file mode 100755 index 00000000..774f2b4f --- /dev/null +++ b/virtualbox-ssh.write @@ -0,0 +1,211 @@ +#!/usr/bin/python +# Copyright (C) 2012-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. + + +'''A Morph deployment write extension for deploying to VirtualBox via ssh. + +VirtualBox is assumed to be running on a remote machine, which is +accessed over ssh. The machine gets created, but not started. + +See file virtualbox-ssh.write.help for documentation + +''' + + +import cliapp +import os +import re +import sys +import time +import tempfile +import urlparse + +import morphlib.writeexts + + +class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + temp_root, location = args + ssh_host, vm_name, vdi_path = self.parse_location(location) + autostart = self.get_environment_boolean('AUTOSTART') + + vagrant = self.get_environment_boolean('VAGRANT') + + fd, raw_disk = tempfile.mkstemp() + os.close(fd) + self.create_local_system(temp_root, raw_disk) + + try: + self.transfer_and_convert_to_vdi( + raw_disk, ssh_host, vdi_path) + self.create_virtualbox_guest(ssh_host, vm_name, vdi_path, + autostart, vagrant) + except BaseException: + sys.stderr.write('Error deploying to VirtualBox') + os.remove(raw_disk) + cliapp.ssh_runcmd(ssh_host, ['rm', '-f', vdi_path]) + raise + else: + os.remove(raw_disk) + self.status( + msg='Virtual machine %(vm_name)s has been created', + vm_name=vm_name) + + def parse_location(self, location): + '''Parse the location argument to get relevant data.''' + + x = urlparse.urlparse(location) + if x.scheme != 'vbox+ssh': + raise cliapp.AppException( + 'URL schema must be vbox+ssh in %s' % location) + m = re.match('^/(?P<guest>[^/]+)(?P<path>/.+)$', x.path) + if not m: + raise cliapp.AppException('Cannot parse location %s' % location) + return x.netloc, m.group('guest'), m.group('path') + + def transfer_and_convert_to_vdi(self, raw_disk, ssh_host, vdi_path): + '''Transfer raw disk image to VirtualBox host, and convert to VDI.''' + + self.status(msg='Transfer disk and convert to VDI') + + st = os.lstat(raw_disk) + xfer_hole_path = morphlib.util.get_data_path('xfer-hole') + recv_hole = morphlib.util.get_data('recv-hole') + + ssh_remote_cmd = [ + 'sh', '-c', recv_hole, + 'dummy-argv0', 'vbox', vdi_path, str(st.st_size), + ] + + cliapp.runcmd( + ['python', xfer_hole_path, raw_disk], + ['ssh', ssh_host] + map(cliapp.shell_quote, ssh_remote_cmd), + stdout=None, stderr=None) + + def virtualbox_version(self, ssh_host): + 'Get the version number of the VirtualBox running on the remote host.' + + # --version gives a build id, which looks something like + # 1.2.3r456789, so we need to strip the suffix off and get a tuple + # of the (major, minor, patch) version, since comparing with a + # tuple is more reliable than a string and more convenient than + # comparing against the major, minor and patch numbers directly + self.status(msg='Checking version of remote VirtualBox') + build_id = cliapp.ssh_runcmd(ssh_host, ['VBoxManage', '--version']) + version_string = re.match(r"^([0-9\.]+).*$", build_id.strip()).group(1) + return tuple(int(s or '0') for s in version_string.split('.')) + + def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path, autostart, + vagrant): + '''Create the VirtualBox virtual machine.''' + + self.status(msg='Create VirtualBox virtual machine') + + ram_mebibytes = str(self.get_ram_size() / (1024**2)) + + vcpu_count = str(self.get_vcpu_count()) + + if not vagrant: + hostonly_iface = self.get_host_interface(ssh_host) + + if self.virtualbox_version(ssh_host) < (4, 3, 0): + sataportcount_option = '--sataportcount' + else: + sataportcount_option = '--portcount' + + commands = [ + ['createvm', '--name', vm_name, '--ostype', 'Linux26_64', + '--register'], + ['modifyvm', vm_name, '--ioapic', 'on', + '--memory', ram_mebibytes, '--cpus', vcpu_count], + ['storagectl', vm_name, '--name', 'SATA Controller', + '--add', 'sata', '--bootable', 'on', sataportcount_option, '2'], + ['storageattach', vm_name, '--storagectl', 'SATA Controller', + '--port', '0', '--device', '0', '--type', 'hdd', '--medium', + vdi_path], + ] + if vagrant: + commands[1].extend(['--nic1', 'nat', + '--natnet1', 'default']) + else: + commands[1].extend(['--nic1', 'hostonly', + '--hostonlyadapter1', hostonly_iface, + '--nic2', 'nat', '--natnet2', 'default']) + + attach_disks = self.parse_attach_disks() + for device_no, disk in enumerate(attach_disks, 1): + cmd = ['storageattach', vm_name, + '--storagectl', 'SATA Controller', + '--port', str(device_no), + '--device', '0', + '--type', 'hdd', + '--medium', disk] + commands.append(cmd) + + if autostart: + commands.append(['startvm', vm_name]) + + for command in commands: + argv = ['VBoxManage'] + command + cliapp.ssh_runcmd(ssh_host, argv) + + def get_host_interface(self, ssh_host): + host_ipaddr = os.environ.get('HOST_IPADDR') + netmask = os.environ.get('NETMASK') + + if host_ipaddr is None: + raise cliapp.AppException('HOST_IPADDR was not given') + + if netmask is None: + raise cliapp.AppException('NETMASK was not given') + + # 'VBoxManage list hostonlyifs' retrieves a list with the hostonly + # interfaces on the host. For each interface, the following lines + # are shown on top: + # + # Name: vboxnet0 + # GUID: 786f6276-656e-4074-8000-0a0027000000 + # Dhcp: Disabled + # IPAddress: 192.168.100.1 + # + # The following command tries to retrieve the hostonly interface + # name (e.g. vboxnet0) associated with the given ip address. + iface = None + lines = cliapp.ssh_runcmd(ssh_host, + ['VBoxManage', 'list', 'hostonlyifs']).splitlines() + for i, v in enumerate(lines): + if host_ipaddr in v: + iface = lines[i-3].split()[1] + break + + if iface is None: + iface = cliapp.ssh_runcmd(ssh_host, + ['VBoxManage', 'hostonlyif', 'create']) + # 'VBoxManage hostonlyif create' shows the name of the + # created hostonly interface inside single quotes + iface = iface[iface.find("'") + 1 : iface.rfind("'")] + cliapp.ssh_runcmd(ssh_host, + ['VBoxManage', 'hostonlyif', + 'ipconfig', iface, + '--ip', host_ipaddr, + '--netmask', netmask]) + + return iface + +VirtualBoxPlusSshWriteExtension().run() diff --git a/virtualbox-ssh.write.help b/virtualbox-ssh.write.help new file mode 100644 index 00000000..2dbf988c --- /dev/null +++ b/virtualbox-ssh.write.help @@ -0,0 +1,135 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see <http://www.gnu.org/licenses/>. + +help: | + + Deploy a Baserock system as a *new* VirtualBox virtual machine. + (Use the `ssh-rsync` write extension to deploy upgrades to an *existing* + VM) + + Connects to HOST via ssh to run VirtualBox's command line management tools. + + Parameters: + + * location: a custom URL scheme of the form `vbox+ssh://HOST/GUEST/PATH`, + where: + * HOST is the name of the host on which VirtualBox is running + * GUEST is the name of the guest VM on that host + * PATH is the path to the disk image that should be created, + on that host. For example, + `vbox+ssh://alice@192.168.122.1/testsys/home/alice/testys.img` where + * `alice@192.168.122.1` is the target host as given to ssh, + **from within the development host** (which may be + different from the target host's normal address); + * `testsys` is the name of the new guest VM'; + * `/home/alice/testys.img` is the pathname of the disk image files + on the target host. + + * HOSTNAME=name: the hostname of the **guest** VM within the network into + which it is being deployed. + + * DISK_SIZE=X: **(MANDATORY)** the size of the VM's primary virtual hard + disk. `X` should use a suffix of `K`, `M`, or `G` (in upper or lower + case) to indicate kilo-, mega-, or gigabytes. For example, + `DISK_SIZE=100G` would create a 100 gigabyte virtual hard disk. + + * RAM_SIZE=X: The amount of RAM that the virtual machine should allocate + for itself from the host. `X` is interpreted in the same as for + DISK_SIZE, and defaults to `1G`. + + * VCPUS=n: the number of virtual CPUs for the VM. Allowed values 1-32. Do + not use more CPU cores than you have available physically (real cores, + no hyperthreads). + + * INITRAMFS_PATH=path: the location of an initramfs for the bootloader to + tell Linux to use, rather than booting the rootfs directly. + + * DTB_PATH=path: **(MANDATORY)** for systems that require a device tree + binary - Give the full path (without a leading /) to the location of the + DTB in the built system image . The deployment will fail if `path` does + not exist. + + * BOOTLOADER_INSTALL=value: the bootloader to be installed + **(MANDATORY)** for non-x86 systems + + allowed values = + - 'extlinux' (default) - the extlinux bootloader will + be installed + - 'none' - no bootloader will be installed by `morph deploy`. A + bootloader must be installed manually. This value must be used when + deploying non-x86 systems such as ARM. + + * BOOTLOADER_CONFIG_FORMAT=value: the bootloader format to be used. + If not specified for x86-32 and x86-64 systems, 'extlinux' will be used + + allowed values = + - 'extlinux' + + * KERNEL_ARGS=args: optional additional kernel command-line parameters to + be appended to the default set. The default set is: + + 'rw init=/sbin/init rootfstype=btrfs \ + rootflags=subvol=systems/default/run \ + root=[name or UUID of root filesystem]' + + (See https://www.kernel.org/doc/Documentation/kernel-parameters.txt) + + * AUTOSTART=<VALUE> - boolean. If it is set, the VM will be started when + it has been deployed. + + * VAGRANT=<VALUE> - boolean. If it is set, then networking is configured + so that the VM will work with Vagrant. Otherwise networking is + configured to run directly in VirtualBox. + + * HOST_IPADDR=<ip_address> - the IP address of the VM host. + + * NETMASK=<netmask> - the netmask of the VM host. + + * NETWORK_CONFIG=<net_config> - `net_config` is used to set up the VM's + network interfaces. It is a string containing semi-colon separated + 'stanzas' where each stanza provides information about a network + interface. Each stanza is of the form name:type[,arg=value] e.g. + + lo:loopback + eth0:dhcp + eth1:static,address=10.0.0.1,netmask=255.255.0.0 + + An example of the NETWORK_CONFIG parameter (It should be in one line) + + `"lo:loopback;eth0:static,address=192.168.100.2,netmask=255.255.255.0; + eth1:dhcp,hostname=$(hostname)"` + + It is useful to configure one interface to use NAT to give the VM access + to the outside world and another interface to use the Virtual Box host + adapter to allow you to access the Trove from the host machine. + + The NAT interface eth1 is set up to use dhcp, the host-only adapter + interface is configured statically. + + Note: you must give the host-only adapter interface an address that lies + **on the same network** as the host adapter. So if the host adapter has + an IP of 192.168.100.1 eth0 should have an address such as + 192.168.100.42. + + The settings of the host adapter, including its IP can be changed either + in the VirtualBox manager UI + (https://www.virtualbox.org/manual/ch03.html#settings-network) + or via the VBoxManage command line + (https://www.virtualbox.org/manual/ch08.html#idp57572192) + + See Chapter 6 of the VirtualBox User Manual for more information about + virtual networking (https://www.virtualbox.org/manual/ch06.html) + + (See `morph help deploy` for details of how to pass parameters to write + extensions) |