diff options
author | Richard Maw <richard.maw@codethink.co.uk> | 2013-02-12 11:51:52 +0000 |
---|---|---|
committer | Richard Maw <richard.maw@codethink.co.uk> | 2013-02-12 11:51:52 +0000 |
commit | c00ee1c6852d5d02cd9d9fdf071b9a3d838ad6ef (patch) | |
tree | 38b4ff8a91b977bb04c700a0e986c21c436266f3 | |
parent | d83af40a3cdff4905af0e41c60a96744078a4b52 (diff) | |
parent | a9ec7e0bdf6b6dc9b15addbe9980f3b03fe342ea (diff) | |
download | morph-c00ee1c6852d5d02cd9d9fdf071b9a3d838ad6ef.tar.gz |
Merge branch 'liw/deployment-refactor' of git://git.baserock.org/baserock/baserock/morph
-rwxr-xr-x | check | 7 | ||||
-rw-r--r-- | morphlib/__init__.py | 2 | ||||
-rwxr-xr-x | morphlib/exts/kvm.write | 110 | ||||
-rwxr-xr-x | morphlib/exts/rawdisk.write | 49 | ||||
-rwxr-xr-x | morphlib/exts/set-hostname.configure | 27 | ||||
-rwxr-xr-x | morphlib/exts/virtualbox-ssh.write | 129 | ||||
-rw-r--r-- | morphlib/morph2.py | 3 | ||||
-rw-r--r-- | morphlib/plugins/__init__.py | 0 | ||||
-rw-r--r-- | morphlib/plugins/branch_and_merge_plugin.py | 1 | ||||
-rw-r--r-- | morphlib/plugins/deploy_plugin.py | 209 | ||||
-rwxr-xr-x | morphlib/writeexts.py | 185 | ||||
-rw-r--r-- | setup.py | 1 | ||||
-rwxr-xr-x | tests.deploy/deploy-rawdisk.script | 31 | ||||
-rwxr-xr-x | tests.deploy/setup | 204 | ||||
-rw-r--r-- | tests.deploy/setup-build | 35 | ||||
-rw-r--r-- | without-test-modules | 4 |
16 files changed, 995 insertions, 2 deletions
@@ -19,10 +19,17 @@ set -e +case "$PYTHONPATH" in + '') PYTHONPATH="$(pwd)" ;; + *) PYTHONPATH="$(pwd):$PYTHONPATH" ;; +esac +export PYTHONPATH + python setup.py clean check cmdtest tests cmdtest tests.branching cmdtest tests.merging +cmdtest tests.deploy if [ $(whoami) = root ] && command -v mkfs.btrfs > /dev/null && python -c " import morphlib, sys diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 0f60642c..4446720e 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -73,4 +73,6 @@ import util import yamlparse +import writeexts + import app # this needs to be last diff --git a/morphlib/exts/kvm.write b/morphlib/exts/kvm.write new file mode 100755 index 00000000..09a7d224 --- /dev/null +++ b/morphlib/exts/kvm.write @@ -0,0 +1,110 @@ +#!/usr/bin/python +# Copyright (C) 2012-2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +'''A Morph deployment write extension for deploying to KVM+libvirt.''' + + +import cliapp +import os +import re +import sys +import tempfile +import urlparse + +import morphlib.writeexts + + +class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): + + '''Create a KVM/LibVirt virtual machine during Morph's deployment. + + The location command line argument is the pathname of the disk image + to be created. The user is expected to provide the location argument + using the following syntax: + + kvm+ssh://HOST/GUEST/PATH + + where: + + * HOST is the host on which KVM/LibVirt is running + * GUEST is the name of the guest virtual machine on that host + * PATH is the path to the disk image that should be created, + on that host + + The extension will connect to HOST via ssh to run libvirt's + command line management tools. + + ''' + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + temp_root, location = args + ssh_host, vm_name, vm_path = self.parse_location(location) + + fd, raw_disk = tempfile.mkstemp() + os.close(fd) + self.create_local_system(temp_root, raw_disk) + + try: + self.transfer(raw_disk, ssh_host, vm_path) + self.create_libvirt_guest(ssh_host, vm_name, vm_path) + except BaseException: + sys.stderr.write('Error deploying to libvirt') + os.remove(raw_disk) + raise + else: + os.remove(raw_disk) + + self.status( + msg='Virtual machine %(vm_name)s has been created', + vm_name=vm_name) + + def parse_location(self, location): + '''Parse the location argument to get relevant data.''' + + x = urlparse.urlparse(location) + if x.scheme != 'kvm+ssh': + raise cliapp.AppException( + 'URL schema must be vbox+ssh in %s' % location) + m = re.match('^/(?P<guest>[^/]+)(?P<path>/.+)$', x.path) + if not m: + raise cliapp.AppException('Cannot parse location %s' % location) + return x.netloc, m.group('guest'), m.group('path') + + def transfer(self, raw_disk, ssh_host, vm_path): + '''Transfer raw disk image to libvirt host.''' + + self.status(msg='Transfer disk image') + target = '%s:%s' % (ssh_host, vm_path) + with open(raw_disk, 'rb') as f: + cliapp.runcmd(['rsync', '-zS', raw_disk, target]) + + def create_libvirt_guest(self, ssh_host, vm_name, vm_path): + '''Create the libvirt virtual machine.''' + + self.status(msg='Create libvirt/kvm virtual machine') + cliapp.runcmd( + ['ssh', ssh_host, + 'virt-install', '--connect qemu:///system', '--import', + '--name', vm_name, '--ram', '1024', '--vnc', '--noreboot', + '--disk path=%s,bus=ide' % vm_path]) + + +KvmPlusSshWriteExtension().run() + diff --git a/morphlib/exts/rawdisk.write b/morphlib/exts/rawdisk.write new file mode 100755 index 00000000..a55473f2 --- /dev/null +++ b/morphlib/exts/rawdisk.write @@ -0,0 +1,49 @@ +#!/usr/bin/python +# Copyright (C) 2012-2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +'''A Morph deployment write extension for raw disk images.''' + + +import os +import sys +import time +import tempfile + +import morphlib.writeexts + + +class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): + + '''Create a raw disk image during Morph's deployment. + + The location command line argument is the pathname of the disk image + to be created. + + ''' + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + temp_root, location = args + + self.create_local_system(temp_root, location) + self.status(msg='Disk image has been created at %s' % location) + + +RawDiskWriteExtension().run() + diff --git a/morphlib/exts/set-hostname.configure b/morphlib/exts/set-hostname.configure new file mode 100755 index 00000000..e44c5d56 --- /dev/null +++ b/morphlib/exts/set-hostname.configure @@ -0,0 +1,27 @@ +#!/bin/sh +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +# Set hostname on system from HOSTNAME. + + +set -e + +if [ -n "$HOSTNAME" ] +then + echo "$HOSTNAME" > "$1/etc/hostname" +fi + diff --git a/morphlib/exts/virtualbox-ssh.write b/morphlib/exts/virtualbox-ssh.write new file mode 100755 index 00000000..c21dcc57 --- /dev/null +++ b/morphlib/exts/virtualbox-ssh.write @@ -0,0 +1,129 @@ +#!/usr/bin/python +# Copyright (C) 2012-2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +'''A Morph deployment write extension for deploying to VirtualBox via ssh. + +VirtualBox is assumed to be running on a remote machine, which is +accessed over ssh. The machine gets created, but not started. + +''' + + +import cliapp +import os +import re +import time +import tempfile +import urlparse + +import morphlib.writeexts + + +class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): + + '''Create a VirtualBox virtual machine during Morph's deployment. + + The location command line argument is the pathname of the disk image + to be created. The user is expected to provide the location argument + using the following syntax: + + vbox+ssh://HOST/GUEST/PATH + + where: + + * HOST is the host on which VirtualBox is running + * GUEST is the name of the guest virtual machine on that host + * PATH is the path to the disk image that should be created, + on that host + + The extension will connect to HOST via ssh to run VirtualBox's + command line management tools. + + ''' + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + temp_root, location = args + ssh_host, vm_name, vdi_path = self.parse_location(location) + + fd, raw_disk = tempfile.mkstemp() + os.close(fd) + self.create_local_system(temp_root, raw_disk) + + try: + self.transfer_and_convert_to_vdi( + raw_disk, size, ssh_host, vdi_path) + self.create_virtualbox_guest(ssh_host, vm_name, vdi_path) + except BaseException: + sys.stderr.write('Error deploying to VirtualBox') + os.remove(raw_disk) + raise + else: + os.remove(raw_disk) + + self.status( + msg='Virtual machine %(vm_name)s has been created', + vm_name=vm_name) + + def parse_location(self, location): + '''Parse the location argument to get relevant data.''' + + x = urlparse.urlparse(location) + if x.scheme != 'vbox+ssh': + raise cliapp.AppException( + 'URL schema must be vbox+ssh in %s' % location) + m = re.match('^/(?P<guest>[^/]+)(?P<path>/.+)$', x.path) + if not m: + raise cliapp.AppException('Cannot parse location %s' % location) + return x.netloc, m.group('guest'), m.group('path') + + def transfer_and_convert_to_vdi(self, raw_disk, size, ssh_host, vdi_path): + '''Transfer raw disk image to VirtualBox host, and convert to VDI.''' + + self.status(msg='Transfer disk and convert to VDI') + with open(raw_disk, 'rb') as f: + cliapp.runcmd( + ['ssh', ssh_host, + 'VBoxManage', 'convertfromraw', 'stdin', vdi_path, str(size)], + stdin=f) + + def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path): + '''Create the VirtualBox virtual machine.''' + + self.status(msg='Create VirtualBox virtual machine') + + commands = [ + ['createvm', '--name', vm_name, '--ostype', 'Linux26_64', + '--register'], + ['modifyvm', vm_name, '--ioapic', 'on', '--memory', '1024', + '--nic1', 'nat'], + ['storagectl', vm_name, '--name', '"SATA Controller"', + '--add', 'sata', '--bootable', 'on', '--sataportcount', '2'], + ['storageattach', vm_name, '--storagectl', '"SATA Controller"', + '--port', '0', '--device', '0', '--type', 'hdd', '--medium', + vdi_path], + ] + + for command in commands: + argv = ['ssh', ssh_host, 'VBoxManage'] + command + cliapp.runcmd(argv) + + +VirtualBoxPlusSshWriteExtension().run() + diff --git a/morphlib/morph2.py b/morphlib/morph2.py index 3a3ad679..6e24765e 100644 --- a/morphlib/morph2.py +++ b/morphlib/morph2.py @@ -49,7 +49,8 @@ class Morphology(object): ('strata', []), ('description', ''), ('arch', None), - ('system-kind', None) + ('system-kind', None), + ('configuration-extensions', []), ] } diff --git a/morphlib/plugins/__init__.py b/morphlib/plugins/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/morphlib/plugins/__init__.py diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index 58aa8931..0f07c431 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -331,6 +331,7 @@ class BranchAndMergePlugin(cliapp.Plugin): 'description', 'disk-size', '_disk-size', + 'configuration-extensions', ], 'stratum': [ 'kind', diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py new file mode 100644 index 00000000..79715e13 --- /dev/null +++ b/morphlib/plugins/deploy_plugin.py @@ -0,0 +1,209 @@ +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import gzip +import os +import shutil +import tarfile +import tempfile +import urlparse +import uuid + +import morphlib + +# UGLY HACK: We need to re-use some code from the branch and merge +# plugin, so we import and instantiate that plugin. This needs to +# be fixed by refactoring the codebase so the shared code is in +# morphlib, not in a plugin. However, this hack lets us re-use +# code without copying it. +import morphlib.plugins.branch_and_merge_plugin + + +class DeployPlugin(cliapp.Plugin): + + def enable(self): + self.app.add_subcommand( + 'deploy', self.deploy, + arg_synopsis='TYPE SYSTEM LOCATION [KEY=VALUE]') + self.other = \ + morphlib.plugins.branch_and_merge_plugin.BranchAndMergePlugin() + self.other.app = self.app + + def disable(self): + pass + + def deploy(self, args): + '''Deploy a built system image.''' + + if len(args) < 3: + raise cliapp.AppException( + 'Too few arguments to deploy command (see help)') + + deployment_type = args[0] + system_name = args[1] + location = args[2] + env_vars = args[3:] + + # Deduce workspace and system branch and branch root repository. + workspace = self.other.deduce_workspace() + branch, branch_dir = self.other.deduce_system_branch() + branch_root = self.other.get_branch_config(branch_dir, 'branch.root') + branch_uuid = self.other.get_branch_config(branch_dir, 'branch.uuid') + + # Generate a UUID for the build. + build_uuid = uuid.uuid4().hex + + build_command = morphlib.buildcommand.BuildCommand(self.app) + build_command = self.app.hookmgr.call('new-build-command', + build_command) + push = self.app.settings['push-build-branches'] + + self.app.status(msg='Starting build %(uuid)s', uuid=build_uuid) + + self.app.status(msg='Collecting morphologies involved in ' + 'building %(system)s from %(branch)s', + system=system_name, branch=branch) + + # Get repositories of morphologies involved in building this system + # from the current system branch. + build_repos = self.other.get_system_build_repos( + branch, branch_dir, branch_root, system_name) + + # Generate temporary build ref names for all these repositories. + self.other.generate_build_ref_names(build_repos, branch_uuid) + + # Create the build refs for all these repositories and commit + # all uncommitted changes to them, updating all references + # to system branch refs to point to the build refs instead. + self.other.update_build_refs(build_repos, branch, build_uuid, push) + + if push: + self.other.push_build_refs(build_repos) + build_branch_root = branch_root + else: + dirname = build_repos[branch_root]['dirname'] + build_branch_root = urlparse.urljoin('file://', dirname) + + # Run the build. + build_ref = build_repos[branch_root]['build-ref'] + order = build_command.compute_build_order( + build_branch_root, + build_ref, + system_name + '.morph') + artifact = order.groups[-1][-1] + + if push: + self.other.delete_remote_build_refs(build_repos) + + # Unpack the artifact (tarball) to a temporary directory. + self.app.status(msg='Unpacking system for configuration') + + system_tree = tempfile.mkdtemp() + + if build_command.lac.has(artifact): + f = build_command.lac.get(artifact) + else: + f = build_command.rac.get(artifact) + ff = gzip.GzipFile(fileobj=f) + tf = tarfile.TarFile(fileobj=ff) + tf.extractall(path=system_tree) + + self.app.status( + msg='System unpacked at %(system_tree)s', + system_tree=system_tree) + + # Set up environment for running extensions. + env = dict(os.environ) + for spec in env_vars: + name, value = spec.split('=', 1) + if name in env: + raise morphlib.Error( + '%s is already set in the enviroment' % name) + env[name] = value + + # Run configuration extensions. + self.app.status(msg='Configure system') + names = artifact.source.morphology['configuration-extensions'] + for name in names: + self._run_extension( + branch_dir, + build_ref, + name, + '.configure', + [system_tree], + env) + + # Run write extension. + self.app.status(msg='Writing to device') + self._run_extension( + branch_dir, + build_ref, + deployment_type, + '.write', + [system_tree, location], + env) + + # Cleanup. + self.app.status(msg='Cleaning up') + shutil.rmtree(system_tree) + + self.app.status(msg='Finished deployment') + + def _run_extension(self, repo_dir, ref, name, kind, args, env): + '''Run an extension. + + The ``kind`` should be either ``.configure`` or ``.write``, + depending on the kind of extension that is sought. + + The extension is found either in the git repository of the + system morphology (repo, ref), or with the Morph code. + + ''' + + # Look for extension in the system morphology's repository. + ext = self._cat_file(repo_dir, ref, name + kind) + if ext is None: + # Not found: look for it in the Morph code. + code_dir = os.path.dirname(morphlib.__file__) + ext_filename = os.path.join(code_dir, 'exts', name + kind) + if not os.path.exists(ext_filename): + raise morphlib.Error( + 'Could not find extension %s%s' % (name, kind)) + delete_ext = False + else: + # Found it in the system morphology's repository. + fd, ext_filename = tempfile.mkstemp() + os.write(fd, ext) + os.close(fd) + os.chmod(ext_filename, 0700) + delete_ext = True + + self.app.runcmd( + [ext_filename] + args, env=env, stdout=None, stderr=None) + + if delete_ext: + os.remove(ext_filename) + + def _cat_file(self, repo_dir, ref, pathname): + '''Retrieve contents of a file from a git repository.''' + + argv = ['git', 'cat-file', 'blob', '%s:%s' % (ref, pathname)] + try: + return self.app.runcmd(argv, cwd=repo_dir) + except cliapp.AppException: + return None + diff --git a/morphlib/writeexts.py b/morphlib/writeexts.py new file mode 100755 index 00000000..60848345 --- /dev/null +++ b/morphlib/writeexts.py @@ -0,0 +1,185 @@ +# Copyright (C) 2012-2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import os +import re +import time +import tempfile + + +class WriteExtension(cliapp.Application): + + '''A base class for deployment write extensions. + + A subclass should subclass this class, and add a + ``process_args`` method. + + Note that it is not necessary to subclass this class for write + extensions. This class is here just to collect common code for + write extensions. + + ''' + + def process_args(self, args): + raise NotImplementedError() + + def status(self, **kwargs): + '''Provide status output. + + The ``msg`` keyword argument is the actual message, + the rest are values for fields in the message as interpolated + by %. + + ''' + + self.output.write('%s\n' % (kwargs['msg'] % kwargs)) + + def create_local_system(self, temp_root, raw_disk): + '''Create a raw system image locally.''' + + size = self.get_disk_size() + self.create_raw_disk_image(raw_disk, size) + try: + self.mkfs_btrfs(raw_disk) + mp = self.mount(raw_disk) + except BaseException: + sys.stderr.write('Error creating disk image') + os.remove(raw_disk) + raise + try: + self.create_factory(mp, temp_root) + self.create_fstab(mp) + self.create_factory_run(mp) + self.install_extlinux(mp) + except BaseException, e: + sys.stderr.write('Error creating disk image') + self.unmount(mp) + os.remove(raw_disk) + raise + else: + self.unmount(mp) + + def get_disk_size(self): + '''Parse disk size from environment.''' + + size = os.environ.get('DISK_SIZE', '1G') + m = re.match('^(\d+)([kmgKMG]?)$', size) + if not m: + raise morphlib.Error('Cannot parse disk size %s' % size) + + factors = { + '': 1, + 'k': 1024, + 'm': 1024**2, + 'g': 1024**3, + } + factor = factors[m.group(2).lower()] + + return int(m.group(1)) * factor + + def create_raw_disk_image(self, filename, size): + '''Create a raw disk image.''' + + self.status(msg='Creating empty disk image') + with open(filename, 'wb') as f: + if size > 0: + f.seek(size-1) + f.write('\0') + + def mkfs_btrfs(self, location): + '''Create a btrfs filesystem on the disk.''' + self.status(msg='Creating btrfs filesystem') + cliapp.runcmd(['mkfs.btrfs', '-L', 'baserock', location]) + + def mount(self, location): + '''Mount the filesystem so it can be tweaked. + + Return path to the mount point. + The mount point is a newly created temporary directory. + The caller must call self.unmount to unmount on the return value. + + ''' + + self.status(msg='Mounting filesystem') + tempdir = tempfile.mkdtemp() + cliapp.runcmd(['mount', '-o', 'loop', location, tempdir]) + return tempdir + + def unmount(self, mount_point): + '''Unmount the filesystem mounted by self.mount. + + Also, remove the temporary directory. + + ''' + + self.status(msg='Unmounting filesystem') + cliapp.runcmd(['umount', mount_point]) + os.rmdir(mount_point) + + def create_factory(self, real_root, temp_root): + '''Create the default "factory" system.''' + + factory = os.path.join(real_root, 'factory') + + self.status(msg='Creating factory subvolume') + cliapp.runcmd(['btrfs', 'subvolume', 'create', factory]) + self.status(msg='Copying files to factory subvolume') + cliapp.runcmd(['cp', '-a', temp_root + '/.', factory + '/.']) + + # The kernel needs to be on the root volume. + self.status(msg='Copying boot directory to root subvolume') + factory_boot = os.path.join(factory, 'boot') + root_boot = os.path.join(real_root, 'boot') + cliapp.runcmd(['cp', '-a', factory_boot, root_boot]) + + def create_factory_run(self, real_root): + '''Create the 'factory-run' snapshot.''' + + self.status(msg='Creating factory-run subvolume') + factory = os.path.join(real_root, 'factory') + factory_run = factory + '-run' + cliapp.runcmd( + ['btrfs', 'subvolume', 'snapshot', factory, factory_run]) + + def create_fstab(self, real_root): + '''Create an fstab.''' + + self.status(msg='Creating fstab') + fstab = os.path.join(real_root, 'factory', 'etc', 'fstab') + with open(fstab, 'w') as f: + f.write('/dev/sda / btrfs defaults,rw,noatime 0 1\n') + + def install_extlinux(self, real_root): + '''Install extlinux on the newly created disk image.''' + + self.status(msg='Creating extlinux.conf') + config = os.path.join(real_root, 'extlinux.conf') + with open(config, 'w') as f: + f.write('default linux\n') + f.write('timeout 1\n') + f.write('label linux\n') + f.write('kernel /boot/vmlinuz\n') + f.write('append root=/dev/sda rootflags=subvol=factory-run ' + 'init=/sbin/init rw\n') + + self.status(msg='Installing extlinux') + cliapp.runcmd(['extlinux', '--install', real_root]) + + # FIXME this hack seems to be necessary to let extlinux finish + cliapp.runcmd(['sync']) + time.sleep(2) + @@ -143,6 +143,7 @@ FIXME package_data={ 'morphlib': [ 'plugins/*_plugin.py', + 'exts/*', 'version', 'commit', 'tree', diff --git a/tests.deploy/deploy-rawdisk.script b/tests.deploy/deploy-rawdisk.script new file mode 100755 index 00000000..4c01ca0d --- /dev/null +++ b/tests.deploy/deploy-rawdisk.script @@ -0,0 +1,31 @@ +#!/bin/sh +# +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +# Test "morph deploy" by deploying to a raw disk image. + + +set -eu + + +. "$SRCDIR/tests.as-root/setup-build" +cd "$DATADIR/workspace/branch1" +"$SRCDIR/scripts/test-morph" build linux-system +"$SRCDIR/scripts/test-morph" --log "$DATADIR/deploy.log" \ + deploy rawdisk linux-system "$DATADIR/disk.img" > /dev/null +[ -e "$DATADIR/disk.img" ] + diff --git a/tests.deploy/setup b/tests.deploy/setup new file mode 100755 index 00000000..1065849d --- /dev/null +++ b/tests.deploy/setup @@ -0,0 +1,204 @@ +#!/bin/bash +# +# Create git repositories for tests. The chunk repository will contain a +# simple "hello, world" C program, and two branches ("master", "farrokh"), +# with the master branch containing just a README. The two branches are there +# so that we can test building a branch that hasn't been checked out. +# The branches are different so that we know that if the wrong branch +# is uses, the build will fail. +# +# The stratum repository contains a single branch, "master", with a +# stratum and a system morphology that include the chunk above. +# +# Copyright (C) 2011, 2012, 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +set -eu + +source "$SRCDIR/scripts/fix-committer-info" + +# The $DATADIR should be empty at the beginnig of each test. +find "$DATADIR" -mindepth 1 -delete + +# Create an empty directory to be used as a morph workspace +mkdir "$DATADIR/workspace" + +# Create chunk repository. + +chunkrepo="$DATADIR/chunk-repo" +mkdir "$chunkrepo" +cd "$chunkrepo" +git init --quiet + +cat <<EOF > README +This is a sample README. +EOF +git add README +git commit --quiet -m "add README" + +git checkout --quiet -b farrokh + +cat <<EOF > hello.c +#include <stdio.h> +int main(void) +{ + puts("hello, world"); + return 0; +} +EOF +git add hello.c + +cat <<EOF > hello.morph +{ + "name": "hello", + "kind": "chunk", + "build-system": "dummy", + "build-commands": [ + "gcc -o hello hello.c" + ], + "install-commands": [ + "install -d \\"\$DESTDIR\\"/etc", + "install -d \\"\$DESTDIR\\"/bin", + "install hello \\"\$DESTDIR\\"/bin/hello" + ] +} +EOF +git add hello.morph + +git commit --quiet -m "add a hello world program and morph" + +git checkout --quiet master + + + +# Create morph repository. + +morphsrepo="$DATADIR/morphs" +mkdir "$morphsrepo" +cd "$morphsrepo" +git init --quiet + +cat <<EOF > hello-stratum.morph +{ + "name": "hello-stratum", + "kind": "stratum", + "chunks": [ + { + "name": "hello", + "repo": "test:chunk-repo", + "ref": "farrokh", + "build-depends": [] + } + ] +} +EOF +git add hello-stratum.morph + +cat <<EOF > hello-system.morph +{ + "name": "hello-system", + "kind": "system", + "system-kind": "rootfs-tarball", + "arch": "$(uname -m)", + "strata": [ + { + "morph": "hello-stratum", + "repo": "test:morphs", + "ref": "master" + } + ] +} +EOF +git add hello-system.morph + +cat <<EOF > linux-stratum.morph +{ + "name": "linux-stratum", + "kind": "stratum", + "build-depends": [ + { + "morph": "hello-stratum", + "repo": "test:morphs", + "ref": "master" + } + ], + "chunks": [ + { + "name": "linux", + "repo": "test:kernel-repo", + "ref": "master", + "build-depends": [] + } + ] +} +EOF +git add linux-stratum.morph + +cat <<EOF > linux-system.morph +{ + "name": "linux-system", + "kind": "system", + "system-kind": "rootfs-tarball", + "arch": "$(uname -m)", + "strata": [ + { + "morph": "hello-stratum", + "repo": "test:morphs", + "ref": "master" + }, + { + "morph": "linux-stratum", + "repo": "test:morphs", + "ref": "master" + } + ] +} +EOF +git add linux-system.morph + +git commit --quiet -m "add morphs" + +# Make a dummy kernel chunk. +mkdir "$DATADIR/kernel-repo" +cat <<EOF > "$DATADIR/kernel-repo/linux.morph" +{ + "name": "linux", + "kind": "chunk", + "install-commands": [ + "mkdir -p \"\$DESTDIR/boot\"", + "touch \"\$DESTDIR\"/extlinux.conf", + "touch \"\$DESTDIR\"/boot/vmlinuz", + "touch \"\$DESTDIR\"/boot/System.map" + ] +} +EOF +"$SRCDIR/scripts/run-git-in" "$DATADIR/kernel-repo" init --quiet +"$SRCDIR/scripts/run-git-in" "$DATADIR/kernel-repo" add . +"$SRCDIR/scripts/run-git-in" "$DATADIR/kernel-repo" commit --quiet -m foo \ + > /dev/null + +# Create a morph configuration file. +cat <<EOF > "$DATADIR/morph.conf" +[config] +repo-alias = test=file://$DATADIR/#file://$DATADIR/ +cachedir = $DATADIR/cache +log = $DATADIR/morph.log +keep-path = true +no-distcc = true +quiet = true +log = /tmp/morph.log +EOF + diff --git a/tests.deploy/setup-build b/tests.deploy/setup-build new file mode 100644 index 00000000..6c0a6252 --- /dev/null +++ b/tests.deploy/setup-build @@ -0,0 +1,35 @@ +#!/bin/bash +# +# Copyright (C) 2012, 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +## Fixture for tests involving 'morph build' + +source "$SRCDIR/scripts/fix-committer-info" + +cd "$DATADIR/workspace" +"$SRCDIR/scripts/test-morph" init +"$SRCDIR/scripts/test-morph" branch test:morphs branch1 +"$SRCDIR/scripts/test-morph" edit linux-system linux-stratum linux + +# Fix UUID's in the checked out repos to make build branch names deterministic +git config -f "$DATADIR/workspace/branch1/.morph-system-branch/config" \ + branch.uuid 123456789 +git config -f "$DATADIR/workspace/branch1/test:morphs/.git/config" \ + morph.uuid 987654321 +git config -f "$DATADIR/workspace/branch1/test:kernel-repo/.git/config" \ + morph.uuid AABBCCDDE + diff --git a/without-test-modules b/without-test-modules index cb0302c8..5bcdb025 100644 --- a/without-test-modules +++ b/without-test-modules @@ -17,5 +17,7 @@ morphlib/plugins/branch_and_merge_plugin.py morphlib/buildcommand.py morphlib/plugins/build_plugin.py morphlib/gitversion.py - morphlib/plugins/expand_repo_plugin.py +morphlib/plugins/deploy_plugin.py +morphlib/plugins/__init__.py +morphlib/writeexts.py |