diff options
28 files changed, 478 insertions, 148 deletions
diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 8319430c..0d445304 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -56,7 +56,7 @@ import buildbranch import buildcommand import buildenvironment import buildsystem -import builder2 +import builder import cachedrepo import cachekeycomputer import extensions diff --git a/morphlib/app.py b/morphlib/app.py index 3f4171a4..177bce45 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,6 +17,7 @@ import cliapp import logging import os +import pipes import sys import time import urlparse @@ -308,6 +309,11 @@ class Morph(cliapp.Application): if (submod.url, submod.commit) not in done: subs_to_process.add((submod.url, submod.commit)) + def _write_status(self, text): + timestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()) + self.output.write('%s %s\n' % (timestamp, text)) + self.output.flush() + def status(self, **kwargs): '''Show user a status update. @@ -344,11 +350,22 @@ class Morph(cliapp.Application): ok = verbose or error or (not quiet and not chatty) if ok: - timestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()) - self.output.write('%s %s\n' % (timestamp, text)) - self.output.flush() + self._write_status(text) - def runcmd(self, argv, *args, **kwargs): + def _commandline_as_message(self, argv, args): + '''Create a status string for a command that's about to be executed.''' + + commands = [] + for command in [argv] + list(args): + if isinstance(command, list): + command_str = ' '.join(map(pipes.quote, command)) + else: + command_str = pipes.quote(command) + commands.append(command_str) + + return '# ' + ' | '.join(commands) + + def _prepare_for_runcmd(self, argv, args, kwargs): if 'env' not in kwargs: kwargs['env'] = dict(os.environ) @@ -358,28 +375,25 @@ class Morph(cliapp.Application): else: print_command = True - # convert the command line arguments into a string - commands = [argv] + list(args) - for command in commands: - if isinstance(command, list): - for i in xrange(0, len(command)): - command[i] = str(command[i]) - commands = [' '.join(command) for command in commands] - - # print the command line - if print_command: - self.status(msg='# %(cmdline)s', - cmdline=' | '.join(commands), - chatty=True) + if print_command and self.settings['verbose']: + # Don't call self.status() here, to avoid writing the message to + # the log as well as to the console. The cliapp.runcmd() function + # will also log the command, and it's messy having it logged twice. + self._write_status(self._commandline_as_message(argv, args)) # Log the environment. prev = getattr(self, 'prev_env', {}) morphlib.util.log_environment_changes(self, kwargs['env'], prev) self.prev_env = kwargs['env'] - # run the command line + def runcmd(self, argv, *args, **kwargs): + self._prepare_for_runcmd(argv, args, kwargs) return cliapp.Application.runcmd(self, argv, *args, **kwargs) + def runcmd_unchecked(self, argv, *args, **kwargs): + self._prepare_for_runcmd(argv, args, kwargs) + return cliapp.Application.runcmd_unchecked(self, argv, *args, **kwargs) + def parse_args(self, args, configs_only=False): return self.settings.parse_args(args, configs_only=configs_only, @@ -432,7 +446,7 @@ class Morph(cliapp.Application): self._help(True) def help_extensions(self, args): - exts = extensions.list_extensions(self.settings['build-ref-prefix']) + exts = extensions.list_extensions() template = "Extensions:\n %s\n" ext_string = '\n '.join(exts) self.output.write(template % (ext_string)) diff --git a/morphlib/buildbranch.py b/morphlib/buildbranch.py index 2d529133..2482bc9a 100644 --- a/morphlib/buildbranch.py +++ b/morphlib/buildbranch.py @@ -35,6 +35,15 @@ class BuildBranchCleanupError(cliapp.AppException): % locals()) +class NoReposError(cliapp.AppException): + def __init__(self, bb, ignored): + self.bb = bb + cliapp.AppException.__init__( + self, "No repos were found in system branch (ignored %i which " + "didn't have the right morph.uuid setting)" % (ignored)) + + + class BuildBranch(object): '''Represent the sources modified in a system branch. @@ -61,12 +70,12 @@ class BuildBranch(object): self._branch_root = sb.get_config('branch.root') branch_uuid = sb.get_config('branch.uuid') - for gd in sb.list_git_directories(): + for count, gd in enumerate(sb.list_git_directories()): try: repo_uuid = gd.get_config('morph.uuid') except cliapp.AppException: # Not a repository cloned by morph, ignore - break + continue build_ref = os.path.join('refs/heads', build_ref_prefix, branch_uuid, repo_uuid) # index is commit of workspace + uncommitted changes may want @@ -76,6 +85,9 @@ class BuildBranch(object): index.set_to_tree(gd.resolve_ref_to_tree(gd.HEAD)) self._to_push[gd] = (build_ref, index) + if len(self._to_push) == 0: + raise NoReposError(self, count) + rootinfo, = ((gd, index) for gd, (build_ref, index) in self._to_push.iteritems() if gd.get_config('morph.repository') == self._branch_root) diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 645a336c..a22e689b 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2014 Codethink Limited +# Copyright (C) 2011-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -126,11 +126,21 @@ class BuildCommand(object): root_arch = root_artifact.source.morphology['arch'] host_arch = morphlib.util.get_host_architecture() - if root_arch != host_arch: - raise morphlib.Error( - 'Are you trying to cross-build? ' - 'Host architecture is %s but target is %s' - % (host_arch, root_arch)) + + if root_arch == host_arch: + return + + # Since the armv8 instruction set is nearly entirely armv7 compatible, + # and since the incompatibilities are appropriately trapped in the + # kernel, we can safely run any armv7 toolchain natively on armv8. + if host_arch == 'armv8l' and root_arch in ('armv7l', 'armv7lhf'): + return + if host_arch == 'armv8b' and root_arch in ('armv7b', 'armv7bhf'): + return + + raise morphlib.Error( + 'Are you trying to cross-build? Host architecture is %s but ' + 'target is %s' % (host_arch, root_arch)) @staticmethod def _validate_has_non_bootstrap_chunks(srcpool): @@ -518,7 +528,7 @@ class BuildCommand(object): staging_area.install_artifact(handle) if target_source.build_mode == 'staging': - morphlib.builder2.ldconfig(self.app.runcmd, staging_area.dirname) + morphlib.builder.ldconfig(self.app.runcmd, staging_area.dirname) def build_and_cache(self, staging_area, source, setup_mounts): '''Build a source and put its artifacts into the local cache.''' @@ -526,7 +536,7 @@ class BuildCommand(object): self.app.status(msg='Starting actual build: %(name)s ' '%(sha1)s', name=source.name, sha1=source.sha1[:7]) - builder = morphlib.builder2.Builder( + builder = morphlib.builder.Builder( self.app, staging_area, self.lac, self.rac, self.lrc, self.app.settings['max-jobs'], setup_mounts) return builder.build_and_cache(source) diff --git a/morphlib/builder2.py b/morphlib/builder.py index f71f21db..9e158ea1 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder.py @@ -665,11 +665,18 @@ class SystemBuilder(BuilderBase): # pragma: no cover ) try: for bin in sorted(os.listdir(sys_integration_dir)): - self.app.runcmd( - morphlib.util.containerised_cmdline( - [os.path.join(SYSTEM_INTEGRATION_PATH, bin)], - root=rootdir, mounts=to_mount, mount_proc=True), - env=env) + argv = [os.path.join(SYSTEM_INTEGRATION_PATH, bin)] + container_config = dict( + root=rootdir, mounts=to_mount, mount_proc=True) + cmdline = morphlib.util.containerised_cmdline( + argv, **container_config) + exit, out, err = self.app.runcmd_unchecked( + cmdline, env=env) + if exit != 0: + logging.debug('Command returned code %i', exit) + msg = error_message_for_containerised_commandline( + argv, err, container_config) + raise cliapp.AppException(msg) except BaseException, e: self.app.status( msg='Error while running system integration commands', diff --git a/morphlib/builder2_tests.py b/morphlib/builder_tests.py index f7a761f2..245648ba 100644 --- a/morphlib/builder2_tests.py +++ b/morphlib/builder_tests.py @@ -156,7 +156,7 @@ class BuilderBaseTests(unittest.TestCase): self.repo_cache = None self.build_env = FakeBuildEnv() self.max_jobs = 1 - self.builder = morphlib.builder2.BuilderBase(self.app, + self.builder = morphlib.builder.BuilderBase(self.app, self.staging_area, self.artifact_cache, None, @@ -195,7 +195,7 @@ class BuilderBaseTests(unittest.TestCase): fh = rac.put(a) fh.write(a.name) fh.close() - morphlib.builder2.download_depends(afacts, lac, rac) + morphlib.builder.download_depends(afacts, lac, rac) self.assertTrue(all(lac.has(a) for a in afacts)) def test_downloads_depends_metadata(self): @@ -209,7 +209,7 @@ class BuilderBaseTests(unittest.TestCase): fh = rac.put_artifact_metadata(a, 'meta') fh.write('metadata') fh.close() - morphlib.builder2.download_depends(afacts, lac, rac, ('meta',)) + morphlib.builder.download_depends(afacts, lac, rac, ('meta',)) self.assertTrue(all(lac.has(a) for a in afacts)) self.assertTrue(all(lac.has_artifact_metadata(a, 'meta') for a in afacts)) @@ -219,5 +219,5 @@ class ChunkBuilderTests(unittest.TestCase): def setUp(self): self.app = FakeApp() - self.build = morphlib.builder2.ChunkBuilder(self.app, None, None, + self.build = morphlib.builder.ChunkBuilder(self.app, None, None, None, None, None, 1, False) diff --git a/morphlib/extensions.py b/morphlib/extensions.py index b270d304..051a54a7 100644 --- a/morphlib/extensions.py +++ b/morphlib/extensions.py @@ -38,6 +38,7 @@ class ExtensionNotExecutableError(ExtensionError): pass def _get_root_repo(): + workspace = morphlib.workspace.open('.') system_branch = morphlib.sysbranchdir.open_from_within('.') root_repo_dir = morphlib.gitdir.GitDirectory( system_branch.get_git_directory_name( @@ -77,7 +78,8 @@ def _list_extensions(kind): try: repo_extension_filenames = \ _list_repo_extension_filenames(kind) - except (sysbranchdir.NotInSystemBranch): + except (morphlib.workspace.NotInWorkspace, + sysbranchdir.NotInSystemBranch): # Squash this and just return no system branch extensions pass morph_extension_filenames = _list_morph_extension_filenames(kind) diff --git a/morphlib/exts/initramfs.write.help b/morphlib/exts/initramfs.write.help index 29a9d266..a4a89f9d 100644 --- a/morphlib/exts/initramfs.write.help +++ b/morphlib/exts/initramfs.write.help @@ -1,4 +1,5 @@ help: | + Create an initramfs for a system by taking an existing system and converting it to the appropriate format. @@ -33,3 +34,8 @@ help: | initramfs: type: initramfs location: boot/initramfs.gz + + Parameters: + + * location: the path where the initramfs will be installed (e.g. + `boot/initramfs.gz`) in the above example diff --git a/morphlib/exts/kvm.write.help b/morphlib/exts/kvm.write.help index 26a54d9c..04393b8a 100644 --- a/morphlib/exts/kvm.write.help +++ b/morphlib/exts/kvm.write.help @@ -42,6 +42,35 @@ help: | * AUTOSTART=<VALUE>` - boolean. If it is set, the VM will be started when it has been deployed. + * DTB_PATH=path: **(MANDATORY)** for systems that require a device tree + binary - Give the full path (without a leading /) to the location of the + DTB in the built system image . The deployment will fail if `path` does + not exist. + + * BOOTLOADER_INSTALL=value: the bootloader to be installed + **(MANDATORY)** for non-x86 systems + + allowed values = + - 'extlinux' (default) - the extlinux bootloader will + be installed + - 'none' - no bootloader will be installed by `morph deploy`. A + bootloader must be installed manually. This value must be used when + deploying non-x86 systems such as ARM. + + * BOOTLOADER_CONFIG_FORMAT=value: the bootloader format to be used. + If not specified for x86-32 and x86-64 systems, 'extlinux' will be used + + allowed values = + - 'extlinux' + + * KERNEL_ARGS=args: optional additional kernel command-line parameters to + be appended to the default set. The default set is: + + 'rw init=/sbin/init rootfstype=btrfs \ + rootflags=subvol=systems/default/run \ + root=[name or UUID of root filesystem]' + + (See https://www.kernel.org/doc/Documentation/kernel-parameters.txt) + (See `morph help deploy` for details of how to pass parameters to write extensions) - diff --git a/morphlib/exts/nfsboot.write b/morphlib/exts/nfsboot.write index 8d3d6df7..49d71174 100755 --- a/morphlib/exts/nfsboot.write +++ b/morphlib/exts/nfsboot.write @@ -17,6 +17,16 @@ '''A Morph deployment write extension for deploying to an nfsboot server +*** DO NOT USE *** +- This was written before 'proper' deployment mechanisms were in place +It is unlikely to work at all and will not work correctly + +Use the pxeboot write extension instead + +*** + + + An nfsboot server is defined as a baserock system that has tftp and nfs servers running, the tftp server is exporting the contents of /srv/nfsboot/tftp/ and the user has sufficient permissions to create nfs roots @@ -125,7 +135,7 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Creating destination directories') try: - cliapp.ssh_runcmd('root@%s' % location, + cliapp.ssh_runcmd('root@%s' % location, ['mkdir', '-p', orig_path, run_path]) except cliapp.AppException: raise cliapp.AppException('Could not create dirs %s and %s on %s' @@ -191,4 +201,3 @@ mv "$temp" "$target" NFSBootWriteExtension().run() - diff --git a/morphlib/exts/nfsboot.write.help b/morphlib/exts/nfsboot.write.help index 598b1b23..310fd7a4 100644 --- a/morphlib/exts/nfsboot.write.help +++ b/morphlib/exts/nfsboot.write.help @@ -1,6 +1,13 @@ help: | + *** DO NOT USE *** + - This was written before 'proper' deployment mechanisms were in place. + It is unlikely to work at all, and will not work correctly. + + Use the pxeboot write extension instead + + *** Deploy a system image and kernel to an nfsboot server. - + An nfsboot server is defined as a baserock system that has tftp and nfs servers running, the tftp server is exporting the contents of /srv/nfsboot/tftp/ and the user has sufficient diff --git a/morphlib/exts/openstack.check b/morphlib/exts/openstack.check index edc37cc1..3850d481 100755 --- a/morphlib/exts/openstack.check +++ b/morphlib/exts/openstack.check @@ -77,9 +77,16 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): '--os-password', password, '--os-auth-url', auth_url, 'image-list'] - try: - cliapp.runcmd(cmdline) - except cliapp.AppException: - raise cliapp.AppException('Wrong OpenStack credentals.') + + exit, out, err = cliapp.runcmd_unchecked(cmdline) + + if exit != 0: + if err.startswith('The request you have made requires ' + 'authentication. (HTTP 401)'): + raise cliapp.AppException('Invalid OpenStack credentials.') + else: + raise cliapp.AppException( + 'Failed to connect to OpenStack instance at %s: %s' % + (auth_url, err)) OpenStackCheckExtension().run() diff --git a/morphlib/exts/openstack.write b/morphlib/exts/openstack.write index b1941d3c..d29d2661 100755 --- a/morphlib/exts/openstack.write +++ b/morphlib/exts/openstack.write @@ -28,40 +28,12 @@ import morphlib.writeexts class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): - '''Configure a raw disk image into an OpenStack host. - - The raw disk image is created during Morph's deployment and the - image is deployed in OpenStack using python-glanceclient. - - The location command line argument is the authentification url - of the OpenStack server using the following syntax: - - http://HOST:PORT/VERSION - - where - - * HOST is the host running OpenStack - * PORT is the port which is using OpenStack for authentifications. - * VERSION is the authentification version of OpenStack (Only v2.0 - supported) - - This extension needs in the environment the following variables: - - * OPENSTACK_USER is the username to use in the deployment. - * OPENSTACK_TENANT is the project name to use in the deployment. - * OPENSTACK_IMAGENAME is the name of the image to create. - * OPENSTACK_PASSWORD is the password of the user. - - - The extension will connect to OpenStack using python-glanceclient - to configure a raw image. - - ''' + '''See openstack.write.help for documentation''' def process_args(self, args): if len(args) != 2: raise cliapp.AppException('Wrong number of command line args') - + temp_root, location = args os_params = self.get_openstack_parameters() @@ -69,7 +41,7 @@ class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): fd, raw_disk = tempfile.mkstemp() os.close(fd) self.create_local_system(temp_root, raw_disk) - self.status(msg='Temporary disk image has been created at %s' + self.status(msg='Temporary disk image has been created at %s' % raw_disk) self.set_extlinux_root_to_virtio(raw_disk) @@ -120,4 +92,3 @@ class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Image configured.') OpenStackWriteExtension().run() - diff --git a/morphlib/exts/openstack.write.help b/morphlib/exts/openstack.write.help new file mode 100644 index 00000000..75ad9f0c --- /dev/null +++ b/morphlib/exts/openstack.write.help @@ -0,0 +1,37 @@ +help: | + + Deploy a Baserock system as a *new* OpenStack virtual machine. + (Use the `ssh-rsync` write extension to deploy upgrades to an *existing* + VM) + + Deploys the system to the OpenStack host using python-glanceclient. + + Parameters: + + * location: the authentication url of the OpenStack server using the + following syntax: + + http://HOST:PORT/VERSION + + where + + * HOST is the host running OpenStack + * PORT is the port which is using OpenStack for authentications. + * VERSION is the authentication version of OpenStack (Only v2.0 + supported) + + * OPENSTACK_USER=username: the username to use in the `--os-username` + argument to `glance`. + + * OPENSTACK_TENANT=tenant: the project name to use in the + `--os-tenant-name` argument to `glance`. + + * OPENSTACK_IMAGENAME=imagename: the name of the image to use in the + `--name` argument to `glance`. + + * OPENSTACK_PASSWORD=password: the password of the OpenStack user. (We + recommend passing this on the command-line, rather than setting an + environment variable or storing it in a cluster cluster definition file.) + + (See `morph help deploy` for details of how to pass parameters to write + extensions) diff --git a/morphlib/exts/rawdisk.write b/morphlib/exts/rawdisk.write index b17f8aa7..d91a4d5f 100755 --- a/morphlib/exts/rawdisk.write +++ b/morphlib/exts/rawdisk.write @@ -29,19 +29,12 @@ import morphlib.writeexts class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): - '''Create a raw disk image during Morph's deployment. - - If the image already exists, it is upgraded. - - The location command line argument is the pathname of the disk image - to be created/upgraded. - - ''' + '''See rawdisk.write.help for documentation''' def process_args(self, args): if len(args) != 2: raise cliapp.AppException('Wrong number of command line args') - + temp_root, location = args upgrade = self.get_environment_boolean('UPGRADE') @@ -114,4 +107,3 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): RawDiskWriteExtension().run() - diff --git a/morphlib/exts/rawdisk.write.help b/morphlib/exts/rawdisk.write.help index 298d441c..54af81c4 100644 --- a/morphlib/exts/rawdisk.write.help +++ b/morphlib/exts/rawdisk.write.help @@ -1,11 +1,68 @@ help: | - Create a raw disk image during Morph's deployment. - - If the image already exists, it is upgraded. - The `location` argument is a pathname to the image to be - created or upgraded. + Write a system produced by Morph to a physical disk, or to a file that can + be used as a virtual disk. The target will be formatted as a single Btrfs + partition, with the system image written to a subvolume in /systems, and + other subvolumes created for /home, /opt, /root, /srv and /var. - The INITRAMFS_PATH option can be used to specify the location of an - initramfs for syslinux to tell Linux to use, rather than booting - the rootfs directly. + When written to a physical drive, the drive can be used as the boot device + for a 'real' machine. + + When written to a file, the file can be used independently of `morph` to + create virtual machines with KVM / libvirt, OpenStack or, after converting + it to VDI format, VirtualBox. + + `morph deploy` will fail if the file specified by `location` already + exists. + + If used in `morph upgrade`, the rootfs produced by 'morph build' is added + to the existing raw disk image or device as an additional btrfs sub-volume. + `morph upgrade` will fail if the file specified by `location` does not + exist, or is not a Baserock raw disk image. (Most users are unlikely to + need or use this functionality: it is useful mainly for developers working + on the Baserock tools.) + + Parameters: + + * location: the pathname of the disk image to be created/upgraded, or the + path to the physical device. + + * VERSION_LABEL=label - should contain only alpha-numeric + characters and the '-' (hyphen) character. Mandatory if being used with + `morph update` + + * INITRAMFS_PATH=path: the location of an initramfs for the bootloader to + tell Linux to use, rather than booting the rootfs directly. + + * DTB_PATH=path: **(MANDATORY)** for systems that require a device tree + binary - Give the full path (without a leading /) to the location of the + DTB in the built system image . The deployment will fail if `path` does + not exist. + + * BOOTLOADER_INSTALL=value: the bootloader to be installed + **(MANDATORY)** for non-x86 systems + + allowed values = + - 'extlinux' (default) - the extlinux bootloader will + be installed + - 'none' - no bootloader will be installed by `morph deploy`. A + bootloader must be installed manually. This value must be used when + deploying non-x86 systems such as ARM. + + * BOOTLOADER_CONFIG_FORMAT=value: the bootloader format to be used. + If not specified for x86-32 and x86-64 systems, 'extlinux' will be used + + allowed values = + - 'extlinux' + + * KERNEL_ARGS=args: optional additional kernel command-line parameters to + be appended to the default set. The default set is: + + 'rw init=/sbin/init rootfstype=btrfs \ + rootflags=subvol=systems/default/run \ + root=[name or UUID of root filesystem]' + + (See https://www.kernel.org/doc/Documentation/kernel-parameters.txt) + + (See `morph help deploy` for details of how to pass parameters to write + extensions) diff --git a/morphlib/exts/ssh-rsync.write b/morphlib/exts/ssh-rsync.write index 2d7258ba..c4577026 100755 --- a/morphlib/exts/ssh-rsync.write +++ b/morphlib/exts/ssh-rsync.write @@ -37,14 +37,8 @@ def ssh_runcmd_ignore_failure(location, command, **kwargs): class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): - '''Upgrade a running baserock system with ssh and rsync. - - It assumes the system is baserock-based and has a btrfs partition. - - The location command line argument is the 'user@hostname' string - that will be passed to ssh and rsync - - ''' + '''See ssh-rsync.write.help for documentation''' + def find_root_disk(self, location): '''Read /proc/mounts on location to find which device contains "/"''' diff --git a/morphlib/exts/ssh-rsync.write.help b/morphlib/exts/ssh-rsync.write.help new file mode 100644 index 00000000..d03508c0 --- /dev/null +++ b/morphlib/exts/ssh-rsync.write.help @@ -0,0 +1,36 @@ +help: | + + Upgrade a Baserock system which is already deployed: + - as a KVM/LibVirt, OpenStack or vbox-ssh virtual machine; + - on a Jetson board. + + Copies a binary delta over to the target system and arranges for it + to be bootable. + + The recommended way to use this extension is by calling `morph upgrade`. + Using `morph deploy --upgrade` is deprecated. + + The upgrade will fail if: + - no VM is deployed and running at `location`; + - the target system is not a Baserock system; + - the target's filesystem and its layout are not compatible with that + created by `morph deploy`." + + See also the 'Upgrading a Baserock installation' section of the 'Using + Baserock` page at wiki.baserock.org + http://wiki.baserock.org/devel-with/#index8h2 + + Parameters: + + * location: the 'user@hostname' string that will be used by ssh and rsync. + 'user' will always be `root` and `hostname` the hostname or address of + the system being upgraded. + + * VERSION_LABEL=label - **(MANDATORY)** should contain only alpha-numeric + characters and the '-' (hyphen) character. + + * AUTOSTART=<VALUE>` - boolean. If it is set, the VM will be started when + it has been deployed. + + (See `morph help deploy` for details of how to pass parameters to write + extensions) diff --git a/morphlib/exts/sysroot.check b/morphlib/exts/sysroot.check new file mode 100755 index 00000000..bfacd3fc --- /dev/null +++ b/morphlib/exts/sysroot.check @@ -0,0 +1,30 @@ +#!/bin/sh +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# Preparatory checks for Morph 'sysroot' write extension + +set -eu + +location="$1" +if [ -d "$location" ]; then + echo >&2 "ERROR: Deployment directory already exists: $location" + exit 1 +fi + +if [ "$UPGRADE" == "yes" ]; then + echo >&2 "ERROR: Cannot upgrade a sysroot deployment" + exit 1 +fi diff --git a/morphlib/exts/sysroot.write b/morphlib/exts/sysroot.write index 1ae4864f..be315365 100755 --- a/morphlib/exts/sysroot.write +++ b/morphlib/exts/sysroot.write @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,9 +18,7 @@ set -eu -# Ensure the target is an empty directory mkdir -p "$2" -find "$2" -mindepth 1 -delete # Move the contents of our source directory to our target # Previously we would (cd "$1" && find -print0 | cpio -0pumd "$absolute_path") diff --git a/morphlib/exts/virtualbox-ssh.write.help b/morphlib/exts/virtualbox-ssh.write.help index b4c59553..cb50acc0 100644 --- a/morphlib/exts/virtualbox-ssh.write.help +++ b/morphlib/exts/virtualbox-ssh.write.help @@ -41,6 +41,36 @@ help: | * INITRAMFS_PATH=path: the location of an initramfs for the bootloader to tell Linux to use, rather than booting the rootfs directly. + * DTB_PATH=path: **(MANDATORY)** for systems that require a device tree + binary - Give the full path (without a leading /) to the location of the + DTB in the built system image . The deployment will fail if `path` does + not exist. + + * BOOTLOADER_INSTALL=value: the bootloader to be installed + **(MANDATORY)** for non-x86 systems + + allowed values = + - 'extlinux' (default) - the extlinux bootloader will + be installed + - 'none' - no bootloader will be installed by `morph deploy`. A + bootloader must be installed manually. This value must be used when + deploying non-x86 systems such as ARM. + + * BOOTLOADER_CONFIG_FORMAT=value: the bootloader format to be used. + If not specified for x86-32 and x86-64 systems, 'extlinux' will be used + + allowed values = + - 'extlinux' + + * KERNEL_ARGS=args: optional additional kernel command-line parameters to + be appended to the default set. The default set is: + + 'rw init=/sbin/init rootfstype=btrfs \ + rootflags=subvol=systems/default/run \ + root=[name or UUID of root filesystem]' + + (See https://www.kernel.org/doc/Documentation/kernel-parameters.txt) + * AUTOSTART=<VALUE> - boolean. If it is set, the VM will be started when it has been deployed. diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py index cd395228..74db6ed6 100644 --- a/morphlib/gitdir.py +++ b/morphlib/gitdir.py @@ -484,11 +484,15 @@ class GitDirectory(object): self._config[key] = value def get_config(self, key): - '''Return value for a git repository configuration variable.''' + '''Return value for a git repository configuration variable. + + If the variable is unset, this will raise cliapp.AppException. + + ''' if key not in self._config: - value = morphlib.git.gitcmd(self._runcmd, 'config', '-z', key) - self._config[key] = value.rstrip('\0') + value = morphlib.git.gitcmd(self._runcmd, 'config', '-z', key) + self._config[key] = value.rstrip('\0') return self._config[key] def get_remote(self, *args, **kwargs): diff --git a/morphlib/plugins/cross-bootstrap_plugin.py b/morphlib/plugins/cross-bootstrap_plugin.py index 7b53a4a5..2b5c8911 100644 --- a/morphlib/plugins/cross-bootstrap_plugin.py +++ b/morphlib/plugins/cross-bootstrap_plugin.py @@ -46,7 +46,7 @@ def escape_source_name(source): # Most of this is ripped from RootfsTarballBuilder, and should be reconciled # with it. -class BootstrapSystemBuilder(morphlib.builder2.BuilderBase): +class BootstrapSystemBuilder(morphlib.builder.BuilderBase): '''Build a bootstrap system tarball The bootstrap system image contains a minimal cross-compiled toolchain @@ -104,7 +104,7 @@ class BootstrapSystemBuilder(morphlib.builder2.BuilderBase): source_dir = os.path.join(path, 'src', escaped_source) if not os.path.exists(source_dir): os.makedirs(source_dir) - morphlib.builder2.extract_sources( + morphlib.builder.extract_sources( self.app, self.repo_cache, s.repo, s.sha1, source_dir) name = s.name diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 87e129e5..0121c110 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -124,7 +124,7 @@ class DeployPlugin(cliapp.Plugin): Each system defined in a cluster morphology can be deployed in multiple ways (`type` in a cluster morphology). Morph provides - five types of deployment: + the following types of deployment: * `tar` where Morph builds a tar archive of the root file system. @@ -144,6 +144,35 @@ class DeployPlugin(cliapp.Plugin): * `nfsboot` where Morph creates a system to be booted over a network. + * `ssh-rsync` where Morph copies a binary delta over to the target + system and arranges for it to be bootable. This requires + `system-version-manager` from the tbdiff chunk + + * `initramfs`, where Morph turns the system into an initramfs image, + suitable for being used as the early userland environment for a + system to be able to locate more complicated storage for its root + file-system, or on its own for diskless deployments. + + There are additional extensions that currently live in the Baserock + definitions repo (baserock:baserock/definitions). These include: + + * `image-package` where Morph creates a tarball that includes scripts + that can be used to make disk images outside of a Baserock + environment. The example in definitions.git will create scripts for + generating disk images and installing to existing disks. + + * `sdk` where Morph generates something resembing a BitBake SDK, which + provides a toolchain for building software to target a system built + by Baserock, from outside of a Baserock environment. This creates a + self-extracting shell archive which you pass a directory to extract + to, and inside that has a shell snippet called + environment-setup-$TARGET which can be used to set environment + variables to use the toolchain. + + * `pxeboot` where Morph temporarily network-boots the system you are + deploying, so it can install a more permanent system onto local + storage. + In addition to the deployment type, the user must also give a value for `location`. Its syntax depends on the deployment types. The deployment types provided by Morph use the diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index b676d4db..fd3f6881 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -190,9 +190,10 @@ class StagingArea(object): shutil.rmtree(self.dirname) - to_mount = ( + to_mount_in_staging = ( ('dev/shm', 'tmpfs', 'none'), ) + to_mount_in_bootstrap = () def ccache_dir(self, source): #pragma: no cover ccache_dir = self._app.settings['compiler-cache-dir'] @@ -266,7 +267,13 @@ class StagingArea(object): do_not_mount_dirs += [temp_dir] logging.debug("Not mounting dirs %r" % do_not_mount_dirs) + if self.use_chroot: + mounts = self.to_mount_in_staging + else: + mounts = [(os.path.join(self.dirname, target), type, source) + for target, type, source in self.to_mount_in_bootstrap] mount_proc = self.use_chroot + if ccache_dir and not self._app.settings['no-ccache']: ccache_target = os.path.join( self.dirname, kwargs['env']['CCACHE_DIR'].lstrip('/')) @@ -274,22 +281,38 @@ class StagingArea(object): else: binds = () + container_config=dict( + cwd=kwargs.pop('cwd', '/'), + root=chroot_dir, + mounts=mounts, + mount_proc=mount_proc, + binds=binds, + writable_paths=do_not_mount_dirs) + cmdline = morphlib.util.containerised_cmdline( - argv, cwd=kwargs.pop('cwd', '/'), - root=chroot_dir, mounts=self.to_mount, - binds=binds, mount_proc=mount_proc, - writable_paths=do_not_mount_dirs) - try: - if kwargs.get('logfile') != None: - logfile = kwargs.pop('logfile') - teecmd = ['tee', '-a', logfile] - return self._app.runcmd(cmdline, teecmd, **kwargs) - else: - return self._app.runcmd(cmdline, **kwargs) - except cliapp.AppException as e: - raise cliapp.AppException('In staging area %s: running ' - 'command \'%s\' failed.' % - (self.dirname, ' '.join(argv))) + argv, **container_config) + + if kwargs.get('logfile') != None: + logfile = kwargs.pop('logfile') + teecmd = ['tee', '-a', logfile] + exit, out, err = self._app.runcmd_unchecked( + cmdline, teecmd, **kwargs) + else: + exit, out, err = self._app.runcmd_unchecked(cmdline, **kwargs) + + if exit == 0: + return out + else: + logging.debug('Command returned code %i', exit) + msg = morphlib.util.error_message_for_containerised_commandline( + argv, err, container_config) + raise cliapp.AppException( + 'In staging area %s: %s' % (self._failed_location(), msg)) + + def _failed_location(self): # pragma: no cover + '''Path this staging area will be moved to if an error occurs.''' + return os.path.join(self._app.settings['tempdir'], 'failed', + os.path.basename(self.dirname)) def abort(self): # pragma: no cover '''Handle what to do with a staging area in the case of failure. @@ -298,9 +321,7 @@ class StagingArea(object): # TODO: when we add the option to throw away failed builds, # hook it up here - - dest_dir = os.path.join(self._app.settings['tempdir'], - 'failed', os.path.basename(self.dirname)) + dest_dir = self._failed_location() os.rename(self.dirname, dest_dir) self.dirname = dest_dir diff --git a/morphlib/sysbranchdir.py b/morphlib/sysbranchdir.py index 4351c6b3..4dbc5f6c 100644 --- a/morphlib/sysbranchdir.py +++ b/morphlib/sysbranchdir.py @@ -72,7 +72,11 @@ class SystemBranchDirectory(object): def _find_git_directory(self, repo_url): for gd in self.list_git_directories(): - if gd.get_config('morph.repository') == repo_url: + try: + gd_repo_url = gd.get_config('morph.repository') + except cliapp.AppException: # pragma: no cover + continue + if gd_repo_url == repo_url: return gd.dirname return None diff --git a/morphlib/util.py b/morphlib/util.py index 6f735387..63e85b6c 100644 --- a/morphlib/util.py +++ b/morphlib/util.py @@ -16,6 +16,7 @@ import contextlib import itertools import os +import pipes import re import subprocess import textwrap @@ -41,7 +42,6 @@ try: from multiprocessing import cpu_count except NotImplementedError: # pragma: no cover cpu_count = lambda: 1 -import os def indent(string, spaces=4): @@ -465,6 +465,10 @@ def get_host_architecture(): # pragma: no cover 'i686': 'x86_32', 'armv7l': 'armv7l', 'armv7b': 'armv7b', + 'armv8l': 'armv8l', + 'armv8b': 'armv8b', + 'aarch64': 'aarch64', + 'aarch64b': 'aarch64b', 'ppc64': 'ppc64' } @@ -626,3 +630,18 @@ def containerised_cmdline(args, cwd='.', root='/', binds=(), cmdargs.append(root) cmdargs.extend(args) return unshared_cmdline(cmdargs, root=root, **kwargs) + + +def error_message_for_containerised_commandline( + argv, err, container_kwargs): # pragma: no cover + '''Return a semi-readable error message for a containerised command.''' + + # This function should do some formatting of the container_kwargs dict, + # rather than just dumping it in the error message, but that is better than + # nothing. + + argv_string = ' '.join(map(pipes.quote, argv)) + return 'Command failed: %s:\n' \ + 'Containerisation settings: %s\n' \ + 'Error output:\n%s' \ + % (argv_string, container_kwargs, err) diff --git a/morphlib/writeexts.py b/morphlib/writeexts.py index 91936f64..6ab2dd55 100644 --- a/morphlib/writeexts.py +++ b/morphlib/writeexts.py @@ -83,14 +83,14 @@ class Fstab(object): class WriteExtension(cliapp.Application): '''A base class for deployment write extensions. - + A subclass should subclass this class, and add a ``process_args`` method. - + Note that it is not necessary to subclass this class for write extensions. This class is here just to collect common code for write extensions. - + ''' def setup_logging(self): @@ -125,13 +125,13 @@ class WriteExtension(cliapp.Application): def status(self, **kwargs): '''Provide status output. - + The ``msg`` keyword argument is the actual message, the rest are values for fields in the message as interpolated by %. - + ''' - + self.output.write('%s\n' % (kwargs['msg'] % kwargs)) self.output.flush() @@ -184,9 +184,9 @@ class WriteExtension(cliapp.Application): def _parse_size(self, size): '''Parse a size from a string. - + Return size in bytes. - + ''' m = re.match('^(\d+)([kmgKMG]?)$', size) @@ -474,6 +474,12 @@ class WriteExtension(cliapp.Application): self.status(msg='Creating extlinux.conf') config = os.path.join(real_root, 'extlinux.conf') + + ''' Please also update the documentation in the following files + if you change these default kernel args: + - kvm.write.help + - rawdisk.write.help + - virtualbox-ssh.write.help ''' kernel_args = ( 'rw ' # ro ought to work, but we don't test that regularly 'init=/sbin/init ' # default, but it doesn't hurt to be explicit @@ -545,9 +551,8 @@ class WriteExtension(cliapp.Application): '''Does the user want to generate a bootloader config? The user may set $BOOTLOADER_CONFIG_FORMAT to the desired - format (u-boot or extlinux). If not set, extlinux is the - default but will be generated on x86-32 and x86-64, but not - otherwise. + format. 'extlinux' is the only allowed value, and is the default + value for x86-32 and x86-64. ''' |