diff options
Diffstat (limited to 'morphlib')
32 files changed, 1174 insertions, 343 deletions
diff --git a/morphlib/app.py b/morphlib/app.py index 7fb71c7b..409e0a12 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -59,6 +59,12 @@ class Morph(cliapp.Application): self.settings.boolean(['quiet', 'q'], 'show no output unless there is an error') + self.settings.boolean(['help', 'h'], + 'show this help message and exit') + + self.settings.boolean(['help-all'], + 'show help message including hidden subcommands') + self.settings.string(['build-ref-prefix'], 'Prefix to use for temporary build refs', metavar='PREFIX', @@ -195,6 +201,14 @@ class Morph(cliapp.Application): def process_args(self, args): self.check_time() + if self.settings['help']: + self.help(args) + sys.exit(0) + + if self.settings['help-all']: + self.help_all(args) + sys.exit(0) + if self.settings['build-ref-prefix'] is None: if self.settings['trove-id']: self.settings['build-ref-prefix'] = os.path.join( @@ -345,7 +359,10 @@ class Morph(cliapp.Application): morphology = resolved_morphologies[reference] visit(reponame, ref, filename, absref, tree, morphology) - if morphology['kind'] == 'system': + if morphology['kind'] == 'cluster': + raise cliapp.AppException( + "Cannot build a morphology of type 'cluster'.") + elif morphology['kind'] == 'system': queue.extend((s.get('repo') or reponame, s.get('ref') or ref, '%s.morph' % s['morph']) @@ -451,27 +468,49 @@ class Morph(cliapp.Application): # run the command line return cliapp.Application.runcmd(self, argv, *args, **kwargs) - # FIXME: This overrides a private method in cliapp. We need - # get cliapp to provide the necessary hooks to do this cleanly. - # As it is, this is a copy of the method in cliapp, with the - # single change that for subcommand helps, the formatting is - # not used. - def _help_helper(self, args, show_all): # pragma: no cover + def parse_args(self, args, configs_only=False): + return self.settings.parse_args(args, + configs_only=configs_only, + arg_synopsis=self.arg_synopsis, + cmd_synopsis=self.cmd_synopsis, + compute_setting_values=self.compute_setting_values, + add_help_option=False) + + class IdentityFormat(): + def format(self, text): + return text + + def _help_helper(self, args, show_all): try: width = int(os.environ.get('COLUMNS', '78')) except ValueError: width = 78 - fmt = cliapp.TextFormat(width=width) - if args: - usage = self._format_usage_for(args[0]) - description = self._format_subcommand_help(args[0]) + cmd = args[0] + if cmd not in self.subcommands: + raise cliapp.AppException('Unknown subcommand %s' % cmd) + # TODO Search for other things we might want help on + # such as write or configuration extensions. + usage = self._format_usage_for(cmd) + fmt = self.IdentityFormat() + description = fmt.format(self._format_subcommand_help(cmd)) text = '%s\n\n%s' % (usage, description) + self.output.write(text) else: - usage = self._format_usage(all=show_all) - description = fmt.format(self._format_description(all=show_all)) - text = '%s\n\n%s' % (usage, description) - - text = self.settings.progname.join(text.split('%prog')) - self.output.write(text) + pp = self.settings.build_parser( + configs_only=True, + arg_synopsis=self.arg_synopsis, + cmd_synopsis=self.cmd_synopsis, + all_options=show_all, + add_help_option=False) + text = pp.format_help() + self.output.write(text) + + def help(self, args): # pragma: no cover + '''Print help.''' + self._help_helper(args, False) + + def help_all(self, args): # pragma: no cover + '''Print help, including hidden subcommands.''' + self._help_helper(args, True) diff --git a/morphlib/artifactsplitrule.py b/morphlib/artifactsplitrule.py index 246691d8..bc92e5fb 100644 --- a/morphlib/artifactsplitrule.py +++ b/morphlib/artifactsplitrule.py @@ -300,4 +300,4 @@ def unify_system_matches(morphology): def unify_cluster_matches(_): - return None + return SplitRules() diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 3c190275..7ad7909d 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -455,10 +455,11 @@ class BuildCommand(object): if artifact.source.build_mode == 'bootstrap': if not self.in_same_stratum(artifact, target_artifact): continue - self.app.status(msg='Installing chunk %(chunk_name)s ' - 'from cache %(cache)s', - chunk_name=artifact.name, - cache=artifact.cache_key[:7]) + self.app.status( + msg='Installing chunk %(chunk_name)s from cache %(cache)s', + chunk_name=artifact.name, + cache=artifact.cache_key[:7], + chatty=True) handle = self.lac.get(artifact) staging_area.install_artifact(handle) diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 2dca738c..02e8b485 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -36,6 +36,8 @@ import morphlib from morphlib.artifactcachereference import ArtifactCacheReference import morphlib.gitversion +SYSTEM_INTEGRATION_PATH = os.path.join('baserock', 'system-integration') + def extract_sources(app, repo_cache, repo, sha1, srcdir): #pragma: no cover '''Get sources from git to a source directory, including submodules''' @@ -419,11 +421,44 @@ class ChunkBuilder(BuilderBase): shutil.copyfileobj(readlog, self.app.output) raise e + def write_system_integration_commands(self, destdir, + integration_commands, artifact_name): # pragma: no cover + + rel_path = SYSTEM_INTEGRATION_PATH + dest_path = os.path.join(destdir, SYSTEM_INTEGRATION_PATH) + + scripts_created = [] + + if not os.path.exists(dest_path): + os.makedirs(dest_path) + + if artifact_name in integration_commands: + prefixes_per_artifact = integration_commands[artifact_name] + for prefix, commands in prefixes_per_artifact.iteritems(): + for index, script in enumerate(commands): + script_name = "%s-%s-%04d" % (prefix, + artifact_name, + index) + script_path = os.path.join(dest_path, script_name) + + with morphlib.savefile.SaveFile(script_path, 'w') as f: + f.write("#!/bin/sh\nset -xeu\n") + f.write(script) + os.chmod(script_path, 0555) + + rel_script_path = os.path.join(SYSTEM_INTEGRATION_PATH, + script_name) + scripts_created += [rel_script_path] + + return scripts_created + def assemble_chunk_artifacts(self, destdir): # pragma: no cover built_artifacts = [] filenames = [] source = self.artifact.source split_rules = source.split_rules + morphology = source.morphology + sys_tag = 'system-integration' def filepaths(destdir): for dirname, subdirs, basenames in os.walk(destdir): @@ -438,6 +473,8 @@ class ChunkBuilder(BuilderBase): matches, overlaps, unmatched = \ split_rules.partition(filepaths(destdir)) + system_integration = morphology.get(sys_tag) or {} + with self.build_watch('create-chunks'): for chunk_artifact_name, chunk_artifact \ in source.artifacts.iteritems(): @@ -455,9 +492,11 @@ class ChunkBuilder(BuilderBase): names.update(all_parents(name)) return sorted(names) - parented_paths = \ - parentify(file_paths + - ['baserock/%s.meta' % chunk_artifact_name]) + extra_files = self.write_system_integration_commands( + destdir, system_integration, + chunk_artifact_name) + extra_files += ['baserock/%s.meta' % chunk_artifact_name] + parented_paths = parentify(file_paths + extra_files) with self.local_artifact_cache.put(chunk_artifact) as f: self.write_metadata(destdir, chunk_artifact_name, @@ -549,7 +588,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover fs_root = self.staging_area.destdir(self.artifact.source) self.unpack_strata(fs_root) self.write_metadata(fs_root, rootfs_name) - self.create_fstab(fs_root) + self.run_system_integration_commands(fs_root) self.copy_kernel_into_artifact_cache(fs_root) unslashy_root = fs_root[1:] def uproot_info(info): @@ -649,29 +688,50 @@ class SystemBuilder(BuilderBase): # pragma: no cover os.chmod(os_release_file, 0644) - def create_fstab(self, path): - '''Create an /etc/fstab inside a system tree. + def run_system_integration_commands(self, rootdir): # pragma: no cover + ''' Run the system integration commands ''' - The fstab is created using assumptions of the disk layout. - If the assumptions are wrong, extend this code so it can deal - with other cases. + sys_integration_dir = os.path.join(rootdir, SYSTEM_INTEGRATION_PATH) + if not os.path.isdir(sys_integration_dir): + return - ''' + env = { + 'PATH': '/bin:/usr/bin:/sbin:/usr/sbin' + } - self.app.status(msg='Creating fstab in %(path)s', - path=path, chatty=True) - with self.build_watch('create-fstab'): - fstab = os.path.join(path, 'etc', 'fstab') - if not os.path.exists(fstab): - # FIXME: should exist - if not os.path.exists(os.path.dirname(fstab)): - os.makedirs(os.path.dirname(fstab)) - # We create an empty fstab: systemd does not require - # /sys and /proc entries, and we can't know what the - # right entry for / is. The fstab gets built during - # deployment instead, when that information is available. - with open(fstab, 'w'): - pass + self.app.status(msg='Running the system integration commands', + error=True) + + mounted = [] + to_mount = ( + ('proc', 'proc', 'none'), + ('dev/shm', 'tmpfs', 'none'), + ) + + try: + for mount_point, mount_type, source in to_mount: + logging.debug('Mounting %s in system root filesystem' + % mount_point) + path = os.path.join(rootdir, mount_point) + if not os.path.exists(path): + os.makedirs(path) + morphlib.fsutils.mount(self.app.runcmd, source, path, + mount_type) + mounted.append(path) + + self.app.runcmd(['chroot', rootdir, 'sh', '-c', + 'cd / && run-parts "$1"', '-', SYSTEM_INTEGRATION_PATH], + env=env) + except BaseException, e: + self.app.status( + msg='Error while running system integration commands', + error=True) + raise + finally: + for mount_path in reversed(mounted): + logging.debug('Unmounting %s in system root filesystem' + % mount_path) + morphlib.fsutils.unmount(self.app.runcmd, mount_path) def copy_kernel_into_artifact_cache(self, path): '''Copy the installed kernel image into the local artifact cache. diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index bb536f82..3efe1cbb 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -114,6 +114,6 @@ class CacheKeyComputer(object): if kind == 'stratum': keys['stratum-format-version'] = 1 elif kind == 'system': - keys['system-compatibility-version'] = "1~ (temporary, root rw)" + keys['system-compatibility-version'] = "2~ (upgradable, root rw)" return keys diff --git a/morphlib/exts/kvm.check b/morphlib/exts/kvm.check new file mode 100755 index 00000000..be7c51c2 --- /dev/null +++ b/morphlib/exts/kvm.check @@ -0,0 +1,35 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''Preparatory checks for Morph 'kvm' write extension''' + +import cliapp + +import morphlib.writeexts + + +class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + upgrade = self.get_environment_boolean('UPGRADE') + if upgrade: + raise cliapp.AppException( + 'Use the `ssh-rsync` write extension to deploy upgrades to an ' + 'existing remote system.') + +KvmPlusSshCheckExtension().run() diff --git a/morphlib/exts/kvm.write b/morphlib/exts/kvm.write index 4f877c22..94560972 100755 --- a/morphlib/exts/kvm.write +++ b/morphlib/exts/kvm.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -56,7 +56,7 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): temp_root, location = args ssh_host, vm_name, vm_path = self.parse_location(location) - autostart = self.parse_autostart() + autostart = self.get_environment_boolean('AUTOSTART') fd, raw_disk = tempfile.mkstemp() os.close(fd) diff --git a/morphlib/exts/nfsboot.check b/morphlib/exts/nfsboot.check new file mode 100755 index 00000000..f84f187f --- /dev/null +++ b/morphlib/exts/nfsboot.check @@ -0,0 +1,101 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''Preparatory checks for Morph 'nfsboot' write extension''' + +import cliapp +import os + +import morphlib.writeexts + + +class NFSBootCheckExtension(morphlib.writeexts.WriteExtension): + + _nfsboot_root = '/srv/nfsboot' + + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + location = args[0] + + upgrade = self.get_environment_boolean('UPGRADE') + if upgrade: + raise cliapp.AppException( + 'Upgrading is not currently supported for NFS deployments.') + + hostname = os.environ.get('HOSTNAME', None) + if hostname is None: + raise cliapp.AppException('You must specify a HOSTNAME.') + if hostname == 'baserock': + raise cliapp.AppException('It is forbidden to nfsboot a system ' + 'with hostname "%s"' % hostname) + + self.test_good_server(location) + + version_label = os.getenv('VERSION_LABEL', 'factory') + versioned_root = os.path.join(self._nfsboot_root, hostname, 'systems', + version_label) + if self.version_exists(versioned_root, location): + raise cliapp.AppException( + 'Root file system for host %s (version %s) already exists on ' + 'the NFS server %s. Deployment aborted.' % (hostname, + version_label, location)) + + def test_good_server(self, server): + # Can be ssh'ed into + try: + cliapp.ssh_runcmd('root@%s' % server, ['true']) + except cliapp.AppException: + raise cliapp.AppException('You are unable to ssh into server %s' + % server) + + # Is an NFS server + try: + cliapp.ssh_runcmd( + 'root@%s' % server, ['test', '-e', '/etc/exports']) + except cliapp.AppException: + raise cliapp.AppException('server %s is not an nfs server' + % server) + try: + cliapp.ssh_runcmd( + 'root@%s' % server, ['systemctl', 'is-enabled', + 'nfs-server.service']) + + except cliapp.AppException: + raise cliapp.AppException('server %s does not control its ' + 'nfs server by systemd' % server) + + # TFTP server exports /srv/nfsboot/tftp + tftp_root = os.path.join(self._nfsboot_root, 'tftp') + try: + cliapp.ssh_runcmd( + 'root@%s' % server, ['test' , '-d', tftp_root]) + except cliapp.AppException: + raise cliapp.AppException('server %s does not export %s' % + (tftp_root, server)) + + def version_exists(self, versioned_root, location): + try: + cliapp.ssh_runcmd('root@%s' % location, + ['test', '-d', versioned_root]) + except cliapp.AppException: + return False + + return True + + +NFSBootCheckExtension().run() diff --git a/morphlib/exts/nfsboot.configure b/morphlib/exts/nfsboot.configure index 8dc6c67c..660d9c39 100755 --- a/morphlib/exts/nfsboot.configure +++ b/morphlib/exts/nfsboot.configure @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,7 +15,9 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# Remove all networking interfaces and stop fstab from mounting '/' +# Remove all networking interfaces. On nfsboot systems, eth0 is set up +# during kernel init, and the normal ifup@eth0.service systemd unit +# would break the NFS connection and cause the system to hang. set -e @@ -26,7 +28,4 @@ auto lo iface lo inet loopback EOF - # Stop fstab from mounting '/' - mv "$1/etc/fstab" "$1/etc/fstab.old" - awk '/^ *#/ || $2 != "/"' "$1/etc/fstab.old" > "$1/etc/fstab" fi diff --git a/morphlib/exts/nfsboot.write b/morphlib/exts/nfsboot.write index 34a72972..8d3d6df7 100755 --- a/morphlib/exts/nfsboot.write +++ b/morphlib/exts/nfsboot.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -60,38 +60,18 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): raise cliapp.AppException('Wrong number of command line args') temp_root, location = args - hostname = self.get_hostname(temp_root) - if hostname == 'baserock': - raise cliapp.AppException('It is forbidden to nfsboot a system ' - 'with hostname "baserock"') - self.test_good_server(location) version_label = os.getenv('VERSION_LABEL', 'factory') + hostname = os.environ['HOSTNAME'] + versioned_root = os.path.join(self._nfsboot_root, hostname, 'systems', version_label) - if self.version_exists(versioned_root, location): - raise cliapp.AppException('Version %s already exists on' - ' this device. Deployment aborted' - % version_label) + self.copy_rootfs(temp_root, location, versioned_root, hostname) self.copy_kernel(temp_root, location, versioned_root, version_label, hostname) self.configure_nfs(location, hostname) - def version_exists(self, versioned_root, location): - try: - cliapp.ssh_runcmd('root@%s' % location, - ['test', '-d', versioned_root]) - except cliapp.AppException: - return False - - return True - - def get_hostname(self, temp_root): - hostnamepath = os.path.join(temp_root, 'etc', 'hostname') - with open(hostnamepath) as f: - return f.readline().strip() - def create_local_state(self, location, hostname): statedir = os.path.join(self._nfsboot_root, hostname, 'state') subdirs = [os.path.join(statedir, 'home'), @@ -209,37 +189,6 @@ mv "$temp" "$target" 'root@%s' % location, ['systemctl', 'restart', 'nfs-server.service']) - def test_good_server(self, server): - # Can be ssh'ed into - try: - cliapp.ssh_runcmd('root@%s' % server, ['true']) - except cliapp.AppException: - raise cliapp.AppException('You are unable to ssh into server %s' - % server) - - # Is an NFS server - try: - cliapp.ssh_runcmd( - 'root@%s' % server, ['test', '-e', '/etc/exports']) - except cliapp.AppException: - raise cliapp.AppException('server %s is not an nfs server' - % server) - try: - cliapp.ssh_runcmd( - 'root@%s' % server, ['systemctl', 'is-enabled', - 'nfs-server.service']) - - except cliapp.AppException: - raise cliapp.AppException('server %s does not control its ' - 'nfs server by systemd' % server) - - # TFTP server exports /srv/nfsboot/tftp - try: - cliapp.ssh_runcmd( - 'root@%s' % server, ['test' , '-d', '/srv/nfsboot/tftp']) - except cliapp.AppException: - raise cliapp.AppException('server %s does not export ' - '/srv/nfsboot/tftp' % server) NFSBootWriteExtension().run() diff --git a/morphlib/exts/openstack.check b/morphlib/exts/openstack.check new file mode 100755 index 00000000..a9a8fe1b --- /dev/null +++ b/morphlib/exts/openstack.check @@ -0,0 +1,35 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''Preparatory checks for Morph 'openstack' write extension''' + +import cliapp + +import morphlib.writeexts + + +class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + upgrade = self.get_environment_boolean('UPGRADE') + if upgrade: + raise cliapp.AppException( + 'Use the `ssh-rsync` write extension to deploy upgrades to an ' + 'existing remote system.') + +OpenStackCheckExtension().run() diff --git a/morphlib/exts/rawdisk.write b/morphlib/exts/rawdisk.write index 8723ac0c..87edf7bf 100755 --- a/morphlib/exts/rawdisk.write +++ b/morphlib/exts/rawdisk.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -50,12 +50,15 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): self.create_local_system(temp_root, location) self.status(msg='Disk image has been created at %s' % location) except Exception: - os.remove(location) self.status(msg='Failure to create disk image at %s' % location) + if os.path.exists(location): + os.remove(location) raise def upgrade_local_system(self, raw_disk, temp_root): + self.complete_fstab_for_btrfs_layout(temp_root) + mp = self.mount(raw_disk) version_label = self.get_version_label(mp) diff --git a/morphlib/exts/ssh-rsync.check b/morphlib/exts/ssh-rsync.check new file mode 100755 index 00000000..90029cb4 --- /dev/null +++ b/morphlib/exts/ssh-rsync.check @@ -0,0 +1,36 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''Preparatory checks for Morph 'ssh-rsync' write extension''' + +import cliapp + +import morphlib.writeexts + + +class SshRsyncCheckExtension(morphlib.writeexts.WriteExtension): + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + upgrade = self.get_environment_boolean('UPGRADE') + if not upgrade: + raise cliapp.AppException( + 'The ssh-rsync write is for upgrading existing remote ' + 'Baserock machines. It cannot be used for an initial ' + 'deployment.') + +SshRsyncCheckExtension().run() diff --git a/morphlib/exts/ssh-rsync.write b/morphlib/exts/ssh-rsync.write index 211dbe5e..509520ae 100755 --- a/morphlib/exts/ssh-rsync.write +++ b/morphlib/exts/ssh-rsync.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,6 +26,14 @@ import tempfile import morphlib.writeexts + +def ssh_runcmd_ignore_failure(location, command, **kwargs): + try: + return cliapp.ssh_runcmd(location, command, **kwargs) + except cliapp.AppException: + pass + + class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): '''Upgrade a running baserock system with ssh and rsync. @@ -47,8 +55,11 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.upgrade_remote_system(location, temp_root) def upgrade_remote_system(self, location, temp_root): + self.complete_fstab_for_btrfs_layout(temp_root) + root_disk = self.find_root_disk(location) version_label = os.environ.get('VERSION_LABEL') + autostart = self.get_environment_boolean('AUTOSTART') self.status(msg='Creating remote mount point') remote_mnt = cliapp.ssh_runcmd(location, ['mktemp', '-d']).strip() @@ -56,15 +67,11 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Mounting root disk') cliapp.ssh_runcmd(location, ['mount', root_disk, remote_mnt]) except Exception as e: - try: - cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) - except: - pass + ssh_runcmd_ignore_failure(location, ['rmdir', remote_mnt]) raise e try: version_root = os.path.join(remote_mnt, 'systems', version_label) - run_dir = os.path.join(version_root, 'run') orig_dir = os.path.join(version_root, 'orig') self.status(msg='Creating %s' % version_root) @@ -73,80 +80,40 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.create_remote_orig(location, version_root, remote_mnt, temp_root) - self.status(msg='Creating "run" subvolume') - cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', - 'snapshot', orig_dir, run_dir]) - - self.status(msg='Updating system configuration') - bscs_loc = os.path.join(run_dir, 'usr', 'bin', - 'baserock-system-config-sync') - - output = cliapp.ssh_runcmd(location, ['sh', '-c', - '"$1" merge "$2" &> /dev/null || echo -n cmdfailed', - '-', bscs_loc, version_label]) - if output == "cmdfailed": - self.status(msg='Updating system configuration failed') - - self.install_remote_kernel(location, version_root, temp_root) - default_path = os.path.join(remote_mnt, 'systems', 'default') - if self.bootloader_is_wanted(): - output = cliapp.ssh_runcmd(location, ['sh', '-c', - 'test -e "$1" && stat -c %F "$1" ' - '|| echo missing file', - '-', default_path]) - if output != "symbolic link": - # we are upgrading and old system that does - # not have an updated extlinux config file - self.update_remote_extlinux(location, remote_mnt, - version_label) - cliapp.ssh_runcmd(location, ['ln', '-sfn', version_label, - default_path]) + # Use the system-version-manager from the new system we just + # installed, so that we can upgrade from systems that don't have + # it installed. + self.status(msg='Calling system-version-manager to deploy upgrade') + deployment = os.path.join('/systems', version_label, 'orig') + system_config_sync = os.path.join( + remote_mnt, 'systems', version_label, 'orig', 'usr', 'bin', + 'baserock-system-config-sync') + system_version_manager = os.path.join( + remote_mnt, 'systems', version_label, 'orig', 'usr', 'bin', + 'system-version-manager') + cliapp.ssh_runcmd(location, + ['env', 'BASEROCK_SYSTEM_CONFIG_SYNC='+system_config_sync, + system_version_manager, 'deploy', deployment]) + + self.status(msg='Setting %s as the new default system' % + version_label) + cliapp.ssh_runcmd(location, + [system_version_manager, 'set-default', version_label]) except Exception as e: - try: - cliapp.ssh_runcmd(location, - ['btrfs', 'subvolume', 'delete', run_dir]) - except: - pass - try: - cliapp.ssh_runcmd(location, - ['btrfs', 'subvolume', 'delete', orig_dir]) - except: - pass - try: - cliapp.ssh_runcmd(location, ['rm', '-rf', version_root]) - except: - pass + self.status(msg='Deployment failed') + ssh_runcmd_ignore_failure( + location, ['btrfs', 'subvolume', 'delete', orig_dir]) + ssh_runcmd_ignore_failure( + location, ['rm', '-rf', version_root]) raise e finally: self.status(msg='Removing temporary mounts') cliapp.ssh_runcmd(location, ['umount', remote_mnt]) cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) - def update_remote_extlinux(self, location, remote_mnt, version_label): - '''Install/reconfigure extlinux on location''' - - self.status(msg='Creating extlinux.conf') - config = os.path.join(remote_mnt, 'extlinux.conf') - temp_fd, temp_path = tempfile.mkstemp() - with os.fdopen(temp_fd, 'w') as f: - f.write('default linux\n') - f.write('timeout 1\n') - f.write('label linux\n') - f.write('kernel /systems/default/kernel\n') - f.write('append root=/dev/sda ' - 'rootflags=subvol=systems/default/run ' - 'init=/sbin/init rw\n') - - try: - cliapp.runcmd(['rsync', '-as', temp_path, - '%s:%s~' % (location, config)]) - cliapp.ssh_runcmd(location, ['mv', config+'~', config]) - except Exception as e: - try: - cliapp.ssh_runcmd(location, ['rm', '-f', config+'~']) - except: - pass - raise e + if autostart: + self.status(msg="Rebooting into new system ...") + ssh_runcmd_ignore_failure(location, ['reboot']) def create_remote_orig(self, location, version_root, remote_mnt, temp_root): @@ -178,18 +145,6 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): if (line_words[1] == '/' and line_words[0] != 'rootfs'): return line_words[0] - def install_remote_kernel(self, location, version_root, temp_root): - '''Install the kernel in temp_root inside version_root on location''' - - self.status(msg='Installing kernel') - image_names = ('vmlinuz', 'zImage', 'uImage') - kernel_dest = os.path.join(version_root, 'kernel') - for name in image_names: - try_path = os.path.join(temp_root, 'boot', name) - if os.path.exists(try_path): - cliapp.runcmd(['rsync', '-as', try_path, - '%s:%s' % (location, kernel_dest)]) - def check_valid_target(self, location): try: cliapp.ssh_runcmd(location, ['true']) @@ -203,10 +158,17 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): raise cliapp.AppException('%s is not a baserock system' % location) - output = cliapp.ssh_runcmd(location, ['sh', '-c', - 'type rsync &> /dev/null || echo -n cmdnotfound']) - if output == 'cmdnotfound': - raise cliapp.AppException('%s does not have rsync' - % location) + def check_command_exists(command): + test = 'type %s > /dev/null 2>&1 || echo -n cmdnotfound' % command + output = cliapp.ssh_runcmd(location, ['sh', '-c', test]) + if output == 'cmdnotfound': + raise cliapp.AppException( + "%s does not have %s" % (location, command)) + + # The deploy requires baserock-system-config-sync and + # system-version-manager in the new system only. The old system doesn't + # need to have them at all. + check_command_exists('rsync') + SshRsyncWriteExtension().run() diff --git a/morphlib/exts/sysroot.write b/morphlib/exts/sysroot.write new file mode 100755 index 00000000..1ae4864f --- /dev/null +++ b/morphlib/exts/sysroot.write @@ -0,0 +1,29 @@ +#!/bin/sh +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# A Morph write extension to deploy to another directory + +set -eu + +# Ensure the target is an empty directory +mkdir -p "$2" +find "$2" -mindepth 1 -delete + +# Move the contents of our source directory to our target +# Previously we would (cd "$1" && find -print0 | cpio -0pumd "$absolute_path") +# to do this, but the source directory is disposable anyway, so we can move +# its contents to save time +find "$1" -maxdepth 1 -mindepth 1 -exec mv {} "$2/." + diff --git a/morphlib/exts/tar.check b/morphlib/exts/tar.check new file mode 100755 index 00000000..cbeaf163 --- /dev/null +++ b/morphlib/exts/tar.check @@ -0,0 +1,24 @@ +#!/bin/sh +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# Preparatory checks for Morph 'tar' write extension + +set -eu + +if [ "$UPGRADE" == "yes" ]; then + echo >&2 "ERROR: Cannot upgrade a tar file deployment." + exit 1 +fi diff --git a/morphlib/exts/virtualbox-ssh.check b/morphlib/exts/virtualbox-ssh.check new file mode 100755 index 00000000..1aeb8999 --- /dev/null +++ b/morphlib/exts/virtualbox-ssh.check @@ -0,0 +1,35 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''Preparatory checks for Morph 'virtualbox-ssh' write extension''' + +import cliapp + +import morphlib.writeexts + + +class VirtualBoxPlusSshCheckExtension(morphlib.writeexts.WriteExtension): + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + upgrade = self.get_environment_boolean('UPGRADE') + if upgrade: + raise cliapp.AppException( + 'Use the `ssh-rsync` write extension to deploy upgrades to an ' + 'existing remote system.') + +VirtualBoxPlusSshCheckExtension().run() diff --git a/morphlib/exts/virtualbox-ssh.write b/morphlib/exts/virtualbox-ssh.write index 204b2447..2a2f3f7b 100755 --- a/morphlib/exts/virtualbox-ssh.write +++ b/morphlib/exts/virtualbox-ssh.write @@ -62,7 +62,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): temp_root, location = args ssh_host, vm_name, vdi_path = self.parse_location(location) - autostart = self.parse_autostart() + autostart = self.get_environment_boolean('AUTOSTART') fd, raw_disk = tempfile.mkstemp() os.close(fd) diff --git a/morphlib/fsutils.py b/morphlib/fsutils.py index 0212b987..751f73f6 100644 --- a/morphlib/fsutils.py +++ b/morphlib/fsutils.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -47,10 +47,14 @@ def create_fs(runcmd, partition): # pragma: no cover runcmd(['mkfs.btrfs', '-L', 'baserock', partition]) -def mount(runcmd, partition, mount_point): # pragma: no cover +def mount(runcmd, partition, mount_point, fstype=None): # pragma: no cover if not os.path.exists(mount_point): os.mkdir(mount_point) - runcmd(['mount', partition, mount_point]) + if not fstype: + fstype = [] + else: + fstype = ['-t', fstype] + runcmd(['mount', partition, mount_point] + fstype) def unmount(runcmd, mount_point): # pragma: no cover diff --git a/morphlib/git.py b/morphlib/git.py index 27146206..ccd06323 100644 --- a/morphlib/git.py +++ b/morphlib/git.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -279,6 +279,10 @@ def copy_repository(runcmd, repo, destdir, is_mirror=True): def checkout_ref(runcmd, gitdir, ref): '''Checks out a specific ref/SHA1 in a git working tree.''' runcmd(['git', 'checkout', ref], cwd=gitdir) + gd = morphlib.gitdir.GitDirectory(gitdir) + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() def index_has_changes(runcmd, gitdir): @@ -308,6 +312,10 @@ def clone_into(runcmd, srcpath, targetpath, ref=None): runcmd(['git', 'checkout', ref], cwd=targetpath) else: runcmd(['git', 'clone', '-b', ref, srcpath, targetpath]) + gd = morphlib.gitdir.GitDirectory(targetpath) + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() def is_valid_sha1(ref): '''Checks whether a string is a valid SHA1.''' diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py index be2137b2..3d0ab53e 100644 --- a/morphlib/gitdir.py +++ b/morphlib/gitdir.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -317,6 +317,14 @@ class Remote(object): self._parse_push_output(out), err) return self._parse_push_output(out) + def pull(self, branch=None): # pragma: no cover + if branch: + repo = self.get_fetch_url() + ret = self.gd._runcmd(['git', 'pull', repo, branch]) + else: + ret = self.gd._runcmd(['git', 'pull']) + return ret + class GitDirectory(object): @@ -330,7 +338,11 @@ class GitDirectory(object): ''' def __init__(self, dirname): - self.dirname = dirname + self.dirname = morphlib.util.find_root(dirname, '.git') + # if we are in a bare repo, self.dirname will now be None + # so we just use the provided dirname + if not self.dirname: + self.dirname = dirname def _runcmd(self, argv, **kwargs): '''Run a command at the root of the git directory. @@ -350,6 +362,9 @@ class GitDirectory(object): def checkout(self, branch_name): # pragma: no cover '''Check out a git branch.''' self._runcmd(['git', 'checkout', branch_name]) + if self.has_fat(): + self.fat_init() + self.fat_pull() def branch(self, new_branch_name, base_ref): # pragma: no cover '''Create a git branch based on an existing ref. @@ -478,7 +493,8 @@ class GitDirectory(object): if dirpath == self.dirname and '.git' in subdirs: subdirs.remove('.git') for filename in filenames: - yield os.path.join(dirpath, filename)[len(self.dirname)+1:] + filepath = os.path.join(dirpath, filename) + yield os.path.relpath(filepath, start=self.dirname) def _list_files_in_ref(self, ref): tree = self.resolve_ref_to_tree(ref) @@ -610,12 +626,35 @@ class GitDirectory(object): except Exception, e: raise RefDeleteError(self, ref, old_sha1, e) + def describe(self): + version = self._runcmd( + ['git', 'describe', '--always', '--dirty=-unreproducible']) + return version.strip() + + def fat_init(self): # pragma: no cover + return self._runcmd(['git', 'fat', 'init']) + + def fat_push(self): # pragma: no cover + return self._runcmd(['git', 'fat', 'push']) + + def fat_pull(self): # pragma: no cover + return self._runcmd(['git', 'fat', 'pull']) + + def has_fat(self): # pragma: no cover + return os.path.isfile(self.join_path('.gitfat')) + + def join_path(self, path): # pragma: no cover + return os.path.join(self.dirname, path) + + def get_relpath(self, path): # pragma: no cover + return os.path.relpath(path, self.dirname) + def init(dirname): '''Initialise a new git repository.''' + cliapp.runcmd(['git', 'init'], cwd=dirname) gd = GitDirectory(dirname) - gd._runcmd(['git', 'init']) return gd diff --git a/morphlib/gitdir_tests.py b/morphlib/gitdir_tests.py index 21a6b5b8..14b2a57a 100644 --- a/morphlib/gitdir_tests.py +++ b/morphlib/gitdir_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -82,9 +82,13 @@ class GitDirectoryContentsTests(unittest.TestCase): shutil.rmtree(self.tempdir) def test_lists_files_in_work_tree(self): + expected = ['bar.morph', 'baz.morph', 'foo.morph', 'quux'] + gd = morphlib.gitdir.GitDirectory(self.dirname) - self.assertEqual(sorted(gd.list_files()), - ['bar.morph', 'baz.morph', 'foo.morph', 'quux']) + self.assertEqual(sorted(gd.list_files()), expected) + + gd = morphlib.gitdir.GitDirectory(self.dirname + '/') + self.assertEqual(sorted(gd.list_files()), expected) def test_read_file_in_work_tree(self): gd = morphlib.gitdir.GitDirectory(self.dirname) @@ -202,6 +206,15 @@ class GitDirectoryContentsTests(unittest.TestCase): ) self.assertEqual(expected, gd.get_commit_contents(commit).split('\n')) + def test_describe(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + + gd._runcmd(['git', 'tag', '-a', '-m', 'Example', 'example', 'HEAD']) + self.assertEqual(gd.describe(), 'example-unreproducible') + + gd._runcmd(['git', 'reset', '--hard']) + self.assertEqual(gd.describe(), 'example') + class GitDirectoryRefTwiddlingTests(unittest.TestCase): diff --git a/morphlib/plugins/add_binary_plugin.py b/morphlib/plugins/add_binary_plugin.py new file mode 100644 index 00000000..1edae0e8 --- /dev/null +++ b/morphlib/plugins/add_binary_plugin.py @@ -0,0 +1,110 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import logging +import os +import urlparse + +import morphlib + + +class AddBinaryPlugin(cliapp.Plugin): + + '''Add a subcommand for dealing with large binary files.''' + + def enable(self): + self.app.add_subcommand( + 'add-binary', self.add_binary, arg_synopsis='FILENAME...') + + def disable(self): + pass + + def add_binary(self, binaries): + '''Add a binary file to the current repository. + + Command line argument: + + * `FILENAME...` is the binaries to be added to the repository. + + This checks for the existence of a .gitfat file in the repository. If + there is one then a line is added to .gitattributes telling it that + the given binary should be handled by git-fat. If there is no .gitfat + file then it is created, with the rsync remote pointing at the correct + directory on the Trove host. A line is then added to .gitattributes to + say that the given binary should be handled by git-fat. + + Example: + + morph add-binary big_binary.tar.gz + + ''' + if not binaries: + raise morphlib.Error('add-binary must get at least one argument') + + gd = morphlib.gitdir.GitDirectory(os.getcwd()) + gd.fat_init() + if not gd.has_fat(): + self._make_gitfat(gd) + self._handle_binaries(binaries, gd) + logging.info('Staged binaries for commit') + + def _handle_binaries(self, binaries, gd): + '''Add a filter for the given file, and then add it to the repo.''' + # begin by ensuring all paths given are relative to the root directory + files = [gd.get_relpath(os.path.realpath(binary)) + for binary in binaries] + + # now add any files that aren't already mentioned in .gitattributes to + # the file so that git fat knows what to do + attr_path = gd.join_path('.gitattributes') + if '.gitattributes' in gd.list_files(): + with open(attr_path, 'r') as attributes: + current = set(f.split()[0] for f in attributes) + else: + current = set() + to_add = set(files) - current + + # if we don't need to change .gitattributes then we can just do + # `git add <binaries>` + if not to_add: + gd.get_index().add_files_from_working_tree(files) + return + + with open(attr_path, 'a') as attributes: + for path in to_add: + attributes.write('%s filter=fat -crlf\n' % path) + + # we changed .gitattributes, so need to stage it for committing + files.append(attr_path) + gd.get_index().add_files_from_working_tree(files) + + def _make_gitfat(self, gd): + '''Make .gitfat point to the rsync directory for the repo.''' + remote = gd.get_remote('origin') + if not remote.get_push_url(): + raise Exception( + 'Remote `origin` does not have a push URL defined.') + url = urlparse.urlparse(remote.get_push_url()) + if url.scheme != 'ssh': + raise Exception( + 'Push URL for `origin` is not an SSH URL: %s' % url.geturl()) + fat_store = '%s:%s' % (url.netloc, url.path) + fat_path = gd.join_path('.gitfat') + with open(fat_path, 'w+') as gitfat: + gitfat.write('[rsync]\n') + gitfat.write('remote = %s' % fat_store) + gd.get_index().add_files_from_working_tree([fat_path]) diff --git a/morphlib/plugins/branch_and_merge_new_plugin.py b/morphlib/plugins/branch_and_merge_new_plugin.py index 94b2381c..51cba401 100644 --- a/morphlib/plugins/branch_and_merge_new_plugin.py +++ b/morphlib/plugins/branch_and_merge_new_plugin.py @@ -190,8 +190,12 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): with self._initializing_system_branch( ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd): + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() + if not self._checkout_has_systems(gd): - raise BranchRootHasNoSystemsError(base_ref) + raise BranchRootHasNoSystemsError(root_url, base_ref) def branch(self, args): @@ -250,9 +254,12 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): gd.branch(system_branch, base_ref) gd.checkout(system_branch) + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() if not self._checkout_has_systems(gd): - raise BranchRootHasNoSystemsError(base_ref) + raise BranchRootHasNoSystemsError(root_url, base_ref) def _save_dirty_morphologies(self, loader, sb, morphs): logging.debug('Saving dirty morphologies: start') @@ -480,6 +487,9 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): gd.checkout(sb.system_branch_name) gd.update_submodules(self.app) gd.update_remotes() + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() # Change the refs to the chunk. if chunk_ref != sb.system_branch_name: diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 90c658a0..ae62b75d 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -16,6 +16,7 @@ import cliapp import contextlib +import json import os import shutil import stat @@ -25,23 +26,24 @@ import uuid import morphlib -# UGLY HACK: We need to re-use some code from the branch and merge -# plugin, so we import and instantiate that plugin. This needs to -# be fixed by refactoring the codebase so the shared code is in -# morphlib, not in a plugin. However, this hack lets us re-use -# code without copying it. -import morphlib.plugins.branch_and_merge_plugin + +class ExtensionNotFoundError(morphlib.Error): + pass class DeployPlugin(cliapp.Plugin): def enable(self): + group_deploy = 'Deploy Options' + self.app.settings.boolean(['upgrade'], + 'specify that you want to upgrade an ' + 'existing cluster of systems rather than do ' + 'an initial deployment', + group=group_deploy) + self.app.add_subcommand( 'deploy', self.deploy, arg_synopsis='CLUSTER [SYSTEM.KEY=VALUE]') - self.other = \ - morphlib.plugins.branch_and_merge_plugin.BranchAndMergePlugin() - self.other.app = self.app def disable(self): pass @@ -250,8 +252,20 @@ class DeployPlugin(cliapp.Plugin): are set as environment variables when either the configuration or the write extension runs (except `type` and `location`). + Deployment configuration is stored in the deployed system as + /baserock/deployment.meta. THIS CONTAINS ALL ENVIRONMENT VARIABLES SET + DURINGR DEPLOYMENT, so make sure you have no sensitive information in + your environment that is being leaked. As a special case, any + environment/deployment variable that contains 'PASSWORD' in its name is + stripped out and not stored in the final system. + ''' + # Nasty hack to allow deploying things of a different architecture + def validate(self, root_artifact): + pass + morphlib.buildcommand.BuildCommand._validate_architecture = validate + if not args: raise cliapp.AppException( 'Too few arguments to deploy command (see help)') @@ -318,65 +332,112 @@ class DeployPlugin(cliapp.Plugin): ref=build_ref, dirname=gd.dirname, remote=remote.get_push_url(), chatty=True) - for system in cluster_morphology['systems']: - self.deploy_system(build_command, root_repo_dir, - bb.root_repo_url, bb.root_ref, - system, env_vars) - - def deploy_system(self, build_command, root_repo_dir, build_repo, ref, - system, env_vars): - # Find the artifact to build - morph = system['morph'] - srcpool = build_command.create_source_pool(build_repo, ref, - morph + '.morph') - def validate(self, root_artifact): - pass - morphlib.buildcommand.BuildCommand._validate_architecture = validate + # Create a tempdir for this deployment to work in + deploy_tempdir = tempfile.mkdtemp( + dir=os.path.join(self.app.settings['tempdir'], 'deployments')) + try: + for system in cluster_morphology['systems']: + self.deploy_system(build_command, deploy_tempdir, + root_repo_dir, bb.root_repo_url, + bb.root_ref, system, env_vars, + parent_location='') + finally: + shutil.rmtree(deploy_tempdir) + + self.app.status(msg='Finished deployment') - artifact = build_command.resolve_artifacts(srcpool) - - deploy_defaults = system['deploy-defaults'] - deployments = system['deploy'] - for system_id, deploy_params in deployments.iteritems(): - user_env = morphlib.util.parse_environment_pairs( - os.environ, - [pair[len(system_id)+1:] - for pair in env_vars - if pair.startswith(system_id)]) - - final_env = dict(deploy_defaults.items() + - deploy_params.items() + - user_env.items()) - - deployment_type = final_env.pop('type', None) - if not deployment_type: - raise morphlib.Error('"type" is undefined ' - 'for system "%s"' % system_id) - - location = final_env.pop('location', None) - if not location: - raise morphlib.Error('"location" is undefined ' - 'for system "%s"' % system_id) - - morphlib.util.sanitize_environment(final_env) - self.do_deploy(build_command, root_repo_dir, ref, artifact, - deployment_type, location, final_env) - - def do_deploy(self, build_command, root_repo_dir, ref, artifact, - deployment_type, location, env): - - # Create a tempdir for this deployment to work in - deploy_tempdir = tempfile.mkdtemp( - dir=os.path.join(self.app.settings['tempdir'], 'deployments')) + def deploy_system(self, build_command, deploy_tempdir, + root_repo_dir, build_repo, ref, system, env_vars, + parent_location): + old_status_prefix = self.app.status_prefix + system_status_prefix = '%s[%s]' % (old_status_prefix, system['morph']) + self.app.status_prefix = system_status_prefix try: - # Create a tempdir to extract the rootfs in - system_tree = tempfile.mkdtemp(dir=deploy_tempdir) + # Find the artifact to build + morph = system['morph'] + srcpool = build_command.create_source_pool(build_repo, ref, + morph + '.morph') + + artifact = build_command.resolve_artifacts(srcpool) + + deploy_defaults = system.get('deploy-defaults', {}) + deployments = system['deploy'] + for system_id, deploy_params in deployments.iteritems(): + deployment_status_prefix = '%s[%s]' % ( + system_status_prefix, system_id) + self.app.status_prefix = deployment_status_prefix + try: + user_env = morphlib.util.parse_environment_pairs( + os.environ, + [pair[len(system_id)+1:] + for pair in env_vars + if pair.startswith(system_id)]) + + final_env = dict(deploy_defaults.items() + + deploy_params.items() + + user_env.items()) + + is_upgrade = ('yes' if self.app.settings['upgrade'] + else 'no') + final_env['UPGRADE'] = is_upgrade + + deployment_type = final_env.pop('type', None) + if not deployment_type: + raise morphlib.Error('"type" is undefined ' + 'for system "%s"' % system_id) + + location = final_env.pop('location', None) + if not location: + raise morphlib.Error('"location" is undefined ' + 'for system "%s"' % system_id) + + morphlib.util.sanitize_environment(final_env) + self.check_deploy(root_repo_dir, ref, deployment_type, + location, final_env) + system_tree = self.setup_deploy(build_command, + deploy_tempdir, + root_repo_dir, + ref, artifact, + deployment_type, + location, final_env) + for subsystem in system.get('subsystems', []): + self.deploy_system(build_command, deploy_tempdir, + root_repo_dir, build_repo, + ref, subsystem, env_vars, + parent_location=system_tree) + if parent_location: + deploy_location = os.path.join(parent_location, + location.lstrip('/')) + else: + deploy_location = location + self.run_deploy_commands(deploy_tempdir, final_env, + artifact, root_repo_dir, + ref, deployment_type, + system_tree, deploy_location) + finally: + self.app.status_prefix = system_status_prefix + finally: + self.app.status_prefix = old_status_prefix - # Extensions get a private tempdir so we can more easily clean - # up any files an extension left behind - deploy_private_tempdir = tempfile.mkdtemp(dir=deploy_tempdir) - env['TMPDIR'] = deploy_private_tempdir + def check_deploy(self, root_repo_dir, ref, deployment_type, location, env): + # Run optional write check extension. These are separate from the write + # extension because it may be several minutes before the write + # extension itself has the chance to raise an error. + try: + self._run_extension( + root_repo_dir, ref, deployment_type, '.check', + [location], env) + except ExtensionNotFoundError: + pass + + def setup_deploy(self, build_command, deploy_tempdir, root_repo_dir, ref, + artifact, deployment_type, location, env): + # deployment_type, location and env are only used for saving metadata + # Create a tempdir to extract the rootfs in + system_tree = tempfile.mkdtemp(dir=deploy_tempdir) + + try: # Unpack the artifact (tarball) to a temporary directory. self.app.status(msg='Unpacking system for configuration') @@ -397,7 +458,27 @@ class DeployPlugin(cliapp.Plugin): msg='System unpacked at %(system_tree)s', system_tree=system_tree) + self.app.status( + msg='Writing deployment metadata file') + metadata = self.create_metadata( + artifact, root_repo_dir, deployment_type, location, env) + metadata_path = os.path.join( + system_tree, 'baserock', 'deployment.meta') + with morphlib.savefile.SaveFile(metadata_path, 'w') as f: + f.write(json.dumps(metadata, indent=4, sort_keys=True)) + return system_tree + except Exception: + shutil.rmtree(system_tree) + raise + + def run_deploy_commands(self, deploy_tempdir, env, artifact, root_repo_dir, + ref, deployment_type, system_tree, location): + # Extensions get a private tempdir so we can more easily clean + # up any files an extension left behind + deploy_private_tempdir = tempfile.mkdtemp(dir=deploy_tempdir) + env['TMPDIR'] = deploy_private_tempdir + try: # Run configuration extensions. self.app.status(msg='Configure system') names = artifact.source.morphology['configuration-extensions'] @@ -423,9 +504,7 @@ class DeployPlugin(cliapp.Plugin): finally: # Cleanup. self.app.status(msg='Cleaning up') - shutil.rmtree(deploy_tempdir) - - self.app.status(msg='Finished deployment') + shutil.rmtree(deploy_private_tempdir) def _run_extension(self, gd, ref, name, kind, args, env): '''Run an extension. @@ -446,7 +525,7 @@ class DeployPlugin(cliapp.Plugin): code_dir = os.path.dirname(morphlib.__file__) ext_filename = os.path.join(code_dir, 'exts', name + kind) if not os.path.exists(ext_filename): - raise morphlib.Error( + raise ExtensionNotFoundError( 'Could not find extension %s%s' % (name, kind)) if not self._is_executable(ext_filename): raise morphlib.Error( @@ -464,7 +543,8 @@ class DeployPlugin(cliapp.Plugin): name=name, kind=kind) self.app.runcmd( [ext_filename] + args, - ['sh', '-c', 'while read l; do echo `date "+%F %T"` $l; done'], + ['sh', '-c', 'while read l; do echo `date "+%F %T"` "$1$l"; done', + '-', '%s[%s]' % (self.app.status_prefix, name + kind)], cwd=gd.dirname, env=env, stdout=None, stderr=None) if delete_ext: @@ -474,3 +554,41 @@ class DeployPlugin(cliapp.Plugin): st = os.stat(filename) mask = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH return (stat.S_IMODE(st.st_mode) & mask) != 0 + + def create_metadata(self, system_artifact, root_repo_dir, deployment_type, + location, env): + '''Deployment-specific metadata. + + The `build` and `deploy` operations must be from the same ref, so full + info on the root repo that the system came from is in + /baserock/${system_artifact}.meta and is not duplicated here. We do + store a `git describe` of the definitions.git repo as a convenience for + post-upgrade hooks that we may need to implement at a future date: + the `git describe` output lists the last tag, which will hopefully help + us to identify which release of a system was deployed without having to + keep a list of SHA1s somewhere or query a Trove. + + ''' + + def remove_passwords(env): + def is_password(key): + return 'PASSWORD' in key + return { k:v for k, v in env.iteritems() if not is_password(k) } + + meta = { + 'system-artifact-name': system_artifact.name, + 'configuration': remove_passwords(env), + 'deployment-type': deployment_type, + 'location': location, + 'definitions-version': { + 'describe': root_repo_dir.describe(), + }, + 'morph-version': { + 'ref': morphlib.gitversion.ref, + 'tree': morphlib.gitversion.tree, + 'commit': morphlib.gitversion.commit, + 'version': morphlib.gitversion.version, + }, + } + + return meta diff --git a/morphlib/plugins/push_pull_plugin.py b/morphlib/plugins/push_pull_plugin.py new file mode 100644 index 00000000..843de1a6 --- /dev/null +++ b/morphlib/plugins/push_pull_plugin.py @@ -0,0 +1,93 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import cliapp +import logging +import os + +import morphlib + + +class PushPullPlugin(cliapp.Plugin): + + '''Add subcommands to wrap the git push and pull commands.''' + + def enable(self): + self.app.add_subcommand( + 'push', self.push, arg_synopsis='REPO TARGET') + self.app.add_subcommand('pull', self.pull, arg_synopsis='[REMOTE]') + + def disable(self): + pass + + def push(self, args): + '''Push a branch to a remote repository. + + Command line arguments: + + * `REPO` is the repository to push your changes to. + + * `TARGET` is the branch to push to the repository. + + This is a wrapper for the `git push` command. It also deals with + pushing any binary files that have been added using git-fat. + + Example: + + morph push origin jrandom/new-feature + + ''' + if len(args) != 2: + raise morphlib.Error('push must get exactly two arguments') + + gd = morphlib.gitdir.GitDirectory(os.getcwd()) + remote, branch = args + rs = morphlib.gitdir.RefSpec(branch) + gd.get_remote(remote).push(rs) + if gd.has_fat(): + gd.fat_init() + gd.fat_push() + + def pull(self, args): + '''Pull changes to the current branch from a repository. + + Command line arguments: + + * `REMOTE` is the remote branch to pull from. By default this is the + branch being tracked by your current git branch (ie origin/master + for branch master) + + This is a wrapper for the `git pull` command. It also deals with + pulling any binary files that have been added to the repository using + git-fat. + + Example: + + morph pull + + ''' + if len(args) > 1: + raise morphlib.Error('pull takes at most one argument') + + gd = morphlib.gitdir.GitDirectory(os.getcwd()) + remote = gd.get_remote('origin') + if args: + branch = args[0] + remote.pull(branch) + else: + remote.pull() + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index 286e5374..61f9e660 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -160,6 +160,9 @@ class StagingArea(object): unpacked_artifact = os.path.join( chunk_cache_dir, os.path.basename(handle.name) + '.d') if not os.path.exists(unpacked_artifact): + self._app.status( + msg='Unpacking chunk from cache %(filename)s', + filename=os.path.basename(handle.name)) savedir = tempfile.mkdtemp(dir=chunk_cache_dir) try: morphlib.bins.unpack_binary_from_file( @@ -233,7 +236,7 @@ class StagingArea(object): path = os.path.join(self.dirname, mount_point) if not os.path.exists(path): os.makedirs(path) - self._app.runcmd(['mount', '-t', mount_type, source, path]) + morphlib.fsutils.mount(self._app.runcmd, source, path, mount_type) self.mounted.append(path) return diff --git a/morphlib/stagingarea_tests.py b/morphlib/stagingarea_tests.py index 27f85dff..52f495eb 100644 --- a/morphlib/stagingarea_tests.py +++ b/morphlib/stagingarea_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -57,7 +57,7 @@ class FakeApplication(object): def runcmd_unchecked(self, *args, **kwargs): return cliapp.runcmd_unchecked(*args, **kwargs) - def status(self, msg): + def status(self, **kwargs): pass diff --git a/morphlib/sysbranchdir.py b/morphlib/sysbranchdir.py index 1a8b898a..9d96e974 100644 --- a/morphlib/sysbranchdir.py +++ b/morphlib/sysbranchdir.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -74,6 +74,11 @@ class SystemBranchDirectory(object): If the URL is a real one (not aliased), the schema and leading // are removed from it, as is a .git suffix. + Any colons in the URL path or network location are replaced + with slashes, so that directory paths do not contain colons. + This avoids problems with PYTHONPATH, PATH, and other things + that use colon as a separator. + ''' # Parse the URL. If the path component is absolute, we assume @@ -93,6 +98,9 @@ class SystemBranchDirectory(object): else: relative = repo_url + # Replace colons with slashes. + relative = '/'.join(relative.split(':')) + # Remove anyleading slashes, or os.path.join below will only # use the relative part (since it's absolute, not relative). relative = relative.lstrip('/') diff --git a/morphlib/sysbranchdir_tests.py b/morphlib/sysbranchdir_tests.py index 7ec8ef5c..8b40f69c 100644 --- a/morphlib/sysbranchdir_tests.py +++ b/morphlib/sysbranchdir_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -149,7 +149,7 @@ class SystemBranchDirectoryTests(unittest.TestCase): self.system_branch_name) self.assertEqual( sb.get_git_directory_name('baserock:baserock/morph'), - os.path.join(self.root_directory, 'baserock:baserock/morph')) + os.path.join(self.root_directory, 'baserock/baserock/morph')) def test_reports_correct_name_for_git_directory_from_real_url(self): stripped = 'git.baserock.org/baserock/baserock/morph' @@ -169,7 +169,7 @@ class SystemBranchDirectoryTests(unittest.TestCase): self.system_branch_name) self.assertEqual( sb.get_filename('test:chunk', 'foo'), - os.path.join(self.root_directory, 'test:chunk/foo')) + os.path.join(self.root_directory, 'test/chunk/foo')) def test_reports_correct_name_for_git_directory_from_file_url(self): stripped = 'foobar/morphs' diff --git a/morphlib/workspace_tests.py b/morphlib/workspace_tests.py index b25be35e..9eef1053 100644 --- a/morphlib/workspace_tests.py +++ b/morphlib/workspace_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -97,7 +97,7 @@ class WorkspaceTests(unittest.TestCase): url = 'test:morphs' branch = 'my/new/thing' sb = ws.create_system_branch_directory(url, branch) - self.assertTrue(type(sb), morphlib.sysbranchdir.SystemBranchDirectory) + self.assertEqual(type(sb), morphlib.sysbranchdir.SystemBranchDirectory) def test_lists_created_system_branches(self): self.create_it() diff --git a/morphlib/writeexts.py b/morphlib/writeexts.py index 9dbc77e6..1849f406 100644 --- a/morphlib/writeexts.py +++ b/morphlib/writeexts.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,12 +17,65 @@ import cliapp import os import re +import shutil import sys import time import tempfile import morphlib + +class Fstab(object): + '''Small helper class for parsing and adding lines to /etc/fstab.''' + + # There is an existing Python helper library for editing of /etc/fstab. + # However it is unmaintained and has an incompatible license (GPL3). + # + # https://code.launchpad.net/~computer-janitor-hackers/python-fstab/trunk + + def __init__(self, filepath='/etc/fstab'): + if os.path.exists(filepath): + with open(filepath, 'r') as f: + self.text= f.read() + else: + self.text = '' + self.filepath = filepath + self.lines_added = 0 + + def get_mounts(self): + '''Return list of mount devices and targets in /etc/fstab. + + Return value is a dict of target -> device. + ''' + mounts = dict() + for line in self.text.splitlines(): + words = line.split() + if len(words) >= 2 and not words[0].startswith('#'): + device, target = words[0:2] + mounts[target] = device + return mounts + + def add_line(self, line): + '''Add a new entry to /etc/fstab. + + Lines are appended, and separated from any entries made by configure + extensions with a comment. + + ''' + if self.lines_added == 0: + if len(self.text) == 0 or self.text[-1] is not '\n': + self.text += '\n' + self.text += '# Morph default system layout\n' + self.lines_added += 1 + + self.text += line + '\n' + + def write(self): + '''Rewrite the fstab file to include all new entries.''' + with morphlib.savefile.SaveFile(self.filepath, 'w') as f: + f.write(self.text) + + class WriteExtension(cliapp.Application): '''A base class for deployment write extensions. @@ -53,7 +106,6 @@ class WriteExtension(cliapp.Application): def create_local_system(self, temp_root, raw_disk): '''Create a raw system image locally.''' - size = self.get_disk_size() if not size: raise cliapp.AppException('DISK_SIZE is not defined') @@ -66,19 +118,10 @@ class WriteExtension(cliapp.Application): os.remove(raw_disk) raise try: - version_label = 'factory' - version_root = os.path.join(mp, 'systems', version_label) - os.makedirs(version_root) - self.create_state(mp) - self.create_orig(version_root, temp_root) - self.create_fstab(version_root) - self.create_run(version_root) - os.symlink(version_label, os.path.join(mp, 'systems', 'default')) - if self.bootloader_is_wanted(): - self.install_kernel(version_root, temp_root) - self.install_extlinux(mp) + self.create_btrfs_system_layout( + temp_root, mp, version_label='factory') except BaseException, e: - sys.stderr.write('Error creating disk image') + sys.stderr.write('Error creating Btrfs system layout') self.unmount(mp) os.remove(raw_disk) raise @@ -129,16 +172,6 @@ class WriteExtension(cliapp.Application): '''Parse the virtual cpu count from environment.''' return self._parse_size_from_environment('VCPUS', '1') - def create_state(self, real_root): - '''Create the state subvolumes that are shared between versions''' - - self.status(msg='Creating state subvolumes') - os.mkdir(os.path.join(real_root, 'state')) - statedirs = ['home', 'opt', 'srv'] - for statedir in statedirs: - dirpath = os.path.join(real_root, 'state', statedir) - cliapp.runcmd(['btrfs', 'subvolume', 'create', dirpath]) - def create_raw_disk_image(self, filename, size): '''Create a raw disk image.''' @@ -178,6 +211,34 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['umount', mount_point]) os.rmdir(mount_point) + def create_btrfs_system_layout(self, temp_root, mountpoint, version_label): + '''Separate base OS versions from state using subvolumes. + + ''' + version_root = os.path.join(mountpoint, 'systems', version_label) + state_root = os.path.join(mountpoint, 'state') + + os.makedirs(version_root) + os.makedirs(state_root) + + self.create_orig(version_root, temp_root) + system_dir = os.path.join(version_root, 'orig') + + state_dirs = self.complete_fstab_for_btrfs_layout(system_dir) + + for state_dir in state_dirs: + self.create_state_subvolume(system_dir, mountpoint, state_dir) + + self.create_run(version_root) + + os.symlink( + version_label, os.path.join(mountpoint, 'systems', 'default')) + + if self.bootloader_is_wanted(): + self.install_kernel(version_root, temp_root) + self.install_syslinux_menu(mountpoint, version_root) + self.install_extlinux(mountpoint) + def create_orig(self, version_root, temp_root): '''Create the default "factory" system.''' @@ -197,29 +258,68 @@ class WriteExtension(cliapp.Application): cliapp.runcmd( ['btrfs', 'subvolume', 'snapshot', orig, run]) - def create_fstab(self, version_root): - '''Create an fstab.''' + def create_state_subvolume(self, system_dir, mountpoint, state_subdir): + '''Create a shared state subvolume. - self.status(msg='Creating fstab') - fstab = os.path.join(version_root, 'orig', 'etc', 'fstab') + We need to move any files added to the temporary rootfs by the + configure extensions to their correct home. For example, they might + have added keys in `/root/.ssh` which we now need to transfer to + `/state/root/.ssh`. - if os.path.exists(fstab): - with open(fstab, 'r') as f: - contents = f.read() - else: - contents = '' + ''' + self.status(msg='Creating %s subvolume' % state_subdir) + subvolume = os.path.join(mountpoint, 'state', state_subdir) + cliapp.runcmd(['btrfs', 'subvolume', 'create', subvolume]) + os.chmod(subvolume, 0755) + + existing_state_dir = os.path.join(system_dir, state_subdir) + files = [] + if os.path.exists(existing_state_dir): + files = os.listdir(existing_state_dir) + if len(files) > 0: + self.status(msg='Moving existing data to %s subvolume' % subvolume) + for filename in files: + filepath = os.path.join(existing_state_dir, filename) + shutil.move(filepath, subvolume) + + def complete_fstab_for_btrfs_layout(self, system_dir): + '''Fill in /etc/fstab entries for the default Btrfs disk layout. + + In the future we should move this code out of the write extension and + in to a configure extension. To do that, though, we need some way of + informing the configure extension what layout should be used. Right now + a configure extension doesn't know if the system is going to end up as + a Btrfs disk image, a tarfile or something else and so it can't come + up with a sensible default fstab. + + Configuration extensions can already create any /etc/fstab that they + like. This function only fills in entries that are missing, so if for + example the user configured /home to be on a separate partition, that + decision will be honoured and /state/home will not be created. - got_root = False - for line in contents.splitlines(): - words = line.split() - if len(words) >= 2 and not words[0].startswith('#'): - got_root = got_root or words[1] == '/' + ''' + shared_state_dirs = {'home', 'root', 'opt', 'srv', 'var'} + + fstab = Fstab(os.path.join(system_dir, 'etc', 'fstab')) + existing_mounts = fstab.get_mounts() + + if '/' in existing_mounts: + root_device = existing_mounts['/'] + else: + root_device = '/dev/sda' + fstab.add_line('/dev/sda / btrfs defaults,rw,noatime 0 1') - if not got_root: - contents += '\n/dev/sda / btrfs defaults,rw,noatime 0 1\n' + state_dirs_to_create = set() + for state_dir in shared_state_dirs: + if '/' + state_dir not in existing_mounts: + state_dirs_to_create.add(state_dir) + state_subvol = os.path.join('/state', state_dir) + fstab.add_line( + '%s /%s btrfs subvol=%s,defaults,rw,noatime 0 2' % + (root_device, state_dir, state_subvol)) - with open(fstab, 'w') as f: - f.write(contents) + fstab.write() + return state_dirs_to_create def install_kernel(self, version_root, temp_root): '''Install the kernel outside of 'orig' or 'run' subvolumes''' @@ -254,6 +354,23 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['sync']) time.sleep(2) + def install_syslinux_menu(self, real_root, version_root): + '''Make syslinux/extlinux menu binary available. + + The syslinux boot menu is compiled to a file named menu.c32. Extlinux + searches a few places for this file but it does not know to look inside + our subvolume, so we copy it to the filesystem root. + + If the file is not available, the bootloader will still work but will + not be able to show a menu. + + ''' + menu_file = os.path.join(version_root, 'orig', + 'usr', 'share', 'syslinux', 'menu.c32') + if os.path.isfile(menu_file): + self.status(msg='Copying menu.c32') + shutil.copy(menu_file, real_root) + def parse_attach_disks(self): '''Parse $ATTACH_DISKS into list of disks to attach.''' @@ -285,14 +402,14 @@ class WriteExtension(cliapp.Application): return value == 'yes' - def parse_autostart(self): - '''Parse $AUTOSTART to determine if VMs should be started.''' + def get_environment_boolean(self, variable): + '''Parse a yes/no boolean passed through the environment.''' - autostart = os.environ.get('AUTOSTART', 'no') - if autostart == 'no': + value = os.environ.get(variable, 'no').lower() + if value in ['no', '0', 'false']: return False - elif autostart == 'yes': + elif value in ['yes', '1', 'true']: return True else: - raise cliapp.AppException('Unexpected value for AUTOSTART: %s' % - autostart) + raise cliapp.AppException('Unexpected value for %s: %s' % + (variable, value)) |