diff options
97 files changed, 1776 insertions, 846 deletions
@@ -2,7 +2,7 @@ # # Run test suite for morph. # -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -23,13 +23,61 @@ set -e # Parse the command line. -full=false +run_style=false +run_unit_tests=false +run_cmdtests=false +run_slow_cmdtests=false +run_yarns=false +if [ "$#" -eq 0 ]; then + run_style=true + run_unit_tests=true + run_cmdtests=true + run_slow_cmdtests=false + run_yarns=true +fi while [ "$#" -gt 0 ] do case "$1" in - --full) full=true; shift ;; - *) echo "ERROR: Unknown argument $1." 1>&2; exit 1 ;; + --full) + run_style=true + run_unit_tests=true + run_cmdtests=true + run_slow_cmdtests=true + run_yarns=true + ;; + --style) + run_style=true + ;; + --no-style) + run_style=false + ;; + --unit-tests) + run_unit_tests=true + ;; + --no-unit-tests) + run_unit_tests=false + ;; + --cmdtests) + run_cmdtests=true + ;; + --no-cmdtests) + run_cmdtests=false + ;; + --slow-cmdtests) + run_slow_cmdtests=true + ;; + --no-slow-cmdtests) + run_slow_cmdtests=false + ;; + --yarns) + run_yarns=true + ;; + --no-yarns) + run_yarns=false + ;; + *) echo "ERROR: Unknown argument $1." 1>&2; exit 1 ;; esac + shift done @@ -45,7 +93,7 @@ export PYTHONPATH # Run the style checks errors=0 -if [ -d .git ]; +if "$run_style" && [ -d .git ]; then echo "Checking copyright statements" if ! (git ls-files -z | xargs -0r scripts/check-copyright-year); then @@ -68,54 +116,82 @@ fi # Clean up artifacts from previous (possibly failed) runs, build, # and run the tests. -python setup.py clean check +if "$run_unit_tests"; then + python setup.py clean check +fi # Run scenario tests with yarn, if yarn is available. +# +# Yarn cleans up the environment when it runs tests, and this removes +# PYTHONPATH from the environment. However, we need our tests to have +# the PYTHONPATH, so that we can get them to, for example, use the right +# versions of updated dependencies. The immediate current need is to +# be able to get them to use an updated version of cliapp, but it is +# a general need. +# +# We solve this by using the yarn --env option, allowing us to tell yarn +# explicitly which environment variables to set in addition to the set +# it sets anyway. -if command -v yarn > /dev/null +if "$run_yarns" && command -v yarn > /dev/null then - yarn -s yarns/morph.shell-lib yarns/*.yarn + yarn --env "PYTHONPATH=$PYTHONPATH" -s yarns/morph.shell-lib yarns/*.yarn fi # cmdtest tests. HOME="$(pwd)/scripts" -cmdtest tests +if "$run_cmdtests" +then + cmdtest tests +else + echo "NOT RUNNING test" +fi -if $full +if "$run_slow_cmdtests" then cmdtest tests.branching else echo "NOT RUNNING test.branching" fi -if $full && false +if false && "$run_cmdtests" then cmdtest tests.merging else echo "NOT RUNNING test.merging" fi -cmdtest tests.deploy +if "$run_cmdtests" +then + cmdtest tests.deploy +else + echo "NOT RUNNING test.deploy" +fi # Building systems requires the 'filter' parameter of tarfile.TarFile.add(): # this was introduced in Python 2.7 -if python --version 2>&1 | grep '^Python 2\.[78]' > /dev/null; then - cmdtest tests.build -else +if ! "$run_cmdtests"; then + echo "NOT RUNNING tests.build" +elif ! (python --version 2>&1 | grep -q '^Python 2\.[78]'); then echo "NOT RUNNING tests.build (requires Python 2.7)" +else + cmdtest tests.build fi # The as-root tests use YAML morphologies, so they require the PyYAML module. -if $full && [ $(whoami) = root ] && command -v mkfs.btrfs > /dev/null && - python -c " +if ! "$run_slow_cmdtests"; then + echo "NOT RUNNING tests.as-root" +elif [ $(whoami) != root ] || ! command -v mkfs.btrfs > /dev/null; then + echo "NOT RUNNING tests.as-root (no btrfs)" +elif ! python -c " import morphlib, sys if not morphlib.got_yaml: sys.exit(1) " > /dev/null 2>&1 then - cmdtest tests.as-root -else echo "NOT RUNNING tests.as-root (requires PyYAML)" +else + cmdtest tests.as-root fi diff --git a/morphlib/app.py b/morphlib/app.py index 7fb71c7b..409e0a12 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -59,6 +59,12 @@ class Morph(cliapp.Application): self.settings.boolean(['quiet', 'q'], 'show no output unless there is an error') + self.settings.boolean(['help', 'h'], + 'show this help message and exit') + + self.settings.boolean(['help-all'], + 'show help message including hidden subcommands') + self.settings.string(['build-ref-prefix'], 'Prefix to use for temporary build refs', metavar='PREFIX', @@ -195,6 +201,14 @@ class Morph(cliapp.Application): def process_args(self, args): self.check_time() + if self.settings['help']: + self.help(args) + sys.exit(0) + + if self.settings['help-all']: + self.help_all(args) + sys.exit(0) + if self.settings['build-ref-prefix'] is None: if self.settings['trove-id']: self.settings['build-ref-prefix'] = os.path.join( @@ -345,7 +359,10 @@ class Morph(cliapp.Application): morphology = resolved_morphologies[reference] visit(reponame, ref, filename, absref, tree, morphology) - if morphology['kind'] == 'system': + if morphology['kind'] == 'cluster': + raise cliapp.AppException( + "Cannot build a morphology of type 'cluster'.") + elif morphology['kind'] == 'system': queue.extend((s.get('repo') or reponame, s.get('ref') or ref, '%s.morph' % s['morph']) @@ -451,27 +468,49 @@ class Morph(cliapp.Application): # run the command line return cliapp.Application.runcmd(self, argv, *args, **kwargs) - # FIXME: This overrides a private method in cliapp. We need - # get cliapp to provide the necessary hooks to do this cleanly. - # As it is, this is a copy of the method in cliapp, with the - # single change that for subcommand helps, the formatting is - # not used. - def _help_helper(self, args, show_all): # pragma: no cover + def parse_args(self, args, configs_only=False): + return self.settings.parse_args(args, + configs_only=configs_only, + arg_synopsis=self.arg_synopsis, + cmd_synopsis=self.cmd_synopsis, + compute_setting_values=self.compute_setting_values, + add_help_option=False) + + class IdentityFormat(): + def format(self, text): + return text + + def _help_helper(self, args, show_all): try: width = int(os.environ.get('COLUMNS', '78')) except ValueError: width = 78 - fmt = cliapp.TextFormat(width=width) - if args: - usage = self._format_usage_for(args[0]) - description = self._format_subcommand_help(args[0]) + cmd = args[0] + if cmd not in self.subcommands: + raise cliapp.AppException('Unknown subcommand %s' % cmd) + # TODO Search for other things we might want help on + # such as write or configuration extensions. + usage = self._format_usage_for(cmd) + fmt = self.IdentityFormat() + description = fmt.format(self._format_subcommand_help(cmd)) text = '%s\n\n%s' % (usage, description) + self.output.write(text) else: - usage = self._format_usage(all=show_all) - description = fmt.format(self._format_description(all=show_all)) - text = '%s\n\n%s' % (usage, description) - - text = self.settings.progname.join(text.split('%prog')) - self.output.write(text) + pp = self.settings.build_parser( + configs_only=True, + arg_synopsis=self.arg_synopsis, + cmd_synopsis=self.cmd_synopsis, + all_options=show_all, + add_help_option=False) + text = pp.format_help() + self.output.write(text) + + def help(self, args): # pragma: no cover + '''Print help.''' + self._help_helper(args, False) + + def help_all(self, args): # pragma: no cover + '''Print help, including hidden subcommands.''' + self._help_helper(args, True) diff --git a/morphlib/artifactsplitrule.py b/morphlib/artifactsplitrule.py index 246691d8..bc92e5fb 100644 --- a/morphlib/artifactsplitrule.py +++ b/morphlib/artifactsplitrule.py @@ -300,4 +300,4 @@ def unify_system_matches(morphology): def unify_cluster_matches(_): - return None + return SplitRules() diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 3c190275..7ad7909d 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -455,10 +455,11 @@ class BuildCommand(object): if artifact.source.build_mode == 'bootstrap': if not self.in_same_stratum(artifact, target_artifact): continue - self.app.status(msg='Installing chunk %(chunk_name)s ' - 'from cache %(cache)s', - chunk_name=artifact.name, - cache=artifact.cache_key[:7]) + self.app.status( + msg='Installing chunk %(chunk_name)s from cache %(cache)s', + chunk_name=artifact.name, + cache=artifact.cache_key[:7], + chatty=True) handle = self.lac.get(artifact) staging_area.install_artifact(handle) diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 2dca738c..02e8b485 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -36,6 +36,8 @@ import morphlib from morphlib.artifactcachereference import ArtifactCacheReference import morphlib.gitversion +SYSTEM_INTEGRATION_PATH = os.path.join('baserock', 'system-integration') + def extract_sources(app, repo_cache, repo, sha1, srcdir): #pragma: no cover '''Get sources from git to a source directory, including submodules''' @@ -419,11 +421,44 @@ class ChunkBuilder(BuilderBase): shutil.copyfileobj(readlog, self.app.output) raise e + def write_system_integration_commands(self, destdir, + integration_commands, artifact_name): # pragma: no cover + + rel_path = SYSTEM_INTEGRATION_PATH + dest_path = os.path.join(destdir, SYSTEM_INTEGRATION_PATH) + + scripts_created = [] + + if not os.path.exists(dest_path): + os.makedirs(dest_path) + + if artifact_name in integration_commands: + prefixes_per_artifact = integration_commands[artifact_name] + for prefix, commands in prefixes_per_artifact.iteritems(): + for index, script in enumerate(commands): + script_name = "%s-%s-%04d" % (prefix, + artifact_name, + index) + script_path = os.path.join(dest_path, script_name) + + with morphlib.savefile.SaveFile(script_path, 'w') as f: + f.write("#!/bin/sh\nset -xeu\n") + f.write(script) + os.chmod(script_path, 0555) + + rel_script_path = os.path.join(SYSTEM_INTEGRATION_PATH, + script_name) + scripts_created += [rel_script_path] + + return scripts_created + def assemble_chunk_artifacts(self, destdir): # pragma: no cover built_artifacts = [] filenames = [] source = self.artifact.source split_rules = source.split_rules + morphology = source.morphology + sys_tag = 'system-integration' def filepaths(destdir): for dirname, subdirs, basenames in os.walk(destdir): @@ -438,6 +473,8 @@ class ChunkBuilder(BuilderBase): matches, overlaps, unmatched = \ split_rules.partition(filepaths(destdir)) + system_integration = morphology.get(sys_tag) or {} + with self.build_watch('create-chunks'): for chunk_artifact_name, chunk_artifact \ in source.artifacts.iteritems(): @@ -455,9 +492,11 @@ class ChunkBuilder(BuilderBase): names.update(all_parents(name)) return sorted(names) - parented_paths = \ - parentify(file_paths + - ['baserock/%s.meta' % chunk_artifact_name]) + extra_files = self.write_system_integration_commands( + destdir, system_integration, + chunk_artifact_name) + extra_files += ['baserock/%s.meta' % chunk_artifact_name] + parented_paths = parentify(file_paths + extra_files) with self.local_artifact_cache.put(chunk_artifact) as f: self.write_metadata(destdir, chunk_artifact_name, @@ -549,7 +588,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover fs_root = self.staging_area.destdir(self.artifact.source) self.unpack_strata(fs_root) self.write_metadata(fs_root, rootfs_name) - self.create_fstab(fs_root) + self.run_system_integration_commands(fs_root) self.copy_kernel_into_artifact_cache(fs_root) unslashy_root = fs_root[1:] def uproot_info(info): @@ -649,29 +688,50 @@ class SystemBuilder(BuilderBase): # pragma: no cover os.chmod(os_release_file, 0644) - def create_fstab(self, path): - '''Create an /etc/fstab inside a system tree. + def run_system_integration_commands(self, rootdir): # pragma: no cover + ''' Run the system integration commands ''' - The fstab is created using assumptions of the disk layout. - If the assumptions are wrong, extend this code so it can deal - with other cases. + sys_integration_dir = os.path.join(rootdir, SYSTEM_INTEGRATION_PATH) + if not os.path.isdir(sys_integration_dir): + return - ''' + env = { + 'PATH': '/bin:/usr/bin:/sbin:/usr/sbin' + } - self.app.status(msg='Creating fstab in %(path)s', - path=path, chatty=True) - with self.build_watch('create-fstab'): - fstab = os.path.join(path, 'etc', 'fstab') - if not os.path.exists(fstab): - # FIXME: should exist - if not os.path.exists(os.path.dirname(fstab)): - os.makedirs(os.path.dirname(fstab)) - # We create an empty fstab: systemd does not require - # /sys and /proc entries, and we can't know what the - # right entry for / is. The fstab gets built during - # deployment instead, when that information is available. - with open(fstab, 'w'): - pass + self.app.status(msg='Running the system integration commands', + error=True) + + mounted = [] + to_mount = ( + ('proc', 'proc', 'none'), + ('dev/shm', 'tmpfs', 'none'), + ) + + try: + for mount_point, mount_type, source in to_mount: + logging.debug('Mounting %s in system root filesystem' + % mount_point) + path = os.path.join(rootdir, mount_point) + if not os.path.exists(path): + os.makedirs(path) + morphlib.fsutils.mount(self.app.runcmd, source, path, + mount_type) + mounted.append(path) + + self.app.runcmd(['chroot', rootdir, 'sh', '-c', + 'cd / && run-parts "$1"', '-', SYSTEM_INTEGRATION_PATH], + env=env) + except BaseException, e: + self.app.status( + msg='Error while running system integration commands', + error=True) + raise + finally: + for mount_path in reversed(mounted): + logging.debug('Unmounting %s in system root filesystem' + % mount_path) + morphlib.fsutils.unmount(self.app.runcmd, mount_path) def copy_kernel_into_artifact_cache(self, path): '''Copy the installed kernel image into the local artifact cache. diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index bb536f82..3efe1cbb 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -114,6 +114,6 @@ class CacheKeyComputer(object): if kind == 'stratum': keys['stratum-format-version'] = 1 elif kind == 'system': - keys['system-compatibility-version'] = "1~ (temporary, root rw)" + keys['system-compatibility-version'] = "2~ (upgradable, root rw)" return keys diff --git a/morphlib/exts/kvm.check b/morphlib/exts/kvm.check new file mode 100755 index 00000000..be7c51c2 --- /dev/null +++ b/morphlib/exts/kvm.check @@ -0,0 +1,35 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''Preparatory checks for Morph 'kvm' write extension''' + +import cliapp + +import morphlib.writeexts + + +class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + upgrade = self.get_environment_boolean('UPGRADE') + if upgrade: + raise cliapp.AppException( + 'Use the `ssh-rsync` write extension to deploy upgrades to an ' + 'existing remote system.') + +KvmPlusSshCheckExtension().run() diff --git a/morphlib/exts/kvm.write b/morphlib/exts/kvm.write index 4f877c22..94560972 100755 --- a/morphlib/exts/kvm.write +++ b/morphlib/exts/kvm.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -56,7 +56,7 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): temp_root, location = args ssh_host, vm_name, vm_path = self.parse_location(location) - autostart = self.parse_autostart() + autostart = self.get_environment_boolean('AUTOSTART') fd, raw_disk = tempfile.mkstemp() os.close(fd) diff --git a/morphlib/exts/nfsboot.check b/morphlib/exts/nfsboot.check new file mode 100755 index 00000000..f84f187f --- /dev/null +++ b/morphlib/exts/nfsboot.check @@ -0,0 +1,101 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''Preparatory checks for Morph 'nfsboot' write extension''' + +import cliapp +import os + +import morphlib.writeexts + + +class NFSBootCheckExtension(morphlib.writeexts.WriteExtension): + + _nfsboot_root = '/srv/nfsboot' + + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + location = args[0] + + upgrade = self.get_environment_boolean('UPGRADE') + if upgrade: + raise cliapp.AppException( + 'Upgrading is not currently supported for NFS deployments.') + + hostname = os.environ.get('HOSTNAME', None) + if hostname is None: + raise cliapp.AppException('You must specify a HOSTNAME.') + if hostname == 'baserock': + raise cliapp.AppException('It is forbidden to nfsboot a system ' + 'with hostname "%s"' % hostname) + + self.test_good_server(location) + + version_label = os.getenv('VERSION_LABEL', 'factory') + versioned_root = os.path.join(self._nfsboot_root, hostname, 'systems', + version_label) + if self.version_exists(versioned_root, location): + raise cliapp.AppException( + 'Root file system for host %s (version %s) already exists on ' + 'the NFS server %s. Deployment aborted.' % (hostname, + version_label, location)) + + def test_good_server(self, server): + # Can be ssh'ed into + try: + cliapp.ssh_runcmd('root@%s' % server, ['true']) + except cliapp.AppException: + raise cliapp.AppException('You are unable to ssh into server %s' + % server) + + # Is an NFS server + try: + cliapp.ssh_runcmd( + 'root@%s' % server, ['test', '-e', '/etc/exports']) + except cliapp.AppException: + raise cliapp.AppException('server %s is not an nfs server' + % server) + try: + cliapp.ssh_runcmd( + 'root@%s' % server, ['systemctl', 'is-enabled', + 'nfs-server.service']) + + except cliapp.AppException: + raise cliapp.AppException('server %s does not control its ' + 'nfs server by systemd' % server) + + # TFTP server exports /srv/nfsboot/tftp + tftp_root = os.path.join(self._nfsboot_root, 'tftp') + try: + cliapp.ssh_runcmd( + 'root@%s' % server, ['test' , '-d', tftp_root]) + except cliapp.AppException: + raise cliapp.AppException('server %s does not export %s' % + (tftp_root, server)) + + def version_exists(self, versioned_root, location): + try: + cliapp.ssh_runcmd('root@%s' % location, + ['test', '-d', versioned_root]) + except cliapp.AppException: + return False + + return True + + +NFSBootCheckExtension().run() diff --git a/morphlib/exts/nfsboot.configure b/morphlib/exts/nfsboot.configure index 8dc6c67c..660d9c39 100755 --- a/morphlib/exts/nfsboot.configure +++ b/morphlib/exts/nfsboot.configure @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -15,7 +15,9 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# Remove all networking interfaces and stop fstab from mounting '/' +# Remove all networking interfaces. On nfsboot systems, eth0 is set up +# during kernel init, and the normal ifup@eth0.service systemd unit +# would break the NFS connection and cause the system to hang. set -e @@ -26,7 +28,4 @@ auto lo iface lo inet loopback EOF - # Stop fstab from mounting '/' - mv "$1/etc/fstab" "$1/etc/fstab.old" - awk '/^ *#/ || $2 != "/"' "$1/etc/fstab.old" > "$1/etc/fstab" fi diff --git a/morphlib/exts/nfsboot.write b/morphlib/exts/nfsboot.write index 34a72972..8d3d6df7 100755 --- a/morphlib/exts/nfsboot.write +++ b/morphlib/exts/nfsboot.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -60,38 +60,18 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): raise cliapp.AppException('Wrong number of command line args') temp_root, location = args - hostname = self.get_hostname(temp_root) - if hostname == 'baserock': - raise cliapp.AppException('It is forbidden to nfsboot a system ' - 'with hostname "baserock"') - self.test_good_server(location) version_label = os.getenv('VERSION_LABEL', 'factory') + hostname = os.environ['HOSTNAME'] + versioned_root = os.path.join(self._nfsboot_root, hostname, 'systems', version_label) - if self.version_exists(versioned_root, location): - raise cliapp.AppException('Version %s already exists on' - ' this device. Deployment aborted' - % version_label) + self.copy_rootfs(temp_root, location, versioned_root, hostname) self.copy_kernel(temp_root, location, versioned_root, version_label, hostname) self.configure_nfs(location, hostname) - def version_exists(self, versioned_root, location): - try: - cliapp.ssh_runcmd('root@%s' % location, - ['test', '-d', versioned_root]) - except cliapp.AppException: - return False - - return True - - def get_hostname(self, temp_root): - hostnamepath = os.path.join(temp_root, 'etc', 'hostname') - with open(hostnamepath) as f: - return f.readline().strip() - def create_local_state(self, location, hostname): statedir = os.path.join(self._nfsboot_root, hostname, 'state') subdirs = [os.path.join(statedir, 'home'), @@ -209,37 +189,6 @@ mv "$temp" "$target" 'root@%s' % location, ['systemctl', 'restart', 'nfs-server.service']) - def test_good_server(self, server): - # Can be ssh'ed into - try: - cliapp.ssh_runcmd('root@%s' % server, ['true']) - except cliapp.AppException: - raise cliapp.AppException('You are unable to ssh into server %s' - % server) - - # Is an NFS server - try: - cliapp.ssh_runcmd( - 'root@%s' % server, ['test', '-e', '/etc/exports']) - except cliapp.AppException: - raise cliapp.AppException('server %s is not an nfs server' - % server) - try: - cliapp.ssh_runcmd( - 'root@%s' % server, ['systemctl', 'is-enabled', - 'nfs-server.service']) - - except cliapp.AppException: - raise cliapp.AppException('server %s does not control its ' - 'nfs server by systemd' % server) - - # TFTP server exports /srv/nfsboot/tftp - try: - cliapp.ssh_runcmd( - 'root@%s' % server, ['test' , '-d', '/srv/nfsboot/tftp']) - except cliapp.AppException: - raise cliapp.AppException('server %s does not export ' - '/srv/nfsboot/tftp' % server) NFSBootWriteExtension().run() diff --git a/morphlib/exts/openstack.check b/morphlib/exts/openstack.check new file mode 100755 index 00000000..a9a8fe1b --- /dev/null +++ b/morphlib/exts/openstack.check @@ -0,0 +1,35 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''Preparatory checks for Morph 'openstack' write extension''' + +import cliapp + +import morphlib.writeexts + + +class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + upgrade = self.get_environment_boolean('UPGRADE') + if upgrade: + raise cliapp.AppException( + 'Use the `ssh-rsync` write extension to deploy upgrades to an ' + 'existing remote system.') + +OpenStackCheckExtension().run() diff --git a/morphlib/exts/rawdisk.write b/morphlib/exts/rawdisk.write index 8723ac0c..87edf7bf 100755 --- a/morphlib/exts/rawdisk.write +++ b/morphlib/exts/rawdisk.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -50,12 +50,15 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): self.create_local_system(temp_root, location) self.status(msg='Disk image has been created at %s' % location) except Exception: - os.remove(location) self.status(msg='Failure to create disk image at %s' % location) + if os.path.exists(location): + os.remove(location) raise def upgrade_local_system(self, raw_disk, temp_root): + self.complete_fstab_for_btrfs_layout(temp_root) + mp = self.mount(raw_disk) version_label = self.get_version_label(mp) diff --git a/morphlib/exts/ssh-rsync.check b/morphlib/exts/ssh-rsync.check new file mode 100755 index 00000000..90029cb4 --- /dev/null +++ b/morphlib/exts/ssh-rsync.check @@ -0,0 +1,36 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''Preparatory checks for Morph 'ssh-rsync' write extension''' + +import cliapp + +import morphlib.writeexts + + +class SshRsyncCheckExtension(morphlib.writeexts.WriteExtension): + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + upgrade = self.get_environment_boolean('UPGRADE') + if not upgrade: + raise cliapp.AppException( + 'The ssh-rsync write is for upgrading existing remote ' + 'Baserock machines. It cannot be used for an initial ' + 'deployment.') + +SshRsyncCheckExtension().run() diff --git a/morphlib/exts/ssh-rsync.write b/morphlib/exts/ssh-rsync.write index 211dbe5e..509520ae 100755 --- a/morphlib/exts/ssh-rsync.write +++ b/morphlib/exts/ssh-rsync.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,6 +26,14 @@ import tempfile import morphlib.writeexts + +def ssh_runcmd_ignore_failure(location, command, **kwargs): + try: + return cliapp.ssh_runcmd(location, command, **kwargs) + except cliapp.AppException: + pass + + class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): '''Upgrade a running baserock system with ssh and rsync. @@ -47,8 +55,11 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.upgrade_remote_system(location, temp_root) def upgrade_remote_system(self, location, temp_root): + self.complete_fstab_for_btrfs_layout(temp_root) + root_disk = self.find_root_disk(location) version_label = os.environ.get('VERSION_LABEL') + autostart = self.get_environment_boolean('AUTOSTART') self.status(msg='Creating remote mount point') remote_mnt = cliapp.ssh_runcmd(location, ['mktemp', '-d']).strip() @@ -56,15 +67,11 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Mounting root disk') cliapp.ssh_runcmd(location, ['mount', root_disk, remote_mnt]) except Exception as e: - try: - cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) - except: - pass + ssh_runcmd_ignore_failure(location, ['rmdir', remote_mnt]) raise e try: version_root = os.path.join(remote_mnt, 'systems', version_label) - run_dir = os.path.join(version_root, 'run') orig_dir = os.path.join(version_root, 'orig') self.status(msg='Creating %s' % version_root) @@ -73,80 +80,40 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.create_remote_orig(location, version_root, remote_mnt, temp_root) - self.status(msg='Creating "run" subvolume') - cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', - 'snapshot', orig_dir, run_dir]) - - self.status(msg='Updating system configuration') - bscs_loc = os.path.join(run_dir, 'usr', 'bin', - 'baserock-system-config-sync') - - output = cliapp.ssh_runcmd(location, ['sh', '-c', - '"$1" merge "$2" &> /dev/null || echo -n cmdfailed', - '-', bscs_loc, version_label]) - if output == "cmdfailed": - self.status(msg='Updating system configuration failed') - - self.install_remote_kernel(location, version_root, temp_root) - default_path = os.path.join(remote_mnt, 'systems', 'default') - if self.bootloader_is_wanted(): - output = cliapp.ssh_runcmd(location, ['sh', '-c', - 'test -e "$1" && stat -c %F "$1" ' - '|| echo missing file', - '-', default_path]) - if output != "symbolic link": - # we are upgrading and old system that does - # not have an updated extlinux config file - self.update_remote_extlinux(location, remote_mnt, - version_label) - cliapp.ssh_runcmd(location, ['ln', '-sfn', version_label, - default_path]) + # Use the system-version-manager from the new system we just + # installed, so that we can upgrade from systems that don't have + # it installed. + self.status(msg='Calling system-version-manager to deploy upgrade') + deployment = os.path.join('/systems', version_label, 'orig') + system_config_sync = os.path.join( + remote_mnt, 'systems', version_label, 'orig', 'usr', 'bin', + 'baserock-system-config-sync') + system_version_manager = os.path.join( + remote_mnt, 'systems', version_label, 'orig', 'usr', 'bin', + 'system-version-manager') + cliapp.ssh_runcmd(location, + ['env', 'BASEROCK_SYSTEM_CONFIG_SYNC='+system_config_sync, + system_version_manager, 'deploy', deployment]) + + self.status(msg='Setting %s as the new default system' % + version_label) + cliapp.ssh_runcmd(location, + [system_version_manager, 'set-default', version_label]) except Exception as e: - try: - cliapp.ssh_runcmd(location, - ['btrfs', 'subvolume', 'delete', run_dir]) - except: - pass - try: - cliapp.ssh_runcmd(location, - ['btrfs', 'subvolume', 'delete', orig_dir]) - except: - pass - try: - cliapp.ssh_runcmd(location, ['rm', '-rf', version_root]) - except: - pass + self.status(msg='Deployment failed') + ssh_runcmd_ignore_failure( + location, ['btrfs', 'subvolume', 'delete', orig_dir]) + ssh_runcmd_ignore_failure( + location, ['rm', '-rf', version_root]) raise e finally: self.status(msg='Removing temporary mounts') cliapp.ssh_runcmd(location, ['umount', remote_mnt]) cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) - def update_remote_extlinux(self, location, remote_mnt, version_label): - '''Install/reconfigure extlinux on location''' - - self.status(msg='Creating extlinux.conf') - config = os.path.join(remote_mnt, 'extlinux.conf') - temp_fd, temp_path = tempfile.mkstemp() - with os.fdopen(temp_fd, 'w') as f: - f.write('default linux\n') - f.write('timeout 1\n') - f.write('label linux\n') - f.write('kernel /systems/default/kernel\n') - f.write('append root=/dev/sda ' - 'rootflags=subvol=systems/default/run ' - 'init=/sbin/init rw\n') - - try: - cliapp.runcmd(['rsync', '-as', temp_path, - '%s:%s~' % (location, config)]) - cliapp.ssh_runcmd(location, ['mv', config+'~', config]) - except Exception as e: - try: - cliapp.ssh_runcmd(location, ['rm', '-f', config+'~']) - except: - pass - raise e + if autostart: + self.status(msg="Rebooting into new system ...") + ssh_runcmd_ignore_failure(location, ['reboot']) def create_remote_orig(self, location, version_root, remote_mnt, temp_root): @@ -178,18 +145,6 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): if (line_words[1] == '/' and line_words[0] != 'rootfs'): return line_words[0] - def install_remote_kernel(self, location, version_root, temp_root): - '''Install the kernel in temp_root inside version_root on location''' - - self.status(msg='Installing kernel') - image_names = ('vmlinuz', 'zImage', 'uImage') - kernel_dest = os.path.join(version_root, 'kernel') - for name in image_names: - try_path = os.path.join(temp_root, 'boot', name) - if os.path.exists(try_path): - cliapp.runcmd(['rsync', '-as', try_path, - '%s:%s' % (location, kernel_dest)]) - def check_valid_target(self, location): try: cliapp.ssh_runcmd(location, ['true']) @@ -203,10 +158,17 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): raise cliapp.AppException('%s is not a baserock system' % location) - output = cliapp.ssh_runcmd(location, ['sh', '-c', - 'type rsync &> /dev/null || echo -n cmdnotfound']) - if output == 'cmdnotfound': - raise cliapp.AppException('%s does not have rsync' - % location) + def check_command_exists(command): + test = 'type %s > /dev/null 2>&1 || echo -n cmdnotfound' % command + output = cliapp.ssh_runcmd(location, ['sh', '-c', test]) + if output == 'cmdnotfound': + raise cliapp.AppException( + "%s does not have %s" % (location, command)) + + # The deploy requires baserock-system-config-sync and + # system-version-manager in the new system only. The old system doesn't + # need to have them at all. + check_command_exists('rsync') + SshRsyncWriteExtension().run() diff --git a/morphlib/exts/sysroot.write b/morphlib/exts/sysroot.write new file mode 100755 index 00000000..1ae4864f --- /dev/null +++ b/morphlib/exts/sysroot.write @@ -0,0 +1,29 @@ +#!/bin/sh +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# A Morph write extension to deploy to another directory + +set -eu + +# Ensure the target is an empty directory +mkdir -p "$2" +find "$2" -mindepth 1 -delete + +# Move the contents of our source directory to our target +# Previously we would (cd "$1" && find -print0 | cpio -0pumd "$absolute_path") +# to do this, but the source directory is disposable anyway, so we can move +# its contents to save time +find "$1" -maxdepth 1 -mindepth 1 -exec mv {} "$2/." + diff --git a/morphlib/exts/tar.check b/morphlib/exts/tar.check new file mode 100755 index 00000000..cbeaf163 --- /dev/null +++ b/morphlib/exts/tar.check @@ -0,0 +1,24 @@ +#!/bin/sh +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# Preparatory checks for Morph 'tar' write extension + +set -eu + +if [ "$UPGRADE" == "yes" ]; then + echo >&2 "ERROR: Cannot upgrade a tar file deployment." + exit 1 +fi diff --git a/morphlib/exts/virtualbox-ssh.check b/morphlib/exts/virtualbox-ssh.check new file mode 100755 index 00000000..1aeb8999 --- /dev/null +++ b/morphlib/exts/virtualbox-ssh.check @@ -0,0 +1,35 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''Preparatory checks for Morph 'virtualbox-ssh' write extension''' + +import cliapp + +import morphlib.writeexts + + +class VirtualBoxPlusSshCheckExtension(morphlib.writeexts.WriteExtension): + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + upgrade = self.get_environment_boolean('UPGRADE') + if upgrade: + raise cliapp.AppException( + 'Use the `ssh-rsync` write extension to deploy upgrades to an ' + 'existing remote system.') + +VirtualBoxPlusSshCheckExtension().run() diff --git a/morphlib/exts/virtualbox-ssh.write b/morphlib/exts/virtualbox-ssh.write index 204b2447..2a2f3f7b 100755 --- a/morphlib/exts/virtualbox-ssh.write +++ b/morphlib/exts/virtualbox-ssh.write @@ -62,7 +62,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): temp_root, location = args ssh_host, vm_name, vdi_path = self.parse_location(location) - autostart = self.parse_autostart() + autostart = self.get_environment_boolean('AUTOSTART') fd, raw_disk = tempfile.mkstemp() os.close(fd) diff --git a/morphlib/fsutils.py b/morphlib/fsutils.py index 0212b987..751f73f6 100644 --- a/morphlib/fsutils.py +++ b/morphlib/fsutils.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -47,10 +47,14 @@ def create_fs(runcmd, partition): # pragma: no cover runcmd(['mkfs.btrfs', '-L', 'baserock', partition]) -def mount(runcmd, partition, mount_point): # pragma: no cover +def mount(runcmd, partition, mount_point, fstype=None): # pragma: no cover if not os.path.exists(mount_point): os.mkdir(mount_point) - runcmd(['mount', partition, mount_point]) + if not fstype: + fstype = [] + else: + fstype = ['-t', fstype] + runcmd(['mount', partition, mount_point] + fstype) def unmount(runcmd, mount_point): # pragma: no cover diff --git a/morphlib/git.py b/morphlib/git.py index 27146206..ccd06323 100644 --- a/morphlib/git.py +++ b/morphlib/git.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -279,6 +279,10 @@ def copy_repository(runcmd, repo, destdir, is_mirror=True): def checkout_ref(runcmd, gitdir, ref): '''Checks out a specific ref/SHA1 in a git working tree.''' runcmd(['git', 'checkout', ref], cwd=gitdir) + gd = morphlib.gitdir.GitDirectory(gitdir) + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() def index_has_changes(runcmd, gitdir): @@ -308,6 +312,10 @@ def clone_into(runcmd, srcpath, targetpath, ref=None): runcmd(['git', 'checkout', ref], cwd=targetpath) else: runcmd(['git', 'clone', '-b', ref, srcpath, targetpath]) + gd = morphlib.gitdir.GitDirectory(targetpath) + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() def is_valid_sha1(ref): '''Checks whether a string is a valid SHA1.''' diff --git a/morphlib/gitdir.py b/morphlib/gitdir.py index be2137b2..3d0ab53e 100644 --- a/morphlib/gitdir.py +++ b/morphlib/gitdir.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -317,6 +317,14 @@ class Remote(object): self._parse_push_output(out), err) return self._parse_push_output(out) + def pull(self, branch=None): # pragma: no cover + if branch: + repo = self.get_fetch_url() + ret = self.gd._runcmd(['git', 'pull', repo, branch]) + else: + ret = self.gd._runcmd(['git', 'pull']) + return ret + class GitDirectory(object): @@ -330,7 +338,11 @@ class GitDirectory(object): ''' def __init__(self, dirname): - self.dirname = dirname + self.dirname = morphlib.util.find_root(dirname, '.git') + # if we are in a bare repo, self.dirname will now be None + # so we just use the provided dirname + if not self.dirname: + self.dirname = dirname def _runcmd(self, argv, **kwargs): '''Run a command at the root of the git directory. @@ -350,6 +362,9 @@ class GitDirectory(object): def checkout(self, branch_name): # pragma: no cover '''Check out a git branch.''' self._runcmd(['git', 'checkout', branch_name]) + if self.has_fat(): + self.fat_init() + self.fat_pull() def branch(self, new_branch_name, base_ref): # pragma: no cover '''Create a git branch based on an existing ref. @@ -478,7 +493,8 @@ class GitDirectory(object): if dirpath == self.dirname and '.git' in subdirs: subdirs.remove('.git') for filename in filenames: - yield os.path.join(dirpath, filename)[len(self.dirname)+1:] + filepath = os.path.join(dirpath, filename) + yield os.path.relpath(filepath, start=self.dirname) def _list_files_in_ref(self, ref): tree = self.resolve_ref_to_tree(ref) @@ -610,12 +626,35 @@ class GitDirectory(object): except Exception, e: raise RefDeleteError(self, ref, old_sha1, e) + def describe(self): + version = self._runcmd( + ['git', 'describe', '--always', '--dirty=-unreproducible']) + return version.strip() + + def fat_init(self): # pragma: no cover + return self._runcmd(['git', 'fat', 'init']) + + def fat_push(self): # pragma: no cover + return self._runcmd(['git', 'fat', 'push']) + + def fat_pull(self): # pragma: no cover + return self._runcmd(['git', 'fat', 'pull']) + + def has_fat(self): # pragma: no cover + return os.path.isfile(self.join_path('.gitfat')) + + def join_path(self, path): # pragma: no cover + return os.path.join(self.dirname, path) + + def get_relpath(self, path): # pragma: no cover + return os.path.relpath(path, self.dirname) + def init(dirname): '''Initialise a new git repository.''' + cliapp.runcmd(['git', 'init'], cwd=dirname) gd = GitDirectory(dirname) - gd._runcmd(['git', 'init']) return gd diff --git a/morphlib/gitdir_tests.py b/morphlib/gitdir_tests.py index 21a6b5b8..14b2a57a 100644 --- a/morphlib/gitdir_tests.py +++ b/morphlib/gitdir_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -82,9 +82,13 @@ class GitDirectoryContentsTests(unittest.TestCase): shutil.rmtree(self.tempdir) def test_lists_files_in_work_tree(self): + expected = ['bar.morph', 'baz.morph', 'foo.morph', 'quux'] + gd = morphlib.gitdir.GitDirectory(self.dirname) - self.assertEqual(sorted(gd.list_files()), - ['bar.morph', 'baz.morph', 'foo.morph', 'quux']) + self.assertEqual(sorted(gd.list_files()), expected) + + gd = morphlib.gitdir.GitDirectory(self.dirname + '/') + self.assertEqual(sorted(gd.list_files()), expected) def test_read_file_in_work_tree(self): gd = morphlib.gitdir.GitDirectory(self.dirname) @@ -202,6 +206,15 @@ class GitDirectoryContentsTests(unittest.TestCase): ) self.assertEqual(expected, gd.get_commit_contents(commit).split('\n')) + def test_describe(self): + gd = morphlib.gitdir.GitDirectory(self.dirname) + + gd._runcmd(['git', 'tag', '-a', '-m', 'Example', 'example', 'HEAD']) + self.assertEqual(gd.describe(), 'example-unreproducible') + + gd._runcmd(['git', 'reset', '--hard']) + self.assertEqual(gd.describe(), 'example') + class GitDirectoryRefTwiddlingTests(unittest.TestCase): diff --git a/morphlib/plugins/add_binary_plugin.py b/morphlib/plugins/add_binary_plugin.py new file mode 100644 index 00000000..1edae0e8 --- /dev/null +++ b/morphlib/plugins/add_binary_plugin.py @@ -0,0 +1,110 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import logging +import os +import urlparse + +import morphlib + + +class AddBinaryPlugin(cliapp.Plugin): + + '''Add a subcommand for dealing with large binary files.''' + + def enable(self): + self.app.add_subcommand( + 'add-binary', self.add_binary, arg_synopsis='FILENAME...') + + def disable(self): + pass + + def add_binary(self, binaries): + '''Add a binary file to the current repository. + + Command line argument: + + * `FILENAME...` is the binaries to be added to the repository. + + This checks for the existence of a .gitfat file in the repository. If + there is one then a line is added to .gitattributes telling it that + the given binary should be handled by git-fat. If there is no .gitfat + file then it is created, with the rsync remote pointing at the correct + directory on the Trove host. A line is then added to .gitattributes to + say that the given binary should be handled by git-fat. + + Example: + + morph add-binary big_binary.tar.gz + + ''' + if not binaries: + raise morphlib.Error('add-binary must get at least one argument') + + gd = morphlib.gitdir.GitDirectory(os.getcwd()) + gd.fat_init() + if not gd.has_fat(): + self._make_gitfat(gd) + self._handle_binaries(binaries, gd) + logging.info('Staged binaries for commit') + + def _handle_binaries(self, binaries, gd): + '''Add a filter for the given file, and then add it to the repo.''' + # begin by ensuring all paths given are relative to the root directory + files = [gd.get_relpath(os.path.realpath(binary)) + for binary in binaries] + + # now add any files that aren't already mentioned in .gitattributes to + # the file so that git fat knows what to do + attr_path = gd.join_path('.gitattributes') + if '.gitattributes' in gd.list_files(): + with open(attr_path, 'r') as attributes: + current = set(f.split()[0] for f in attributes) + else: + current = set() + to_add = set(files) - current + + # if we don't need to change .gitattributes then we can just do + # `git add <binaries>` + if not to_add: + gd.get_index().add_files_from_working_tree(files) + return + + with open(attr_path, 'a') as attributes: + for path in to_add: + attributes.write('%s filter=fat -crlf\n' % path) + + # we changed .gitattributes, so need to stage it for committing + files.append(attr_path) + gd.get_index().add_files_from_working_tree(files) + + def _make_gitfat(self, gd): + '''Make .gitfat point to the rsync directory for the repo.''' + remote = gd.get_remote('origin') + if not remote.get_push_url(): + raise Exception( + 'Remote `origin` does not have a push URL defined.') + url = urlparse.urlparse(remote.get_push_url()) + if url.scheme != 'ssh': + raise Exception( + 'Push URL for `origin` is not an SSH URL: %s' % url.geturl()) + fat_store = '%s:%s' % (url.netloc, url.path) + fat_path = gd.join_path('.gitfat') + with open(fat_path, 'w+') as gitfat: + gitfat.write('[rsync]\n') + gitfat.write('remote = %s' % fat_store) + gd.get_index().add_files_from_working_tree([fat_path]) diff --git a/morphlib/plugins/branch_and_merge_new_plugin.py b/morphlib/plugins/branch_and_merge_new_plugin.py index 94b2381c..51cba401 100644 --- a/morphlib/plugins/branch_and_merge_new_plugin.py +++ b/morphlib/plugins/branch_and_merge_new_plugin.py @@ -190,8 +190,12 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): with self._initializing_system_branch( ws, root_url, system_branch, cached_repo, base_ref) as (sb, gd): + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() + if not self._checkout_has_systems(gd): - raise BranchRootHasNoSystemsError(base_ref) + raise BranchRootHasNoSystemsError(root_url, base_ref) def branch(self, args): @@ -250,9 +254,12 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): gd.branch(system_branch, base_ref) gd.checkout(system_branch) + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() if not self._checkout_has_systems(gd): - raise BranchRootHasNoSystemsError(base_ref) + raise BranchRootHasNoSystemsError(root_url, base_ref) def _save_dirty_morphologies(self, loader, sb, morphs): logging.debug('Saving dirty morphologies: start') @@ -480,6 +487,9 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): gd.checkout(sb.system_branch_name) gd.update_submodules(self.app) gd.update_remotes() + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() # Change the refs to the chunk. if chunk_ref != sb.system_branch_name: diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 90c658a0..ae62b75d 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -16,6 +16,7 @@ import cliapp import contextlib +import json import os import shutil import stat @@ -25,23 +26,24 @@ import uuid import morphlib -# UGLY HACK: We need to re-use some code from the branch and merge -# plugin, so we import and instantiate that plugin. This needs to -# be fixed by refactoring the codebase so the shared code is in -# morphlib, not in a plugin. However, this hack lets us re-use -# code without copying it. -import morphlib.plugins.branch_and_merge_plugin + +class ExtensionNotFoundError(morphlib.Error): + pass class DeployPlugin(cliapp.Plugin): def enable(self): + group_deploy = 'Deploy Options' + self.app.settings.boolean(['upgrade'], + 'specify that you want to upgrade an ' + 'existing cluster of systems rather than do ' + 'an initial deployment', + group=group_deploy) + self.app.add_subcommand( 'deploy', self.deploy, arg_synopsis='CLUSTER [SYSTEM.KEY=VALUE]') - self.other = \ - morphlib.plugins.branch_and_merge_plugin.BranchAndMergePlugin() - self.other.app = self.app def disable(self): pass @@ -250,8 +252,20 @@ class DeployPlugin(cliapp.Plugin): are set as environment variables when either the configuration or the write extension runs (except `type` and `location`). + Deployment configuration is stored in the deployed system as + /baserock/deployment.meta. THIS CONTAINS ALL ENVIRONMENT VARIABLES SET + DURINGR DEPLOYMENT, so make sure you have no sensitive information in + your environment that is being leaked. As a special case, any + environment/deployment variable that contains 'PASSWORD' in its name is + stripped out and not stored in the final system. + ''' + # Nasty hack to allow deploying things of a different architecture + def validate(self, root_artifact): + pass + morphlib.buildcommand.BuildCommand._validate_architecture = validate + if not args: raise cliapp.AppException( 'Too few arguments to deploy command (see help)') @@ -318,65 +332,112 @@ class DeployPlugin(cliapp.Plugin): ref=build_ref, dirname=gd.dirname, remote=remote.get_push_url(), chatty=True) - for system in cluster_morphology['systems']: - self.deploy_system(build_command, root_repo_dir, - bb.root_repo_url, bb.root_ref, - system, env_vars) - - def deploy_system(self, build_command, root_repo_dir, build_repo, ref, - system, env_vars): - # Find the artifact to build - morph = system['morph'] - srcpool = build_command.create_source_pool(build_repo, ref, - morph + '.morph') - def validate(self, root_artifact): - pass - morphlib.buildcommand.BuildCommand._validate_architecture = validate + # Create a tempdir for this deployment to work in + deploy_tempdir = tempfile.mkdtemp( + dir=os.path.join(self.app.settings['tempdir'], 'deployments')) + try: + for system in cluster_morphology['systems']: + self.deploy_system(build_command, deploy_tempdir, + root_repo_dir, bb.root_repo_url, + bb.root_ref, system, env_vars, + parent_location='') + finally: + shutil.rmtree(deploy_tempdir) + + self.app.status(msg='Finished deployment') - artifact = build_command.resolve_artifacts(srcpool) - - deploy_defaults = system['deploy-defaults'] - deployments = system['deploy'] - for system_id, deploy_params in deployments.iteritems(): - user_env = morphlib.util.parse_environment_pairs( - os.environ, - [pair[len(system_id)+1:] - for pair in env_vars - if pair.startswith(system_id)]) - - final_env = dict(deploy_defaults.items() + - deploy_params.items() + - user_env.items()) - - deployment_type = final_env.pop('type', None) - if not deployment_type: - raise morphlib.Error('"type" is undefined ' - 'for system "%s"' % system_id) - - location = final_env.pop('location', None) - if not location: - raise morphlib.Error('"location" is undefined ' - 'for system "%s"' % system_id) - - morphlib.util.sanitize_environment(final_env) - self.do_deploy(build_command, root_repo_dir, ref, artifact, - deployment_type, location, final_env) - - def do_deploy(self, build_command, root_repo_dir, ref, artifact, - deployment_type, location, env): - - # Create a tempdir for this deployment to work in - deploy_tempdir = tempfile.mkdtemp( - dir=os.path.join(self.app.settings['tempdir'], 'deployments')) + def deploy_system(self, build_command, deploy_tempdir, + root_repo_dir, build_repo, ref, system, env_vars, + parent_location): + old_status_prefix = self.app.status_prefix + system_status_prefix = '%s[%s]' % (old_status_prefix, system['morph']) + self.app.status_prefix = system_status_prefix try: - # Create a tempdir to extract the rootfs in - system_tree = tempfile.mkdtemp(dir=deploy_tempdir) + # Find the artifact to build + morph = system['morph'] + srcpool = build_command.create_source_pool(build_repo, ref, + morph + '.morph') + + artifact = build_command.resolve_artifacts(srcpool) + + deploy_defaults = system.get('deploy-defaults', {}) + deployments = system['deploy'] + for system_id, deploy_params in deployments.iteritems(): + deployment_status_prefix = '%s[%s]' % ( + system_status_prefix, system_id) + self.app.status_prefix = deployment_status_prefix + try: + user_env = morphlib.util.parse_environment_pairs( + os.environ, + [pair[len(system_id)+1:] + for pair in env_vars + if pair.startswith(system_id)]) + + final_env = dict(deploy_defaults.items() + + deploy_params.items() + + user_env.items()) + + is_upgrade = ('yes' if self.app.settings['upgrade'] + else 'no') + final_env['UPGRADE'] = is_upgrade + + deployment_type = final_env.pop('type', None) + if not deployment_type: + raise morphlib.Error('"type" is undefined ' + 'for system "%s"' % system_id) + + location = final_env.pop('location', None) + if not location: + raise morphlib.Error('"location" is undefined ' + 'for system "%s"' % system_id) + + morphlib.util.sanitize_environment(final_env) + self.check_deploy(root_repo_dir, ref, deployment_type, + location, final_env) + system_tree = self.setup_deploy(build_command, + deploy_tempdir, + root_repo_dir, + ref, artifact, + deployment_type, + location, final_env) + for subsystem in system.get('subsystems', []): + self.deploy_system(build_command, deploy_tempdir, + root_repo_dir, build_repo, + ref, subsystem, env_vars, + parent_location=system_tree) + if parent_location: + deploy_location = os.path.join(parent_location, + location.lstrip('/')) + else: + deploy_location = location + self.run_deploy_commands(deploy_tempdir, final_env, + artifact, root_repo_dir, + ref, deployment_type, + system_tree, deploy_location) + finally: + self.app.status_prefix = system_status_prefix + finally: + self.app.status_prefix = old_status_prefix - # Extensions get a private tempdir so we can more easily clean - # up any files an extension left behind - deploy_private_tempdir = tempfile.mkdtemp(dir=deploy_tempdir) - env['TMPDIR'] = deploy_private_tempdir + def check_deploy(self, root_repo_dir, ref, deployment_type, location, env): + # Run optional write check extension. These are separate from the write + # extension because it may be several minutes before the write + # extension itself has the chance to raise an error. + try: + self._run_extension( + root_repo_dir, ref, deployment_type, '.check', + [location], env) + except ExtensionNotFoundError: + pass + + def setup_deploy(self, build_command, deploy_tempdir, root_repo_dir, ref, + artifact, deployment_type, location, env): + # deployment_type, location and env are only used for saving metadata + # Create a tempdir to extract the rootfs in + system_tree = tempfile.mkdtemp(dir=deploy_tempdir) + + try: # Unpack the artifact (tarball) to a temporary directory. self.app.status(msg='Unpacking system for configuration') @@ -397,7 +458,27 @@ class DeployPlugin(cliapp.Plugin): msg='System unpacked at %(system_tree)s', system_tree=system_tree) + self.app.status( + msg='Writing deployment metadata file') + metadata = self.create_metadata( + artifact, root_repo_dir, deployment_type, location, env) + metadata_path = os.path.join( + system_tree, 'baserock', 'deployment.meta') + with morphlib.savefile.SaveFile(metadata_path, 'w') as f: + f.write(json.dumps(metadata, indent=4, sort_keys=True)) + return system_tree + except Exception: + shutil.rmtree(system_tree) + raise + + def run_deploy_commands(self, deploy_tempdir, env, artifact, root_repo_dir, + ref, deployment_type, system_tree, location): + # Extensions get a private tempdir so we can more easily clean + # up any files an extension left behind + deploy_private_tempdir = tempfile.mkdtemp(dir=deploy_tempdir) + env['TMPDIR'] = deploy_private_tempdir + try: # Run configuration extensions. self.app.status(msg='Configure system') names = artifact.source.morphology['configuration-extensions'] @@ -423,9 +504,7 @@ class DeployPlugin(cliapp.Plugin): finally: # Cleanup. self.app.status(msg='Cleaning up') - shutil.rmtree(deploy_tempdir) - - self.app.status(msg='Finished deployment') + shutil.rmtree(deploy_private_tempdir) def _run_extension(self, gd, ref, name, kind, args, env): '''Run an extension. @@ -446,7 +525,7 @@ class DeployPlugin(cliapp.Plugin): code_dir = os.path.dirname(morphlib.__file__) ext_filename = os.path.join(code_dir, 'exts', name + kind) if not os.path.exists(ext_filename): - raise morphlib.Error( + raise ExtensionNotFoundError( 'Could not find extension %s%s' % (name, kind)) if not self._is_executable(ext_filename): raise morphlib.Error( @@ -464,7 +543,8 @@ class DeployPlugin(cliapp.Plugin): name=name, kind=kind) self.app.runcmd( [ext_filename] + args, - ['sh', '-c', 'while read l; do echo `date "+%F %T"` $l; done'], + ['sh', '-c', 'while read l; do echo `date "+%F %T"` "$1$l"; done', + '-', '%s[%s]' % (self.app.status_prefix, name + kind)], cwd=gd.dirname, env=env, stdout=None, stderr=None) if delete_ext: @@ -474,3 +554,41 @@ class DeployPlugin(cliapp.Plugin): st = os.stat(filename) mask = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH return (stat.S_IMODE(st.st_mode) & mask) != 0 + + def create_metadata(self, system_artifact, root_repo_dir, deployment_type, + location, env): + '''Deployment-specific metadata. + + The `build` and `deploy` operations must be from the same ref, so full + info on the root repo that the system came from is in + /baserock/${system_artifact}.meta and is not duplicated here. We do + store a `git describe` of the definitions.git repo as a convenience for + post-upgrade hooks that we may need to implement at a future date: + the `git describe` output lists the last tag, which will hopefully help + us to identify which release of a system was deployed without having to + keep a list of SHA1s somewhere or query a Trove. + + ''' + + def remove_passwords(env): + def is_password(key): + return 'PASSWORD' in key + return { k:v for k, v in env.iteritems() if not is_password(k) } + + meta = { + 'system-artifact-name': system_artifact.name, + 'configuration': remove_passwords(env), + 'deployment-type': deployment_type, + 'location': location, + 'definitions-version': { + 'describe': root_repo_dir.describe(), + }, + 'morph-version': { + 'ref': morphlib.gitversion.ref, + 'tree': morphlib.gitversion.tree, + 'commit': morphlib.gitversion.commit, + 'version': morphlib.gitversion.version, + }, + } + + return meta diff --git a/morphlib/plugins/push_pull_plugin.py b/morphlib/plugins/push_pull_plugin.py new file mode 100644 index 00000000..843de1a6 --- /dev/null +++ b/morphlib/plugins/push_pull_plugin.py @@ -0,0 +1,93 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import cliapp +import logging +import os + +import morphlib + + +class PushPullPlugin(cliapp.Plugin): + + '''Add subcommands to wrap the git push and pull commands.''' + + def enable(self): + self.app.add_subcommand( + 'push', self.push, arg_synopsis='REPO TARGET') + self.app.add_subcommand('pull', self.pull, arg_synopsis='[REMOTE]') + + def disable(self): + pass + + def push(self, args): + '''Push a branch to a remote repository. + + Command line arguments: + + * `REPO` is the repository to push your changes to. + + * `TARGET` is the branch to push to the repository. + + This is a wrapper for the `git push` command. It also deals with + pushing any binary files that have been added using git-fat. + + Example: + + morph push origin jrandom/new-feature + + ''' + if len(args) != 2: + raise morphlib.Error('push must get exactly two arguments') + + gd = morphlib.gitdir.GitDirectory(os.getcwd()) + remote, branch = args + rs = morphlib.gitdir.RefSpec(branch) + gd.get_remote(remote).push(rs) + if gd.has_fat(): + gd.fat_init() + gd.fat_push() + + def pull(self, args): + '''Pull changes to the current branch from a repository. + + Command line arguments: + + * `REMOTE` is the remote branch to pull from. By default this is the + branch being tracked by your current git branch (ie origin/master + for branch master) + + This is a wrapper for the `git pull` command. It also deals with + pulling any binary files that have been added to the repository using + git-fat. + + Example: + + morph pull + + ''' + if len(args) > 1: + raise morphlib.Error('pull takes at most one argument') + + gd = morphlib.gitdir.GitDirectory(os.getcwd()) + remote = gd.get_remote('origin') + if args: + branch = args[0] + remote.pull(branch) + else: + remote.pull() + if gd.has_fat(): + gd.fat_init() + gd.fat_pull() diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index 286e5374..61f9e660 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -160,6 +160,9 @@ class StagingArea(object): unpacked_artifact = os.path.join( chunk_cache_dir, os.path.basename(handle.name) + '.d') if not os.path.exists(unpacked_artifact): + self._app.status( + msg='Unpacking chunk from cache %(filename)s', + filename=os.path.basename(handle.name)) savedir = tempfile.mkdtemp(dir=chunk_cache_dir) try: morphlib.bins.unpack_binary_from_file( @@ -233,7 +236,7 @@ class StagingArea(object): path = os.path.join(self.dirname, mount_point) if not os.path.exists(path): os.makedirs(path) - self._app.runcmd(['mount', '-t', mount_type, source, path]) + morphlib.fsutils.mount(self._app.runcmd, source, path, mount_type) self.mounted.append(path) return diff --git a/morphlib/stagingarea_tests.py b/morphlib/stagingarea_tests.py index 27f85dff..52f495eb 100644 --- a/morphlib/stagingarea_tests.py +++ b/morphlib/stagingarea_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -57,7 +57,7 @@ class FakeApplication(object): def runcmd_unchecked(self, *args, **kwargs): return cliapp.runcmd_unchecked(*args, **kwargs) - def status(self, msg): + def status(self, **kwargs): pass diff --git a/morphlib/sysbranchdir.py b/morphlib/sysbranchdir.py index 1a8b898a..9d96e974 100644 --- a/morphlib/sysbranchdir.py +++ b/morphlib/sysbranchdir.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -74,6 +74,11 @@ class SystemBranchDirectory(object): If the URL is a real one (not aliased), the schema and leading // are removed from it, as is a .git suffix. + Any colons in the URL path or network location are replaced + with slashes, so that directory paths do not contain colons. + This avoids problems with PYTHONPATH, PATH, and other things + that use colon as a separator. + ''' # Parse the URL. If the path component is absolute, we assume @@ -93,6 +98,9 @@ class SystemBranchDirectory(object): else: relative = repo_url + # Replace colons with slashes. + relative = '/'.join(relative.split(':')) + # Remove anyleading slashes, or os.path.join below will only # use the relative part (since it's absolute, not relative). relative = relative.lstrip('/') diff --git a/morphlib/sysbranchdir_tests.py b/morphlib/sysbranchdir_tests.py index 7ec8ef5c..8b40f69c 100644 --- a/morphlib/sysbranchdir_tests.py +++ b/morphlib/sysbranchdir_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -149,7 +149,7 @@ class SystemBranchDirectoryTests(unittest.TestCase): self.system_branch_name) self.assertEqual( sb.get_git_directory_name('baserock:baserock/morph'), - os.path.join(self.root_directory, 'baserock:baserock/morph')) + os.path.join(self.root_directory, 'baserock/baserock/morph')) def test_reports_correct_name_for_git_directory_from_real_url(self): stripped = 'git.baserock.org/baserock/baserock/morph' @@ -169,7 +169,7 @@ class SystemBranchDirectoryTests(unittest.TestCase): self.system_branch_name) self.assertEqual( sb.get_filename('test:chunk', 'foo'), - os.path.join(self.root_directory, 'test:chunk/foo')) + os.path.join(self.root_directory, 'test/chunk/foo')) def test_reports_correct_name_for_git_directory_from_file_url(self): stripped = 'foobar/morphs' diff --git a/morphlib/workspace_tests.py b/morphlib/workspace_tests.py index b25be35e..9eef1053 100644 --- a/morphlib/workspace_tests.py +++ b/morphlib/workspace_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -97,7 +97,7 @@ class WorkspaceTests(unittest.TestCase): url = 'test:morphs' branch = 'my/new/thing' sb = ws.create_system_branch_directory(url, branch) - self.assertTrue(type(sb), morphlib.sysbranchdir.SystemBranchDirectory) + self.assertEqual(type(sb), morphlib.sysbranchdir.SystemBranchDirectory) def test_lists_created_system_branches(self): self.create_it() diff --git a/morphlib/writeexts.py b/morphlib/writeexts.py index 9dbc77e6..1849f406 100644 --- a/morphlib/writeexts.py +++ b/morphlib/writeexts.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -17,12 +17,65 @@ import cliapp import os import re +import shutil import sys import time import tempfile import morphlib + +class Fstab(object): + '''Small helper class for parsing and adding lines to /etc/fstab.''' + + # There is an existing Python helper library for editing of /etc/fstab. + # However it is unmaintained and has an incompatible license (GPL3). + # + # https://code.launchpad.net/~computer-janitor-hackers/python-fstab/trunk + + def __init__(self, filepath='/etc/fstab'): + if os.path.exists(filepath): + with open(filepath, 'r') as f: + self.text= f.read() + else: + self.text = '' + self.filepath = filepath + self.lines_added = 0 + + def get_mounts(self): + '''Return list of mount devices and targets in /etc/fstab. + + Return value is a dict of target -> device. + ''' + mounts = dict() + for line in self.text.splitlines(): + words = line.split() + if len(words) >= 2 and not words[0].startswith('#'): + device, target = words[0:2] + mounts[target] = device + return mounts + + def add_line(self, line): + '''Add a new entry to /etc/fstab. + + Lines are appended, and separated from any entries made by configure + extensions with a comment. + + ''' + if self.lines_added == 0: + if len(self.text) == 0 or self.text[-1] is not '\n': + self.text += '\n' + self.text += '# Morph default system layout\n' + self.lines_added += 1 + + self.text += line + '\n' + + def write(self): + '''Rewrite the fstab file to include all new entries.''' + with morphlib.savefile.SaveFile(self.filepath, 'w') as f: + f.write(self.text) + + class WriteExtension(cliapp.Application): '''A base class for deployment write extensions. @@ -53,7 +106,6 @@ class WriteExtension(cliapp.Application): def create_local_system(self, temp_root, raw_disk): '''Create a raw system image locally.''' - size = self.get_disk_size() if not size: raise cliapp.AppException('DISK_SIZE is not defined') @@ -66,19 +118,10 @@ class WriteExtension(cliapp.Application): os.remove(raw_disk) raise try: - version_label = 'factory' - version_root = os.path.join(mp, 'systems', version_label) - os.makedirs(version_root) - self.create_state(mp) - self.create_orig(version_root, temp_root) - self.create_fstab(version_root) - self.create_run(version_root) - os.symlink(version_label, os.path.join(mp, 'systems', 'default')) - if self.bootloader_is_wanted(): - self.install_kernel(version_root, temp_root) - self.install_extlinux(mp) + self.create_btrfs_system_layout( + temp_root, mp, version_label='factory') except BaseException, e: - sys.stderr.write('Error creating disk image') + sys.stderr.write('Error creating Btrfs system layout') self.unmount(mp) os.remove(raw_disk) raise @@ -129,16 +172,6 @@ class WriteExtension(cliapp.Application): '''Parse the virtual cpu count from environment.''' return self._parse_size_from_environment('VCPUS', '1') - def create_state(self, real_root): - '''Create the state subvolumes that are shared between versions''' - - self.status(msg='Creating state subvolumes') - os.mkdir(os.path.join(real_root, 'state')) - statedirs = ['home', 'opt', 'srv'] - for statedir in statedirs: - dirpath = os.path.join(real_root, 'state', statedir) - cliapp.runcmd(['btrfs', 'subvolume', 'create', dirpath]) - def create_raw_disk_image(self, filename, size): '''Create a raw disk image.''' @@ -178,6 +211,34 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['umount', mount_point]) os.rmdir(mount_point) + def create_btrfs_system_layout(self, temp_root, mountpoint, version_label): + '''Separate base OS versions from state using subvolumes. + + ''' + version_root = os.path.join(mountpoint, 'systems', version_label) + state_root = os.path.join(mountpoint, 'state') + + os.makedirs(version_root) + os.makedirs(state_root) + + self.create_orig(version_root, temp_root) + system_dir = os.path.join(version_root, 'orig') + + state_dirs = self.complete_fstab_for_btrfs_layout(system_dir) + + for state_dir in state_dirs: + self.create_state_subvolume(system_dir, mountpoint, state_dir) + + self.create_run(version_root) + + os.symlink( + version_label, os.path.join(mountpoint, 'systems', 'default')) + + if self.bootloader_is_wanted(): + self.install_kernel(version_root, temp_root) + self.install_syslinux_menu(mountpoint, version_root) + self.install_extlinux(mountpoint) + def create_orig(self, version_root, temp_root): '''Create the default "factory" system.''' @@ -197,29 +258,68 @@ class WriteExtension(cliapp.Application): cliapp.runcmd( ['btrfs', 'subvolume', 'snapshot', orig, run]) - def create_fstab(self, version_root): - '''Create an fstab.''' + def create_state_subvolume(self, system_dir, mountpoint, state_subdir): + '''Create a shared state subvolume. - self.status(msg='Creating fstab') - fstab = os.path.join(version_root, 'orig', 'etc', 'fstab') + We need to move any files added to the temporary rootfs by the + configure extensions to their correct home. For example, they might + have added keys in `/root/.ssh` which we now need to transfer to + `/state/root/.ssh`. - if os.path.exists(fstab): - with open(fstab, 'r') as f: - contents = f.read() - else: - contents = '' + ''' + self.status(msg='Creating %s subvolume' % state_subdir) + subvolume = os.path.join(mountpoint, 'state', state_subdir) + cliapp.runcmd(['btrfs', 'subvolume', 'create', subvolume]) + os.chmod(subvolume, 0755) + + existing_state_dir = os.path.join(system_dir, state_subdir) + files = [] + if os.path.exists(existing_state_dir): + files = os.listdir(existing_state_dir) + if len(files) > 0: + self.status(msg='Moving existing data to %s subvolume' % subvolume) + for filename in files: + filepath = os.path.join(existing_state_dir, filename) + shutil.move(filepath, subvolume) + + def complete_fstab_for_btrfs_layout(self, system_dir): + '''Fill in /etc/fstab entries for the default Btrfs disk layout. + + In the future we should move this code out of the write extension and + in to a configure extension. To do that, though, we need some way of + informing the configure extension what layout should be used. Right now + a configure extension doesn't know if the system is going to end up as + a Btrfs disk image, a tarfile or something else and so it can't come + up with a sensible default fstab. + + Configuration extensions can already create any /etc/fstab that they + like. This function only fills in entries that are missing, so if for + example the user configured /home to be on a separate partition, that + decision will be honoured and /state/home will not be created. - got_root = False - for line in contents.splitlines(): - words = line.split() - if len(words) >= 2 and not words[0].startswith('#'): - got_root = got_root or words[1] == '/' + ''' + shared_state_dirs = {'home', 'root', 'opt', 'srv', 'var'} + + fstab = Fstab(os.path.join(system_dir, 'etc', 'fstab')) + existing_mounts = fstab.get_mounts() + + if '/' in existing_mounts: + root_device = existing_mounts['/'] + else: + root_device = '/dev/sda' + fstab.add_line('/dev/sda / btrfs defaults,rw,noatime 0 1') - if not got_root: - contents += '\n/dev/sda / btrfs defaults,rw,noatime 0 1\n' + state_dirs_to_create = set() + for state_dir in shared_state_dirs: + if '/' + state_dir not in existing_mounts: + state_dirs_to_create.add(state_dir) + state_subvol = os.path.join('/state', state_dir) + fstab.add_line( + '%s /%s btrfs subvol=%s,defaults,rw,noatime 0 2' % + (root_device, state_dir, state_subvol)) - with open(fstab, 'w') as f: - f.write(contents) + fstab.write() + return state_dirs_to_create def install_kernel(self, version_root, temp_root): '''Install the kernel outside of 'orig' or 'run' subvolumes''' @@ -254,6 +354,23 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['sync']) time.sleep(2) + def install_syslinux_menu(self, real_root, version_root): + '''Make syslinux/extlinux menu binary available. + + The syslinux boot menu is compiled to a file named menu.c32. Extlinux + searches a few places for this file but it does not know to look inside + our subvolume, so we copy it to the filesystem root. + + If the file is not available, the bootloader will still work but will + not be able to show a menu. + + ''' + menu_file = os.path.join(version_root, 'orig', + 'usr', 'share', 'syslinux', 'menu.c32') + if os.path.isfile(menu_file): + self.status(msg='Copying menu.c32') + shutil.copy(menu_file, real_root) + def parse_attach_disks(self): '''Parse $ATTACH_DISKS into list of disks to attach.''' @@ -285,14 +402,14 @@ class WriteExtension(cliapp.Application): return value == 'yes' - def parse_autostart(self): - '''Parse $AUTOSTART to determine if VMs should be started.''' + def get_environment_boolean(self, variable): + '''Parse a yes/no boolean passed through the environment.''' - autostart = os.environ.get('AUTOSTART', 'no') - if autostart == 'no': + value = os.environ.get(variable, 'no').lower() + if value in ['no', '0', 'false']: return False - elif autostart == 'yes': + elif value in ['yes', '1', 'true']: return True else: - raise cliapp.AppException('Unexpected value for AUTOSTART: %s' % - autostart) + raise cliapp.AppException('Unexpected value for %s: %s' % + (variable, value)) diff --git a/scripts/edit-morph b/scripts/edit-morph index 2970cc6e..465a3ea3 100755 --- a/scripts/edit-morph +++ b/scripts/edit-morph @@ -16,6 +16,7 @@ import cliapp +import contextlib import os import re import yaml @@ -201,6 +202,15 @@ class EditMorph(cliapp.Application): return result + @staticmethod + @contextlib.contextmanager + def _open_yaml(path): + with open(path, 'r') as f: + d = yaml.load(f) + yield d + with open(path, 'w') as f: + yaml.dump(d, f) + def cmd_set_system_artifact_depends(self, args): '''Change the artifacts used by a System. @@ -219,13 +229,10 @@ class EditMorph(cliapp.Application): file_path = args[0] stratum_name = args[1] artifacts = re.split(r"\s+and\s+|,?\s*", args[2]) - with open(file_path, "r") as f: - d = yaml.load(f) - for spec in d["strata"]: - if spec.get("alias", spec["name"]) == stratum_name: - spec["artifacts"] = artifacts - with open(file_path, "w") as f: - yaml.dump(d, f) + with self._open_yaml(file_path) as d: + for spec in d["strata"]: + if spec.get("alias", spec["name"]) == stratum_name: + spec["artifacts"] = artifacts def cmd_set_stratum_match_rules(self, (file_path, match_rules)): '''Set a stratum's match rules. @@ -239,10 +246,89 @@ class EditMorph(cliapp.Application): The match rules must be a string that yaml can parse. ''' - with open(file_path, "r") as f: - d = yaml.load(f) - d['products'] = yaml.load(match_rules) - with open(file_path, "w") as f: + with self._open_yaml(file_path) as d: + d['products'] = yaml.load(match_rules) + + @classmethod + def _splice_cluster_system(cls, syslist, syspath): + sysname = syspath[0] + syspath = syspath[1:] + for system in syslist: + if sysname in system['deploy']: + break + else: + system = { + 'morph': None, + 'deploy': { + sysname: { + 'type': None, + 'location': None, + }, + }, + } + syslist.append(system) + if syspath: + cls._splice_cluster_system( + system.setdefault('subsystems', []), syspath) + + @classmethod + def _find_cluster_system(cls, syslist, syspath): + sysname = syspath[0] + syspath = syspath[1:] + for system in syslist: + if sysname in system['deploy']: + break + if syspath: + return cls._find_cluster_system(system['subsystems'], syspath) + return system + + def cmd_cluster_init(self, (cluster_file,)): + suffix = '.morph' + if not cluster_file.endswith(suffix): + raise cliapp.AppException( + "Morphology file path must end with .morph") + with open(cluster_file, 'w') as f: + d = { + 'name': os.path.basename(cluster_file)[:-len(suffix)], + 'kind': 'cluster', + } yaml.dump(d, f) + def cmd_cluster_system_init(self, (cluster_file, system_path)): + syspath = system_path.split('.') + with self._open_yaml(cluster_file) as d: + self._splice_cluster_system(d.setdefault('systems', []), syspath) + + def cmd_cluster_system_set_morphology(self, + (cluster_file, system_path, morphology)): + + syspath = system_path.split('.') + with self._open_yaml(cluster_file) as d: + system = self._find_cluster_system(d['systems'], syspath) + system['morph'] = morphology + + def cmd_cluster_system_set_deploy_type(self, + (cluster_file, system_path, deploy_type)): + + syspath = system_path.split('.') + with self._open_yaml(cluster_file) as d: + system = self._find_cluster_system(d['systems'], syspath) + system['deploy'][syspath[-1]]['type'] = deploy_type + + def cmd_cluster_system_set_deploy_location(self, + (cluster_file, system_path, deploy_location)): + + syspath = system_path.split('.') + with self._open_yaml(cluster_file) as d: + system = self._find_cluster_system(d['systems'], syspath) + system['deploy'][syspath[-1]]['location'] = deploy_location + + def cmd_cluster_system_set_deploy_variable(self, + (cluster_file, system_path, key, val)): + + syspath = system_path.split('.') + with self._open_yaml(cluster_file) as d: + system = self._find_cluster_system(d['systems'], syspath) + system['deploy'][syspath[-1]][key] = val + EditMorph().run() diff --git a/tests.as-root/branch-from-image-works.script b/tests.as-root/branch-from-image-works.script index c9d50bbb..fb0b09c9 100755 --- a/tests.as-root/branch-from-image-works.script +++ b/tests.as-root/branch-from-image-works.script @@ -50,7 +50,7 @@ workspace="$DATADIR/workspace" cd "$workspace" "$SRCDIR/scripts/test-morph" branch-from-image mybranch \ --metadata-dir="$extracted/baserock" -cd mybranch/test:morphs +cd mybranch/test/morphs grep -qFe "$hello_chunk_commit" hello-stratum.morph tar=$("$SRCDIR/scripts/test-morph" --find-system-artifact build hello-tarball) tar -xf "$tar" bin/hello diff --git a/tests.as-root/build-handles-stratum-build-depends.script b/tests.as-root/build-handles-stratum-build-depends.script index 22d07c72..6e6f82da 100755 --- a/tests.as-root/build-handles-stratum-build-depends.script +++ b/tests.as-root/build-handles-stratum-build-depends.script @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -33,7 +33,7 @@ set -eu cd "$DATADIR/workspace" "$SRCDIR/scripts/test-morph" branch test:morphs test/stratum-build-depends -cd test/stratum-build-depends/test:morphs +cd test/stratum-build-depends/test/morphs # 'linux-system' and the build-depends fields of 'linux-stratum' need to # be updated here. Any build-depends of any altered strata also need to diff --git a/tests.as-root/build-with-external-strata.script b/tests.as-root/build-with-external-strata.script index f5d86dfe..e43d0262 100755 --- a/tests.as-root/build-with-external-strata.script +++ b/tests.as-root/build-with-external-strata.script @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -35,7 +35,7 @@ cd "$DATADIR/workspace" # 'morph build'. cd "branch1" "$SRCDIR/scripts/test-morph" edit hello-system stratum2 -cd "test:external-strata" +cd "test/external-strata" awk ' /^chunks:/ { diff --git a/tests.as-root/building-a-system-branch-multiple-times-doesnt-generate-new-artifacts.script b/tests.as-root/building-a-system-branch-multiple-times-doesnt-generate-new-artifacts.script index 8852b96d..ac6cffec 100755 --- a/tests.as-root/building-a-system-branch-multiple-times-doesnt-generate-new-artifacts.script +++ b/tests.as-root/building-a-system-branch-multiple-times-doesnt-generate-new-artifacts.script @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -38,6 +38,6 @@ cd "$DATADIR/workspace/branch1" [ "$ARTIFACT_COUNT" -eq $(ls "$DATADIR/cache/artifacts" | wc -l) ] # Build thrice, and that should be enough. -cd "$DATADIR/workspace/branch1/test:morphs" +cd "$DATADIR/workspace/branch1/test/morphs" "$SRCDIR/scripts/test-morph" build linux-system [ "$ARTIFACT_COUNT" -eq $(ls "$DATADIR/cache/artifacts" | wc -l) ] diff --git a/tests.as-root/building-a-system-branch-picks-up-committed-removes.script b/tests.as-root/building-a-system-branch-picks-up-committed-removes.script index fbfd2c0f..64ae82c7 100755 --- a/tests.as-root/building-a-system-branch-picks-up-committed-removes.script +++ b/tests.as-root/building-a-system-branch-picks-up-committed-removes.script @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,7 +24,7 @@ set -e . "$SRCDIR/tests.as-root/setup-build" KERNEL_BRANCH=baserock/builds/123456789/AABBCCDDE -cd "$DATADIR/workspace/branch1/test:kernel-repo" +cd "$DATADIR/workspace/branch1/test/kernel-repo" git checkout --quiet master echo Use Morph >README git add README @@ -36,7 +36,7 @@ cd "$DATADIR/workspace" # Make a change elsewhere to be pulled in PEER_REPO="$DATADIR/peer-kernel" -git clone --quiet "file://$DATADIR/workspace/branch1/test:kernel-repo" \ +git clone --quiet "file://$DATADIR/workspace/branch1/test/kernel-repo" \ "$PEER_REPO" cd "$PEER_REPO" git checkout --quiet -b fix @@ -47,7 +47,7 @@ git checkout --quiet master git merge --no-ff fix >/dev/null 2>&1 # Pull a commit in to the linux morphology. -cd "$DATADIR/workspace/branch1/test:kernel-repo" +cd "$DATADIR/workspace/branch1/test/kernel-repo" git remote add peer "file://$PEER_REPO" git remote update >/dev/null 2>&1 git merge --quiet peer/master @@ -55,8 +55,8 @@ git merge --quiet peer/master # Build the linux system again without comitting. cd "$DATADIR/workspace" "$SRCDIR/scripts/test-morph" build linux-system -cd branch1/test:kernel-repo +cd branch1/test/kernel-repo # Check whether the new morphology exists in the temporary build ref -cd "$DATADIR/workspace/branch1/test:kernel-repo" +cd "$DATADIR/workspace/branch1/test/kernel-repo" ! git cat-file blob "$KERNEL_BRANCH:README" >/dev/null 2>&1 diff --git a/tests.as-root/building-a-system-branch-picks-up-uncommitted-changes.script b/tests.as-root/building-a-system-branch-picks-up-uncommitted-changes.script index 8d298010..4dacb23e 100755 --- a/tests.as-root/building-a-system-branch-picks-up-uncommitted-changes.script +++ b/tests.as-root/building-a-system-branch-picks-up-uncommitted-changes.script @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -32,13 +32,13 @@ cd "$DATADIR/workspace" "$SRCDIR/scripts/test-morph" build linux-system # Print tree SHA1s of the build ref of morphs and kernel. -cd "$DATADIR/workspace/branch1/test:morphs" +cd "$DATADIR/workspace/branch1/test/morphs" MORPHS_SHA1="$(git rev-parse baserock/builds/123456789/987654321)" -cd "$DATADIR/workspace/branch1/test:kernel-repo" +cd "$DATADIR/workspace/branch1/test/kernel-repo" KERNEL_SHA1="$(git rev-parse baserock/builds/123456789/AABBCCDDE)" # Make an uncommitted change to the linux morphology. -cd "$DATADIR/workspace/branch1/test:kernel-repo" +cd "$DATADIR/workspace/branch1/test/kernel-repo" sed -i -e 's@touch@touch foo@g' linux.morph # Build the linux system again without comitting. @@ -49,7 +49,7 @@ cd "$DATADIR/workspace" # This time the tree SHA1 of morphs should be the same # but that of the kernel repo should be different because we # made a change. -cd "$DATADIR/workspace/branch1/test:morphs" +cd "$DATADIR/workspace/branch1/test/morphs" [ "$(git rev-parse baserock/builds/123456789/987654321)" != "$MORPHS_SHA1" ] -cd "$DATADIR/workspace/branch1/test:kernel-repo" +cd "$DATADIR/workspace/branch1/test/kernel-repo" [ "$(git rev-parse baserock/builds/123456789/AABBCCDDE)" != "$KERNEL_SHA1" ] diff --git a/tests.as-root/building-a-system-branch-works-anywhere.script b/tests.as-root/building-a-system-branch-works-anywhere.script index d5d1e52d..cf946cd5 100755 --- a/tests.as-root/building-a-system-branch-works-anywhere.script +++ b/tests.as-root/building-a-system-branch-works-anywhere.script @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -41,13 +41,13 @@ cd "$DATADIR/workspace/branch1" rm -rf "$DATADIR/cache/artifacts"/* # Build form the branch root repository. -cd "$DATADIR/workspace/branch1/test:morphs" +cd "$DATADIR/workspace/branch1/test/morphs" "$SRCDIR/scripts/test-morph" build linux-system "$SRCDIR/scripts/list-tree" "$DATADIR/cache/artifacts" > "$DATADIR/output3" rm -rf "$DATADIR/cache/artifacts"/* # Build from the linux directory. -cd "$DATADIR/workspace/branch1/test:kernel-repo" +cd "$DATADIR/workspace/branch1/test/kernel-repo" "$SRCDIR/scripts/test-morph" build linux-system "$SRCDIR/scripts/list-tree" "$DATADIR/cache/artifacts" > "$DATADIR/output4" rm -rf "$DATADIR/cache/artifacts"/* diff --git a/tests.as-root/building-creates-correct-temporary-refs.script b/tests.as-root/building-creates-correct-temporary-refs.script index c0bf6a1e..6fb6c83a 100755 --- a/tests.as-root/building-creates-correct-temporary-refs.script +++ b/tests.as-root/building-creates-correct-temporary-refs.script @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -35,10 +35,10 @@ cd "$DATADIR/workspace" # Verify that the right temporary refs were created. echo "Refs of morphs repo after first build:" -"$SRCDIR/scripts/run-git-in" "$DATADIR/workspace/branch1/test:morphs" \ +"$SRCDIR/scripts/run-git-in" "$DATADIR/workspace/branch1/test/morphs" \ show-ref | cut -d' ' -f2 echo "Refs of kernel repo after first build:" -"$SRCDIR/scripts/run-git-in" "$DATADIR/workspace/branch1/test:kernel-repo" \ +"$SRCDIR/scripts/run-git-in" "$DATADIR/workspace/branch1/test/kernel-repo" \ show-ref | cut -d' ' -f2 echo @@ -48,8 +48,8 @@ cd "$DATADIR/workspace" # Verify that the right temporary refs were created. echo "Refs of morphs repo after second build:" -"$SRCDIR/scripts/run-git-in" "$DATADIR/workspace/branch1/test:morphs" \ +"$SRCDIR/scripts/run-git-in" "$DATADIR/workspace/branch1/test/morphs" \ show-ref | cut -d' ' -f2 echo "Refs of kernel repo after second build:" -"$SRCDIR/scripts/run-git-in" "$DATADIR/workspace/branch1/test:kernel-repo" \ +"$SRCDIR/scripts/run-git-in" "$DATADIR/workspace/branch1/test/kernel-repo" \ show-ref | cut -d' ' -f2 diff --git a/tests.as-root/metadata-includes-morph-version.setup b/tests.as-root/metadata-includes-morph-version.setup index 31829b01..d7fc96e3 100755 --- a/tests.as-root/metadata-includes-morph-version.setup +++ b/tests.as-root/metadata-includes-morph-version.setup @@ -27,7 +27,7 @@ cat <<EOF > hello-tarball.morph { "name": "hello-tarball", "kind": "system", - "arch": "$(uname -m)", + "arch": "$("$SRCDIR/scripts/test-morph" print-architecture)", "strata": [ { "morph": "hello-stratum", diff --git a/tests.as-root/metadata-includes-repo-alias.setup b/tests.as-root/metadata-includes-repo-alias.setup index 31829b01..d7fc96e3 100755 --- a/tests.as-root/metadata-includes-repo-alias.setup +++ b/tests.as-root/metadata-includes-repo-alias.setup @@ -27,7 +27,7 @@ cat <<EOF > hello-tarball.morph { "name": "hello-tarball", "kind": "system", - "arch": "$(uname -m)", + "arch": "$("$SRCDIR/scripts/test-morph" print-architecture)", "strata": [ { "morph": "hello-stratum", diff --git a/tests.as-root/rootfs-tarball-builds-rootfs-and-kernel.script b/tests.as-root/rootfs-tarball-builds-rootfs-and-kernel.script index 93cd135f..e0829968 100755 --- a/tests.as-root/rootfs-tarball-builds-rootfs-and-kernel.script +++ b/tests.as-root/rootfs-tarball-builds-rootfs-and-kernel.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011, 2012, 2013 Codethink Limited +# Copyright (C) 2011, 2012, 2013, 2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -23,7 +23,7 @@ set -eu cache="$DATADIR/cache/artifacts" -arch=$(uname -m) +arch=$("$SRCDIR/scripts/test-morph" print-architecture) cd "$DATADIR/kernel-repo" cat <<EOF >linux.morph diff --git a/tests.as-root/run-in-artifact-with-different-artifacts.script b/tests.as-root/run-in-artifact-with-different-artifacts.script index ff944af4..57d408e3 100755 --- a/tests.as-root/run-in-artifact-with-different-artifacts.script +++ b/tests.as-root/run-in-artifact-with-different-artifacts.script @@ -44,4 +44,4 @@ echo # Run 'run-in-artifact' with the statum artifact. echo "Stratum:" "$SRCDIR/scripts/test-morph" run-in-artifact "$stratum" -- ls baserock/ \ - || echo "Failed" + 2>/dev/null || echo "Failed" diff --git a/tests.as-root/run-in-artifact-with-different-artifacts.stderr b/tests.as-root/run-in-artifact-with-different-artifacts.stderr deleted file mode 100644 index 9241fbb5..00000000 --- a/tests.as-root/run-in-artifact-with-different-artifacts.stderr +++ /dev/null @@ -1 +0,0 @@ -ERROR: Artifact TMP/cache/artifacts/e8e771e742c8f2199bd9f1e29cbeca07dbd51dd8880136d6521093711d37d8bf.stratum.linux-stratum-runtime cannot be extracted or mounted diff --git a/tests.as-root/setup b/tests.as-root/setup index b865f42d..1cf9dd04 100755 --- a/tests.as-root/setup +++ b/tests.as-root/setup @@ -132,7 +132,7 @@ git add tools-stratum.morph cat <<EOF > hello-system.morph name: hello-system kind: system -arch: `uname -m` +arch: `"$SRCDIR/scripts/test-morph" print-architecture` strata: - morph: hello-stratum EOF @@ -155,7 +155,7 @@ git add linux-stratum.morph cat <<EOF > linux-system.morph name: linux-system kind: system -arch: `uname -m` +arch: `"$SRCDIR/scripts/test-morph" print-architecture` strata: - morph: hello-stratum - morph: linux-stratum diff --git a/tests.as-root/system-overlap.script b/tests.as-root/system-overlap.script index c6154e0e..1ce8379d 100755 --- a/tests.as-root/system-overlap.script +++ b/tests.as-root/system-overlap.script @@ -31,7 +31,7 @@ cat <<EOF >overlap-system.morph { "name": "overlap-system", "kind": "system", - "arch": "$(uname -m)", + "arch": "$("$SRCDIR/scripts/test-morph" print-architecture)", "strata": [ { "morph": "foo-baz-stratum", diff --git a/tests.as-root/tarball-image-is-sensible.setup b/tests.as-root/tarball-image-is-sensible.setup index 505707b3..c47a5336 100755 --- a/tests.as-root/tarball-image-is-sensible.setup +++ b/tests.as-root/tarball-image-is-sensible.setup @@ -46,7 +46,7 @@ cat <<EOF > hello-tarball.morph { "name": "hello-tarball", "kind": "system", - "arch": "$(uname -m)", + "arch": "$("$SRCDIR/scripts/test-morph" print-architecture)", "strata": [ { "morph": "link-stratum", diff --git a/tests.as-root/tarball-image-is-sensible.stdout b/tests.as-root/tarball-image-is-sensible.stdout index 4141dee8..cf74a1ec 100644 --- a/tests.as-root/tarball-image-is-sensible.stdout +++ b/tests.as-root/tarball-image-is-sensible.stdout @@ -33,7 +33,6 @@ ./boot/System.map ./boot/vmlinuz ./etc/ -./etc/fstab ./etc/os-release ./extlinux.conf NAME="Baserock" diff --git a/tests.as-root/unimportant-morphology-contents-do-not-change-cache-keys.script b/tests.as-root/unimportant-morphology-contents-do-not-change-cache-keys.script index ca92b2cb..a540cdee 100755 --- a/tests.as-root/unimportant-morphology-contents-do-not-change-cache-keys.script +++ b/tests.as-root/unimportant-morphology-contents-do-not-change-cache-keys.script @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -36,8 +36,8 @@ cd "$DATADIR/workspace" ARTIFACT_COUNT="$(ls "$DATADIR/cache/artifacts" | wc -l)" # Petrify the refs, so the morphologies will be different -(set -e && cd branch1/test:morphs && git push --quiet origin HEAD) -(set -e && cd branch1/test:kernel-repo && git push --quiet origin HEAD) +(set -e && cd branch1/test/morphs && git push --quiet origin HEAD) +(set -e && cd branch1/test/kernel-repo && git push --quiet origin HEAD) "$SRCDIR/scripts/test-morph" petrify # Build with the petrified morphologies. diff --git a/tests.branching/add-then-edit.script b/tests.branching/add-then-edit.script index 5cd6e842..2dd62254 100755 --- a/tests.branching/add-then-edit.script +++ b/tests.branching/add-then-edit.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -27,7 +27,7 @@ cd "$DATADIR/workspace" cd "me/add-then-edit" # add a chunk -cd test:morphs +cd test/morphs ## Sub-optimally, to alter the stratum, you have to `morph edit` it first "$SRCDIR/scripts/test-morph" edit hello-system hello-stratum @@ -51,4 +51,4 @@ with open("hello-stratum.morph", "w") as f: grep -qFe goodbye hello-stratum.morph # check whether edit has cloned the repository to the right branch -git --git-dir="../test:goodbye/.git" rev-parse --abbrev-ref HEAD +git --git-dir="../goodbye/.git" rev-parse --abbrev-ref HEAD diff --git a/tests.branching/ambiguous-refs.script b/tests.branching/ambiguous-refs.script index ed72f9e3..aeec61a1 100755 --- a/tests.branching/ambiguous-refs.script +++ b/tests.branching/ambiguous-refs.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -37,10 +37,10 @@ cd "$DATADIR/workspace" "$SRCDIR/scripts/test-morph" branch test:morphs release # Create an extra ref to confuse any users of git show-ref -cd release/test:morphs +cd release/test/morphs git checkout --quiet -b alpha/master HEAD~1 git checkout --quiet release # The petrify will fail if we resolved 'master' as 'alpha/master' by mistake. -cd "$DATADIR/workspace/release/test:morphs" +cd "$DATADIR/workspace/release/test/morphs" "$SRCDIR/scripts/test-morph" petrify diff --git a/tests.branching/branch-creates-new-system-branch-not-from-master.script b/tests.branching/branch-creates-new-system-branch-not-from-master.script index 72e21740..c561f191 100755 --- a/tests.branching/branch-creates-new-system-branch-not-from-master.script +++ b/tests.branching/branch-creates-new-system-branch-not-from-master.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -31,8 +31,8 @@ echo "File tree:" grep -v 'cache/gits/file_[^/]*/' echo "Current branches:" -"$SRCDIR/scripts/run-git-in" newbranch/test:morphs branch +"$SRCDIR/scripts/run-git-in" newbranch/test/morphs branch echo "Current origin:" -"$SRCDIR/scripts/run-git-in" newbranch/test:morphs remote show origin | +"$SRCDIR/scripts/run-git-in" newbranch/test/morphs remote show origin | sed 's,\(TMP/workspace/\.morph/cache/gits/file_\).*_,\1,g' diff --git a/tests.branching/branch-creates-new-system-branch-not-from-master.stdout b/tests.branching/branch-creates-new-system-branch-not-from-master.stdout index af65eb7b..c61624b4 100644 --- a/tests.branching/branch-creates-new-system-branch-not-from-master.stdout +++ b/tests.branching/branch-creates-new-system-branch-not-from-master.stdout @@ -3,12 +3,13 @@ d . d ./.morph d ./newbranch d ./newbranch/.morph-system-branch -d ./newbranch/test:morphs -d ./newbranch/test:morphs/.git +d ./newbranch/test +d ./newbranch/test/morphs +d ./newbranch/test/morphs/.git f ./newbranch/.morph-system-branch/config -f ./newbranch/test:morphs/hello-stratum.morph -f ./newbranch/test:morphs/hello-system.morph -f ./newbranch/test:morphs/this.is.alfred +f ./newbranch/test/morphs/hello-stratum.morph +f ./newbranch/test/morphs/hello-system.morph +f ./newbranch/test/morphs/this.is.alfred Current branches: alfred * newbranch diff --git a/tests.branching/branch-creates-new-system-branch.script b/tests.branching/branch-creates-new-system-branch.script index c2d7f640..784bed62 100755 --- a/tests.branching/branch-creates-new-system-branch.script +++ b/tests.branching/branch-creates-new-system-branch.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -31,8 +31,8 @@ echo "File tree:" grep -v 'cache/gits/file_[^/]*/' echo "Current branches:" -"$SRCDIR/scripts/run-git-in" newbranch/test:morphs branch +"$SRCDIR/scripts/run-git-in" newbranch/test/morphs branch echo "Current origin:" -"$SRCDIR/scripts/run-git-in" newbranch/test:morphs remote show origin | +"$SRCDIR/scripts/run-git-in" newbranch/test/morphs remote show origin | sed 's,\(TMP/workspace/\.morph/cache/gits/file_\).*_,\1,g' diff --git a/tests.branching/branch-creates-new-system-branch.stdout b/tests.branching/branch-creates-new-system-branch.stdout index ba1651e5..a7318378 100644 --- a/tests.branching/branch-creates-new-system-branch.stdout +++ b/tests.branching/branch-creates-new-system-branch.stdout @@ -3,11 +3,12 @@ d . d ./.morph d ./newbranch d ./newbranch/.morph-system-branch -d ./newbranch/test:morphs -d ./newbranch/test:morphs/.git +d ./newbranch/test +d ./newbranch/test/morphs +d ./newbranch/test/morphs/.git f ./newbranch/.morph-system-branch/config -f ./newbranch/test:morphs/hello-stratum.morph -f ./newbranch/test:morphs/hello-system.morph +f ./newbranch/test/morphs/hello-stratum.morph +f ./newbranch/test/morphs/hello-system.morph Current branches: master * newbranch diff --git a/tests.branching/branch-works-anywhere.script b/tests.branching/branch-works-anywhere.script index ee8f5bfa..7f6156ce 100755 --- a/tests.branching/branch-works-anywhere.script +++ b/tests.branching/branch-works-anywhere.script @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -53,7 +53,7 @@ echo "Workspace after creating the third branch:" # Now, go into the morphs repository of that third branch and # create a fourth system branch from in there. This, too, should # end up being created in the toplevel workspace directory. -cd "$DATADIR/workspace/branch3/test:morphs" +cd "$DATADIR/workspace/branch3/test/morphs" "$SRCDIR/scripts/test-morph" branch test:morphs branch4 echo "Workspace after creating the fourth branch:" diff --git a/tests.branching/branch-works-anywhere.stdout b/tests.branching/branch-works-anywhere.stdout index 0b59ce71..4e317902 100644 --- a/tests.branching/branch-works-anywhere.stdout +++ b/tests.branching/branch-works-anywhere.stdout @@ -3,80 +3,90 @@ d . d ./.morph d ./branch1 d ./branch1/.morph-system-branch -d ./branch1/test:morphs -d ./branch1/test:morphs/.git +d ./branch1/test +d ./branch1/test/morphs +d ./branch1/test/morphs/.git f ./branch1/.morph-system-branch/config -f ./branch1/test:morphs/hello-stratum.morph -f ./branch1/test:morphs/hello-system.morph +f ./branch1/test/morphs/hello-stratum.morph +f ./branch1/test/morphs/hello-system.morph Workspace after creating the second branch: d . d ./.morph d ./branch1 d ./branch1/.morph-system-branch -d ./branch1/test:morphs -d ./branch1/test:morphs/.git +d ./branch1/test +d ./branch1/test/morphs +d ./branch1/test/morphs/.git d ./branch2 d ./branch2/.morph-system-branch -d ./branch2/test:morphs -d ./branch2/test:morphs/.git +d ./branch2/test +d ./branch2/test/morphs +d ./branch2/test/morphs/.git f ./branch1/.morph-system-branch/config -f ./branch1/test:morphs/hello-stratum.morph -f ./branch1/test:morphs/hello-system.morph +f ./branch1/test/morphs/hello-stratum.morph +f ./branch1/test/morphs/hello-system.morph f ./branch2/.morph-system-branch/config -f ./branch2/test:morphs/hello-stratum.morph -f ./branch2/test:morphs/hello-system.morph +f ./branch2/test/morphs/hello-stratum.morph +f ./branch2/test/morphs/hello-system.morph Workspace after creating the third branch: d . d ./.morph d ./branch1 d ./branch1/.morph-system-branch -d ./branch1/test:morphs -d ./branch1/test:morphs/.git +d ./branch1/test +d ./branch1/test/morphs +d ./branch1/test/morphs/.git d ./branch2 d ./branch2/.morph-system-branch -d ./branch2/test:morphs -d ./branch2/test:morphs/.git +d ./branch2/test +d ./branch2/test/morphs +d ./branch2/test/morphs/.git d ./branch3 d ./branch3/.morph-system-branch -d ./branch3/test:morphs -d ./branch3/test:morphs/.git +d ./branch3/test +d ./branch3/test/morphs +d ./branch3/test/morphs/.git f ./branch1/.morph-system-branch/config -f ./branch1/test:morphs/hello-stratum.morph -f ./branch1/test:morphs/hello-system.morph +f ./branch1/test/morphs/hello-stratum.morph +f ./branch1/test/morphs/hello-system.morph f ./branch2/.morph-system-branch/config -f ./branch2/test:morphs/hello-stratum.morph -f ./branch2/test:morphs/hello-system.morph +f ./branch2/test/morphs/hello-stratum.morph +f ./branch2/test/morphs/hello-system.morph f ./branch3/.morph-system-branch/config -f ./branch3/test:morphs/hello-stratum.morph -f ./branch3/test:morphs/hello-system.morph +f ./branch3/test/morphs/hello-stratum.morph +f ./branch3/test/morphs/hello-system.morph Workspace after creating the fourth branch: d . d ./.morph d ./branch1 d ./branch1/.morph-system-branch -d ./branch1/test:morphs -d ./branch1/test:morphs/.git +d ./branch1/test +d ./branch1/test/morphs +d ./branch1/test/morphs/.git d ./branch2 d ./branch2/.morph-system-branch -d ./branch2/test:morphs -d ./branch2/test:morphs/.git +d ./branch2/test +d ./branch2/test/morphs +d ./branch2/test/morphs/.git d ./branch3 d ./branch3/.morph-system-branch -d ./branch3/test:morphs -d ./branch3/test:morphs/.git +d ./branch3/test +d ./branch3/test/morphs +d ./branch3/test/morphs/.git d ./branch4 d ./branch4/.morph-system-branch -d ./branch4/test:morphs -d ./branch4/test:morphs/.git +d ./branch4/test +d ./branch4/test/morphs +d ./branch4/test/morphs/.git f ./branch1/.morph-system-branch/config -f ./branch1/test:morphs/hello-stratum.morph -f ./branch1/test:morphs/hello-system.morph +f ./branch1/test/morphs/hello-stratum.morph +f ./branch1/test/morphs/hello-system.morph f ./branch2/.morph-system-branch/config -f ./branch2/test:morphs/hello-stratum.morph -f ./branch2/test:morphs/hello-system.morph +f ./branch2/test/morphs/hello-stratum.morph +f ./branch2/test/morphs/hello-system.morph f ./branch3/.morph-system-branch/config -f ./branch3/test:morphs/hello-stratum.morph -f ./branch3/test:morphs/hello-system.morph +f ./branch3/test/morphs/hello-stratum.morph +f ./branch3/test/morphs/hello-system.morph f ./branch4/.morph-system-branch/config -f ./branch4/test:morphs/hello-stratum.morph -f ./branch4/test:morphs/hello-system.morph +f ./branch4/test/morphs/hello-stratum.morph +f ./branch4/test/morphs/hello-system.morph diff --git a/tests.branching/checkout-existing-branch.script b/tests.branching/checkout-existing-branch.script index 653fffb5..b1740d9c 100755 --- a/tests.branching/checkout-existing-branch.script +++ b/tests.branching/checkout-existing-branch.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -30,4 +30,4 @@ echo "File tree:" grep -v 'cache/gits/file_[^/]*/' echo "Current branches:" -"$SRCDIR/scripts/run-git-in" master/test:morphs branch +"$SRCDIR/scripts/run-git-in" master/test/morphs branch diff --git a/tests.branching/checkout-existing-branch.stdout b/tests.branching/checkout-existing-branch.stdout index 0f51893a..a6026269 100644 --- a/tests.branching/checkout-existing-branch.stdout +++ b/tests.branching/checkout-existing-branch.stdout @@ -3,10 +3,11 @@ d . d ./.morph d ./master d ./master/.morph-system-branch -d ./master/test:morphs -d ./master/test:morphs/.git +d ./master/test +d ./master/test/morphs +d ./master/test/morphs/.git f ./master/.morph-system-branch/config -f ./master/test:morphs/hello-stratum.morph -f ./master/test:morphs/hello-system.morph +f ./master/test/morphs/hello-stratum.morph +f ./master/test/morphs/hello-system.morph Current branches: * master diff --git a/tests.branching/checkout-works-anywhere.script b/tests.branching/checkout-works-anywhere.script index 02deb4d7..14d18842 100755 --- a/tests.branching/checkout-works-anywhere.script +++ b/tests.branching/checkout-works-anywhere.script @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -41,7 +41,7 @@ cd workspace # This time, create a new branch and check out the master branch # from within that branch. "$SRCDIR/scripts/test-morph" branch test:morphs newbranch -cd newbranch/test:morphs +cd newbranch/test/morphs "$SRCDIR/scripts/test-morph" checkout test:morphs master echo "Workspace after checking out master from within a new branch:" diff --git a/tests.branching/checkout-works-anywhere.stdout b/tests.branching/checkout-works-anywhere.stdout index ba83058f..ed8b1567 100644 --- a/tests.branching/checkout-works-anywhere.stdout +++ b/tests.branching/checkout-works-anywhere.stdout @@ -3,25 +3,28 @@ d . d ./.morph d ./master d ./master/.morph-system-branch -d ./master/test:morphs -d ./master/test:morphs/.git +d ./master/test +d ./master/test/morphs +d ./master/test/morphs/.git f ./master/.morph-system-branch/config -f ./master/test:morphs/hello-stratum.morph -f ./master/test:morphs/hello-system.morph +f ./master/test/morphs/hello-stratum.morph +f ./master/test/morphs/hello-system.morph Workspace after checking out master from within a new branch: d . d ./.morph d ./master d ./master/.morph-system-branch -d ./master/test:morphs -d ./master/test:morphs/.git +d ./master/test +d ./master/test/morphs +d ./master/test/morphs/.git d ./newbranch d ./newbranch/.morph-system-branch -d ./newbranch/test:morphs -d ./newbranch/test:morphs/.git +d ./newbranch/test +d ./newbranch/test/morphs +d ./newbranch/test/morphs/.git f ./master/.morph-system-branch/config -f ./master/test:morphs/hello-stratum.morph -f ./master/test:morphs/hello-system.morph +f ./master/test/morphs/hello-stratum.morph +f ./master/test/morphs/hello-system.morph f ./newbranch/.morph-system-branch/config -f ./newbranch/test:morphs/hello-stratum.morph -f ./newbranch/test:morphs/hello-system.morph +f ./newbranch/test/morphs/hello-stratum.morph +f ./newbranch/test/morphs/hello-system.morph diff --git a/tests.branching/edit-checkouts-existing-chunk.script b/tests.branching/edit-checkouts-existing-chunk.script index 9e66ceb0..c8fb9312 100755 --- a/tests.branching/edit-checkouts-existing-chunk.script +++ b/tests.branching/edit-checkouts-existing-chunk.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -34,4 +34,4 @@ echo "Current branches:" echo echo "Files in hello:" -ls "$DATADIR/workspace/alfred/test:hello" +ls "$DATADIR/workspace/alfred/test/hello" diff --git a/tests.branching/edit-handles-submodules.script b/tests.branching/edit-handles-submodules.script index 72344119..2ab39420 100755 --- a/tests.branching/edit-handles-submodules.script +++ b/tests.branching/edit-handles-submodules.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -28,6 +28,6 @@ cd "$DATADIR/workspace" # Submodules should be set up automatically "$SRCDIR/scripts/test-morph" edit hello-system hello-stratum hello -cd "$DATADIR/workspace/newbranch/test:hello" +cd "$DATADIR/workspace/newbranch/test/hello" [ -e foolib/README ] diff --git a/tests.branching/edit-updates-stratum.script b/tests.branching/edit-updates-stratum.script index bfe16c8b..cf5fc26d 100755 --- a/tests.branching/edit-updates-stratum.script +++ b/tests.branching/edit-updates-stratum.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -29,4 +29,4 @@ cd "$DATADIR/workspace" "$SRCDIR/scripts/test-morph" edit hello-system hello-stratum hello # See what effect the editing had. -"$SRCDIR/scripts/run-git-in" "newbranch/test:morphs" diff +"$SRCDIR/scripts/run-git-in" "newbranch/test/morphs" diff diff --git a/tests.branching/morph-repository-stored-in-cloned-repositories.script b/tests.branching/morph-repository-stored-in-cloned-repositories.script index c2dc8690..342c3d0b 100755 --- a/tests.branching/morph-repository-stored-in-cloned-repositories.script +++ b/tests.branching/morph-repository-stored-in-cloned-repositories.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -29,7 +29,7 @@ cd "$DATADIR/workspace" "$SRCDIR/scripts/test-morph" branch test:morphs newbranch echo "morph.repository in branch root repository:" -cd "$DATADIR/workspace/newbranch/test:morphs" +cd "$DATADIR/workspace/newbranch/test/morphs" git config morph.repository echo @@ -37,7 +37,7 @@ cd "$DATADIR/workspace" "$SRCDIR/scripts/test-morph" checkout test:morphs master echo "morph.repository in branch root repository of a checkout:" -cd "$DATADIR/workspace/master/test:morphs" +cd "$DATADIR/workspace/master/test/morphs" git config morph.repository echo @@ -45,5 +45,5 @@ cd "$DATADIR/workspace/master" "$SRCDIR/scripts/test-morph" edit hello-system hello-stratum hello echo "morph.repository of an edited repository:" -cd "$DATADIR/workspace/master/test:hello" +cd "$DATADIR/workspace/master/test/hello" git config morph.repository diff --git a/tests.branching/petrify-no-double-petrify.script b/tests.branching/petrify-no-double-petrify.script index 9484aa58..3c9185dc 100755 --- a/tests.branching/petrify-no-double-petrify.script +++ b/tests.branching/petrify-no-double-petrify.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -25,7 +25,7 @@ cd "$DATADIR/workspace" "$SRCDIR/scripts/test-morph" init "$SRCDIR/scripts/test-morph" branch test:morphs test/petrify -cd test/petrify/test:morphs +cd test/petrify/test/morphs git push --quiet origin HEAD "$SRCDIR/scripts/test-morph" petrify "$SRCDIR/scripts/test-morph" petrify diff --git a/tests.branching/petrify.script b/tests.branching/petrify.script index fed8e965..5a3cb8c4 100755 --- a/tests.branching/petrify.script +++ b/tests.branching/petrify.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -27,10 +27,10 @@ cd "$DATADIR/workspace" "$SRCDIR/scripts/test-morph" init "$SRCDIR/scripts/test-morph" branch test:morphs test/petrify master -cd test/petrify/test:morphs +cd test/petrify/test/morphs git push --quiet origin HEAD "$SRCDIR/scripts/test-morph" edit hello-system hello-stratum goodbye -(cd ../test:goodbye && git push --quiet origin HEAD) +(cd ../goodbye && git push --quiet origin HEAD) "$SRCDIR/scripts/test-morph" petrify echo "Petrified:" diff --git a/tests.branching/setup b/tests.branching/setup index 06a05e90..22c51c24 100755 --- a/tests.branching/setup +++ b/tests.branching/setup @@ -50,7 +50,7 @@ ln -s "$DATADIR/morphs" "$DATADIR/morphs.git" cat <<EOF > "$DATADIR/morphs/hello-system.morph" name: hello-system kind: system -arch: $(uname -m) +arch: $("$SRCDIR/scripts/test-morph" print-architecture) strata: - morph: hello-stratum EOF diff --git a/tests.branching/status-in-dirty-branch.script b/tests.branching/status-in-dirty-branch.script index cc1dd46e..7fdd8862 100755 --- a/tests.branching/status-in-dirty-branch.script +++ b/tests.branching/status-in-dirty-branch.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -37,7 +37,7 @@ cd "$DATADIR/workspace" cd branch1 "$SRCDIR/scripts/test-morph" edit hello-system stratum2 hello -cd test:stratum2-hello +cd test/stratum2-hello git checkout -q master cd .. diff --git a/tests.branching/tag-creates-commit-and-tag.script b/tests.branching/tag-creates-commit-and-tag.script deleted file mode 100755 index 8fd45d16..00000000 --- a/tests.branching/tag-creates-commit-and-tag.script +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2012-2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Check that 'morph tag example-tag' successfully creates a dangling -## commit and an annotated tag pointing to this commit. - -set -eu - -# Make sure the commits always have the same SHA1s. -. "$SRCDIR/scripts/fix-committer-info" - -# Create a workspace and branch. -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" init -"$SRCDIR/scripts/test-morph" checkout test:morphs master - -# Tag the system branch. -"$SRCDIR/scripts/test-morph" tag example-tag -- -m Message - -# Show the tag itself and its log. This allows to verify a couple of things, -# including that the commit and tag are created, that the commit message is -# set correctly and that all references are petrified. -"$SRCDIR/scripts/test-morph" foreach -- git show example-tag -"$SRCDIR/scripts/test-morph" foreach -- git log --format=fuller example-tag diff --git a/tests.branching/tag-creates-commit-and-tag.stdout b/tests.branching/tag-creates-commit-and-tag.stdout deleted file mode 100644 index 335e6a31..00000000 --- a/tests.branching/tag-creates-commit-and-tag.stdout +++ /dev/null @@ -1,44 +0,0 @@ -test:morphs -tag example-tag -Tagger: developer <developer@example.com> -Date: Tue Jul 31 16:51:54 2012 +0000 - -Message - -commit 74b7fcdd21ac4756e473eb8a577caaabeab208b0 -Author: developer <developer@example.com> -Date: Tue Jul 31 16:51:54 2012 +0000 - - Message - -diff --git a/hello-stratum.morph b/hello-stratum.morph -index e012b5f..50da61b 100644 ---- a/hello-stratum.morph -+++ b/hello-stratum.morph -@@ -3,6 +3,7 @@ kind: stratum - chunks: - - name: hello - repo: test:hello -- ref: master -+ ref: 293fa0b08f0382c63181c36b6efa602876aa8c87 -+ unpetrify-ref: master - build-depends: [] - build-mode: test - -test:morphs -commit 74b7fcdd21ac4756e473eb8a577caaabeab208b0 -Author: developer <developer@example.com> -AuthorDate: Tue Jul 31 16:51:54 2012 +0000 -Commit: developer <developer@example.com> -CommitDate: Tue Jul 31 16:51:54 2012 +0000 - - Message - -commit dc323150575be26e3df0d2dab678560c04281b9f -Author: developer <developer@example.com> -AuthorDate: Tue Jul 31 16:51:54 2012 +0000 -Commit: developer <developer@example.com> -CommitDate: Tue Jul 31 16:51:54 2012 +0000 - - initial - diff --git a/tests.branching/tag-tag-works-as-expected.script b/tests.branching/tag-tag-works-as-expected.script deleted file mode 100755 index 006b98bf..00000000 --- a/tests.branching/tag-tag-works-as-expected.script +++ /dev/null @@ -1,46 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2012-2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## Check that tagging an existing tag commit with 'morph tag' preserves -## the unpetrify-ref and does not "double-petrify" apart from updating -## references to "example-tag" to "tagged-tag". - -set -eu - -# Make sure the commits always have the same SHA1s. -. "$SRCDIR/scripts/fix-committer-info" - -# Create a workspace and branch. -cd "$DATADIR/workspace" -"$SRCDIR/scripts/test-morph" init -"$SRCDIR/scripts/test-morph" checkout test:morphs master - -# Tag the system branch. -"$SRCDIR/scripts/test-morph" tag example-tag -- -m First - -# Check out the tag. -"$SRCDIR/scripts/run-git-in" master/test:morphs checkout -b example-tag \ - 2>/dev/null - -# Tag the tag. -"$SRCDIR/scripts/test-morph" tag tagged-tag -- -m Second - -# List all tags and show the second one. -"$SRCDIR/scripts/test-morph" foreach -- git tag -l -"$SRCDIR/scripts/test-morph" foreach -- git show tagged-tag -"$SRCDIR/scripts/test-morph" foreach -- git log --format=fuller tagged-tag diff --git a/tests.branching/tag-tag-works-as-expected.stdout b/tests.branching/tag-tag-works-as-expected.stdout deleted file mode 100644 index c3d723d7..00000000 --- a/tests.branching/tag-tag-works-as-expected.stdout +++ /dev/null @@ -1,48 +0,0 @@ -test:morphs -example-tag -tagged-tag - -test:morphs -tag tagged-tag -Tagger: developer <developer@example.com> -Date: Tue Jul 31 16:51:54 2012 +0000 - -Second - -commit ed4fa3a98076e92d61983202ed44455b3689bc16 -Author: developer <developer@example.com> -Date: Tue Jul 31 16:51:54 2012 +0000 - - Second - -diff --git a/hello-stratum.morph b/hello-stratum.morph -index e012b5f..50da61b 100644 ---- a/hello-stratum.morph -+++ b/hello-stratum.morph -@@ -3,6 +3,7 @@ kind: stratum - chunks: - - name: hello - repo: test:hello -- ref: master -+ ref: 293fa0b08f0382c63181c36b6efa602876aa8c87 -+ unpetrify-ref: master - build-depends: [] - build-mode: test - -test:morphs -commit ed4fa3a98076e92d61983202ed44455b3689bc16 -Author: developer <developer@example.com> -AuthorDate: Tue Jul 31 16:51:54 2012 +0000 -Commit: developer <developer@example.com> -CommitDate: Tue Jul 31 16:51:54 2012 +0000 - - Second - -commit dc323150575be26e3df0d2dab678560c04281b9f -Author: developer <developer@example.com> -AuthorDate: Tue Jul 31 16:51:54 2012 +0000 -Commit: developer <developer@example.com> -CommitDate: Tue Jul 31 16:51:54 2012 +0000 - - initial - diff --git a/tests.branching/workflow-separate-stratum-repos.script b/tests.branching/workflow-separate-stratum-repos.script index 3faf23f5..f2fd519b 100755 --- a/tests.branching/workflow-separate-stratum-repos.script +++ b/tests.branching/workflow-separate-stratum-repos.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -35,23 +35,23 @@ cd "$DATADIR/workspace" # Edit one chunk cd "me/readme-fixes" "$SRCDIR/scripts/test-morph" edit hello-system stratum2 hello -cd "$DATADIR/workspace/me/readme-fixes/test:stratum2-hello" +cd "$DATADIR/workspace/me/readme-fixes/test/stratum2-hello" echo > README yoyoyo git add README git commit -m "Fix README in hello" --quiet # Edit the other chunk too "$SRCDIR/scripts/test-morph" edit hello-system stratum3 hello -cd "$DATADIR/workspace/me/readme-fixes/test:stratum3-hello" +cd "$DATADIR/workspace/me/readme-fixes/test/stratum3-hello" echo > README yoyoyo git add README git commit -m "Fix README in hello" --quiet # Update the morphology repos -cd ../test:external-strata +cd ../test/external-strata git commit --quiet --all -m "Commit changes for system branch" -cd ../test:morphs +cd ../test/morphs git commit --quiet --all -m "Commit changes for system branch" # Merge our system branch into master @@ -60,13 +60,13 @@ cd master "$SRCDIR/scripts/test-morph" merge me/readme-fixes # Check the changes have appeared -cd test:morphs +cd test/morphs [ $(git rev-parse HEAD) = $(git rev-parse master) ] -cd ../test:stratum2-hello +cd ../test/stratum2-hello [ -e README ] [ $(git rev-parse HEAD) = $(git rev-parse master) ] -cd ../test:stratum3-hello +cd ../test/stratum3-hello [ -e README ] [ $(git rev-parse HEAD) = $(git rev-parse master) ] diff --git a/tests.branching/workflow.script b/tests.branching/workflow.script index 10383132..51a8d106 100755 --- a/tests.branching/workflow.script +++ b/tests.branching/workflow.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012,2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,12 +24,12 @@ cd "$DATADIR/workspace" "$SRCDIR/scripts/test-morph" init "$SRCDIR/scripts/test-morph" branch test:morphs me/readme-fix "$SRCDIR/scripts/test-morph" edit hello-system hello-stratum hello -cd me/readme-fix/test:hello +cd me/readme-fix/test/hello echo > README yoyoyo git add README git commit -m "Fix README, yo!" --quiet -cd ../test:morphs +cd ../morphs git commit --quiet --all -m "Commit changes for system branch" cd "$DATADIR/workspace" diff --git a/tests.build/bootstrap-mode.script b/tests.build/bootstrap-mode.script deleted file mode 100755 index 0ac66220..00000000 --- a/tests.build/bootstrap-mode.script +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2011-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## 'bootstrap' mode is similar to 'test' mode, but they should not be included -## in the final stratum artifact. This feature is used to bootstrap the -## build-essential stratum using the toolchain of the host. - -set -eu - -# Create a fake 'compiler' chunk to go into build-essential stratum -"$SRCDIR/tests.build/setup-build-essential" - -"$SRCDIR/scripts/test-morph" build-morphology \ - test:morphs-repo master hello-system - -cd "$DATADIR/cache/artifacts" -echo "build-essential strata:" -for stratum in $(find . -regex '.*\.stratum\.build-essential-[^.]*$' | sort) -do - echo "$stratum" - sed 's/[a-f0-9]\{64\}/xxxx/g' "$stratum" - echo -done -echo -echo "hello-system:" -system=$(ls *hello-system-rootfs) -tar tf "$system" | LC_ALL=C sort -u | sed '/^\.\/./s:^\./::' \ - | grep -v '^baserock' diff --git a/tests.build/bootstrap-mode.stdout b/tests.build/bootstrap-mode.stdout deleted file mode 100644 index e1747b15..00000000 --- a/tests.build/bootstrap-mode.stdout +++ /dev/null @@ -1,14 +0,0 @@ -build-essential strata: -./2271bef9cb222b1fd49bd2bcf3da435e02b4e0ce9fdddbcdf9358f608f51b547.stratum.build-essential-devel -["xxxx.chunk.cc-devel", "xxxx.chunk.cc-doc"] -./2a908a170acf10b4e9e131a007c8f3f0fa4961a7227c60793b3550c107f4c312.stratum.build-essential-runtime -["xxxx.chunk.cc-bins", "xxxx.chunk.cc-libs", "xxxx.chunk.cc-locale", "xxxx.chunk.cc-misc"] - -hello-system: -./ -etc/ -etc/fstab -etc/os-release -usr/ -usr/bin/ -usr/bin/hello diff --git a/tests.build/build-stratum-with-submodules.stdout b/tests.build/build-stratum-with-submodules.stdout index 6dda5049..d4d03e13 100644 --- a/tests.build/build-stratum-with-submodules.stdout +++ b/tests.build/build-stratum-with-submodules.stdout @@ -1,4 +1,3 @@ ./ etc/ -etc/fstab etc/os-release diff --git a/tests.build/build-system.stdout b/tests.build/build-system.stdout index 2e8270dc..4d0fac2f 100644 --- a/tests.build/build-system.stdout +++ b/tests.build/build-system.stdout @@ -2,5 +2,4 @@ bin/ bin/hello etc/ -etc/fstab etc/os-release diff --git a/tests.build/setup b/tests.build/setup index 8b1dfd0e..559825b1 100755 --- a/tests.build/setup +++ b/tests.build/setup @@ -107,7 +107,7 @@ cat <<EOF > hello-system.morph { "name": "hello-system", "kind": "system", - "arch": "$(uname -m)", + "arch": "$("$SRCDIR/scripts/test-morph" print-architecture)", "strata": [ { "morph": "hello-stratum", diff --git a/tests.deploy/deploy-cluster.script b/tests.deploy/deploy-cluster.script index 0efc8d3c..acc65a54 100755 --- a/tests.deploy/deploy-cluster.script +++ b/tests.deploy/deploy-cluster.script @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -20,7 +20,6 @@ set -eu - . "$SRCDIR/tests.deploy/setup-build" cd "$DATADIR/workspace/branch1" @@ -29,12 +28,15 @@ cd "$DATADIR/workspace/branch1" "$SRCDIR/scripts/test-morph" build linux-system +GIT_DIR=test/morphs/.git git tag -a my-test-tag -m "Example tag" HEAD + "$SRCDIR/scripts/test-morph" --log "$DATADIR/deploy.log" \ deploy test_cluster \ + linux-system-2.EXAMPLE_PASSWORD="secret" \ linux-system-2.HOSTNAME="baserock-rocks-even-more" \ > /dev/null -outputdir=test:morphs +outputdir=test/morphs test -e $outputdir/hello-system.img test -e $outputdir/linux-system-1.tar test -e $outputdir/linux-system-2.tar @@ -44,3 +46,15 @@ hostname2=$(tar -xf $outputdir/linux-system-2.tar ./etc/hostname -O) [ "$hostname1" = baserock-rocks ] [ "$hostname2" = baserock-rocks-even-more ] + +tar -xf $outputdir/linux-system-2.tar ./baserock/deployment.meta +metadata=baserock/deployment.meta + +# Check that 'git describe' of definitions repo was stored correctly +echo -n "definitions-version: " +"$SRCDIR/scripts/yaml-extract" $metadata definitions-version + +echo -n "configuration.HOSTNAME: " +"$SRCDIR/scripts/yaml-extract" $metadata configuration HOSTNAME + +! (grep -q "EXAMPLE_PASSWORD" $metadata) diff --git a/tests.deploy/deploy-cluster.stdout b/tests.deploy/deploy-cluster.stdout new file mode 100644 index 00000000..16b78015 --- /dev/null +++ b/tests.deploy/deploy-cluster.stdout @@ -0,0 +1,2 @@ +definitions-version: {'describe': 'my-test-tag-unreproducible'} +configuration.HOSTNAME: baserock-rocks-even-more diff --git a/tests.deploy/deploy-rawdisk.script b/tests.deploy/deploy-rawdisk.script index 257ef0dd..3489a198 100755 --- a/tests.deploy/deploy-rawdisk.script +++ b/tests.deploy/deploy-rawdisk.script @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -28,6 +28,6 @@ cd "$DATADIR/workspace/branch1" "$SRCDIR/scripts/test-morph" --log "$DATADIR/deploy.log" \ deploy rawdisk_test_cluster > /dev/null -outputdir="test:morphs" +outputdir="test/morphs" test -e $outputdir/disk.img diff --git a/tests.deploy/setup b/tests.deploy/setup index 5fbe0262..ece8819a 100755 --- a/tests.deploy/setup +++ b/tests.deploy/setup @@ -112,7 +112,7 @@ cat <<EOF > hello-system.morph { "name": "hello-system", "kind": "system", - "arch": "$(uname -m)", + "arch": "$("$SRCDIR/scripts/test-morph" print-architecture)", "strata": [ { "morph": "hello-stratum", @@ -148,7 +148,7 @@ cat <<EOF > linux-system.morph { "name": "linux-system", "kind": "system", - "arch": "$(uname -m)", + "arch": "$("$SRCDIR/scripts/test-morph" print-architecture)", "strata": [ { "morph": "hello-stratum", diff --git a/tests.deploy/setup-build b/tests.deploy/setup-build index 6c0a6252..0fc561f9 100644 --- a/tests.deploy/setup-build +++ b/tests.deploy/setup-build @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright (C) 2012, 2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -28,8 +28,8 @@ cd "$DATADIR/workspace" # Fix UUID's in the checked out repos to make build branch names deterministic git config -f "$DATADIR/workspace/branch1/.morph-system-branch/config" \ branch.uuid 123456789 -git config -f "$DATADIR/workspace/branch1/test:morphs/.git/config" \ +git config -f "$DATADIR/workspace/branch1/test/morphs/.git/config" \ morph.uuid 987654321 -git config -f "$DATADIR/workspace/branch1/test:kernel-repo/.git/config" \ +git config -f "$DATADIR/workspace/branch1/test/kernel-repo/.git/config" \ morph.uuid AABBCCDDE diff --git a/tests.merging/setup b/tests.merging/setup index 2fdf9db0..77b8bebc 100755 --- a/tests.merging/setup +++ b/tests.merging/setup @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -51,7 +51,7 @@ cat <<EOF > "$DATADIR/morphs/hello-system.morph" { "name": "hello-system", "kind": "system", - "arch": "$(uname -m)", + "arch": "$("$SRCDIR/scripts/test-morph" print-architecture)", "strata": [ { "morph": "hello-stratum", diff --git a/tests/show-dependencies.setup b/tests/show-dependencies.setup index e1cc71ae..510656e6 100755 --- a/tests/show-dependencies.setup +++ b/tests/show-dependencies.setup @@ -347,7 +347,7 @@ cat <<EOF > xfce-system.morph { "name": "xfce-system", "kind": "system", - "arch": "$(uname -m)", + "arch": "$("$SRCDIR/scripts/test-morph" print-architecture)", "strata": [ { "build-mode": "test", diff --git a/without-test-modules b/without-test-modules index 774e8ac1..9b893300 100644 --- a/without-test-modules +++ b/without-test-modules @@ -29,6 +29,8 @@ morphlib/plugins/trovectl_plugin.py morphlib/plugins/gc_plugin.py morphlib/plugins/branch_and_merge_new_plugin.py morphlib/plugins/print_architecture_plugin.py +morphlib/plugins/add_binary_plugin.py +morphlib/plugins/push_pull_plugin.py morphlib/plugins/distbuild_plugin.py # Not unit tested, since it needs a full system branch morphlib/buildbranch.py diff --git a/yarns/branches-workspaces.yarn b/yarns/branches-workspaces.yarn index c3b73ee3..a9cfb19b 100644 --- a/yarns/branches-workspaces.yarn +++ b/yarns/branches-workspaces.yarn @@ -112,7 +112,7 @@ We leak a little bit of the implementation here, to keep things simple: the (mocked) git server the implementation sets up has the `test:morphs` repository, which is the system branch root repository. - WHEN the user reports the workspace from the directory master/test:morphs + WHEN the user reports the workspace from the directory master/test/morphs THEN the workspace is reported correctly However, running it outside a workspace should fail. @@ -134,7 +134,7 @@ current working directory, it will find it and report it correctly. AND the user reports the system branch from the directory master THEN the system branch is reported as master - WHEN the user reports the system branch from the directory master/test:morphs + WHEN the user reports the system branch from the directory master/test/morphs THEN the system branch is reported as master WHEN the user reports the system branch from the directory . @@ -230,7 +230,7 @@ repositories referenced in the system branch. AND the user edits the chunk test-chunk in the stratum test-stratum in the system test-system in branch foo THEN morph reports changes in foo in test:morphs only - WHEN creating file foo in test:test-chunk in branch foo + WHEN creating file foo in test/test-chunk in branch foo THEN morph reports changes in foo in test:morphs only WHEN adding file foo in test:test-chunk in branch foo to git @@ -254,8 +254,8 @@ branch checkout. WHEN the user creates a system branch called foo AND the user edits the chunk test-chunk in the stratum test-stratum in the system test-system in branch foo AND running shell command in each repo in foo - THEN morph ran command in test:morphs in foo - AND morph ran command in test:test-chunk in foo + THEN morph ran command in test/morphs in foo + AND morph ran command in test/test-chunk in foo Explicit petrification ---------------------- diff --git a/yarns/deployment.yarn b/yarns/deployment.yarn index 855ecc52..fc21b826 100644 --- a/yarns/deployment.yarn +++ b/yarns/deployment.yarn @@ -9,11 +9,96 @@ Morph Deployment Tests THEN morph failed AND the deploy error message includes the string "morph deploy is only supported for cluster morphologies" - SCENARIO deploying a cluster morphology + SCENARIO deploying a cluster morphology as a tarfile GIVEN a workspace AND a git server WHEN the user checks out the system branch called master - GIVEN a cluster called test-cluster for deploying only the test-system system as type tar in system branch master + GIVEN a cluster called test-cluster in system branch master + AND a system in cluster test-cluster in branch master called test-system + AND system test-system in cluster test-cluster in branch master builds test-system + AND system test-system in cluster test-cluster in branch master has deployment type: tar + AND system test-system in cluster test-cluster in branch master has deployment location: test.tar WHEN the user builds the system test-system in branch master - AND the user attempts to deploy the cluster test-cluster in branch master with options system.location=test.tar + AND the user attempts to deploy the cluster test-cluster in branch master THEN morph succeeded + +Some deployment types support upgrades, but some do not and Morph needs to make +this clear. + + SCENARIO attempting to upgrade a tarfile deployment + GIVEN a workspace + AND a git server + WHEN the user checks out the system branch called master + GIVEN a cluster called test-cluster in system branch master + AND a system in cluster test-cluster in branch master called test-system + AND system test-system in cluster test-cluster in branch master builds test-system + AND system test-system in cluster test-cluster in branch master has deployment type: tar + AND system test-system in cluster test-cluster in branch master has deployment location: test.tar + WHEN the user builds the system test-system in branch master + AND the user attempts to upgrade the cluster test-cluster in branch master + THEN morph failed + +The rawdisk write extension supports both initial deployment and subsequent +upgrades. Note that the rawdisk upgrade code needs bringing up to date to use +the new Baserock OS version manager tool. Also, the test deploys an identical +base OS as an upgrade. While pointless, this is permitted and does exercise +the same code paths as a real upgrade. + + SCENARIO deploying a cluster morphology as rawdisk and then upgrading it + GIVEN a workspace + AND a git server + WHEN the user checks out the system branch called master + GIVEN a cluster called test-cluster in system branch master + AND a system in cluster test-cluster in branch master called test-system + AND system test-system in cluster test-cluster in branch master builds test-system + AND system test-system in cluster test-cluster in branch master has deployment type: rawdisk + AND system test-system in cluster test-cluster in branch master has deployment location: test.tar + WHEN the user builds the system test-system in branch master + AND the user attempts to deploy the cluster test-cluster in branch master with options test-system.DISK_SIZE=20M test-system.VERSION_LABEL=test1 + THEN morph succeeded + WHEN the user attempts to upgrade the cluster test-cluster in branch master with options test-system.VERSION_LABEL=test2 + THEN morph succeeded + +Nested deployments +================== + +For the use-cases of: + +1. Installer CD/USB +2. NFS/VM host +3. System with multiple containerised applications +4. System with a toolchain targetting the sysroot of another target +5. Any nested combination of the above + +It is convenient to be able to deploy one system inside another. + + SCENARIO deploying a cluster morphology with nested systems + GIVEN a workspace + AND a git server + WHEN the user checks out the system branch called master + GIVEN a cluster called test-cluster in system branch master + AND a system in cluster test-cluster in branch master called test-system + AND system test-system in cluster test-cluster in branch master builds test-system + AND system test-system in cluster test-cluster in branch master has deployment type: tar + +After the usual setup, we also add a subsystem to the cluster. + + GIVEN a subsystem in cluster test-cluster in branch master called test-system.sysroot + AND subsystem test-system.sysroot in cluster test-cluster in branch master builds test-system + AND subsystem test-system.sysroot in cluster test-cluster in branch master has deployment type: sysroot + +We specify the location as a file path, this is relative to the parent +system's extracted rootfs, before it is configured. + + AND subsystem test-system.sysroot in cluster test-cluster in branch master has deployment location: var/lib/sysroots/test-system + WHEN the user builds the system test-system in branch master + AND the user attempts to deploy the cluster test-cluster in branch master with options test-system.location="$DATADIR/test.tar" + THEN morph succeeded + +Morph succeeding alone is not sufficient to check whether it actually +worked, since if it ignored the subsystems field, or got the location +wrong for the subsystem. To actually test it, we have to check that our +deployed system contains the other. Since the baserock directory is in +every system, we can check for that. + + AND tarball test.tar contains var/lib/sysroots/test-system/baserock diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index ccebabca..e4f36399 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -340,9 +340,9 @@ Attempt to check out a system branch from a root that has no systems. We also need to verify that a system branch has been checked out. IMPLEMENTS THEN the system branch (\S+) is checked out - is_dir "$DATADIR/workspace/$MATCH_1/test:morphs" - is_file "$DATADIR/workspace/$MATCH_1/test:morphs/test-system.morph" - is_file "$DATADIR/workspace/$MATCH_1/test:morphs/test-stratum.morph" + is_dir "$DATADIR/workspace/$MATCH_1/test/morphs" + is_file "$DATADIR/workspace/$MATCH_1/test/morphs/test-system.morph" + is_file "$DATADIR/workspace/$MATCH_1/test/morphs/test-stratum.morph" We can create a new branch, off master. @@ -368,7 +368,7 @@ Pushing all changes in a system branch checkout to the git server. IMPLEMENTS WHEN the user pushes the system branch called (\S+) to the git server # FIXME: For now, this is just the morphs checkout. - run_in "$DATADIR/workspace/$MATCH_1/test:morphs" git push origin HEAD + run_in "$DATADIR/workspace/$MATCH_1/test/morphs" git push origin HEAD Report workspace path. @@ -411,7 +411,7 @@ Editing morphologies with `morph edit`. IMPLEMENTS THEN in branch (\S+), stratum (\S+) refs (\S+) in (\S+) "$SRCDIR/scripts/yaml-extract" \ - "$DATADIR/workspace/$MATCH_1/test:morphs/$MATCH_2.morph" \ + "$DATADIR/workspace/$MATCH_1/test/morphs/$MATCH_2.morph" \ chunks name="$MATCH_3" ref > "$DATADIR/ref.actual" echo "$MATCH_4" > "$DATADIR/ref.wanted" diff -u "$DATADIR/ref.wanted" "$DATADIR/ref.actual" @@ -419,21 +419,22 @@ Editing morphologies with `morph edit`. IMPLEMENTS THEN in branch (\S+), (system|stratum) (\S+) refers to (\S+) without (\S+) if [ $MATCH_2 == system ]; then field=strata; else field=build-depends; fi { ! "$SRCDIR/scripts/yaml-extract" \ - "$DATADIR/workspace/$MATCH_1/test:morphs/$MATCH_3.morph" \ + "$DATADIR/workspace/$MATCH_1/test/morphs/$MATCH_3.morph" \ "$field" name="$MATCH_4" "$MATCH_5"; } 2>&1 | grep -qFe "Object does not contain $MATCH_5" IMPLEMENTS WHEN the user edits the stratum (\S+) in the system (\S+) in branch (\S+) - cd "$DATADIR/workspace/$MATCH_3/test:morphs" + cd "$DATADIR/workspace/$MATCH_3/test/morphs" run_morph edit "$MATCH_2" "$MATCH_1" IMPLEMENTS WHEN the user edits the chunk (\S+) in the stratum (\S+) in the system (\S+) in branch (\S+) - cd "$DATADIR/workspace/$MATCH_4/test:morphs" + cd "$DATADIR/workspace/$MATCH_4/test/morphs" run_morph edit "$MATCH_3" "$MATCH_2" "$MATCH_1" IMPLEMENTS THEN the edited chunk (\S+) has git branch (\S+) ls -l "$DATADIR/workspace/$MATCH_2" - cd "$DATADIR/workspace/$MATCH_2/$MATCH_1" + chunkdir="$(slashify_colons "$MATCH_1")" + cd "$DATADIR/workspace/$MATCH_2/$chunkdir" git branch | awk '$1 == "*" { print $2 }' > "$DATADIR/git-branch.actual" echo "$MATCH_2" > "$DATADIR/git-branch.wanted" diff -u "$DATADIR/git-branch.wanted" "$DATADIR/git-branch.actual" @@ -448,7 +449,7 @@ print-architecture` to get a value appropriate for morph. IMPLEMENTS WHEN the user creates an uncommitted system morphology called (\S+) for our architecture in system branch (\S+) arch=$(morph print-architecture) - cat << EOF > "$DATADIR/workspace/$MATCH_2/test:morphs/$MATCH_1.morph" + cat << EOF > "$DATADIR/workspace/$MATCH_2/test/morphs/$MATCH_1.morph" arch: $arch configuration-extensions: [] description: A system called $MATCH_1 for architectures $arch @@ -494,11 +495,12 @@ Reporting status of checked out repositories: touch "$DATADIR/workspace/$MATCH_3/$MATCH_2/$MATCH_1" IMPLEMENTS WHEN adding file (\S+) in (\S+) in branch (\S+) to git - cd "$DATADIR/workspace/$MATCH_3/$MATCH_2" + chunkdir="$(slashify_colons "$MATCH_2")" + cd "$DATADIR/workspace/$MATCH_3/$chunkdir" git add "$MATCH_1" IMPLEMENTS WHEN committing changes in (\S+) in branch (\S+) - cd "$DATADIR/workspace/$MATCH_2/$MATCH_1" + cd "$DATADIR/workspace/$MATCH_2/$(slashify_colons "$MATCH_1")" git commit -a -m test-commit Running shell command in each checked out repository: @@ -514,11 +516,11 @@ Running shell command in each checked out repository: Petrification and unpetrification: IMPLEMENTS WHEN remembering all refs in (\S+) - cd "$DATADIR/workspace/$MATCH_1/test:morphs" + cd "$DATADIR/workspace/$MATCH_1/test/morphs" list_refs *.morph > "$DATADIR/refs.remembered" IMPLEMENTS THEN (\S+) refs are as remembered - cd "$DATADIR/workspace/$MATCH_1/test:morphs" + cd "$DATADIR/workspace/$MATCH_1/test/morphs" # FIXME: petrify/unpetrify doesn't work quite right at this time: # petrify can change a ref to a stratum to point at the system @@ -548,41 +550,41 @@ Petrification and unpetrification: done IMPLEMENTS WHEN petrifying (\S+) - cd "$DATADIR/workspace/$MATCH_1/test:morphs" + cd "$DATADIR/workspace/$MATCH_1/test/morphs" run_morph petrify IMPLEMENTS WHEN unpetrifying (\S+) - cd "$DATADIR/workspace/$MATCH_1/test:morphs" + cd "$DATADIR/workspace/$MATCH_1/test/morphs" run_morph unpetrify IMPLEMENTS THEN (\S+) is petrified - cd "$DATADIR/workspace/$MATCH_1/test:morphs" + cd "$DATADIR/workspace/$MATCH_1/test/morphs" assert_morphologies_are_petrified "$MATCH_1" *.morph IMPLEMENTS THEN (\S+) is not petrified - cd "$DATADIR/workspace/$MATCH_1/test:morphs" + cd "$DATADIR/workspace/$MATCH_1/test/morphs" ! assert_morphologies_are_petrified "$MATCH_1" *.morph Tagging. IMPLEMENTS WHEN the user tags the system branch called (\S+) as (\S+) - cd "$DATADIR/workspace/$MATCH_1/test:morphs" + cd "$DATADIR/workspace/$MATCH_1/test/morphs" set tag "$MATCH_2" -- -m "testing morph tagging" run_morph tag "$MATCH_2" -- -m "testing morph tagging" IMPLEMENTS WHEN the user attempts to tag the system branch called (\S+) as (\S+) - cd "$DATADIR/workspace/$MATCH_1/test:morphs" + cd "$DATADIR/workspace/$MATCH_1/test/morphs" attempt_morph tag "$MATCH_2" -- -m "testing morph tagging" IMPLEMENTS THEN morph tag (\S+) in (\S+) is an annotated git tag - cd "$DATADIR/workspace/$MATCH_2/test:morphs" + cd "$DATADIR/workspace/$MATCH_2/test/morphs" if git show "$MATCH_1" | head -n1 | grep -v '^tag ' then die "git tag $MATCH_1 is not an annotated tag" fi IMPLEMENTS THEN morph tag (\S+) in (\S+) refers to a petrified commit - cd "$DATADIR/workspace/$MATCH_2/test:morphs" + cd "$DATADIR/workspace/$MATCH_2/test/morphs" git ls-tree "$MATCH_1" | awk '$NF ~ /\.morph$/ { print $NF }' | while read x @@ -663,26 +665,15 @@ them, so they can be added to the end of the implements section. if [ $MATCH_1 == "deploys" ]; then run_morph "$@" else attempt_morph "$@"; fi -To successfully deploy systems, we need a cluster morphology. Since the -common case is to just have one system, we generate a stub morphology -with only the minimal information. - - IMPLEMENTS GIVEN a cluster called (\S+) for deploying only the (\S+) system as type (\S+) in system branch (\S+) - name="$MATCH_1" - system="$MATCH_2" - type="$MATCH_3" - branch="$MATCH_4" - cat << EOF > "$DATADIR/workspace/$branch/test:morphs/$name.morph" - name: $name - kind: cluster - systems: - - morph: $system - repo: test:morphs - ref: $branch - deploy: - system: - type: $type - EOF + IMPLEMENTS WHEN the user (attempts to upgrade|upgrades) the (system|cluster) (\S+) in branch (\S+)( with options (.*))? + cd "$DATADIR/workspace/$MATCH_4" + set -- deploy --upgrade "$MATCH_3" + if [ "$MATCH_5" != '' ]; then + # eval used so word splitting in the text is preserved + eval set -- '"$@"' $MATCH_6 + fi + if [ $MATCH_1 == "upgrades" ]; then run_morph "$@" + else attempt_morph "$@"; fi Implementations sections for reading error messages =================================================== @@ -805,7 +796,61 @@ Altering morphologies in their source repositories Altering morphologies in the workspace -------------------------------------- +### Altering strata ### + IMPLEMENTS GIVEN stratum (\S+) in system branch (\S+) has match rules: (.*) - cd "$DATADIR/workspace/$MATCH_2/test:morphs" + cd "$DATADIR/workspace/$MATCH_2/test/morphs" "$SRCDIR/scripts/edit-morph" set-stratum-match-rules \ "$MATCH_1.morph" "$MATCH_3" + +### Altering clusters ### + + IMPLEMENTS GIVEN a cluster called (\S+) in system branch (\S+) + name="$MATCH_1" + branch="$MATCH_2" + "$SRCDIR/scripts/edit-morph" cluster-init \ + "$DATADIR/workspace/$branch/test/morphs/$name.morph" + + IMPLEMENTS GIVEN a (sub)?system in cluster (\S+) in branch (\S+) called (\S+) + cluster="$MATCH_2" + branch="$MATCH_3" + name="$MATCH_4" + "$SRCDIR/scripts/edit-morph" cluster-system-init \ + "$DATADIR/workspace/$branch/test/morphs/$cluster.morph" "$name" + + IMPLEMENTS GIVEN (sub)?system (\S+) in cluster (\S+) in branch (\S+) builds (\S+) + name="$MATCH_2" + cluster="$MATCH_3" + branch="$MATCH_4" + morphology="$MATCH_5" + "$SRCDIR/scripts/edit-morph" cluster-system-set-morphology \ + "$DATADIR/workspace/$branch/test/morphs/$cluster.morph" "$name" \ + "$morphology" + + IMPLEMENTS GIVEN (sub)?system (\S+) in cluster (\S+) in branch (\S+) has deployment type: (\S+) + name="$MATCH_2" + cluster="$MATCH_3" + branch="$MATCH_4" + type="$MATCH_5" + "$SRCDIR/scripts/edit-morph" cluster-system-set-deploy-type \ + "$DATADIR/workspace/$branch/test/morphs/$cluster.morph" "$name" \ + "$type" + + IMPLEMENTS GIVEN (sub)?system (\S+) in cluster (\S+) in branch (\S+) has deployment location: (\S+) + name="$MATCH_2" + cluster="$MATCH_3" + branch="$MATCH_4" + location="$MATCH_5" + "$SRCDIR/scripts/edit-morph" cluster-system-set-deploy-location \ + "$DATADIR/workspace/$branch/test/morphs/$cluster.morph" "$name" \ + "$location" + + IMPLEMENTS GIVEN (sub)?system (\S+) in cluster (\S+) in branch (\S+) has deployment variable: ([^=]+)=(.*) + name="$MATCH_2" + cluster="$MATCH_3" + branch="$MATCH_4" + key="$MATCH_5" + val="$MATCH_6" + "$SRCDIR/scripts/edit-morph" cluster-system-set-deploy-variable \ + "$DATADIR/workspace/$branch/test/morphs/$cluster.morph" "$name" \ + "$key" "$val" diff --git a/yarns/morph.shell-lib b/yarns/morph.shell-lib index 31dcc7af..05c11bcc 100644 --- a/yarns/morph.shell-lib +++ b/yarns/morph.shell-lib @@ -19,20 +19,25 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# Add $SRCDIR to PYTHONPATH. + +case "$PYTHONPATH" in + '') PYTHONPATH="$SRCDIR)" ;; + *) PYTHONPATH="$SRCDIR:$PYTHONPATH" ;; +esac +export PYTHONPATH + + # Run Morph from the source tree, ignoring any configuration files. # This way the test suite is not affected by any configuration the user # or system may have. Instead, we'll use the `$DATADIR/morph.conf` file, -# which tests can create, if they want to. Unfortunately, currently yarn -# does not set a $SRCDIR that points at the source tree, so if the test -# needs to cd away from there, things can break. We work around this -# by allowing the caller to set $SRCDIR if they want to, and if it isn't -# set, we default to . (current working directory). +# which tests can create, if they want to. run_morph() { { set +e - "${SRCDIR:-.}"/morph \ + "$SRCDIR"/morph \ --cachedir-min-space=0 --tempdir-min-space=0 \ --no-default-config --config "$DATADIR/morph.conf" "$@" \ 2> "$DATADIR/result-$1" @@ -152,15 +157,6 @@ assert_morphologies_are_petrified() } -# Currently, yarn isn't setting $SRCDIR to point at the project source -# directory. We simulate this here. - -if ! env | grep '^SRCDIR=' > /dev/null -then - export SRCDIR="$(pwd)" -fi - - # Added until it's fixed in upstream. # It's a solution to create an empty home directory each execution export HOME="$DATADIR/home" @@ -178,3 +174,12 @@ then email = tomjon@codethink.co.uk EOF fi + + +# Change colons to slashes. This is used when converting an aliases +# repository URL (e.g., test:morphs) into a directory path. + +slashify_colons() +{ + echo "$1" | sed s,:,/,g +} diff --git a/yarns/regression.yarn b/yarns/regression.yarn index d8eedea2..b0f4d112 100644 --- a/yarns/regression.yarn +++ b/yarns/regression.yarn @@ -58,11 +58,14 @@ source it depended on. AND system test-system uses test-stratum-runtime from test-stratum AND stratum test-stratum has match rules: [{artifact: test-stratum-runtime, include: [.*-(bins|libs|locale)]}, {artifact: test-stratum-devel, include: [.*-(devel|doc|misc)]}] WHEN the user checks out the system branch called master - GIVEN a cluster called test-cluster for deploying only the test-system system as type tar in system branch master + GIVEN a cluster called test-cluster in system branch master + AND a system in cluster test-cluster in branch master called test-system + AND system test-system in cluster test-cluster in branch master builds test-system + AND system test-system in cluster test-cluster in branch master has deployment type: tar WHEN the user builds the system test-system in branch master GIVEN stratum test-stratum in system branch master has match rules: [{artifact: test-stratum-runtime, include: [.*-(bins|libs|misc)]}, {artifact: test-stratum-devel, include: [.*-(devel|doc|locale)]}] WHEN the user builds the system test-system in branch master - AND the user deploys the cluster test-cluster in branch master with options system.location="$DATADIR/test.tar" + AND the user deploys the cluster test-cluster in branch master with options test-system.location="$DATADIR/test.tar" THEN tarball test.tar contains baserock/test-chunk-misc.meta diff --git a/yarns/splitting.yarn b/yarns/splitting.yarn index 3da397f5..ee587e11 100644 --- a/yarns/splitting.yarn +++ b/yarns/splitting.yarn @@ -47,9 +47,12 @@ The best way to test that only using some stratum artifacts works is to check which files the output has, so we deploy a tarball and inspect its contents. - GIVEN a cluster called test-cluster for deploying only the test-system system as type tar in system branch master + GIVEN a cluster called test-cluster in system branch master + AND a system in cluster test-cluster in branch master called test-system + AND system test-system in cluster test-cluster in branch master builds test-system + AND system test-system in cluster test-cluster in branch master has deployment type: tar WHEN the user builds the system test-system in branch master - AND the user attempts to deploy the cluster test-cluster in branch master with options system.location="$DATADIR/test.tar" + AND the user attempts to deploy the cluster test-cluster in branch master with options test-system.location="$DATADIR/test.tar" The -runtime artifacts include executables and shared libraries. |