diff options
40 files changed, 530 insertions, 368 deletions
@@ -4,6 +4,16 @@ NEWS for Morph This file contains high-level summaries of user-visible changes in each Morph release. +Version 14.23, released 2014-06-06 +---------------------------------- + +New feature: + +* Initramfs support + +There have also been a number of fixes to distbuild, and the +`morph copy-artifacts` command has been replaced by `morph list-artifacts`. + Version 14.22, released 2014-05-29 ---------------------------------- diff --git a/distbuild/build_controller.py b/distbuild/build_controller.py index c399939e..987f01f4 100644 --- a/distbuild/build_controller.py +++ b/distbuild/build_controller.py @@ -39,8 +39,9 @@ class _Built(object): pass class _AnnotationFailed(object): - def __init__(self, http_status_code): + def __init__(self, http_status_code, error_msg): self.http_status_code = http_status_code + self.error_msg = error_msg class _GotGraph(object): @@ -359,12 +360,13 @@ class BuildController(distbuild.StateMachine): logging.debug('Got cache response: %s' % repr(event.msg)) http_status_code = event.msg['status'] + error_msg = event.msg['body'] if http_status_code != httplib.OK: logging.debug('Cache request failed with status: %s' % event.msg['status']) self.mainloop.queue_event(self, - _AnnotationFailed(http_status_code)) + _AnnotationFailed(http_status_code, error_msg)) return cache_state = json.loads(event.msg['body']) @@ -577,8 +579,8 @@ class BuildController(distbuild.StateMachine): self._queue_worker_builds(None, event) def _notify_annotation_failed(self, event_source, event): - errmsg = ('Failed to annotate build graph: http request got %d' - % event.http_status_code) + errmsg = ('Failed to annotate build graph: http request got %d: %s' + % (event.http_status_code, event.error_msg)) logging.error(errmsg) failed = BuildFailed(self._request['id'], errmsg) diff --git a/distbuild/worker_build_scheduler.py b/distbuild/worker_build_scheduler.py index a7f7eb80..57cc0224 100644 --- a/distbuild/worker_build_scheduler.py +++ b/distbuild/worker_build_scheduler.py @@ -100,7 +100,7 @@ class Job(object): self.artifact = artifact self.initiators = [initiator_id] self.who = None # we don't know who's going to do this yet - self.is_building = False + self.running = False self.failed = False @@ -120,18 +120,18 @@ class Jobs(object): return job def remove(self, job): - del self._jobs[job.artifact.basename()] + if job.artifact.basename() in self._jobs: + del self._jobs[job.artifact.basename()] + else: + logging.warning("Tried to remove a job that doesn't exist " + "(%s)", job.artifact.basename()) def get_jobs(self): return self._jobs def remove_jobs(self, jobs): for job in jobs: - if job.artifact.basename() in self._jobs: - self.remove(job) - else: - logging.warning("Tried to remove a job that doesn't exist " - "(%s)", job.artifact.basename()) + self.remove(job) def exists(self, artifact_basename): return artifact_basename in self._jobs @@ -168,19 +168,19 @@ class _Cached(object): pass -class _ExecStarted(object): +class _JobStarted(object): def __init__(self, job): self.job = job -class _ExecEnded(object): +class _JobFinished(object): def __init__(self, job): self.job = job -class _ExecFailed(object): +class _JobFailed(object): def __init__(self, job): self.job = job @@ -215,30 +215,30 @@ class WorkerBuildQueuer(distbuild.StateMachine): self._handle_cancel), ('idle', WorkerConnection, _NeedJob, 'idle', self._handle_worker), - ('idle', WorkerConnection, _ExecStarted, 'idle', - self._set_exec_started), - ('idle', WorkerConnection, _ExecEnded, 'idle', - self._set_exec_ended), - ('idle', WorkerConnection, _ExecFailed, 'idle', - self._set_exec_failed) + ('idle', WorkerConnection, _JobStarted, 'idle', + self._set_job_started), + ('idle', WorkerConnection, _JobFinished, 'idle', + self._set_job_finished), + ('idle', WorkerConnection, _JobFailed, 'idle', + self._set_job_failed) ] self.add_transitions(spec) - def _set_exec_started(self, event_source, event): + def _set_job_started(self, event_source, event): logging.debug('Setting job state for job %s with id %s: ' - 'Job is building', + 'Job is running', event.job.artifact.basename(), event.job.id) - event.job.is_building = True + event.job.running = True - def _set_exec_ended(self, event_source, event): + def _set_job_finished(self, event_source, event): logging.debug('Setting job state for job %s with id %s: ' - 'Job is NOT building', + 'Job is NOT running', event.job.artifact.basename(), event.job.id) - event.job.is_building = False + event.job.running = False - def _set_exec_failed(self, event_source, event): + def _set_job_failed(self, event_source, event): logging.debug('Job %s with id %s failed', event.job.artifact.basename(), event.job.id) event.job.failed = True @@ -258,7 +258,7 @@ class WorkerBuildQueuer(distbuild.StateMachine): job = self._jobs.get(event.artifact.basename()) job.initiators.append(event.initiator_id) - if job.is_building: + if job.running: logging.debug('Worker build step already started: %s' % event.artifact.basename()) progress = WorkerBuildStepAlreadyStarted(event.initiator_id, @@ -295,7 +295,7 @@ class WorkerBuildQueuer(distbuild.StateMachine): name, job_id) if len(job.initiators) == 1: - if job.is_building or job.failed: + if job.running or job.failed: logging.debug('NOT removing running job %s with job id %s ' '(WorkerConnection will cancel job)', name, job_id) @@ -485,7 +485,7 @@ class WorkerConnection(distbuild.StateMachine): started = WorkerBuildStepStarted(self._job.initiators, self._job.artifact.cache_key, self.name()) - self.mainloop.queue_event(WorkerConnection, _ExecStarted(self._job)) + self.mainloop.queue_event(WorkerConnection, _JobStarted(self._job)) self.mainloop.queue_event(WorkerConnection, started) def _handle_json_message(self, event_source, event): @@ -524,15 +524,13 @@ class WorkerConnection(distbuild.StateMachine): # Build failed. new_event = WorkerBuildFailed(new, self._job.artifact.cache_key) self.mainloop.queue_event(WorkerConnection, new_event) - self.mainloop.queue_event(WorkerConnection, _ExecFailed(self._job)) + self.mainloop.queue_event(WorkerConnection, _JobFailed(self._job)) self.mainloop.queue_event(self, _BuildFailed()) else: # Build succeeded. We have more work to do: caching the result. self.mainloop.queue_event(self, _BuildFinished()) self._exec_response_msg = new - self.mainloop.queue_event(WorkerConnection, _ExecEnded(self._job)) - def _request_job(self, event_source, event): distbuild.crash_point() self.mainloop.queue_event(WorkerConnection, _NeedJob(self)) @@ -600,7 +598,20 @@ class WorkerConnection(distbuild.StateMachine): logging.error( 'Failed to populate artifact cache: %s %s' % (event.msg['status'], event.msg['body'])) + + # We will attempt to remove this job twice + # unless we mark it as failed before the BuildController + # processes the WorkerBuildFailed event. + # + # The BuildController will not try to cancel jobs that have + # been marked as failed. + self.mainloop.queue_event(WorkerConnection, + _JobFailed(self._job)) + new_event = WorkerBuildFailed( self._exec_response_msg, self._job.artifact.cache_key) self.mainloop.queue_event(WorkerConnection, new_event) + self.mainloop.queue_event(self, _BuildFailed()) + + self.mainloop.queue_event(WorkerConnection, _JobFinished(self._job)) diff --git a/morphlib/exts/initramfs.write b/morphlib/exts/initramfs.write new file mode 100755 index 00000000..f8af6d84 --- /dev/null +++ b/morphlib/exts/initramfs.write @@ -0,0 +1,27 @@ +#!/bin/sh +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + +set -e + +ROOTDIR="$1" +INITRAMFS_PATH="$2" + +(cd "$ROOTDIR" && + find . -print0 | + cpio -0 -H newc -o) | + gzip -c | install -D -m644 /dev/stdin "$INITRAMFS_PATH" diff --git a/morphlib/exts/initramfs.write.help b/morphlib/exts/initramfs.write.help new file mode 100644 index 00000000..29a9d266 --- /dev/null +++ b/morphlib/exts/initramfs.write.help @@ -0,0 +1,35 @@ +help: | + Create an initramfs for a system by taking an existing system and + converting it to the appropriate format. + + The system must have a `/init` executable as the userland entry-point. + This can have a different path, if `rdinit=$path` is added to + the kernel command line. This can be added to the `rawdisk`, + `virtualbox-ssh` and `kvm` write extensions with the `KERNEL_CMDLINE` + option. + + It is possible to use a ramfs as the final rootfs without a `/init` + executable, by setting `root=/dev/mem`, or `rdinit=/sbin/init`, + but this is beyond the scope for the `initramfs.write` extension. + + The intended use of initramfs.write is to be part of a nested + deployment, so the parent system has an initramfs stored as + `/boot/initramfs.gz`. See the following example: + + name: initramfs-test + kind: cluster + systems: + - morph: minimal-system-x86_64-generic + deploy: + system: + type: rawdisk + location: initramfs-system-x86_64.img + DISK_SIZE: 1G + HOSTNAME: initramfs-system + INITRAMFS_PATH: boot/initramfs.gz + subsystems: + - morph: initramfs-x86_64 + deploy: + initramfs: + type: initramfs + location: boot/initramfs.gz diff --git a/morphlib/exts/kvm.write.help b/morphlib/exts/kvm.write.help new file mode 100644 index 00000000..8b5053a5 --- /dev/null +++ b/morphlib/exts/kvm.write.help @@ -0,0 +1,4 @@ +help: | + The INITRAMFS_PATH option can be used to specify the location of an + initramfs for syslinux to tell Linux to use, rather than booting + the rootfs directly. diff --git a/morphlib/exts/rawdisk.write.help b/morphlib/exts/rawdisk.write.help index a514a4e8..298d441c 100644 --- a/morphlib/exts/rawdisk.write.help +++ b/morphlib/exts/rawdisk.write.help @@ -5,3 +5,7 @@ help: | The `location` argument is a pathname to the image to be created or upgraded. + + The INITRAMFS_PATH option can be used to specify the location of an + initramfs for syslinux to tell Linux to use, rather than booting + the rootfs directly. diff --git a/morphlib/exts/virtualbox-ssh.write.help b/morphlib/exts/virtualbox-ssh.write.help new file mode 100644 index 00000000..8b5053a5 --- /dev/null +++ b/morphlib/exts/virtualbox-ssh.write.help @@ -0,0 +1,4 @@ +help: | + The INITRAMFS_PATH option can be used to specify the location of an + initramfs for syslinux to tell Linux to use, rather than booting + the rootfs directly. diff --git a/morphlib/plugins/branch_and_merge_new_plugin.py b/morphlib/plugins/branch_and_merge_new_plugin.py index 51cba401..5ac8353a 100644 --- a/morphlib/plugins/branch_and_merge_new_plugin.py +++ b/morphlib/plugins/branch_and_merge_new_plugin.py @@ -363,124 +363,32 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): Command line arguments: - * `SYSTEM` is the name of a system morphology in the root repository - of the current system branch. - * `STRATUM` is the name of a stratum inside the system. - * `CHUNK` is the name of a chunk inside the stratum. + * `CHUNK` is the name of a chunk - This marks the specified stratum or chunk (if given) as being - changed within the system branch, by creating the git branches in - the affected repositories, and changing the relevant morphologies - to point at those branches. It also creates a local clone of - the git repository of the stratum or chunk. - - For example: - - morph edit devel-system-x86-64-generic devel - - The above command will mark the `devel` stratum as being - modified in the current system branch. In this case, the stratum's - morphology is in the same git repository as the system morphology, - so there is no need to create a new git branch. However, the - system morphology is modified to point at the stratum morphology - in the same git branch, rather than the original branch. - - In other words, where the system morphology used to say this: - - morph: devel - repo: baserock:baserock/morphs - ref: master - - The updated system morphology will now say this instead: - - morph: devel - repo: baserock:baserock/morphs - ref: jrandom/new-feature - - (Assuming the system branch is called `jrandom/new-feature`.) - - Another example: - - morph edit devel-system-x86_64-generic devel gcc - - The above command will mark the `gcc` chunk as being edited in - the current system branch. Morph will clone the `gcc` repository - locally, into the current workspace, and create a new (local) - branch named after the system branch. It will also change the - stratum morphology to refer to the new git branch, instead of - whatever branch it was referring to originally. - - If the `gcc` repository already had a git branch named after - the system branch, that is reused. Similarly, if the stratum - morphology was already pointing that that branch, it doesn't - need to be changed again. In that case, the only action Morph - does is to clone the chunk repository locally, and if that was - also done already, Morph does nothing. + This makes a local checkout of CHUNK in the current system branch + and edits any stratum morphology file(s) containing the chunk ''' - if len(args) not in (2, 3): - raise cliapp.AppException('morph edit needs the names of a system,' - ' a stratum and optionally a chunk' - ' as parameters') - - system_name = morphlib.util.strip_morph_extension(args[0]) - stratum_name = morphlib.util.strip_morph_extension(args[1]) - chunk_name = None - if len(args) == 3: - chunk_name = morphlib.util.strip_morph_extension(args[2]) + if len(args) != 1: + raise cliapp.AppException('morph edit needs a chunk ' + 'as parameter') ws = morphlib.workspace.open('.') sb = morphlib.sysbranchdir.open_from_within('.') loader = morphlib.morphloader.MorphologyLoader() + morphs = self._load_all_sysbranch_morphologies(sb, loader) - # Load the system morphology, and all stratum morphologies, including - # all the strata that are being build-depended on. - - logging.debug('Loading system morphology') - system_morph = loader.load_from_file( - sb.get_filename(sb.root_repository_url, system_name + '.morph')) - if system_morph['kind'] != 'system': - raise cliapp.AppException("%s is not a system" % system_name) - system_morph.repo_url = sb.root_repository_url - system_morph.ref = sb.system_branch_name - system_morph.filename = system_name + '.morph' - - logging.debug('Loading stratum morphologies') - morphs = self._load_stratum_morphologies(loader, sb, system_morph) - morphs.add_morphology(system_morph) - logging.debug('morphs: %s' % repr(morphs.morphologies)) - - # Change refs to the stratum to be to the system branch. - # Note: this currently only supports strata in root repository. - - logging.debug('Changing refs to stratum %s' % stratum_name) - stratum_morph = morphs.get_stratum_in_system( - system_morph, stratum_name) - morphs.change_ref( - stratum_morph.repo_url, stratum_morph.ref, stratum_morph.filename, - sb.system_branch_name) - logging.debug('morphs: %s' % repr(morphs.morphologies)) - - # If we're editing a chunk, make it available locally, with the - # relevant git branch checked out. This also invents the new branch - # name. - - if chunk_name: - logging.debug('Editing chunk %s' % chunk_name) - - chunk_url, chunk_ref, chunk_morph = morphs.get_chunk_triplet( - stratum_morph, chunk_name) + def edit_chunk(morph, chunk_name): + chunk_url, chunk_ref, chunk_morph = ( + morphs.get_chunk_triplet(morph, chunk_name)) chunk_dirname = sb.get_git_directory_name(chunk_url) + if not os.path.exists(chunk_dirname): lrc, rrc = morphlib.util.new_repo_caches(self.app) cached_repo = lrc.get_updated_repo(chunk_url) - # FIXME: This makes the simplifying assumption that - # a chunk branch must have the same name as the system - # branch. - gd = sb.clone_cached_repo(cached_repo, chunk_ref) if chunk_ref != sb.system_branch_name: gd.branch(sb.system_branch_name, chunk_ref) @@ -491,16 +399,48 @@ class SimpleBranchAndMergePlugin(cliapp.Plugin): gd.fat_init() gd.fat_pull() - # Change the refs to the chunk. - if chunk_ref != sb.system_branch_name: - morphs.change_ref( - chunk_url, chunk_ref, chunk_morph + '.morph', - sb.system_branch_name) + # Change the refs to the chunk. + if chunk_ref != sb.system_branch_name: + morphs.change_ref( + chunk_url, chunk_ref, chunk_morph + '.morph', + sb.system_branch_name) + + return chunk_dirname + + chunk_name = morphlib.util.strip_morph_extension(args[0]) + dirs = set() + found = 0 + + for morph in morphs.morphologies: + if morph['kind'] == 'stratum': + for chunk in morph['chunks']: + if chunk['name'] == chunk_name: + self.app.status( + msg='Editing %(chunk)s in %(stratum)s stratum', + chunk=chunk_name, stratum=morph['name']) + chunk_dirname = edit_chunk(morph, chunk_name) + dirs.add(chunk_dirname) + found = found + 1 # Save any modified strata. self._save_dirty_morphologies(loader, sb, morphs.morphologies) + if found == 0: + self.app.status( + msg="No chunk %(chunk)s found. If you want to create one, add " + "an entry to a stratum morph file.", chunk=chunk_name) + + if found >= 1: + dirs_list = ', '.join(sorted(dirs)) + self.app.status( + msg="Chunk %(chunk)s source is available at %(dirs)s", + chunk=chunk_name, dirs=dirs_list) + + if found > 1: + self.app.status( + msg="Notice that this chunk appears in more than one stratum") + def show_system_branch(self, args): '''Show the name of the current system branch.''' diff --git a/morphlib/plugins/copy-artifacts_plugin.py b/morphlib/plugins/copy-artifacts_plugin.py deleted file mode 100644 index 577d0ef2..00000000 --- a/morphlib/plugins/copy-artifacts_plugin.py +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - - -# FIXME: no tests - - - -import os -import glob -import json - - -import cliapp - - - -import morphlib - - - -class CopyArtifactsPlugin(cliapp.Plugin): - - def enable(self): - self.app.add_subcommand( - 'list-artifacts', self.list_artifacts, - arg_synopsis='SYSTEM-ARTIFACT') - self.app.add_subcommand( - 'copy-artifacts', self.copy_artifacts, - arg_synopsis='SYSTEM-ARTIFACT DESTINATION') - - def disable(self): - pass - - def list_artifacts(self, args): - '''List every artifact that makes up a system artifact. - - Command line arguments: - - * `SYSTEM-ARTIFACT` is the filename for a build artifact for - a system (ending in `-rootfs`). - - list-artifacts reads the system artifact and writes out a list - of the filenames, in the local artifact cache, of all the - component artifacts included in the system. It does not include - build-dependencies of the system, unless they're included in - the system. - - ''' - - if len(args) != 1: - raise cliapp.AppException( - 'Wrong number of arguments to list-artifacts command' - '(see help)') - - system = args[0] - artifacts = self.find_artifacts(system) - for artifact in artifacts: - self.app.output.write(artifact + "\n") - - def copy_artifacts(self, args): - '''Copy every artifact that makes up a system to an rsync path. - - Command line arguments: - - * `SYSTEM-ARTIFACT` is the filename for a build artifact for - a system (ending in `-rootfs`). - * `DESTINATION` is a URL for where the artifacts are to be - copied to, in a form suitable for the rsync program. - - This command is useful for copying artifacts from a local - system to a Morph artifact cache server, so they can be - shared by other people. It can also be used to archive - artifacts used for a release. Note, however, that it does - not include artifacts that are build-dependencies of the - strata included in the system, but not actually in cluded - in the system. - - Example: - - morph copy-artifacts /src/cache/artifacts/*.system.* \ - cache.example.com:/srv/cache/ - - ''' - - if len(args) != 2: - raise cliapp.AppException( - 'Wrong number of arguments to copy-artifacts command' - '(see help)') - - system = args[0] - rsync_dest = args[1] - artifacts = self.find_artifacts(system) - cmdline = ['rsync'] - cmdline.extend(artifacts) - cmdline.append(rsync_dest) - cliapp.runcmd(cmdline) - - def find_artifacts(self, system): - artifacts_dir = os.path.join( - self.app.settings['cachedir'], 'artifacts') - artifacts = [] - - def find_in_system(dirname): - metadirs = [ - os.path.join(dirname, 'factory', 'baserock'), - os.path.join(dirname, 'baserock') - ] - existing_metadirs = [x for x in metadirs if os.path.isdir(x)] - if not existing_metadirs: - raise NotASystemArtifactError(system) - metadir = existing_metadirs[0] - for basename in glob.glob(os.path.join(metadir, '*.meta')): - metafile = os.path.join(metadir, basename) - metadata = json.load(open(metafile)) - cache_key = metadata['cache-key'] - artifact_glob = os.path.join(artifacts_dir, cache_key) + '*' - found_artifacts = glob.glob(artifact_glob) - if not found_artifacts: - raise cliapp.AppException('Could not find cache-key ' - + cache_key + 'for artifact ' - + metajson['artifact-name']) - artifacts.extend(found_artifacts) - - morphlib.bins.call_in_artifact_directory( - self.app, system, find_in_system) - - return artifacts - diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index 1d582949..3afb7b17 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -274,6 +274,7 @@ class DeployPlugin(cliapp.Plugin): self.app.settings['tempdir-min-space'], '/', 0) + self.app.settings['no-git-update'] = True cluster_name = morphlib.util.strip_morph_extension(args[0]) env_vars = args[1:] diff --git a/morphlib/plugins/gc_plugin.py b/morphlib/plugins/gc_plugin.py index df736afe..43de173e 100644 --- a/morphlib/plugins/gc_plugin.py +++ b/morphlib/plugins/gc_plugin.py @@ -61,6 +61,11 @@ class GCPlugin(cliapp.Plugin): It also removes any left over temporary chunks and staging areas from failed builds. + In addition we remove failed deployments, generally these are + cleared up by morph during deployment but in some cases they + won't be e.g. if morph gets a SIGKILL or the machine running + morph loses power. + ''' tempdir = self.app.settings['tempdir'] @@ -76,7 +81,7 @@ class GCPlugin(cliapp.Plugin): def cleanup_tempdir(self, temp_path, min_space): self.app.status(msg='Cleaning up temp dir %(temp_path)s', temp_path=temp_path, chatty=True) - for subdir in ('failed', 'chunks'): + for subdir in ('deployments', 'failed', 'chunks'): if morphlib.util.get_bytes_free_in_path(temp_path) >= min_space: self.app.status(msg='Not Removing subdirectory ' '%(subdir)s, enough space already cleared', diff --git a/morphlib/plugins/list_artifacts_plugin.py b/morphlib/plugins/list_artifacts_plugin.py new file mode 100644 index 00000000..5e64f708 --- /dev/null +++ b/morphlib/plugins/list_artifacts_plugin.py @@ -0,0 +1,124 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# This plugin is used as part of the Baserock automated release process. +# +# See: <http://wiki.baserock.org/guides/release-process> for more information. + + +import cliapp +import morphlib + + +class ListArtifactsPlugin(cliapp.Plugin): + + def enable(self): + self.app.add_subcommand( + 'list-artifacts', self.list_artifacts, + arg_synopsis='REPO REF MORPH [MORPH]...') + + def disable(self): + pass + + def list_artifacts(self, args): + '''List every artifact in the build graph of a system. + + Command line arguments: + + * `REPO` is a git repository URL. + * `REF` is a branch or other commit reference in that repository. + * `MORPH` is a system morphology name at that ref. + + You can pass multiple values for `MORPH`, in which case the command + outputs the union of the build graphs of all the systems passed in. + + The output includes any meta-artifacts such as .meta and .build-log + files. + + ''' + + if len(args) < 3: + raise cliapp.AppException( + 'Wrong number of arguments to list-artifacts command ' + '(see help)') + + repo, ref = args[0], args[1] + system_names = map(morphlib.util.strip_morph_extension, args[2:]) + + self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) + self.resolver = morphlib.artifactresolver.ArtifactResolver() + + artifact_files = set() + for system_name in system_names: + system_artifact_files = self.list_artifacts_for_system( + repo, ref, system_name) + artifact_files.update(system_artifact_files) + + for artifact_file in sorted(artifact_files): + print artifact_file + + def list_artifacts_for_system(self, repo, ref, system_name): + '''List all artifact files in the build graph of a single system.''' + + # Sadly, we must use a fresh source pool and a fresh list of artifacts + # for each system. Creating a source pool is slow (queries every Git + # repo involved in the build) and resolving artifacts isn't so quick + # either. Unfortunately, each Source object can only have one set of + # Artifact objects associated, which means the source pool cannot mix + # sources that are being built for multiple architectures: the build + # graph representation does not distinguish chunks or strata of + # different architectures right now. + + self.app.status( + msg='Creating source pool for %s' % system_name, chatty=True) + source_pool = self.app.create_source_pool( + self.lrc, self.rrc, (repo, ref, system_name + '.morph')) + + self.app.status( + msg='Resolving artifacts for %s' % system_name, chatty=True) + artifacts = self.resolver.resolve_artifacts(source_pool) + + def find_artifact_by_name(artifacts_list, name): + for a in artifacts_list: + if a.source.filename == name + '.morph': + return a + raise ValueError + + system_artifact = find_artifact_by_name(artifacts, system_name) + + self.app.status( + msg='Computing cache keys for %s' % system_name, chatty=True) + build_env = morphlib.buildenvironment.BuildEnvironment( + self.app.settings, system_artifact.source.morphology['arch']) + ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) + + artifact_files = set() + for artifact in system_artifact.walk(): + artifact.cache_key = ckc.compute_key(artifact) + artifact.cache_id = ckc.get_cache_id(artifact) + + artifact_files.add(artifact.basename()) + + if artifact.source.morphology.needs_artifact_metadata_cached: + artifact_files.add('%s.meta' % artifact.basename()) + + # This is unfortunate hardwiring of behaviour; in future we + # should list all artifacts in the meta-artifact file, so we + # don't have to guess what files there will be. + artifact_files.add('%s.meta' % artifact.cache_key) + if artifact.source.morphology['kind'] == 'chunk': + artifact_files.add('%s.build-log' % artifact.cache_key) + + return artifact_files diff --git a/morphlib/writeexts.py b/morphlib/writeexts.py index b4912db1..d6f23e0d 100644 --- a/morphlib/writeexts.py +++ b/morphlib/writeexts.py @@ -120,7 +120,8 @@ class WriteExtension(cliapp.Application): raise try: self.create_btrfs_system_layout( - temp_root, mp, version_label='factory') + temp_root, mp, version_label='factory', + disk_uuid=self.get_uuid(raw_disk)) except BaseException, e: sys.stderr.write('Error creating Btrfs system layout') self.unmount(mp) @@ -186,6 +187,13 @@ class WriteExtension(cliapp.Application): '''Create a btrfs filesystem on the disk.''' self.status(msg='Creating btrfs filesystem') cliapp.runcmd(['mkfs.btrfs', '-L', 'baserock', location]) + + def get_uuid(self, location): + '''Get the UUID of a block device's file system.''' + # Requires util-linux blkid; busybox one ignores options and + # lies by exiting successfully. + return cliapp.runcmd(['blkid', '-s', 'UUID', '-o', 'value', + location]).strip() def mount(self, location): '''Mount the filesystem so it can be tweaked. @@ -212,10 +220,12 @@ class WriteExtension(cliapp.Application): cliapp.runcmd(['umount', mount_point]) os.rmdir(mount_point) - def create_btrfs_system_layout(self, temp_root, mountpoint, version_label): + def create_btrfs_system_layout(self, temp_root, mountpoint, version_label, + disk_uuid): '''Separate base OS versions from state using subvolumes. ''' + initramfs = self.find_initramfs(temp_root) version_root = os.path.join(mountpoint, 'systems', version_label) state_root = os.path.join(mountpoint, 'state') @@ -225,7 +235,8 @@ class WriteExtension(cliapp.Application): self.create_orig(version_root, temp_root) system_dir = os.path.join(version_root, 'orig') - state_dirs = self.complete_fstab_for_btrfs_layout(system_dir) + state_dirs = self.complete_fstab_for_btrfs_layout(system_dir, + disk_uuid) for state_dir in state_dirs: self.create_state_subvolume(system_dir, mountpoint, state_dir) @@ -238,7 +249,12 @@ class WriteExtension(cliapp.Application): if self.bootloader_is_wanted(): self.install_kernel(version_root, temp_root) self.install_syslinux_menu(mountpoint, version_root) - self.install_extlinux(mountpoint) + if initramfs is not None: + self.install_initramfs(initramfs, version_root) + self.install_extlinux(mountpoint, disk_uuid) + else: + self.install_extlinux(mountpoint) + def create_orig(self, version_root, temp_root): '''Create the default "factory" system.''' @@ -283,7 +299,7 @@ class WriteExtension(cliapp.Application): filepath = os.path.join(existing_state_dir, filename) cliapp.runcmd(['mv', filepath, subvolume]) - def complete_fstab_for_btrfs_layout(self, system_dir): + def complete_fstab_for_btrfs_layout(self, system_dir, rootfs_uuid=None): '''Fill in /etc/fstab entries for the default Btrfs disk layout. In the future we should move this code out of the write extension and @@ -307,8 +323,9 @@ class WriteExtension(cliapp.Application): if '/' in existing_mounts: root_device = existing_mounts['/'] else: - root_device = '/dev/sda' - fstab.add_line('/dev/sda / btrfs defaults,rw,noatime 0 1') + root_device = ('/dev/sda' if rootfs_uuid is None else + 'UUID=%s' % rootfs_uuid) + fstab.add_line('%s / btrfs defaults,rw,noatime 0 1' % root_device) state_dirs_to_create = set() for state_dir in shared_state_dirs: @@ -322,6 +339,29 @@ class WriteExtension(cliapp.Application): fstab.write() return state_dirs_to_create + def find_initramfs(self, temp_root): + '''Check whether the rootfs has an initramfs. + + Uses the INITRAMFS_PATH option to locate it. + ''' + if 'INITRAMFS_PATH' in os.environ: + initramfs = os.path.join(temp_root, os.environ['INITRAMFS_PATH']) + if not os.path.exists(initramfs): + raise morphlib.Error('INITRAMFS_PATH specified, ' + 'but file does not exist') + return initramfs + return None + + def install_initramfs(self, initramfs_path, version_root): + '''Install the initramfs outside of 'orig' or 'run' subvolumes. + + This is required because syslinux doesn't traverse subvolumes when + loading the kernel or initramfs. + ''' + self.status(msg='Installing initramfs') + initramfs_dest = os.path.join(version_root, 'initramfs') + cliapp.runcmd(['cp', '-a', initramfs_path, initramfs_dest]) + def install_kernel(self, version_root, temp_root): '''Install the kernel outside of 'orig' or 'run' subvolumes''' @@ -337,20 +377,28 @@ class WriteExtension(cliapp.Application): def get_extra_kernel_args(self): return os.environ.get('KERNEL_ARGS', '') - def install_extlinux(self, real_root): + def install_extlinux(self, real_root, disk_uuid=None): '''Install extlinux on the newly created disk image.''' self.status(msg='Creating extlinux.conf') config = os.path.join(real_root, 'extlinux.conf') - kernel_args = self.get_extra_kernel_args() + kernel_args = ( + 'rw ' # ro ought to work, but we don't test that regularly + 'init=/sbin/init ' # default, but it doesn't hurt to be explicit + 'rootfstype=btrfs ' # required when using initramfs, also boots + # faster when specified without initramfs + 'rootflags=subvol=systems/default/run ') # boot runtime subvol + kernel_args += 'root=%s ' % ('/dev/sda' if disk_uuid is None + else 'UUID=%s' % disk_uuid) + kernel_args += self.get_extra_kernel_args() with open(config, 'w') as f: f.write('default linux\n') f.write('timeout 1\n') f.write('label linux\n') f.write('kernel /systems/default/kernel\n') - f.write('append root=/dev/sda ' - 'rootflags=subvol=systems/default/run ' - '%s init=/sbin/init rw\n' % (kernel_args)) + if disk_uuid is not None: + f.write('initrd /systems/default/initramfs\n') + f.write('append %s\n' % kernel_args) self.status(msg='Installing extlinux') cliapp.runcmd(['extlinux', '--install', real_root]) diff --git a/tests.as-root/build-with-external-strata.script b/tests.as-root/build-with-external-strata.script index e43d0262..be870053 100755 --- a/tests.as-root/build-with-external-strata.script +++ b/tests.as-root/build-with-external-strata.script @@ -34,7 +34,6 @@ cd "$DATADIR/workspace" # don't commit it, in one of the external strata, as a challenge for # 'morph build'. cd "branch1" -"$SRCDIR/scripts/test-morph" edit hello-system stratum2 cd "test/external-strata" awk ' diff --git a/tests.branching.disabled/workflow-petrify.script b/tests.branching.disabled/workflow-petrify.script index 79279340..3c561d5b 100755 --- a/tests.branching.disabled/workflow-petrify.script +++ b/tests.branching.disabled/workflow-petrify.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -34,7 +34,7 @@ cat test:morphs/hello-system.morph cat test:external-strata/stratum2.morph cat test:external-strata/stratum3.morph -"$SRCDIR/scripts/test-morph" edit hello-system stratum2 hello +"$SRCDIR/scripts/test-morph" edit hello echo echo "test/petrify after editing a chunk:" diff --git a/tests.branching/add-then-edit.script b/tests.branching/add-then-edit.script index 2dd62254..be3315d9 100755 --- a/tests.branching/add-then-edit.script +++ b/tests.branching/add-then-edit.script @@ -29,9 +29,6 @@ cd "me/add-then-edit" # add a chunk cd test/morphs -## Sub-optimally, to alter the stratum, you have to `morph edit` it first -"$SRCDIR/scripts/test-morph" edit hello-system hello-stratum - python -c 'import yaml with open("hello-stratum.morph", "r") as f: stratum = yaml.load(f) @@ -45,7 +42,7 @@ with open("hello-stratum.morph", "w") as f: yaml.dump(stratum, f) ' -"$SRCDIR/scripts/test-morph" edit hello-system hello-stratum goodbye +"$SRCDIR/scripts/test-morph" edit goodbye # check whether the stratum still contains the goodbye chunk grep -qFe goodbye hello-stratum.morph diff --git a/tests.branching/edit-checkouts-existing-chunk.script b/tests.branching/edit-checkouts-existing-chunk.script index c8fb9312..df2a7d85 100755 --- a/tests.branching/edit-checkouts-existing-chunk.script +++ b/tests.branching/edit-checkouts-existing-chunk.script @@ -27,7 +27,7 @@ cd "$DATADIR/workspace" # Edit the hello chunk in alfred. cd "alfred" -"$SRCDIR/scripts/test-morph" edit hello-system hello-stratum hello +"$SRCDIR/scripts/test-morph" edit hello echo "Current branches:" "$SRCDIR/scripts/test-morph" foreach git branch diff --git a/tests.branching/edit-clones-chunk.script b/tests.branching/edit-clones-chunk.script index d3324078..a6313ca6 100755 --- a/tests.branching/edit-clones-chunk.script +++ b/tests.branching/edit-clones-chunk.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,7 +26,7 @@ cd "$DATADIR/workspace" "$SRCDIR/scripts/test-morph" branch test:morphs newbranch # Edit chunk. -"$SRCDIR/scripts/test-morph" edit hello-system hello-stratum hello +"$SRCDIR/scripts/test-morph" edit hello echo "Current branches:" "$SRCDIR/scripts/test-morph" foreach git branch diff --git a/tests.branching/edit-handles-submodules.script b/tests.branching/edit-handles-submodules.script index 2ab39420..09592f74 100755 --- a/tests.branching/edit-handles-submodules.script +++ b/tests.branching/edit-handles-submodules.script @@ -26,7 +26,7 @@ cd "$DATADIR/workspace" "$SRCDIR/scripts/test-morph" branch test:morphs newbranch # Submodules should be set up automatically -"$SRCDIR/scripts/test-morph" edit hello-system hello-stratum hello +"$SRCDIR/scripts/test-morph" edit hello cd "$DATADIR/workspace/newbranch/test/hello" [ -e foolib/README ] diff --git a/tests.branching/edit-updates-stratum.script b/tests.branching/edit-updates-stratum.script index cf5fc26d..b60c46e7 100755 --- a/tests.branching/edit-updates-stratum.script +++ b/tests.branching/edit-updates-stratum.script @@ -26,7 +26,7 @@ cd "$DATADIR/workspace" "$SRCDIR/scripts/test-morph" branch test:morphs newbranch # Edit chunk. -"$SRCDIR/scripts/test-morph" edit hello-system hello-stratum hello +"$SRCDIR/scripts/test-morph" edit hello # See what effect the editing had. "$SRCDIR/scripts/run-git-in" "newbranch/test/morphs" diff diff --git a/tests.branching/edit-works-after-branch-root-was-renamed.script b/tests.branching/edit-works-after-branch-root-was-renamed.script index c7043e27..e28ab7df 100755 --- a/tests.branching/edit-works-after-branch-root-was-renamed.script +++ b/tests.branching/edit-works-after-branch-root-was-renamed.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -35,7 +35,7 @@ cd "$DATADIR/workspace" cd "$DATADIR/workspace/master" mv test:morphs my-renamed-morphs -"$SRCDIR/scripts/test-morph" edit hello-system hello-stratum hello +"$SRCDIR/scripts/test-morph" edit hello "$SRCDIR/scripts/list-tree" "$DATADIR/workspace" | grep -v '/\.git/' | sed 's,/cache/gits/file_[^/]*_,/cache/gits/file_,' | diff --git a/tests.branching/foreach-handles-command-failure.script b/tests.branching/foreach-handles-command-failure.script index eea381c8..4bc71c78 100755 --- a/tests.branching/foreach-handles-command-failure.script +++ b/tests.branching/foreach-handles-command-failure.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -23,6 +23,6 @@ set -eu cd "$DATADIR/workspace" "$SRCDIR/scripts/test-morph" init "$SRCDIR/scripts/test-morph" checkout test:morphs master -"$SRCDIR/scripts/test-morph" edit hello-system hello-stratum hello +"$SRCDIR/scripts/test-morph" edit hello "$SRCDIR/scripts/test-morph" foreach git remote update non-existant-remote diff --git a/tests.branching/morph-repository-stored-in-cloned-repositories.script b/tests.branching/morph-repository-stored-in-cloned-repositories.script index 342c3d0b..f60b16ae 100755 --- a/tests.branching/morph-repository-stored-in-cloned-repositories.script +++ b/tests.branching/morph-repository-stored-in-cloned-repositories.script @@ -42,7 +42,7 @@ git config morph.repository echo cd "$DATADIR/workspace/master" -"$SRCDIR/scripts/test-morph" edit hello-system hello-stratum hello +"$SRCDIR/scripts/test-morph" edit hello echo "morph.repository of an edited repository:" cd "$DATADIR/workspace/master/test/hello" diff --git a/tests.branching/petrify.script b/tests.branching/petrify.script index 5a3cb8c4..f8e7c1e9 100755 --- a/tests.branching/petrify.script +++ b/tests.branching/petrify.script @@ -29,7 +29,7 @@ cd "$DATADIR/workspace" cd test/petrify/test/morphs git push --quiet origin HEAD -"$SRCDIR/scripts/test-morph" edit hello-system hello-stratum goodbye +"$SRCDIR/scripts/test-morph" edit goodbye (cd ../goodbye && git push --quiet origin HEAD) "$SRCDIR/scripts/test-morph" petrify diff --git a/tests.branching/status-in-dirty-branch.script b/tests.branching/status-in-dirty-branch.script index 7fdd8862..37fca97b 100755 --- a/tests.branching/status-in-dirty-branch.script +++ b/tests.branching/status-in-dirty-branch.script @@ -35,7 +35,7 @@ cd "$DATADIR/workspace" # Make the branch have some interesting changes and pitfalls cd branch1 -"$SRCDIR/scripts/test-morph" edit hello-system stratum2 hello +"$SRCDIR/scripts/test-morph" edit hello cd test/stratum2-hello git checkout -q master diff --git a/tests.branching/workflow-separate-stratum-repos.script b/tests.branching/workflow-separate-stratum-repos.script index f2fd519b..1d8cc1e5 100755 --- a/tests.branching/workflow-separate-stratum-repos.script +++ b/tests.branching/workflow-separate-stratum-repos.script @@ -34,14 +34,14 @@ cd "$DATADIR/workspace" # Edit one chunk cd "me/readme-fixes" -"$SRCDIR/scripts/test-morph" edit hello-system stratum2 hello +"$SRCDIR/scripts/test-morph" edit hello cd "$DATADIR/workspace/me/readme-fixes/test/stratum2-hello" echo > README yoyoyo git add README git commit -m "Fix README in hello" --quiet # Edit the other chunk too -"$SRCDIR/scripts/test-morph" edit hello-system stratum3 hello +"$SRCDIR/scripts/test-morph" edit hello cd "$DATADIR/workspace/me/readme-fixes/test/stratum3-hello" echo > README yoyoyo git add README diff --git a/tests.branching/workflow.script b/tests.branching/workflow.script index 51a8d106..f84489db 100755 --- a/tests.branching/workflow.script +++ b/tests.branching/workflow.script @@ -23,7 +23,7 @@ set -eu cd "$DATADIR/workspace" "$SRCDIR/scripts/test-morph" init "$SRCDIR/scripts/test-morph" branch test:morphs me/readme-fix -"$SRCDIR/scripts/test-morph" edit hello-system hello-stratum hello +"$SRCDIR/scripts/test-morph" edit hello cd me/readme-fix/test/hello echo > README yoyoyo git add README diff --git a/tests.deploy/setup-build b/tests.deploy/setup-build index 0fc561f9..c6b24da5 100644 --- a/tests.deploy/setup-build +++ b/tests.deploy/setup-build @@ -23,7 +23,7 @@ source "$SRCDIR/scripts/fix-committer-info" cd "$DATADIR/workspace" "$SRCDIR/scripts/test-morph" init "$SRCDIR/scripts/test-morph" branch test:morphs branch1 -"$SRCDIR/scripts/test-morph" edit linux-system linux-stratum linux +"$SRCDIR/scripts/test-morph" edit linux # Fix UUID's in the checked out repos to make build branch names deterministic git config -f "$DATADIR/workspace/branch1/.morph-system-branch/config" \ diff --git a/tests.merging/basic.script b/tests.merging/basic.script index 6d01ba16..5a1c1842 100755 --- a/tests.merging/basic.script +++ b/tests.merging/basic.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -34,7 +34,7 @@ git push --quiet origin test/stable cd "$DATADIR/workspace/test/feature" # Edit hello in FROM -"$SRCDIR/scripts/test-morph" edit hello-system hello-stratum hello +"$SRCDIR/scripts/test-morph" edit hello cd test:hello touch newfile.txt git add newfile.txt diff --git a/tests.merging/conflict-chunks.script b/tests.merging/conflict-chunks.script index 5a0c5f52..b0d118ee 100755 --- a/tests.merging/conflict-chunks.script +++ b/tests.merging/conflict-chunks.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -44,14 +44,12 @@ add_text_in_repo() { # Sow the seeds of conflict cd "$DATADIR/workspace/test/stable" -"$SRCDIR/scripts/test-morph" edit hello-system stratum2 hello -"$SRCDIR/scripts/test-morph" edit hello-system stratum3 hello +"$SRCDIR/scripts/test-morph" edit hello add_text_in_repo "test:stratum2-hello" "xyzzy" add_text_in_repo "test:stratum3-hello" "xyzzy" cd "$DATADIR/workspace/test/feature" -"$SRCDIR/scripts/test-morph" edit hello-system stratum2 hello -"$SRCDIR/scripts/test-morph" edit hello-system stratum3 hello +"$SRCDIR/scripts/test-morph" edit hello add_text_in_repo "test:stratum2-hello" "plugh" add_text_in_repo "test:stratum3-hello" "plover" diff --git a/tests.merging/conflict-morphology-kind.script b/tests.merging/conflict-morphology-kind.script index ca8403b8..cd2a24f5 100755 --- a/tests.merging/conflict-morphology-kind.script +++ b/tests.merging/conflict-morphology-kind.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -25,7 +25,6 @@ cd "$DATADIR/workspace" "$SRCDIR/scripts/test-morph" branch test:morphs test/unmergable cd "$DATADIR/workspace/test/unmergable/test:morphs" -"$SRCDIR/scripts/test-morph" edit hello-system hello-stratum sed -ie 's/"kind": "stratum"/"kind": "chunk"/' hello-stratum.morph git commit --quiet --all -m "Unmergeable because kind has changed" diff --git a/tests.merging/conflict-stratum-field-ordering.script b/tests.merging/conflict-stratum-field-ordering.script index c96c59ab..b83358bf 100755 --- a/tests.merging/conflict-stratum-field-ordering.script +++ b/tests.merging/conflict-stratum-field-ordering.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -41,7 +41,6 @@ git push --quiet origin test/stable # Make a change in TO cd "$DATADIR/workspace/test/stable" -"$SRCDIR/scripts/test-morph" edit hello-system hello-stratum cd test:morphs cat <<EOF > "hello-stratum.morph" { @@ -69,7 +68,6 @@ git commit --quiet --all -m "Split up 'hello' chunk into runtime and devel" # Make a change in FROM that isn't very mergable cd "$DATADIR/workspace/test/feature" -"$SRCDIR/scripts/test-morph" edit hello-system hello-stratum cd test:morphs cat <<EOF > "hello-stratum.morph" { diff --git a/tests.merging/move-chunk-repo.script b/tests.merging/move-chunk-repo.script index 3a00015b..405e6c88 100755 --- a/tests.merging/move-chunk-repo.script +++ b/tests.merging/move-chunk-repo.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,7 +26,7 @@ cd "$DATADIR/workspace" "$SRCDIR/scripts/test-morph" init "$SRCDIR/scripts/test-morph" branch test:morphs baserock/newbranch -"$SRCDIR/scripts/test-morph" edit hello-system hello-stratum hello +"$SRCDIR/scripts/test-morph" edit hello # Chunk moves to a new location (we manually update the ref back to master # here, so 'morph edit' can create a system branch in the new repo from it). @@ -38,7 +38,7 @@ sed -e 's/"repo": "test:hello"/"repo": "test:hello-lorried"/' \ git commit -q --all -m "'hello' repository has moved" # Now we further edit the chunk, just for fun. -"$SRCDIR/scripts/test-morph" edit hello-system hello-stratum hello +"$SRCDIR/scripts/test-morph" edit hello cd "$DATADIR/workspace/baserock/newbranch/test:hello-lorried" touch newfile.txt git add newfile.txt diff --git a/tests.merging/rename-chunk.script b/tests.merging/rename-chunk.script index 8c323798..ac63cdd7 100755 --- a/tests.merging/rename-chunk.script +++ b/tests.merging/rename-chunk.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -27,7 +27,7 @@ cd "$DATADIR/workspace" "$SRCDIR/scripts/test-morph" branch test:morphs baserock/newbranch # Rename the chunk, and then commit a seperate change -"$SRCDIR/scripts/test-morph" edit hello-system hello-stratum hello +"$SRCDIR/scripts/test-morph" edit hello cd baserock/newbranch/test:hello cat hello.morph | sed -e 's/"name": "hello"/"name": "goodbye"/' > goodbye.morph diff --git a/tests.merging/rename-stratum.script b/tests.merging/rename-stratum.script index ba759fa3..11c4cb50 100755 --- a/tests.merging/rename-stratum.script +++ b/tests.merging/rename-stratum.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,14 +26,8 @@ cd "$DATADIR/workspace" "$SRCDIR/scripts/test-morph" init "$SRCDIR/scripts/test-morph" branch test:morphs baserock/newbranch -# The user may 'morph edit hello-system hello-stratum hello' and commit here: -# we currently silently ignore her changes on merge, because we don't -# associate hello-stratum and goodbye-stratum at all. - # Rename the stratum -"$SRCDIR/scripts/test-morph" edit hello-system hello-stratum cd baserock/newbranch/test:morphs - sed -e 's/"morph": "hello-stratum"/"morph": "goodbye-stratum"/'\ -i hello-system.morph sed -e 's/"name": "hello-stratum"/"name": "goodbye-stratum"/' \ @@ -42,9 +36,6 @@ git rm -q hello-stratum.morph git add goodbye-stratum.morph git commit -q --all -m "Rename hello-stratum to goodbye-stratum" -# The user may 'morph edit hello-system goodbye-stratum hello' and commit -# here, too: same problem. - # Merge changes back to master (this should fail, because we don't support # adding strata inside branches yet). cd "$DATADIR/workspace" diff --git a/without-test-modules b/without-test-modules index a42cce97..a847da86 100644 --- a/without-test-modules +++ b/without-test-modules @@ -25,7 +25,7 @@ morphlib/plugins/expand_repo_plugin.py morphlib/plugins/deploy_plugin.py morphlib/plugins/__init__.py morphlib/writeexts.py -morphlib/plugins/copy-artifacts_plugin.py +morphlib/plugins/list_artifacts_plugin.py morphlib/plugins/trovectl_plugin.py morphlib/plugins/gc_plugin.py morphlib/plugins/branch_and_merge_new_plugin.py diff --git a/yarns/branches-workspaces.yarn b/yarns/branches-workspaces.yarn index a9cfb19b..c542994a 100644 --- a/yarns/branches-workspaces.yarn +++ b/yarns/branches-workspaces.yarn @@ -203,16 +203,6 @@ fields when referring to strata, when it didn't before. AND in branch foo, system test-system refers to test-stratum without repo AND in branch foo, system test-system refers to test-stratum without ref -Morph edit should only work with a system argument. - - SCENARIO morph edit errors when supplied only a stratum and chunk as arguments - GIVEN a workspace - AND a git server - WHEN the user checks out the system branch called master - AND the user edits the chunk test-chunk in the stratum test-stratum with no system specified in branch master - THEN morph failed - AND the edit error message includes the string "is not a system" - Status of system branch checkout -------------------------------- @@ -226,8 +216,7 @@ repositories referenced in the system branch. WHEN the user creates a system branch called foo THEN morph reports no outstanding changes in foo - WHEN the user edits the stratum test-stratum in the system test-system in branch foo - AND the user edits the chunk test-chunk in the stratum test-stratum in the system test-system in branch foo + WHEN the user edits the chunk test-chunk in the stratum test-stratum in the system test-system in branch foo THEN morph reports changes in foo in test:morphs only WHEN creating file foo in test/test-chunk in branch foo diff --git a/yarns/deployment.yarn b/yarns/deployment.yarn index fc21b826..67aecce2 100644 --- a/yarns/deployment.yarn +++ b/yarns/deployment.yarn @@ -102,3 +102,86 @@ deployed system contains the other. Since the baserock directory is in every system, we can check for that. AND tarball test.tar contains var/lib/sysroots/test-system/baserock + +Initramfs deployments +===================== + +There's a few ways of creating an initramfs. We could: +1. Build a sysroot and: + 1. Have a chunk turn that into a cpio archive, written into /boot. + 2. Embed it in the Linux kernel image, having the initramfs as part + of the BSP. +2. Deploy an existing system as a cpio archive + 1. As a stand-alone system, without a rootfs + 2. Nested inside another system + +1.1 and 1.2 require system engineering work, so won't be mentioned here. + + SCENARIO deploying a system with an initramfs + ASSUMING there is space for 5 512M disk images + GIVEN a workspace + AND a git server + WHEN the user checks out the system branch called master + GIVEN a cluster called C in system branch master + AND a system in cluster C in branch master called S + +2.2 needs a nested system that is deployed with the initramfs write +extension. + + GIVEN a subsystem in cluster C in branch master called S.I + AND subsystem S.I in cluster C in branch master builds test-system + AND subsystem S.I in cluster C in branch master has deployment type: initramfs + +The nested system needs to be placed somewhere in the parent. The +traditional place for an initramfs is `/boot`. + + AND subsystem S.I in cluster C in branch master has deployment location: boot/initramfs.gz + +1.1 and 2.2 need the write extension to configure the boot-loader to +use the produced initramfs. Only write extensions that involve creating a disk image care, so we'll use `rawdisk.write`. + + GIVEN system S in cluster C in branch master builds test-system + AND system S in cluster C in branch master has deployment type: rawdisk + AND system S in cluster C in branch master has deployment location: test.img + AND system S in cluster C in branch master has deployment variable: DISK_SIZE=512M + +Initramfs support is triggered by the `INITRAMFS_PATH` variable. It could have been made automatic, triggering the behaviour if `/boot/initramfs.gz` exists, but: + +1. There are a bunch of possible names, some of which imply different formats. +2. If we decide on one specific name, how do we pick. +3. If we allow multiple possible names, how do we handle multiple being possible. +4. We may need to pick a non-standard name: e.g. We have a deployment + where the system loads a kernel and initramfs from a disk, then boots + the target in KVM, so the bootloader we want to use for the guest is + `initramfs.gz`, while the host's initramfs is `hyp-initramfs.gz`. +5. We may have the initramfs come from a chunk the system built, but + for speed, we want this particular deployment not to use an initramfs, + even though we have a generic image that may support one. + +For all these reasons, despite there being redundancy in some cases, +we're going to set `INITRAMFS_PATH` to the same as the nested deployment's +location. + + GIVEN system S in cluster C in branch master has deployment variable: INITRAMFS_PATH=boot/initramfs.gz + +Fully testing that the system is bootable requires a lot more time, +infrastructure and dependencies, so we're just going to build it and +inspect the result of the deployment. + + WHEN the user builds the system test-system in branch master + AND the user attempts to deploy the cluster C in branch master + THEN morph succeeded + AND file workspace/master/test/morphs/test.img exists + +If the initramfs write extension works, the rootfs image should contain +`boot/initramfs.gz`. + + WHEN disk image workspace/master/test/morphs/test.img is mounted at mnt + THEN file mnt/systems/default/run/boot/initramfs.gz exists + +If the `rawdisk` write extension worked, then the bootloader config file +will mention the initramfs, and the UUID of the disk. + + AND file mnt/extlinux.conf matches initramfs + AND file mnt/extlinux.conf matches root=UUID= + FINALLY mnt is unmounted diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index e4f36399..66d47bfd 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -423,13 +423,9 @@ Editing morphologies with `morph edit`. "$field" name="$MATCH_4" "$MATCH_5"; } 2>&1 | grep -qFe "Object does not contain $MATCH_5" - IMPLEMENTS WHEN the user edits the stratum (\S+) in the system (\S+) in branch (\S+) - cd "$DATADIR/workspace/$MATCH_3/test/morphs" - run_morph edit "$MATCH_2" "$MATCH_1" - IMPLEMENTS WHEN the user edits the chunk (\S+) in the stratum (\S+) in the system (\S+) in branch (\S+) cd "$DATADIR/workspace/$MATCH_4/test/morphs" - run_morph edit "$MATCH_3" "$MATCH_2" "$MATCH_1" + run_morph edit "$MATCH_1" IMPLEMENTS THEN the edited chunk (\S+) has git branch (\S+) ls -l "$DATADIR/workspace/$MATCH_2" @@ -439,10 +435,6 @@ Editing morphologies with `morph edit`. echo "$MATCH_2" > "$DATADIR/git-branch.wanted" diff -u "$DATADIR/git-branch.wanted" "$DATADIR/git-branch.actual" - IMPLEMENTS WHEN the user edits the chunk (\S+) in the stratum (\S+) with no system specified in branch (\S+) - cd "$DATADIR/workspace/$MATCH_3" - attempt_morph edit "$MATCH_2" "$MATCH_1" - To produce buildable morphologies, we need them to be of the same architecture as the machine doing the testing. This uses `morph print-architecture` to get a value appropriate for morph. @@ -732,6 +724,49 @@ Check attributes of a file on the filesystem IMPLEMENTS THEN file (\S+) is empty stat -c %s "$DATADIR/$MATCH_1" | grep -Fx 0 + IMPLEMENTS THEN file (\S+) matches (.*) + grep -q "$MATCH_2" "$DATADIR/$MATCH_1" + +Disk image manipulation +----------------------- + +We need to test disk images we create. In the absence of tools for +inspecting disks without mounting them, we need commands to handle this. + + IMPLEMENTS WHEN disk image (\S+) is mounted at (.*) + mkdir -p "$DATADIR/$MATCH_2" + mount -o loop "$DATADIR/$MATCH_1" "$DATADIR/$MATCH_2" + + IMPLEMENTS FINALLY (\S+) is unmounted + umount -d "$DATADIR/$MATCH_1" + +We may not have enough space to run some tests that have disk images. + + IMPLEMENTS ASSUMING there is space for (\d+) (\d+)(\S*) disk images? + # Count is included as an argument, so that if we change the disk + # image sizes then it's more obvious when we need to change the + # assumption, since it's the same value. + count="$MATCH_1" + case "$MATCH_3" in + '') + size="$MATCH_2" + ;; + M) + size=$(expr "$MATCH_2" '*' 1024 '*' 1024 ) + ;; + G) + size=$(expr "$MATCH_2" '*' 1024 '*' 1024 '*' 1024 ) + ;; + *) + echo Unrecognized size suffix: "$MATCH_3" >&2 + exit 1 + esac + total_image_size="$(expr "$size" '*' "$count" )" + blocks="$(stat -f -c %a "$DATADIR")" + block_size="$(stat -f -c %S "$DATADIR")" + disk_free=$(expr "$blocks" '*' "$block_size" ) + test "$disk_free" -gt "$total_image_size" + Check contents of a file ------------------------ |