diff options
| -rw-r--r-- | NEWS | 10 | ||||
| -rw-r--r-- | distbuild/build_controller.py | 10 | ||||
| -rw-r--r-- | distbuild/worker_build_scheduler.py | 69 | ||||
| -rw-r--r-- | morphlib/plugins/copy-artifacts_plugin.py | 142 | ||||
| -rw-r--r-- | morphlib/plugins/list_artifacts_plugin.py | 124 | ||||
| -rw-r--r-- | without-test-modules | 2 |
6 files changed, 181 insertions, 176 deletions
@@ -4,6 +4,16 @@ NEWS for Morph This file contains high-level summaries of user-visible changes in each Morph release. +Version 14.23, released 2014-06-06 +---------------------------------- + +New feature: + +* Initramfs support + +There have also been a number of fixes to distbuild, and the +`morph copy-artifacts` command has been replaced by `morph list-artifacts`. + Version 14.22, released 2014-05-29 ---------------------------------- diff --git a/distbuild/build_controller.py b/distbuild/build_controller.py index c399939e..987f01f4 100644 --- a/distbuild/build_controller.py +++ b/distbuild/build_controller.py @@ -39,8 +39,9 @@ class _Built(object): pass class _AnnotationFailed(object): - def __init__(self, http_status_code): + def __init__(self, http_status_code, error_msg): self.http_status_code = http_status_code + self.error_msg = error_msg class _GotGraph(object): @@ -359,12 +360,13 @@ class BuildController(distbuild.StateMachine): logging.debug('Got cache response: %s' % repr(event.msg)) http_status_code = event.msg['status'] + error_msg = event.msg['body'] if http_status_code != httplib.OK: logging.debug('Cache request failed with status: %s' % event.msg['status']) self.mainloop.queue_event(self, - _AnnotationFailed(http_status_code)) + _AnnotationFailed(http_status_code, error_msg)) return cache_state = json.loads(event.msg['body']) @@ -577,8 +579,8 @@ class BuildController(distbuild.StateMachine): self._queue_worker_builds(None, event) def _notify_annotation_failed(self, event_source, event): - errmsg = ('Failed to annotate build graph: http request got %d' - % event.http_status_code) + errmsg = ('Failed to annotate build graph: http request got %d: %s' + % (event.http_status_code, event.error_msg)) logging.error(errmsg) failed = BuildFailed(self._request['id'], errmsg) diff --git a/distbuild/worker_build_scheduler.py b/distbuild/worker_build_scheduler.py index a7f7eb80..57cc0224 100644 --- a/distbuild/worker_build_scheduler.py +++ b/distbuild/worker_build_scheduler.py @@ -100,7 +100,7 @@ class Job(object): self.artifact = artifact self.initiators = [initiator_id] self.who = None # we don't know who's going to do this yet - self.is_building = False + self.running = False self.failed = False @@ -120,18 +120,18 @@ class Jobs(object): return job def remove(self, job): - del self._jobs[job.artifact.basename()] + if job.artifact.basename() in self._jobs: + del self._jobs[job.artifact.basename()] + else: + logging.warning("Tried to remove a job that doesn't exist " + "(%s)", job.artifact.basename()) def get_jobs(self): return self._jobs def remove_jobs(self, jobs): for job in jobs: - if job.artifact.basename() in self._jobs: - self.remove(job) - else: - logging.warning("Tried to remove a job that doesn't exist " - "(%s)", job.artifact.basename()) + self.remove(job) def exists(self, artifact_basename): return artifact_basename in self._jobs @@ -168,19 +168,19 @@ class _Cached(object): pass -class _ExecStarted(object): +class _JobStarted(object): def __init__(self, job): self.job = job -class _ExecEnded(object): +class _JobFinished(object): def __init__(self, job): self.job = job -class _ExecFailed(object): +class _JobFailed(object): def __init__(self, job): self.job = job @@ -215,30 +215,30 @@ class WorkerBuildQueuer(distbuild.StateMachine): self._handle_cancel), ('idle', WorkerConnection, _NeedJob, 'idle', self._handle_worker), - ('idle', WorkerConnection, _ExecStarted, 'idle', - self._set_exec_started), - ('idle', WorkerConnection, _ExecEnded, 'idle', - self._set_exec_ended), - ('idle', WorkerConnection, _ExecFailed, 'idle', - self._set_exec_failed) + ('idle', WorkerConnection, _JobStarted, 'idle', + self._set_job_started), + ('idle', WorkerConnection, _JobFinished, 'idle', + self._set_job_finished), + ('idle', WorkerConnection, _JobFailed, 'idle', + self._set_job_failed) ] self.add_transitions(spec) - def _set_exec_started(self, event_source, event): + def _set_job_started(self, event_source, event): logging.debug('Setting job state for job %s with id %s: ' - 'Job is building', + 'Job is running', event.job.artifact.basename(), event.job.id) - event.job.is_building = True + event.job.running = True - def _set_exec_ended(self, event_source, event): + def _set_job_finished(self, event_source, event): logging.debug('Setting job state for job %s with id %s: ' - 'Job is NOT building', + 'Job is NOT running', event.job.artifact.basename(), event.job.id) - event.job.is_building = False + event.job.running = False - def _set_exec_failed(self, event_source, event): + def _set_job_failed(self, event_source, event): logging.debug('Job %s with id %s failed', event.job.artifact.basename(), event.job.id) event.job.failed = True @@ -258,7 +258,7 @@ class WorkerBuildQueuer(distbuild.StateMachine): job = self._jobs.get(event.artifact.basename()) job.initiators.append(event.initiator_id) - if job.is_building: + if job.running: logging.debug('Worker build step already started: %s' % event.artifact.basename()) progress = WorkerBuildStepAlreadyStarted(event.initiator_id, @@ -295,7 +295,7 @@ class WorkerBuildQueuer(distbuild.StateMachine): name, job_id) if len(job.initiators) == 1: - if job.is_building or job.failed: + if job.running or job.failed: logging.debug('NOT removing running job %s with job id %s ' '(WorkerConnection will cancel job)', name, job_id) @@ -485,7 +485,7 @@ class WorkerConnection(distbuild.StateMachine): started = WorkerBuildStepStarted(self._job.initiators, self._job.artifact.cache_key, self.name()) - self.mainloop.queue_event(WorkerConnection, _ExecStarted(self._job)) + self.mainloop.queue_event(WorkerConnection, _JobStarted(self._job)) self.mainloop.queue_event(WorkerConnection, started) def _handle_json_message(self, event_source, event): @@ -524,15 +524,13 @@ class WorkerConnection(distbuild.StateMachine): # Build failed. new_event = WorkerBuildFailed(new, self._job.artifact.cache_key) self.mainloop.queue_event(WorkerConnection, new_event) - self.mainloop.queue_event(WorkerConnection, _ExecFailed(self._job)) + self.mainloop.queue_event(WorkerConnection, _JobFailed(self._job)) self.mainloop.queue_event(self, _BuildFailed()) else: # Build succeeded. We have more work to do: caching the result. self.mainloop.queue_event(self, _BuildFinished()) self._exec_response_msg = new - self.mainloop.queue_event(WorkerConnection, _ExecEnded(self._job)) - def _request_job(self, event_source, event): distbuild.crash_point() self.mainloop.queue_event(WorkerConnection, _NeedJob(self)) @@ -600,7 +598,20 @@ class WorkerConnection(distbuild.StateMachine): logging.error( 'Failed to populate artifact cache: %s %s' % (event.msg['status'], event.msg['body'])) + + # We will attempt to remove this job twice + # unless we mark it as failed before the BuildController + # processes the WorkerBuildFailed event. + # + # The BuildController will not try to cancel jobs that have + # been marked as failed. + self.mainloop.queue_event(WorkerConnection, + _JobFailed(self._job)) + new_event = WorkerBuildFailed( self._exec_response_msg, self._job.artifact.cache_key) self.mainloop.queue_event(WorkerConnection, new_event) + self.mainloop.queue_event(self, _BuildFailed()) + + self.mainloop.queue_event(WorkerConnection, _JobFinished(self._job)) diff --git a/morphlib/plugins/copy-artifacts_plugin.py b/morphlib/plugins/copy-artifacts_plugin.py deleted file mode 100644 index 577d0ef2..00000000 --- a/morphlib/plugins/copy-artifacts_plugin.py +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - - -# FIXME: no tests - - - -import os -import glob -import json - - -import cliapp - - - -import morphlib - - - -class CopyArtifactsPlugin(cliapp.Plugin): - - def enable(self): - self.app.add_subcommand( - 'list-artifacts', self.list_artifacts, - arg_synopsis='SYSTEM-ARTIFACT') - self.app.add_subcommand( - 'copy-artifacts', self.copy_artifacts, - arg_synopsis='SYSTEM-ARTIFACT DESTINATION') - - def disable(self): - pass - - def list_artifacts(self, args): - '''List every artifact that makes up a system artifact. - - Command line arguments: - - * `SYSTEM-ARTIFACT` is the filename for a build artifact for - a system (ending in `-rootfs`). - - list-artifacts reads the system artifact and writes out a list - of the filenames, in the local artifact cache, of all the - component artifacts included in the system. It does not include - build-dependencies of the system, unless they're included in - the system. - - ''' - - if len(args) != 1: - raise cliapp.AppException( - 'Wrong number of arguments to list-artifacts command' - '(see help)') - - system = args[0] - artifacts = self.find_artifacts(system) - for artifact in artifacts: - self.app.output.write(artifact + "\n") - - def copy_artifacts(self, args): - '''Copy every artifact that makes up a system to an rsync path. - - Command line arguments: - - * `SYSTEM-ARTIFACT` is the filename for a build artifact for - a system (ending in `-rootfs`). - * `DESTINATION` is a URL for where the artifacts are to be - copied to, in a form suitable for the rsync program. - - This command is useful for copying artifacts from a local - system to a Morph artifact cache server, so they can be - shared by other people. It can also be used to archive - artifacts used for a release. Note, however, that it does - not include artifacts that are build-dependencies of the - strata included in the system, but not actually in cluded - in the system. - - Example: - - morph copy-artifacts /src/cache/artifacts/*.system.* \ - cache.example.com:/srv/cache/ - - ''' - - if len(args) != 2: - raise cliapp.AppException( - 'Wrong number of arguments to copy-artifacts command' - '(see help)') - - system = args[0] - rsync_dest = args[1] - artifacts = self.find_artifacts(system) - cmdline = ['rsync'] - cmdline.extend(artifacts) - cmdline.append(rsync_dest) - cliapp.runcmd(cmdline) - - def find_artifacts(self, system): - artifacts_dir = os.path.join( - self.app.settings['cachedir'], 'artifacts') - artifacts = [] - - def find_in_system(dirname): - metadirs = [ - os.path.join(dirname, 'factory', 'baserock'), - os.path.join(dirname, 'baserock') - ] - existing_metadirs = [x for x in metadirs if os.path.isdir(x)] - if not existing_metadirs: - raise NotASystemArtifactError(system) - metadir = existing_metadirs[0] - for basename in glob.glob(os.path.join(metadir, '*.meta')): - metafile = os.path.join(metadir, basename) - metadata = json.load(open(metafile)) - cache_key = metadata['cache-key'] - artifact_glob = os.path.join(artifacts_dir, cache_key) + '*' - found_artifacts = glob.glob(artifact_glob) - if not found_artifacts: - raise cliapp.AppException('Could not find cache-key ' - + cache_key + 'for artifact ' - + metajson['artifact-name']) - artifacts.extend(found_artifacts) - - morphlib.bins.call_in_artifact_directory( - self.app, system, find_in_system) - - return artifacts - diff --git a/morphlib/plugins/list_artifacts_plugin.py b/morphlib/plugins/list_artifacts_plugin.py new file mode 100644 index 00000000..5e64f708 --- /dev/null +++ b/morphlib/plugins/list_artifacts_plugin.py @@ -0,0 +1,124 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# This plugin is used as part of the Baserock automated release process. +# +# See: <http://wiki.baserock.org/guides/release-process> for more information. + + +import cliapp +import morphlib + + +class ListArtifactsPlugin(cliapp.Plugin): + + def enable(self): + self.app.add_subcommand( + 'list-artifacts', self.list_artifacts, + arg_synopsis='REPO REF MORPH [MORPH]...') + + def disable(self): + pass + + def list_artifacts(self, args): + '''List every artifact in the build graph of a system. + + Command line arguments: + + * `REPO` is a git repository URL. + * `REF` is a branch or other commit reference in that repository. + * `MORPH` is a system morphology name at that ref. + + You can pass multiple values for `MORPH`, in which case the command + outputs the union of the build graphs of all the systems passed in. + + The output includes any meta-artifacts such as .meta and .build-log + files. + + ''' + + if len(args) < 3: + raise cliapp.AppException( + 'Wrong number of arguments to list-artifacts command ' + '(see help)') + + repo, ref = args[0], args[1] + system_names = map(morphlib.util.strip_morph_extension, args[2:]) + + self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) + self.resolver = morphlib.artifactresolver.ArtifactResolver() + + artifact_files = set() + for system_name in system_names: + system_artifact_files = self.list_artifacts_for_system( + repo, ref, system_name) + artifact_files.update(system_artifact_files) + + for artifact_file in sorted(artifact_files): + print artifact_file + + def list_artifacts_for_system(self, repo, ref, system_name): + '''List all artifact files in the build graph of a single system.''' + + # Sadly, we must use a fresh source pool and a fresh list of artifacts + # for each system. Creating a source pool is slow (queries every Git + # repo involved in the build) and resolving artifacts isn't so quick + # either. Unfortunately, each Source object can only have one set of + # Artifact objects associated, which means the source pool cannot mix + # sources that are being built for multiple architectures: the build + # graph representation does not distinguish chunks or strata of + # different architectures right now. + + self.app.status( + msg='Creating source pool for %s' % system_name, chatty=True) + source_pool = self.app.create_source_pool( + self.lrc, self.rrc, (repo, ref, system_name + '.morph')) + + self.app.status( + msg='Resolving artifacts for %s' % system_name, chatty=True) + artifacts = self.resolver.resolve_artifacts(source_pool) + + def find_artifact_by_name(artifacts_list, name): + for a in artifacts_list: + if a.source.filename == name + '.morph': + return a + raise ValueError + + system_artifact = find_artifact_by_name(artifacts, system_name) + + self.app.status( + msg='Computing cache keys for %s' % system_name, chatty=True) + build_env = morphlib.buildenvironment.BuildEnvironment( + self.app.settings, system_artifact.source.morphology['arch']) + ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) + + artifact_files = set() + for artifact in system_artifact.walk(): + artifact.cache_key = ckc.compute_key(artifact) + artifact.cache_id = ckc.get_cache_id(artifact) + + artifact_files.add(artifact.basename()) + + if artifact.source.morphology.needs_artifact_metadata_cached: + artifact_files.add('%s.meta' % artifact.basename()) + + # This is unfortunate hardwiring of behaviour; in future we + # should list all artifacts in the meta-artifact file, so we + # don't have to guess what files there will be. + artifact_files.add('%s.meta' % artifact.cache_key) + if artifact.source.morphology['kind'] == 'chunk': + artifact_files.add('%s.build-log' % artifact.cache_key) + + return artifact_files diff --git a/without-test-modules b/without-test-modules index a42cce97..a847da86 100644 --- a/without-test-modules +++ b/without-test-modules @@ -25,7 +25,7 @@ morphlib/plugins/expand_repo_plugin.py morphlib/plugins/deploy_plugin.py morphlib/plugins/__init__.py morphlib/writeexts.py -morphlib/plugins/copy-artifacts_plugin.py +morphlib/plugins/list_artifacts_plugin.py morphlib/plugins/trovectl_plugin.py morphlib/plugins/gc_plugin.py morphlib/plugins/branch_and_merge_new_plugin.py |