diff options
author | Lars Wirzenius <lars.wirzenius@codethink.co.uk> | 2014-07-22 18:11:49 +0000 |
---|---|---|
committer | Lars Wirzenius <lars.wirzenius@codethink.co.uk> | 2014-07-24 16:31:23 +0000 |
commit | 313234e1e6f59ed8dfa4f0a11d0b91aab2d33c34 (patch) | |
tree | d9145d6ffa2e3d450af02d3e4dbbdc074a198f1c /scripts/release-build | |
parent | ef11b3db09526bc568261ef72a8ee0e346b753d3 (diff) | |
download | definitions-313234e1e6f59ed8dfa4f0a11d0b91aab2d33c34.tar.gz |
Add new scripts for building, uploading release
These scripts are a rewrite of scripts/do-release.py and
scripts/distbuild-cluster. The biggest difference is that they split
the tasks of building the things that are to be released, and uploading
them to git.baserock.org / download.baserock.org, where do-release.py
combines both (and distbuild-cluster only builds chunk/stratum/system
artifacts, not the release images). The new scripts are also configurable
using command line options or a configuration file rather than requiring
editing of the source.
These changes will allow, for example, a CI job that builds a release,
but doesn't upload it to download.baserock.org.
The new scripts are coupled with a change to the release process, which
will be documented as a change to the release process page on
wiki.baserock.org.
The 14.29 release of Baserock was done with slightly different versions
of these scripts to make it feasible to upload things over multiple
network connections.
Diffstat (limited to 'scripts/release-build')
-rwxr-xr-x | scripts/release-build | 161 |
1 files changed, 161 insertions, 0 deletions
diff --git a/scripts/release-build b/scripts/release-build new file mode 100755 index 00000000..36a38deb --- /dev/null +++ b/scripts/release-build @@ -0,0 +1,161 @@ +#!/usr/bin/env python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import cliapp +import morphlib +import os +import subprocess +import sys +import time + + +class Build(object): + '''A single distbuild instance.''' + + def __init__(self, name, arch, app): + self.system_name = name + self.controller = app.controllers[arch] + self.command = [ + 'morph', 'distbuild-morphology', + '--controller-initiator-address=%s' % self.controller, + 'baserock:baserock/definitions', app.ref, self.system_name] + + def start(self): + self.process = subprocess.Popen(self.command) + + def completed(self): + return (self.process.poll() is not None) + + +class ReleaseApp(cliapp.Application): + + '''Cliapp app that handles distbuilding and deploying a cluster.''' + + def add_settings(self): + self.settings.string_list(['controllers'], + 'a list of distbuild controllers and their ' + 'architecture') + + self.settings.string(['trove-host'], + 'hostname of Trove instance') + + self.settings.string(['release-number'], + 'Baserock version of the systems being built', + default='yy.ww') + + def process_args(self, args): + '''Process the command line''' + self.controllers = {} + controllers_list = self.settings['controllers'] + for item in controllers_list: + arch, controller = item.split(':') + self.controllers[arch] = controller + + self.ref = cliapp.runcmd(['git', 'rev-parse', 'HEAD']).strip() + + sb = morphlib.sysbranchdir.open_from_within('.') + definitions = sb.get_git_directory_name(sb.root_repository_url) + defs_repo = morphlib.gitdir.GitDirectory(definitions) + self.loader = morphlib.morphloader.MorphologyLoader() + self.finder = morphlib.morphologyfinder.MorphologyFinder(defs_repo) + + cluster_name = args[0] + cluster, cluster_path = self.load_morphology(cluster_name) + + builds = self.prepare_builds(cluster) + if not os.path.exists('builds'): + os.mkdir('builds') + os.chdir('builds') + for build in builds: + build.start() + + while not all(build.completed() for build in builds): + time.sleep(1) + + fail = False + for build in builds: + if build.process.returncode != 0: + fail = True + sys.stderr.write( + 'Building failed for %s\n' % build.system_name) + if fail: + raise cliapp.AppException('Building of systems failed') + + os.chdir('..') + if not os.path.exists('release'): + os.mkdir('release') + os.chdir('release') + self.deploy_images(cluster, cluster_path) + + def load_morphology(self, name, kind=None): + path = morphlib.util.sanitise_morphology_path(name) + morph = self.loader.load_from_string( + self.finder.read_morphology(path)) + if kind: + assert morph['kind'] == kind + return morph, path + + def prepare_builds(self, cluster): + '''Prepare a list of builds''' + systems = [system['morph'] for system in cluster['systems']] + builds = [] + for system_name in systems: + system, _ = self.load_morphology(system_name) + builds.append(Build(system_name, system['arch'], self)) + return builds + + def deploy_images(self, cluster, cluster_path): + version_label = 'baserock-%s' % self.settings['release-number'] + outputs = {} + + for system in cluster['systems']: + name = system['morph'] + if name not in system['deploy']: + raise cliapp.AppException( + 'In %s: system %s ID should be "%s"' % + (cluster_path, name, name)) + + # The release.morph cluster must specify a basename for the file, + # of name and extension. This script knows about name, but it + # can't find out the appropriate file extension without second + # guessing the behaviour of write extensions. + basename = system['deploy'][name]['location'] + + if '/' in basename or basename.startswith(version_label): + raise cliapp.AppException( + 'In %s: system %s.location should be just the base name, ' + 'e.g. "%s.img"' % (cluster_path, name, name)) + + filename = '%s-%s' % (version_label, basename) + if os.path.exists(filename): + self.output.write('Reusing existing deployment of %s\n' % filename) + else: + self.output.write('Creating %s from release.morph\n' % filename) + self.deploy_single_image(cluster_path, name, filename, version_label) + + def deploy_single_image(self, cluster_path, name, location, version_label): + deploy_command = [ + 'morph', 'deploy', cluster_path, name, + '--trove-host=%s' % self.settings['trove-host'], + '%s.location=%s' % (name, location), + '%s.VERSION_LABEL=%s' % (name, version_label) + ] + + cliapp.runcmd(deploy_command, stdout=sys.stdout) + + +ReleaseApp().run() |