summaryrefslogtreecommitdiff
path: root/scripts/release-build
diff options
context:
space:
mode:
authorLars Wirzenius <lars.wirzenius@codethink.co.uk>2014-07-22 18:11:49 +0000
committerLars Wirzenius <lars.wirzenius@codethink.co.uk>2014-07-24 16:31:23 +0000
commit313234e1e6f59ed8dfa4f0a11d0b91aab2d33c34 (patch)
treed9145d6ffa2e3d450af02d3e4dbbdc074a198f1c /scripts/release-build
parentef11b3db09526bc568261ef72a8ee0e346b753d3 (diff)
downloaddefinitions-313234e1e6f59ed8dfa4f0a11d0b91aab2d33c34.tar.gz
Add new scripts for building, uploading release
These scripts are a rewrite of scripts/do-release.py and scripts/distbuild-cluster. The biggest difference is that they split the tasks of building the things that are to be released, and uploading them to git.baserock.org / download.baserock.org, where do-release.py combines both (and distbuild-cluster only builds chunk/stratum/system artifacts, not the release images). The new scripts are also configurable using command line options or a configuration file rather than requiring editing of the source. These changes will allow, for example, a CI job that builds a release, but doesn't upload it to download.baserock.org. The new scripts are coupled with a change to the release process, which will be documented as a change to the release process page on wiki.baserock.org. The 14.29 release of Baserock was done with slightly different versions of these scripts to make it feasible to upload things over multiple network connections.
Diffstat (limited to 'scripts/release-build')
-rwxr-xr-xscripts/release-build161
1 files changed, 161 insertions, 0 deletions
diff --git a/scripts/release-build b/scripts/release-build
new file mode 100755
index 00000000..36a38deb
--- /dev/null
+++ b/scripts/release-build
@@ -0,0 +1,161 @@
+#!/usr/bin/env python
+# Copyright (C) 2014 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import cliapp
+import morphlib
+import os
+import subprocess
+import sys
+import time
+
+
+class Build(object):
+ '''A single distbuild instance.'''
+
+ def __init__(self, name, arch, app):
+ self.system_name = name
+ self.controller = app.controllers[arch]
+ self.command = [
+ 'morph', 'distbuild-morphology',
+ '--controller-initiator-address=%s' % self.controller,
+ 'baserock:baserock/definitions', app.ref, self.system_name]
+
+ def start(self):
+ self.process = subprocess.Popen(self.command)
+
+ def completed(self):
+ return (self.process.poll() is not None)
+
+
+class ReleaseApp(cliapp.Application):
+
+ '''Cliapp app that handles distbuilding and deploying a cluster.'''
+
+ def add_settings(self):
+ self.settings.string_list(['controllers'],
+ 'a list of distbuild controllers and their '
+ 'architecture')
+
+ self.settings.string(['trove-host'],
+ 'hostname of Trove instance')
+
+ self.settings.string(['release-number'],
+ 'Baserock version of the systems being built',
+ default='yy.ww')
+
+ def process_args(self, args):
+ '''Process the command line'''
+ self.controllers = {}
+ controllers_list = self.settings['controllers']
+ for item in controllers_list:
+ arch, controller = item.split(':')
+ self.controllers[arch] = controller
+
+ self.ref = cliapp.runcmd(['git', 'rev-parse', 'HEAD']).strip()
+
+ sb = morphlib.sysbranchdir.open_from_within('.')
+ definitions = sb.get_git_directory_name(sb.root_repository_url)
+ defs_repo = morphlib.gitdir.GitDirectory(definitions)
+ self.loader = morphlib.morphloader.MorphologyLoader()
+ self.finder = morphlib.morphologyfinder.MorphologyFinder(defs_repo)
+
+ cluster_name = args[0]
+ cluster, cluster_path = self.load_morphology(cluster_name)
+
+ builds = self.prepare_builds(cluster)
+ if not os.path.exists('builds'):
+ os.mkdir('builds')
+ os.chdir('builds')
+ for build in builds:
+ build.start()
+
+ while not all(build.completed() for build in builds):
+ time.sleep(1)
+
+ fail = False
+ for build in builds:
+ if build.process.returncode != 0:
+ fail = True
+ sys.stderr.write(
+ 'Building failed for %s\n' % build.system_name)
+ if fail:
+ raise cliapp.AppException('Building of systems failed')
+
+ os.chdir('..')
+ if not os.path.exists('release'):
+ os.mkdir('release')
+ os.chdir('release')
+ self.deploy_images(cluster, cluster_path)
+
+ def load_morphology(self, name, kind=None):
+ path = morphlib.util.sanitise_morphology_path(name)
+ morph = self.loader.load_from_string(
+ self.finder.read_morphology(path))
+ if kind:
+ assert morph['kind'] == kind
+ return morph, path
+
+ def prepare_builds(self, cluster):
+ '''Prepare a list of builds'''
+ systems = [system['morph'] for system in cluster['systems']]
+ builds = []
+ for system_name in systems:
+ system, _ = self.load_morphology(system_name)
+ builds.append(Build(system_name, system['arch'], self))
+ return builds
+
+ def deploy_images(self, cluster, cluster_path):
+ version_label = 'baserock-%s' % self.settings['release-number']
+ outputs = {}
+
+ for system in cluster['systems']:
+ name = system['morph']
+ if name not in system['deploy']:
+ raise cliapp.AppException(
+ 'In %s: system %s ID should be "%s"' %
+ (cluster_path, name, name))
+
+ # The release.morph cluster must specify a basename for the file,
+ # of name and extension. This script knows about name, but it
+ # can't find out the appropriate file extension without second
+ # guessing the behaviour of write extensions.
+ basename = system['deploy'][name]['location']
+
+ if '/' in basename or basename.startswith(version_label):
+ raise cliapp.AppException(
+ 'In %s: system %s.location should be just the base name, '
+ 'e.g. "%s.img"' % (cluster_path, name, name))
+
+ filename = '%s-%s' % (version_label, basename)
+ if os.path.exists(filename):
+ self.output.write('Reusing existing deployment of %s\n' % filename)
+ else:
+ self.output.write('Creating %s from release.morph\n' % filename)
+ self.deploy_single_image(cluster_path, name, filename, version_label)
+
+ def deploy_single_image(self, cluster_path, name, location, version_label):
+ deploy_command = [
+ 'morph', 'deploy', cluster_path, name,
+ '--trove-host=%s' % self.settings['trove-host'],
+ '%s.location=%s' % (name, location),
+ '%s.VERSION_LABEL=%s' % (name, version_label)
+ ]
+
+ cliapp.runcmd(deploy_command, stdout=sys.stdout)
+
+
+ReleaseApp().run()