summaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
authorLars Wirzenius <lars.wirzenius@codethink.co.uk>2014-07-24 16:36:45 (GMT)
committerLars Wirzenius <lars.wirzenius@codethink.co.uk>2014-07-24 16:36:45 (GMT)
commit983f3e0ce7b5e757769229837512687bca446834 (patch)
tree13bae33c325d10632e9f1738a0ebb78e886a2daf /scripts
parentb688c4672680a3cdeee0d1dcf9492dd4c02f1524 (diff)
parent312e84f5061782ca4851c3667fc2ac941140835d (diff)
downloaddefinitions-983f3e0ce7b5e757769229837512687bca446834.tar.gz
Merge branch 'baserock/liw/release-upload-script'
Reviewed-by: Richard Maw Reviewed-by: Sam Thursfield I fixed commit meessages and made some small code changes while merging, as discussed on the mailing list.
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/release-build161
-rw-r--r--scripts/release-build.test.conf6
-rwxr-xr-xscripts/release-upload387
-rw-r--r--scripts/release-upload.test.conf10
4 files changed, 564 insertions, 0 deletions
diff --git a/scripts/release-build b/scripts/release-build
new file mode 100755
index 0000000..36a38de
--- /dev/null
+++ b/scripts/release-build
@@ -0,0 +1,161 @@
+#!/usr/bin/env python
+# Copyright (C) 2014 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import cliapp
+import morphlib
+import os
+import subprocess
+import sys
+import time
+
+
+class Build(object):
+ '''A single distbuild instance.'''
+
+ def __init__(self, name, arch, app):
+ self.system_name = name
+ self.controller = app.controllers[arch]
+ self.command = [
+ 'morph', 'distbuild-morphology',
+ '--controller-initiator-address=%s' % self.controller,
+ 'baserock:baserock/definitions', app.ref, self.system_name]
+
+ def start(self):
+ self.process = subprocess.Popen(self.command)
+
+ def completed(self):
+ return (self.process.poll() is not None)
+
+
+class ReleaseApp(cliapp.Application):
+
+ '''Cliapp app that handles distbuilding and deploying a cluster.'''
+
+ def add_settings(self):
+ self.settings.string_list(['controllers'],
+ 'a list of distbuild controllers and their '
+ 'architecture')
+
+ self.settings.string(['trove-host'],
+ 'hostname of Trove instance')
+
+ self.settings.string(['release-number'],
+ 'Baserock version of the systems being built',
+ default='yy.ww')
+
+ def process_args(self, args):
+ '''Process the command line'''
+ self.controllers = {}
+ controllers_list = self.settings['controllers']
+ for item in controllers_list:
+ arch, controller = item.split(':')
+ self.controllers[arch] = controller
+
+ self.ref = cliapp.runcmd(['git', 'rev-parse', 'HEAD']).strip()
+
+ sb = morphlib.sysbranchdir.open_from_within('.')
+ definitions = sb.get_git_directory_name(sb.root_repository_url)
+ defs_repo = morphlib.gitdir.GitDirectory(definitions)
+ self.loader = morphlib.morphloader.MorphologyLoader()
+ self.finder = morphlib.morphologyfinder.MorphologyFinder(defs_repo)
+
+ cluster_name = args[0]
+ cluster, cluster_path = self.load_morphology(cluster_name)
+
+ builds = self.prepare_builds(cluster)
+ if not os.path.exists('builds'):
+ os.mkdir('builds')
+ os.chdir('builds')
+ for build in builds:
+ build.start()
+
+ while not all(build.completed() for build in builds):
+ time.sleep(1)
+
+ fail = False
+ for build in builds:
+ if build.process.returncode != 0:
+ fail = True
+ sys.stderr.write(
+ 'Building failed for %s\n' % build.system_name)
+ if fail:
+ raise cliapp.AppException('Building of systems failed')
+
+ os.chdir('..')
+ if not os.path.exists('release'):
+ os.mkdir('release')
+ os.chdir('release')
+ self.deploy_images(cluster, cluster_path)
+
+ def load_morphology(self, name, kind=None):
+ path = morphlib.util.sanitise_morphology_path(name)
+ morph = self.loader.load_from_string(
+ self.finder.read_morphology(path))
+ if kind:
+ assert morph['kind'] == kind
+ return morph, path
+
+ def prepare_builds(self, cluster):
+ '''Prepare a list of builds'''
+ systems = [system['morph'] for system in cluster['systems']]
+ builds = []
+ for system_name in systems:
+ system, _ = self.load_morphology(system_name)
+ builds.append(Build(system_name, system['arch'], self))
+ return builds
+
+ def deploy_images(self, cluster, cluster_path):
+ version_label = 'baserock-%s' % self.settings['release-number']
+ outputs = {}
+
+ for system in cluster['systems']:
+ name = system['morph']
+ if name not in system['deploy']:
+ raise cliapp.AppException(
+ 'In %s: system %s ID should be "%s"' %
+ (cluster_path, name, name))
+
+ # The release.morph cluster must specify a basename for the file,
+ # of name and extension. This script knows about name, but it
+ # can't find out the appropriate file extension without second
+ # guessing the behaviour of write extensions.
+ basename = system['deploy'][name]['location']
+
+ if '/' in basename or basename.startswith(version_label):
+ raise cliapp.AppException(
+ 'In %s: system %s.location should be just the base name, '
+ 'e.g. "%s.img"' % (cluster_path, name, name))
+
+ filename = '%s-%s' % (version_label, basename)
+ if os.path.exists(filename):
+ self.output.write('Reusing existing deployment of %s\n' % filename)
+ else:
+ self.output.write('Creating %s from release.morph\n' % filename)
+ self.deploy_single_image(cluster_path, name, filename, version_label)
+
+ def deploy_single_image(self, cluster_path, name, location, version_label):
+ deploy_command = [
+ 'morph', 'deploy', cluster_path, name,
+ '--trove-host=%s' % self.settings['trove-host'],
+ '%s.location=%s' % (name, location),
+ '%s.VERSION_LABEL=%s' % (name, version_label)
+ ]
+
+ cliapp.runcmd(deploy_command, stdout=sys.stdout)
+
+
+ReleaseApp().run()
diff --git a/scripts/release-build.test.conf b/scripts/release-build.test.conf
new file mode 100644
index 0000000..5008335
--- /dev/null
+++ b/scripts/release-build.test.conf
@@ -0,0 +1,6 @@
+[config]
+trove-host = ct-mcr-1.ducie.codethink.co.uk
+controllers = x86_64:ct-mcr-1-distbuild-x86-64-majikthise-controller.dyn.ducie.codethink.co.uk,
+ x86_32:ct-mcr-1-distbuild-x86-32-majikthise-controller.dyn.ducie.codethink.co.uk,
+ armv7lhf:ct-mcr-1-distbuild-armv7lhf-jetson.dyn.ducie.codethink.co.uk
+release-number = 14.29
diff --git a/scripts/release-upload b/scripts/release-upload
new file mode 100755
index 0000000..2e7f54e
--- /dev/null
+++ b/scripts/release-upload
@@ -0,0 +1,387 @@
+#!/usr/bin/python
+# Copyright (C) 2014 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+'''Upload and publish Baserock binaries for a release.
+
+This utility is used for the Baserock release process. See
+http://wiki.baserock.org/guides/release-process/ for details on the
+release process.
+
+This utility uploads two sets of binaries:
+
+* The build artifacts (built chunks and strata) used to construct the
+ systems being released. The systems are found in `release.morph` and
+ the artifacts from the Trove used to prepare the release. They get
+ uploaded to a public Trove (by default git.baserock.org). If they're
+ the same Trove, then nothing happens.
+
+* The released system images (disk images, tar archives, etc)
+ specified in `release.morph` get uploaded to a download server (by
+ default download.baserock.org).
+
+'''
+
+
+import json
+import logging
+import os
+import pwd
+import shutil
+import sys
+import urllib
+import urllib2
+import urlparse
+
+import cliapp
+import yaml
+
+
+class ReleaseUploader(cliapp.Application):
+
+ def add_settings(self):
+ group = 'Release upload settings'
+
+ local_username = self.get_local_username()
+
+ self.settings.string(
+ ['build-trove-host'],
+ 'get build artifacts from Trove at ADDRESS',
+ metavar='ADDRESS',
+ group=group)
+
+ self.settings.string(
+ ['public-trove-host'],
+ 'publish build artifacts on Trove at ADDRESS',
+ metavar='ADDRESS',
+ default='git.baserock.org',
+ group=group)
+
+ self.settings.string(
+ ['public-trove-username'],
+ 'log into public trove as USER',
+ metavar='USER',
+ default=local_username,
+ group=group)
+
+ self.settings.string(
+ ['public-trove-artifact-dir'],
+ 'put published artifacts into DIR',
+ metavar='DIR',
+ default='/home/cache/artifacts',
+ group=group)
+
+ self.settings.string(
+ ['release-artifact-dir'],
+ 'get release artifacts from DIR (all files from there)',
+ metavar='DIR',
+ default='.',
+ group=group)
+
+ self.settings.string(
+ ['download-server-address'],
+ 'publish release artifacts on server at ADDRESS',
+ metavar='ADDRESS',
+ default='download.baserock.org',
+ group=group)
+
+ self.settings.string(
+ ['download-server-username'],
+ 'log into download server as USER',
+ metavar='USER',
+ default=local_username,
+ group=group)
+
+ self.settings.string(
+ ['download-server-private-dir'],
+ 'use DIR as the temporary location for uploaded release '
+ 'artifacts',
+ metavar='DIR',
+ default='/srv/download.baserock.org/baserock/.publish-temp',
+ group=group)
+
+ self.settings.string(
+ ['download-server-public-dir'],
+ 'put published release artifacts in DIR',
+ metavar='DIR',
+ default='/srv/download.baserock.org/baserock',
+ group=group)
+
+ self.settings.string(
+ ['local-build-artifacts-dir'],
+ 'keep build artifacts to be uploaded temporarily in DIR',
+ metavar='DIR',
+ default='build-artifacts',
+ group=group)
+
+ self.settings.string(
+ ['morph-cmd'],
+ 'run FILE to invoke morph',
+ metavar='FILE',
+ default='morph',
+ group=group)
+
+ def get_local_username(self):
+ uid = os.getuid()
+ return pwd.getpwuid(uid)[0]
+
+ def process_args(self, args):
+ self.status(msg='Uploading and publishing Baserock release')
+ BuildArtifactPublisher(self.settings, self.status).publish_build_artifacts()
+ ReleaseArtifactPublisher(self.settings, self.status).publish_release_artifacts()
+ self.status(msg='Release has been uploaded and published')
+
+ def status(self, msg, **kwargs):
+ formatted = msg.format(**kwargs)
+ logging.info(formatted)
+ sys.stdout.write(formatted + '\n')
+ sys.stdout.flush()
+
+
+class BuildArtifactPublisher(object):
+
+ '''Publish build artifacts related to the release.'''
+
+ def __init__(self, settings, status):
+ self.settings = settings
+ self.status = status
+
+ def publish_build_artifacts(self):
+ artifact_basenames = self.list_build_artifacts_for_release()
+ self.status(
+ msg='Found {count} build artifact files in release',
+ count=len(artifact_basenames))
+
+ to_be_uploaded = self.filter_away_build_artifacts_on_public_trove(
+ artifact_basenames)
+
+ logging.debug('List of artifacts (basenames) to upload (without already uploaded):')
+ for i, basename in enumerate(to_be_uploaded):
+ logging.debug(' {0}: {1}'.format(i, basename))
+ logging.debug('End of artifact list (to_be_uploaded)')
+
+ self.status(
+ msg='Need to fetch locally, then upload {count} build artifacts',
+ count=len(to_be_uploaded))
+
+ self.upload_build_artifacts_to_public_trove(to_be_uploaded)
+
+ def list_build_artifacts_for_release(self):
+ self.status(msg='Find build artifacts included in release')
+
+ # FIXME: These are hardcoded for simplicity. They would be
+ # possible to deduce automatically from the workspace, but
+ # that can happen later.
+ repo = 'baserock:baserock/definitions'
+ ref = 'HEAD'
+
+ argv = [self.settings['morph-cmd'], 'list-artifacts', '--quiet', repo, ref]
+ argv += self.find_system_morphologies()
+ output = cliapp.runcmd(argv)
+ basenames = output.splitlines()
+
+ return basenames
+
+ def find_system_morphologies(self):
+ cluster_morphology_pathname = 'release.morph'
+ with open(cluster_morphology_pathname) as f:
+ obj = yaml.load(f)
+ return [system_dict['morph'] for system_dict in obj['systems']]
+
+ def filter_away_build_artifacts_on_public_trove(self, basenames):
+ result = []
+ logging.debug('Filtering away already existing artifacts:')
+ for basename, exists in self.query_public_trove_for_artifacts(basenames):
+ logging.debug(' {0}: {1}'.format(basename, exists))
+ if not exists:
+ result.append(basename)
+ logging.debug('End of filtering away')
+ return result
+
+ def query_public_trove_for_artifacts(self, basenames):
+ host = self.settings['public-trove-host']
+
+ # FIXME: This could use
+ # contextlib.closing(urllib2.urlopen(url, data=data) instead
+ # of explicit closing.
+ url = 'http://{host}:8080/1.0/artifacts'.format(host=host)
+ data = json.dumps(basenames)
+ f = urllib2.urlopen(url, data=data)
+ obj = json.load(f)
+ return obj.items()
+
+ def upload_build_artifacts_to_public_trove(self, basenames):
+ self.download_artifacts_locally(basenames)
+ self.upload_artifacts_to_public_trove(basenames)
+
+ def download_artifacts_locally(self, basenames):
+ dirname = self.settings['local-build-artifacts-dir']
+ self.create_directory_if_missing(dirname)
+ for i, basename in enumerate(basenames):
+ url = self.construct_artifact_url(basename)
+ pathname = os.path.join(dirname, basename)
+ if not os.path.exists(pathname):
+ self.status(
+ msg='Downloading {i}/{total} {basename}',
+ basename=repr(basename), i=i, total=len(basenames))
+ self.download_from_url(url, dirname, pathname)
+
+ def create_directory_if_missing(self, dirname):
+ if not os.path.exists(dirname):
+ os.makedirs(dirname)
+
+ def construct_artifact_url(self, basename):
+ scheme = 'http'
+ netloc = '{host}:8080'.format(host=self.settings['build-trove-host'])
+ path = '/1.0/artifacts'
+ query = 'filename={0}'.format(urllib.quote_plus(basename))
+ fragment = ''
+ components = (scheme, netloc, path, query, fragment)
+ return urlparse.urlunsplit(components)
+
+ def download_from_url(self, url, dirname, pathname):
+ logging.info(
+ 'Downloading {url} to {pathname}'.format(
+ url=url, pathname=pathname))
+ with open(pathname, 'wb') as output:
+ try:
+ incoming = urllib2.urlopen(url)
+ shutil.copyfileobj(incoming, output)
+ incoming.close()
+ except urllib2.HTTPError as e:
+ if pathname.endswith('.meta'):
+ return
+ self.status(
+ msg="ERROR: Can't download {url}: {explanation}",
+ url=url,
+ explanation=str(e))
+ os.remove(pathname)
+ raise
+
+ def upload_artifacts_to_public_trove(self, basenames):
+ self.status(
+ msg='Upload build artifacts to {trove}',
+ trove=self.settings['public-trove-host'])
+ rsync_files_to_server(
+ self.settings['local-build-artifacts-dir'],
+ basenames,
+ self.settings['public-trove-username'],
+ self.settings['public-trove-host'],
+ self.settings['public-trove-artifact-dir'])
+ set_permissions_on_server(
+ self.settings['public-trove-username'],
+ self.settings['public-trove-host'],
+ self.settings['public-trove-artifact-dir'],
+ basenames)
+
+class ReleaseArtifactPublisher(object):
+
+ '''Publish release artifacts for a release.'''
+
+ def __init__(self, settings, status):
+ self.settings = settings
+ self.status = status
+
+ def publish_release_artifacts(self):
+ files = self.list_release_artifacts()
+ if files:
+ self.upload_release_artifacts_to_private_dir(files)
+ self.move_release_artifacts_to_public_dir(files)
+ self.create_symlinks_to_new_release_artifacts(files)
+
+ def list_release_artifacts(self):
+ self.status(msg='Find release artifacts to publish')
+ return os.listdir(self.settings['release-artifact-dir'])
+
+ def upload_release_artifacts_to_private_dir(self, files):
+ self.status(msg='Upload release artifacts to private directory')
+ path = self.settings['download-server-private-dir']
+ self.create_directory_on_download_server(path)
+ self.rsync_files_to_download_server(files, path)
+
+ def create_directory_on_download_server(self, path):
+ user = self.settings['download-server-username']
+ host = self.settings['download-server-address']
+ self.status(msg='Create {host}:{path}', host=host, path=path)
+ target = '{user}@{host}'.format(user=user, host=host)
+ cliapp.ssh_runcmd(target, ['mkdir', '-p', path])
+
+ def rsync_files_to_download_server(self, files, path):
+ self.status(msg='Upload release artifacts to download server')
+ rsync_files_to_server(
+ self.settings['release-artifact-dir'],
+ files,
+ self.settings['download-server-username'],
+ self.settings['download-server-address'],
+ path)
+ set_permissions_on_server(
+ self.settings['download-server-username'],
+ self.settings['download-server-address'],
+ path,
+ files)
+
+ def move_release_artifacts_to_public_dir(self, files):
+ self.status(msg='Move release artifacts to public directory')
+ private_dir = self.settings['download-server-private-dir']
+ public_dir = self.settings['download-server-public-dir']
+ self.create_directory_on_download_server(public_dir)
+
+ # Move just the contents of the private dir, not the dir
+ # itself (-mindepth). Avoid overwriting existing files (mv
+ # -n).
+ argv = ['find', private_dir, '-mindepth', '1',
+ '-exec', 'mv', '-n', '{}', public_dir + '/.', ';']
+
+ target = '{user}@{host}'.format(
+ user=self.settings['download-server-username'],
+ host=self.settings['download-server-address'])
+ cliapp.ssh_runcmd(target, argv)
+
+ def create_symlinks_to_new_release_artifacts(self, files):
+ self.status(msg='FIXME: Create symlinks to new releas artifacts')
+
+
+def rsync_files_to_server(
+ source_dir, source_filenames, user, host, target_dir):
+
+ argv = [
+ 'rsync',
+ '-a',
+ '--progress',
+ '--partial',
+ '--human-readable',
+ '--sparse',
+ '--protect-args',
+ '-0',
+ '--files-from=-',
+ source_dir,
+ '{user}@{host}:{path}'.format(user=user, host=host, path=target_dir),
+ ]
+
+ files_list = ''.join(
+ '{0}\0'.format(filename) for filename in source_filenames)
+ cliapp.runcmd(argv, feed_stdin=files_list, stdout=None, stderr=None)
+
+
+def set_permissions_on_server(user, host, target_dir, filenames):
+ target = '{user}@{host}'.format(user=user, host=host)
+ argv = ['chmod', '0644']
+ for filename in filenames:
+ argv.append(os.path.join(target_dir, filename))
+ cliapp.ssh_runcmd(target, argv)
+
+
+ReleaseUploader(description=__doc__).run()
diff --git a/scripts/release-upload.test.conf b/scripts/release-upload.test.conf
new file mode 100644
index 0000000..1322798
--- /dev/null
+++ b/scripts/release-upload.test.conf
@@ -0,0 +1,10 @@
+[config]
+download-server-address = localhost
+download-server-private-dir = /tmp/private
+download-server-public-dir = /tmp/public
+build-trove-host = ct-mcr-1.ducie.codethink.co.uk
+public-trove-host = localhost
+public-trove-username = root
+public-trove-artifact-dir = /tmp/artifacts
+release-artifact-dir = t.release-files
+morph-cmd = /home/root/git-morph