diff options
Diffstat (limited to 'scripts/release-upload')
-rwxr-xr-x | scripts/release-upload | 473 |
1 files changed, 473 insertions, 0 deletions
diff --git a/scripts/release-upload b/scripts/release-upload new file mode 100755 index 00000000..f8b3337c --- /dev/null +++ b/scripts/release-upload @@ -0,0 +1,473 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +'''Upload and publish Baserock binaries for a release. + +This utility is used for the Baserock release process. See +http://wiki.baserock.org/guides/release-process/ for details on the +release process. + +This utility uploads two sets of binaries: + +* The build artifacts (built chunks and strata) used to construct the + systems being released. The systems are found in `release.morph` and + the artifacts from the Trove used to prepare the release. They get + uploaded to a public Trove (by default git.baserock.org). If they're + the same Trove, then nothing happens. + +* The released system images (disk images, tar archives, etc) + specified in `release.morph` get uploaded to a download server (by + default download.baserock.org). + +''' + + +import json +import logging +import os +import pwd +import shutil +import sys +import urllib +import urllib2 +import urlparse + +import cliapp +import yaml + +import morphlib + +class ReleaseUploader(cliapp.Application): + + def add_settings(self): + group = 'Release upload settings' + + local_username = self.get_local_username() + + self.settings.string( + ['build-trove-host'], + 'get build artifacts from Trove at ADDRESS', + metavar='ADDRESS', + group=group) + + self.settings.string( + ['public-trove-host'], + 'publish build artifacts on Trove at ADDRESS', + metavar='ADDRESS', + default='git.baserock.org', + group=group) + + self.settings.string( + ['public-trove-username'], + 'log into public trove as USER', + metavar='USER', + default=local_username, + group=group) + + self.settings.string( + ['public-trove-artifact-dir'], + 'put published artifacts into DIR', + metavar='DIR', + default='/home/cache/artifacts', + group=group) + + self.settings.string( + ['release-artifact-dir'], + 'get release artifacts from DIR (all files from there)', + metavar='DIR', + default='.', + group=group) + + self.settings.string( + ['download-server-address'], + 'publish release artifacts on server at ADDRESS', + metavar='ADDRESS', + default='download.baserock.org', + group=group) + + self.settings.string( + ['download-server-username'], + 'log into download server as USER', + metavar='USER', + default=local_username, + group=group) + + self.settings.string( + ['download-server-private-dir'], + 'use DIR as the temporary location for uploaded release ' + 'artifacts', + metavar='DIR', + default='/srv/download.baserock.org/baserock/.publish-temp', + group=group) + + self.settings.string( + ['download-server-public-dir'], + 'put published release artifacts in DIR', + metavar='DIR', + default='/srv/download.baserock.org/baserock', + group=group) + + self.settings.string( + ['local-build-artifacts-dir'], + 'keep build artifacts to be uploaded temporarily in DIR', + metavar='DIR', + default='build-artifacts', + group=group) + + self.settings.string( + ['morph-cmd'], + 'run FILE to invoke morph', + metavar='FILE', + default='morph', + group=group) + + self.settings.string_list( + ['arch'], + 'Upload files from morphologies of ARCH', + metavar='ARCH', + default=[], + group=group) + + self.settings.boolean( + ['upload-build-artifacts'], + 'upload build artifacts?', + default=True) + + self.settings.boolean( + ['upload-release-artifacts'], + 'upload release artifacts (disk images etc)?', + default=True) + + def get_local_username(self): + uid = os.getuid() + return pwd.getpwuid(uid)[0] + + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Usage: release-upload CLUSTER') + cluster_morphology_path = args[0] + self.status(msg='Uploading and publishing Baserock release') + + if self.settings['upload-build-artifacts']: + self.publish_build_artifacts(cluster_morphology_path) + else: + self.status( + msg='Not uploading build artifacts ' + '(upload-build-artifacts set to false') + + if self.settings['upload-release-artifacts']: + self.publish_release_artifacts() + else: + self.status( + msg='Not uploading release artifacts ' + '(upload-release-artifacts set to false') + + def publish_build_artifacts(self, cluster_morphology_path): + publisher = BuildArtifactPublisher(self.settings, self.status) + publisher.publish_build_artifacts(cluster_morphology_path) + self.status(msg='Build artifacts have been published') + + def publish_release_artifacts(self): + publisher = ReleaseArtifactPublisher(self.settings, self.status) + publisher.publish_release_artifacts() + self.status(msg='Release artifacts have been published') + + def status(self, msg, **kwargs): + formatted = msg.format(**kwargs) + logging.info(formatted) + sys.stdout.write(formatted + '\n') + sys.stdout.flush() + + +class BuildArtifactPublisher(object): + + '''Publish build artifacts related to the release.''' + + def __init__(self, settings, status): + self.settings = settings + self.status = status + + def publish_build_artifacts(self, cluster_path): + artifact_basenames = self.list_build_artifacts_for_release(cluster_path) + self.status( + msg='Found {count} build artifact files in release', + count=len(artifact_basenames)) + + to_be_uploaded = self.filter_away_build_artifacts_on_public_trove( + artifact_basenames) + + logging.debug('List of artifacts (basenames) to upload (without already uploaded):') + for i, basename in enumerate(to_be_uploaded): + logging.debug(' {0}: {1}'.format(i, basename)) + logging.debug('End of artifact list (to_be_uploaded)') + + self.status( + msg='Need to fetch locally, then upload {count} build artifacts', + count=len(to_be_uploaded)) + + self.upload_build_artifacts_to_public_trove(to_be_uploaded) + + def list_build_artifacts_for_release(self, cluster_morphology_path): + self.status(msg='Find build artifacts included in release') + + # FIXME: These are hardcoded for simplicity. They would be + # possible to deduce automatically from the workspace, but + # that can happen later. + repo = 'file://%s' % os.path.abspath('.') + ref = 'HEAD' + + argv = [self.settings['morph-cmd'], 'list-artifacts', '--quiet', + '--repo', repo, '--ref', ref] + argv += self.find_system_morphologies(cluster_morphology_path) + output = cliapp.runcmd(argv) + basenames = output.splitlines() + logging.debug('List of build artifacts in release:') + for basename in basenames: + logging.debug(' {0}'.format(basename)) + logging.debug('End of list of build artifacts in release') + + return basenames + + def find_system_morphologies(self, cluster_morphology_path): + cluster = self.load_cluster_morphology(cluster_morphology_path) + system_dicts = self.find_systems_in_parsed_cluster_morphology(cluster) + if self.settings['arch']: + system_dicts = self.choose_systems_for_wanted_architectures( + system_dicts, self.settings['arch']) + return [sd['morph'] for sd in system_dicts] + + def load_cluster_morphology(self, pathname): + with open(pathname) as f: + return yaml.load(f) + + def find_systems_in_parsed_cluster_morphology(self, cluster): + return cluster['systems'] + + def choose_systems_for_wanted_architectures(self, system_dicts, archs): + return [ + sd + for sd in system_dicts + if self.system_is_for_wanted_arch(sd, archs)] + + def system_is_for_wanted_arch(self, system_dict, archs): + morph = self.load_system_morphology(system_dict) + return morph['arch'] in archs + + def load_system_morphology(self, system_dict): + pathname = morphlib.util.sanitise_morphology_path(system_dict['morph']) + return self.load_morphology_from_named_file(pathname) + + def load_morphology_from_named_file(self, pathname): + finder = self.get_morphology_finder_for_root_repository() + morphology_text = finder.read_file(pathname) + loader = morphlib.morphloader.MorphologyLoader() + return loader.load_from_string(morphology_text) + + def get_morphology_finder_for_root_repository(self): + definitions_repo = morphlib.definitions_repo.open( + '.', search_for_root=True) + return morphlib.morphologyfinder.MorphologyFinder(definitions_repo) + + def filter_away_build_artifacts_on_public_trove(self, basenames): + result = [] + logging.debug('Filtering away already existing artifacts:') + for basename, exists in self.query_public_trove_for_artifacts(basenames): + logging.debug(' {0}: {1}'.format(basename, exists)) + if not exists: + result.append(basename) + logging.debug('End of filtering away') + return result + + def query_public_trove_for_artifacts(self, basenames): + host = self.settings['public-trove-host'] + + # FIXME: This could use + # contextlib.closing(urllib2.urlopen(url, data=data) instead + # of explicit closing. + url = 'http://{host}:8080/1.0/artifacts'.format(host=host) + data = json.dumps(basenames) + f = urllib2.urlopen(url, data=data) + obj = json.load(f) + return obj.items() + + def upload_build_artifacts_to_public_trove(self, basenames): + self.download_artifacts_locally(basenames) + self.upload_artifacts_to_public_trove(basenames) + + def download_artifacts_locally(self, basenames): + dirname = self.settings['local-build-artifacts-dir'] + self.create_directory_if_missing(dirname) + for i, basename in enumerate(basenames): + url = self.construct_artifact_url(basename) + pathname = os.path.join(dirname, basename) + if not os.path.exists(pathname): + self.status( + msg='Downloading {i}/{total} {basename}', + basename=repr(basename), i=i, total=len(basenames)) + self.download_from_url(url, dirname, pathname) + + def create_directory_if_missing(self, dirname): + if not os.path.exists(dirname): + os.makedirs(dirname) + + def construct_artifact_url(self, basename): + scheme = 'http' + netloc = '{host}:8080'.format(host=self.settings['build-trove-host']) + path = '/1.0/artifacts' + query = 'filename={0}'.format(urllib.quote_plus(basename)) + fragment = '' + components = (scheme, netloc, path, query, fragment) + return urlparse.urlunsplit(components) + + def download_from_url(self, url, dirname, pathname): + logging.info( + 'Downloading {url} to {pathname}'.format( + url=url, pathname=pathname)) + with open(pathname, 'wb') as output: + try: + incoming = urllib2.urlopen(url) + shutil.copyfileobj(incoming, output) + incoming.close() + except urllib2.HTTPError as e: + if pathname.endswith('.meta'): + return + self.status( + msg="ERROR: Can't download {url}: {explanation}", + url=url, + explanation=str(e)) + os.remove(pathname) + raise + + def upload_artifacts_to_public_trove(self, basenames): + self.status( + msg='Upload build artifacts to {trove}', + trove=self.settings['public-trove-host']) + rsync_files_to_server( + self.settings['local-build-artifacts-dir'], + basenames, + self.settings['public-trove-username'], + self.settings['public-trove-host'], + self.settings['public-trove-artifact-dir']) + set_permissions_on_server( + self.settings['public-trove-username'], + self.settings['public-trove-host'], + self.settings['public-trove-artifact-dir'], + basenames) + +class ReleaseArtifactPublisher(object): + + '''Publish release artifacts for a release.''' + + def __init__(self, settings, status): + self.settings = settings + self.status = status + + def publish_release_artifacts(self): + files = self.list_release_artifacts() + if files: + self.upload_release_artifacts_to_private_dir(files) + self.move_release_artifacts_to_public_dir(files) + self.create_symlinks_to_new_release_artifacts(files) + + def list_release_artifacts(self): + self.status(msg='Find release artifacts to publish') + return os.listdir(self.settings['release-artifact-dir']) + + def upload_release_artifacts_to_private_dir(self, files): + self.status(msg='Upload release artifacts to private directory') + path = self.settings['download-server-private-dir'] + self.create_directory_on_download_server(path) + self.rsync_files_to_download_server(files, path) + + def create_directory_on_download_server(self, path): + user = self.settings['download-server-username'] + host = self.settings['download-server-address'] + self.status(msg='Create {host}:{path}', host=host, path=path) + target = '{user}@{host}'.format(user=user, host=host) + cliapp.ssh_runcmd(target, ['mkdir', '-p', path]) + + def rsync_files_to_download_server(self, files, path): + self.status(msg='Upload release artifacts to download server') + rsync_files_to_server( + self.settings['release-artifact-dir'], + files, + self.settings['download-server-username'], + self.settings['download-server-address'], + path) + set_permissions_on_server( + self.settings['download-server-username'], + self.settings['download-server-address'], + path, + files) + + def move_release_artifacts_to_public_dir(self, files): + self.status(msg='Move release artifacts to public directory') + private_dir = self.settings['download-server-private-dir'] + public_dir = self.settings['download-server-public-dir'] + self.create_directory_on_download_server(public_dir) + + # Move just the contents of the private dir, not the dir + # itself (-mindepth). Avoid overwriting existing files (mv + # -n). + argv = ['find', private_dir, '-mindepth', '1', + '-exec', 'mv', '-n', '{}', public_dir + '/.', ';'] + + target = '{user}@{host}'.format( + user=self.settings['download-server-username'], + host=self.settings['download-server-address']) + cliapp.ssh_runcmd(target, argv) + + def create_symlinks_to_new_release_artifacts(self, files): + self.status(msg='FIXME: Create symlinks to new releas artifacts') + + +def rsync_files_to_server( + source_dir, source_filenames, user, host, target_dir): + + if not source_filenames: + return + + argv = [ + 'rsync', + '-a', + '--progress', + '--partial', + '--human-readable', + '--sparse', + '--protect-args', + '-0', + '--files-from=-', + source_dir, + '{user}@{host}:{path}'.format(user=user, host=host, path=target_dir), + ] + + files_list = '\0'.join(filename for filename in source_filenames) + cliapp.runcmd(argv, feed_stdin=files_list, stdout=None, stderr=None) + + +def set_permissions_on_server(user, host, target_dir, filenames): + # If we have no files, we can't form a valid command to run on the server + if not filenames: + return + target = '{user}@{host}'.format(user=user, host=host) + argv = ['xargs', '-0', 'chmod', '0644'] + files_list = ''.join( + '{0}\0'.format(os.path.join(target_dir, filename)) for filename in filenames) + cliapp.ssh_runcmd(target, argv, feed_stdin=files_list, stdout=None, stderr=None) + + +ReleaseUploader(description=__doc__).run() |