diff options
Diffstat (limited to 'scripts/release-upload')
| -rwxr-xr-x | scripts/release-upload | 473 |
1 files changed, 0 insertions, 473 deletions
diff --git a/scripts/release-upload b/scripts/release-upload deleted file mode 100755 index f8b3337c..00000000 --- a/scripts/release-upload +++ /dev/null @@ -1,473 +0,0 @@ -#!/usr/bin/python -# Copyright (C) 2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -'''Upload and publish Baserock binaries for a release. - -This utility is used for the Baserock release process. See -http://wiki.baserock.org/guides/release-process/ for details on the -release process. - -This utility uploads two sets of binaries: - -* The build artifacts (built chunks and strata) used to construct the - systems being released. The systems are found in `release.morph` and - the artifacts from the Trove used to prepare the release. They get - uploaded to a public Trove (by default git.baserock.org). If they're - the same Trove, then nothing happens. - -* The released system images (disk images, tar archives, etc) - specified in `release.morph` get uploaded to a download server (by - default download.baserock.org). - -''' - - -import json -import logging -import os -import pwd -import shutil -import sys -import urllib -import urllib2 -import urlparse - -import cliapp -import yaml - -import morphlib - -class ReleaseUploader(cliapp.Application): - - def add_settings(self): - group = 'Release upload settings' - - local_username = self.get_local_username() - - self.settings.string( - ['build-trove-host'], - 'get build artifacts from Trove at ADDRESS', - metavar='ADDRESS', - group=group) - - self.settings.string( - ['public-trove-host'], - 'publish build artifacts on Trove at ADDRESS', - metavar='ADDRESS', - default='git.baserock.org', - group=group) - - self.settings.string( - ['public-trove-username'], - 'log into public trove as USER', - metavar='USER', - default=local_username, - group=group) - - self.settings.string( - ['public-trove-artifact-dir'], - 'put published artifacts into DIR', - metavar='DIR', - default='/home/cache/artifacts', - group=group) - - self.settings.string( - ['release-artifact-dir'], - 'get release artifacts from DIR (all files from there)', - metavar='DIR', - default='.', - group=group) - - self.settings.string( - ['download-server-address'], - 'publish release artifacts on server at ADDRESS', - metavar='ADDRESS', - default='download.baserock.org', - group=group) - - self.settings.string( - ['download-server-username'], - 'log into download server as USER', - metavar='USER', - default=local_username, - group=group) - - self.settings.string( - ['download-server-private-dir'], - 'use DIR as the temporary location for uploaded release ' - 'artifacts', - metavar='DIR', - default='/srv/download.baserock.org/baserock/.publish-temp', - group=group) - - self.settings.string( - ['download-server-public-dir'], - 'put published release artifacts in DIR', - metavar='DIR', - default='/srv/download.baserock.org/baserock', - group=group) - - self.settings.string( - ['local-build-artifacts-dir'], - 'keep build artifacts to be uploaded temporarily in DIR', - metavar='DIR', - default='build-artifacts', - group=group) - - self.settings.string( - ['morph-cmd'], - 'run FILE to invoke morph', - metavar='FILE', - default='morph', - group=group) - - self.settings.string_list( - ['arch'], - 'Upload files from morphologies of ARCH', - metavar='ARCH', - default=[], - group=group) - - self.settings.boolean( - ['upload-build-artifacts'], - 'upload build artifacts?', - default=True) - - self.settings.boolean( - ['upload-release-artifacts'], - 'upload release artifacts (disk images etc)?', - default=True) - - def get_local_username(self): - uid = os.getuid() - return pwd.getpwuid(uid)[0] - - def process_args(self, args): - if len(args) != 1: - raise cliapp.AppException('Usage: release-upload CLUSTER') - cluster_morphology_path = args[0] - self.status(msg='Uploading and publishing Baserock release') - - if self.settings['upload-build-artifacts']: - self.publish_build_artifacts(cluster_morphology_path) - else: - self.status( - msg='Not uploading build artifacts ' - '(upload-build-artifacts set to false') - - if self.settings['upload-release-artifacts']: - self.publish_release_artifacts() - else: - self.status( - msg='Not uploading release artifacts ' - '(upload-release-artifacts set to false') - - def publish_build_artifacts(self, cluster_morphology_path): - publisher = BuildArtifactPublisher(self.settings, self.status) - publisher.publish_build_artifacts(cluster_morphology_path) - self.status(msg='Build artifacts have been published') - - def publish_release_artifacts(self): - publisher = ReleaseArtifactPublisher(self.settings, self.status) - publisher.publish_release_artifacts() - self.status(msg='Release artifacts have been published') - - def status(self, msg, **kwargs): - formatted = msg.format(**kwargs) - logging.info(formatted) - sys.stdout.write(formatted + '\n') - sys.stdout.flush() - - -class BuildArtifactPublisher(object): - - '''Publish build artifacts related to the release.''' - - def __init__(self, settings, status): - self.settings = settings - self.status = status - - def publish_build_artifacts(self, cluster_path): - artifact_basenames = self.list_build_artifacts_for_release(cluster_path) - self.status( - msg='Found {count} build artifact files in release', - count=len(artifact_basenames)) - - to_be_uploaded = self.filter_away_build_artifacts_on_public_trove( - artifact_basenames) - - logging.debug('List of artifacts (basenames) to upload (without already uploaded):') - for i, basename in enumerate(to_be_uploaded): - logging.debug(' {0}: {1}'.format(i, basename)) - logging.debug('End of artifact list (to_be_uploaded)') - - self.status( - msg='Need to fetch locally, then upload {count} build artifacts', - count=len(to_be_uploaded)) - - self.upload_build_artifacts_to_public_trove(to_be_uploaded) - - def list_build_artifacts_for_release(self, cluster_morphology_path): - self.status(msg='Find build artifacts included in release') - - # FIXME: These are hardcoded for simplicity. They would be - # possible to deduce automatically from the workspace, but - # that can happen later. - repo = 'file://%s' % os.path.abspath('.') - ref = 'HEAD' - - argv = [self.settings['morph-cmd'], 'list-artifacts', '--quiet', - '--repo', repo, '--ref', ref] - argv += self.find_system_morphologies(cluster_morphology_path) - output = cliapp.runcmd(argv) - basenames = output.splitlines() - logging.debug('List of build artifacts in release:') - for basename in basenames: - logging.debug(' {0}'.format(basename)) - logging.debug('End of list of build artifacts in release') - - return basenames - - def find_system_morphologies(self, cluster_morphology_path): - cluster = self.load_cluster_morphology(cluster_morphology_path) - system_dicts = self.find_systems_in_parsed_cluster_morphology(cluster) - if self.settings['arch']: - system_dicts = self.choose_systems_for_wanted_architectures( - system_dicts, self.settings['arch']) - return [sd['morph'] for sd in system_dicts] - - def load_cluster_morphology(self, pathname): - with open(pathname) as f: - return yaml.load(f) - - def find_systems_in_parsed_cluster_morphology(self, cluster): - return cluster['systems'] - - def choose_systems_for_wanted_architectures(self, system_dicts, archs): - return [ - sd - for sd in system_dicts - if self.system_is_for_wanted_arch(sd, archs)] - - def system_is_for_wanted_arch(self, system_dict, archs): - morph = self.load_system_morphology(system_dict) - return morph['arch'] in archs - - def load_system_morphology(self, system_dict): - pathname = morphlib.util.sanitise_morphology_path(system_dict['morph']) - return self.load_morphology_from_named_file(pathname) - - def load_morphology_from_named_file(self, pathname): - finder = self.get_morphology_finder_for_root_repository() - morphology_text = finder.read_file(pathname) - loader = morphlib.morphloader.MorphologyLoader() - return loader.load_from_string(morphology_text) - - def get_morphology_finder_for_root_repository(self): - definitions_repo = morphlib.definitions_repo.open( - '.', search_for_root=True) - return morphlib.morphologyfinder.MorphologyFinder(definitions_repo) - - def filter_away_build_artifacts_on_public_trove(self, basenames): - result = [] - logging.debug('Filtering away already existing artifacts:') - for basename, exists in self.query_public_trove_for_artifacts(basenames): - logging.debug(' {0}: {1}'.format(basename, exists)) - if not exists: - result.append(basename) - logging.debug('End of filtering away') - return result - - def query_public_trove_for_artifacts(self, basenames): - host = self.settings['public-trove-host'] - - # FIXME: This could use - # contextlib.closing(urllib2.urlopen(url, data=data) instead - # of explicit closing. - url = 'http://{host}:8080/1.0/artifacts'.format(host=host) - data = json.dumps(basenames) - f = urllib2.urlopen(url, data=data) - obj = json.load(f) - return obj.items() - - def upload_build_artifacts_to_public_trove(self, basenames): - self.download_artifacts_locally(basenames) - self.upload_artifacts_to_public_trove(basenames) - - def download_artifacts_locally(self, basenames): - dirname = self.settings['local-build-artifacts-dir'] - self.create_directory_if_missing(dirname) - for i, basename in enumerate(basenames): - url = self.construct_artifact_url(basename) - pathname = os.path.join(dirname, basename) - if not os.path.exists(pathname): - self.status( - msg='Downloading {i}/{total} {basename}', - basename=repr(basename), i=i, total=len(basenames)) - self.download_from_url(url, dirname, pathname) - - def create_directory_if_missing(self, dirname): - if not os.path.exists(dirname): - os.makedirs(dirname) - - def construct_artifact_url(self, basename): - scheme = 'http' - netloc = '{host}:8080'.format(host=self.settings['build-trove-host']) - path = '/1.0/artifacts' - query = 'filename={0}'.format(urllib.quote_plus(basename)) - fragment = '' - components = (scheme, netloc, path, query, fragment) - return urlparse.urlunsplit(components) - - def download_from_url(self, url, dirname, pathname): - logging.info( - 'Downloading {url} to {pathname}'.format( - url=url, pathname=pathname)) - with open(pathname, 'wb') as output: - try: - incoming = urllib2.urlopen(url) - shutil.copyfileobj(incoming, output) - incoming.close() - except urllib2.HTTPError as e: - if pathname.endswith('.meta'): - return - self.status( - msg="ERROR: Can't download {url}: {explanation}", - url=url, - explanation=str(e)) - os.remove(pathname) - raise - - def upload_artifacts_to_public_trove(self, basenames): - self.status( - msg='Upload build artifacts to {trove}', - trove=self.settings['public-trove-host']) - rsync_files_to_server( - self.settings['local-build-artifacts-dir'], - basenames, - self.settings['public-trove-username'], - self.settings['public-trove-host'], - self.settings['public-trove-artifact-dir']) - set_permissions_on_server( - self.settings['public-trove-username'], - self.settings['public-trove-host'], - self.settings['public-trove-artifact-dir'], - basenames) - -class ReleaseArtifactPublisher(object): - - '''Publish release artifacts for a release.''' - - def __init__(self, settings, status): - self.settings = settings - self.status = status - - def publish_release_artifacts(self): - files = self.list_release_artifacts() - if files: - self.upload_release_artifacts_to_private_dir(files) - self.move_release_artifacts_to_public_dir(files) - self.create_symlinks_to_new_release_artifacts(files) - - def list_release_artifacts(self): - self.status(msg='Find release artifacts to publish') - return os.listdir(self.settings['release-artifact-dir']) - - def upload_release_artifacts_to_private_dir(self, files): - self.status(msg='Upload release artifacts to private directory') - path = self.settings['download-server-private-dir'] - self.create_directory_on_download_server(path) - self.rsync_files_to_download_server(files, path) - - def create_directory_on_download_server(self, path): - user = self.settings['download-server-username'] - host = self.settings['download-server-address'] - self.status(msg='Create {host}:{path}', host=host, path=path) - target = '{user}@{host}'.format(user=user, host=host) - cliapp.ssh_runcmd(target, ['mkdir', '-p', path]) - - def rsync_files_to_download_server(self, files, path): - self.status(msg='Upload release artifacts to download server') - rsync_files_to_server( - self.settings['release-artifact-dir'], - files, - self.settings['download-server-username'], - self.settings['download-server-address'], - path) - set_permissions_on_server( - self.settings['download-server-username'], - self.settings['download-server-address'], - path, - files) - - def move_release_artifacts_to_public_dir(self, files): - self.status(msg='Move release artifacts to public directory') - private_dir = self.settings['download-server-private-dir'] - public_dir = self.settings['download-server-public-dir'] - self.create_directory_on_download_server(public_dir) - - # Move just the contents of the private dir, not the dir - # itself (-mindepth). Avoid overwriting existing files (mv - # -n). - argv = ['find', private_dir, '-mindepth', '1', - '-exec', 'mv', '-n', '{}', public_dir + '/.', ';'] - - target = '{user}@{host}'.format( - user=self.settings['download-server-username'], - host=self.settings['download-server-address']) - cliapp.ssh_runcmd(target, argv) - - def create_symlinks_to_new_release_artifacts(self, files): - self.status(msg='FIXME: Create symlinks to new releas artifacts') - - -def rsync_files_to_server( - source_dir, source_filenames, user, host, target_dir): - - if not source_filenames: - return - - argv = [ - 'rsync', - '-a', - '--progress', - '--partial', - '--human-readable', - '--sparse', - '--protect-args', - '-0', - '--files-from=-', - source_dir, - '{user}@{host}:{path}'.format(user=user, host=host, path=target_dir), - ] - - files_list = '\0'.join(filename for filename in source_filenames) - cliapp.runcmd(argv, feed_stdin=files_list, stdout=None, stderr=None) - - -def set_permissions_on_server(user, host, target_dir, filenames): - # If we have no files, we can't form a valid command to run on the server - if not filenames: - return - target = '{user}@{host}'.format(user=user, host=host) - argv = ['xargs', '-0', 'chmod', '0644'] - files_list = ''.join( - '{0}\0'.format(os.path.join(target_dir, filename)) for filename in filenames) - cliapp.ssh_runcmd(target, argv, feed_stdin=files_list, stdout=None, stderr=None) - - -ReleaseUploader(description=__doc__).run() |