summaryrefslogtreecommitdiff
path: root/scripts/release-upload
diff options
context:
space:
mode:
Diffstat (limited to 'scripts/release-upload')
-rwxr-xr-xscripts/release-upload473
1 files changed, 0 insertions, 473 deletions
diff --git a/scripts/release-upload b/scripts/release-upload
deleted file mode 100755
index f8b3337..0000000
--- a/scripts/release-upload
+++ /dev/null
@@ -1,473 +0,0 @@
-#!/usr/bin/python
-# Copyright (C) 2014 Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-'''Upload and publish Baserock binaries for a release.
-
-This utility is used for the Baserock release process. See
-http://wiki.baserock.org/guides/release-process/ for details on the
-release process.
-
-This utility uploads two sets of binaries:
-
-* The build artifacts (built chunks and strata) used to construct the
- systems being released. The systems are found in `release.morph` and
- the artifacts from the Trove used to prepare the release. They get
- uploaded to a public Trove (by default git.baserock.org). If they're
- the same Trove, then nothing happens.
-
-* The released system images (disk images, tar archives, etc)
- specified in `release.morph` get uploaded to a download server (by
- default download.baserock.org).
-
-'''
-
-
-import json
-import logging
-import os
-import pwd
-import shutil
-import sys
-import urllib
-import urllib2
-import urlparse
-
-import cliapp
-import yaml
-
-import morphlib
-
-class ReleaseUploader(cliapp.Application):
-
- def add_settings(self):
- group = 'Release upload settings'
-
- local_username = self.get_local_username()
-
- self.settings.string(
- ['build-trove-host'],
- 'get build artifacts from Trove at ADDRESS',
- metavar='ADDRESS',
- group=group)
-
- self.settings.string(
- ['public-trove-host'],
- 'publish build artifacts on Trove at ADDRESS',
- metavar='ADDRESS',
- default='git.baserock.org',
- group=group)
-
- self.settings.string(
- ['public-trove-username'],
- 'log into public trove as USER',
- metavar='USER',
- default=local_username,
- group=group)
-
- self.settings.string(
- ['public-trove-artifact-dir'],
- 'put published artifacts into DIR',
- metavar='DIR',
- default='/home/cache/artifacts',
- group=group)
-
- self.settings.string(
- ['release-artifact-dir'],
- 'get release artifacts from DIR (all files from there)',
- metavar='DIR',
- default='.',
- group=group)
-
- self.settings.string(
- ['download-server-address'],
- 'publish release artifacts on server at ADDRESS',
- metavar='ADDRESS',
- default='download.baserock.org',
- group=group)
-
- self.settings.string(
- ['download-server-username'],
- 'log into download server as USER',
- metavar='USER',
- default=local_username,
- group=group)
-
- self.settings.string(
- ['download-server-private-dir'],
- 'use DIR as the temporary location for uploaded release '
- 'artifacts',
- metavar='DIR',
- default='/srv/download.baserock.org/baserock/.publish-temp',
- group=group)
-
- self.settings.string(
- ['download-server-public-dir'],
- 'put published release artifacts in DIR',
- metavar='DIR',
- default='/srv/download.baserock.org/baserock',
- group=group)
-
- self.settings.string(
- ['local-build-artifacts-dir'],
- 'keep build artifacts to be uploaded temporarily in DIR',
- metavar='DIR',
- default='build-artifacts',
- group=group)
-
- self.settings.string(
- ['morph-cmd'],
- 'run FILE to invoke morph',
- metavar='FILE',
- default='morph',
- group=group)
-
- self.settings.string_list(
- ['arch'],
- 'Upload files from morphologies of ARCH',
- metavar='ARCH',
- default=[],
- group=group)
-
- self.settings.boolean(
- ['upload-build-artifacts'],
- 'upload build artifacts?',
- default=True)
-
- self.settings.boolean(
- ['upload-release-artifacts'],
- 'upload release artifacts (disk images etc)?',
- default=True)
-
- def get_local_username(self):
- uid = os.getuid()
- return pwd.getpwuid(uid)[0]
-
- def process_args(self, args):
- if len(args) != 1:
- raise cliapp.AppException('Usage: release-upload CLUSTER')
- cluster_morphology_path = args[0]
- self.status(msg='Uploading and publishing Baserock release')
-
- if self.settings['upload-build-artifacts']:
- self.publish_build_artifacts(cluster_morphology_path)
- else:
- self.status(
- msg='Not uploading build artifacts '
- '(upload-build-artifacts set to false')
-
- if self.settings['upload-release-artifacts']:
- self.publish_release_artifacts()
- else:
- self.status(
- msg='Not uploading release artifacts '
- '(upload-release-artifacts set to false')
-
- def publish_build_artifacts(self, cluster_morphology_path):
- publisher = BuildArtifactPublisher(self.settings, self.status)
- publisher.publish_build_artifacts(cluster_morphology_path)
- self.status(msg='Build artifacts have been published')
-
- def publish_release_artifacts(self):
- publisher = ReleaseArtifactPublisher(self.settings, self.status)
- publisher.publish_release_artifacts()
- self.status(msg='Release artifacts have been published')
-
- def status(self, msg, **kwargs):
- formatted = msg.format(**kwargs)
- logging.info(formatted)
- sys.stdout.write(formatted + '\n')
- sys.stdout.flush()
-
-
-class BuildArtifactPublisher(object):
-
- '''Publish build artifacts related to the release.'''
-
- def __init__(self, settings, status):
- self.settings = settings
- self.status = status
-
- def publish_build_artifacts(self, cluster_path):
- artifact_basenames = self.list_build_artifacts_for_release(cluster_path)
- self.status(
- msg='Found {count} build artifact files in release',
- count=len(artifact_basenames))
-
- to_be_uploaded = self.filter_away_build_artifacts_on_public_trove(
- artifact_basenames)
-
- logging.debug('List of artifacts (basenames) to upload (without already uploaded):')
- for i, basename in enumerate(to_be_uploaded):
- logging.debug(' {0}: {1}'.format(i, basename))
- logging.debug('End of artifact list (to_be_uploaded)')
-
- self.status(
- msg='Need to fetch locally, then upload {count} build artifacts',
- count=len(to_be_uploaded))
-
- self.upload_build_artifacts_to_public_trove(to_be_uploaded)
-
- def list_build_artifacts_for_release(self, cluster_morphology_path):
- self.status(msg='Find build artifacts included in release')
-
- # FIXME: These are hardcoded for simplicity. They would be
- # possible to deduce automatically from the workspace, but
- # that can happen later.
- repo = 'file://%s' % os.path.abspath('.')
- ref = 'HEAD'
-
- argv = [self.settings['morph-cmd'], 'list-artifacts', '--quiet',
- '--repo', repo, '--ref', ref]
- argv += self.find_system_morphologies(cluster_morphology_path)
- output = cliapp.runcmd(argv)
- basenames = output.splitlines()
- logging.debug('List of build artifacts in release:')
- for basename in basenames:
- logging.debug(' {0}'.format(basename))
- logging.debug('End of list of build artifacts in release')
-
- return basenames
-
- def find_system_morphologies(self, cluster_morphology_path):
- cluster = self.load_cluster_morphology(cluster_morphology_path)
- system_dicts = self.find_systems_in_parsed_cluster_morphology(cluster)
- if self.settings['arch']:
- system_dicts = self.choose_systems_for_wanted_architectures(
- system_dicts, self.settings['arch'])
- return [sd['morph'] for sd in system_dicts]
-
- def load_cluster_morphology(self, pathname):
- with open(pathname) as f:
- return yaml.load(f)
-
- def find_systems_in_parsed_cluster_morphology(self, cluster):
- return cluster['systems']
-
- def choose_systems_for_wanted_architectures(self, system_dicts, archs):
- return [
- sd
- for sd in system_dicts
- if self.system_is_for_wanted_arch(sd, archs)]
-
- def system_is_for_wanted_arch(self, system_dict, archs):
- morph = self.load_system_morphology(system_dict)
- return morph['arch'] in archs
-
- def load_system_morphology(self, system_dict):
- pathname = morphlib.util.sanitise_morphology_path(system_dict['morph'])
- return self.load_morphology_from_named_file(pathname)
-
- def load_morphology_from_named_file(self, pathname):
- finder = self.get_morphology_finder_for_root_repository()
- morphology_text = finder.read_file(pathname)
- loader = morphlib.morphloader.MorphologyLoader()
- return loader.load_from_string(morphology_text)
-
- def get_morphology_finder_for_root_repository(self):
- definitions_repo = morphlib.definitions_repo.open(
- '.', search_for_root=True)
- return morphlib.morphologyfinder.MorphologyFinder(definitions_repo)
-
- def filter_away_build_artifacts_on_public_trove(self, basenames):
- result = []
- logging.debug('Filtering away already existing artifacts:')
- for basename, exists in self.query_public_trove_for_artifacts(basenames):
- logging.debug(' {0}: {1}'.format(basename, exists))
- if not exists:
- result.append(basename)
- logging.debug('End of filtering away')
- return result
-
- def query_public_trove_for_artifacts(self, basenames):
- host = self.settings['public-trove-host']
-
- # FIXME: This could use
- # contextlib.closing(urllib2.urlopen(url, data=data) instead
- # of explicit closing.
- url = 'http://{host}:8080/1.0/artifacts'.format(host=host)
- data = json.dumps(basenames)
- f = urllib2.urlopen(url, data=data)
- obj = json.load(f)
- return obj.items()
-
- def upload_build_artifacts_to_public_trove(self, basenames):
- self.download_artifacts_locally(basenames)
- self.upload_artifacts_to_public_trove(basenames)
-
- def download_artifacts_locally(self, basenames):
- dirname = self.settings['local-build-artifacts-dir']
- self.create_directory_if_missing(dirname)
- for i, basename in enumerate(basenames):
- url = self.construct_artifact_url(basename)
- pathname = os.path.join(dirname, basename)
- if not os.path.exists(pathname):
- self.status(
- msg='Downloading {i}/{total} {basename}',
- basename=repr(basename), i=i, total=len(basenames))
- self.download_from_url(url, dirname, pathname)
-
- def create_directory_if_missing(self, dirname):
- if not os.path.exists(dirname):
- os.makedirs(dirname)
-
- def construct_artifact_url(self, basename):
- scheme = 'http'
- netloc = '{host}:8080'.format(host=self.settings['build-trove-host'])
- path = '/1.0/artifacts'
- query = 'filename={0}'.format(urllib.quote_plus(basename))
- fragment = ''
- components = (scheme, netloc, path, query, fragment)
- return urlparse.urlunsplit(components)
-
- def download_from_url(self, url, dirname, pathname):
- logging.info(
- 'Downloading {url} to {pathname}'.format(
- url=url, pathname=pathname))
- with open(pathname, 'wb') as output:
- try:
- incoming = urllib2.urlopen(url)
- shutil.copyfileobj(incoming, output)
- incoming.close()
- except urllib2.HTTPError as e:
- if pathname.endswith('.meta'):
- return
- self.status(
- msg="ERROR: Can't download {url}: {explanation}",
- url=url,
- explanation=str(e))
- os.remove(pathname)
- raise
-
- def upload_artifacts_to_public_trove(self, basenames):
- self.status(
- msg='Upload build artifacts to {trove}',
- trove=self.settings['public-trove-host'])
- rsync_files_to_server(
- self.settings['local-build-artifacts-dir'],
- basenames,
- self.settings['public-trove-username'],
- self.settings['public-trove-host'],
- self.settings['public-trove-artifact-dir'])
- set_permissions_on_server(
- self.settings['public-trove-username'],
- self.settings['public-trove-host'],
- self.settings['public-trove-artifact-dir'],
- basenames)
-
-class ReleaseArtifactPublisher(object):
-
- '''Publish release artifacts for a release.'''
-
- def __init__(self, settings, status):
- self.settings = settings
- self.status = status
-
- def publish_release_artifacts(self):
- files = self.list_release_artifacts()
- if files:
- self.upload_release_artifacts_to_private_dir(files)
- self.move_release_artifacts_to_public_dir(files)
- self.create_symlinks_to_new_release_artifacts(files)
-
- def list_release_artifacts(self):
- self.status(msg='Find release artifacts to publish')
- return os.listdir(self.settings['release-artifact-dir'])
-
- def upload_release_artifacts_to_private_dir(self, files):
- self.status(msg='Upload release artifacts to private directory')
- path = self.settings['download-server-private-dir']
- self.create_directory_on_download_server(path)
- self.rsync_files_to_download_server(files, path)
-
- def create_directory_on_download_server(self, path):
- user = self.settings['download-server-username']
- host = self.settings['download-server-address']
- self.status(msg='Create {host}:{path}', host=host, path=path)
- target = '{user}@{host}'.format(user=user, host=host)
- cliapp.ssh_runcmd(target, ['mkdir', '-p', path])
-
- def rsync_files_to_download_server(self, files, path):
- self.status(msg='Upload release artifacts to download server')
- rsync_files_to_server(
- self.settings['release-artifact-dir'],
- files,
- self.settings['download-server-username'],
- self.settings['download-server-address'],
- path)
- set_permissions_on_server(
- self.settings['download-server-username'],
- self.settings['download-server-address'],
- path,
- files)
-
- def move_release_artifacts_to_public_dir(self, files):
- self.status(msg='Move release artifacts to public directory')
- private_dir = self.settings['download-server-private-dir']
- public_dir = self.settings['download-server-public-dir']
- self.create_directory_on_download_server(public_dir)
-
- # Move just the contents of the private dir, not the dir
- # itself (-mindepth). Avoid overwriting existing files (mv
- # -n).
- argv = ['find', private_dir, '-mindepth', '1',
- '-exec', 'mv', '-n', '{}', public_dir + '/.', ';']
-
- target = '{user}@{host}'.format(
- user=self.settings['download-server-username'],
- host=self.settings['download-server-address'])
- cliapp.ssh_runcmd(target, argv)
-
- def create_symlinks_to_new_release_artifacts(self, files):
- self.status(msg='FIXME: Create symlinks to new releas artifacts')
-
-
-def rsync_files_to_server(
- source_dir, source_filenames, user, host, target_dir):
-
- if not source_filenames:
- return
-
- argv = [
- 'rsync',
- '-a',
- '--progress',
- '--partial',
- '--human-readable',
- '--sparse',
- '--protect-args',
- '-0',
- '--files-from=-',
- source_dir,
- '{user}@{host}:{path}'.format(user=user, host=host, path=target_dir),
- ]
-
- files_list = '\0'.join(filename for filename in source_filenames)
- cliapp.runcmd(argv, feed_stdin=files_list, stdout=None, stderr=None)
-
-
-def set_permissions_on_server(user, host, target_dir, filenames):
- # If we have no files, we can't form a valid command to run on the server
- if not filenames:
- return
- target = '{user}@{host}'.format(user=user, host=host)
- argv = ['xargs', '-0', 'chmod', '0644']
- files_list = ''.join(
- '{0}\0'.format(os.path.join(target_dir, filename)) for filename in filenames)
- cliapp.ssh_runcmd(target, argv, feed_stdin=files_list, stdout=None, stderr=None)
-
-
-ReleaseUploader(description=__doc__).run()