diff options
Diffstat (limited to 'mason/publishers.py')
| -rw-r--r-- | mason/publishers.py | 239 |
1 files changed, 239 insertions, 0 deletions
diff --git a/mason/publishers.py b/mason/publishers.py new file mode 100644 index 0000000..7c81310 --- /dev/null +++ b/mason/publishers.py @@ -0,0 +1,239 @@ +# Copyright 2014 Codethink Ltd +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import cliapp +import json +import logging + +import mason + + +class BuildArtifactPublisher(object): + + '''Publish build artifacts related to the release.''' + + logging.getLogger('mason.publishers.BuildArtifactPublisher') + + def __init__(self, config, defs_repo): + self.config = config + self.morph_helper = mason.util.MorphologyHelper(defs_repo) + + def publish_build_artifacts(self): + artifact_basenames = self.list_build_artifacts_for_release( + self.config['cluster-morphology']) + logging.info( + 'Found %s build artifact files in release', + len(artifact_basenames)) + + to_be_uploaded = self.filter_away_build_artifacts_on_public_trove( + artifact_basenames) + + logging.debug('List of artifacts (basenames) to upload (without already uploaded):') + for i, basename in enumerate(to_be_uploaded): + logging.debug(' {0}: {1}'.format(i, basename)) + logging.debug('End of artifact list (to_be_uploaded)') + + logging.info( + 'Need to fetch locally, then upload %s build artifacts', + len(to_be_uploaded)) + + self.upload_build_artifacts_to_public_trove(to_be_uploaded) + + def list_build_artifacts_for_release(self, cluster_morphology_path): + logging.info('Find build artifacts included in release') + + # FIXME: These are hardcoded for simplicity. They would be + # possible to deduce automatically from the workspace, but + # that can happen later. + repo = 'file://%s' % \ + os.path.abspath(self.morph_helper.defs_repo.dirname) + ref = 'HEAD' + + argv = [self.config['morph-cmd'], + 'list-artifacts', '--quiet', + repo, ref] + argv += self.morph_helper.find_systems_by_arch( + cluster_morphology_path, self.config['architecture']) + + output = cliapp.runcmd(argv) + basenames = output.splitlines() + logging.debug('List of build artifacts in release:') + for basename in basenames: + logging.debug(' {0}'.format(basename)) + logging.debug('End of list of build artifacts in release') + + return basenames + + def filter_away_build_artifacts_on_public_trove(self, basenames): + result = [] + logging.debug('Filtering away already existing artifacts:') + for basename, exists in self.query_public_trove_for_artifacts(basenames): + logging.debug(' {0}: {1}'.format(basename, exists)) + if not exists: + result.append(basename) + logging.debug('End of filtering away') + return result + + def query_public_trove_for_artifacts(self, basenames): + host = self.config['public-trove-host'] + + # FIXME: This could use + # contextlib.closing(urllib2.urlopen(url, data=data) instead + # of explicit closing. + url = 'http://{host}:8080/1.0/artifacts'.format(host=host) + data = json.dumps(basenames) + f = urllib2.urlopen(url, data=data) + obj = json.load(f) + return obj.items() + + def upload_build_artifacts_to_public_trove(self, basenames): + self.download_artifacts_locally(basenames) + self.upload_artifacts_to_public_trove(basenames) + + def download_artifacts_locally(self, basenames): + dirname = self.config['local-build-artifacts-dir'] + self.create_directory_if_missing(dirname) + for i, basename in enumerate(basenames): + pathname = os.path.join(dirname, basename) + if not os.path.exists(pathname): + logging.info( + 'Downloading %s/%s %s', + i, len(basenames), repr(basename)) + orig = '/srv/distbuild/artifacts/%s' % basename + #TODO: download the artifacts from a shared cache + shutil.copy2(orig, pathname) + + def create_directory_if_missing(self, dirname): + if not os.path.exists(dirname): + os.makedirs(dirname) + + def upload_artifacts_to_public_trove(self, basenames): + logging.info( + 'Upload build artifacts to %s', + self.config['public-trove-host']) + rsync_files_to_server( + self.config['local-build-artifacts-dir'], + basenames, + self.config['public-trove-username'], + self.config['public-trove-host'], + self.config['public-trove-artifact-dir']) + set_permissions_on_server( + self.config['public-trove-username'], + self.config['public-trove-host'], + self.config['public-trove-artifact-dir'], + basenames) + + +class ReleaseArtifactPublisher(object): + + '''Publish release artifacts for a release.''' + + logging.getLogger('mason.publishers.ReleaseArtifactPublisher') + + def __init__(self, config): + self.config = config + + def publish_release_artifacts(self): + files = self.list_release_artifacts() + if files: + self.upload_release_artifacts_to_private_dir(files) + self.move_release_artifacts_to_public_dir(files) + self.create_symlinks_to_new_release_artifacts(files) + + def list_release_artifacts(self): + logging.info('Find release artifacts to publish') + return os.listdir(self.config['release-artifact-dir']) + + def upload_release_artifacts_to_private_dir(self, files): + logging.info('Upload release artifacts to private directory') + path = self.config['download-server-private-dir'] + self.create_directory_on_download_server(path) + self.rsync_files_to_download_server(files, path) + + def create_directory_on_download_server(self, path): + user = self.config['download-server-username'] + host = self.config['download-server-address'] + logging.info(msg='Create {host}:{path}', host=host, path=path) + target = '{user}@{host}'.format(user=user, host=host) + cliapp.ssh_runcmd(target, ['mkdir', '-p', path]) + + def rsync_files_to_download_server(self, files, path): + logging.info('Upload release artifacts to download server') + rsync_files_to_server( + self.config['release-artifact-dir'], + files, + self.config['download-server-username'], + self.config['download-server-address'], + path) + set_permissions_on_server( + self.config['download-server-username'], + self.config['download-server-address'], + path, + files) + + def move_release_artifacts_to_public_dir(self, files): + logging.info('Move release artifacts to public directory') + private_dir = self.config['download-server-private-dir'] + public_dir = self.config['download-server-public-dir'] + self.create_directory_on_download_server(public_dir) + + # Move just the contents of the private dir, not the dir + # itself (-mindepth). Avoid overwriting existing files (mv + # -n). + argv = ['find', private_dir, '-mindepth', '1', + '-exec', 'mv', '-n', '{}', public_dir + '/.', ';'] + + target = '{user}@{host}'.format( + user=self.config['download-server-username'], + host=self.config['download-server-address']) + cliapp.ssh_runcmd(target, argv) + + def create_symlinks_to_new_release_artifacts(self, files): + logging.info('FIXME: Create symlinks to new release artifacts') + + +def rsync_files_to_server( + source_dir, source_filenames, user, host, target_dir): + + if not source_filenames: + return + + argv = [ + 'rsync', + '-a', + '--progress', + '--partial', + '--human-readable', + '--sparse', + '--protect-args', + '-0', + '--files-from=-', + source_dir, + '{user}@{host}:{path}'.format(user=user, host=host, path=target_dir), + ] + + files_list = '\0'.join(filename for filename in source_filenames) + cliapp.runcmd(argv, feed_stdin=files_list, stdout=None, stderr=None) + + +def set_permissions_on_server(user, host, target_dir, filenames): + # If we have no files, we can't form a valid command to run on the server + if not filenames: + return + target = '{user}@{host}'.format(user=user, host=host) + argv = ['xargs', '-0', 'chmod', '0644'] + files_list = ''.join( + '{0}\0'.format(os.path.join(target_dir, filename)) for filename in filenames) + cliapp.ssh_runcmd(target, argv, feed_stdin=files_list, stdout=None, stderr=None) |