# Copyright 2014 Codethink Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import cliapp import json import logging import mason class BuildArtifactPublisher(object): '''Publish build artifacts related to the release.''' logging.getLogger('mason.publishers.BuildArtifactPublisher') def __init__(self, config, defs_repo): self.config = config self.morph_helper = mason.util.MorphologyHelper(defs_repo) def publish_build_artifacts(self): artifact_basenames = self.list_build_artifacts_for_release( self.config['cluster-morphology']) logging.info( 'Found %s build artifact files in release', len(artifact_basenames)) to_be_uploaded = self.filter_away_build_artifacts_on_public_trove( artifact_basenames) logging.debug('List of artifacts (basenames) to upload (without already uploaded):') for i, basename in enumerate(to_be_uploaded): logging.debug(' {0}: {1}'.format(i, basename)) logging.debug('End of artifact list (to_be_uploaded)') logging.info( 'Need to fetch locally, then upload %s build artifacts', len(to_be_uploaded)) self.upload_build_artifacts_to_public_trove(to_be_uploaded) def list_build_artifacts_for_release(self, cluster_morphology_path): logging.info('Find build artifacts included in release') # FIXME: These are hardcoded for simplicity. They would be # possible to deduce automatically from the workspace, but # that can happen later. repo = 'file://%s' % \ os.path.abspath(self.morph_helper.defs_repo.dirname) ref = 'HEAD' argv = [self.config['morph-cmd'], 'list-artifacts', '--quiet', repo, ref] argv += self.morph_helper.find_systems_by_arch( cluster_morphology_path, self.config['architecture']) output = cliapp.runcmd(argv) basenames = output.splitlines() logging.debug('List of build artifacts in release:') for basename in basenames: logging.debug(' {0}'.format(basename)) logging.debug('End of list of build artifacts in release') return basenames def filter_away_build_artifacts_on_public_trove(self, basenames): result = [] logging.debug('Filtering away already existing artifacts:') for basename, exists in self.query_public_trove_for_artifacts(basenames): logging.debug(' {0}: {1}'.format(basename, exists)) if not exists: result.append(basename) logging.debug('End of filtering away') return result def query_public_trove_for_artifacts(self, basenames): host = self.config['public-trove-host'] # FIXME: This could use # contextlib.closing(urllib2.urlopen(url, data=data) instead # of explicit closing. url = 'http://{host}:8080/1.0/artifacts'.format(host=host) data = json.dumps(basenames) f = urllib2.urlopen(url, data=data) obj = json.load(f) return obj.items() def upload_build_artifacts_to_public_trove(self, basenames): self.download_artifacts_locally(basenames) self.upload_artifacts_to_public_trove(basenames) def download_artifacts_locally(self, basenames): dirname = self.config['local-build-artifacts-dir'] self.create_directory_if_missing(dirname) for i, basename in enumerate(basenames): pathname = os.path.join(dirname, basename) if not os.path.exists(pathname): logging.info( 'Downloading %s/%s %s', i, len(basenames), repr(basename)) orig = '/srv/distbuild/artifacts/%s' % basename #TODO: download the artifacts from a shared cache shutil.copy2(orig, pathname) def create_directory_if_missing(self, dirname): if not os.path.exists(dirname): os.makedirs(dirname) def upload_artifacts_to_public_trove(self, basenames): logging.info( 'Upload build artifacts to %s', self.config['public-trove-host']) rsync_files_to_server( self.config['local-build-artifacts-dir'], basenames, self.config['public-trove-username'], self.config['public-trove-host'], self.config['public-trove-artifact-dir']) set_permissions_on_server( self.config['public-trove-username'], self.config['public-trove-host'], self.config['public-trove-artifact-dir'], basenames) class ReleaseArtifactPublisher(object): '''Publish release artifacts for a release.''' logging.getLogger('mason.publishers.ReleaseArtifactPublisher') def __init__(self, config): self.config = config def publish_release_artifacts(self): files = self.list_release_artifacts() if files: self.upload_release_artifacts_to_private_dir(files) self.move_release_artifacts_to_public_dir(files) self.create_symlinks_to_new_release_artifacts(files) def list_release_artifacts(self): logging.info('Find release artifacts to publish') return os.listdir(self.config['release-artifact-dir']) def upload_release_artifacts_to_private_dir(self, files): logging.info('Upload release artifacts to private directory') path = self.config['download-server-private-dir'] self.create_directory_on_download_server(path) self.rsync_files_to_download_server(files, path) def create_directory_on_download_server(self, path): user = self.config['download-server-username'] host = self.config['download-server-address'] logging.info(msg='Create {host}:{path}', host=host, path=path) target = '{user}@{host}'.format(user=user, host=host) cliapp.ssh_runcmd(target, ['mkdir', '-p', path]) def rsync_files_to_download_server(self, files, path): logging.info('Upload release artifacts to download server') rsync_files_to_server( self.config['release-artifact-dir'], files, self.config['download-server-username'], self.config['download-server-address'], path) set_permissions_on_server( self.config['download-server-username'], self.config['download-server-address'], path, files) def move_release_artifacts_to_public_dir(self, files): logging.info('Move release artifacts to public directory') private_dir = self.config['download-server-private-dir'] public_dir = self.config['download-server-public-dir'] self.create_directory_on_download_server(public_dir) # Move just the contents of the private dir, not the dir # itself (-mindepth). Avoid overwriting existing files (mv # -n). argv = ['find', private_dir, '-mindepth', '1', '-exec', 'mv', '-n', '{}', public_dir + '/.', ';'] target = '{user}@{host}'.format( user=self.config['download-server-username'], host=self.config['download-server-address']) cliapp.ssh_runcmd(target, argv) def create_symlinks_to_new_release_artifacts(self, files): logging.info('FIXME: Create symlinks to new release artifacts') def rsync_files_to_server( source_dir, source_filenames, user, host, target_dir): if not source_filenames: return argv = [ 'rsync', '-a', '--progress', '--partial', '--human-readable', '--sparse', '--protect-args', '-0', '--files-from=-', source_dir, '{user}@{host}:{path}'.format(user=user, host=host, path=target_dir), ] files_list = '\0'.join(filename for filename in source_filenames) cliapp.runcmd(argv, feed_stdin=files_list, stdout=None, stderr=None) def set_permissions_on_server(user, host, target_dir, filenames): # If we have no files, we can't form a valid command to run on the server if not filenames: return target = '{user}@{host}'.format(user=user, host=host) argv = ['xargs', '-0', 'chmod', '0644'] files_list = ''.join( '{0}\0'.format(os.path.join(target_dir, filename)) for filename in filenames) cliapp.ssh_runcmd(target, argv, feed_stdin=files_list, stdout=None, stderr=None)