diff options
author | Lars Wirzenius <lars.wirzenius@codethink.co.uk> | 2014-07-30 17:21:58 +0000 |
---|---|---|
committer | Lars Wirzenius <lars.wirzenius@codethink.co.uk> | 2014-07-30 17:21:58 +0000 |
commit | 0946a88d22c25d9678feecbb1d80254508b60256 (patch) | |
tree | b3628b8f1660d7ef7f514a21f24e487107a4b7d1 | |
parent | dcbf761d786bd67ee77f996bc6f4d6befb489dc7 (diff) | |
parent | d239aeed53a52e5dbc10aad6131334e51776ef95 (diff) | |
download | definitions-0946a88d22c25d9678feecbb1d80254508b60256.tar.gz |
Merge branch 'baserock/liw/merge-michaels-ci-release-scripts'
This is a merge of the baserock/michaeldrake/ci-release-scripts branch,
but with some cleanups of the code for cleanliness.
-rwxr-xr-x | scripts/release-build | 59 | ||||
-rwxr-xr-x | scripts/release-upload | 109 |
2 files changed, 129 insertions, 39 deletions
diff --git a/scripts/release-build b/scripts/release-build index 36a38deb..a3e25ebb 100755 --- a/scripts/release-build +++ b/scripts/release-build @@ -98,7 +98,6 @@ class ReleaseApp(cliapp.Application): os.chdir('..') if not os.path.exists('release'): os.mkdir('release') - os.chdir('release') self.deploy_images(cluster, cluster_path) def load_morphology(self, name, kind=None): @@ -109,13 +108,21 @@ class ReleaseApp(cliapp.Application): assert morph['kind'] == kind return morph, path + def iterate_systems(self, system_list): + for system in system_list: + yield system['morph'] + if 'subsystems' in system: + for subsystem in self.iterate_systems(system['subsystems']): + yield subsystem + def prepare_builds(self, cluster): '''Prepare a list of builds''' - systems = [system['morph'] for system in cluster['systems']] + systems = set(self.iterate_systems(cluster['systems'])) builds = [] for system_name in systems: system, _ = self.load_morphology(system_name) - builds.append(Build(system_name, system['arch'], self)) + if system['arch'] in self.controllers: + builds.append(Build(system_name, system['arch'], self)) return builds def deploy_images(self, cluster, cluster_path): @@ -123,29 +130,29 @@ class ReleaseApp(cliapp.Application): outputs = {} for system in cluster['systems']: - name = system['morph'] - if name not in system['deploy']: - raise cliapp.AppException( - 'In %s: system %s ID should be "%s"' % - (cluster_path, name, name)) - - # The release.morph cluster must specify a basename for the file, - # of name and extension. This script knows about name, but it - # can't find out the appropriate file extension without second - # guessing the behaviour of write extensions. - basename = system['deploy'][name]['location'] - - if '/' in basename or basename.startswith(version_label): - raise cliapp.AppException( - 'In %s: system %s.location should be just the base name, ' - 'e.g. "%s.img"' % (cluster_path, name, name)) - - filename = '%s-%s' % (version_label, basename) - if os.path.exists(filename): - self.output.write('Reusing existing deployment of %s\n' % filename) - else: - self.output.write('Creating %s from release.morph\n' % filename) - self.deploy_single_image(cluster_path, name, filename, version_label) + morphology_name = system['morph'] + morphology = self.load_morphology(morphology_name)[0] + if morphology['arch'] not in self.controllers: + continue + + for deployment_name, deployment_info in system['deploy'].iteritems(): + # The release.morph cluster must specify a basename for the file, + # of name and extension. This script knows about name, but it + # can't find out the appropriate file extension without second + # guessing the behaviour of write extensions. + basename = deployment_info['location'] + + if '/' in basename or basename.startswith(version_label): + raise cliapp.AppException( + 'In %s: system %s.location should be just the base name, ' + 'e.g. "%s.img"' % (cluster_path, deployment_name, deployment_name)) + + filename = os.path.join('release', '%s-%s' % (version_label, basename)) + if os.path.exists(filename): + self.output.write('Reusing existing deployment of %s\n' % filename) + else: + self.output.write('Creating %s from release.morph\n' % filename) + self.deploy_single_image(cluster_path, deployment_name, filename, version_label) def deploy_single_image(self, cluster_path, name, location, version_label): deploy_command = [ diff --git a/scripts/release-upload b/scripts/release-upload index 2e7f54e8..0ab6b957 100755 --- a/scripts/release-upload +++ b/scripts/release-upload @@ -49,6 +49,7 @@ import urlparse import cliapp import yaml +import morphlib class ReleaseUploader(cliapp.Application): @@ -134,15 +135,53 @@ class ReleaseUploader(cliapp.Application): default='morph', group=group) + self.settings.string_list( + ['arch'], + 'Upload files from morphologies of ARCH', + metavar='ARCH', + default=[], + group=group) + + self.settings.boolean( + ['upload-build-artifacts'], + 'upload build artifacts?', + default=True) + + self.settings.boolean( + ['upload-release-artifacts'], + 'upload release artifacts (disk images etc)?', + default=True) + def get_local_username(self): uid = os.getuid() return pwd.getpwuid(uid)[0] def process_args(self, args): self.status(msg='Uploading and publishing Baserock release') - BuildArtifactPublisher(self.settings, self.status).publish_build_artifacts() - ReleaseArtifactPublisher(self.settings, self.status).publish_release_artifacts() - self.status(msg='Release has been uploaded and published') + + if self.settings['upload-build-artifacts']: + self.publish_build_artifacts() + else: + self.status( + msg='Not uploading build artifacts ' + '(upload-build-artifacts set to false') + + if self.settings['upload-release-artifacts']: + self.publish_release_artifacts() + else: + self.status( + msg='Not uploading release artifacts ' + '(upload-release-artifacts set to false') + + def publish_build_artifacts(self): + publisher = BuildArtifactPublisher(self.settings, self.status) + publisher.publish_build_artifacts() + self.status(msg='Build artifacts have been published') + + def publish_release_artifacts(self): + publisher = ReleaseArtifactPublisher(self.settings, self.status) + publisher.publish_release_artifacts() + self.status(msg='Release artifacts have been published') def status(self, msg, **kwargs): formatted = msg.format(**kwargs) @@ -192,14 +231,53 @@ class BuildArtifactPublisher(object): argv += self.find_system_morphologies() output = cliapp.runcmd(argv) basenames = output.splitlines() + logging.debug('List of build artifacts in release:') + for basename in basenames: + logging.debug(' {0}'.format(basename)) + logging.debug('End of list of build artifacts in release') return basenames def find_system_morphologies(self): - cluster_morphology_pathname = 'release.morph' - with open(cluster_morphology_pathname) as f: - obj = yaml.load(f) - return [system_dict['morph'] for system_dict in obj['systems']] + cluster = self.load_cluster_morphology('release.morph') + system_dicts = self.find_systems_in_parsed_cluster_morphology(cluster) + if self.settings['arch']: + system_dicts = self.choose_systems_for_wanted_architectures( + system_dicts, self.settings['arch']) + return [sd['morph'] for sd in system_dicts] + + def load_cluster_morphology(self, pathname): + with open(pathname) as f: + return yaml.load(f) + + def find_systems_in_parsed_cluster_morphology(self, cluster): + return cluster['systems'] + + def choose_systems_for_wanted_architectures(self, system_dicts, archs): + return [ + sd + for sd in system_dicts + if self.system_is_for_wanted_arch(sd, archs)] + + def system_is_for_wanted_arch(self, system_dict, archs): + morph = self.load_system_morphology(system_dict) + return morph['arch'] in archs + + def load_system_morphology(self, system_dict): + pathname = morphlib.util.sanitise_morphology_path(system_dict['morph']) + return self.load_morphology_from_named_file(pathname) + + def load_morphology_from_named_file(self, pathname): + finder = self.get_morphology_finder_for_root_repository() + morphology_text = finder.read_morphology(pathname) + loader = morphlib.morphloader.MorphologyLoader() + return loader.load_from_string(morphology_text) + + def get_morphology_finder_for_root_repository(self): + sb = morphlib.sysbranchdir.open_from_within('.') + definitions = sb.get_git_directory_name(sb.root_repository_url) + definitions_repo = morphlib.gitdir.GitDirectory(definitions) + return morphlib.morphologyfinder.MorphologyFinder(definitions_repo) def filter_away_build_artifacts_on_public_trove(self, basenames): result = [] @@ -357,6 +435,9 @@ class ReleaseArtifactPublisher(object): def rsync_files_to_server( source_dir, source_filenames, user, host, target_dir): + if not source_filenames: + return + argv = [ 'rsync', '-a', @@ -371,17 +452,19 @@ def rsync_files_to_server( '{user}@{host}:{path}'.format(user=user, host=host, path=target_dir), ] - files_list = ''.join( - '{0}\0'.format(filename) for filename in source_filenames) + files_list = '\0'.join(filename for filename in source_filenames) cliapp.runcmd(argv, feed_stdin=files_list, stdout=None, stderr=None) def set_permissions_on_server(user, host, target_dir, filenames): + # If we have no files, we can't form a valid command to run on the server + if not filenames: + return target = '{user}@{host}'.format(user=user, host=host) - argv = ['chmod', '0644'] - for filename in filenames: - argv.append(os.path.join(target_dir, filename)) - cliapp.ssh_runcmd(target, argv) + argv = ['xargs', '-0', 'chmod', '0644'] + files_list = ''.join( + '{0}\0'.format(os.path.join(target_dir, filename)) for filename in filenames) + cliapp.ssh_runcmd(target, argv, feed_stdin=files_list, stdout=None, stderr=None) ReleaseUploader(description=__doc__).run() |