From 0b8ceeca7540faab5f7750e2c95d6496ae904391 Mon Sep 17 00:00:00 2001 From: Michael Drake Date: Wed, 30 Jul 2014 14:51:48 +0100 Subject: Write release images into release subdirectory The script used to chdir into the release directory before running morph deploy. Unfortunately, this didn't work because deployments are run from the top of the definitons repository. So now the release directory is included in the path to be deployed. --- scripts/release-build | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/release-build b/scripts/release-build index 36a38deb..361d32a0 100755 --- a/scripts/release-build +++ b/scripts/release-build @@ -98,7 +98,6 @@ class ReleaseApp(cliapp.Application): os.chdir('..') if not os.path.exists('release'): os.mkdir('release') - os.chdir('release') self.deploy_images(cluster, cluster_path) def load_morphology(self, name, kind=None): @@ -140,7 +139,7 @@ class ReleaseApp(cliapp.Application): 'In %s: system %s.location should be just the base name, ' 'e.g. "%s.img"' % (cluster_path, name, name)) - filename = '%s-%s' % (version_label, basename) + filename = os.path.join('release', '%s-%s' % (version_label, basename)) if os.path.exists(filename): self.output.write('Reusing existing deployment of %s\n' % filename) else: -- cgit v1.2.1 From b2199074d608eee43a4dbdb4732eee71f52201f4 Mon Sep 17 00:00:00 2001 From: Michael Drake Date: Wed, 30 Jul 2014 14:54:13 +0100 Subject: Allow release-build to build subsystems of defined systems --- scripts/release-build | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/scripts/release-build b/scripts/release-build index 361d32a0..34305c67 100755 --- a/scripts/release-build +++ b/scripts/release-build @@ -108,9 +108,16 @@ class ReleaseApp(cliapp.Application): assert morph['kind'] == kind return morph, path + def _iterate_systems(self, system_list): + for system in system_list: + yield system['morph'] + if 'subsystems' in system: + for subsystem in self._iterate_systems(system['subsystems']): + yield subsystem + def prepare_builds(self, cluster): '''Prepare a list of builds''' - systems = [system['morph'] for system in cluster['systems']] + systems = set(self._iterate_systems(cluster['systems'])) builds = [] for system_name in systems: system, _ = self.load_morphology(system_name) -- cgit v1.2.1 From 822a1457660a6b46d5f8c5ab8b28efeb8cb51b59 Mon Sep 17 00:00:00 2001 From: Michael Drake Date: Wed, 30 Jul 2014 14:55:57 +0100 Subject: Allow release-upload to be configured not to upload release images For continuous artifact cache population, we don't care so much about the large disk images that we make available at release time. This patch allows omitting any of the configuration required to upload the release images to mean that we didn't want to upload them, and continue without doing so. --- scripts/release-upload | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/scripts/release-upload b/scripts/release-upload index 2e7f54e8..e7367f13 100755 --- a/scripts/release-upload +++ b/scripts/release-upload @@ -140,9 +140,27 @@ class ReleaseUploader(cliapp.Application): def process_args(self, args): self.status(msg='Uploading and publishing Baserock release') - BuildArtifactPublisher(self.settings, self.status).publish_build_artifacts() - ReleaseArtifactPublisher(self.settings, self.status).publish_release_artifacts() - self.status(msg='Release has been uploaded and published') + BuildArtifactPublisher(self.settings, + self.status).publish_build_artifacts() + self.status(msg='Build artifiacts have been published') + + RELEASE_OPTIONS = ('release-artifact-dir', + 'download-server-username', + 'download-server-address', + 'download-server-private-dir', + 'download-server-public-dir') + missing_options = set(option for option in RELEASE_OPTIONS + if self.settings[option] == '') + + if not missing_options: + ReleaseArtifactPublisher(self.settings, + self.status).publish_release_artifacts() + self.status(msg='Release images have been published') + else: + self.status(msg='Not uploading release images: ' + 'options not provided: {options}', + options=', '.join(missing_options)) + def status(self, msg, **kwargs): formatted = msg.format(**kwargs) -- cgit v1.2.1 From 0ad896ece6ce1d1fe78b9f8e35acfccaadb18a1f Mon Sep 17 00:00:00 2001 From: Michael Drake Date: Wed, 30 Jul 2014 15:01:50 +0100 Subject: Allow release scripts to be configured to only handle a subset of architectures We currently build all architectures at once during the release process, however for our CD pipeline we operate with one CD pipeline per architecture. This is not just useful for the CD pipeline work though, as it allows one organisation to handle releases for x86, where the infrastructure may be located in the cloud, and one organisation to handle ARM systems, which may be located in an office. --- scripts/release-build | 5 +++++ scripts/release-upload | 29 ++++++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/scripts/release-build b/scripts/release-build index 34305c67..fe10ba3b 100755 --- a/scripts/release-build +++ b/scripts/release-build @@ -121,6 +121,8 @@ class ReleaseApp(cliapp.Application): builds = [] for system_name in systems: system, _ = self.load_morphology(system_name) + if system['arch'] not in self.controllers: + continue builds.append(Build(system_name, system['arch'], self)) return builds @@ -130,6 +132,9 @@ class ReleaseApp(cliapp.Application): for system in cluster['systems']: name = system['morph'] + morphology = self.load_morphology(name)[0] + if morphology['arch'] not in self.controllers: + continue if name not in system['deploy']: raise cliapp.AppException( 'In %s: system %s ID should be "%s"' % diff --git a/scripts/release-upload b/scripts/release-upload index e7367f13..f97e7e98 100755 --- a/scripts/release-upload +++ b/scripts/release-upload @@ -49,6 +49,7 @@ import urlparse import cliapp import yaml +import morphlib class ReleaseUploader(cliapp.Application): @@ -134,6 +135,13 @@ class ReleaseUploader(cliapp.Application): default='morph', group=group) + self.settings.string_list( + ['arch'], + 'Upload files from morphologies of ARCH', + metavar='ARCH', + default=[], + group=group) + def get_local_username(self): uid = os.getuid() return pwd.getpwuid(uid)[0] @@ -210,14 +218,33 @@ class BuildArtifactPublisher(object): argv += self.find_system_morphologies() output = cliapp.runcmd(argv) basenames = output.splitlines() + for basename in basenames: + logging.debug(' {0}'.format(basename)) return basenames def find_system_morphologies(self): cluster_morphology_pathname = 'release.morph' + systems = [] + sb = morphlib.sysbranchdir.open_from_within('.') + definitions = sb.get_git_directory_name(sb.root_repository_url) + defs_repo = morphlib.gitdir.GitDirectory(definitions) + loader = morphlib.morphloader.MorphologyLoader() + finder = morphlib.morphologyfinder.MorphologyFinder(defs_repo) with open(cluster_morphology_pathname) as f: obj = yaml.load(f) - return [system_dict['morph'] for system_dict in obj['systems']] + for system_dict in obj['systems']: + system_path = system_dict['morph'] + if self.settings['arch']: + path = morphlib.util.sanitise_morphology_path(system_path) + morph = loader.load_from_string( + finder.read_morphology(path)) + if morph['arch'] in self.settings['arch']: + systems.append(system_path) + else: + systems.append(system_path) + + return systems def filter_away_build_artifacts_on_public_trove(self, basenames): result = [] -- cgit v1.2.1 From 9b744d4b2536ec6c75f341bc4535eb93b203ee5a Mon Sep 17 00:00:00 2001 From: Michael Drake Date: Wed, 30 Jul 2014 15:06:07 +0100 Subject: Fix releasing when no changes to the cache are required Without this change the rsync and xargs commands will wait forever for input that will never arrive. --- scripts/release-upload | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/scripts/release-upload b/scripts/release-upload index f97e7e98..d3cc39e6 100755 --- a/scripts/release-upload +++ b/scripts/release-upload @@ -416,17 +416,20 @@ def rsync_files_to_server( '{user}@{host}:{path}'.format(user=user, host=host, path=target_dir), ] - files_list = ''.join( - '{0}\0'.format(filename) for filename in source_filenames) + files_list = '\0'.join( + filename for filename in source_filenames) + '\0' cliapp.runcmd(argv, feed_stdin=files_list, stdout=None, stderr=None) def set_permissions_on_server(user, host, target_dir, filenames): + # If we have no files, we can't form a valid command to run on the server + if not filenames: + return target = '{user}@{host}'.format(user=user, host=host) - argv = ['chmod', '0644'] - for filename in filenames: - argv.append(os.path.join(target_dir, filename)) - cliapp.ssh_runcmd(target, argv) + argv = ['xargs', '-0', 'chmod', '0644'] + files_list = ''.join( + '{0}\0'.format(os.path.join(target_dir, filename)) for filename in filenames) + cliapp.ssh_runcmd(target, argv, feed_stdin=files_list, stdout=None, stderr=None) ReleaseUploader(description=__doc__).run() -- cgit v1.2.1 From d91a916069b6a0acfbee21ce26e68633fa667258 Mon Sep 17 00:00:00 2001 From: Michael Drake Date: Wed, 30 Jul 2014 15:07:59 +0100 Subject: =?UTF-8?q?Allow=20release=20deployments=20with=20system=20name=20?= =?UTF-8?q?=E2=89=A0=20deployment=20name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There used to be a check that prevented deployments with names different to the system. I don't know why this was, but I don't think we need it. --- scripts/release-build | 45 +++++++++++++++++++++------------------------ 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/scripts/release-build b/scripts/release-build index fe10ba3b..7713aab8 100755 --- a/scripts/release-build +++ b/scripts/release-build @@ -131,32 +131,29 @@ class ReleaseApp(cliapp.Application): outputs = {} for system in cluster['systems']: - name = system['morph'] - morphology = self.load_morphology(name)[0] + morphology_name = system['morph'] + morphology = self.load_morphology(morphology_name)[0] if morphology['arch'] not in self.controllers: continue - if name not in system['deploy']: - raise cliapp.AppException( - 'In %s: system %s ID should be "%s"' % - (cluster_path, name, name)) - - # The release.morph cluster must specify a basename for the file, - # of name and extension. This script knows about name, but it - # can't find out the appropriate file extension without second - # guessing the behaviour of write extensions. - basename = system['deploy'][name]['location'] - - if '/' in basename or basename.startswith(version_label): - raise cliapp.AppException( - 'In %s: system %s.location should be just the base name, ' - 'e.g. "%s.img"' % (cluster_path, name, name)) - - filename = os.path.join('release', '%s-%s' % (version_label, basename)) - if os.path.exists(filename): - self.output.write('Reusing existing deployment of %s\n' % filename) - else: - self.output.write('Creating %s from release.morph\n' % filename) - self.deploy_single_image(cluster_path, name, filename, version_label) + + for deployment_name, deployment_info in system['deploy'].iteritems(): + # The release.morph cluster must specify a basename for the file, + # of name and extension. This script knows about name, but it + # can't find out the appropriate file extension without second + # guessing the behaviour of write extensions. + basename = deployment_info['location'] + + if '/' in basename or basename.startswith(version_label): + raise cliapp.AppException( + 'In %s: system %s.location should be just the base name, ' + 'e.g. "%s.img"' % (cluster_path, deployment_name, deployment_name)) + + filename = os.path.join('release', '%s-%s' % (version_label, basename)) + if os.path.exists(filename): + self.output.write('Reusing existing deployment of %s\n' % filename) + else: + self.output.write('Creating %s from release.morph\n' % filename) + self.deploy_single_image(cluster_path, deployment_name, filename, version_label) def deploy_single_image(self, cluster_path, name, location, version_label): deploy_command = [ -- cgit v1.2.1 From 3d950f7787a2de5c0d495642f9337bd998f2edd7 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 30 Jul 2014 16:53:26 +0000 Subject: Remove unnecessary leading _ from method --- scripts/release-build | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/release-build b/scripts/release-build index 7713aab8..36ecb423 100755 --- a/scripts/release-build +++ b/scripts/release-build @@ -108,16 +108,16 @@ class ReleaseApp(cliapp.Application): assert morph['kind'] == kind return morph, path - def _iterate_systems(self, system_list): + def iterate_systems(self, system_list): for system in system_list: yield system['morph'] if 'subsystems' in system: - for subsystem in self._iterate_systems(system['subsystems']): + for subsystem in self.iterate_systems(system['subsystems']): yield subsystem def prepare_builds(self, cluster): '''Prepare a list of builds''' - systems = set(self._iterate_systems(cluster['systems'])) + systems = set(self.iterate_systems(cluster['systems'])) builds = [] for system_name in systems: system, _ = self.load_morphology(system_name) -- cgit v1.2.1 From 497334fd9c0190437cefa9f089537be0556e3d2d Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 30 Jul 2014 16:55:50 +0000 Subject: Add --upload-release-artifacts setting --- scripts/release-upload | 22 +++++++++------------- 1 file changed, 9 insertions(+), 13 deletions(-) diff --git a/scripts/release-upload b/scripts/release-upload index d3cc39e6..68de800a 100755 --- a/scripts/release-upload +++ b/scripts/release-upload @@ -142,6 +142,11 @@ class ReleaseUploader(cliapp.Application): default=[], group=group) + self.settings.boolean( + ['upload-release-artifacts'], + 'upload release artifacts (disk images etc)?', + default=True) + def get_local_username(self): uid = os.getuid() return pwd.getpwuid(uid)[0] @@ -152,23 +157,14 @@ class ReleaseUploader(cliapp.Application): self.status).publish_build_artifacts() self.status(msg='Build artifiacts have been published') - RELEASE_OPTIONS = ('release-artifact-dir', - 'download-server-username', - 'download-server-address', - 'download-server-private-dir', - 'download-server-public-dir') - missing_options = set(option for option in RELEASE_OPTIONS - if self.settings[option] == '') - - if not missing_options: + if self.settings['upload-release-artifacts']: ReleaseArtifactPublisher(self.settings, self.status).publish_release_artifacts() self.status(msg='Release images have been published') else: - self.status(msg='Not uploading release images: ' - 'options not provided: {options}', - options=', '.join(missing_options)) - + self.status( + msg=('Not uploading release artifacts ' + '(--upload-release-artifacts not set)')) def status(self, msg, **kwargs): formatted = msg.format(**kwargs) -- cgit v1.2.1 From 389b259fb586454c3dc447613545c280d512fc51 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 30 Jul 2014 16:57:16 +0000 Subject: Refactor process_args to be clearer Move stuff into new methods to make overall logic clearer and to avoid stuffing too much into each method. --- scripts/release-upload | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/scripts/release-upload b/scripts/release-upload index 68de800a..cfbe06b2 100755 --- a/scripts/release-upload +++ b/scripts/release-upload @@ -153,18 +153,23 @@ class ReleaseUploader(cliapp.Application): def process_args(self, args): self.status(msg='Uploading and publishing Baserock release') - BuildArtifactPublisher(self.settings, - self.status).publish_build_artifacts() - self.status(msg='Build artifiacts have been published') - + self.publish_build_artifacts() if self.settings['upload-release-artifacts']: - ReleaseArtifactPublisher(self.settings, - self.status).publish_release_artifacts() - self.status(msg='Release images have been published') + self.publish_release_artifacts() else: self.status( - msg=('Not uploading release artifacts ' - '(--upload-release-artifacts not set)')) + msg='Not uploading release artifacts ' + '(upload-release-artifacts set to false') + + def publish_build_artifacts(self): + publisher = BuildArtifactPublisher(self.settings, self.status) + publisher.publish_build_artifacts() + self.status(msg='Build artifacts have been published') + + def publish_release_artifacts(self): + publisher = ReleaseArtifactPublisher(self.settings, self.status) + publisher.publish_release_artifacts() + self.status(msg='Release artifacts have been published') def status(self, msg, **kwargs): formatted = msg.format(**kwargs) -- cgit v1.2.1 From bedec8e16e987f7f4acb89ae7def5a702e1fcc9b Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 30 Jul 2014 16:58:34 +0000 Subject: Simplify logic and condition Avoiding a condition that has a negation tends to be a bit simpler for humans to understand. --- scripts/release-build | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/release-build b/scripts/release-build index 36ecb423..a3e25ebb 100755 --- a/scripts/release-build +++ b/scripts/release-build @@ -121,9 +121,8 @@ class ReleaseApp(cliapp.Application): builds = [] for system_name in systems: system, _ = self.load_morphology(system_name) - if system['arch'] not in self.controllers: - continue - builds.append(Build(system_name, system['arch'], self)) + if system['arch'] in self.controllers: + builds.append(Build(system_name, system['arch'], self)) return builds def deploy_images(self, cluster, cluster_path): -- cgit v1.2.1 From 39c006d33c81213f847c83425907297e72918471 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 30 Jul 2014 17:02:14 +0000 Subject: Add markers around list in debug log --- scripts/release-upload | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/release-upload b/scripts/release-upload index cfbe06b2..ad865001 100755 --- a/scripts/release-upload +++ b/scripts/release-upload @@ -219,8 +219,10 @@ class BuildArtifactPublisher(object): argv += self.find_system_morphologies() output = cliapp.runcmd(argv) basenames = output.splitlines() + logging.debug('List of build artifacts in release:') for basename in basenames: logging.debug(' {0}'.format(basename)) + logging.debug('End of list of build artifacts in release') return basenames -- cgit v1.2.1 From f3005dd222410f0fe8a75bd3e3b161ca2ef1cbbe Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 30 Jul 2014 17:03:25 +0000 Subject: Refactor long method into smaller ones For comprehensibility. --- scripts/release-upload | 56 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 37 insertions(+), 19 deletions(-) diff --git a/scripts/release-upload b/scripts/release-upload index ad865001..35630668 100755 --- a/scripts/release-upload +++ b/scripts/release-upload @@ -227,27 +227,45 @@ class BuildArtifactPublisher(object): return basenames def find_system_morphologies(self): - cluster_morphology_pathname = 'release.morph' - systems = [] + cluster = self.load_cluster_morphology('release.morph') + system_dicts = self.find_systems_in_parsed_cluster_morphology(cluster) + if self.settings['arch']: + system_dicts = self.choose_systems_for_wanted_architectures( + system_dicts, self.settings['arch']) + return [sd['morph'] for sd in system_dicts] + + def load_cluster_morphology(self, pathname): + with open(pathname) as f: + return yaml.load(f) + + def find_systems_in_parsed_cluster_morphology(self, cluster): + return cluster['systems'] + + def choose_systems_for_wanted_architectures(self, system_dicts, archs): + return [ + sd + for sd in system_dicts + if self.system_is_for_wanted_arch(sd, archs)] + + def system_is_for_wanted_arch(self, system_dict, archs): + morph = self.load_system_morphology(system_dict) + return morph['arch'] in archs + + def load_system_morphology(self, system_dict): + pathname = morphlib.util.sanitise_morphology_path(system_dict['morph']) + return self.load_morphology_from_named_file(pathname) + + def load_morphology_from_named_file(self, pathname): + finder = self.get_morphology_finder_for_root_repository() + morphology_text = finder.read_morphology(pathname) + loader = morphlib.morphloader.MorphologyLoader() + return loader.load_from_string(morphology_text) + + def get_morphology_finder_for_root_repository(self): sb = morphlib.sysbranchdir.open_from_within('.') definitions = sb.get_git_directory_name(sb.root_repository_url) - defs_repo = morphlib.gitdir.GitDirectory(definitions) - loader = morphlib.morphloader.MorphologyLoader() - finder = morphlib.morphologyfinder.MorphologyFinder(defs_repo) - with open(cluster_morphology_pathname) as f: - obj = yaml.load(f) - for system_dict in obj['systems']: - system_path = system_dict['morph'] - if self.settings['arch']: - path = morphlib.util.sanitise_morphology_path(system_path) - morph = loader.load_from_string( - finder.read_morphology(path)) - if morph['arch'] in self.settings['arch']: - systems.append(system_path) - else: - systems.append(system_path) - - return systems + definitions_repo = morphlib.gitdir.GitDirectory(definitions) + return morphlib.morphologyfinder.MorphologyFinder(definitions_repo) def filter_away_build_artifacts_on_public_trove(self, basenames): result = [] -- cgit v1.2.1 From 6e785c77ba7d7338d9541d425094e2bbf66b2275 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 30 Jul 2014 17:04:25 +0000 Subject: Avoid running rsync if source file list is empty --- scripts/release-upload | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/scripts/release-upload b/scripts/release-upload index 35630668..75030502 100755 --- a/scripts/release-upload +++ b/scripts/release-upload @@ -423,6 +423,9 @@ class ReleaseArtifactPublisher(object): def rsync_files_to_server( source_dir, source_filenames, user, host, target_dir): + if not source_filenames: + return + argv = [ 'rsync', '-a', @@ -438,7 +441,7 @@ def rsync_files_to_server( ] files_list = '\0'.join( - filename for filename in source_filenames) + '\0' + filename for filename in source_filenames) cliapp.runcmd(argv, feed_stdin=files_list, stdout=None, stderr=None) -- cgit v1.2.1 From f31353b4eb6fcb191a402b645bc3233d26e6527f Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 30 Jul 2014 17:04:41 +0000 Subject: Join short lines into one line, for readability --- scripts/release-upload | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/release-upload b/scripts/release-upload index 75030502..6711e725 100755 --- a/scripts/release-upload +++ b/scripts/release-upload @@ -440,8 +440,7 @@ def rsync_files_to_server( '{user}@{host}:{path}'.format(user=user, host=host, path=target_dir), ] - files_list = '\0'.join( - filename for filename in source_filenames) + files_list = '\0'.join(filename for filename in source_filenames) cliapp.runcmd(argv, feed_stdin=files_list, stdout=None, stderr=None) -- cgit v1.2.1 From d239aeed53a52e5dbc10aad6131334e51776ef95 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 30 Jul 2014 17:17:53 +0000 Subject: Add --upload-build-artifacts setting --- scripts/release-upload | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/scripts/release-upload b/scripts/release-upload index 6711e725..0ab6b957 100755 --- a/scripts/release-upload +++ b/scripts/release-upload @@ -142,6 +142,11 @@ class ReleaseUploader(cliapp.Application): default=[], group=group) + self.settings.boolean( + ['upload-build-artifacts'], + 'upload build artifacts?', + default=True) + self.settings.boolean( ['upload-release-artifacts'], 'upload release artifacts (disk images etc)?', @@ -153,7 +158,14 @@ class ReleaseUploader(cliapp.Application): def process_args(self, args): self.status(msg='Uploading and publishing Baserock release') - self.publish_build_artifacts() + + if self.settings['upload-build-artifacts']: + self.publish_build_artifacts() + else: + self.status( + msg='Not uploading build artifacts ' + '(upload-build-artifacts set to false') + if self.settings['upload-release-artifacts']: self.publish_release_artifacts() else: -- cgit v1.2.1