From 950d6dddfe9cbf294e01973cc39040181f5525e5 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 1 Jul 2014 16:41:48 +0000 Subject: do-release: take advantage of partial deployment The 'morph deploy' command now lets us deploy system images one at a time, so let's do that. This means that if all but one image is deployed successfully, on the next run the user just needs to deploy one further image. Also, since each deployment has a unique name in release.morph now, we can override the location and VERSION_LABEL fields instead of requiring the user to update them manually before each release. The release.morph cluster should now specify the *basename* of the image in the location field only. By basename, I mean the system name plus the appropriate extension (normally .tar or .img). The do-release script will then prepend the image path and the version label to get a filename. The release.morph cluster has been updated accordingly. --- release.morph | 10 +++--- scripts/do-release.py | 87 +++++++++++++++++++-------------------------------- 2 files changed, 37 insertions(+), 60 deletions(-) diff --git a/release.morph b/release.morph index 4ec38837..cb069f23 100644 --- a/release.morph +++ b/release.morph @@ -7,23 +7,21 @@ systems: deploy: devel-system-x86_32-chroot: type: tar - location: /src/release/baserock-14.26-devel-system-x86_32-chroot.tar + location: devel-system-x86_32-chroot.tar - morph: devel-system-x86_32-generic deploy: devel-system-x86_32-generic: type: rawdisk - location: /src/release/baserock-14.26-devel-system-x86_32-generic.img + location: devel-system-x86_32-generic.img DISK_SIZE: 4G - VERSION_LABEL: baserock-14.26 - morph: devel-system-x86_64-chroot deploy: devel-system-x86_64-chroot: type: tar - location: /src/release/baserock-14.26-devel-system-x86_64-chroot.tar + location: devel-system-x86_64-chroot.tar - morph: devel-system-x86_64-generic deploy: devel-system-x86_64-generic: type: rawdisk - location: /src/release/baserock-14.26-devel-system-x86_64-generic.img + location: devel-system-x86_64-generic.img DISK_SIZE: 4G - VERSION_LABEL: baserock-14.26 diff --git a/scripts/do-release.py b/scripts/do-release.py index e8efe814..43ce7201 100644 --- a/scripts/do-release.py +++ b/scripts/do-release.py @@ -137,70 +137,53 @@ class DeployImages(object): assert morph['kind'] == kind return morph - def parse_release_cluster(self, release_cluster): - '''Validate release cluster and list the systems being released. - - This function returns a dict mapping the system name to the location - of its deployed image. + def deploy_single_image(self, system_name, location, version_label): + deploy_command = [ + 'morph', 'deploy', 'release.morph', system_name, + '--trove-host=%s' % config.build_trove, + '%s.location=%s' % (system_name, location), + '%s.VERSION_LABEL=%s' % (system_name, version_label) + ] - It's an open question how we should detect and handle the case where a - write extension creates more than one file. ARM kernels and GENIVI - manifest files are possible examples of this. + cliapp.runcmd(deploy_command, stdout=sys.stdout) - ''' + def deploy_images(self, release_cluster): + '''Use `morph deploy` to create the release images.''' version_label = 'baserock-%s' % config.release_number - outputs = {} + for system in release_cluster['systems']: - system_morph = system['morph'] + system_name = system['morph'] - if system_morph not in system['deploy']: + if system_name not in system['deploy']: raise cliapp.AppException( 'In release.morph: system %s ID should be "%s"' % - (system_morph, system_morph)) + (system_name, system_name)) - # We can't override 'location' with a different value. We must use - # what's already in the morphology, and check that it makes sense. - location = system['deploy'][system_morph]['location'] - if not os.path.samefile(os.path.dirname(location), - config.images_dir): - raise cliapp.AppException( - 'In release.morph: system location %s is not inside ' - 'configured images_dir %s' % (location, config.images_dir)) - if not os.path.basename(location).startswith(version_label): - raise cliapp.AppException( - 'In release.morph: system image name %s does not start ' - 'with version label %s' % (location, version_label)) - - outputs[system_morph] = location - - return outputs + # The release.morph cluster must specify a basename for the file, + # of system-name + extension. This script knows system-name, but it + # can't find out the appropriate file extension without + # second-guessing the behaviour of write extensions. + basename = system['deploy'][system_name]['location'] - def deploy_images(self, outputs): - '''Use `morph deploy` to create the release images.''' - - # FIXME: once `morph deploy` supports partial deployment, this should - # deploy only the images which aren't already deployed... it should - # also check if they need redeploying based on the SHA1 they were - # deployed from, perhaps. That's getting fancy! - - todo = [f for f in outputs.itervalues() if not os.path.exists(f)] + if '/' in basename or basename.startswith(version_label): + raise cliapp.AppException( + 'In release.morph: system %s.location should be just the ' + 'base name, e.g. "%s.img"' % (system_name, system_name)) - if len(todo) == 0: - status('Reusing existing release images') - else: - logging.debug('Need to deploy images: %s' % ', '.join(todo)) - status('Creating release images from release.morph') + filename = '%s-%s' % (version_label, basename) + location = os.path.join(config.images_dir, filename) - version_label = 'baserock-%s' % config.release_number + if os.path.exists(location): + status('Reusing existing deployment of %s', filename) + else: + status('Creating %s from release.morph', filename) + self.deploy_single_image(system_name, location, version_label) - morph_config = ['--trove-host=%s' % config.build_trove] - deploy_config = ['release.VERSION_LABEL=%s' % version_label] + outputs[system_name] = location - cliapp.runcmd( - ['morph', 'deploy', 'release.morph'] + morph_config + - deploy_config, stdout=sys.stdout) + return outputs def compress_images(self, outputs): for name, source_file in outputs.iteritems(): @@ -221,11 +204,7 @@ class DeployImages(object): with cwd(definitions_dir): release_cluster = self.read_morph('release.morph', kind='cluster') - - outputs = self.parse_release_cluster(release_cluster) - - with cwd(definitions_dir): - self.deploy_images(outputs) + outputs = self.deploy_images(release_cluster) self.compress_images(outputs) -- cgit v1.2.1