From adc082e1de3aaf8e3c3f0bb92a75b97962b1fce0 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Wed, 28 May 2014 08:47:44 +0000 Subject: Add release automation script. --- release/do-release.py | 444 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 444 insertions(+) create mode 100644 release/do-release.py diff --git a/release/do-release.py b/release/do-release.py new file mode 100644 index 00000000..e11e6625 --- /dev/null +++ b/release/do-release.py @@ -0,0 +1,444 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import cliapp +import morphlib +import yaml + +import contextlib +import gzip +import json +import logging +import os +import re +import sys +import tarfile +import urllib2 + + +''' do-release: Baserock release tooling. + +See: . + +''' + + +class config(object): + release_number = RELEASE NUMBER + + build_trove = 'hawkdevtrove' + release_trove = 'git.baserock.org' + + # Note that the 'location' field of the various systems in release.morph + # should match 'images_dir' here. + deploy_workspace = '/src/ws-release' + images_dir = '/src/release' + artifacts_dir = '/src/release/artifacts' + + # These locations should be appropriate 'staging' directories on the public + # servers that host images and artifacts. Remember not to upload to the + # public directories directly, or you risk exposing partially uploaded + # files. Once everything has uploaded you can 'mv' the release artifacts + # to the public directories in one quick operation. + # FIXME: we should probably warn if the dir exists and is not empty. + images_upload_location = \ + '@download.baserock.org:baserock-release-staging' + artifacts_upload_location = \ + 'root@git.baserock.org:/home/cache/baserock-release-staging' + + # The Codethink Manchester office currently has 8Mbits/s upload available. + # This setting ensures we use no more than half of the available bandwidth. + bandwidth_limit_kbytes_sec = 512 + + +def status(message, *args): + sys.stdout.write(message % args) + sys.stdout.write('\n') + + +@contextlib.contextmanager +def cwd(path): + ''' + Context manager to set current working directory.''' + old_cwd = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(old_cwd) + + +def transfer(f_in, f_out, block_size=10*1024*1024, show_status=True): + '''Stream from f_in to f_out until the end of f_in is reached. + + This function is rather like shutil.copyfileobj(), but it doesn't seem + possible to output progress info using that function. + + ''' + total_bytes = 0 + while True: + data = f_in.read(block_size) + total_bytes += len(data) + if len(data) == 0: + break + f_out.write(data) + if show_status: + sys.stdout.write( + '\rProcessed %iMB ...' % (total_bytes / (1024 * 1024))) + sys.stdout.flush() + if show_status: + sys.stdout.write('\rCompleted transfer\n') + + +class DeployImages(object): + '''Stage 1: deploy release images.''' + + def create_deploy_workspace(self, path): + '''Create or enter existing workspace for deploying release images.''' + + if not os.path.exists(path): + status('Creating workspace %s' % path) + cliapp.runcmd(['morph', 'init', path]) + else: + status('Reusing existing workspace %s' % path) + + repo = 'baserock:baserock/definitions' + branch = 'master' + + with cwd(path): + if not os.path.exists(branch): + status('Checking out %s branch %s' % (repo, branch)) + cliapp.runcmd(['morph', 'checkout', repo, branch]) + else: + status('Reusing checkout of %s %s' % (repo, branch)) + + definitions_dir = os.path.join( + config.deploy_workspace, branch, 'baserock/baserock/definitions') + + return definitions_dir + + def read_morph(self, filename, kind=None): + with open(filename) as f: + morph = yaml.load(f) + if kind is not None: + assert morph['kind'] == kind + return morph + + def parse_release_cluster(self, release_cluster): + '''Validate release cluster and list the systems being released. + + This function returns a dict mapping the system name to the location + of its deployed image. + + It's an open question how we should detect and handle the case where a + write extension creates more than one file. ARM kernels and GENIVI + manifest files are possible examples of this. + + ''' + + version_label = 'baserock-%s' % config.release_number + + outputs = {} + for system in release_cluster['systems']: + system_morph = system['morph'] + + if 'release' not in system['deploy']: + raise cliapp.AppException( + 'In release.morph: system %s ID should be "release"' % + system_morph) + + # We can't override 'location' with a different value. We must use + # what's already in the morphology, and check that it makes sense. + location = system['deploy']['release']['location'] + if not os.path.samefile(os.path.dirname(location), + config.images_dir): + raise cliapp.AppException( + 'In release.morph: system location %s is not inside ' + 'configured images_dir %s' % (location, config.images_dir)) + if not os.path.basename(location).startswith(version_label): + raise cliapp.AppException( + 'In release.morph: system image name %s does not start ' + 'with version label %s' % (location, version_label)) + + outputs[system_morph] = location + + return outputs + + def deploy_images(self, outputs): + '''Use `morph deploy` to create the release images.''' + + # FIXME: once `morph deploy` supports partial deployment, this should + # deploy only the images which aren't already deployed... it should + # also check if they need redeploying based on the SHA1 they were + # deployed from, perhaps. That's getting fancy! + + todo = [f for f in outputs.itervalues() if not os.path.exists(f)] + + if len(todo) == 0: + status('Reusing existing release images') + else: + logging.debug('Need to deploy images: %s' % ', '.join(todo)) + status('Creating release images from release.morph') + + version_label = 'baserock-%s' % config.release_number + + morph_config = ['--trove-host=%s' % config.build_trove] + deploy_config = ['release.VERSION_LABEL=%s' % version_label] + + cliapp.runcmd( + ['morph', 'deploy', 'release.morph'] + morph_config + + deploy_config, stdout=sys.stdout) + + def compress_images(self, outputs): + for name, source_file in outputs.iteritems(): + target_file = source_file + '.gz' + + if os.path.exists(target_file): + status('Reusing compressed image %s' % target_file) + else: + status('Compressing %s to %s', source_file, target_file) + with open(source_file, 'r') as f_in: + with gzip.open(target_file, 'w', compresslevel=4) as f_out: + transfer(f_in, f_out) + + outputs[name] = target_file + + def run(self): + definitions_dir = self.create_deploy_workspace(config.deploy_workspace) + + with cwd(definitions_dir): + release_cluster = self.read_morph('release.morph', kind='cluster') + + outputs = self.parse_release_cluster(release_cluster) + + with cwd(definitions_dir): + self.deploy_images(outputs) + + self.compress_images(outputs) + + return outputs + + +class PrepareArtifacts(object): + '''Stage 2: Fetch all artifacts and archive them. + + This includes the system artifacts. While these are large, it's very + helpful to have the system artifacts available in the trove.baserock.org + artifact cache because it allows users to deploy them with `morph deploy`. + If they are not available in the cache they must be built, which requires + access to a system of the same architecture as the target system. + + ''' + + def get_artifact_list(self, system_morphs): + '''Return list of artifacts involved in the release. + + List is also written to a file. + + Note that this function requires the `list-artifacts` command from + Morph of Baserock 14.23 or later. + + ''' + artifact_list_file = os.path.join( + config.artifacts_dir, 'baserock-%s-artifacts.txt' % + config.release_number) + if os.path.exists(artifact_list_file): + with open(artifact_list_file) as f: + artifact_basenames = [line.strip() for line in f] + else: + text = cliapp.runcmd( + ['morph', '--quiet', '--trove-host=%s' % config.build_trove, + 'list-artifacts', 'baserock:baserock/definitions', 'master'] + + system_morphs) + artifact_basenames = text.strip().split('\n') + with morphlib.savefile.SaveFile(artifact_list_file, 'w') as f: + f.write(text) + return artifact_list_file, artifact_basenames + + def query_remote_artifacts(self, trove, artifact_basenames): + url = 'http://%s:8080/1.0/artifacts' % trove + logging.debug('Querying %s' % url) + f = urllib2.urlopen(url, data=json.dumps(list(artifact_basenames))) + response = json.load(f) + return response + + def fetch_artifact(self, remote_cache, artifact): + f_in = remote_cache._get_file(artifact) + artifact_local = os.path.join(config.artifacts_dir, artifact) + with morphlib.savefile.SaveFile(artifact_local, 'wb') as f_out: + try: + logging.debug('Writing to %s' % artifact_local) + transfer(f_in, f_out) + except BaseException: + logging.debug( + 'Cleaning up %s after error' % artifact_local) + f_out.abort() + raise + f_in.close() + + def fetch_artifacts(self, artifact_basenames): + remote_cache = morphlib.remoteartifactcache.RemoteArtifactCache( + 'http://%s:8080' % config.build_trove) + found_artifacts = set() + + artifacts_to_query = [] + for artifact in artifact_basenames: + artifact_local = os.path.join(config.artifacts_dir, artifact) + # FIXME: no checksumming of artifacts done; we could get corruption + # introduced here and we would have no way of knowing. Cached + # artifact validation is planned for Morph; see: + # http://listmaster.pepperfish.net/pipermail/baserock-dev-baserock.org/2014-May/005675.html + if os.path.exists(artifact_local): + status('%s already cached' % artifact) + found_artifacts.add(artifact) + else: + artifacts_to_query.append(artifact) + + if len(artifacts_to_query) > 0: + result = self.query_remote_artifacts(config.build_trove, + artifacts_to_query) + for artifact, present in result.iteritems(): + if present: + status('Downloading %s from remote cache' % artifact) + self.fetch_artifact(remote_cache, artifact) + found_artifacts.add(artifact) + elif artifact.endswith('build-log'): + # For historical reasons, not all chunks have their + # build logs. Fixed here: + # http://git.baserock.org/cgi-bin/cgit.cgi/baserock/baserock/morph.git/commit/?id=6fb5fbad4f2876f30f482133c53f3a138911498b + # We still need to work around it for now, though. + logging.debug('Ignoring missing build log %s' % artifact) + elif re.match('[0-9a-f]{64}\.meta', artifact): + # FIXME: We still don't seem to share the .meta files. + # We should. Note that *artifact* meta files + # (.stratum.meta files) can't be ignored, they are an + # essential part of the stratum and it's an error if + # such a file is missing. + logging.debug('Ignoring missing source metadata %s' % + artifact) + else: + raise cliapp.AppException( + 'Remote artifact cache is missing artifact %s' % + artifact) + + return found_artifacts + + def prepare_artifacts_archive(self, tar_name, files): + if os.path.exists(tar_name): + status('Reusing tarball of artifacts at %s', tar_name) + else: + try: + status('Creating tarball of artifacts at %s', tar_name) + tar = tarfile.TarFile.gzopen(name=tar_name, mode='w', + compresslevel=4) + n_files = len(files) + for i, filename in enumerate(sorted(files)): + logging.debug('Add %s to tar file' % filename) + tar.add(filename, arcname=os.path.basename(filename)) + sys.stdout.write('\rAdded %i files of %i' % (i, n_files)) + sys.stdout.flush() + sys.stdout.write('\rFinished creating %s\n' % tar_name) + tar.close() + except BaseException: + logging.debug('Cleaning up %s after error' % tar_name) + os.unlink(tar_name) + raise + + def run(self, system_morphs): + if not os.path.exists(config.artifacts_dir): + os.makedirs(config.artifacts_dir) + + artifact_list_file, all_artifacts = \ + self.get_artifact_list(system_morphs) + + found_artifacts = self.fetch_artifacts(all_artifacts) + + tar_name = 'baserock-%s-artifacts.tar.gz' % config.release_number + artifacts_tar_file = os.path.join(config.artifacts_dir, tar_name) + artifact_files = [ + os.path.join(config.artifacts_dir, a) for a in found_artifacts] + + self.prepare_artifacts_archive(artifacts_tar_file, artifact_files) + + tar_name = 'baserock-%s-new-artifacts.tar.gz' % config.release_number + new_artifacts_tar_file = os.path.join(config.artifacts_dir, tar_name) + result = self.query_remote_artifacts(config.release_trove, + found_artifacts) + new_artifacts = [a for a, present in result.iteritems() if not present] + new_artifact_files = [ + os.path.join(config.artifacts_dir, a) for a in new_artifacts + if a.split('.')[1] != 'system'] + + self.prepare_artifacts_archive(new_artifacts_tar_file, + new_artifact_files) + + return (artifact_list_file, artifacts_tar_file, new_artifacts_tar_file) + + +class Upload(object): + '''Stage 3: upload images and artifacts to public servers.''' + + def run_rsync(self, sources, target): + if isinstance(sources, str): + sources = [sources] + settings = [ + '--bwlimit=%s' % config.bandwidth_limit_kbytes_sec, + '--partial', + '--progress', + ] + cliapp.runcmd( + ['rsync'] + settings + sources + [target], stdout=sys.stdout) + + def upload_release_images(self, images): + self.run_rsync(images, config.images_upload_location) + + def upload_artifacts(self, artifacts_list_file, artifacts_tar_file): + host, path = config.artifacts_upload_location.split(':', 1) + + self.run_rsync([artifacts_list_file, artifacts_tar_file], + config.artifacts_upload_location) + + # UGH! Perhaps morph-cache-server should grow an authorised-users-only + # API call receive artifacts, to avoid this. + remote_artifacts_tar = os.path.join( + path, os.path.basename(artifacts_tar_file)) + extract_tar_cmd = 'cd "%s" && tar xf "%s" && chown cache:cache *' % \ + (path, remote_artifacts_tar) + cliapp.ssh_runcmd( + host, ['sh', '-c', extract_tar_cmd]) + + +def main(): + logging.basicConfig(level=logging.DEBUG) + + deploy_images = DeployImages() + outputs = deploy_images.run() + + prepare_artifacts = PrepareArtifacts() + artifacts_list_file, artifacts_tar_file, new_artifacts_tar_file = \ + prepare_artifacts.run(outputs.keys()) + + upload = Upload() + upload.upload_release_images(outputs.values()) + upload.upload_artifacts(artifacts_list_file, new_artifacts_tar_file) + + sys.stdout.writelines([ + '\nPreparation for %s release complete!\n' % config.release_number, + 'Images uploaded to %s\n' % config.images_upload_location, + 'Artifacts uploaded to %s\n' % config.artifacts_upload_location + ]) + + +main() -- cgit v1.2.1