summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLars Wirzenius <lars.wirzenius@codethink.co.uk>2014-07-30 17:21:58 +0000
committerLars Wirzenius <lars.wirzenius@codethink.co.uk>2014-07-30 17:21:58 +0000
commit0946a88d22c25d9678feecbb1d80254508b60256 (patch)
treeb3628b8f1660d7ef7f514a21f24e487107a4b7d1
parentdcbf761d786bd67ee77f996bc6f4d6befb489dc7 (diff)
parentd239aeed53a52e5dbc10aad6131334e51776ef95 (diff)
downloaddefinitions-0946a88d22c25d9678feecbb1d80254508b60256.tar.gz
Merge branch 'baserock/liw/merge-michaels-ci-release-scripts'
This is a merge of the baserock/michaeldrake/ci-release-scripts branch, but with some cleanups of the code for cleanliness.
-rwxr-xr-xscripts/release-build59
-rwxr-xr-xscripts/release-upload109
2 files changed, 129 insertions, 39 deletions
diff --git a/scripts/release-build b/scripts/release-build
index 36a38deb..a3e25ebb 100755
--- a/scripts/release-build
+++ b/scripts/release-build
@@ -98,7 +98,6 @@ class ReleaseApp(cliapp.Application):
os.chdir('..')
if not os.path.exists('release'):
os.mkdir('release')
- os.chdir('release')
self.deploy_images(cluster, cluster_path)
def load_morphology(self, name, kind=None):
@@ -109,13 +108,21 @@ class ReleaseApp(cliapp.Application):
assert morph['kind'] == kind
return morph, path
+ def iterate_systems(self, system_list):
+ for system in system_list:
+ yield system['morph']
+ if 'subsystems' in system:
+ for subsystem in self.iterate_systems(system['subsystems']):
+ yield subsystem
+
def prepare_builds(self, cluster):
'''Prepare a list of builds'''
- systems = [system['morph'] for system in cluster['systems']]
+ systems = set(self.iterate_systems(cluster['systems']))
builds = []
for system_name in systems:
system, _ = self.load_morphology(system_name)
- builds.append(Build(system_name, system['arch'], self))
+ if system['arch'] in self.controllers:
+ builds.append(Build(system_name, system['arch'], self))
return builds
def deploy_images(self, cluster, cluster_path):
@@ -123,29 +130,29 @@ class ReleaseApp(cliapp.Application):
outputs = {}
for system in cluster['systems']:
- name = system['morph']
- if name not in system['deploy']:
- raise cliapp.AppException(
- 'In %s: system %s ID should be "%s"' %
- (cluster_path, name, name))
-
- # The release.morph cluster must specify a basename for the file,
- # of name and extension. This script knows about name, but it
- # can't find out the appropriate file extension without second
- # guessing the behaviour of write extensions.
- basename = system['deploy'][name]['location']
-
- if '/' in basename or basename.startswith(version_label):
- raise cliapp.AppException(
- 'In %s: system %s.location should be just the base name, '
- 'e.g. "%s.img"' % (cluster_path, name, name))
-
- filename = '%s-%s' % (version_label, basename)
- if os.path.exists(filename):
- self.output.write('Reusing existing deployment of %s\n' % filename)
- else:
- self.output.write('Creating %s from release.morph\n' % filename)
- self.deploy_single_image(cluster_path, name, filename, version_label)
+ morphology_name = system['morph']
+ morphology = self.load_morphology(morphology_name)[0]
+ if morphology['arch'] not in self.controllers:
+ continue
+
+ for deployment_name, deployment_info in system['deploy'].iteritems():
+ # The release.morph cluster must specify a basename for the file,
+ # of name and extension. This script knows about name, but it
+ # can't find out the appropriate file extension without second
+ # guessing the behaviour of write extensions.
+ basename = deployment_info['location']
+
+ if '/' in basename or basename.startswith(version_label):
+ raise cliapp.AppException(
+ 'In %s: system %s.location should be just the base name, '
+ 'e.g. "%s.img"' % (cluster_path, deployment_name, deployment_name))
+
+ filename = os.path.join('release', '%s-%s' % (version_label, basename))
+ if os.path.exists(filename):
+ self.output.write('Reusing existing deployment of %s\n' % filename)
+ else:
+ self.output.write('Creating %s from release.morph\n' % filename)
+ self.deploy_single_image(cluster_path, deployment_name, filename, version_label)
def deploy_single_image(self, cluster_path, name, location, version_label):
deploy_command = [
diff --git a/scripts/release-upload b/scripts/release-upload
index 2e7f54e8..0ab6b957 100755
--- a/scripts/release-upload
+++ b/scripts/release-upload
@@ -49,6 +49,7 @@ import urlparse
import cliapp
import yaml
+import morphlib
class ReleaseUploader(cliapp.Application):
@@ -134,15 +135,53 @@ class ReleaseUploader(cliapp.Application):
default='morph',
group=group)
+ self.settings.string_list(
+ ['arch'],
+ 'Upload files from morphologies of ARCH',
+ metavar='ARCH',
+ default=[],
+ group=group)
+
+ self.settings.boolean(
+ ['upload-build-artifacts'],
+ 'upload build artifacts?',
+ default=True)
+
+ self.settings.boolean(
+ ['upload-release-artifacts'],
+ 'upload release artifacts (disk images etc)?',
+ default=True)
+
def get_local_username(self):
uid = os.getuid()
return pwd.getpwuid(uid)[0]
def process_args(self, args):
self.status(msg='Uploading and publishing Baserock release')
- BuildArtifactPublisher(self.settings, self.status).publish_build_artifacts()
- ReleaseArtifactPublisher(self.settings, self.status).publish_release_artifacts()
- self.status(msg='Release has been uploaded and published')
+
+ if self.settings['upload-build-artifacts']:
+ self.publish_build_artifacts()
+ else:
+ self.status(
+ msg='Not uploading build artifacts '
+ '(upload-build-artifacts set to false')
+
+ if self.settings['upload-release-artifacts']:
+ self.publish_release_artifacts()
+ else:
+ self.status(
+ msg='Not uploading release artifacts '
+ '(upload-release-artifacts set to false')
+
+ def publish_build_artifacts(self):
+ publisher = BuildArtifactPublisher(self.settings, self.status)
+ publisher.publish_build_artifacts()
+ self.status(msg='Build artifacts have been published')
+
+ def publish_release_artifacts(self):
+ publisher = ReleaseArtifactPublisher(self.settings, self.status)
+ publisher.publish_release_artifacts()
+ self.status(msg='Release artifacts have been published')
def status(self, msg, **kwargs):
formatted = msg.format(**kwargs)
@@ -192,14 +231,53 @@ class BuildArtifactPublisher(object):
argv += self.find_system_morphologies()
output = cliapp.runcmd(argv)
basenames = output.splitlines()
+ logging.debug('List of build artifacts in release:')
+ for basename in basenames:
+ logging.debug(' {0}'.format(basename))
+ logging.debug('End of list of build artifacts in release')
return basenames
def find_system_morphologies(self):
- cluster_morphology_pathname = 'release.morph'
- with open(cluster_morphology_pathname) as f:
- obj = yaml.load(f)
- return [system_dict['morph'] for system_dict in obj['systems']]
+ cluster = self.load_cluster_morphology('release.morph')
+ system_dicts = self.find_systems_in_parsed_cluster_morphology(cluster)
+ if self.settings['arch']:
+ system_dicts = self.choose_systems_for_wanted_architectures(
+ system_dicts, self.settings['arch'])
+ return [sd['morph'] for sd in system_dicts]
+
+ def load_cluster_morphology(self, pathname):
+ with open(pathname) as f:
+ return yaml.load(f)
+
+ def find_systems_in_parsed_cluster_morphology(self, cluster):
+ return cluster['systems']
+
+ def choose_systems_for_wanted_architectures(self, system_dicts, archs):
+ return [
+ sd
+ for sd in system_dicts
+ if self.system_is_for_wanted_arch(sd, archs)]
+
+ def system_is_for_wanted_arch(self, system_dict, archs):
+ morph = self.load_system_morphology(system_dict)
+ return morph['arch'] in archs
+
+ def load_system_morphology(self, system_dict):
+ pathname = morphlib.util.sanitise_morphology_path(system_dict['morph'])
+ return self.load_morphology_from_named_file(pathname)
+
+ def load_morphology_from_named_file(self, pathname):
+ finder = self.get_morphology_finder_for_root_repository()
+ morphology_text = finder.read_morphology(pathname)
+ loader = morphlib.morphloader.MorphologyLoader()
+ return loader.load_from_string(morphology_text)
+
+ def get_morphology_finder_for_root_repository(self):
+ sb = morphlib.sysbranchdir.open_from_within('.')
+ definitions = sb.get_git_directory_name(sb.root_repository_url)
+ definitions_repo = morphlib.gitdir.GitDirectory(definitions)
+ return morphlib.morphologyfinder.MorphologyFinder(definitions_repo)
def filter_away_build_artifacts_on_public_trove(self, basenames):
result = []
@@ -357,6 +435,9 @@ class ReleaseArtifactPublisher(object):
def rsync_files_to_server(
source_dir, source_filenames, user, host, target_dir):
+ if not source_filenames:
+ return
+
argv = [
'rsync',
'-a',
@@ -371,17 +452,19 @@ def rsync_files_to_server(
'{user}@{host}:{path}'.format(user=user, host=host, path=target_dir),
]
- files_list = ''.join(
- '{0}\0'.format(filename) for filename in source_filenames)
+ files_list = '\0'.join(filename for filename in source_filenames)
cliapp.runcmd(argv, feed_stdin=files_list, stdout=None, stderr=None)
def set_permissions_on_server(user, host, target_dir, filenames):
+ # If we have no files, we can't form a valid command to run on the server
+ if not filenames:
+ return
target = '{user}@{host}'.format(user=user, host=host)
- argv = ['chmod', '0644']
- for filename in filenames:
- argv.append(os.path.join(target_dir, filename))
- cliapp.ssh_runcmd(target, argv)
+ argv = ['xargs', '-0', 'chmod', '0644']
+ files_list = ''.join(
+ '{0}\0'.format(os.path.join(target_dir, filename)) for filename in filenames)
+ cliapp.ssh_runcmd(target, argv, feed_stdin=files_list, stdout=None, stderr=None)
ReleaseUploader(description=__doc__).run()