From 743152f1662498afcf395e02095fb40b1d711393 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Fri, 27 Feb 2015 16:20:21 +0000 Subject: Fix copyright years --- morphlib/writeexts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/morphlib/writeexts.py b/morphlib/writeexts.py index ad4fabe9..ab451d14 100644 --- a/morphlib/writeexts.py +++ b/morphlib/writeexts.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -- cgit v1.2.1 From 7fde92e3a77e6416a645e76fef37284c8dbd7d1e Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 10 Feb 2015 17:06:59 +0000 Subject: Only update Git submodules in cache when necessary This saves a duplicate `git remote update origin` that was being run as part of each chunk build. For any repos that have submodules, it also avoids updating repos if the SHA1 we need to build is already present locally. As well as speeding up builds slightly, this means Morph can now build without being connected to a network, as long as the local Git cache all of the necessary repos and commits in the build, without needing the '--no-git-update' option. The code is also now in a more logical place than before. --- morphlib/app.py | 20 ------------ morphlib/buildcommand.py | 35 ++------------------- morphlib/localrepocache.py | 76 ++++++++++++++++++++++++++++++++++++++-------- 3 files changed, 65 insertions(+), 66 deletions(-) diff --git a/morphlib/app.py b/morphlib/app.py index 0c87f814..b8bae850 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -297,26 +297,6 @@ class Morph(cliapp.Application): morphlib.util.sanitise_morphology_path(args[2])) args = args[3:] - def cache_repo_and_submodules(self, cache, url, ref, done): - subs_to_process = set() - subs_to_process.add((url, ref)) - while subs_to_process: - url, ref = subs_to_process.pop() - done.add((url, ref)) - cached_repo = cache.cache_repo(url) - cached_repo.update() - - try: - submodules = morphlib.git.Submodules(self, cached_repo.path, - ref) - submodules.load() - except morphlib.git.NoModulesFileError: - pass - else: - for submod in submodules: - if (submod.url, submod.commit) not in done: - subs_to_process.add((submod.url, submod.commit)) - def _write_status(self, text): timestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()) self.output.write('%s %s\n' % (timestamp, text)) diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 0aa50a3b..8572450d 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -387,39 +387,8 @@ class BuildCommand(object): '''Update the local git repository cache with the sources.''' repo_name = source.repo_name - if self.app.settings['no-git-update']: - self.app.status(msg='Not updating existing git repository ' - '%(repo_name)s ' - 'because of no-git-update being set', - chatty=True, - repo_name=repo_name) - source.repo = self.lrc.get_repo(repo_name) - return - - if self.lrc.has_repo(repo_name): - source.repo = self.lrc.get_repo(repo_name) - try: - sha1 = source.sha1 - source.repo.resolve_ref_to_commit(sha1) - self.app.status(msg='Not updating git repository ' - '%(repo_name)s because it ' - 'already contains sha1 %(sha1)s', - chatty=True, repo_name=repo_name, - sha1=sha1) - except morphlib.gitdir.InvalidRefError: - self.app.status(msg='Updating %(repo_name)s', - repo_name=repo_name) - source.repo.update() - else: - self.app.status(msg='Cloning %(repo_name)s', - repo_name=repo_name) - source.repo = self.lrc.cache_repo(repo_name) - - # Update submodules. - done = set() - self.app.cache_repo_and_submodules( - self.lrc, source.repo.url, - source.sha1, done) + source.repo = self.lrc.get_updated_repo(repo_name, ref=source.sha1) + self.lrc.ensure_submodules(source.repo, source.sha1) def cache_artifacts_locally(self, artifacts): '''Get artifacts missing from local cache from remote cache.''' diff --git a/morphlib/localrepocache.py b/morphlib/localrepocache.py index 39fbd200..1565b913 100644 --- a/morphlib/localrepocache.py +++ b/morphlib/localrepocache.py @@ -14,10 +14,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -import logging import os -import re -import urllib2 import urlparse import string import sys @@ -246,15 +243,68 @@ class LocalRepoCache(object): return repo raise NotCached(reponame) - def get_updated_repo(self, reponame): # pragma: no cover - '''Return object representing cached repository, which is updated.''' + def get_updated_repo(self, repo_name, ref=None): # pragma: no cover + '''Return object representing cached repository. - if not self._app.settings['no-git-update']: - cached_repo = self.cache_repo(reponame) - self._app.status( - msg='Updating git repository %s in cache' % reponame) - cached_repo.update() - else: - cached_repo = self.get_repo(reponame) - return cached_repo + If 'ref' is None, the repo will be updated unless + app.settings['no-git-update'] is set. + + If 'ref' is set to a SHA1, the repo will only be updated if 'ref' isn't + already available locally. + ''' + + if self._app.settings['no-git-update']: + self._app.status(msg='Not updating existing git repository ' + '%(repo_name)s ' + 'because of no-git-update being set', + chatty=True, + repo_name=repo_name) + return self.get_repo(repo_name) + + if self.has_repo(repo_name): + repo = self.get_repo(repo_name) + if ref and morphlib.git.is_valid_sha1(ref): + try: + repo.resolve_ref_to_commit(ref) + self._app.status(msg='Not updating git repository ' + '%(repo_name)s because it ' + 'already contains sha1 %(sha1)s', + chatty=True, repo_name=repo_name, + sha1=ref) + return repo + except morphlib.gitdir.InvalidRefError: + pass + + self._app.status(msg='Updating %(repo_name)s', + repo_name=repo_name) + repo.update() + return repo + else: + self._app.status(msg='Cloning %(repo_name)s', + repo_name=repo_name) + return self.cache_repo(repo_name) + + def ensure_submodules(self, toplevel_repo, + toplevel_ref): # pragma: no cover + '''Ensure any submodules of a given repo are cached and up to date.''' + + def submodules_for_repo(repo_path, ref): + try: + submodules = morphlib.git.Submodules(self._app, repo_path, ref) + submodules.load() + return [(submod.url, submod.commit) for submod in submodules] + except morphlib.git.NoModulesFileError: + return [] + + done = set() + subs_to_process = submodules_for_repo(toplevel_repo.path, toplevel_ref) + while subs_to_process: + url, ref = subs_to_process.pop() + done.add((url, ref)) + + cached_repo = self.get_updated_repo(url, ref=ref) + + for submod in submodules_for_repo(cached_repo.path, ref): + if (submod.url, submod.commit) not in done: + subs_to_process.add((submod.url, submod.commit)) -- cgit v1.2.1 From 5955559f87c8516ab496f99f56fd07eaf452c80f Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 3 Feb 2015 15:27:19 +0000 Subject: distbuild: Be more robust when a worker disconnects The logic to handle a worker disconnecting was broken. The WorkerConnection object would remove itself from the main loop as soon as the worker disconnected. But it would not get removed from the list of available workers that the WorkerBuildQueue maintains. So the controller would continue sending messages to this dead connection, and the builds it sent would hang forever for a response. --- distbuild/worker_build_scheduler.py | 43 ++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 8 deletions(-) diff --git a/distbuild/worker_build_scheduler.py b/distbuild/worker_build_scheduler.py index be732153..4f7ff98f 100644 --- a/distbuild/worker_build_scheduler.py +++ b/distbuild/worker_build_scheduler.py @@ -1,6 +1,6 @@ # distbuild/worker_build_scheduler.py -- schedule worker-builds on workers # -# Copyright (C) 2012, 2014 Codethink Limited +# Copyright (C) 2012, 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -93,6 +93,12 @@ class _HaveAJob(object): def __init__(self, job): self.job = job +class _Disconnected(object): + + def __init__(self, who): + self.who = who + + class Job(object): def __init__(self, job_id, artifact, initiator_id): @@ -220,7 +226,10 @@ class WorkerBuildQueuer(distbuild.StateMachine): ('idle', WorkerConnection, _JobFinished, 'idle', self._set_job_finished), ('idle', WorkerConnection, _JobFailed, 'idle', - self._set_job_failed) + self._set_job_failed), + + ('idle', WorkerConnection, _Disconnected, 'idle', + self._handle_worker_disconnected), ] self.add_transitions(spec) @@ -355,8 +364,22 @@ class WorkerBuildQueuer(distbuild.StateMachine): (job.artifact.name, worker.who.name())) self.mainloop.queue_event(worker.who, _HaveAJob(job)) - - + + def _handle_worker_disconnected(self, event): + self._remove_worker(self, event.who) + + def _remove_worker(self, worker): + logging.debug('WBQ: Removing worker %s from queue', worker.name()) + + # There should only be one InitiatorConnection instance per worker in + # the _available_workers list. But anything can happen in space! So we + # take care to remove all GiveJob messages in the list that came from + # the disconnected worker, not the first. + self._available_workers = filter( + lambda worker_msg: worker_msg.who != worker, + self._available_workers) + + class WorkerConnection(distbuild.StateMachine): '''Communicate with a single worker.''' @@ -397,14 +420,15 @@ class WorkerConnection(distbuild.StateMachine): spec = [ # state, source, event_class, new_state, callback - ('idle', self._jm, distbuild.JsonEof, None, self._reconnect), + ('idle', self._jm, distbuild.JsonEof, None, self._disconnected), ('idle', self, _HaveAJob, 'building', self._start_build), ('building', distbuild.BuildController, distbuild.BuildCancel, 'building', self._maybe_cancel), - ('building', self._jm, distbuild.JsonEof, None, self._reconnect), + ('building', self._jm, distbuild.JsonEof, None, + self._disconnected), ('building', self._jm, distbuild.JsonNewMessage, 'building', self._handle_json_message), ('building', self, _BuildFailed, 'idle', self._request_job), @@ -412,6 +436,7 @@ class WorkerConnection(distbuild.StateMachine): ('building', self, _BuildFinished, 'caching', self._request_caching), + ('caching', self._jm, distbuild.JsonEof, None, self._disconnected), ('caching', distbuild.HelperRouter, distbuild.HelperResult, 'caching', self._maybe_handle_helper_result), ('caching', self, _Cached, 'idle', self._request_job), @@ -449,10 +474,12 @@ class WorkerConnection(distbuild.StateMachine): self._job.initiators.remove(build_cancel.id) - def _reconnect(self, event_source, event): + def _disconnected(self, event_source, event): distbuild.crash_point() - logging.debug('WC: Triggering reconnect') + logging.debug('WC: Disconnected from worker %s' % self.name()) + self.mainloop.queue_event(InitiatorConnection, _Disconnected(self)) + self.mainloop.queue_event(self._cm, distbuild.Reconnect()) def _start_build(self, event_source, event): -- cgit v1.2.1 From c96dfa28f2b6f5c7aeb482c505ae010b607b1305 Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Wed, 28 Jan 2015 11:32:07 +0000 Subject: Check file can be created at location --- morphlib/exts/kvm.check | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/morphlib/exts/kvm.check b/morphlib/exts/kvm.check index 1bb4007a..3c6accbf 100755 --- a/morphlib/exts/kvm.check +++ b/morphlib/exts/kvm.check @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -43,6 +43,7 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): ssh_host, vm_name, vm_path = self.check_and_parse_location(location) self.check_ssh_connectivity(ssh_host) + self.check_can_create_file_at_given_path(ssh_host, vm_path) self.check_no_existing_libvirt_vm(ssh_host, vm_name) self.check_extra_disks_exist(ssh_host, self.parse_attach_disks()) @@ -73,6 +74,26 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): 'write extension to deploy upgrades to existing machines.' % (ssh_host, vm_name)) + def check_can_create_file_at_given_path(self, ssh_host, vm_path): + + def check_can_write_to_given_path(): + try: + cliapp.ssh_runcmd(ssh_host, ['touch', vm_path]) + except cliapp.AppException as e: + raise cliapp.AppException("Can't write to location %s on %s" + % (vm_path, ssh_host)) + else: + cliapp.ssh_runcmd(ssh_host, ['rm', vm_path]) + + try: + cliapp.ssh_runcmd(ssh_host, ['test', '-e', vm_path]) + except cliapp.AppException as e: + # vm_path doesn't already exist, so let's test we can write + check_can_write_to_given_path() + else: + raise cliapp.AppException('%s already exists on %s' + % (vm_path, ssh_host)) + def check_extra_disks_exist(self, ssh_host, filename_list): for filename in filename_list: try: -- cgit v1.2.1 From 774d708f969553a7f571b734c21642c19605385a Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Wed, 28 Jan 2015 17:28:00 +0000 Subject: Add check for virtual networks An exception will be raised if any needed networks are not started --- morphlib/exts/kvm.check | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/morphlib/exts/kvm.check b/morphlib/exts/kvm.check index 3c6accbf..b8877a89 100755 --- a/morphlib/exts/kvm.check +++ b/morphlib/exts/kvm.check @@ -17,6 +17,7 @@ '''Preparatory checks for Morph 'kvm' write extension''' import cliapp +import os import re import urlparse @@ -46,6 +47,7 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): self.check_can_create_file_at_given_path(ssh_host, vm_path) self.check_no_existing_libvirt_vm(ssh_host, vm_name) self.check_extra_disks_exist(ssh_host, self.parse_attach_disks()) + self.check_virtual_networks_are_started(ssh_host) def check_and_parse_location(self, location): '''Check and parse the location argument to get relevant data.''' @@ -102,4 +104,50 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): raise cliapp.AppException('Did not find file %s on host %s' % (filename, ssh_host)) + def check_virtual_networks_are_started(self, ssh_host): + + def check_virtual_network_is_started(network_name): + cmd = ['virsh', '-c', 'qemu:///system', 'net-info', network_name] + net_info = cliapp.ssh_runcmd(ssh_host, cmd).split('\n') + + def pretty_concat(lines): + return '\n'.join(['\t%s' % line for line in lines]) + + for line in net_info: + m = re.match('^Active:\W*(\w+)\W*', line) + if m: + break + else: + raise cliapp.AppException( + "Got unexpected output parsing output of `%s':\n%s" + % (' '.join(cmd), pretty_concat(net_info))) + + network_active = m.group(1) == 'yes' + + if not network_active: + raise cliapp.AppException("Network '%s' is not started" + % network_name) + + def name(nic_entry): + if ',' in nic_entry: + # NETWORK_NAME,mac=12:34,model=e1000... + return nic_entry[:nic_entry.find(',')] + else: + return nic_entry # NETWORK_NAME + + if 'NIC_CONFIG' in os.environ: + nics = os.environ['NIC_CONFIG'].split() + + # --network bridge= is used to specify a bridge + # --network user is used to specify a form of NAT + # (see the virt-install(1) man page) + networks = [name(n) for n in nics if not n.startswith('bridge=') + and not n.startswith('user')] + else: + networks = ['default'] + + for network in networks: + check_virtual_network_is_started(network) + + KvmPlusSshCheckExtension().run() -- cgit v1.2.1 From 8221465890030d95fd7d32e9de2353d8801daf54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 27 Feb 2015 15:23:09 +0000 Subject: morphlib/sourceresolver.py: Add _get_file_contents_from_repo() And make _get_morphology_from_repo() use it --- morphlib/sourceresolver.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index 22e643d2..6347861e 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -219,7 +219,8 @@ class SourceResolver(object): else: return None - def _get_morphology_from_repo(self, loader, reponame, sha1, filename): + def _get_file_contents_from_repo(self, reponame, + sha1, filename): # pragma: no cover if self.lrc.has_repo(reponame): self.status(msg="Looking for %(reponame)s:%(filename)s in the " "local repo cache.", @@ -227,21 +228,26 @@ class SourceResolver(object): try: repo = self.lrc.get_repo(reponame) text = repo.read_file(filename, sha1) - morph = loader.load_from_string(text) except IOError: - morph = None + text = None elif self.rrc is not None: self.status(msg="Looking for %(reponame)s:%(filename)s in the " "remote repo cache.", reponame=reponame, filename=filename, chatty=True) try: text = self.rrc.cat_file(reponame, sha1, filename) - morph = loader.load_from_string(text) except morphlib.remoterepocache.CatFileError: - morph = None + text = None else: # pragma: no cover repo = self.cache_repo_locally(reponame) text = repo.read_file(filename, sha1) + + return text + + def _get_morphology_from_repo(self, loader, reponame, sha1, filename): + text = self._get_file_contents_from_repo(self, reponame, sha1, filename) + + if file not None: morph = loader.load_from_string(text) return morph -- cgit v1.2.1 From 0ad137344b5712a7a42c54dc744f7367c56a3212 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 27 Feb 2015 15:36:49 +0000 Subject: morphlib/sourceresolver.py: Add _get_file_contents_from_definitions() And make _get_morphology_from_definitions() use it --- morphlib/sourceresolver.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index 6347861e..dd020803 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -212,10 +212,19 @@ class SourceResolver(object): return absref, tree + def _get_file_contents_from_definitions(self, + filename): # pragma: no cover + if os.path.exists(filename): + with open(filename) as f: + return f.read() + else: + return None + def _get_morphology_from_definitions(self, loader, filename): # pragma: no cover if os.path.exists(filename): - return loader.load_from_file(filename) + text = self._get_file_contents_from_definitions(filename) + return self.load_from_string(text, filename=filename) else: return None -- cgit v1.2.1 From 1cf8cfda9e7def942f3fe568373c55b251e7e2f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 27 Feb 2015 15:48:13 +0000 Subject: morphlib/sourceresolver.py: Add _get_file_contents() And make _get_morphology() use it --- morphlib/sourceresolver.py | 35 +++++++++++++++++++++++------------ 1 file changed, 23 insertions(+), 12 deletions(-) diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index dd020803..cd183ee0 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -261,18 +261,13 @@ class SourceResolver(object): return morph - def _get_morphology(self, reponame, sha1, filename): - '''Read the morphology at the specified location. + def _get_file_contents(self, reponame, sha1, filename): # pragma: no cover + '''Read the file at the specified location. Returns None if the file does not exist in the specified commit. ''' - key = (reponame, sha1, filename) - if key in self._resolved_morphologies: - return self._resolved_morphologies[key] - - loader = morphlib.morphloader.MorphologyLoader() - morph = None + text = None if reponame == self._definitions_repo and \ sha1 == self._definitions_absref: # pragma: no cover @@ -280,11 +275,27 @@ class SourceResolver(object): # we can quickly read definitions files from. defs_filename = os.path.join(self._definitions_checkout_dir, filename) - morph = self._get_morphology_from_definitions(loader, - defs_filename) + text = self._get_file_contents_from_definitions(defs_filename) else: - morph = self._get_morphology_from_repo(loader, reponame, sha1, - filename) + text = self._get_file_contents_from_repo(reponame, sha1, filename) + + return text + + def _get_morphology(self, reponame, sha1, filename): # pragma: no cover + '''Read the morphology at the specified location. + + Returns None if the file does not exist in the specified commit. + + ''' + key = (reponame, sha1, filename) + if key in self._resolved_morphologies: + return self._resolved_morphologies[key] + + loader = morphlib.morphloader.MorphologyLoader() + morph = None + + text = self._get_file_contents(reponame, sha1, filename) + morph = loader.load_from_string(text) if morph is None: return None -- cgit v1.2.1 From e7e54155ce92b9fa24c2711dca45a87d37ffecee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 27 Feb 2015 18:39:57 +0000 Subject: morphlib/morphloader.py: Update copyrigth --- morphlib/morphloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index 8289b01e..d257dced 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -- cgit v1.2.1 From 940ce227c4ca4700e6557cc110141312a5f73131 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 27 Feb 2015 18:40:06 +0000 Subject: morphlib/morphloader.py: Add a check to load_from_string() --- morphlib/morphloader.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index d257dced..e0f0952b 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -398,13 +398,17 @@ class MorphologyLoader(object): return morphlib.morphology.Morphology(obj) - def load_from_string(self, string, filename='string'): + def load_from_string(self, string, + filename='string'): # pragma: no cover '''Load a morphology from a string. Return the Morphology object. ''' + if string is None: + return None + m = self.parse_morphology_text(string, filename) m.filename = filename self.validate(m) -- cgit v1.2.1 From dbb482ec95b6e0110a32df000893880f4b92abeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 27 Feb 2015 16:05:17 +0000 Subject: morphlib/sourceresolver.py: Do not duplicate what is already done in load_from_string() --- morphlib/sourceresolver.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index cd183ee0..5be373a8 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -292,19 +292,14 @@ class SourceResolver(object): return self._resolved_morphologies[key] loader = morphlib.morphloader.MorphologyLoader() - morph = None text = self._get_file_contents(reponame, sha1, filename) morph = loader.load_from_string(text) - if morph is None: - return None - else: - loader.validate(morph) - loader.set_commands(morph) - loader.set_defaults(morph) + if morph is not None: self._resolved_morphologies[key] = morph - return morph + + return morph def _detect_build_system(self, reponame, sha1, expected_filename): '''Attempt to detect buildsystem of the given commit. -- cgit v1.2.1 From d2b5e4c36681909e097f9baea1316f93e3462535 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 27 Feb 2015 15:48:51 +0000 Subject: morphlib/sourceresolver.py: Remove not used functions _get_morphology_from_definitions() and _get_morphology_from_repo() --- morphlib/sourceresolver.py | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index 5be373a8..f571d76b 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -220,14 +220,6 @@ class SourceResolver(object): else: return None - def _get_morphology_from_definitions(self, loader, - filename): # pragma: no cover - if os.path.exists(filename): - text = self._get_file_contents_from_definitions(filename) - return self.load_from_string(text, filename=filename) - else: - return None - def _get_file_contents_from_repo(self, reponame, sha1, filename): # pragma: no cover if self.lrc.has_repo(reponame): @@ -253,14 +245,6 @@ class SourceResolver(object): return text - def _get_morphology_from_repo(self, loader, reponame, sha1, filename): - text = self._get_file_contents_from_repo(self, reponame, sha1, filename) - - if file not None: - morph = loader.load_from_string(text) - - return morph - def _get_file_contents(self, reponame, sha1, filename): # pragma: no cover '''Read the file at the specified location. -- cgit v1.2.1 From 915e37724a26e5a9d376844fb8f247fb9470aa94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Mon, 2 Mar 2015 19:14:57 +0000 Subject: morphlib/sourceresolver.py: Check and parse VERSION file Only fail if: - VERSION file exists - and its a yaml file - and its a dict - and has the key 'version' - and the contents of the key 'version' is an int - and that int is in the list of non_supported_versions (empty at the moment) --- morphlib/sourceresolver.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index f571d76b..387d2e0d 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -21,6 +21,7 @@ import os import pylru import shutil import tempfile +import yaml import cliapp @@ -31,6 +32,7 @@ tree_cache_filename = 'trees.cache.pickle' buildsystem_cache_size = 10000 buildsystem_cache_filename = 'detected-chunk-buildsystems.cache.pickle' +not_supported_versions = [] class PickleCacheManager(object): # pragma: no cover '''Cache manager for PyLRU that reads and writes to Pickle files. @@ -89,6 +91,11 @@ class MorphologyNotFoundError(SourceResolverError): # pragma: no cover SourceResolverError.__init__( self, "Couldn't find morphology: %s" % filename) +class UnknownVersionError(SourceResolverError): # pragma: no cover + def __init__(self, version): + SourceResolverError.__init__( + self, "Definitions format version %s is not supported" % version) + class SourceResolver(object): '''Provides a way of resolving the set of sources for a given system. @@ -338,6 +345,22 @@ class SourceResolver(object): loader.set_defaults(morph) return morph + def _check_version_file(self,definitions_repo, + definitions_absref): # pragma: no cover + version_file = self._get_file_contents( + definitions_repo, definitions_absref, 'VERSION') + + if version_file is None: + return + + try: + version = yaml.safe_load(version_file)['version'] + except (yaml.error.YAMLError, KeyError, TypeError): + version = 0 + + if version in not_supported_versions: + raise UnknownVersionError(version) + def _process_definitions_with_children(self, system_filenames, definitions_repo, definitions_ref, @@ -347,6 +370,8 @@ class SourceResolver(object): definitions_queue = collections.deque(system_filenames) chunk_queue = set() + self._check_version_file(definitions_repo, definitions_absref) + while definitions_queue: filename = definitions_queue.popleft() -- cgit v1.2.1 From 507380e0e4052b63c310d68c20a6e2d688965ce4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Wed, 4 Mar 2015 19:03:28 +0000 Subject: morphlib/sourceresolver_tests.py: Remove test to require 'build-depends' --- morphlib/sourceresolver_tests.py | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/morphlib/sourceresolver_tests.py b/morphlib/sourceresolver_tests.py index 638f593f..1239b437 100644 --- a/morphlib/sourceresolver_tests.py +++ b/morphlib/sourceresolver_tests.py @@ -69,15 +69,6 @@ class FakeLocalRepo(object): build-mode: bootstrap build-depends: [] ''', - 'stratum-no-chunk-bdeps.morph': ''' - name: stratum-no-chunk-bdeps - kind: stratum - chunks: - - name: chunk - repo: test:repo - ref: sha1 - build-mode: bootstrap - ''', 'stratum-no-bdeps-no-bootstrap.morph': ''' name: stratum-no-bdeps-no-bootstrap kind: stratum @@ -328,11 +319,6 @@ class SourceResolverTests(unittest.TestCase): self.assertRaises(morphlib.Error, self.sr._get_morphology, 'reponame', 'sha1', 'parse-error.morph') - def test_fails_on_no_chunk_bdeps(self): - self.assertRaises(morphlib.morphloader.NoBuildDependenciesError, - self.sr._get_morphology, 'reponame', 'sha1', - 'stratum-no-chunk-bdeps.morph') - def test_fails_on_no_bdeps_or_bootstrap(self): self.assertRaises( morphlib.morphloader.NoStratumBuildDependenciesError, -- cgit v1.2.1 From ecc2961699488cb409e45f1a3560c0181caf56d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Wed, 4 Mar 2015 19:08:02 +0000 Subject: morphlib/morphloader_tests.py: Remove test to require 'build-depends' --- morphlib/morphloader_tests.py | 22 +--------------------- 1 file changed, 1 insertion(+), 21 deletions(-) diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py index dd70c824..a1fe1674 100644 --- a/morphlib/morphloader_tests.py +++ b/morphlib/morphloader_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -344,26 +344,6 @@ build-system: dummy self.loader.validate(m) self.assertEqual(m['arch'], 'armv7l') - def test_validate_requires_build_deps_for_chunks_in_strata(self): - m = morphlib.morphology.Morphology( - { - "kind": "stratum", - "name": "foo", - "chunks": [ - { - "name": "foo", - "repo": "foo", - "ref": "foo", - "morph": "foo", - "build-mode": "bootstrap", - } - ], - }) - - self.assertRaises( - morphlib.morphloader.NoBuildDependenciesError, - self.loader.validate, m) - def test_validate_requires_build_deps_or_bootstrap_mode_for_strata(self): m = morphlib.morphology.Morphology( { -- cgit v1.2.1 From 8cf5c70943817c5c24b1f8b79a67cc8ac1d7dab0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Wed, 4 Mar 2015 19:03:58 +0000 Subject: Do not fail if a chunk doesnt have a 'build-depends' parameter defined --- morphlib/artifactresolver.py | 17 +++++++++-------- morphlib/morphloader.py | 13 +------------ 2 files changed, 10 insertions(+), 20 deletions(-) diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index e53c7511..5062f854 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -195,13 +195,14 @@ class ArtifactResolver(object): chunk_source.add_dependency(other_stratum) # Add dependencies between chunks mentioned in this stratum - for name in build_depends: # pragma: no cover - if name not in name_to_processed_artifacts: - raise DependencyOrderError( - source, info['name'], name) - other_artifacts = name_to_processed_artifacts[name] - for other_artifact in other_artifacts: - chunk_source.add_dependency(other_artifact) + if build_depends is not None: + for name in build_depends: # pragma: no cover + if name not in name_to_processed_artifacts: + raise DependencyOrderError( + source, info['name'], name) + other_artifacts = name_to_processed_artifacts[name] + for other_artifact in other_artifacts: + chunk_source.add_dependency(other_artifact) # Add build dependencies between our stratum's artifacts # and the chunk artifacts produced by this stratum. diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index e0f0952b..7d51dc1e 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -111,14 +111,6 @@ class UnknownArchitectureError(MorphologyValidationError): % (arch, morph_filename)) -class NoBuildDependenciesError(MorphologyValidationError): - - def __init__(self, stratum_name, chunk_name, morph_filename): - self.msg = ( - 'Stratum %s has no build dependencies for chunk %s in %s' % - (stratum_name, chunk_name, morph_filename)) - - class NoStratumBuildDependenciesError(MorphologyValidationError): def __init__(self, stratum_name, morph_filename): @@ -556,7 +548,7 @@ class MorphologyLoader(object): # Validate build-dependencies if specified self._validate_stratum_specs_fields(morph, 'build-depends') - # Require build-dependencies for each chunk. + # Check build-dependencies for each chunk. for spec in morph['chunks']: chunk_name = spec.get('alias', spec['name']) if 'build-depends' in spec: @@ -564,9 +556,6 @@ class MorphologyLoader(object): raise InvalidTypeError( '%s.build-depends' % chunk_name, list, type(spec['build-depends']), morph['name']) - else: - raise NoBuildDependenciesError( - morph['name'], chunk_name, morph.filename) @classmethod def _validate_chunk(cls, morphology): -- cgit v1.2.1