diff options
Diffstat (limited to 'lorrycontroller/givemejob.py')
-rw-r--r-- | lorrycontroller/givemejob.py | 24 |
1 files changed, 21 insertions, 3 deletions
diff --git a/lorrycontroller/givemejob.py b/lorrycontroller/givemejob.py index 2f7580f..973e141 100644 --- a/lorrycontroller/givemejob.py +++ b/lorrycontroller/givemejob.py @@ -16,6 +16,7 @@ import json import logging import re +import traceback import urllib.parse import bottle @@ -39,17 +40,34 @@ class GiveMeJob(lorrycontroller.LorryControllerRoute): now = statedb.get_current_time() for lorry_info in lorry_infos: if self.ready_to_run(lorry_info, now): + lorry_path = lorry_info['path'] metadata = self.get_repo_metadata(statedb, lorry_info) downstream_type = lorrycontroller.downstream_types[ self.app_settings['git-server-type']] - downstream_type(self.app_settings) \ - .prepare_repo(lorry_info['path'], metadata) + + # Skip over any repos that fail to prepare + # otherwise job queue will get stuck here + try: + downstream_type(self.app_settings).prepare_repo( + lorry_path, metadata) + # Catching base Exception because we don't want + # unexpected exception types to block the queue + except Exception: + logging.exception( + 'Skipping lorry %s due to an encountered exception', + lorry_path) + # Failure should be visible to user on status page + statedb.set_lorry_last_run_exit_and_output( + lorry_path, '1', traceback.format_exc()) + # Prevent repeating failure + statedb.set_lorry_last_run(lorry_path, int(now)) + continue self.give_job_to_minion(statedb, lorry_info, now) logging.info( 'Giving job %s to lorry %s to MINION %s:%s', lorry_info['job_id'], - lorry_info['path'], + lorry_path, bottle.request.forms.host, bottle.request.forms.pid) return lorry_info |