diff options
| author | Ben Brown <ben@demerara.io> | 2022-01-12 11:19:06 +0000 |
|---|---|---|
| committer | Ben Brown <ben@demerara.io> | 2022-01-12 11:19:06 +0000 |
| commit | dc2ab24774afb5386f1ef1e8471e8c2efa488319 (patch) | |
| tree | ecff3c56c2b61f77cd65bfd99859e47f253d210b | |
| parent | 286b50ee969bad45a9f11df96c6a36d513310a51 (diff) | |
| parent | 1a1acf5ca41c4ffca5a9f94d770f852335111048 (diff) | |
| download | lorry-controller-dc2ab24774afb5386f1ef1e8471e8c2efa488319.tar.gz | |
Merge branch 'km/fix-block' into 'master'
Fix job queue getting stuck on repo preparation
Closes #32 and #25
See merge request CodethinkLabs/lorry/lorry-controller!32
| -rw-r--r-- | lorrycontroller/givemejob.py | 24 |
1 files changed, 21 insertions, 3 deletions
diff --git a/lorrycontroller/givemejob.py b/lorrycontroller/givemejob.py index 2f7580f..973e141 100644 --- a/lorrycontroller/givemejob.py +++ b/lorrycontroller/givemejob.py @@ -16,6 +16,7 @@ import json import logging import re +import traceback import urllib.parse import bottle @@ -39,17 +40,34 @@ class GiveMeJob(lorrycontroller.LorryControllerRoute): now = statedb.get_current_time() for lorry_info in lorry_infos: if self.ready_to_run(lorry_info, now): + lorry_path = lorry_info['path'] metadata = self.get_repo_metadata(statedb, lorry_info) downstream_type = lorrycontroller.downstream_types[ self.app_settings['git-server-type']] - downstream_type(self.app_settings) \ - .prepare_repo(lorry_info['path'], metadata) + + # Skip over any repos that fail to prepare + # otherwise job queue will get stuck here + try: + downstream_type(self.app_settings).prepare_repo( + lorry_path, metadata) + # Catching base Exception because we don't want + # unexpected exception types to block the queue + except Exception: + logging.exception( + 'Skipping lorry %s due to an encountered exception', + lorry_path) + # Failure should be visible to user on status page + statedb.set_lorry_last_run_exit_and_output( + lorry_path, '1', traceback.format_exc()) + # Prevent repeating failure + statedb.set_lorry_last_run(lorry_path, int(now)) + continue self.give_job_to_minion(statedb, lorry_info, now) logging.info( 'Giving job %s to lorry %s to MINION %s:%s', lorry_info['job_id'], - lorry_info['path'], + lorry_path, bottle.request.forms.host, bottle.request.forms.pid) return lorry_info |