# Copyright (C) 2014-2016,2020 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import json import logging import re import traceback import urllib.parse import bottle import cliapp import lorrycontroller class GiveMeJob(lorrycontroller.LorryControllerRoute): http_method = 'POST' path = '/1.0/give-me-job' def run(self, **kwargs): logging.info('%s %s called', self.http_method, self.path) statedb = self.open_statedb() with statedb: if statedb.get_running_queue() and not self.max_jobs_reached(statedb): lorry_infos = statedb.get_all_lorries_info() now = statedb.get_current_time() for lorry_info in lorry_infos: if self.ready_to_run(lorry_info, now): lorry_path = lorry_info['path'] metadata = self.get_repo_metadata(statedb, lorry_info) downstream_type = lorrycontroller.downstream_types[ self.app_settings['git-server-type']] # Skip over any repos that fail to prepare # otherwise job queue will get stuck here try: downstream_type(self.app_settings).prepare_repo( lorry_path, metadata) # Catching base Exception because we don't want # unexpected exception types to block the queue except Exception: logging.exception( 'Skipping lorry %s due to an encountered exception', lorry_path) # Failure should be visible to user on status page statedb.set_lorry_last_run_exit_and_output( lorry_path, '1', traceback.format_exc()) # Prevent repeating failure statedb.set_lorry_last_run(lorry_path, int(now)) continue self.give_job_to_minion(statedb, lorry_info, now) logging.info( 'Giving job %s to lorry %s to MINION %s:%s', lorry_info['job_id'], lorry_path, bottle.request.forms.host, bottle.request.forms.pid) return lorry_info logging.info('No job to give MINION') return { 'job_id': None } def max_jobs_reached(self, statedb): max_jobs = statedb.get_max_jobs() if max_jobs is None: return False running_jobs = statedb.get_running_jobs() return len(running_jobs) >= max_jobs def ready_to_run(self, lorry_info, now): due = lorry_info['last_run'] + lorry_info['interval'] return (lorry_info['running_job'] is None and due <= now) @staticmethod def get_upstream_host_repo_metadata(statedb, lorry_info): assert lorry_info['from_host'] assert lorry_info['from_path'] try: host_info = statedb.get_host_info(lorry_info['from_host']) except lorrycontroller.HostNotFoundError: # XXX Shouldn't happen, but currently the database schema # does not prevent it return {} else: return lorrycontroller.get_upstream_host(host_info) \ .get_repo_metadata(lorry_info['from_path']) @staticmethod def get_single_repo_metadata(lorry_info): assert not lorry_info['from_host'] lorry_dict = json.loads(lorry_info['text']) _, upstream_config = lorry_dict.popitem() upstream_type = upstream_config['type'] # Get the repository URL url = None try: url = upstream_config['url'].strip() except KeyError: if upstream_type == 'bzr': try: url = upstream_config['branches']['trunk'].strip() except KeyError: pass # Extract the host-name and repo path host_name, repo_path = None, None if url: # Handle pseudo-URLs if upstream_type == 'bzr': if url.startswith('lp:'): host_name = 'launchpad.net' repo_path = url[3:] elif upstream_type == 'cvs': # :pserver:user@host:/path, user@host:/path, etc. match = re.match(r'^(?::[^:@/]+:)?(?:[^:@/]+@)?([^:@/]+):/', url) if match: host_name = match.group(1) repo_path = url[match.end():].rstrip('/') elif upstream_type == 'git': # user@host:path, host:path. Path must not start with # '//' as that indicates a real URL. match = re.match(r'^(?:[^:@/]+@)?([^:@/]+):(?!//)', url) if match: host_name = match.group(1) repo_path = url[match.end():].strip('/') # Default to parsing as a real URL if not host_name: try: url_obj = urllib.parse.urlparse(url) except ValueError: pass else: host_name = url_obj.hostname repo_path = url_obj.path.strip('/') metadata = {} # Determine the default branch if upstream_type == 'bzr': # Default in Bazaar is 'trunk' and we don't remap it metadata['head'] = 'trunk' elif upstream_type == 'git': if url: # Query the remote to find its default try: output = cliapp.runcmd(['git', 'ls-remote', '--symref', '--', url, 'HEAD']) \ .decode('utf-8', errors='replace') match = re.match(r'^ref: refs/heads/([^\s]+)\tHEAD\n', output) if match: metadata['head'] = match.group(1) except cliapp.AppException: pass else: # We currently produce 'master' for all other types metadata['head'] = 'master' # Use description from .lorry file, or repository name try: metadata['description'] = upstream_config['description'] except KeyError: if repo_path: metadata['description'] = repo_path return host_name, metadata def get_repo_metadata(self, statedb, lorry_info): '''Get repository head and description.''' host_name = lorry_info['from_host'] if host_name: metadata = self.get_upstream_host_repo_metadata(statedb, lorry_info) else: host_name, metadata = self.get_single_repo_metadata(lorry_info) if host_name and 'description' in metadata: # Prepend Upstream Host name metadata['description'] = '{host}: {desc}'.format( host=host_name, desc=metadata['description']) return metadata def give_job_to_minion(self, statedb, lorry_info, now): path = lorry_info['path'] minion_host = bottle.request.forms.host minion_pid = bottle.request.forms.pid running_job = statedb.get_next_job_id() statedb.set_running_job(path, running_job) statedb.add_new_job( running_job, minion_host, minion_pid, path, int(now)) lorry_info['job_id'] = running_job return lorry_info