# Copyright (C) 2014-2021 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import errno import glob import json import logging import os import re import bottle import cliapp import yaml import lorrycontroller class LorryControllerConfParseError(Exception): def __init__(self, filename, exc): Exception.__init__( self, 'ERROR reading %s: %s' % (filename, str(exc))) class ReadConfiguration(lorrycontroller.LorryControllerRoute): http_method = 'POST' path = '/1.0/read-configuration' DEFAULT_LORRY_TIMEOUT = 3600 # in seconds def run(self, **kwargs): logging.info('%s %s called', self.http_method, self.path) self.get_confgit() try: conf_obj = self.read_config_file() except LorryControllerConfParseError as e: return str(e) error = self.validate_config(conf_obj) if error: return 'ERROR: %s: %r' % (error, conf_obj) self.fix_up_parsed_fields(conf_obj) statedb = self.open_statedb() with statedb: lorries_to_remove = set(statedb.get_lorries_paths()) hosts_to_remove = set(statedb.get_hosts()) for section in conf_obj: if not 'type' in section: return 'ERROR: no type field in section' if section['type'] == 'lorries': added = self.add_matching_lorries_to_statedb( statedb, section) lorries_to_remove = lorries_to_remove.difference(added) elif section['type'] in lorrycontroller.upstream_types: self.add_host(statedb, section) host = section.get('host') or section['trovehost'] if host in hosts_to_remove: hosts_to_remove.remove(host) lorries_to_remove = lorries_to_remove.difference( statedb.get_lorries_for_host(host)) else: logging.error( 'Unknown section in configuration: %r', section) return ( 'ERROR: Unknown section type in configuration: %r' % section) for path in lorries_to_remove: statedb.remove_lorry(path) for host in hosts_to_remove: statedb.remove_host(host) statedb.remove_lorries_for_host(host) if 'redirect' in bottle.request.forms: bottle.redirect(bottle.request.forms.redirect) return 'Configuration has been updated.' def get_confgit(self): if self.app_settings['debug-real-confgit']: confdir = self.app_settings['configuration-directory'] self.fetch_confgit(confdir) def fetch_confgit(self, confdir): url = self.app_settings['confgit-url'] branch = self.app_settings['confgit-branch'] logging.info('Fetching CONFGIT in %s', confdir) if not os.path.exists(confdir): cliapp.runcmd(['git', 'init', confdir]) # Fetch updates to remote branch. cliapp.runcmd(['git', 'fetch', '--prune', url, branch], cwd=confdir) # Get rid of any files not known by git. This might be, # say, core dumps. cliapp.runcmd(['git', 'clean', '-fdx'], cwd=confdir) # Now move the current HEAD to whatever we just fetched, no # questions asked. This doesn't do merging or any of the other # things we don't want in this situation. cliapp.runcmd(['git', 'reset', '--hard', 'FETCH_HEAD'], cwd=confdir) @property def config_file_name(self): return os.path.join( self.app_settings['configuration-directory'], 'lorry-controller.conf') def read_config_file(self): '''Read the configuration file, return as Python object.''' filename = self.config_file_name logging.debug('Reading configuration file %s', filename) try: with open(filename) as f: return json.load(f) except IOError as e: if e.errno == errno.ENOENT: logging.debug( '%s: does not exist, returning empty config', filename) return [] bottle.abort(500, 'Error reading %s: %s' % (filename, e)) except ValueError as e: logging.error('Error parsing configuration: %s', e) raise LorryControllerConfParseError(filename, e) def validate_config(self, obj): validator = LorryControllerConfValidator() return validator.validate_config(obj) def fix_up_parsed_fields(self, obj): for item in obj: item['interval'] = self.fix_up_interval(item.get('interval')) item['ls-interval'] = self.fix_up_interval(item.get('ls-interval')) def fix_up_interval(self, value): default_interval = 86400 # 1 day if not value: return default_interval m = re.match('(\d+)\s*(s|m|h|d)?', value, re.I) if not m: return default_interval number, factor = m.groups() factors = { 's': 1, 'm': 60, 'h': 60*60, 'd': 60*60*24, } if factor is None: factor = 's' factor = factors.get(factor.lower(), 1) return int(number) * factor def add_matching_lorries_to_statedb(self, statedb, section): logging.debug('Adding matching lorries to STATEDB') added_paths = set() filenames = self.find_lorry_files_for_section(section) logging.debug('filenames=%r', filenames) lorry_specs = [] for filename in sorted(filenames): logging.debug('Reading .lorry: %s', filename) for subpath, obj in self.get_valid_lorry_specs(filename): self.add_refspecs_if_missing(obj) lorry_specs.append((subpath, obj)) for subpath, obj in sorted(lorry_specs, key=(lambda x: x[0])): path = self.deduce_repo_path(section, subpath) text = self.serialise_lorry_spec(path, obj) interval = section['interval'] timeout = section.get( 'lorry-timeout', self.DEFAULT_LORRY_TIMEOUT) statedb.add_to_lorries( path=path, text=text, from_host='', from_path='', interval=interval, timeout=timeout) added_paths.add(path) return added_paths def find_lorry_files_for_section(self, section): result = [] dirname = os.path.dirname(self.config_file_name) for base_pattern in section['globs']: pattern = os.path.join(dirname, base_pattern) result.extend(glob.glob(pattern)) return result def get_valid_lorry_specs(self, filename): # We do some basic validation of the .lorry file and the Lorry # specs contained within it. We silently ignore anything that # doesn't look OK. We don't have a reasonable mechanism to # communicate any problems to the user, but we do log them to # the log file. try: with open(filename) as f: try: obj = yaml.safe_load(f) except yaml.YAMLError: f.seek(0) obj = json.load(f) except ValueError: logging.error('YAML and JSON problem in %s', filename) return [] if type(obj) != dict: logging.error('%s: does not contain a dict', filename) return [] items = [] for key in obj: if type(obj[key]) != dict: logging.error( '%s: key %s does not map to a dict', filename, key) continue if 'type' not in obj[key]: logging.error( '%s: key %s does not have type field', filename, key) continue logging.debug('Happy with Lorry spec %r: %r', key, obj[key]) items.append((key, obj[key])) return items def add_refspecs_if_missing(self, obj): if 'refspecs' not in obj: obj['refspecs'] = [ '+refs/heads/*', '+refs/tags/*', ] def deduce_repo_path(self, section, subpath): return '%s/%s' % (section['prefix'], subpath) def serialise_lorry_spec(self, path, obj): new_obj = { path: obj } return json.dumps(new_obj, indent=4) def add_host(self, statedb, section): username = None password = None if 'auth' in section: auth = section['auth'] username = auth.get('username') password = auth.get('password') type_params = lorrycontroller.upstream_types[section['type']] \ .get_host_type_params(section) statedb.add_host( host=section.get('host') or section['trovehost'], protocol=section['protocol'], username=username, password=password, host_type=section['type'], type_params=type_params, lorry_interval=section['interval'], lorry_timeout=section.get( 'lorry-timeout', self.DEFAULT_LORRY_TIMEOUT), ls_interval=section['ls-interval'], prefixmap=json.dumps(section['prefixmap']), ignore=json.dumps(section.get('ignore', []))) class ValidationError(Exception): def __init__(self, msg): Exception.__init__(self, msg) class LorryControllerConfValidator(object): def validate_config(self, conf_obj): try: self.check_is_list(conf_obj) self.check_is_list_of_dicts(conf_obj) for section in conf_obj: if 'type' not in section: raise ValidationError( 'section without type: %r' % section) # Backward compatibility if section['type'] == 'troves': section['type'] = 'trove' if section['type'] in lorrycontroller.upstream_types: self._check_host_section(section) lorrycontroller.upstream_types[section['type']] \ .check_host_type_params(self, section) elif section['type'] == 'lorries': self._check_lorries_section(section) else: raise ValidationError( 'unknown section type %r' % section['type']) except ValidationError as e: return str(e) return None def check_is_list(self, conf_obj): if type(conf_obj) is not list: raise ValidationError( 'type %r is not a JSON list' % type(conf_obj)) def check_is_list_of_dicts(self, conf_obj): for item in conf_obj: if type(item) is not dict: raise ValidationError('all items must be dicts') def _check_host_section(self, section): if not any(i in ('trovehost', 'host') for i in section): self.check_has_required_fields(section, ['host']) self.check_has_required_fields( section, ['protocol', 'interval', 'ls-interval', 'prefixmap']) self._check_protocol(section) self._check_prefixmap(section) if 'ignore' in section: self.check_is_list_of_strings(section, 'ignore') def _check_protocol(self, section): valid = ('ssh', 'http', 'https') if section['protocol'] not in valid: raise ValidationError( 'protocol field has value "%s", but valid ones are %s' % (section['protocol'], ', '.join(valid))) def _check_prefixmap(self, section): # FIXME: We should be checking the prefixmap for things like # mapping to a prefix that starts with the local Trove ID, but # since we don't have easy access to that, we don't do that # yet. This should be fixed later. pass def _check_lorries_section(self, section): self.check_has_required_fields( section, ['interval', 'prefix', 'globs']) self.check_is_list_of_strings(section, 'globs') def check_has_required_fields(self, section, fields): for field in fields: if field not in section: raise ValidationError( 'mandatory field %s missing in section %r' % (field, section)) def check_is_list_of_strings(self, section, field): obj = section[field] if not isinstance(obj, list) or not all( isinstance(s, str) for s in obj): raise ValidationError( '%s field in %r must be a list of strings' % (field, section))