path: root/lorrycontroller/readconf.py

# Copyright (C) 2014-2016  Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.


import errno
import glob
import json
import logging
import os
import re

import bottle
import cliapp

import lorrycontroller


class LorryControllerConfParseError(Exception):

    def __init__(self, filename, exc):
        Exception.__init__(
            self, 'ERROR reading %s: %s' % (filename, str(exc)))


class ReadConfiguration(lorrycontroller.LorryControllerRoute):

    http_method = 'POST'
    path = '/1.0/read-configuration'

    DEFAULT_LORRY_TIMEOUT = 3600 # in seconds

    def run(self, **kwargs):
        logging.info('%s %s called', self.http_method, self.path)

        self.get_confgit()

        try:
            conf_obj = self.read_config_file()
        except LorryControllerConfParseError as e:
            return str(e)

        error = self.validate_config(conf_obj)
        if error:
            return 'ERROR: %s: %r' % (error, conf_obj)

        self.fix_up_parsed_fields(conf_obj)

        statedb = self.open_statedb()
        with statedb:
            lorries_to_remove = set(statedb.get_lorries_paths())
            troves_to_remove = set(statedb.get_troves())

            for section in conf_obj:
                if not 'type' in section:
                    return 'ERROR: no type field in section'
                if section['type'] == 'lorries':
                    added = self.add_matching_lorries_to_statedb(
                        statedb, section)
                    lorries_to_remove = lorries_to_remove.difference(added)
                elif section['type'] in ('trove', 'troves', 'gitlab'):
                    self.add_trove(statedb, section)
                    trovehost = section.get('host') or section['trovehost']
                    if trovehost in troves_to_remove:
                        troves_to_remove.remove(trovehost)
                        lorries_to_remove = lorries_to_remove.difference(
                            statedb.get_lorries_for_trove(trovehost))
                else:
                    logging.error(
                        'Unknown section in configuration: %r', section)
                    return (
                        'ERROR: Unknown section type in configuration: %r' %
                        section)

            for path in lorries_to_remove:
                statedb.remove_lorry(path)

            for trovehost in troves_to_remove:
                statedb.remove_trove(trovehost)
                statedb.remove_lorries_for_trovehost(trovehost)

        if 'redirect' in bottle.request.forms:
            bottle.redirect(bottle.request.forms.redirect)

        return 'Configuration has been updated.'

    def get_confgit(self):
        if self.app_settings['debug-real-confgit']:
            confdir = self.app_settings['configuration-directory']
            if not os.path.exists(confdir):
                self.git_clone_confgit(confdir)
            else:
                self.update_confgit(confdir)

    def git_clone_confgit(self, confdir):
        url = self.app_settings['confgit-url']
        branch = self.app_settings['confgit-branch']
        logging.info('Cloning %s to %s', url, confdir)
        cliapp.runcmd(['git', 'clone', '-b', branch, url, confdir])

    def update_confgit(self, confdir):
        logging.info('Updating CONFGIT in %s', confdir)

        # The following sequence makes the working tree mirror
        # current upstream git repository as closely as possible.

        # Get rid of any local changes. No human is meant to edit
        # anything locally, but there may be remnants of failed
        # runs.
        cliapp.runcmd(['git', 'reset', '--hard'], cwd=confdir)

        # Get rid of any files not known by git. This might be,
        # say, core dumps.
        cliapp.runcmd(['git', 'clean', '-fdx'], cwd=confdir)

        # Fetch updates to remote branches.
        cliapp.runcmd(['git', 'remote', 'update', 'origin'], cwd=confdir)

        # Now move the current HEAD to wherever the remote master
        # branch is, no questions asked. This doesn't do merging
        # or any of the other things we don't want in this situation.
        cliapp.runcmd(
            ['git', 'reset', '--hard', 'origin/master'], cwd=confdir)

    @property
    def config_file_name(self):
        return os.path.join(
            self.app_settings['configuration-directory'],
            'lorry-controller.conf')

    def read_config_file(self):
        '''Read the configuration file, return as Python object.'''

        filename = self.config_file_name
        logging.debug('Reading configuration file %s', filename)

        try:
            with open(filename) as f:
                return json.load(f)
        except IOError as e:
            if e.errno == errno.ENOENT:
                logging.debug(
                    '%s: does not exist, returning empty config', filename)
                return []
            bottle.abort(500, 'Error reading %s: %s' % (filename, e))
        except ValueError as e:
            logging.error('Error parsing configuration: %s', e)
            raise LorryControllerConfParseError(filename, e)

    def validate_config(self, obj):
        validator = LorryControllerConfValidator()
        return validator.validate_config(obj)

    def fix_up_parsed_fields(self, obj):
        for item in obj:
            item['interval'] = self.fix_up_interval(item.get('interval'))
            item['ls-interval'] = self.fix_up_interval(item.get('ls-interval'))

    def fix_up_interval(self, value):
        default_interval = 86400 # 1 day
        if not value:
            return default_interval
        m = re.match('(\d+)\s*(s|m|h|d)?', value, re.I)
        if not m:
            return default_value

        number, factor = m.groups()
        factors = {
            's': 1,
            'm': 60,
            'h': 60*60,
            'd': 60*60*24,
            }
        if factor is None:
            factor = 's'
        factor = factors.get(factor.lower(), 1)
        return int(number) * factor

    def add_matching_lorries_to_statedb(self, statedb, section):
        logging.debug('Adding matching lorries to STATEDB')

        added_paths = set()

        filenames = self.find_lorry_files_for_section(section)
        logging.debug('filenames=%r', filenames)
        lorry_specs = []
        for filename in sorted(filenames):
            logging.debug('Reading .lorry: %s', filename)
            for subpath, obj in self.get_valid_lorry_specs(filename):
                self.add_refspecs_if_missing(obj)
                lorry_specs.append((subpath, obj))

        for subpath, obj in sorted(lorry_specs):
            path = self.deduce_repo_path(section, subpath)
            text = self.serialise_lorry_spec(path, obj)
            interval = section['interval']
            timeout = section.get(
                'lorry-timeout', self.DEFAULT_LORRY_TIMEOUT)

            try:
                old_lorry_info = statedb.get_lorry_info(path)
            except lorrycontroller.LorryNotFoundError:
                old_lorry_info = None

            statedb.add_to_lorries(
                path=path, text=text, from_trovehost='', from_path='',
                interval=interval, timeout=timeout)

            added_paths.add(path)

        return added_paths

    def find_lorry_files_for_section(self, section):
        result = []
        dirname = os.path.dirname(self.config_file_name)
        for base_pattern in section['globs']:
            pattern = os.path.join(dirname, base_pattern)
            result.extend(glob.glob(pattern))
        return result

    def get_valid_lorry_specs(self, filename):
        # We do some basic validation of the .lorry file and the Lorry
        # specs contained within it. We silently ignore anything that
        # doesn't look OK. We don't have a reasonable mechanism to
        # communicate any problems to the user, but we do log them to
        # the log file.

        try:
            with open(filename) as f:
                obj = json.load(f)
        except ValueError as e:
            logging.error('JSON problem in %s', filename)
            return []

        if type(obj) != dict:
            logging.error('%s: does not contain a dict', filename)
            return []

        items = []
        for key in obj:
            if type(obj[key]) != dict:
                logging.error(
                    '%s: key %s does not map to a dict', filename, key)
                continue

            if 'type' not in obj[key]:
                logging.error(
                    '%s: key %s does not have type field', filename, key)
                continue

            logging.debug('Happy with Lorry spec %r: %r', key, obj[key])
            items.append((key, obj[key]))

        return items

    def add_refspecs_if_missing(self, obj):
        if 'refspecs' not in obj:
            obj['refspecs'] = [
                '+refs/heads/*',
                '+refs/tags/*',
                ]

    def deduce_repo_path(self, section, subpath):
        return '%s/%s' % (section['prefix'], subpath)

    def serialise_lorry_spec(self, path, obj):
        new_obj = { path: obj }
        return json.dumps(new_obj, indent=4)

    def add_trove(self, statedb, section):
        username = None
        password = None
        if 'auth' in section:
            auth = section['auth']
            username = auth.get('username')
            password = auth.get('password')

        gitlab_token = None
        if section['type'] == 'gitlab':
            gitlab_token = section['private-token']

        statedb.add_trove(
            trovehost=section.get('host') or section['trovehost'],
            protocol=section['protocol'],
            username=username,
            password=password,
            lorry_interval=section['interval'],
            lorry_timeout=section.get(
                'lorry-timeout', self.DEFAULT_LORRY_TIMEOUT),
            ls_interval=section['ls-interval'],
            prefixmap=json.dumps(section['prefixmap']),
            ignore=json.dumps(section.get('ignore', [])),
            gitlab_token=gitlab_token)


class ValidationError(Exception):

    def __init__(self, msg):
        Exception.__init__(self, msg)


class LorryControllerConfValidator(object):

    def validate_config(self, conf_obj):
        try:
            self._check_is_list(conf_obj)
            self._check_is_list_of_dicts(conf_obj)

            for section in conf_obj:
                if 'type' not in section:
                    raise ValidationError(
                        'section without type: %r' % section)
                elif section['type'] in ('trove', 'troves'):
                    self._check_troves_section(section)
                elif section['type'] == 'lorries':
                    self._check_lorries_section(section)
                elif section['type'] == 'gitlab':
                    self._check_gitlab_section(section)
                else:
                    raise ValidationError(
                        'unknown section type %r' % section['type'])
        except ValidationError as e:
            return str(e)

        return None

    def _check_is_list(self, conf_obj):
        if type(conf_obj) is not list:
            raise ValidationError(
                'type %r is not a JSON list' % type(conf_obj))

    def _check_is_list_of_dicts(self, conf_obj):
        for item in conf_obj:
            if type(item) is not dict:
                raise ValidationError('all items must be dicts')

    def _check_gitlab_section(self, section):
        # gitlab section inherits trove configurations, perform the same checks.
        self._check_troves_section(section)
        self._check_has_required_fields(section, ['private-token'])

    def _check_troves_section(self, section):
        if not any(i in ('trovehost', 'host') for i in section):
            self._check_has_required_fields(section, ['host'])
        self._check_has_required_fields(
            section,
            ['protocol', 'interval', 'ls-interval', 'prefixmap'])
        self._check_protocol(section)
        self._check_prefixmap(section)
        if 'ignore' in section:
            self._check_is_list_of_strings(section, 'ignore')

    def _check_protocol(self, section):
        valid = ('ssh', 'http', 'https')
        if section['protocol'] not in valid:
            raise ValidationError(
                'protocol field has value "%s", but valid ones are %s' %
                (section['protocol'], ', '.join(valid)))

    def _check_prefixmap(self, section):
        # FIXME: We should be checking the prefixmap for things like
        # mapping to a prefix that starts with the local Trove ID, but
        # since we don't have easy access to that, we don't do that
        # yet. This should be fixed later.
        pass

    def _check_lorries_section(self, section):
        self._check_has_required_fields(
            section, ['interval', 'prefix', 'globs'])
        self._check_is_list_of_strings(section, 'globs')

    def _check_has_required_fields(self, section, fields):
        for field in fields:
            if field not in section:
                raise ValidationError(
                    'mandatory field %s missing in section %r' %
                    (field, section))

    def _check_is_list_of_strings(self, section, field):
        obj = section[field]
        if not isinstance(obj, list) or not all(
                isinstance(s, basestring) for s in obj):
            raise ValidationError(
                '%s field in %r must be a list of strings' %
                (field, section))