Merge branch 'bwh/cleanup-host-types' into 'master'

Clean up and document how host type connectors work

Closes #10 and #5

See merge request CodethinkLabs/lorry-controller!7

11 files changed, 479 insertions, 214 deletions

diff --git a/ARCH b/ARCH
index 152ae33..5f7ce7b 100644
--- a/ARCH
+++ b/ARCH
@@ -431,8 +431,13 @@ The Lorry Controller code base is laid out as follows:
   way (with the `@app.route` decorator) seemed to make that harder and
   require everything in the same class.
 
-* `lorrycontroller` is a Python package with the HTTP request
-  handlers, management of STATEDB, plus some helpful utilities.
+* `lorrycontroller` is a Python package with:
+
+  - The HTTP request handlers (`LorryControllerRoute` and its subclasses)
+  - Management of STATEDB (`statedb` module)
+  - Support for various Downstream and Upstream Host types
+    (`hosts`, `gitano`, `gerrit`, `gitlab`, `local` modules)
+  - Some helpful utilities (`proxy` module)
 
 * `lorry-controller-minion` is the entirety of the MINION, except that
   it uses the `lorrycontroller.setup_proxy` function.
diff --git a/lorry-controller-webapp b/lorry-controller-webapp
index 9e3a310..6fc827e 100755
--- a/lorry-controller-webapp
+++ b/lorry-controller-webapp
@@ -131,17 +131,21 @@ class WEBAPP(cliapp.Application):
 
         self.settings.choice(
             ['downstream-host-type', 'git-server-type'],
-            ['gitano', 'gerrit', 'gitlab', 'local'],
+            # Default is the first choice, and must be 'gitano' for backward
+            # compatibility
+            ['gitano']
+            + sorted(host_type
+                     for host_type in lorrycontroller.downstream_types.keys()
+                     if host_type != 'gitano'),
             'what API the Downstream Host speaks')
 
-        self.settings.string(
-            ['gitlab-private-token'],
-            'private token for GitLab API access')
-
         self.settings.boolean(
             ['publish-failures'],
             'make the status page show failure logs from lorry')
 
+        for downstream_type in lorrycontroller.downstream_types.values():
+            downstream_type.add_app_settings(self.settings)
+
     def find_routes(self):
         '''Return all classes that are API routes.
 
@@ -166,11 +170,8 @@ class WEBAPP(cliapp.Application):
 
     def process_args(self, args):
         self.settings.require('statedb')
-        if (self.settings['git-server-type'] == 'gitlab' and
-            not self.settings['gitlab-private-token']):
-                logging.error('A private token must be provided to create '
-                              'repositories on a GitLab instance.')
-                self.settings.require('gitlab-private-token')
+        lorrycontroller.downstream_types[self.settings['git-server-type']] \
+                       .check_app_settings(self.settings)
 
         self.setup_proxy()
 
diff --git a/lorrycontroller/__init__.py b/lorrycontroller/__init__.py
index 64c4a6f..ddc2f74 100644
--- a/lorrycontroller/__init__.py
+++ b/lorrycontroller/__init__.py
@@ -37,15 +37,30 @@ from .removejob import RemoveJob
 from .lsupstreams import LsUpstreams, ForceLsUpstream
 from .pretendtime import PretendTime
 from .maxjobs import GetMaxJobs, SetMaxJobs
-from .gitano import (
-    GitanoCommand,
-    LocalTroveGitanoCommand,
-    GitanoCommandFailure,
-    new_gitano_command)
 from .static import StaticFile
 from .proxy import setup_proxy
-from .gerrit import Gerrit
-from .gitlab import Gitlab
+from . import gerrit
+from . import gitano
+from . import gitlab
+from . import local
+
+
+downstream_types = {
+    'gerrit': gerrit.GerritDownstream,
+    'gitano': gitano.GitanoDownstream,
+    'gitlab': gitlab.GitlabDownstream,
+    'local':  local.LocalDownstream,
+}
+
+
+upstream_types = {
+    'gitlab': gitlab.GitlabUpstream,
+    'trove':  gitano.TroveUpstream,
+}
+
+
+def get_upstream_host(host_info):
+    return upstream_types[host_info['type']](host_info)
 
 
 __all__ = locals()
diff --git a/lorrycontroller/gerrit.py b/lorrycontroller/gerrit.py
index 4e77ba7..c18f2ed 100644
--- a/lorrycontroller/gerrit.py
+++ b/lorrycontroller/gerrit.py
@@ -13,11 +13,14 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
+import logging
 
 import cliapp
 
+from . import hosts
 
-class Gerrit(object):
+
+class GerritDownstream(hosts.DownstreamHost):
 
     '''Run commands on a Gerrit instance.
 
@@ -28,7 +31,12 @@ class Gerrit(object):
 
     '''
 
-    def __init__(self, host, user, port=29418):
+    def __init__(self, app_settings):
+        # XXX These need to be configurable
+        host = 'localhost'
+        port = 29418
+        user = 'lorry'
+
         self._ssh_command_args = [
             'ssh', '-oStrictHostKeyChecking=no', '-oBatchMode=yes', '-p%i' % port,
             '%s@%s' % (user, host)]
@@ -40,7 +48,7 @@ class Gerrit(object):
             out = out.decode('utf-8', errors='replace')
         return out
 
-    def has_project(self, name):
+    def _has_project(self, name):
         # There's no 'does this project exist' command in Gerrit 2.9.4; 'list
         # all projects with this prefix' is as close we can get.
 
@@ -53,5 +61,28 @@ class Gerrit(object):
         else:
             return False
 
-    def create_project(self, name):
-        self._ssh_command(['gerrit', 'create-project', name])
+    def prepare_repo(self, name, metadata):
+        '''Create a project in the local Gerrit server.
+
+        The 'lorry' user must have createProject capability in the Gerrit.
+
+        '''
+
+        if self._has_project(name):
+            logging.info('Project %s exists in local Gerrit already.',
+                         name)
+        else:
+            self._ssh_command(['gerrit', 'create-project', name])
+            logging.info('Created %s project in local Gerrit.', name)
+
+        # We can only set this metadata if we're the owner of the
+        # repository.  For now, ignore failures.
+        try:
+            if 'head' in metadata:
+                self._ssh_command(['gerrit', 'set-head', name,
+                                   '--new-head', metadata['head']])
+            if 'description' in metadata:
+                self._ssh_command(['gerrit', 'set-project', name,
+                                   '-d', metadata['description']])
+        except cliapp.AppException:
+            pass
diff --git a/lorrycontroller/gitano.py b/lorrycontroller/gitano.py
index 7d9c436..499bb5d 100644
--- a/lorrycontroller/gitano.py
+++ b/lorrycontroller/gitano.py
@@ -23,9 +23,10 @@ import cliapp
 import requests
 
 import lorrycontroller
+from . import hosts
 
 
-class GitanoCommandFailure(Exception):
+class _GitanoCommandFailure(Exception):
 
     def __init__(self, trovehost, command, stderr):
         Exception.__init__(
@@ -34,7 +35,7 @@ class GitanoCommandFailure(Exception):
             (command, trovehost, stderr))
 
 
-class GitanoCommand(object):
+class _GitanoCommand(object):
 
     '''Run a Gitano command on a Trove.'''
 
@@ -49,7 +50,7 @@ class GitanoCommand(object):
         elif protocol in ('http', 'https'):
             self._command = self._http_command
         else:
-            raise GitanoCommandFailure(
+            raise _GitanoCommandFailure(
                 self.trovehost, '__init__', 'unknown protocol %s' % protocol)
 
     def whoami(self):
@@ -98,7 +99,7 @@ class GitanoCommand(object):
             logging.error(
                 'Failed to run "%s" for %s:\n%s',
                 quoted_args, self.trovehost, stdout + stderr)
-            raise GitanoCommandFailure(
+            raise _GitanoCommandFailure(
                 self.trovehost,
                 ' '.join(gitano_args),
                 stdout + stderr)
@@ -122,13 +123,13 @@ class GitanoCommand(object):
             else:
                 response = requests.get(url)
         except (requests.exceptions.RequestException) as e:
-            raise GitanoCommandFailure(
+            raise _GitanoCommandFailure(
                 self.trovehost, ' '.join(gitano_args), str(e))
 
         return response.text
 
 
-class LocalTroveGitanoCommand(GitanoCommand):
+class _LocalTroveGitanoCommand(_GitanoCommand):
 
     '''Run commands on the local Trove's Gitano.
 
@@ -138,14 +139,89 @@ class LocalTroveGitanoCommand(GitanoCommand):
     '''
 
     def __init__(self):
-        GitanoCommand.__init__(self, 'localhost', 'ssh', '', '')
+        _GitanoCommand.__init__(self, 'localhost', 'ssh', '', '')
 
 
 
-def new_gitano_command(statedb, trovehost):
-    trove_info = statedb.get_trove_info(trovehost)
-    return lorrycontroller.GitanoCommand(
-        trovehost,
-        trove_info['protocol'],
-        trove_info['username'],
-        trove_info['password'])
+class GitanoDownstream(hosts.DownstreamHost):
+    def __init__(self, app_settings):
+        self._gitano = _LocalTroveGitanoCommand()
+
+    def prepare_repo(self, repo_path, metadata):
+        # Create repository on local Trove. If it fails, assume
+        # it failed because the repository already existed, and
+        # ignore the failure (but log message).
+
+        try:
+            self._gitano.create(repo_path)
+        except _GitanoCommandFailure as e:
+            logging.debug(
+                'Ignoring error creating %s on local Trove: %s',
+                repo_path, e)
+        else:
+            logging.info('Created %s on local repo', repo_path)
+
+        try:
+            local_config = self._gitano.get_gitano_config(repo_path)
+            if 'head' in metadata \
+               and metadata['head'] != local_config['project.head']:
+                self._gitano.set_gitano_config(repo_path,
+                                               'project.head',
+                                               metadata['head'])
+            if 'description' in metadata \
+               and metadata['description'] != \
+                   local_config['project.description']:
+                self._gitano.set_gitano_config(repo_path,
+                                               'project.description',
+                                               metadata['description'])
+        except _GitanoCommandFailure as e:
+            logging.error('ERROR: %s' % str(e))
+            # FIXME: We need a good way to report these errors to the
+            # user. However, we probably don't want to fail the
+            # request, so that's not the way to do this. Needs
+            # thinking.
+
+
+class TroveUpstream(hosts.UpstreamHost):
+    def __init__(self, host_info):
+        self._host_info = host_info
+        self._gitano = _GitanoCommand(host_info['host'],
+                                      host_info['protocol'],
+                                      host_info['username'],
+                                      host_info['password'])
+
+    def list_repos(self):
+        ls_output = self._gitano.ls()
+        repo_paths = []
+        for line in ls_output.splitlines():
+            words = line.split(None, 1)
+            if words[0].startswith('R') and len(words) == 2:
+                repo_paths.append(words[1])
+        return repo_paths
+
+    def get_repo_url(self, remote_path):
+        vars = dict(self._host_info)
+        vars['remote_path'] = remote_path
+
+        patterns = {
+            'ssh':  'ssh://git@{host}/{remote_path}',
+            'https':'https://{username}:{password}@{host}/git/{remote_path}',
+            'http': 'http://{host}/git/{remote_path}',
+            }
+
+        return patterns[self._host_info['protocol']].format(**vars)
+
+    def get_repo_metadata(self, repo_path):
+        try:
+            remote_config = self._gitano.get_gitano_config(repo_path)
+            return {
+                'head':        remote_config['project.head'],
+                'description': remote_config['project.description'],
+            }
+        except _GitanoCommandFailure as e:
+            logging.error('ERROR: %s' % str(e))
+            # FIXME: We need a good way to report these errors to the
+            # user. However, we probably don't want to fail the
+            # request, so that's not the way to do this. Needs
+            # thinking.
+            return {}
diff --git a/lorrycontroller/gitlab.py b/lorrycontroller/gitlab.py
index 13becfe..4f70f0a 100644
--- a/lorrycontroller/gitlab.py
+++ b/lorrycontroller/gitlab.py
@@ -13,7 +13,15 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
+'''
+Run commands on a GitLab instance.
 
+This uses the python wrapper around the GitLab API.
+Use of the API requires the private token of a user with master access
+to the targetted group.
+'''
+
+import logging
 import re
 import urllib.parse
 
@@ -22,40 +30,60 @@ try:
 except ImportError:
     gitlab = None
 
+from . import hosts
+
 
 class MissingGitlabModuleError(Exception):
     pass
 
 
-class Gitlab(object):
+def _init_gitlab(host, token):
+    if gitlab:
+        url = "http://" + host
+        return gitlab.Gitlab(url, token)
+    else:
+        raise MissingGitlabModuleError('gitlab module missing\n'
+            '\tpython-gitlab is required with GitLab as the git server')
 
-    '''Run commands on a GitLab instance.
 
-    This uses the python wrapper around the GitLab API.
-    Use of the API requires the private token of a user with master access
-    to the targetted group.
+class GitlabDownstream(hosts.DownstreamHost):
+    @staticmethod
+    def add_app_settings(app_settings):
+        app_settings.string(
+            ['gitlab-private-token'],
+            'private token for GitLab API access')
 
-    '''
+    @staticmethod
+    def check_app_settings(app_settings):
+        if not app_settings['gitlab-private-token']:
+            logging.error('A private token must be provided to create '
+                          'repositories on a GitLab instance.')
+            app_settings.require('gitlab-private-token')
 
-    def __init__(self, host, token):
-        if gitlab:
-            url = "http://" + host
-            self.gl = gitlab.Gitlab(url, token)
-        else:
-            raise MissingGitlabModuleError('gitlab module missing\n'
-                '\tpython-gitlab is required with GitLab as the git server')
+    def __init__(self, app_settings):
+        # XXX This needs to be configurable
+        host = 'localhost'
 
-    def find_project(self, repo_path):
-        return self.gl.projects.get(repo_path)
+        self.gl = _init_gitlab(host, app_settings['gitlab-private-token'])
+
+    def prepare_repo(self, repo_path, metadata):
 
-    def has_project(self, repo_path):
         try:
-            self.find_project(repo_path)
-            return True
+            project = self.gl.projects.get(repo_path)
         except gitlab.GitlabGetError:
-            return False
+            pass
+        else:
+            logging.info('Project %s exists in local GitLab already.',
+                         repo_path)
+            if 'head' in metadata \
+               and project.default_branch != metadata['head']:
+                project.default_branch = metadata['head']
+            if 'description' in metadata \
+               and project.description != metadata['description']:
+                project.description = metadata['description']
+            project.save()
+            return
 
-    def create_project(self, repo_path):
         path_comps = repo_path.split('/')
 
         if len(path_comps) < 2:
@@ -84,15 +112,34 @@ class Gitlab(object):
             'public': True,
             'merge_requests_enabled': False,
             'namespace_id': group.id,
+            'default_branch': metadata.get('head'),
+            'description': metadata.get('description'),
         }
         self.gl.projects.create(project)
 
-    def list_projects(self):
+        logging.info('Created %s project in local GitLab.', repo_path)
+
+
+class GitlabUpstream(hosts.UpstreamHost):
+    @staticmethod
+    def check_host_type_params(validator, section):
+        validator.check_has_required_fields(section, ['private-token'])
+
+    @staticmethod
+    def get_host_type_params(section):
+        return {'private-token': section['private-token']}
+
+    def __init__(self, host_info):
+        self._protocol = host_info['protocol']
+        self.gl = _init_gitlab(host_info['host'],
+                               host_info['type_params']['private-token'])
+
+    def list_repos(self):
         '''List projects on a GitLab instance.'''
 
         return [x.path_with_namespace for x in self.gl.projects.list()]
 
-    def get_project_url(self, protocol, project_path):
+    def get_repo_url(self, repo_path):
         '''Return the clone url for a GitLab project.
 
         Depending on the protocol specified, will return a suitable clone url.
@@ -105,11 +152,20 @@ class Gitlab(object):
         format matching 'http(s)://host/group/project.git'.
         '''
 
-        project = self.find_project(project_path)
+        project = self.gl.projects.get(repo_path)
 
-        if protocol == 'ssh':
+        if self._protocol == 'ssh':
             return project.ssh_url_to_repo
-        elif protocol in ('http', 'https'):
+        elif self._protocol in ('http', 'https'):
             split = urllib.parse.urlsplit(project.http_url_to_repo)
             return urllib.parse.urlunsplit((
-                protocol, split.netloc, split.path, '', ''))
+                self._protocol, split.netloc, split.path, '', ''))
+
+    def get_repo_metadata(self, repo_path):
+        project = self.gl.projects.get(repo_path)
+        metadata = {}
+        if project.default_branch is not None:
+            metadata['head'] = project.default_branch
+        if project.description is not None:
+            metadata['description'] = project.description
+        return metadata
diff --git a/lorrycontroller/givemejob.py b/lorrycontroller/givemejob.py
index 9d4d4d2..6736b35 100644
--- a/lorrycontroller/givemejob.py
+++ b/lorrycontroller/givemejob.py
@@ -36,9 +36,12 @@ class GiveMeJob(lorrycontroller.LorryControllerRoute):
                 now = statedb.get_current_time()
                 for lorry_info in lorry_infos:
                     if self.ready_to_run(lorry_info, now):
-                        self.create_repository(statedb, lorry_info)
-                        if lorry_info['from_host']:
-                            self.copy_repository_metadata(statedb, lorry_info)
+                        metadata = self.get_repo_metadata(statedb, lorry_info)
+                        downstream_type = lorrycontroller.downstream_types[
+                            self.app_settings['git-server-type']]
+                        downstream_type(self.app_settings) \
+                            .prepare_repo(lorry_info['path'], metadata)
+
                         self.give_job_to_minion(statedb, lorry_info, now)
                         logging.info(
                             'Giving job %s to lorry %s to MINION %s:%s',
@@ -62,98 +65,36 @@ class GiveMeJob(lorrycontroller.LorryControllerRoute):
         due = lorry_info['last_run'] + lorry_info['interval']
         return (lorry_info['running_job'] is None and due <= now)
 
-    def create_repository(self, statedb, lorry_info):
-        api = self.app_settings['git-server-type']
-        if api == 'gitano':
-            self.create_repository_in_local_trove(statedb, lorry_info)
-        elif api == 'gerrit':
-            self.create_gerrit_project(statedb, lorry_info)
-        elif api == 'gitlab':
-            self.create_gitlab_project(statedb, lorry_info)
-        elif api == 'local':
-            pass
-
-    def create_repository_in_local_trove(self, statedb, lorry_info):
-        # Create repository on local Trove. If it fails, assume
-        # it failed because the repository already existed, and
-        # ignore the failure (but log message).
-
-        local = lorrycontroller.LocalTroveGitanoCommand()
-        try:
-            local.create(lorry_info['path'])
-        except lorrycontroller.GitanoCommandFailure as e:
-            logging.debug(
-                'Ignoring error creating %s on local Trove: %s',
-                lorry_info['path'], e)
-        else:
-            logging.info('Created %s on local repo', lorry_info['path'])
-
-    def create_gerrit_project(self, statedb, lorry_info):
-        '''Create a project in the local Gerrit server.
-
-        The 'lorry' user must have createProject capability in the Gerrit.
-
-        '''
-        gerrit = lorrycontroller.Gerrit(
-            host='localhost', user='lorry')
-        project_name = lorry_info['path']
-
-        if gerrit.has_project(project_name):
-            logging.info('Project %s exists in local Gerrit already.',
-                         project_name)
-        else:
-            gerrit.create_project(project_name)
-            logging.info('Created %s project in local Gerrit.', project_name)
-
-    def create_gitlab_project(self, statedb, lorry_info):
-        gitlab = lorrycontroller.Gitlab(
-            'localhost', self.app_settings['gitlab-private-token'])
-        project_name = lorry_info['path']
-
-        if gitlab.has_project(project_name):
-            logging.info('Project %s exists in local GitLab already.',
-                         project_name)
-        else:
-            gitlab.create_project(lorry_info['path'])
-            logging.info('Created %s project in local GitLab.', project_name)
-
-    def copy_repository_metadata(self, statedb, lorry_info):
-        '''Copy project.head and project.description to the local Trove.'''
-
-        assert lorry_info['from_host']
-        assert lorry_info['from_path']
+    def get_repo_metadata(self, statedb, lorry_info):
+        '''Get repository head and description.'''
 
-        if self.app_settings['git-server-type'] != 'gitano':
-            # FIXME: would be good to have this info in Gerrit too
-            return
+        if not lorry_info['from_host']:
+            return {}
 
-        remote = lorrycontroller.new_gitano_command(statedb, lorry_info['from_host'])
-        local = lorrycontroller.LocalTroveGitanoCommand()
+        assert lorry_info['from_path']
 
         try:
-            remote_config = remote.get_gitano_config(lorry_info['from_path'])
-            local_config = local.get_gitano_config(lorry_info['path'])
-
-            if remote_config['project.head'] != local_config['project.head']:
-                local.set_gitano_config(
-                    lorry_info['path'],
-                    'project.head',
-                    remote_config['project.head'])
-
-            if not local_config['project.description']:
-                desc = '{host}: {desc}'.format(
-                    host=lorry_info['from_host'],
-                    desc=remote_config['project.description'])
-                local.set_gitano_config(
-                    lorry_info['path'],
-                    'project.description',
-                    desc)
-        except lorrycontroller.GitanoCommandFailure as e:
-            logging.error('ERROR: %s' % str(e))
-            # FIXME: We need a good way to report these errors to the
-            # user. However, we probably don't want to fail the
-            # request, so that's not the way to do this. Needs
-            # thinking.
+            host_info = statedb.get_host_info(lorry_info['from_host'])
+        except lorrycontroller.HostNotFoundError:
+            # XXX We don't know whether upstream is Trove.  It should be
+            # possible to set host type for single repositories.
+            host_info = {
+                'host':        lorry_info['from_host'],
+                'protocol':    'ssh',
+                'username':    None,
+                'password':    None,
+                'type':        'trove',
+                'type_params': {},
+            }
+
+        metadata = lorrycontroller.get_upstream_host(host_info) \
+                                  .get_repo_metadata(lorry_info['from_path'])
+        if 'description' in metadata:
+            # Prepend Upstream Host name
+            metadata['description'] = '{host}: {desc}'.format(
+                host=lorry_info['from_host'],
+                desc=metadata['description'])
+        return metadata
 
     def give_job_to_minion(self, statedb, lorry_info, now):
         path = lorry_info['path']
diff --git a/lorrycontroller/hosts.py b/lorrycontroller/hosts.py
new file mode 100644
index 0000000..39cae57
--- /dev/null
+++ b/lorrycontroller/hosts.py
@@ -0,0 +1,121 @@
+# Copyright (C) 2020  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import abc
+
+
+class DownstreamHost(abc.ABC):
+    @staticmethod
+    def add_app_settings(app_settings):
+        '''Add any application settings that are specific to this Downstream
+        Host type.
+        '''
+        pass
+
+    @staticmethod
+    def check_app_settings(app_settings):
+        '''Validate any fields in the application settings that are specific
+        to this Downstream Host type.
+        '''
+        pass
+
+    @abc.abstractmethod
+    def __init__(self, app_settings):
+        '''Construct a Downstream Host connector from the application
+        settings.
+        '''
+        pass
+
+    @abc.abstractmethod
+    def prepare_repo(self, repo_path, metadata):
+        '''Prepare a repository on the Host.  If the repository does not
+        exist, this method must create it.  It should also set any
+        given metadata on the repository, whether or not it already
+        exists.
+
+        repo_path is the path that the repository should appear at
+        within the Host.
+
+        metadata is a dictionary with the following (optional) keys
+        defined:
+
+        - head: Name of the default branch (a.k.a. HEAD)
+        - description: Short string describing the repository
+        '''
+        pass
+
+
+class UpstreamHost(abc.ABC):
+    @staticmethod
+    def check_host_type_params(validator, section):
+        '''Validate any type-specific fields in a CONFGIT host section.
+
+        validator is an instance of LorryControllerConfValidator that
+        may be used to check the types of configuration fields.
+
+        section is the dictionary of fields for the section.
+
+        Returns None if the configuration is valid; raises an
+        exception on error.
+        '''
+        pass
+
+    @staticmethod
+    def get_host_type_params(section):
+        '''Convert any type-specific fields in a CONFGIT host section into a
+        dictionary that will be stored in STATEDB.
+
+        section is the dictionary of fields for the section.
+
+        Returns a dictionary, which may be empty.  This will be stored
+        in STATEDB as the type_params of the host.
+        '''
+        return {}
+
+    @abc.abstractmethod
+    def __init__(self, host_info):
+        '''Construct an Upstream Host connector from the given host_info.
+        The host_info comes directly from STATEDB.
+        '''
+        pass
+
+    @abc.abstractmethod
+    def list_repos(self):
+        '''List all visible repositories on the Host.
+
+        Returns a list of path strings.
+        '''
+        pass
+
+    @abc.abstractmethod
+    def get_repo_url(self, repo_path):
+        '''Get URL for a repository.
+
+        repo_path is the path to the repository within the Host.
+
+        Returns a URL string suitable for passing to git clone.
+        '''
+        pass
+
+    @abc.abstractmethod
+    def get_repo_metadata(self, repo_path):
+        '''Get metadata for a repository.
+
+        repo_path is the path to the repository within the Host.
+
+        Returns a dictionary of metadata suitable for passing to
+        DownstreamHost.prepare_repo.
+        '''
+        pass
diff --git a/lorrycontroller/local.py b/lorrycontroller/local.py
new file mode 100644
index 0000000..d55214d
--- /dev/null
+++ b/lorrycontroller/local.py
@@ -0,0 +1,56 @@
+# Copyright (C) 2020  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import logging
+import os
+import os.path
+
+import cliapp
+
+from . import hosts
+
+
+class LocalDownstream(hosts.DownstreamHost):
+    @staticmethod
+    def add_app_settings(app_settings):
+        app_settings.string(
+            ['local-base-directory'],
+            'Base directory for local Downstream Host')
+
+    @staticmethod
+    def check_app_settings(app_settings):
+        if not app_settings['local-base-directory']:
+            logging.error('A base directory must be provided to create '
+                          'repositories on a local filesystem.')
+            app_settings.require('local-base-directory')
+
+    def __init__(self, app_settings):
+        self._base_dir = app_settings['local-base-directory']
+
+    def prepare_repo(self, repo_path, metadata):
+        repo_path = '%s/%s.git' % (self._base_dir, repo_path)
+
+        # These are idempotent, so we don't need to explicitly check
+        # whether the repository already exists
+        os.makedirs(repo_path, exist_ok=True)
+        cliapp.runcmd(['git', 'init', '--bare', repo_path])
+
+        if 'head' in metadata:
+            cliapp.runcmd(['git', '--git-dir', repo_path,
+                           'symbolic-ref', 'HEAD',
+                           'refs/heads/' + metadata['head']])
+        if 'description' in metadata:
+            with open(os.path.join(repo_path, 'description'), 'w') as f:
+                print(metadata['description'], file=f)
diff --git a/lorrycontroller/lsupstreams.py b/lorrycontroller/lsupstreams.py
index b31a385..a535174 100644
--- a/lorrycontroller/lsupstreams.py
+++ b/lorrycontroller/lsupstreams.py
@@ -54,7 +54,7 @@ class HostRepositoryLister(object):
         if self.app_settings['debug-fake-upstream-host']:
             repo_paths = self.get_fake_ls_output(host_info)
         else:
-            repo_paths = self.get_real_ls_output(statedb, host_info)
+            repo_paths = self.get_real_ls_output(host_info)
 
         return repo_paths
 
@@ -68,25 +68,8 @@ class HostRepositoryLister(object):
                 return obj['ls-output']
         return None
 
-    def get_real_ls_output(self, statedb, host_info):
-        if host_info['type'] == 'gitlab':
-            return lorrycontroller.Gitlab(host_info['host'],
-                                          host_info['type_params']
-                                          ['private-token']) \
-                                  .list_projects()
-
-        gitano = lorrycontroller.new_gitano_command(
-                statedb, host_info['host'])
-        output = gitano.ls()
-        return self.parse_ls_output(output)
-
-    def parse_ls_output(self, ls_output):
-        repo_paths = []
-        for line in ls_output.splitlines():
-            words = line.split(None, 1)
-            if words[0].startswith('R') and len(words) == 2:
-                repo_paths.append(words[1])
-        return repo_paths
+    def get_real_ls_output(self, host_info):
+        return lorrycontroller.get_upstream_host(host_info).list_repos()
 
     def skip_ignored_repos(self, host, repo_paths):
         ignored_patterns = json.loads(host['ignore'])
@@ -156,23 +139,8 @@ class HostRepositoryLister(object):
             }
 
     def construct_lorry_url(self, host_info, remote_path):
-        if host_info['type'] == 'gitlab':
-            return lorrycontroller.Gitlab(host_info['host'],
-                                          host_info['type_params']
-                                          ['private-token']) \
-                                  .get_project_url(host_info['protocol'],
-                                                   remote_path)
-
-        vars = dict(host_info)
-        vars['remote_path'] = remote_path
-
-        patterns = {
-            'ssh':  'ssh://git@{host}/{remote_path}',
-            'https':'https://{username}:{password}@{host}/git/{remote_path}',
-            'http': 'http://{host}/git/{remote_path}',
-            }
-
-        return patterns[host_info['protocol']].format(**vars)
+        return lorrycontroller.get_upstream_host(host_info) \
+                              .get_repo_url(remote_path)
 
 
 class ForceLsUpstream(lorrycontroller.LorryControllerRoute):
diff --git a/lorrycontroller/readconf.py b/lorrycontroller/readconf.py
index b95b9de..3303f68 100644
--- a/lorrycontroller/readconf.py
+++ b/lorrycontroller/readconf.py
@@ -70,7 +70,7 @@ class ReadConfiguration(lorrycontroller.LorryControllerRoute):
                     added = self.add_matching_lorries_to_statedb(
                         statedb, section)
                     lorries_to_remove = lorries_to_remove.difference(added)
-                elif section['type'] in ('trove', 'gitlab'):
+                elif section['type'] in lorrycontroller.upstream_types:
                     self.add_host(statedb, section)
                     host = section.get('host') or section['trovehost']
                     if host in hosts_to_remove:
@@ -290,9 +290,8 @@ class ReadConfiguration(lorrycontroller.LorryControllerRoute):
             username = auth.get('username')
             password = auth.get('password')
 
-        type_params = {}
-        if section['type'] == 'gitlab':
-            type_params['private-token'] = section['private-token']
+        type_params = lorrycontroller.upstream_types[section['type']] \
+                                     .get_host_type_params(section)
 
         statedb.add_host(
             host=section.get('host') or section['trovehost'],
@@ -319,8 +318,8 @@ class LorryControllerConfValidator(object):
 
     def validate_config(self, conf_obj):
         try:
-            self._check_is_list(conf_obj)
-            self._check_is_list_of_dicts(conf_obj)
+            self.check_is_list(conf_obj)
+            self.check_is_list_of_dicts(conf_obj)
 
             for section in conf_obj:
                 if 'type' not in section:
@@ -329,12 +328,12 @@ class LorryControllerConfValidator(object):
                 # Backward compatibility
                 if section['type'] == 'troves':
                     section['type'] = 'trove'
-                if section['type'] == 'trove':
+                if section['type'] in lorrycontroller.upstream_types:
                     self._check_host_section(section)
+                    lorrycontroller.upstream_types[section['type']] \
+                                   .check_host_type_params(self, section)
                 elif section['type'] == 'lorries':
                     self._check_lorries_section(section)
-                elif section['type'] == 'gitlab':
-                    self._check_gitlab_section(section)
                 else:
                     raise ValidationError(
                         'unknown section type %r' % section['type'])
@@ -343,30 +342,26 @@ class LorryControllerConfValidator(object):
 
         return None
 
-    def _check_is_list(self, conf_obj):
+    def check_is_list(self, conf_obj):
         if type(conf_obj) is not list:
             raise ValidationError(
                 'type %r is not a JSON list' % type(conf_obj))
 
-    def _check_is_list_of_dicts(self, conf_obj):
+    def check_is_list_of_dicts(self, conf_obj):
         for item in conf_obj:
             if type(item) is not dict:
                 raise ValidationError('all items must be dicts')
 
-    def _check_gitlab_section(self, section):
-        self._check_host_section(section)
-        self._check_has_required_fields(section, ['private-token'])
-
     def _check_host_section(self, section):
         if not any(i in ('trovehost', 'host') for i in section):
-            self._check_has_required_fields(section, ['host'])
-        self._check_has_required_fields(
+            self.check_has_required_fields(section, ['host'])
+        self.check_has_required_fields(
             section,
             ['protocol', 'interval', 'ls-interval', 'prefixmap'])
         self._check_protocol(section)
         self._check_prefixmap(section)
         if 'ignore' in section:
-            self._check_is_list_of_strings(section, 'ignore')
+            self.check_is_list_of_strings(section, 'ignore')
 
     def _check_protocol(self, section):
         valid = ('ssh', 'http', 'https')
@@ -383,18 +378,18 @@ class LorryControllerConfValidator(object):
         pass
 
     def _check_lorries_section(self, section):
-        self._check_has_required_fields(
+        self.check_has_required_fields(
             section, ['interval', 'prefix', 'globs'])
-        self._check_is_list_of_strings(section, 'globs')
+        self.check_is_list_of_strings(section, 'globs')
 
-    def _check_has_required_fields(self, section, fields):
+    def check_has_required_fields(self, section, fields):
         for field in fields:
             if field not in section:
                 raise ValidationError(
                     'mandatory field %s missing in section %r' %
                     (field, section))
 
-    def _check_is_list_of_strings(self, section, field):
+    def check_is_list_of_strings(self, section, field):
         obj = section[field]
         if not isinstance(obj, list) or not all(
                 isinstance(s, str) for s in obj):