Add support for lorrying from a Gitlab server

Change-Id: I2bb0aaf428e331a0bcd5a1e3111d4c7bca4afede

5 files changed, 93 insertions, 24 deletions

diff --git a/README b/README
index c62d024..5fe1d93 100644
--- a/README
+++ b/README
@@ -64,9 +64,10 @@ The `lorry-controller.conf` file
 --------------------------------
 
 `lorry-controller.conf` is a JSON file containing a list of maps. Each
-map specifies another Trove, or one set of `.lorry` files. Here's an
-example that tells LC to mirror the `git.baserock.org` Trove and
-anything in the `open-source-lorries/*.lorry` files (if any exist).
+map specifies another Trove, a GitLab instance, or one set of `.lorry`
+files. Here's an example that tells LC to mirror the `git.baserock.org`
+Trove and anything in the `open-source-lorries/*.lorry` files (if any
+exist).
 
     [
         {
@@ -130,6 +131,15 @@ specifications:
   (only). It should be a dictionary with the fields `username` and
   `password`.
 
+A GitLab specification (map) makes use of the same keys as a Trove,
+however it uses an additional mandatory key:
+
+* `type: gitlab` -- specify it's a GitLab specification.
+
+* `private-token` -- the GitLab private token for a user with the
+  minimum permissions of master of any group you may wish to create
+  repositories under.
+
 A Lorry specification (map) uses the following keys, all of them
 mandatory:
 
diff --git a/lorrycontroller/gitlab.py b/lorrycontroller/gitlab.py
index 5c7e579..659d179 100644
--- a/lorrycontroller/gitlab.py
+++ b/lorrycontroller/gitlab.py
@@ -14,6 +14,7 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 from __future__ import absolute_import
+import urlparse
 import itertools
 try:
     import gitlab
@@ -82,3 +83,28 @@ class Gitlab(object):
         }
         self.gl.projects.create(project)
 
+    def list_projects(self):
+        return [x.path_with_namespace for x in self.gl.projects.list()]
+
+    def get_project_url(self, protocol, project_path):
+        '''Return the clone url for a GitLab project.
+
+        Depending on the protocol specified, will return a suitable clone url.
+        If 'ssh', a url in the format 'git@host:group/project.git' will be
+        returned.
+        If 'http' or 'https', the http_url_to_repo from the GitLab API is split
+        with urlparse into its constituent parts: the protocol (http by
+        default), the host, and the path ('group/project.git').  This is then
+        rejoined, replacing the protocol with what is specified.  The resulting
+        format matching 'http(s)://host/group/project.git'.
+        '''
+
+        group, project = self.split_path(project_path)
+        project = self.find_project(group, project)
+
+        if protocol == 'ssh':
+            return project.ssh_url_to_repo
+        elif protocol in ('http', 'https'):
+            split = urlparse.urlsplit(project.http_url_to_repo)
+            return urlparse.urlunsplit((
+                protocol, split.netloc, split.path, '', ''))
diff --git a/lorrycontroller/lstroves.py b/lorrycontroller/lstroves.py
index 72515f5..456359c 100644
--- a/lorrycontroller/lstroves.py
+++ b/lorrycontroller/lstroves.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2015  Codethink Limited
+# Copyright (C) 2014-2016  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -23,14 +23,14 @@ import bottle
 import lorrycontroller
 
 
-class GitanoLsError(Exception):
+class ServerLsError(Exception):
 
-    def __init__(self, trovehost, output):
+    def __init__(self, remote_host, output):
         Exception.__init__(
             self,
             'Failed to get list of git repositories '
-            'on remote host %s:\n%s' % (trovehost, output))
-        self.trovehost = trovehost
+            'on remote host %s:\n%s' % (remote_host, output))
+        self.remote_host = remote_host
 
 
 class TroveRepositoryLister(object):
@@ -69,7 +69,13 @@ class TroveRepositoryLister(object):
         return None
 
     def get_real_ls_output(self, statedb, trove_info):
-        gitano = lorrycontroller.new_gitano_command(statedb, trove_info['trovehost'])
+        gitlab_token = trove_info.get('gitlab_token')
+        if gitlab_token:
+            return lorrycontroller.Gitlab(
+                trove_info['trovehost'], gitlab_token).list_projects()
+
+        gitano = lorrycontroller.new_gitano_command(
+                statedb, trove_info['trovehost'])
         output = gitano.ls()
         return self.parse_ls_output(output)
 
@@ -149,6 +155,12 @@ class TroveRepositoryLister(object):
             }
 
     def construct_lorry_url(self, trove_info, remote_path):
+        gitlab_token = trove_info.get('gitlab_token')
+        if gitlab_token:
+            return lorrycontroller.Gitlab(
+                trove_info['trovehost'], gitlab_token).get_project_url(
+                    trove_info['protocol'], remote_path)
+
         vars = dict(trove_info)
         vars['remote_path'] = remote_path
 
@@ -177,7 +189,7 @@ class ForceLsTrove(lorrycontroller.LorryControllerRoute):
         trove_info = statedb.get_trove_info(trovehost)
         try:
             updated = lister.list_trove_into_statedb(statedb, trove_info)
-        except GitanoLsError as e:
+        except ServerLsError as e:
             raise bottle.abort(500, str(e))
 
         return { 'updated-troves': updated }
@@ -199,7 +211,7 @@ class LsTroves(lorrycontroller.LorryControllerRoute):
             logging.info('Trove %r is due an ls', trove_info['trovehost'])
             try:
                 lister.list_trove_into_statedb(statedb, trove_info)
-            except GitanoLsError as e:
+            except ServerLsError as e:
                 bottle.abort(500, str(e))
 
         return {
diff --git a/lorrycontroller/readconf.py b/lorrycontroller/readconf.py
index a108c41..5323a3f 100644
--- a/lorrycontroller/readconf.py
+++ b/lorrycontroller/readconf.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2014  Codethink Limited
+# Copyright (C) 2014-2016  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -69,7 +69,7 @@ class ReadConfiguration(lorrycontroller.LorryControllerRoute):
                     added = self.add_matching_lorries_to_statedb(
                         statedb, section)
                     lorries_to_remove = lorries_to_remove.difference(added)
-                elif section['type'] in ('trove', 'troves'):
+                elif section['type'] in ('trove', 'troves', 'gitlab'):
                     self.add_trove(statedb, section)
                     trovehost = section['trovehost']
                     if trovehost in troves_to_remove:
@@ -286,6 +286,10 @@ class ReadConfiguration(lorrycontroller.LorryControllerRoute):
             username = auth.get('username')
             password = auth.get('password')
 
+        gitlab_token = None
+        if section['type'] == 'gitlab':
+            gitlab_token = section['private-token']
+
         statedb.add_trove(
             trovehost=section['trovehost'],
             protocol=section['protocol'],
@@ -296,7 +300,8 @@ class ReadConfiguration(lorrycontroller.LorryControllerRoute):
                 'lorry-timeout', self.DEFAULT_LORRY_TIMEOUT),
             ls_interval=section['ls-interval'],
             prefixmap=json.dumps(section['prefixmap']),
-            ignore=json.dumps(section.get('ignore', [])))
+            ignore=json.dumps(section.get('ignore', [])),
+            gitlab_token=gitlab_token)
 
 
 class ValidationError(Exception):
@@ -320,6 +325,8 @@ class LorryControllerConfValidator(object):
                     self._check_troves_section(section)
                 elif section['type'] == 'lorries':
                     self._check_lorries_section(section)
+                elif section['type'] == 'gitlab':
+                    self._check_gitlab_section(section)
                 else:
                     raise ValidationError(
                         'unknown section type %r' % section['type'])
@@ -338,6 +345,11 @@ class LorryControllerConfValidator(object):
             if type(item) is not dict:
                 raise ValidationError('all items must be dicts')
 
+    def _check_gitlab_section(self, section):
+        # gitlab section inherits trove configurations, perform the same checks.
+        self._check_troves_section(section)
+        self._check_has_required_fields(section, ['private-token'])
+
     def _check_troves_section(self, section):
         self._check_has_required_fields(
             section,
diff --git a/lorrycontroller/statedb.py b/lorrycontroller/statedb.py
index 7f537f3..27e6fae 100644
--- a/lorrycontroller/statedb.py
+++ b/lorrycontroller/statedb.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2014  Codethink Limited
+# Copyright (C) 2014-2016  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -80,6 +80,13 @@ class StateDB(object):
             logging.debug('New connection is %r', self._conn)
             if not existed:
                 self._initialise_tables()
+            else:
+                c = self._conn.cursor()
+                c.execute('PRAGMA table_info(troves)')
+                columns = (info[1] for info in c.fetchall())
+                if 'gitlab_token' not in columns:
+                    c.execute('ALTER TABLE troves ADD COLUMN gitlab_token')
+                    self._conn.commit()
 
     def _initialise_tables(self):
         logging.debug('Initialising tables in database')
@@ -106,7 +113,8 @@ class StateDB(object):
             'ls_interval INT, '
             'ls_last_run INT, '
             'prefixmap TEXT, '
-            'ignore TEXT '
+            'ignore TEXT, '
+            'gitlab_token TEXT '
             ')')
 
         # Table for all the known lorries (the "run queue").
@@ -215,7 +223,7 @@ class StateDB(object):
         c.execute(
             'SELECT protocol, username, password, lorry_interval, '
             'lorry_timeout, ls_interval, ls_last_run, '
-            'prefixmap, ignore '
+            'prefixmap, ignore, gitlab_token '
             'FROM troves WHERE trovehost IS ?',
             (trovehost,))
         row = c.fetchone()
@@ -232,12 +240,13 @@ class StateDB(object):
             'ls_last_run': row[6],
             'prefixmap': row[7],
             'ignore': row[8],
-            }
+            'gitlab_token': row[9]
+        }
 
     def add_trove(self, trovehost=None, protocol=None, username=None,
                   password=None, lorry_interval=None,
                   lorry_timeout=None, ls_interval=None,
-                  prefixmap=None, ignore=None):
+                  prefixmap=None, ignore=None, gitlab_token=None):
         logging.debug(
             'StateDB.add_trove(%r,%r,%r,%r,%r,%r) called',
             trovehost, lorry_interval, lorry_timeout, ls_interval,
@@ -261,20 +270,20 @@ class StateDB(object):
                 '(trovehost, protocol, username, password, '
                 'lorry_interval, lorry_timeout, '
                 'ls_interval, ls_last_run, '
-                'prefixmap, ignore) '
-                'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
+                'prefixmap, ignore, gitlab_token) '
+                'VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)',
                 (trovehost, protocol, username, password,
                  lorry_interval, lorry_timeout, ls_interval, 0,
-                 prefixmap, ignore))
+                 prefixmap, ignore, gitlab_token))
         else:
             c = self.get_cursor()
             c.execute(
                 'UPDATE troves '
                 'SET lorry_interval=?, lorry_timeout=?, ls_interval=?, '
-                'prefixmap=?, ignore=?, protocol=? '
+                'prefixmap=?, ignore=?, protocol=?, gitlab_token=? '
                 'WHERE trovehost IS ?',
                 (lorry_interval, lorry_timeout, ls_interval, prefixmap,
-                 ignore, protocol, trovehost))
+                 ignore, protocol, gitlab_token, trovehost))
 
     def remove_trove(self, trovehost):
         logging.debug('StateDB.remove_trove(%r) called', trovehost)