Merge branch 'bwh/single-lorry-metadata' into 'master'

Add metadata for single repositories

Closes #15

See merge request CodethinkLabs/lorry/lorry-controller!17

3 files changed, 124 insertions, 25 deletions

diff --git a/README.md b/README.md
index 32c804d..8723214 100644
--- a/README.md
+++ b/README.md
@@ -167,7 +167,9 @@ hours (`h`), and days (`d`), expressed as single-letter codes in upper
 or lower case.
 
 The syntax of `.lorry` files is specified by the Lorry program; see
-its documentation for details.
+its documentation for details.  Lorry Controller supports an
+optional `description` field in `.lorry` files that is used to set
+the repository description on the Downstream Host.
 
 
 HTTP proxy configuration: `proxy.conf`
diff --git a/lorrycontroller/gitlab.py b/lorrycontroller/gitlab.py
index 0b5e1c2..266861c 100644
--- a/lorrycontroller/gitlab.py
+++ b/lorrycontroller/gitlab.py
@@ -76,13 +76,22 @@ class GitlabDownstream(hosts.DownstreamHost):
         else:
             logging.info('Project %s exists in local GitLab already.',
                          repo_path)
-            if 'head' in metadata \
-               and project.default_branch != metadata['head']:
-                project.default_branch = metadata['head']
+
             if 'description' in metadata \
                and project.description != metadata['description']:
                 project.description = metadata['description']
             project.save()
+
+            # This will fail if we haven't created the branch yet.
+            # We'll fix it next time round.
+            try:
+                if 'head' in metadata \
+                   and project.default_branch != metadata['head']:
+                    project.default_branch = metadata['head']
+                project.save()
+            except gitlab.GitlabUpdateError:
+                pass
+
             return
 
         path_comps = repo_path.split('/')
diff --git a/lorrycontroller/givemejob.py b/lorrycontroller/givemejob.py
index 1a0fe35..ee998ea 100644
--- a/lorrycontroller/givemejob.py
+++ b/lorrycontroller/givemejob.py
@@ -13,10 +13,13 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-
+import json
 import logging
+import re
+import urllib.parse
 
 import bottle
+import cliapp
 
 import lorrycontroller
 
@@ -65,35 +68,120 @@ class GiveMeJob(lorrycontroller.LorryControllerRoute):
         due = lorry_info['last_run'] + lorry_info['interval']
         return (lorry_info['running_job'] is None and due <= now)
 
-    def get_repo_metadata(self, statedb, lorry_info):
-        '''Get repository head and description.'''
-
-        if not lorry_info['from_host']:
-            return {}
-
+    @staticmethod
+    def get_upstream_host_repo_metadata(lorry_info):
+        assert lorry_info['from_host']
         assert lorry_info['from_path']
 
         try:
             host_info = statedb.get_host_info(lorry_info['from_host'])
         except lorrycontroller.HostNotFoundError:
-            # XXX We don't know whether upstream is Trove.  It should be
-            # possible to set host type for single repositories.
-            host_info = {
-                'host':        lorry_info['from_host'],
-                'protocol':    'ssh',
-                'username':    None,
-                'password':    None,
-                'type':        'trove',
-                'type_params': {},
-            }
-
-        metadata = lorrycontroller.get_upstream_host(host_info) \
+            # XXX Shouldn't happen, but currently the database schema
+            # does not prevent it
+            return {}
+        else:
+            return lorrycontroller.get_upstream_host(host_info) \
                                   .get_repo_metadata(lorry_info['from_path'])
-        if 'description' in metadata:
+
+    @staticmethod
+    def get_single_repo_metadata(lorry_info):
+        assert not lorry_info['from_host']
+
+        lorry_dict = json.loads(lorry_info['text'])
+        _, upstream_config = lorry_dict.popitem()
+        upstream_type = upstream_config['type']
+
+        # Get the repository URL
+        url = None
+        try:
+            url = upstream_config['url'].strip()
+        except KeyError:
+            if upstream_type == 'bzr':
+                try:
+                    url = upstream_config['branches']['trunk'].strip()
+                except KeyError:
+                    pass
+
+        # Extract the host-name and repo path
+        host_name, repo_path = None, None
+        if url:
+            # Handle pseudo-URLs
+            if upstream_type == 'bzr':
+                if url.startswith('lp:'):
+                    host_name = 'launchpad.net'
+                    repo_path = url[3:]
+            elif upstream_type == 'cvs':
+                # :pserver:user@host:/path, user@host:/path, etc.
+                match = re.match(r'^(?::[^:@/]+:)?(?:[^:@/]+@)?([^:@/]+):/',
+                                 url)
+                if match:
+                    host_name = match.group(1)
+                    repo_path = url[match.end():].rstrip('/')
+            elif upstream_type == 'git':
+                # user@host:path, host:path.  Path must not start with
+                # '//' as that indicates a real URL.
+                match = re.match(r'^(?:[^:@/]+@)?([^:@/]+):(?!//)', url)
+                if match:
+                    host_name = match.group(1)
+                    repo_path = url[match.end():].strip('/')
+
+            # Default to parsing as a real URL
+            if not host_name:
+                try:
+                    url_obj = urllib.parse.urlparse(url)
+                except ValueError:
+                    pass
+                else:
+                    host_name = url_obj.hostname
+                    repo_path = url_obj.path.strip('/')
+
+        metadata = {}
+
+        # Determine the default branch
+        if upstream_type == 'bzr':
+            # Default in Bazaar is 'trunk' and we don't remap it
+            metadata['head'] = 'trunk'
+        elif upstream_type == 'git':
+            if url:
+                # Query the remote to find its default
+                try:
+                    output = cliapp.runcmd(['git', 'ls-remote', '--symref',
+                                            '--', url, 'HEAD']) \
+                                   .decode('utf-8', errors='replace')
+                    match = re.match(r'^ref: refs/heads/([^\s]+)\tHEAD\n',
+                                     output)
+                    if match:
+                        metadata['head'] = match.group(1)
+                except cliapp.AppException:
+                    pass
+        else:
+            # We currently produce 'master' for all other types
+            metadata['head'] = 'master'
+
+        # Use description from .lorry file, or repository name
+        try:
+            metadata['description'] = upstream_config['description']
+        except KeyError:
+            if repo_path:
+                metadata['description'] = repo_path
+
+        return host_name, metadata
+
+    def get_repo_metadata(self, statedb, lorry_info):
+        '''Get repository head and description.'''
+
+        host_name = lorry_info['from_host']
+        if host_name:
+            metadata = self.get_upstream_host_repo_metadata(lorry_info)
+        else:
+            host_name, metadata = self.get_single_repo_metadata(lorry_info)
+
+        if host_name and 'description' in metadata:
             # Prepend Upstream Host name
             metadata['description'] = '{host}: {desc}'.format(
-                host=lorry_info['from_host'],
+                host=host_name,
                 desc=metadata['description'])
+
         return metadata
 
     def give_job_to_minion(self, statedb, lorry_info, now):