From 1263e62a2fafc590017b52f3e8c5ee4a94d56212 Mon Sep 17 00:00:00 2001
From: Ben Hutchings <ben.hutchings@codethink.co.uk>
Date: Tue, 14 Jul 2020 23:52:14 +0100
Subject: givemejob: Add metadata for single repositories

When mirroring a repository found through a 'lorries' configuration
section (instead of 'trove' or 'gitlab'), we currently don't set a
description or default branch.

* Set the description to the upstream repository path, but allow this
  to be overridden by a description field in the .lorry file.  Prepend
  the host-name, just as we do when mirroring an Upstream Host.

* Set the default branch to:
  - Bazaar: 'trunk'
  - Git: upstream default branch, found using 'git ls-remote'
  - others: 'master'

Closes #15.
---
 README.md                    |  4 +-
 lorrycontroller/givemejob.py | 91 +++++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 92 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 32c804d..8723214 100644
--- a/README.md
+++ b/README.md
@@ -167,7 +167,9 @@ hours (`h`), and days (`d`), expressed as single-letter codes in upper
 or lower case.
 
 The syntax of `.lorry` files is specified by the Lorry program; see
-its documentation for details.
+its documentation for details.  Lorry Controller supports an
+optional `description` field in `.lorry` files that is used to set
+the repository description on the Downstream Host.
 
 
 HTTP proxy configuration: `proxy.conf`
diff --git a/lorrycontroller/givemejob.py b/lorrycontroller/givemejob.py
index 721b55e..ee998ea 100644
--- a/lorrycontroller/givemejob.py
+++ b/lorrycontroller/givemejob.py
@@ -13,10 +13,13 @@
 # with this program; if not, write to the Free Software Foundation, Inc.,
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
-
+import json
 import logging
+import re
+import urllib.parse
 
 import bottle
+import cliapp
 
 import lorrycontroller
 
@@ -80,6 +83,90 @@ class GiveMeJob(lorrycontroller.LorryControllerRoute):
             return lorrycontroller.get_upstream_host(host_info) \
                                   .get_repo_metadata(lorry_info['from_path'])
 
+    @staticmethod
+    def get_single_repo_metadata(lorry_info):
+        assert not lorry_info['from_host']
+
+        lorry_dict = json.loads(lorry_info['text'])
+        _, upstream_config = lorry_dict.popitem()
+        upstream_type = upstream_config['type']
+
+        # Get the repository URL
+        url = None
+        try:
+            url = upstream_config['url'].strip()
+        except KeyError:
+            if upstream_type == 'bzr':
+                try:
+                    url = upstream_config['branches']['trunk'].strip()
+                except KeyError:
+                    pass
+
+        # Extract the host-name and repo path
+        host_name, repo_path = None, None
+        if url:
+            # Handle pseudo-URLs
+            if upstream_type == 'bzr':
+                if url.startswith('lp:'):
+                    host_name = 'launchpad.net'
+                    repo_path = url[3:]
+            elif upstream_type == 'cvs':
+                # :pserver:user@host:/path, user@host:/path, etc.
+                match = re.match(r'^(?::[^:@/]+:)?(?:[^:@/]+@)?([^:@/]+):/',
+                                 url)
+                if match:
+                    host_name = match.group(1)
+                    repo_path = url[match.end():].rstrip('/')
+            elif upstream_type == 'git':
+                # user@host:path, host:path.  Path must not start with
+                # '//' as that indicates a real URL.
+                match = re.match(r'^(?:[^:@/]+@)?([^:@/]+):(?!//)', url)
+                if match:
+                    host_name = match.group(1)
+                    repo_path = url[match.end():].strip('/')
+
+            # Default to parsing as a real URL
+            if not host_name:
+                try:
+                    url_obj = urllib.parse.urlparse(url)
+                except ValueError:
+                    pass
+                else:
+                    host_name = url_obj.hostname
+                    repo_path = url_obj.path.strip('/')
+
+        metadata = {}
+
+        # Determine the default branch
+        if upstream_type == 'bzr':
+            # Default in Bazaar is 'trunk' and we don't remap it
+            metadata['head'] = 'trunk'
+        elif upstream_type == 'git':
+            if url:
+                # Query the remote to find its default
+                try:
+                    output = cliapp.runcmd(['git', 'ls-remote', '--symref',
+                                            '--', url, 'HEAD']) \
+                                   .decode('utf-8', errors='replace')
+                    match = re.match(r'^ref: refs/heads/([^\s]+)\tHEAD\n',
+                                     output)
+                    if match:
+                        metadata['head'] = match.group(1)
+                except cliapp.AppException:
+                    pass
+        else:
+            # We currently produce 'master' for all other types
+            metadata['head'] = 'master'
+
+        # Use description from .lorry file, or repository name
+        try:
+            metadata['description'] = upstream_config['description']
+        except KeyError:
+            if repo_path:
+                metadata['description'] = repo_path
+
+        return host_name, metadata
+
     def get_repo_metadata(self, statedb, lorry_info):
         '''Get repository head and description.'''
 
@@ -87,7 +174,7 @@ class GiveMeJob(lorrycontroller.LorryControllerRoute):
         if host_name:
             metadata = self.get_upstream_host_repo_metadata(lorry_info)
         else:
-            metadata = {}
+            host_name, metadata = self.get_single_repo_metadata(lorry_info)
 
         if host_name and 'description' in metadata:
             # Prepend Upstream Host name
-- 
cgit v1.2.1