From 90dbbffbc2c3beb44768a03107516978d3a4da1a Mon Sep 17 00:00:00 2001
From: Pedro Alvarez <pedro.alvarez@codethink.co.uk>
Date: Sun, 27 Mar 2016 02:24:16 +0000
Subject: Speed up migration 008

Given that the script already needs access to the Trove
git.baserock.org, use it to read the .gitmodules files without
cloning the repositories.

Also, avoid creating a 'submodules' field when .gitmodules is empty.

Change-Id: I7587587e35b5f52251503fe52af92114fbad4644
---
 migrations/008-submodules-in-strata.py | 37 +++++++++++++++++++++++++---------
 1 file changed, 27 insertions(+), 10 deletions(-)

diff --git a/migrations/008-submodules-in-strata.py b/migrations/008-submodules-in-strata.py
index 92ec865..a536852 100755
--- a/migrations/008-submodules-in-strata.py
+++ b/migrations/008-submodules-in-strata.py
@@ -143,10 +143,29 @@ def move_dict_entry_last(dict_object, key, error_if_missing=False):
         if error_if_missing:
             raise KeyError(key)
 
-def submodules_to_dict(path):
-    with open(os.path.join(path, '.gitmodules'), "r") as gitfile:
-        content = '\n'.join([l.strip() for l in gitfile.read().splitlines()])
-    io = StringIO(content)
+def submodules_to_dict(url, ref):
+    try:
+        response = requests.get(
+            GIT_CACHE_SERVER_URL + '1.0/files',
+            params={'repo': url, 'ref': ref, 'filename':'.gitmodules'},
+            headers={'Accept': 'application/json'},
+            timeout=9)
+        logging.debug("Got response: %s" % response)
+        try:
+            response.raise_for_status()
+        except Exception as e:
+            raise RuntimeError(
+                "Unexpected response from server %s for repo %s: %s" %
+                (GIT_CACHE_SERVER_URL, url, e.message))
+    except requests.exceptions.ConnectionError as e:
+        raise RuntimeError("Unable to connect to cache server %s while trying "
+                           "to read file '.gitmodules' of repo %s. Error "
+                           "was: %s" % (GIT_CACHE_SERVER_URL, url, e.message))
+
+    gitmodules = ''
+    for line in response.text.splitlines():
+        gitmodules += "%s\n" % (line.strip())
+    io = StringIO(gitmodules)
     parser = RawConfigParser()
     parser.readfp(io)
     stuff = {}
@@ -190,12 +209,10 @@ def add_submodules_to_strata(contents, filename):
 
         path = get_repo_name(chunk_git_url)
         if u'.gitmodules' in toplevel_file_list:
-            call(['git', 'clone', chunk_git_url, path])
-            p_co = Popen(['git', 'checkout', chunk_git_ref], cwd=path)
-            p_co.wait()
-            chunk_ref['submodules'] = submodules_to_dict(path)
-            call(['rm', '-rf', path])
-            changed = True
+            submodules = submodules_to_dict(chunk_git_url, chunk_git_ref)
+            if submodules:
+                chunk_ref['submodules'] = submodules
+                changed = True
 
     return changed
 try:
-- 
cgit v1.2.1