Use gitmachine to handle recursive submodule checkouts and overrideslc/gm-rebase

7 files changed, 37 insertions, 201 deletions

diff --git a/install_dependencies.sh b/install_dependencies.sh
index 19a36ab..389cbeb 100755
--- a/install_dependencies.sh
+++ b/install_dependencies.sh
@@ -118,4 +118,5 @@ jsonschema
 bottle
 cherrypy
 riemann-client
+gitmachine
 EOM
diff --git a/ybd/__main__.py b/ybd/__main__.py
index 68553f8..5a7abb8 100755
--- a/ybd/__main__.py
+++ b/ybd/__main__.py
@@ -20,7 +20,7 @@
 import os
 import sys
 import fcntl
-from ybd import app, cache, sandbox
+from ybd import app, cache, sandbox, repos
 from ybd.app import cleanup, RetryException, setup, spawn
 from ybd.assembly import compose
 from ybd import config
@@ -31,6 +31,7 @@ from ybd.release_note import do_release_note
 from ybd.utils import log, timer
 import sandboxlib
 import yaml
+from gitmachine import GitMachine
 
 
 def write_cache_key():
@@ -69,6 +70,9 @@ with timer('TOTAL'):
         Pipeline(target)
         os._exit(0)
 
+    repos.gitmachine = GitMachine(config.config['gits'], config.config['tmp'],
+                                  config.config['aliases'])
+
     with timer('CACHE-KEYS', 'cache-key calculations'):
         cache.cache_key(target)
 
diff --git a/ybd/cache.py b/ybd/cache.py
index acbec74..eb7b0b5 100644
--- a/ybd/cache.py
+++ b/ybd/cache.py
@@ -22,9 +22,9 @@ import os
 import shutil
 from subprocess import call
 
-from ybd import utils, config
+from ybd import repos, config, utils
+from ybd.repos import get_tree
 from ybd.utils import log
-from ybd.repos import get_repo_url, get_tree
 import tempfile
 import yaml
 import re
@@ -178,7 +178,8 @@ def update_manifest(dn, manifest):
         if manifest.endswith('text'):
             format = '%s %s %s %s %s %s\n'
             m.write(format % (dn['name'], dn['cache'],
-                              get_repo_url(dn.get('repo', 'None')),
+                              repos.gitmachine.normalize_repo_url(
+                                  dn.get('repo', 'None')),
                               dn.get('ref', 'None'),
                               dn.get('unpetrify-ref', 'None'),
                               md5(get_cache(dn))))
@@ -187,7 +188,8 @@ def update_manifest(dn, manifest):
 
         text = {'name': dn['name'],
                 'summary': {'artifact': dn['cache'],
-                            'repo': get_repo_url(dn.get('repo', None)),
+                            'repo': repos.gitmachine.normalize_repo_url(
+                                dn.get('repo', None)),
                             'sha': dn.get('ref', None),
                             'ref': dn.get('unpetrify-ref', None),
                             'md5': md5(get_cache(dn))}}
diff --git a/ybd/config/ybd.conf b/ybd/config/ybd.conf
index 269f290..1388dff 100644
--- a/ybd/config/ybd.conf
+++ b/ybd/config/ybd.conf
@@ -135,7 +135,7 @@ kbas-password: 'insecure'
 kbas-upload: ['chunk']
 
 # Where to look for artifacts already built by other instances of YBD
-kbas-url: 'http://artifacts1.baserock.org:8000/'
+kbas-url: 'NO'
 
 # log-timings (previously this was log-elapsed)
 # - 'elapsed' (default) show time since the start of the run
diff --git a/ybd/release_note.py b/ybd/release_note.py
index 53a4a90..87e6b72 100644
--- a/ybd/release_note.py
+++ b/ybd/release_note.py
@@ -20,8 +20,6 @@ import tempfile
 from ybd import app
 from ybd.config import config
 from ybd.morphs import Morphs
-from ybd.repos import explore, get_last_tag, get_repo_name
-from ybd.repos import mirror, mirror_has_ref
 from ybd.utils import log, chdir
 
 
@@ -84,9 +82,10 @@ def log_changes(dn, tmpdir, old_defs, ref):
             log(dn, 'Logging git change history', tmpdir)
             try:
                 gitdir = os.path.join(config['gits'],
-                                      get_repo_name(dn['repo']))
+                                      repos.gitmachine.normalize_repo_name(
+                                          dn['repo']))
                 if not os.path.exists(gitdir):
-                    mirror(dn['name'], dn['repo'])
+                    repos.gitmachine.mirror_repository(dn['repo'])
                 elif not mirror_has_ref(gitdir, ref):
                     update_mirror(dn['name'], dn['repo'], gitdir)
                 with chdir(gitdir):
diff --git a/ybd/repos.py b/ybd/repos.py
index 8ab6e52..629330d 100644
--- a/ybd/repos.py
+++ b/ybd/repos.py
@@ -36,29 +36,7 @@ else:
     from io import StringIO
 
 
-def get_repo_url(repo):
-    if repo:
-        for alias, url in config.config.get('aliases', {}).items():
-            repo = repo.replace(alias, url)
-        if repo[:4] == "http" and not repo.endswith('.git'):
-            repo = repo + '.git'
-    return repo
-
-
-def get_repo_name(repo):
-    ''' Convert URIs to strings that only contain digits, letters, _ and %.
-
-    NOTE: this naming scheme is based on what lorry uses
-
-    '''
-    def transl(x):
-        return x if x in valid_chars else '_'
-
-    valid_chars = string.digits + string.ascii_letters + '%_'
-    url = get_repo_url(repo)
-    if url.endswith('.git'):
-        url = url[:-4]
-    return ''.join([transl(x) for x in url])
+gitmachine = None
 
 
 def get_version(gitdir, ref='HEAD'):
@@ -87,9 +65,23 @@ def get_last_tag(gitdir):
         return None
 
 
+def mirror_has_ref(gitdir, ref):
+    with utils.chdir(gitdir), open(os.devnull, "w") as fnull:
+        out = call(['git', 'cat-file', '-t', ref], stdout=fnull, stderr=fnull)
+        return out == 0
+
+
+def checkout(dn):
+    gitmachine.arrange_into_folder(
+        dn['repo'], dn['ref'], dn.get('submodules', {}), dn['checkout'])
+
+    utils.set_mtime_recursively(dn['checkout'])
+
+
 def get_tree(dn):
     ref = str(dn['ref'])
-    gitdir = os.path.join(config.config['gits'], get_repo_name(dn['repo']))
+    gitdir = os.path.join(config.config['gits'],
+                          gitmachine.normalize_repo_name(dn['repo']))
     if dn['repo'].startswith('file://') or dn['repo'].startswith('/'):
         gitdir = dn['repo'].replace('file://', '')
         if not os.path.isdir(gitdir):
@@ -97,7 +89,8 @@ def get_tree(dn):
 
     if not os.path.exists(gitdir):
         try:
-            params = {'repo': get_repo_url(dn['repo']), 'ref': ref}
+            params = {'repo': gitmachine.normalize_repo_url(dn['repo']),
+                      'ref': ref}
             r = requests.get(url=config.config['tree-server'], params=params)
             return r.json()['tree']
         except:
@@ -124,173 +117,11 @@ def get_tree(dn):
             log(dn, 'No tree for ref', (ref, gitdir), exit=True)
 
 
-def mirror(name, repo):
-    tempfile.tempdir = config.config['tmp']
-    tmpdir = tempfile.mkdtemp()
-    repo_url = get_repo_url(repo)
-    try:
-        tar_file = get_repo_name(repo_url) + '.tar'
-        log(name, 'Try fetching tarball %s' % tar_file)
-        # try tarball first
-        with utils.chdir(tmpdir), open(os.devnull, "w") as fnull:
-            call(['wget', os.path.join(config.config['tar-url'], tar_file)],
-                 stdout=fnull, stderr=fnull)
-            call(['tar', 'xf', tar_file], stderr=fnull)
-            call(['git', 'config', 'gc.autodetach', 'false'], stderr=fnull)
-            os.remove(tar_file)
-            update_mirror(name, repo, tmpdir)
-    except:
-        log(name, 'Try git clone from', repo_url)
-        with open(os.devnull, "w") as fnull:
-            if call(['git', 'clone', '--mirror', '-n', repo_url, tmpdir]):
-                log(name, 'Failed to clone', repo, exit=True)
-
-    with utils.chdir(tmpdir):
-        if call(['git', 'rev-parse']):
-            log(name, 'Problem mirroring git repo at', tmpdir, exit=True)
-
-    gitdir = os.path.join(config.config['gits'], get_repo_name(repo))
-    try:
-        shutil.move(tmpdir, gitdir)
-        log(name, 'Git repo is mirrored at', gitdir)
-    except:
-        pass
-
-
-def fetch(repo):
-    with utils.chdir(repo), open(os.devnull, "w") as fnull:
-        call(['git', 'fetch', 'origin'], stdout=fnull, stderr=fnull)
-
-
-def mirror_has_ref(gitdir, ref):
-    with utils.chdir(gitdir), open(os.devnull, "w") as fnull:
-        out = call(['git', 'cat-file', '-t', ref], stdout=fnull, stderr=fnull)
-        return out == 0
-
-
-def update_mirror(name, repo, gitdir):
-    with utils.chdir(gitdir), open(os.devnull, "w") as fnull:
-        log(name, 'Refreshing mirror for %s' % repo)
-        repo_url = get_repo_url(repo)
-        if call(['git', 'fetch', repo_url, '+refs/*:refs/*', '--prune'],
-                stdout=fnull, stderr=fnull):
-            log(name, 'Git update mirror failed', repo, exit=True)
-
-
-def checkout(dn):
-    _checkout(dn['name'], dn['repo'], dn['ref'], dn['checkout'])
-
-    with utils.chdir(dn['checkout']):
-        if os.path.exists('.gitmodules') or dn.get('submodules'):
-            checkout_submodules(dn)
-
-    utils.set_mtime_recursively(dn['checkout'])
-
-
-def _checkout(name, repo, ref, checkout):
-    gitdir = os.path.join(config.config['gits'], get_repo_name(repo))
-    if not os.path.exists(gitdir):
-        mirror(name, repo)
-    elif not mirror_has_ref(gitdir, ref):
-        update_mirror(name, repo, gitdir)
-    # checkout the required version from git
-    with open(os.devnull, "w") as fnull:
-        # We need to pass '--no-hardlinks' because right now there's nothing to
-        # stop the build from overwriting the files in the .git directory
-        # inside the sandbox. If they were hardlinks, it'd be possible for a
-        # build to corrupt the repo cache. I think it would be faster if we
-        # removed --no-hardlinks, though.
-        if call(['git', 'clone', '--no-hardlinks', gitdir, checkout],
-                stdout=fnull, stderr=fnull):
-            log(name, 'Git clone failed for', gitdir, exit=True)
-
-        with utils.chdir(checkout):
-            if call(['git', 'checkout', '--force', ref], stdout=fnull,
-                    stderr=fnull):
-                log(name, 'Git checkout failed for', ref, exit=True)
-
-            log(name, 'Git checkout %s in %s' % (repo, checkout))
-            log(name, 'Upstream version %s' % get_version(checkout, ref))
-
-
 def source_date_epoch(checkout):
     with utils.chdir(checkout):
         return check_output(['git', 'log', '-1', '--pretty=%ct'])[:-1]
 
 
-def extract_commit(name, repo, ref, target_dir):
-    '''Check out a single commit (or tree) from a Git repo.
-    The checkout() function actually clones the entire repo, so this
-    function is much quicker when you don't need to copy the whole repo into
-    target_dir.
-    '''
-    gitdir = os.path.join(config.config['gits'], get_repo_name(repo))
-    if not os.path.exists(gitdir):
-        mirror(name, repo)
-    elif not mirror_has_ref(gitdir, ref):
-        update_mirror(name, repo, gitdir)
-
-    with tempfile.NamedTemporaryFile() as git_index_file:
-        git_env = os.environ.copy()
-        git_env['GIT_INDEX_FILE'] = git_index_file.name
-        git_env['GIT_WORK_TREE'] = target_dir
-
-        log(name, 'Extracting commit', ref)
-        if call(['git', 'read-tree', ref], env=git_env, cwd=gitdir):
-            log(name, 'git read-tree failed for', ref, exit=True)
-        log(name, 'Then checkout index', ref)
-        if call(['git', 'checkout-index', '--all'], env=git_env, cwd=gitdir):
-            log(name, 'Git checkout-index failed for', ref, exit=True)
-        log(name, 'Done', ref)
-
-    utils.set_mtime_recursively(target_dir)
-
-
-def checkout_submodules(dn):
-    log(dn, 'Checking git submodules')
-    with open('.gitmodules', "r") as gitfile:
-        # drop indentation in sections, as RawConfigParser cannot handle it
-        content = '\n'.join([l.strip() for l in gitfile.read().splitlines()])
-    io = StringIO(content)
-    parser = RawConfigParser()
-    parser.readfp(io)
-
-    for section in parser.sections():
-        # validate section name against the 'submodule "foo"' pattern
-        submodule = re.sub(r'submodule "(.*)"', r'\1', section)
-        path = parser.get(section, 'path')
-        try:
-            url = dn['submodules'][path]['url']
-            log(dn, 'Processing submodule %s from' % path, url)
-        except:
-            url = parser.get(section, 'url')
-            log(dn, 'WARNING: fallback to submodule %s from' % path, url)
-
-        try:
-            # list objects in the parent repo tree to find the commit
-            # object that corresponds to the submodule
-            commit = check_output(['git', 'ls-tree', dn['ref'], path])
-
-            # read the commit hash from the output
-            fields = commit.split()
-            if len(fields) >= 2 and fields[1] == 'commit':
-                submodule_commit = commit.split()[2]
-
-                # fail if the commit hash is invalid
-                if len(submodule_commit) != 40:
-                    raise Exception
-
-                fulldir = os.path.join(os.getcwd(), path)
-                _checkout(dn['name'], url, submodule_commit, fulldir)
-
-            else:
-                log(dn, 'Skipping submodule %s, not a commit:' % path,
-                        fields)
-
-        except:
-            log(dn, "Git submodules problem", exit=True)
-
-
 @contextlib.contextmanager
 def explore(ref):
     try:
diff --git a/ybd/sandbox.py b/ybd/sandbox.py
index 554f1b2..81805fc 100644
--- a/ybd/sandbox.py
+++ b/ybd/sandbox.py
@@ -23,9 +23,7 @@ import shutil
 import stat
 import tempfile
 from subprocess import call, PIPE
-
-from ybd import app, cache, utils, config
-from ybd.repos import get_repo_url
+from ybd import app, cache, config, repos, utils
 from ybd.utils import log
 
 # This must be set to a sandboxlib backend before the run_sandboxed() function
@@ -248,7 +246,8 @@ def ccache_mounts(dn, ccache_target):
     if config.config['no-ccache'] or 'repo' not in dn:
         mounts = []
     else:
-        name = os.path.basename(get_repo_url(dn['repo']))
+        name = os.path.basename(repos.gitmachine.normalize_repo_url(
+            dn['repo']))
         if name.endswith('.git'):
             name = name[:-4]
         ccache_dir = os.path.join(config.config['ccache_dir'], name)