diff options
| -rw-r--r-- | requirements.freeze.txt | 1 | ||||
| -rw-r--r-- | requirements.txt | 1 | ||||
| -rwxr-xr-x | ybd/__main__.py | 6 | ||||
| -rw-r--r-- | ybd/cache.py | 10 | ||||
| -rw-r--r-- | ybd/config.py | 2 | ||||
| -rw-r--r-- | ybd/release_note.py | 11 | ||||
| -rw-r--r-- | ybd/repos.py | 200 | ||||
| -rw-r--r-- | ybd/sandbox.py | 8 |
8 files changed, 42 insertions, 197 deletions
diff --git a/requirements.freeze.txt b/requirements.freeze.txt index e3956ac..f7dac27 100644 --- a/requirements.freeze.txt +++ b/requirements.freeze.txt @@ -7,3 +7,4 @@ bottle==0.12.1 cherrypy==8.1.2 riemann-client==6.3.0 fs==0.5.0 +gitmachine==0.1.5 diff --git a/requirements.txt b/requirements.txt index 5e618c1..2b8c20e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ bottle cherrypy riemann-client fs +gitmachine diff --git a/ybd/__main__.py b/ybd/__main__.py index 2aba7cd..b0c65b5 100755 --- a/ybd/__main__.py +++ b/ybd/__main__.py @@ -20,7 +20,7 @@ import os import sys import fcntl -from ybd import app, cache, sandbox +from ybd import app, cache, sandbox, repos from ybd.app import cleanup, config, log, RetryException, setup, spawn, timer from ybd.assembly import compose from ybd.deployment import deploy @@ -29,6 +29,7 @@ from ybd.concourse import Pipeline from ybd.release_note import do_release_note import sandboxlib import yaml +from gitmachine import GitMachine if sys.version_info < (3,5,2): @@ -70,6 +71,9 @@ with timer('TOTAL'): Pipeline(target) os._exit(0) + repos.gitmachine = GitMachine(config['gits'], config['tmp'], + config['aliases']) + with timer('CACHE-KEYS', 'cache-key calculations'): cache.cache_key(target) diff --git a/ybd/cache.py b/ybd/cache.py index c37d790..d93b2b0 100644 --- a/ybd/cache.py +++ b/ybd/cache.py @@ -22,8 +22,8 @@ import os import shutil from subprocess import call -from ybd import app, utils -from ybd.repos import get_repo_url, get_tree +from ybd import app, repos, utils +from ybd.repos import get_tree import tempfile import yaml import re @@ -174,7 +174,8 @@ def update_manifest(dn, manifest): if manifest.endswith('text'): format = '%s %s %s %s %s %s\n' m.write(format % (dn['name'], dn['cache'], - get_repo_url(dn.get('repo', 'None')), + repos.gitmachine.normalize_repo_url( + dn.get('repo', 'None')), dn.get('ref', 'None'), dn.get('unpetrify-ref', 'None'), md5(get_cache(dn)))) @@ -183,7 +184,8 @@ def update_manifest(dn, manifest): text = {'name': dn['name'], 'summary': {'artifact': dn['cache'], - 'repo': get_repo_url(dn.get('repo', None)), + 'repo': repos.gitmachine.normalize_repo_url( + dn.get('repo', None)), 'sha': dn.get('ref', None), 'ref': dn.get('unpetrify-ref', None), 'md5': md5(get_cache(dn))}} diff --git a/ybd/config.py b/ybd/config.py new file mode 100644 index 0000000..fc8f3a7 --- /dev/null +++ b/ybd/config.py @@ -0,0 +1,2 @@ +config = {} +defs = {} diff --git a/ybd/release_note.py b/ybd/release_note.py index 68d20fd..d3485f8 100644 --- a/ybd/release_note.py +++ b/ybd/release_note.py @@ -17,11 +17,11 @@ import os from subprocess import check_output import tempfile -from ybd import app +from ybd import app, repos from ybd.app import chdir, config, log from ybd.morphs import Morphs -from ybd.repos import explore, get_last_tag, get_repo_name -from ybd.repos import mirror, mirror_has_ref +from ybd.repos import explore, get_last_tag, mirror_has_ref +from ybd.morphs import Morphs def do_release_note(release_note): @@ -83,9 +83,10 @@ def log_changes(dn, tmpdir, old_defs, ref): log(dn, 'Logging git change history', tmpdir) try: gitdir = os.path.join(config['gits'], - get_repo_name(dn['repo'])) + repos.gitmachine.normalize_repo_name( + dn['repo'])) if not os.path.exists(gitdir): - mirror(dn['name'], dn['repo']) + repos.gitmachine.mirror_repository(dn['repo']) elif not mirror_has_ref(gitdir, ref): update_mirror(dn['name'], dn['repo'], gitdir) with chdir(gitdir): diff --git a/ybd/repos.py b/ybd/repos.py index a1235b9..4bc4b5b 100644 --- a/ybd/repos.py +++ b/ybd/repos.py @@ -34,30 +34,7 @@ else: from configparser import RawConfigParser from io import StringIO - -def get_repo_url(repo): - if repo: - for alias, url in app.config.get('aliases', {}).items(): - repo = repo.replace(alias, url) - if repo[:4] == "http" and not repo.endswith('.git'): - repo = repo + '.git' - return repo - - -def get_repo_name(repo): - ''' Convert URIs to strings that only contain digits, letters, _ and %. - - NOTE: this naming scheme is based on what lorry uses - - ''' - def transl(x): - return x if x in valid_chars else '_' - - valid_chars = string.digits + string.ascii_letters + '%_' - url = get_repo_url(repo) - if url.endswith('.git'): - url = url[:-4] - return ''.join([transl(x) for x in url]) +gitmachine = None def get_version(gitdir, ref='HEAD'): @@ -86,9 +63,23 @@ def get_last_tag(gitdir): return None +def mirror_has_ref(gitdir, ref): + with utils.chdir(gitdir), open(os.devnull, "w") as fnull: + out = call(['git', 'cat-file', '-t', ref], stdout=fnull, stderr=fnull) + return out == 0 + + +def checkout(dn): + gitmachine.arrange_into_folder( + dn['repo'], dn['ref'], dn.get('submodules', {}), dn['checkout']) + + utils.set_mtime_recursively(dn['checkout']) + + def get_tree(dn): ref = str(dn['ref']) - gitdir = os.path.join(app.config['gits'], get_repo_name(dn['repo'])) + gitdir = os.path.join(app.config['gits'], + gitmachine.normalize_repo_name(dn['repo'])) if dn['repo'].startswith('file://') or dn['repo'].startswith('/'): gitdir = dn['repo'].replace('file://', '') if not os.path.isdir(gitdir): @@ -96,7 +87,8 @@ def get_tree(dn): if not os.path.exists(gitdir): try: - params = {'repo': get_repo_url(dn['repo']), 'ref': ref} + params = {'repo': gitmachine.normalize_repo_url(dn['repo']), + 'ref': ref} r = requests.get(url=app.config['tree-server'], params=params) return r.json()['tree'] except: @@ -123,167 +115,11 @@ def get_tree(dn): app.log(dn, 'No tree for ref', (ref, gitdir), exit=True) -def mirror(name, repo): - tempfile.tempdir = app.config['tmp'] - tmpdir = tempfile.mkdtemp() - repo_url = get_repo_url(repo) - try: - tar_file = get_repo_name(repo_url) + '.tar' - app.log(name, 'Try fetching tarball %s' % tar_file) - # try tarball first - with app.chdir(tmpdir), open(os.devnull, "w") as fnull: - call(['wget', os.path.join(app.config['tar-url'], tar_file)], - stdout=fnull, stderr=fnull) - call(['tar', 'xf', tar_file], stderr=fnull) - call(['git', 'config', 'gc.autodetach', 'false'], stderr=fnull) - os.remove(tar_file) - update_mirror(name, repo, tmpdir) - except: - app.log(name, 'Try git clone from', repo_url) - with open(os.devnull, "w") as fnull: - if call(['git', 'clone', '--mirror', '-n', repo_url, tmpdir]): - app.log(name, 'Failed to clone', repo, exit=True) - - with app.chdir(tmpdir): - if call(['git', 'rev-parse']): - app.log(name, 'Problem mirroring git repo at', tmpdir, exit=True) - - gitdir = os.path.join(app.config['gits'], get_repo_name(repo)) - try: - shutil.move(tmpdir, gitdir) - app.log(name, 'Git repo is mirrored at', gitdir) - except: - pass - - -def fetch(repo): - with app.chdir(repo), open(os.devnull, "w") as fnull: - call(['git', 'fetch', 'origin'], stdout=fnull, stderr=fnull) - - -def mirror_has_ref(gitdir, ref): - with app.chdir(gitdir), open(os.devnull, "w") as fnull: - out = call(['git', 'cat-file', '-t', ref], stdout=fnull, stderr=fnull) - return out == 0 - - -def update_mirror(name, repo, gitdir): - with app.chdir(gitdir), open(os.devnull, "w") as fnull: - app.log(name, 'Refreshing mirror for %s' % repo) - repo_url = get_repo_url(repo) - if call(['git', 'fetch', repo_url, '+refs/*:refs/*', '--prune'], - stdout=fnull, stderr=fnull): - app.log(name, 'Git update mirror failed', repo, exit=True) - - -def checkout(dn): - _checkout(dn['name'], dn['repo'], dn['ref'], dn['checkout']) - - with app.chdir(dn['checkout']): - if os.path.exists('.gitmodules') or dn.get('submodules'): - checkout_submodules(dn) - - utils.set_mtime_recursively(dn['checkout']) - - -def _checkout(name, repo, ref, checkout): - gitdir = os.path.join(app.config['gits'], get_repo_name(repo)) - if not os.path.exists(gitdir): - mirror(name, repo) - elif not mirror_has_ref(gitdir, ref): - update_mirror(name, repo, gitdir) - # checkout the required version from git - with open(os.devnull, "w") as fnull: - # We need to pass '--no-hardlinks' because right now there's nothing to - # stop the build from overwriting the files in the .git directory - # inside the sandbox. If they were hardlinks, it'd be possible for a - # build to corrupt the repo cache. I think it would be faster if we - # removed --no-hardlinks, though. - if call(['git', 'clone', '--no-hardlinks', gitdir, checkout], - stdout=fnull, stderr=fnull): - app.log(name, 'Git clone failed for', gitdir, exit=True) - - with app.chdir(checkout): - if call(['git', 'checkout', '--force', ref], stdout=fnull, - stderr=fnull): - app.log(name, 'Git checkout failed for', ref, exit=True) - - app.log(name, 'Git checkout %s in %s' % (repo, checkout)) - app.log(name, 'Upstream version %s' % get_version(checkout, ref)) - - def source_date_epoch(checkout): with app.chdir(checkout): return check_output(['git', 'log', '-1', '--pretty=%ct'])[:-1] -def extract_commit(name, repo, ref, target_dir): - '''Check out a single commit (or tree) from a Git repo. - The checkout() function actually clones the entire repo, so this - function is much quicker when you don't need to copy the whole repo into - target_dir. - ''' - gitdir = os.path.join(app.config['gits'], get_repo_name(repo)) - if not os.path.exists(gitdir): - mirror(name, repo) - elif not mirror_has_ref(gitdir, ref): - update_mirror(name, repo, gitdir) - - with tempfile.NamedTemporaryFile() as git_index_file: - git_env = os.environ.copy() - git_env['GIT_INDEX_FILE'] = git_index_file.name - git_env['GIT_WORK_TREE'] = target_dir - - app.log(name, 'Extracting commit', ref) - if call(['git', 'read-tree', ref], env=git_env, cwd=gitdir): - app.log(name, 'git read-tree failed for', ref, exit=True) - app.log(name, 'Then checkout index', ref) - if call(['git', 'checkout-index', '--all'], env=git_env, cwd=gitdir): - app.log(name, 'Git checkout-index failed for', ref, exit=True) - app.log(name, 'Done', ref) - - utils.set_mtime_recursively(target_dir) - - -def checkout_submodules(dn): - app.log(dn, 'Checking git submodules') - with open('.gitmodules', "r") as gitfile: - # drop indentation in sections, as RawConfigParser cannot handle it - content = '\n'.join([l.strip() for l in gitfile.read().splitlines()]) - io = StringIO(content) - parser = RawConfigParser() - parser.readfp(io) - - for section in parser.sections(): - # validate section name against the 'submodule "foo"' pattern - submodule = re.sub(r'submodule "(.*)"', r'\1', section) - path = parser.get(section, 'path') - try: - url = dn['submodules'][path]['url'] - app.log(dn, 'Processing submodule %s from' % path, url) - except: - url = parser.get(section, 'url') - app.log(dn, 'WARNING: fallback to submodule %s from' % path, url) - - # list objects in the parent repo tree to find the commit - # object that corresponds to the submodule - commit = check_output(['git', 'ls-tree', dn['ref'], path]).split() - - # read the commit hash from the output - fields = list(map(lambda x: x.decode('unicode-escape'), commit)) - if len(fields) >= 2 and fields[1] == 'commit': - submodule_commit = fields[2] - - # fail if the commit hash is invalid - if len(submodule_commit) != 40: - raise Exception - - fulldir = os.path.join(os.getcwd(), path) - _checkout(dn['name'], url, submodule_commit, fulldir) - else: - app.log(dn, 'Skipping submodule %s, not a commit:' % path, fields) - - @contextlib.contextmanager def explore(ref): try: diff --git a/ybd/sandbox.py b/ybd/sandbox.py index 8d869c2..38a49f9 100644 --- a/ybd/sandbox.py +++ b/ybd/sandbox.py @@ -23,10 +23,7 @@ import shutil import stat import tempfile from subprocess import call, PIPE - -from ybd import app, cache, utils -from ybd.repos import get_repo_url - +from ybd import app, cache, repos, utils # This must be set to a sandboxlib backend before the run_sandboxed() function # can be used. @@ -248,7 +245,8 @@ def ccache_mounts(dn, ccache_target): if app.config['no-ccache'] or 'repo' not in dn: mounts = [] else: - name = os.path.basename(get_repo_url(dn['repo'])) + name = os.path.basename(repos.gitmachine.normalize_repo_url( + dn['repo'])) if name.endswith('.git'): name = name[:-4] ccache_dir = os.path.join(app.config['ccache_dir'], name) |