diff options
| -rw-r--r-- | morphlib/builder2.py | 20 | ||||
| -rw-r--r-- | morphlib/cachedrepo.py | 50 | ||||
| -rw-r--r-- | morphlib/git.py | 51 | ||||
| -rw-r--r-- | morphlib/localrepocache.py | 4 | ||||
| -rw-r--r-- | morphlib/plugins/branch_and_merge_plugin.py | 5 |
5 files changed, 107 insertions, 23 deletions
diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 0b986540..72165ce7 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -308,16 +308,14 @@ class ChunkBuilder(BuilderBase): cache_dir = os.path.dirname(self.artifact.source.repo.path) - def extract_repo(path, sha1, destdir): + def extract_repo(repo, sha1, destdir): self.app.status(msg='Extracting %(source)s into %(target)s', - source=path, + source=repo.original_name, target=destdir) - if not os.path.exists(destdir): - os.mkdir(destdir) - morphlib.git.copy_repository(self.app.runcmd, path, destdir) - morphlib.git.checkout_ref(self.app.runcmd, destdir, sha1) + + repo.checkout(sha1, destdir) morphlib.git.reset_workdir(self.app.runcmd, destdir) - submodules = morphlib.git.Submodules(self.app, path, sha1) + submodules = morphlib.git.Submodules(self.app, repo.path, sha1) try: submodules.load() except morphlib.git.NoModulesFileError: @@ -327,14 +325,14 @@ class ChunkBuilder(BuilderBase): for sub in submodules: cached_repo = self.repo_cache.get_repo(sub.url) sub_dir = os.path.join(destdir, sub.path) - tuples.append((cached_repo.path, sub.commit, sub_dir)) + tuples.append((cached_repo, sub.commit, sub_dir)) return tuples s = self.artifact.source - todo = [(s.repo.path, s.sha1, srcdir)] + todo = [(s.repo, s.sha1, srcdir)] while todo: - path, sha1, srcdir = todo.pop() - todo += extract_repo(path, sha1, srcdir) + repo, sha1, srcdir = todo.pop() + todo += extract_repo(repo, sha1, srcdir) self.set_mtime_recursively(srcdir) def set_mtime_recursively(self, root): # pragma: no cover diff --git a/morphlib/cachedrepo.py b/morphlib/cachedrepo.py index ad17785a..61526cd7 100644 --- a/morphlib/cachedrepo.py +++ b/morphlib/cachedrepo.py @@ -49,6 +49,14 @@ class CloneError(cliapp.AppException): def __init__(self, repo, target_dir): cliapp.AppException.__init__( self, + 'Failed to clone %s into %s' % (repo.original_name, target_dir)) + + +class CopyError(cliapp.AppException): + + def __init__(self, repo, target_dir): + cliapp.AppException.__init__( + self, 'Failed to copy %s into %s' % (repo.original_name, target_dir)) @@ -75,6 +83,9 @@ class CachedRepo(object): remote Git repository. This remote repository is set up as the 'origin' remote. + Cached repositories are bare mirrors of the upstream. Locally created + branches will be lost the next time the repository updates. + CachedRepo objects can resolve Git refs into SHA1s. Given a SHA1 ref, they can also be asked to return the contents of a file via the cat() method. They can furthermore check out the repository into @@ -109,8 +120,7 @@ class CachedRepo(object): if not self.is_valid_sha1(ref): try: refs = self._show_ref(ref).split('\n') - # split each ref line into an array, drop non-origin branches - refs = [x.split() for x in refs if 'origin' in x] + refs = [x.split() for x in refs] absref = refs[0][0] except cliapp.AppException: raise InvalidReferenceError(self, ref) @@ -146,8 +156,8 @@ class CachedRepo(object): raise IOError('File %s does not exist in ref %s of repo %s' % (filename, ref, self)) - def checkout(self, ref, target_dir): - '''Unpacks the repository in a directory and checks out a commit ref. + def clone_checkout(self, ref, target_dir): + '''Clone from the cache into the target path and check out a given ref. Raises a CheckoutDirectoryExistsError if the target directory already exists. Raises an InvalidReferenceError if the @@ -160,9 +170,28 @@ class CachedRepo(object): if os.path.exists(target_dir): raise CheckoutDirectoryExistsError(self, target_dir) - os.mkdir(target_dir) + self.resolve_ref(ref) + + self._clone_into(target_dir, ref) + + def checkout(self, ref, target_dir): + '''Unpacks the repository in a directory and checks out a commit ref. + + Raises an InvalidReferenceError if the ref is not found in the + repository. Raises a CopyError if something goes wrong with the copy + of the repository. Raises a CheckoutError if something else goes wrong + while copying the repository or checking out the SHA1 ref. + + ''' + + if not os.path.exists(target_dir): + os.mkdir(target_dir) + # Note, we copy instead of cloning because it's much faster in the case + # that the target is on a different filesystem from the cache. We then + # take care to turn the copy into something as good as a real clone. self._copy_repository(self.path, target_dir) + self._checkout_ref(ref, target_dir) def ls_tree(self, ref): @@ -219,11 +248,18 @@ class CachedRepo(object): return self._runcmd(['git', 'cat-file', 'blob', '%s:%s' % (ref, filename)]) + def _clone_into(self, target_dir, ref): #pragma: no cover + '''Actually perform the clone''' + try: + morphlib.git.clone_into(self._runcmd, self.path, target_dir, ref) + except cliapp.AppException: + raise CloneError(self, target_dir) + def _copy_repository(self, source_dir, target_dir): # pragma: no cover try: morphlib.git.copy_repository(self._runcmd, source_dir, target_dir) except cliapp.AppException: - raise CloneError(self, target_dir) + raise CopyError(self, target_dir) def _checkout_ref(self, ref, target_dir): # pragma: no cover try: @@ -232,7 +268,7 @@ class CachedRepo(object): raise CheckoutError(self, ref, target_dir) def _update(self): # pragma: no cover - self._runcmd(['git', 'remote', 'update', 'origin']) + self._runcmd(['git', 'remote', 'update', 'origin', '--prune']) def __str__(self): # pragma: no cover return self.url diff --git a/morphlib/git.py b/morphlib/git.py index d8169c93..9ab98ad0 100644 --- a/morphlib/git.py +++ b/morphlib/git.py @@ -154,8 +154,42 @@ def set_remote(runcmd, gitdir, name, url): def copy_repository(runcmd, repo, destdir): - '''Copies a cached repository into a directory using cp.''' - return runcmd(['cp', '-a', os.path.join(repo, '.git'), destdir]) + '''Copies a cached repository into a directory using cp. + + This also fixes up the repository afterwards, so that it can contain + code etc. It does not leave any given branch ready for use. + + ''' + runcmd(['cp', '-a', repo, os.path.join(destdir, '.git')]) + # core.bare should be false so that git believes work trees are possible + runcmd(['git', 'config', 'core.bare', 'false'], cwd=destdir) + # we do not want the origin remote to behave as a mirror for pulls + runcmd(['git', 'config', '--unset', 'remote.origin.mirror'], cwd=destdir) + # we want a traditional refs/heads -> refs/remotes/origin ref mapping + runcmd(['git', 'config', 'remote.origin.fetch', + '+refs/heads/*:refs/remotes/origin/*'], cwd=destdir) + # set the origin url to the cached repo so that we can quickly clean up + runcmd(['git', 'config', 'remote.origin.url', repo], cwd=destdir) + # by packing the refs, we can then edit then en-masse easily + runcmd(['git', 'pack-refs', '--all', '--prune'], cwd=destdir) + # turn refs/heads/* into refs/remotes/origin/* in the packed refs + # so that the new copy behaves more like a traditional clone. + logging.debug("Adjusting packed refs for %s" % destdir) + with open(os.path.join(destdir, ".git", "packed-refs"), "r") as ref_fh: + pack_lines = ref_fh.read().split("\n") + with open(os.path.join(destdir, ".git", "packed-refs"), "w") as ref_fh: + ref_fh.write(pack_lines.pop(0) + "\n") + for refline in pack_lines: + if ' refs/remotes/' in refline: + continue + if ' refs/heads/' in refline: + sha, ref = refline[:40], refline[41:] + if ref.startswith("refs/heads/"): + ref = "refs/remotes/origin/" + ref[11:] + refline = "%s %s" % (sha, ref) + ref_fh.write("%s\n" % (refline)) + # Finally run a remote update to clear up the refs ready for use. + runcmd(['git', 'remote', 'update', 'origin', '--prune'], cwd=destdir) def checkout_ref(runcmd, gitdir, ref): @@ -167,3 +201,16 @@ def reset_workdir(runcmd, gitdir): '''and the status of the working directory''' runcmd(['git', 'clean', '-fxd'], cwd=gitdir) runcmd(['git', 'reset', '--hard', 'HEAD'], cwd=gitdir) + + +def clone_into(runcmd, srcpath, targetpath, ref=None): + '''Clones a repo in srcpath into targetpath, optionally directly at ref.''' + if ref is None: + runcmd(['git', 'clone', srcpath, targetpath]) + else: + runcmd(['git', 'clone', '-b', ref, srcpath, targetpath]) + +def find_first_ref(runcmd, gitdir, ref): + '''Find the *first* ref match and returns its sha1.''' + return runcmd(['git', 'show-ref', ref], + cwd=gitdir).split("\n")[0].split(" ")[0] diff --git a/morphlib/localrepocache.py b/morphlib/localrepocache.py index c5a95ebc..89be56c8 100644 --- a/morphlib/localrepocache.py +++ b/morphlib/localrepocache.py @@ -197,7 +197,7 @@ class LocalRepoCache(object): return False, 'Unable to fetch bundle %s: %s' % (bundle_url, e) try: - self._git(['clone', '-n', bundle_path, path]) + self._git(['clone', '--mirror', '-n', bundle_path, path]) self._git(['remote', 'set-url', 'origin', repourl], cwd=path) except cliapp.AppException, e: # pragma: no cover if self._exists(path): @@ -236,7 +236,7 @@ class LocalRepoCache(object): repourl = self._resolver.pull_url(reponame) path = self._cache_name(repourl) try: - self._git(['clone', '-n', repourl, path]) + self._git(['clone', '--mirror', '-n', repourl, path]) except cliapp.AppException, e: errors.append('Unable to clone from %s to %s: %s' % (repourl, path, e)) diff --git a/morphlib/plugins/branch_and_merge_plugin.py b/morphlib/plugins/branch_and_merge_plugin.py index d3aced4b..82d83a58 100644 --- a/morphlib/plugins/branch_and_merge_plugin.py +++ b/morphlib/plugins/branch_and_merge_plugin.py @@ -194,7 +194,7 @@ class BranchAndMergePlugin(cliapp.Plugin): os.makedirs(parent_dir) # Clone it from cache to target directory. - repo.checkout(ref, os.path.abspath(dirname)) + repo.clone_checkout(ref, os.path.abspath(dirname)) # Remember the repo name we cloned from in order to be able # to identify the repo again later using the same name, even @@ -221,6 +221,9 @@ class BranchAndMergePlugin(cliapp.Plugin): with open(filename) as f: text = f.read() else: + ref = morphlib.git.find_first_ref(self.app.runcmd, repo_dir, ref) + logging.warning("Running git cat-file blob %s:%s.morph in %s" % ( + ref, name, repo_dir)) text = self.app.runcmd(['git', 'cat-file', 'blob', '%s:%s.morph' % (ref, name)], cwd=repo_dir) morphology = morphlib.morph2.Morphology(text) |