summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSam Thursfield <sam.thursfield@codethink.co.uk>2015-01-06 18:13:28 +0000
committerAdam Coldrick <adam.coldrick@codethink.co.uk>2015-02-18 10:31:43 +0000
commit2ff375e68935f17a63eebedb55b15f8e3aec3b40 (patch)
tree66026f1a40f41426b90db2f0d4aa67cc89f0cded
parent0c9a8f3bcd13428bdde3fd7bf7201818aec554f0 (diff)
downloadmorph-2ff375e68935f17a63eebedb55b15f8e3aec3b40.tar.gz
Read files from a local clone of definitions where possible
Most morphologies involved in a build are in the definitions repo these days. Currently we read each of them using `git cat-file`, which is slow. It's quicker to check out all the files in one go to a temporary directory and then read them from there. With the current workflow users often have definitions.git checked out on disk. It seems strange to not just read the files from there. There are two reasons why I don't want to do that yet: - there are commands which don't run inside a system branch, which would be broken if we expected to always be in a system branch - there may be local changes in the checked-out repo, and it takes around 5 seconds on each build to check if there aren't any local changes. It actually seems faster to just check out a known clean version from the cache.
-rw-r--r--morphlib/sourceresolver.py31
1 files changed, 30 insertions, 1 deletions
diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py
index 0e520b67..b8af8aee 100644
--- a/morphlib/sourceresolver.py
+++ b/morphlib/sourceresolver.py
@@ -19,6 +19,8 @@ import cPickle
import logging
import os
import pylru
+import shutil
+import tempfile
import cliapp
@@ -151,6 +153,8 @@ class SourceResolver(object):
self._resolved_morphologies = {}
self._resolved_buildsystems = {}
+ self._definitions_checkout_dir = None
+
def _resolve_ref(self, reponame, ref):
'''Resolves commit and tree sha1s of the ref in a repo and returns it.
@@ -220,8 +224,18 @@ class SourceResolver(object):
if key in self._resolved_morphologies:
return self._resolved_morphologies[key]
+ if reponame == self._definitions_repo and \
+ sha1 == self._definitions_absref:
+ defs_filename = os.path.join(self._definitions_checkout_dir,
+ filename)
+ else:
+ defs_filename = None
+
+
loader = morphlib.morphloader.MorphologyLoader()
- if self.lrc.has_repo(reponame):
+ if defs_filename and os.path.exists(defs_filename):
+ morph = loader.load_from_file(defs_filename)
+ elif self.lrc.has_repo(reponame):
self.status(msg="Looking for %(reponame)s:%(filename)s in the "
"local repo cache.",
reponame=reponame, filename=filename, chatty=True)
@@ -394,7 +408,19 @@ class SourceResolver(object):
if definitions_original_ref:
definitions_ref = definitions_original_ref
+ self._definitions_checkout_dir = tempfile.mkdtemp()
+
try:
+ # FIXME: not an ideal way of passing this info across
+ self._definitions_repo = definitions_repo
+ self._definitions_absref = definitions_absref
+ try:
+ definitions_cached_repo = self.lrc.get_repo(definitions_repo)
+ except morphlib.localrepocache.NotCached:
+ definitions_cached_repo = self.lrc.cache_repo(definitions_repo)
+ definitions_cached_repo.extract_commit(
+ definitions_absref, self._definitions_checkout_dir)
+
# First, process the system and its stratum morphologies. These
# will all live in the same Git repository, and will point to
# various chunk morphologies.
@@ -414,6 +440,9 @@ class SourceResolver(object):
for repo, ref, filename in chunk_in_source_repo_queue:
self.process_chunk(repo, ref, repo, ref, filename, visit)
finally:
+ shutil.rmtree(self._definitions_checkout_dir)
+ self._definitions_checkout_dir = None
+
logging.debug('Saving contents of resolved tree cache')
self.tree_cache_manager.save_cache(self._resolved_trees)