diff options
| -rwxr-xr-x | morph | 23 | ||||
| -rw-r--r-- | morphlib/__init__.py | 1 | ||||
| -rw-r--r-- | morphlib/cachedrepo.py | 6 | ||||
| -rw-r--r-- | morphlib/localrepocache.py | 38 | ||||
| -rw-r--r-- | morphlib/morph2.py | 2 |
5 files changed, 42 insertions, 28 deletions
@@ -53,7 +53,8 @@ class Morph(cliapp.Application): default=defaults['git-base-url']) self.settings.string(['bundle-server'], 'base URL to download bundles', - metavar='URL') + metavar='URL', + default=None) self.settings.string(['cachedir'], 'put build results in DIR', metavar='DIR', @@ -222,29 +223,33 @@ class Morph(cliapp.Application): ''' - cachedir = self.settings['cachedir'] + cachedir = os.path.join(self.settings['cachedir'], 'gits') baseurls = self.settings['git-base-url'] - cache = LocalRepoCache(cachedir, baseurls) + bundle_base_url = self.settings['bundle-server'] + cache = morphlib.localrepocache.LocalRepoCache( + cachedir, baseurls, bundle_base_url) # Reverse so we process things in the order given by the user. queue = collections.deque(self._itertriplets(args)) while queue: - repo, ref, filename = queue.popleft() - cache.cache_repo(repo) - repo = cache.get_repo(repo) + reponame, ref, filename = queue.popleft() + self.msg('updating %s:%s:%s' % (reponame, ref, filename)) + cache.cache_repo(reponame) + repo = cache.get_repo(reponame) repo.update() absref = repo.resolve_ref(ref) text = repo.cat(absref, filename) morphology = morphlib.morph2.Morphology(text) if morphology['kind'] == 'system': for stratum in morphology['strata']: - queue.append((repo, ref, '%s.morph' % stratum)) + queue.append((reponame, ref, '%s.morph' % stratum)) elif morphology['kind'] == 'stratum': for stratum in morphology['build-depends']: - queue.append((repo, ref, '%s.morph' % stratum)) + queue.append((reponame, ref, '%s.morph' % stratum)) for x in morphology['sources']: - queue.append((x.repo, x.ref, x.morph)) + queue.append((x['repo'], x['ref'], + '%s.morph' % x['morph'])) def cmd_build_single(self, args): '''Build a binary from a morphology but do not build its dependencies. diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 62a4ab7c..f6b8ccb4 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -30,6 +30,7 @@ import execute import fsutils import git import localrepocache +import morph2 import morphology import morphologyloader import savefile diff --git a/morphlib/cachedrepo.py b/morphlib/cachedrepo.py index df0ebf9d..37c53c7b 100644 --- a/morphlib/cachedrepo.py +++ b/morphlib/cachedrepo.py @@ -106,7 +106,7 @@ class CachedRepo(object): if not self.is_valid_sha1(ref): raise InvalidReferenceError(self, ref) try: - return self._rev_list(ref) + return self._rev_list(ref).strip() except morphlib.execute.CommandFailure: raise InvalidReferenceError(self, ref) @@ -123,7 +123,7 @@ class CachedRepo(object): if not self.is_valid_sha1(ref): raise UnresolvedNamedReferenceError(self, ref) try: - sha1 = self._rev_list(ref) + sha1 = self._rev_list(ref).strip() except morphlib.execute.CommandFailure: raise InvalidReferenceError(self, ref) @@ -154,7 +154,7 @@ class CachedRepo(object): os.mkdir(target_dir) try: - sha1 = self._rev_list(ref) + sha1 = self._rev_list(ref).strip() except morphlib.execute.CommandFailure: raise InvalidReferenceError(self, ref) diff --git a/morphlib/localrepocache.py b/morphlib/localrepocache.py index 46f57289..6587a0e5 100644 --- a/morphlib/localrepocache.py +++ b/morphlib/localrepocache.py @@ -16,7 +16,7 @@ import logging import os -import urllib +import urllib2 import urlparse import morphlib @@ -64,10 +64,11 @@ class LocalRepoCache(object): ''' - def __init__(self, cachedir, baseurls, bundle_base_url): - assert bundle_base_url.endswith('/') + def __init__(self, cachedir, baseurls, bundle_base_url=None): self._cachedir = cachedir self._baseurls = baseurls + if bundle_base_url and not bundle_base_url.endswith('/'): + bundle_base_url += '/' self._bundle_base_url = bundle_base_url self._ex = morphlib.execute.Execute(cachedir, logging.debug) @@ -98,16 +99,19 @@ class LocalRepoCache(object): ''' - source_handle = urllib2.urlopen(url) - target_handle = open(filename, 'wb') + try: + source_handle = urllib2.urlopen(url) + target_handle = open(filename, 'wb') - data = source_handle.read(4096) - while data: - target_handle.write(data) data = source_handle.read(4096) + while data: + target_handle.write(data) + data = source_handle.read(4096) - source_handle.close() - target_handle.close() + source_handle.close() + target_handle.close() + except urllib2.URLError: + return False def _remove(self, filename): # pragma: no cover '''Remove given file. @@ -136,6 +140,8 @@ class LocalRepoCache(object): def _base_iterate(self, reponame): for baseurl in self._baseurls: + if not baseurl.endswith('/'): + baseurl += '/' repourl = urlparse.urljoin(baseurl, reponame) path = self._cache_name(repourl) yield repourl, path @@ -148,15 +154,17 @@ class LocalRepoCache(object): return False def _clone_with_bundle(self, repourl, path): + if not self._bundle_base_url: + return False escaped = self._escape(repourl) bundle_url = urlparse.urljoin(self._bundle_base_url, escaped) bundle_path = path + '.bundle' - if self._fetch(bundle_url, bundle_path): + success = self._fetch(bundle_url, bundle_path) + if success: self._git(['clone', bundle_path, path]) + if os.path.exists(bundle_path): self._remove(bundle_path) - return True - else: - return False + return success def cache_repo(self, reponame): '''Clone the given repo into the cache. @@ -173,7 +181,7 @@ class LocalRepoCache(object): break try: - self._git(['clone', reponame, path]) + self._git(['clone', repourl, path]) except morphlib.execute.CommandFailure: pass else: diff --git a/morphlib/morph2.py b/morphlib/morph2.py index 8987b848..717d855c 100644 --- a/morphlib/morph2.py +++ b/morphlib/morph2.py @@ -34,7 +34,7 @@ class Morphology(object): ('sources', []), ('max-jobs', None), ('description', ''), - ('build-depends', None), + ('build-depends', []), ('build-system', 'manual'), ] |