diff options
author | Rob Taylor <rob.taylor@codethink.co.uk> | 2012-01-19 10:28:21 +0000 |
---|---|---|
committer | Rob Taylor <rob.taylor@codethink.co.uk> | 2012-01-19 10:28:21 +0000 |
commit | e946a1e66d1a236a475f9c83351305a0082e0c25 (patch) | |
tree | d12d26f8ba462571cd212319f825fe9cc9a31394 /morphlib | |
parent | eb36c1b9da1e629836264c1ff4fa6f4fb78d3c74 (diff) | |
download | morph-e946a1e66d1a236a475f9c83351305a0082e0c25.tar.gz |
Complete SourceManager tests
Diffstat (limited to 'morphlib')
-rw-r--r-- | morphlib/git.py | 21 | ||||
-rw-r--r-- | morphlib/sourcemanager.py | 62 | ||||
-rw-r--r-- | morphlib/sourcemanager_tests.py | 99 |
3 files changed, 149 insertions, 33 deletions
diff --git a/morphlib/git.py b/morphlib/git.py index 655d4716..e8c9eb70 100644 --- a/morphlib/git.py +++ b/morphlib/git.py @@ -18,6 +18,7 @@ import logging import urlparse import binascii import morphlib +import os class NoMorphs(Exception): @@ -44,14 +45,15 @@ class InvalidTreeish(Exception): class Treeish: - def __init__(self, repo, ref): + def __init__(self, repo, ref, msg=logging.debug): self.repo = repo + self.msg = msg self.sha1 = None self.ref = None self._resolve_ref(ref) def _resolve_ref(self, ref): - ex = morphlib.execute.Execute(self.repo, msg=logging.debug) + ex = morphlib.execute.Execute(self.repo, self.msg) try: refs = ex.runv(['git', 'show-ref', ref]).split() binascii.unhexlify(refs[0]) #Valid hex? @@ -64,10 +66,10 @@ class Treeish: try: if len(ref)==40: binascii.unhexlify(ref) - ex = morphlib.execute.Execute(self.repo, msg=logging.debug) + ex = morphlib.execute.Execute(self.repo, self.msg) try: refs = ex.runv(['git', 'rev-list', '--no-walk', ref]) - self.sha1=REF + self.sha1=ref except morphlib.execute.CommandFailure: raise InvalidTreeish(self.repo,ref) @@ -94,3 +96,14 @@ def clone(location, repo): ex = morphlib.execute.Execute('.', msg=logging.debug) return ex.runv(['git', 'clone', repo, location]) +def init(location): + '''initialise git repo at location''' + os.mkdir(location) + ex = morphlib.execute.Execute(location, msg=logging.debug) + return ex.runv(['git', 'init']) + +def add_remote(gitdir, name, url): + '''add remote with name 'name' for url at gitdir''' + ex = morphlib.execute.Execute(gitdir, msg=logging.debug) + return ex.runv(['git', 'remote', 'add', '-f', name, url]) + diff --git a/morphlib/sourcemanager.py b/morphlib/sourcemanager.py index aca45c97..45a20d86 100644 --- a/morphlib/sourcemanager.py +++ b/morphlib/sourcemanager.py @@ -21,6 +21,7 @@ import urlparse import urllib import urllib2 import errno +import string import morphlib from morphlib.git import Treeish @@ -32,6 +33,19 @@ urlparse.uses_params.extend(gitscheme) urlparse.uses_query.extend(gitscheme) urlparse.uses_fragment.extend(gitscheme) +_valid_chars = string.digits + string.letters + ':%_' + +def quote_url(url): + transl = lambda x: x if x in _valid_chars else '_' + return ''.join([transl(x) for x in url]) + +class SourceNotFound(Exception): + + def __init__(self, repo, ref): + Exception.__init__(self, + 'No source found at %s:%s' % + (repo, ref)) + class SourceManager(object): @@ -42,7 +56,7 @@ class SourceManager(object): self.settings = app.settings def _get_git_cache(self, repo): - name = urllib.quote_plus(repo) + name = quote_url(repo) location = self.source_cache_dir + '/' + name if os.path.exists(location): @@ -52,28 +66,34 @@ class SourceManager(object): self.msg('Making sure we have a local cache of the git repo') - bundle_server = self.settings['bundle-server'] bundle=None - if bundle_server: - bundle = location + ".bndl" + if self.settings.has_key('bundle-server'): + bundle_server = self.settings['bundle-server'] + if not bundle_server.endswith('/'): + bundle_server += '/' + self.msg("Using bundle server %s, looking for bundle for %s" % (bundle_server, name)) + bundle = name + ".bndl" lookup_url = urlparse.urljoin(bundle_server, bundle) self.msg('Checking for bundle %s' % lookup_url) req = urllib2.Request(lookup_url) + try: urllib2.urlopen(req) - except urllib2.HTTPError, e: - bundle_exists=False + self._wget(lookup_url) + bundle = self.source_cache_dir + '/' + bundle + except urllib2.URLError, e: + self.msg("Unable to find bundle %s on %s" % (bundle, bundle_server)) bundle=None - if bundle_exists: - ex = morphlib.execute.Execute(self.source_cache_dir, msg=logging.debug) - ex.runv(['wget', '-c', lookup_url]) - try: if bundle: + self.msg("initialising git repo at %s" % location) morphlib.git.init(location) + self.msg("extracting bundle %s into %s" % (bundle, location)) morphlib.git.extract_bundle(location, bundle) - morphlib.git.add_remotes(location,remote) + self.msg("adding origin %s" % repo) + morphlib.git.add_remote(location,'origin', repo) else: + self.msg("cloning %s into %s" % (repo, location)) morphlib.git.clone(location, repo) success=True except morphlib.execute.CommandFailure: @@ -81,32 +101,38 @@ class SourceManager(object): return success, location + def _wget(self,url): + ex = morphlib.execute.Execute(self.source_cache_dir, msg=self.msg) # pragma: no cover + ex.runv(['wget', '-c', url]) # pragma: no cover def get_treeish(self, repo, ref): self.msg('checking cache for git %s|%s' % (repo, ref)) + #TODO is it actually an error to have no base url? base_urls = self.settings['git-base-url'] success = False; - #TODO should i check if we have full repo before or after checking with base_url? - #TODO is it actually an error to have no base url? - assert(base_urls != None) for base_url in base_urls: if success: + self.msg("success!") break + self.msg("looking in base url %s" % base_url) + if not base_url.endswith('/'): base_url += '/' full_repo = urlparse.urljoin(base_url, repo) - self.msg('cache git base_url=%s full repo url=%s' % (base_url,full_repo)) + self.msg('cache git base_url=\'%s\' full repo url=\'%s\'' % (base_url,full_repo)) success, gitcache = self._get_git_cache(full_repo); - print "repo=%s, gitcache=%s" % (repo, gitcache) - treeish = Treeish(gitcache, ref) - return treeish + if not success: + raise SourceNotFound(repo,ref) + self.msg("creating treeish for %s ref %s" % (gitcache,ref)) + treeish = Treeish(gitcache, ref, self.msg) + return treeish diff --git a/morphlib/sourcemanager_tests.py b/morphlib/sourcemanager_tests.py index 5f29d306..9315b664 100644 --- a/morphlib/sourcemanager_tests.py +++ b/morphlib/sourcemanager_tests.py @@ -19,31 +19,108 @@ import StringIO import unittest import tempfile import shutil +import os +import urllib +from urlparse import urlparse from tempfile import mkdtemp import morphlib class DummyApp(object): - def __init__(self, git_base_url='.'): - self.settings = { 'git-base-url': git_base_url } + def __init__(self): + self.settings = { 'git-base-url': ['.',] } self.msg = lambda msg: None class SourceManagerTests(unittest.TestCase): def setUp(self): - self.tempdir = tempfile.mkdtemp() + pass def tearDown(self): - shutil.rmtree(self.tempdir) + pass - def test_get_treeish_for_self(self): - s = morphlib.sourcemanager.SourceManager(self.tempdir, DummyApp()) - t = s.get_treeish('.','41ee528492db9bd41604311b100da5a871098b3a') - assert(t.sha == '41ee528492db9bd41604311b100da5a871098b3a') - print t.repo + def test_get_sha1_treeish_for_self(self): + tempdir = tempfile.mkdtemp() - - + s = morphlib.sourcemanager.SourceManager(tempdir, DummyApp()) + t = s.get_treeish(os.getcwd(),'41ee528492db9bd41604311b100da5a871098b3a') + assert(t.sha1 == '41ee528492db9bd41604311b100da5a871098b3a') + + shutil.rmtree(tempdir) + + def test_get_sha1_treeish_for_self_twice(self): + tempdir = tempfile.mkdtemp() + + s = morphlib.sourcemanager.SourceManager(tempdir, DummyApp()) + t = s.get_treeish(os.getcwd(),'41ee528492db9bd41604311b100da5a871098b3a') + assert(t.sha1 == '41ee528492db9bd41604311b100da5a871098b3a') + + s = morphlib.sourcemanager.SourceManager(tempdir, DummyApp()) + t = s.get_treeish(os.getcwd(),'41ee528492db9bd41604311b100da5a871098b3a') + assert(t.sha1 == '41ee528492db9bd41604311b100da5a871098b3a') + + shutil.rmtree(tempdir) + + def test_get_ref_treeish_for_self(self): + tempdir = tempfile.mkdtemp() + + s = morphlib.sourcemanager.SourceManager(tempdir, DummyApp()) + t = s.get_treeish(os.getcwd(),'master') + assert(t.ref == 'refs/heads/master') + + shutil.rmtree(tempdir) + + def test_get_sha1_treeish_for_self_bundle(self): + tempdir = tempfile.mkdtemp() + bundle_server_loc = tempdir+'/bundle_server' + os.mkdir(bundle_server_loc) + bundle_name = morphlib.sourcemanager.quote_url(os.getcwd()) + '.bndl' + shutil.copy(os.getcwd() +'/testdata/morph.bndl', bundle_server_loc + '/' +bundle_name) + + app = DummyApp() + app.settings['bundle-server'] = 'file://' + bundle_server_loc + + s = morphlib.sourcemanager.SourceManager(tempdir, app) + + def wget(url): + path=urlparse(url).path + shutil.copy(path, s.source_cache_dir) + + s._wget = wget + + t = s.get_treeish(os.getcwd(),'41ee528492db9bd41604311b100da5a871098b3a') + assert(t.sha1 == '41ee528492db9bd41604311b100da5a871098b3a') + + + + def test_get_sha1_treeish_for_self_bundle_fail(self): + tempdir = tempfile.mkdtemp() + app = DummyApp() + app.settings['bundle-server'] = 'file://' + os.getcwd() +'/testdata' + + s = morphlib.sourcemanager.SourceManager(tempdir, app) + + def wget(url): + path=urlparse(url).path + shutil.copy(path, s.source_cache_dir) + + s._wget = wget + self.assertRaises(morphlib.sourcemanager.SourceNotFound, s.get_treeish, 'asdf','41ee528492db9bd41604311b100da5a871098b3a') + + shutil.rmtree(tempdir) + + def test_get_sha1_treeish_for_self_multple_base(self): + + tempdir = tempfile.mkdtemp() + app = DummyApp() + app.settings['git-base-url'] = ['.', '/somewhere/else'] + + + s = morphlib.sourcemanager.SourceManager(tempdir, app) + t = s.get_treeish(os.getcwd(),'41ee528492db9bd41604311b100da5a871098b3a') + assert(t.sha1 == '41ee528492db9bd41604311b100da5a871098b3a') + + shutil.rmtree(tempdir) |