diff options
| -rw-r--r-- | morphlib/__init__.py | 1 | ||||
| -rw-r--r-- | morphlib/git.py | 73 | ||||
| -rw-r--r-- | morphlib/sourcemanager.py | 112 | ||||
| -rw-r--r-- | morphlib/sourcemanager_tests.py | 49 |
4 files changed, 215 insertions, 20 deletions
diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 35939c45..5302139d 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -28,6 +28,7 @@ import execute import git import morphology import morphologyloader +import sourcemanager import stopwatch import tempdir import tester diff --git a/morphlib/git.py b/morphlib/git.py index 59f59c9b..655d4716 100644 --- a/morphlib/git.py +++ b/morphlib/git.py @@ -16,7 +16,7 @@ import logging import urlparse - +import binascii import morphlib @@ -35,29 +35,62 @@ class TooManyMorphs(Exception): 'Too many morphologies at %s:%s: %s' % (repo, ref, ', '.join(morphs))) +class InvalidTreeish(Exception): -def export_sources(repo, ref, tar_filename): - '''Export the contents of a specific commit into a compressed tarball.''' - ex = morphlib.execute.Execute('.', msg=logging.debug) - ex.runv(['git', 'archive', '-o', tar_filename, '--remote', repo, ref]) - -def get_commit_id(repo, ref): - '''Return the full SHA-1 commit id for a repo+ref.''' - # FIXME: This assumes repo is a file:/// URL. + def __init__(self, repo, ref): + Exception.__init__(self, + '%s is an invalid reference for repo %s' % + (ref,repo)) - scheme, netlock, path, params, query, frag = urlparse.urlparse(repo) - assert scheme == 'file' - ex = morphlib.execute.Execute(path, msg=logging.debug) - out = ex.runv(['git', 'rev-list', '-n1', ref]) - return out.strip() +class Treeish: + def __init__(self, repo, ref): + self.repo = repo + self.sha1 = None + self.ref = None + self._resolve_ref(ref) + + def _resolve_ref(self, ref): + ex = morphlib.execute.Execute(self.repo, msg=logging.debug) + try: + refs = ex.runv(['git', 'show-ref', ref]).split() + binascii.unhexlify(refs[0]) #Valid hex? + self.sha1 = refs[0] + self.ref = refs[1] + except morphlib.execute.CommandFailure: + self._is_treeish(ref) + + def _is_treeish(self, ref): + try: + if len(ref)==40: + binascii.unhexlify(ref) + ex = morphlib.execute.Execute(self.repo, msg=logging.debug) + try: + refs = ex.runv(['git', 'rev-list', '--no-walk', ref]) + self.sha1=REF + except morphlib.execute.CommandFailure: + raise InvalidTreeish(self.repo,ref) + + except TypeError: + raise InvalidTreeish(self.repo,ref) + +def export_sources(treeish, tar_filename): + '''Export the contents of a specific commit into a compressed tarball.''' + ex = morphlib.execute.Execute('.', msg=logging.debug) + ex.runv(['git', 'archive', '-o', tar_filename, '--remote', treeish.repo, treeish.sha1]) -def get_morph_text(repo, ref, filename): +def get_morph_text(treeish, filename): '''Return a morphology from a git repository.''' - # FIXME: This implementation assumes a local repo. + ex = morphlib.execute.Execute(treeish.repo, msg=logging.debug) + return ex.runv(['git', 'cat-file', 'blob', '%s:%s' % (treeish.sha1, filename)]) - scheme, netlock, path, params, query, frag = urlparse.urlparse(repo) - assert scheme == 'file' - ex = morphlib.execute.Execute(path, msg=logging.debug) - return ex.runv(['git', 'cat-file', 'blob', '%s:%s' % (ref, filename)]) +def extract_bundle(location, bundle): + '''Extract a bundle into git at location''' + ex = morphlib.execute.Execute(location, msg=logging.debug) + return ex.runv(['git', 'bundle', 'unbundle', bundle]) + +def clone(location, repo): + '''clone at git repo into location''' + ex = morphlib.execute.Execute('.', msg=logging.debug) + return ex.runv(['git', 'clone', repo, location]) diff --git a/morphlib/sourcemanager.py b/morphlib/sourcemanager.py new file mode 100644 index 00000000..aca45c97 --- /dev/null +++ b/morphlib/sourcemanager.py @@ -0,0 +1,112 @@ +# Copyright (C) 2011 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import json +import logging +import os +import urlparse +import urllib +import urllib2 +import errno + +import morphlib +from morphlib.git import Treeish + +gitscheme=["git",] +urlparse.uses_relative.extend(gitscheme) +urlparse.uses_netloc.extend(gitscheme) +urlparse.uses_params.extend(gitscheme) +urlparse.uses_query.extend(gitscheme) +urlparse.uses_fragment.extend(gitscheme) + + + +class SourceManager(object): + + def __init__(self, cachedir, app): + self.source_cache_dir = cachedir + self.msg = app.msg + self.settings = app.settings + + def _get_git_cache(self, repo): + name = urllib.quote_plus(repo) + location = self.source_cache_dir + '/' + name + + if os.path.exists(location): + return True, location + + success=False + + self.msg('Making sure we have a local cache of the git repo') + + bundle_server = self.settings['bundle-server'] + bundle=None + if bundle_server: + bundle = location + ".bndl" + lookup_url = urlparse.urljoin(bundle_server, bundle) + self.msg('Checking for bundle %s' % lookup_url) + req = urllib2.Request(lookup_url) + try: + urllib2.urlopen(req) + except urllib2.HTTPError, e: + bundle_exists=False + bundle=None + if bundle_exists: + ex = morphlib.execute.Execute(self.source_cache_dir, msg=logging.debug) + ex.runv(['wget', '-c', lookup_url]) + + try: + if bundle: + morphlib.git.init(location) + morphlib.git.extract_bundle(location, bundle) + morphlib.git.add_remotes(location,remote) + else: + morphlib.git.clone(location, repo) + success=True + except morphlib.execute.CommandFailure: + success=False + + return success, location + + + def get_treeish(self, repo, ref): + self.msg('checking cache for git %s|%s' % (repo, ref)) + + base_urls = self.settings['git-base-url'] + success = False; + + #TODO should i check if we have full repo before or after checking with base_url? + #TODO is it actually an error to have no base url? + assert(base_urls != None) + + for base_url in base_urls: + if success: + break + + if not base_url.endswith('/'): + base_url += '/' + full_repo = urlparse.urljoin(base_url, repo) + + self.msg('cache git base_url=%s full repo url=%s' % (base_url,full_repo)) + + success, gitcache = self._get_git_cache(full_repo); + + print "repo=%s, gitcache=%s" % (repo, gitcache) + treeish = Treeish(gitcache, ref) + return treeish + + + diff --git a/morphlib/sourcemanager_tests.py b/morphlib/sourcemanager_tests.py new file mode 100644 index 00000000..5f29d306 --- /dev/null +++ b/morphlib/sourcemanager_tests.py @@ -0,0 +1,49 @@ +# Copyright (C) 2011 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import json +import StringIO +import unittest +import tempfile +import shutil +from tempfile import mkdtemp + +import morphlib + +class DummyApp(object): + def __init__(self, git_base_url='.'): + self.settings = { 'git-base-url': git_base_url } + self.msg = lambda msg: None + +class SourceManagerTests(unittest.TestCase): + + def setUp(self): + self.tempdir = tempfile.mkdtemp() + + def tearDown(self): + shutil.rmtree(self.tempdir) + + + def test_get_treeish_for_self(self): + s = morphlib.sourcemanager.SourceManager(self.tempdir, DummyApp()) + t = s.get_treeish('.','41ee528492db9bd41604311b100da5a871098b3a') + assert(t.sha == '41ee528492db9bd41604311b100da5a871098b3a') + print t.repo + + + + + |