From eb362413076846559ac1c22a32e2e29174055fcb Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 9 Apr 2012 16:16:03 +0100 Subject: Add a LocalRepoCache class Jannis and I discussed a refactoring of most of morph's internals to reduce coupling and increase cohesion. This is one of the results: we'll want a couple of classes to manage locally cached git repositories. This commit adds the LocalRepoCache class to manage a the git cache directory. Later on, we'll add the CachedRepo class to represent individual repositories. --- morphlib/__init__.py | 1 + morphlib/localrepocache.py | 144 +++++++++++++++++++++++++++++++++++++++ morphlib/localrepocache_tests.py | 98 ++++++++++++++++++++++++++ 3 files changed, 243 insertions(+) create mode 100644 morphlib/localrepocache.py create mode 100644 morphlib/localrepocache_tests.py diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 7d4f1293..a9281b85 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -28,6 +28,7 @@ import cachedir import execute import fsutils import git +import localrepocache import morphology import morphologyloader import savefile diff --git a/morphlib/localrepocache.py b/morphlib/localrepocache.py new file mode 100644 index 00000000..673f9854 --- /dev/null +++ b/morphlib/localrepocache.py @@ -0,0 +1,144 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import logging +import os +import urllib +import urlparse + +import morphlib + + +# urlparse.urljoin needs to know details of the URL scheme being used. +# It does not know about git:// by default, so we teach it here. +gitscheme=['git'] +urlparse.uses_relative.extend(gitscheme) +urlparse.uses_netloc.extend(gitscheme) +urlparse.uses_params.extend(gitscheme) +urlparse.uses_query.extend(gitscheme) +urlparse.uses_fragment.extend(gitscheme) + + + +class CachedRepoPlaceholder(object): + + '''A placeholder until the real CachedRepo exists.''' + + def __init__(self, url, dirname): + pass + + +class NoRemote(Exception): + + def __init__(self, reponame): + self.reponame = reponame + + def __str__(self): + return 'Cannot find remote git repository: %s' % self.reponame + + +class LocalRepoCache(object): + + '''Manage locally cached git repositories. + + When we build stuff, we need a local copy of the git repository. + To avoid having to clone the repositories for every build, we + maintain a local cache of the repositories: we first clone the + remote repository to the cache, and then make a local clone from + the cache to the build environment. This class manages the local + cached repositories. + + Repositories may be specified either using a full URL, in a form + understood by git(1), or as a repository name to which a base url + is prepended. The base urls are given to the class when it is + created. + + ''' + + def __init__(self, cachedir, baseurls): + self._cachedir = cachedir + self._baseurls = baseurls + self._ex = morphlib.execute.Execute(cachedir, logging.debug) + + def _exists(self, filename): # pragma: no cover + '''Does a file exist? + + This is a wrapper around os.path.exists, so that unit tests may + override it. + + ''' + + return os.path.exists(filename) + + def _git(self, args): # pragma: no cover + '''Execute git command. + + This is a method of its own so that unit tests can easily override + all use of the external git command. + + ''' + + self._ex.runv(['git'] + args) + + def _escape(self, url): + '''Escape a URL so it can be used as a basename in a file.''' + return urllib.quote(url, safe='') + + def _cache_name(self, url): + basename = self._escape(url) + path = os.path.join(self._cachedir, basename) + return path + + def _base_iterate(self, reponame): + for baseurl in self._baseurls: + repourl = urlparse.urljoin(baseurl, reponame) + path = self._cache_name(repourl) + yield repourl, path + + def has_repo(self, reponame): + '''Have we already got a cache of a given repo?''' + for repourl, path in self._base_iterate(reponame): + if self._exists(path): + return True + return False + + def cache_repo(self, reponame): + '''Clone the given repo into the cache. + + If the repo is already clone, do nothing. + + ''' + + for repourl, path in self._base_iterate(reponame): + if self._exists(path): + break + try: + self._git(['clone', reponame, path]) + except morphlib.execute.CommandFailure: + pass + else: + break + else: + raise NoRemote(reponame) + + def get_repo(self, reponame): + '''Return an object representing a cached repository.''' + + for repourl, path in self._base_iterate(reponame): + if self._exists(path): + return CachedRepoPlaceholder(repourl, path) + raise Exception('Repository %s is not cached yet' % reponame) + diff --git a/morphlib/localrepocache_tests.py b/morphlib/localrepocache_tests.py new file mode 100644 index 00000000..13a1c415 --- /dev/null +++ b/morphlib/localrepocache_tests.py @@ -0,0 +1,98 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import unittest + +import morphlib + + +class LocalRepoCacheTests(unittest.TestCase): + + def setUp(self): + baseurls = ['git://example.com/'] + self.reponame = 'reponame' + self.repourl = 'git://example.com/reponame' + self.cachedir = '/cache/dir' + self.cache = set() + self.remotes = [] + self.lrc = morphlib.localrepocache.LocalRepoCache(self.cachedir, + baseurls) + self.lrc._git = self.fake_git + self.lrc._exists = self.fake_exists + + def fake_git(self, args): + if args[0] == 'clone': + self.assertEqual(len(args), 3) + remote = args[1] + local = args[2] + if local in self.cache: + raise Exception('cloning twice to %s' % local) + self.remotes.append(remote) + self.cache.add(local) + else: + raise NotImplementedError() + + def fake_exists(self, filename): + return filename in self.cache + + def test_has_not_got_relative_repo_initially(self): + self.assertFalse(self.lrc.has_repo(self.reponame)) + + def test_has_not_got_absolute_repo_initially(self): + self.assertFalse(self.lrc.has_repo(self.repourl)) + + def test_caches_relative_repository_on_request(self): + self.lrc.cache_repo(self.reponame) + self.assertTrue(self.lrc.has_repo(self.reponame)) + self.assertTrue(self.lrc.has_repo(self.repourl)) + + def test_caches_absolute_repository_on_request(self): + self.lrc.cache_repo(self.repourl) + self.assertTrue(self.lrc.has_repo(self.reponame)) + self.assertTrue(self.lrc.has_repo(self.repourl)) + + def test_happily_caches_same_repo_twice(self): + self.lrc.cache_repo(self.repourl) + self.lrc.cache_repo(self.repourl) + + def test_fails_to_cache_when_remote_does_not_exist(self): + def fail(args): + raise morphlib.execute.CommandFailure('', '') + self.lrc._git = fail + self.assertRaises(morphlib.localrepocache.NoRemote, + self.lrc.cache_repo, self.repourl) + + def test_gets_cached_relative_repo(self): + self.lrc.cache_repo(self.reponame) + cached = self.lrc.get_repo(self.reponame) + self.assertTrue(cached is not None) + + def test_gets_cached_absolute_repo(self): + self.lrc.cache_repo(self.repourl) + cached = self.lrc.get_repo(self.repourl) + self.assertTrue(cached is not None) + + def test_get_repo_raises_exception_if_repo_is_not_cached(self): + self.assertRaises(Exception, self.lrc.get_repo, self.repourl) + + def test_escapes_repourl_as_filename(self): + escaped = self.lrc._escape(self.repourl) + self.assertFalse('/' in escaped) + + def test_noremote_error_message_contains_repo_name(self): + e = morphlib.localrepocache.NoRemote(self.repourl) + self.assertTrue(self.repourl in str(e)) + -- cgit v1.2.1