From 98b802e36b2a291b39dbdb83b55546b1acae1573 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Fri, 24 Aug 2012 17:58:42 +0100 Subject: Use git ls-tree to autodetect build system The cost of one git ls-tree call is roughly the same as one git cat-file call. Therefore, when autodetecting the build system, it is much faster to list the tree once and then search for the required files than to call git cat-file for every possible one. --- morphlib/buildsystem.py | 4 +- morphlib/buildsystem_tests.py | 76 +++++++++++-------------------------- morphlib/morphologyfactory.py | 19 ++-------- morphlib/morphologyfactory_tests.py | 14 ++++++- 4 files changed, 41 insertions(+), 72 deletions(-) diff --git a/morphlib/buildsystem.py b/morphlib/buildsystem.py index 8e213b75..86eac850 100644 --- a/morphlib/buildsystem.py +++ b/morphlib/buildsystem.py @@ -195,13 +195,15 @@ build_systems = [ ] -def detect_build_system(exists): +def detect_build_system(file_list): '''Automatically detect the build system, if possible. If the build system cannot be detected automatically, return None. For ``exists`` see the ``BuildSystem.exists`` method. ''' + def exists(filename): + return filename in file_list for bs in build_systems: if bs.used_by_project(exists): diff --git a/morphlib/buildsystem_tests.py b/morphlib/buildsystem_tests.py index c955f12a..53b4fc17 100644 --- a/morphlib/buildsystem_tests.py +++ b/morphlib/buildsystem_tests.py @@ -26,13 +26,8 @@ def touch(pathname): with open(pathname, 'w'): pass - -def create_manual_project(srcdir): - pass - - -def create_autotools_project(srcdir): - touch(os.path.join(srcdir, 'configure.in')) +manual_project = [] +autotools_project = ['configure.in'] class BuildSystemTests(unittest.TestCase): @@ -62,85 +57,58 @@ class ManualBuildSystemTests(unittest.TestCase): def setUp(self): self.bs = morphlib.buildsystem.ManualBuildSystem() - self.tempdir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self.tempdir) - - def exists(self, filename): - return os.path.exists(os.path.join(self.tempdir, filename)) def test_does_not_autodetect_empty(self): - create_manual_project(self.tempdir) - self.assertFalse(self.bs.used_by_project(self.exists)) + def exists(filename): + return filename in manual_project + self.assertFalse(self.bs.used_by_project(exists)) def test_does_not_autodetect_autotools(self): - create_autotools_project(self.tempdir) - self.assertFalse(self.bs.used_by_project(self.exists)) + def exists(filename): + return filename in autotools_project + self.assertFalse(self.bs.used_by_project(exists)) class DummyBuildSystemTests(unittest.TestCase): def setUp(self): self.bs = morphlib.buildsystem.DummyBuildSystem() - self.tempdir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self.tempdir) - - def exists(self, filename): - return os.path.exists(os.path.join(self.tempdir, filename)) def test_does_not_autodetect_empty(self): - create_manual_project(self.tempdir) - self.assertFalse(self.bs.used_by_project(self.exists)) + def exists(filename): + return filename in manual_project + self.assertFalse(self.bs.used_by_project(exists)) def test_does_not_autodetect_autotools(self): - create_autotools_project(self.tempdir) - self.assertFalse(self.bs.used_by_project(self.exists)) + def exists(filename): + return filename in autotools_project + self.assertFalse(self.bs.used_by_project(exists)) class AutotoolsBuildSystemTests(unittest.TestCase): def setUp(self): self.bs = morphlib.buildsystem.AutotoolsBuildSystem() - self.tempdir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self.tempdir) - - def exists(self, filename): - return os.path.exists(os.path.join(self.tempdir, filename)) def test_does_not_autodetect_empty(self): - create_manual_project(self.tempdir) - self.assertFalse(self.bs.used_by_project(self.exists)) + def exists(filename): + return filename in manual_project + self.assertFalse(self.bs.used_by_project(exists)) def test_autodetects_autotools(self): - create_autotools_project(self.tempdir) - self.assertTrue(self.bs.used_by_project(self.exists)) + def exists(filename): + return filename in autotools_project + self.assertTrue(self.bs.used_by_project(exists)) class DetectBuildSystemTests(unittest.TestCase): - def setUp(self): - self.bs = morphlib.buildsystem.ManualBuildSystem() - self.tempdir = tempfile.mkdtemp() - - def tearDown(self): - shutil.rmtree(self.tempdir) - - def exists(self, filename): - return os.path.exists(os.path.join(self.tempdir, filename)) - def test_does_not_autodetect_manual(self): - create_manual_project(self.tempdir) - bs = morphlib.buildsystem.detect_build_system(self.exists) + bs = morphlib.buildsystem.detect_build_system(manual_project) self.assertEqual(bs, None) def test_autodetects_autotools(self): - create_autotools_project(self.tempdir) - bs = morphlib.buildsystem.detect_build_system(self.exists) + bs = morphlib.buildsystem.detect_build_system(autotools_project) self.assertEqual(type(bs), morphlib.buildsystem.AutotoolsBuildSystem) diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py index 89288b33..c03d541e 100644 --- a/morphlib/morphologyfactory.py +++ b/morphlib/morphologyfactory.py @@ -72,26 +72,15 @@ class MorphologyFactory(object): raise NotcachedError(reponame) def _autodetect_text(self, reponame, sha1, filename): - # TODO get lists of files from the cache to reduce round trips if self._lrc.has_repo(reponame): repo = self._lrc.get_repo(reponame) - - def has_file(filename): - try: - repo.cat(sha1, filename) - return True - except IOError: - return False + file_list = repo.ls_tree(sha1) elif self._rrc is not None: - def has_file(filename): - try: - text = self._rrc.cat_file(reponame, sha1, filename) - return True - except morphlib.remoterepocache.CatFileError: - return False + file_list = self._rrc.ls_tree(reponame, sha1) else: raise NotcachedError(reponame) - bs = morphlib.buildsystem.detect_build_system(has_file) + + bs = morphlib.buildsystem.detect_build_system(file_list) if bs is None: raise AutodetectError(reponame, sha1) # TODO consider changing how morphs are located to be by morph diff --git a/morphlib/morphologyfactory_tests.py b/morphlib/morphologyfactory_tests.py index 39331262..9d6900d7 100644 --- a/morphlib/morphologyfactory_tests.py +++ b/morphlib/morphologyfactory_tests.py @@ -35,6 +35,8 @@ class FakeRemoteRepoCache(object): }''' return 'text' + def ls_tree(self, reponame, sha1): + return [] class FakeLocalRepo(object): @@ -87,6 +89,8 @@ class FakeLocalRepo(object): }''' return 'text' + def ls_tree(self, sha1): + return [] class FakeLocalRepoCache(object): @@ -120,6 +124,9 @@ class MorphologyFactoryTests(unittest.TestCase): raise IOError('File not found') return 'text' + def autotoolsbuildsystem(self, *args): + return ['configure.in'] + def noremotemorph(self, *args): if args[-1].endswith('.morph'): raise CatFileError('reponame', 'ref', 'filename') @@ -141,13 +148,15 @@ class MorphologyFactoryTests(unittest.TestCase): def test_autodetects_local_morphology(self): self.lr.cat = self.nolocalmorph + self.lr.ls_tree = self.autotoolsbuildsystem morph = self.mf.get_morphology('reponame', 'sha1', - 'assumed-local.morph') + 'assumed-local.morph') self.assertEqual('assumed-local', morph['name']) def test_autodetects_remote_morphology(self): self.lrc.has_repo = self.doesnothaverepo self.rrc.cat_file = self.noremotemorph + self.rrc.ls_tree = self.autotoolsbuildsystem morph = self.mf.get_morphology('reponame', 'sha1', 'assumed-remote.morph') self.assertEqual('assumed-remote', morph['name']) @@ -171,7 +180,8 @@ class MorphologyFactoryTests(unittest.TestCase): def test_autodetects_locally_with_no_remote(self): self.lr.cat = self.nolocalmorph - morph = self.lmf.get_morphology('reponame', 'sha1', + self.lr.ls_tree = self.autotoolsbuildsystem + morph = self.mf.get_morphology('reponame', 'sha1', 'assumed-local.morph') self.assertEqual('assumed-local', morph['name']) -- cgit v1.2.1