diff options
Diffstat (limited to 'test/performance')
| -rw-r--r-- | test/performance/lib.py | 43 | ||||
| -rw-r--r-- | test/performance/test_commit.py | 52 | ||||
| -rw-r--r-- | test/performance/test_odb.py | 39 | ||||
| -rw-r--r-- | test/performance/test_streams.py | 87 |
4 files changed, 142 insertions, 79 deletions
diff --git a/test/performance/lib.py b/test/performance/lib.py index 86f87757..101e2cd4 100644 --- a/test/performance/lib.py +++ b/test/performance/lib.py @@ -3,27 +3,21 @@ import logging import os import tempfile -from git import ( - Repo -) -from git.db import ( - GitCmdObjectDB, - GitDB -) -from test.lib import ( - TestBase -) +from git import Repo +from git.db import GitCmdObjectDB, GitDB +from test.lib import TestBase from git.util import rmtree import os.path as osp -#{ Invariants +# { Invariants k_env_git_repo = "GIT_PYTHON_TEST_GIT_REPO_BASE" -#} END invariants +# } END invariants -#{ Base Classes +# { Base Classes + class TestBigRepoR(TestBase): @@ -39,8 +33,8 @@ class TestBigRepoR(TestBase): * As gitrepo, but uses pure python implementation """ - #{ Invariants - #} END invariants + # { Invariants + # } END invariants def setUp(self): try: @@ -51,11 +45,17 @@ class TestBigRepoR(TestBase): repo_path = os.environ.get(k_env_git_repo) if repo_path is None: logging.info( - ("You can set the %s environment variable to a .git repository of" % k_env_git_repo) + - "your choice - defaulting to the gitpython repository") + ( + "You can set the %s environment variable to a .git repository of" + % k_env_git_repo + ) + + "your choice - defaulting to the gitpython repository" + ) repo_path = osp.dirname(__file__) # end set some repo path - self.gitrorepo = Repo(repo_path, odbt=GitCmdObjectDB, search_parent_directories=True) + self.gitrorepo = Repo( + repo_path, odbt=GitCmdObjectDB, search_parent_directories=True + ) self.puregitrorepo = Repo(repo_path, odbt=GitDB, search_parent_directories=True) def tearDown(self): @@ -79,7 +79,9 @@ class TestBigRepoRW(TestBigRepoR): pass dirname = tempfile.mktemp() os.mkdir(dirname) - self.gitrwrepo = self.gitrorepo.clone(dirname, shared=True, bare=True, odbt=GitCmdObjectDB) + self.gitrwrepo = self.gitrorepo.clone( + dirname, shared=True, bare=True, odbt=GitCmdObjectDB + ) self.puregitrwrepo = Repo(dirname, odbt=GitDB) def tearDown(self): @@ -91,4 +93,5 @@ class TestBigRepoRW(TestBigRepoR): self.puregitrwrepo.git.clear_cache() self.puregitrwrepo = None -#} END base classes + +# } END base classes diff --git a/test/performance/test_commit.py b/test/performance/test_commit.py index 8158a1e6..25cc34b8 100644 --- a/test/performance/test_commit.py +++ b/test/performance/test_commit.py @@ -14,13 +14,13 @@ from test.test_commit import TestCommitSerialization class TestPerformance(TestBigRepoRW, TestCommitSerialization): - def tearDown(self): import gc + gc.collect() # ref with about 100 commits in its history - ref_100 = '0.1.6' + ref_100 = "0.1.6" def _query_commit_info(self, c): c.author @@ -50,8 +50,11 @@ class TestPerformance(TestBigRepoRW, TestCommitSerialization): # END for each object # END for each commit elapsed_time = time() - st - print("Traversed %i Trees and a total of %i uncached objects in %s [s] ( %f objs/s )" - % (nc, no, elapsed_time, no / elapsed_time), file=sys.stderr) + print( + "Traversed %i Trees and a total of %i uncached objects in %s [s] ( %f objs/s )" + % (nc, no, elapsed_time, no / elapsed_time), + file=sys.stderr, + ) def test_commit_traversal(self): # bound to cat-file parsing performance @@ -62,8 +65,11 @@ class TestPerformance(TestBigRepoRW, TestCommitSerialization): self._query_commit_info(c) # END for each traversed commit elapsed_time = time() - st - print("Traversed %i Commits in %s [s] ( %f commits/s )" - % (nc, elapsed_time, nc / elapsed_time), file=sys.stderr) + print( + "Traversed %i Commits in %s [s] ( %f commits/s )" + % (nc, elapsed_time, nc / elapsed_time), + file=sys.stderr, + ) def test_commit_iteration(self): # bound to stream parsing performance @@ -74,11 +80,14 @@ class TestPerformance(TestBigRepoRW, TestCommitSerialization): self._query_commit_info(c) # END for each traversed commit elapsed_time = time() - st - print("Iterated %i Commits in %s [s] ( %f commits/s )" - % (nc, elapsed_time, nc / elapsed_time), file=sys.stderr) + print( + "Iterated %i Commits in %s [s] ( %f commits/s )" + % (nc, elapsed_time, nc / elapsed_time), + file=sys.stderr, + ) def test_commit_serialization(self): - self.assert_commit_serialization(self.gitrwrepo, '58c78e6', True) + self.assert_commit_serialization(self.gitrwrepo, "58c78e6", True) rwrepo = self.gitrwrepo make_object = rwrepo.odb.store @@ -89,10 +98,20 @@ class TestPerformance(TestBigRepoRW, TestCommitSerialization): nc = 5000 st = time() for i in range(nc): - cm = Commit(rwrepo, Commit.NULL_BIN_SHA, hc.tree, - hc.author, hc.authored_date, hc.author_tz_offset, - hc.committer, hc.committed_date, hc.committer_tz_offset, - str(i), parents=hc.parents, encoding=hc.encoding) + cm = Commit( + rwrepo, + Commit.NULL_BIN_SHA, + hc.tree, + hc.author, + hc.authored_date, + hc.author_tz_offset, + hc.committer, + hc.committed_date, + hc.committer_tz_offset, + str(i), + parents=hc.parents, + encoding=hc.encoding, + ) stream = BytesIO() cm._serialize(stream) @@ -103,5 +122,8 @@ class TestPerformance(TestBigRepoRW, TestCommitSerialization): # END commit creation elapsed = time() - st - print("Serialized %i commits to loose objects in %f s ( %f commits / s )" - % (nc, elapsed, nc / elapsed), file=sys.stderr) + print( + "Serialized %i commits to loose objects in %f s ( %f commits / s )" + % (nc, elapsed, nc / elapsed), + file=sys.stderr, + ) diff --git a/test/performance/test_odb.py b/test/performance/test_odb.py index c9521c56..680464c9 100644 --- a/test/performance/test_odb.py +++ b/test/performance/test_odb.py @@ -2,13 +2,10 @@ import sys from time import time -from .lib import ( - TestBigRepoR -) +from .lib import TestBigRepoR class TestObjDBPerformance(TestBigRepoR): - def test_random_access(self): results = [["Iterate Commits"], ["Iterate Blobs"], ["Retrieve Blob Data"]] for repo in (self.gitrorepo, self.puregitrorepo): @@ -19,8 +16,11 @@ class TestObjDBPerformance(TestBigRepoR): nc = len(commits) elapsed = time() - st - print("%s: Retrieved %i commits from ObjectStore in %g s ( %f commits / s )" - % (type(repo.odb), nc, elapsed, nc / elapsed), file=sys.stderr) + print( + "%s: Retrieved %i commits from ObjectStore in %g s ( %f commits / s )" + % (type(repo.odb), nc, elapsed, nc / elapsed), + file=sys.stderr, + ) results[0].append(elapsed) # GET TREES @@ -33,7 +33,7 @@ class TestObjDBPerformance(TestBigRepoR): blobs = [] for item in tree.traverse(): nt += 1 - if item.type == 'blob': + if item.type == "blob": blobs.append(item) # direct access for speed # END while trees are there for walking @@ -41,8 +41,11 @@ class TestObjDBPerformance(TestBigRepoR): # END for each commit elapsed = time() - st - print("%s: Retrieved %i objects from %i commits in %g s ( %f objects / s )" - % (type(repo.odb), nt, len(commits), elapsed, nt / elapsed), file=sys.stderr) + print( + "%s: Retrieved %i objects from %i commits in %g s ( %f objects / s )" + % (type(repo.odb), nt, len(commits), elapsed, nt / elapsed), + file=sys.stderr, + ) results[1].append(elapsed) # GET BLOBS @@ -60,13 +63,25 @@ class TestObjDBPerformance(TestBigRepoR): # END for each bloblist elapsed = time() - st - msg = "%s: Retrieved %i blob (%i KiB) and their data in %g s ( %f blobs / s, %f KiB / s )"\ - % (type(repo.odb), nb, data_bytes / 1000, elapsed, nb / elapsed, (data_bytes / 1000) / elapsed) + msg = ( + "%s: Retrieved %i blob (%i KiB) and their data in %g s ( %f blobs / s, %f KiB / s )" + % ( + type(repo.odb), + nb, + data_bytes / 1000, + elapsed, + nb / elapsed, + (data_bytes / 1000) / elapsed, + ) + ) print(msg, file=sys.stderr) results[2].append(elapsed) # END for each repo type # final results for test_name, a, b in results: - print("%s: %f s vs %f s, pure is %f times slower" % (test_name, a, b, b / a), file=sys.stderr) + print( + "%s: %f s vs %f s, pure is %f times slower" % (test_name, a, b, b / a), + file=sys.stderr, + ) # END for each result diff --git a/test/performance/test_streams.py b/test/performance/test_streams.py index 28e6b13e..2ae94e29 100644 --- a/test/performance/test_streams.py +++ b/test/performance/test_streams.py @@ -4,36 +4,29 @@ import subprocess import sys from time import time -from test.lib import ( - with_rw_repo -) +from test.lib import with_rw_repo from git.util import bin_to_hex -from gitdb import ( - LooseObjectDB, - IStream -) +from gitdb import LooseObjectDB, IStream from gitdb.test.lib import make_memory_file import os.path as osp -from .lib import ( - TestBigRepoR -) +from .lib import TestBigRepoR class TestObjDBPerformance(TestBigRepoR): - large_data_size_bytes = 1000 * 1000 * 10 # some MiB should do it - moderate_data_size_bytes = 1000 * 1000 * 1 # just 1 MiB + large_data_size_bytes = 1000 * 1000 * 10 # some MiB should do it + moderate_data_size_bytes = 1000 * 1000 * 1 # just 1 MiB - @with_rw_repo('HEAD', bare=True) + @with_rw_repo("HEAD", bare=True) def test_large_data_streaming(self, rwrepo): # TODO: This part overlaps with the same file in gitdb.test.performance.test_stream # It should be shared if possible - ldb = LooseObjectDB(osp.join(rwrepo.git_dir, 'objects')) + ldb = LooseObjectDB(osp.join(rwrepo.git_dir, "objects")) for randomize in range(2): - desc = (randomize and 'random ') or '' + desc = (randomize and "random ") or "" print("Creating %s data ..." % desc, file=sys.stderr) st = time() size, stream = make_memory_file(self.large_data_size_bytes, randomize) @@ -42,7 +35,7 @@ class TestObjDBPerformance(TestBigRepoR): # writing - due to the compression it will seem faster than it is st = time() - binsha = ldb.store(IStream('blob', size, stream)).binsha + binsha = ldb.store(IStream("blob", size, stream)).binsha elapsed_add = time() - st assert ldb.has_object(binsha) db_file = ldb.readable_db_object_path(bin_to_hex(binsha)) @@ -79,33 +72,45 @@ class TestObjDBPerformance(TestBigRepoR): elapsed_readchunks = time() - st stream.seek(0) - assert b''.join(chunks) == stream.getvalue() + assert b"".join(chunks) == stream.getvalue() cs_kib = cs / 1000 - print("Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" - % (size_kib, desc, cs_kib, elapsed_readchunks, size_kib / elapsed_readchunks), file=sys.stderr) + print( + "Read %i KiB of %s data in %i KiB chunks from loose odb in %f s ( %f Read KiB / s)" + % ( + size_kib, + desc, + cs_kib, + elapsed_readchunks, + size_kib / elapsed_readchunks, + ), + file=sys.stderr, + ) # del db file so git has something to do ostream = None import gc + gc.collect() os.remove(db_file) # VS. CGIT ########## # CGIT ! Can using the cgit programs be faster ? - proc = rwrepo.git.hash_object('-w', '--stdin', as_process=True, istream=subprocess.PIPE) + proc = rwrepo.git.hash_object( + "-w", "--stdin", as_process=True, istream=subprocess.PIPE + ) # write file - pump everything in at once to be a fast as possible - data = stream.getvalue() # cache it + data = stream.getvalue() # cache it st = time() proc.stdin.write(data) proc.stdin.close() gitsha = proc.stdout.read().strip() proc.wait() gelapsed_add = time() - st - del(data) - assert gitsha == bin_to_hex(binsha) # we do it the same way, right ? + del data + assert gitsha == bin_to_hex(binsha) # we do it the same way, right ? # as its the same sha, we reuse our path fsize_kib = osp.getsize(db_file) / 1000 @@ -114,19 +119,28 @@ class TestObjDBPerformance(TestBigRepoR): print(msg, file=sys.stderr) # compare ... - print("Git-Python is %f %% faster than git when adding big %s files" - % (100.0 - (elapsed_add / gelapsed_add) * 100, desc), file=sys.stderr) + print( + "Git-Python is %f %% faster than git when adding big %s files" + % (100.0 - (elapsed_add / gelapsed_add) * 100, desc), + file=sys.stderr, + ) # read all st = time() _hexsha, _typename, size, data = rwrepo.git.get_object_data(gitsha) gelapsed_readall = time() - st - print("Read %i KiB of %s data at once using git-cat-file in %f s ( %f Read KiB / s)" - % (size_kib, desc, gelapsed_readall, size_kib / gelapsed_readall), file=sys.stderr) + print( + "Read %i KiB of %s data at once using git-cat-file in %f s ( %f Read KiB / s)" + % (size_kib, desc, gelapsed_readall, size_kib / gelapsed_readall), + file=sys.stderr, + ) # compare - print("Git-Python is %f %% faster than git when reading big %sfiles" - % (100.0 - (elapsed_readall / gelapsed_readall) * 100, desc), file=sys.stderr) + print( + "Git-Python is %f %% faster than git when reading big %sfiles" + % (100.0 - (elapsed_readall / gelapsed_readall) * 100, desc), + file=sys.stderr, + ) # read chunks st = time() @@ -138,10 +152,19 @@ class TestObjDBPerformance(TestBigRepoR): # END read stream gelapsed_readchunks = time() - st msg = "Read %i KiB of %s data in %i KiB chunks from git-cat-file in %f s ( %f Read KiB / s)" - msg %= (size_kib, desc, cs_kib, gelapsed_readchunks, size_kib / gelapsed_readchunks) + msg %= ( + size_kib, + desc, + cs_kib, + gelapsed_readchunks, + size_kib / gelapsed_readchunks, + ) print(msg, file=sys.stderr) # compare - print("Git-Python is %f %% faster than git when reading big %s files in chunks" - % (100.0 - (elapsed_readchunks / gelapsed_readchunks) * 100, desc), file=sys.stderr) + print( + "Git-Python is %f %% faster than git when reading big %s files in chunks" + % (100.0 - (elapsed_readchunks / gelapsed_readchunks) * 100, desc), + file=sys.stderr, + ) # END for each randomization factor |