diff options
| -rw-r--r-- | morphlib/__init__.py | 1 | ||||
| -rw-r--r-- | morphlib/cachekeycomputer.py | 79 | ||||
| -rw-r--r-- | morphlib/cachekeycomputer_tests.py | 95 |
3 files changed, 175 insertions, 0 deletions
diff --git a/morphlib/__init__.py b/morphlib/__init__.py index ab21e4c0..26727929 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -28,6 +28,7 @@ import buildworker import builder import cachedir import cachedrepo +import cachekeycomputer import dependencyresolver import execute import fsutils diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py new file mode 100644 index 00000000..ade74b79 --- /dev/null +++ b/morphlib/cachekeycomputer.py @@ -0,0 +1,79 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import hashlib + +import morphlib + +class CacheKeyComputer(): + + def __init__(self, env): + self._arch = morphlib.util.arch() + self._env = self._filterenv(env) + self._calculated = {} + + def _filterenv(self, env): + return dict([(k, env[k]) for k in ("USER", "USERNAME", "LOGNAME", + "TOOLCHAIN_TARGET", "PREFIX", + "BOOTSTRAP", "CFLAGS")]) + + def get_cache_key(self, source): + return self._hash_id(self.get_cache_id(source)) + + def _hash_id(self, cache_id): + sha = hashlib.sha256() + self._hash_dict(sha, cache_id) + return sha.hexdigest() + + def _hash_thing(self, sha, thing): + if type(thing) == dict: + self._hash_dict(sha, thing) + elif type(thing) == list: + self._hash_list(sha, thing) + elif type(thing) == tuple: + self._hash_tuple(sha, thing) + else: + sha.update(str(thing)) + + def _hash_dict(self, sha, d): + for tup in sorted(d.iteritems()): + self._hash_thing(sha, tup) + + def _hash_list(self, sha, l): + for item in l: + self._hash_thing(sha, item) + + def _hash_tuple(self, sha, tup): + for item in tup: + self._hash_thing(sha, item) + + def get_cache_id(self, source): + try: + return self._calculated[source] + except KeyError: + cacheid = self._calculate(source) + self._calculated[source] = cacheid + return cacheid + + def _calculate(self, source): + return { + 'arch': self._arch, + 'env': self._env, + 'ref': source.sha1, + 'filename': source.filename, + 'kids': [self.get_cache_id(dependency) + for dependency in source.dependencies], + } diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py new file mode 100644 index 00000000..5304307a --- /dev/null +++ b/morphlib/cachekeycomputer_tests.py @@ -0,0 +1,95 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import unittest +import json + +import morphlib + + +class CacheKeyComputerTests(unittest.TestCase): + + def setUp(self): + self.env = {"USER": "foouser", + "USERNAME": "foouser", + "LOGNAME": "foouser", + "TOOLCHAIN_TARGET": "dummy-baserock-linux-gnu", + "PREFIX": "/baserock", + "BOOTSTRAP": "false", + "CFLAGS": "-O4"} + self.ckc = morphlib.cachekeycomputer.CacheKeyComputer(self.env) + pool = morphlib.sourcepool.SourcePool() + self.sources = {} + for name, text in { + 'chunk.morph': '''{ + "name": "chunk", + "kind": "chunk" + }''', + 'stratum.morph': '''{ + "name": "stratum", + "kind": "stratum", + "sources": [ + { + "name": "chunk", + "repo": "repo", + "ref": "original/ref" + } + ] + }''', + }.iteritems(): + source = morphlib.source.Source('repo', 'original/ref', 'sha', + morphlib.morph2.Morphology(text), name) + pool.add(source) + self.sources[name] = source + + morphlib.buildgraph.BuildGraph().compute_build_order(pool) + + def _valid_sha256(self, s): + validchars = '0123456789abcdef' + return len(s) == 64 and all(c in validchars for c in s) + + def test_get_cache_key_hashes_all_types(self): + runcount = {'thing': 0, 'dict': 0, 'list': 0, 'tuple': 0} + def inccount(func, name): + def f(sha, item): + runcount[name] = runcount[name] + 1 + func(sha, item) + return f + self.ckc._hash_thing = inccount(self.ckc._hash_thing, 'thing') + self.ckc._hash_dict = inccount(self.ckc._hash_dict, 'dict') + self.ckc._hash_list = inccount(self.ckc._hash_list, 'list') + self.ckc._hash_tuple = inccount(self.ckc._hash_tuple, 'tuple') + self.ckc.get_cache_key(self.sources['stratum.morph']) + self.assertNotEqual(runcount['thing'], 0) + self.assertNotEqual(runcount['dict'], 0) + self.assertNotEqual(runcount['list'], 0) + self.assertNotEqual(runcount['tuple'], 0) + + def test_get_cache_key_returns_sha256(self): + self.assertTrue(self._valid_sha256( + self.ckc.get_cache_key(self.sources['stratum.morph']))) + + def test_get_cache_id_returns_dict(self): + print json.dumps(self.ckc.get_cache_id( + self.sources['stratum.morph']), + indent=2) + self.assertEqual('', ) + + def test_different_env_gives_different_key(self): + oldsha = self.ckc.get_cache_key(self.sources['stratum.morph']) + self.ckc._env['CFLAGS'] = "-Os" + self.assertNotEqual(oldsha, + self.ckc.get_cache_key(self.sources['stratum.morph'])) |