diff options
-rw-r--r-- | morphlib/localartifactcache.py | 48 | ||||
-rw-r--r-- | morphlib/localartifactcache_tests.py | 34 | ||||
-rw-r--r-- | morphlib/plugins/gc_plugin.py | 3 | ||||
-rw-r--r-- | morphlib/util.py | 8 |
4 files changed, 40 insertions, 53 deletions
diff --git a/morphlib/localartifactcache.py b/morphlib/localartifactcache.py index 76d085d1..341bbb56 100644 --- a/morphlib/localartifactcache.py +++ b/morphlib/localartifactcache.py @@ -16,6 +16,7 @@ import collections import os +import time import morphlib @@ -44,8 +45,8 @@ class LocalArtifactCache(object): sense to put the complication there. ''' - def __init__(self, cachedir): - self.cachedir = cachedir + def __init__(self, cachefs): + self.cachefs = cachefs def put(self, artifact): filename = self.artifact_filename(artifact) @@ -93,28 +94,23 @@ class LocalArtifactCache(object): return open(filename) def artifact_filename(self, artifact): - basename = artifact.basename() - return os.path.join(self.cachedir, basename) + return self.cachefs.getsyspath(artifact.basename()) def _artifact_metadata_filename(self, artifact, name): - basename = artifact.metadata_basename(name) - return os.path.join(self.cachedir, basename) + return self.cachefs.getsyspath(artifact.metadata_basename(name)) def _source_metadata_filename(self, source, cachekey, name): - basename = '%s.%s' % (cachekey, name) - return os.path.join(self.cachedir, basename) + return self.cachefs.getsyspath('%s.%s' % (cachekey, name)) def clear(self): '''Clear everything from the artifact cache directory. After calling this, the artifact cache will be entirely empty. Caveat caller. - - ''' - for dirname, subdirs, basenames in os.walk(self.cachedir): - for basename in basenames: - os.remove(os.path.join(dirname, basename)) + ''' + for filename in self.cachefs.walkfiles(): + self.cachefs.remove(filename) def list_contents(self): '''Return the set of sources cached and related information. @@ -124,22 +120,20 @@ class LocalArtifactCache(object): ''' CacheInfo = collections.namedtuple('CacheInfo', ('artifacts', 'mtime')) contents = collections.defaultdict(lambda: CacheInfo(set(), 0)) - for dirpath, dirnames, filenames in os.walk(self.cachedir): - for filename in filenames: - cachekey = filename[:63] - artifact = filename[65:] - artifacts, max_mtime = contents[cachekey] - artifacts.add(artifact) - this_mtime = os.stat(os.path.join(dirpath, filename)).st_mtime - contents[cachekey] = CacheInfo(artifacts, - max(max_mtime, this_mtime)) + for filename in self.cachefs.walkfiles(): + cachekey = filename[:63] + artifact = filename[65:] + artifacts, max_mtime = contents[cachekey] + artifacts.add(artifact) + art_info = self.cachefs.getinfo(filename) + time_t = art_info['modified_time'].timetuple() + contents[cachekey] = CacheInfo(artifacts, + max(max_mtime, time.mktime(time_t))) return ((cache_key, info.artifacts, info.mtime) for cache_key, info in contents.iteritems()) - def remove(self, cachekey): '''Remove all artifacts associated with the given cachekey.''' - for dirpath, dirnames, filenames in os.walk(self.cachedir): - for filename in filenames: - if filename.startswith(cachekey): - os.remove(os.path.join(dirpath, filename)) + for filename in (x for x in self.cachefs.walkfiles() + if x.startswith(cachekey)): + self.cachefs.remove(filename) diff --git a/morphlib/localartifactcache_tests.py b/morphlib/localartifactcache_tests.py index 082b926a..d7743359 100644 --- a/morphlib/localartifactcache_tests.py +++ b/morphlib/localartifactcache_tests.py @@ -17,13 +17,15 @@ import unittest import os +import fs.tempfs + import morphlib class LocalArtifactCacheTests(unittest.TestCase): def setUp(self): - self.tempdir = morphlib.tempdir.Tempdir() + self.tempfs = fs.tempfs.TempFS() morph = morphlib.morph2.Morphology( ''' @@ -52,20 +54,14 @@ class LocalArtifactCacheTests(unittest.TestCase): self.source, 'chunk-devel') self.devel_artifact.cache_key = '0'*64 - def tearDown(self): - self.tempdir.remove() - def test_artifact_filename(self): - cache = morphlib.localartifactcache.LocalArtifactCache( - self.tempdir.dirname) + cache = morphlib.localartifactcache.LocalArtifactCache(self.tempfs) filename = cache.artifact_filename(self.devel_artifact) - expected_name = os.path.join(self.tempdir.dirname, - self.devel_artifact.basename()) + expected_name = self.tempfs.getsyspath(self.devel_artifact.basename()) self.assertEqual(filename, expected_name) def test_put_artifacts_and_check_whether_the_cache_has_them(self): - cache = morphlib.localartifactcache.LocalArtifactCache( - self.tempdir.dirname) + cache = morphlib.localartifactcache.LocalArtifactCache(self.tempfs) handle = cache.put(self.runtime_artifact) handle.write('runtime') @@ -81,8 +77,7 @@ class LocalArtifactCacheTests(unittest.TestCase): self.assertTrue(cache.has(self.devel_artifact)) def test_put_artifacts_and_get_them_afterwards(self): - cache = morphlib.localartifactcache.LocalArtifactCache( - self.tempdir.dirname) + cache = morphlib.localartifactcache.LocalArtifactCache(self.tempfs) handle = cache.put(self.runtime_artifact) handle.write('runtime') @@ -111,8 +106,7 @@ class LocalArtifactCacheTests(unittest.TestCase): self.assertEqual(stored_data, 'devel') def test_put_check_and_get_artifact_metadata(self): - cache = morphlib.localartifactcache.LocalArtifactCache( - self.tempdir.dirname) + cache = morphlib.localartifactcache.LocalArtifactCache(self.tempfs) handle = cache.put_artifact_metadata(self.runtime_artifact, 'log') handle.write('log line 1\nlog line 2\n') @@ -128,8 +122,7 @@ class LocalArtifactCacheTests(unittest.TestCase): self.assertEqual(stored_metadata, 'log line 1\nlog line 2\n') def test_put_check_and_get_source_metadata(self): - cache = morphlib.localartifactcache.LocalArtifactCache( - self.tempdir.dirname) + cache = morphlib.localartifactcache.LocalArtifactCache(self.tempfs) handle = cache.put_source_metadata(self.source, 'mycachekey', 'log') handle.write('source log line 1\nsource log line 2\n') @@ -146,8 +139,7 @@ class LocalArtifactCacheTests(unittest.TestCase): 'source log line 1\nsource log line 2\n') def test_clears_artifact_cache(self): - cache = morphlib.localartifactcache.LocalArtifactCache( - self.tempdir.dirname) + cache = morphlib.localartifactcache.LocalArtifactCache(self.tempfs) handle = cache.put(self.runtime_artifact) handle.write('runtime') @@ -158,8 +150,7 @@ class LocalArtifactCacheTests(unittest.TestCase): self.assertFalse(cache.has(self.runtime_artifact)) def test_put_artifacts_and_list_them_afterwards(self): - cache = morphlib.localartifactcache.LocalArtifactCache( - self.tempdir.dirname) + cache = morphlib.localartifactcache.LocalArtifactCache(self.tempfs) handle = cache.put(self.runtime_artifact) handle.write('runtime') @@ -174,8 +165,7 @@ class LocalArtifactCacheTests(unittest.TestCase): self.assertTrue(len(list(cache.list_contents())) == 1) def test_put_artifacts_and_remove_them_afterwards(self): - cache = morphlib.localartifactcache.LocalArtifactCache( - self.tempdir.dirname) + cache = morphlib.localartifactcache.LocalArtifactCache(self.tempfs) handle = cache.put(self.runtime_artifact) handle.write('runtime') diff --git a/morphlib/plugins/gc_plugin.py b/morphlib/plugins/gc_plugin.py index 1adabe78..abfa1a30 100644 --- a/morphlib/plugins/gc_plugin.py +++ b/morphlib/plugins/gc_plugin.py @@ -19,6 +19,7 @@ import os import shutil import time +import fs.osfs import cliapp import morphlib @@ -114,7 +115,7 @@ class GCPlugin(cliapp.Plugin): chatty=True) return lac = morphlib.localartifactcache.LocalArtifactCache( - os.path.join(cache_path, 'artifacts')) + fs.osfs.OSFS(os.path.join(cache_path, 'artifacts'))) max_age, min_age = self.calculate_delete_range() logging.debug('Must remove artifacts older than timestamp %d' % max_age) diff --git a/morphlib/util.py b/morphlib/util.py index 90ad425e..68aed608 100644 --- a/morphlib/util.py +++ b/morphlib/util.py @@ -17,6 +17,8 @@ import itertools import os import re +import fs.osfs + import morphlib '''Utility functions for morph.''' @@ -118,13 +120,13 @@ def new_artifact_caches(settings): # pragma: no cover if not os.path.exists(artifact_cachedir): os.mkdir(artifact_cachedir) - lac = morphlib.localartifactcache.LocalArtifactCache(artifact_cachedir) + lac = morphlib.localartifactcache.LocalArtifactCache( + fs.osfs.OSFS(artifact_cachedir)) rac_url = get_artifact_cache_server(settings) + rac = None if rac_url: rac = morphlib.remoteartifactcache.RemoteArtifactCache(rac_url) - else: - rac = None return lac, rac |