From 8f88b10048c858179a2c19c6eb479e59406d8b73 Mon Sep 17 00:00:00 2001
From: Daniel Firth <dan.firth@codethink.co.uk>
Date: Tue, 10 Dec 2013 15:06:57 +0000
Subject: Refactored localrepocache

---
 morphlib/localrepocache.py       | 130 ++++++++++-----------------------------
 morphlib/localrepocache_tests.py |  60 ++++++++----------
 2 files changed, 59 insertions(+), 131 deletions(-)

diff --git a/morphlib/localrepocache.py b/morphlib/localrepocache.py
index aa45cd3d..9c20e4bc 100644
--- a/morphlib/localrepocache.py
+++ b/morphlib/localrepocache.py
@@ -19,10 +19,11 @@ import os
 import re
 import urllib2
 import urlparse
-import shutil
 import string
+import tempfile
 
 import cliapp
+import fs.osfs
 
 import morphlib
 
@@ -93,6 +94,7 @@ class LocalRepoCache(object):
 
     def __init__(self, app, cachedir, resolver, tarball_base_url=None):
         self._app = app
+        self.fs = fs.osfs.OSFS('/')
         self._cachedir = cachedir
         self._resolver = resolver
         if tarball_base_url and not tarball_base_url.endswith('/'):
@@ -100,16 +102,6 @@ class LocalRepoCache(object):
         self._tarball_base_url = tarball_base_url
         self._cached_repo_objects = {}
 
-    def _exists(self, filename):  # pragma: no cover
-        '''Does a file exist?
-
-        This is a wrapper around os.path.exists, so that unit tests may
-        override it.
-
-        '''
-
-        return os.path.exists(filename)
-
     def _git(self, args, cwd=None):  # pragma: no cover
         '''Execute git command.
 
@@ -120,67 +112,24 @@ class LocalRepoCache(object):
 
         self._app.runcmd(['git'] + args, cwd=cwd)
 
-    def _runcmd(self, args, cwd=None):  # pragma: no cover
-        '''Execute a command.
-
-        This is a method of its own so that unit tests can easily override
-        all use of the external git command.
-
-        '''
-
-        self._app.runcmd(args, cwd=cwd)
-
-    def _fetch(self, url, filename):  # pragma: no cover
+    def _fetch(self, url, path):  # pragma: no cover
         '''Fetch contents of url into a file.
 
         This method is meant to be overridden by unit tests.
 
         '''
         self._app.status(msg="Trying to fetch %(tarball)s to seed the cache",
-                         tarball=url,
-                         chatty=True)
-        source_handle = None
-        try:
-            source_handle = urllib2.urlopen(url)
-            with open(filename, 'wb') as target_handle:
-                shutil.copyfileobj(source_handle, target_handle)
-            self._app.status(msg="Tarball fetch successful",
-                             chatty=True)
-        except Exception, e:
-            self._app.status(msg="Tarball fetch failed: %(reason)s",
-                             reason=e,
-                             chatty=True)
-            raise
-        finally:
-            if source_handle is not None:
-                source_handle.close()
-
-    def _mkdir(self, dirname):  # pragma: no cover
-        '''Create a directory.
-
-        This method is meant to be overridden by unit tests.
-
-        '''
-
-        os.mkdir(dirname)
+                         tarball=url, chatty=True)
+        self._app.runcmd(['wget', '-q', '-O-', url],
+                         ['tar', 'xf', '-'], cwd=path)
 
-    def _remove(self, filename):  # pragma: no cover
-        '''Remove given file.
+    def _mkdtemp(self, dirname):  # pragma: no cover
+        '''Creates a temporary directory.
 
         This method is meant to be overridden by unit tests.
 
         '''
-
-        os.remove(filename)
-
-    def _rmtree(self, dirname):  # pragma: no cover
-        '''Remove given directory tree.
-
-        This method is meant to be overridden by unit tests.
-
-        '''
-
-        shutil.rmtree(dirname)
+        return tempfile.mkdtemp(dir=dirname)
 
     def _escape(self, url):
         '''Escape a URL so it can be used as a basename in a file.'''
@@ -194,45 +143,31 @@ class LocalRepoCache(object):
 
     def _cache_name(self, url):
         scheme, netloc, path, query, fragment = urlparse.urlsplit(url)
-        if scheme == 'file':
-            return path
-        else:
-            basename = self._escape(url)
-            path = os.path.join(self._cachedir, basename)
-            return path
+        if scheme != 'file':
+            path = os.path.join(self._cachedir, self._escape(url))
+        return path
 
     def has_repo(self, reponame):
         '''Have we already got a cache of a given repo?'''
         url = self._resolver.pull_url(reponame)
         path = self._cache_name(url)
-        return self._exists(path)
+        return self.fs.exists(path)
 
     def _clone_with_tarball(self, repourl, path):
-        escaped = self._escape(repourl)
         tarball_url = urlparse.urljoin(self._tarball_base_url,
-                                       escaped) + '.tar'
-        tarball_path = path + '.tar'
-
+                                       self._escape(repourl)) + '.tar'
         try:
-            self._fetch(tarball_url, tarball_path)
-        except urllib2.URLError, e:
-            return False, 'Unable to fetch tarball %s: %s' % (tarball_url, e)
-
-        try:
-            self._mkdir(path)
-            self._runcmd(['tar', 'xf', tarball_path], cwd=path)
+            self.fs.makedir(path)
+            self._fetch(tarball_url, path)
             self._git(['config', 'remote.origin.url', repourl], cwd=path)
             self._git(['config', 'remote.origin.mirror', 'true'], cwd=path)
             self._git(['config', 'remote.origin.fetch', '+refs/*:refs/*'],
                       cwd=path)
-        except cliapp.AppException, e:  # pragma: no cover
-            if self._exists(path):
-                shutil.rmtree(path)
+        except BaseException, e:  # pragma: no cover
+            if self.fs.exists(path):
+                self.fs.removedir(path, force=True)
             return False, 'Unable to extract tarball %s: %s' % (
-                tarball_path, e)
-        finally:
-            if self._exists(tarball_path):
-                self._remove(tarball_path)
+                tarball_url, e)
 
         return True, None
 
@@ -243,32 +178,35 @@ class LocalRepoCache(object):
 
         '''
         errors = []
-        if not self._exists(self._cachedir):
-            self._mkdir(self._cachedir)
+        if not self.fs.exists(self._cachedir):
+            self.fs.makedir(self._cachedir, recursive=True)
 
         try:
             return self.get_repo(reponame)
         except NotCached, e:
             pass
 
+        repourl = self._resolver.pull_url(reponame)
+        path = self._cache_name(repourl)
         if self._tarball_base_url:
-            repourl = self._resolver.pull_url(reponame)
-            path = self._cache_name(repourl)
             ok, error = self._clone_with_tarball(repourl, path)
             if ok:
                 return self.get_repo(reponame)
             else:
                 errors.append(error)
-
-        repourl = self._resolver.pull_url(reponame)
-        path = self._cache_name(repourl)
+                self._app.status(
+                    msg='Failed to fetch tarball, falling back to git clone.')
+        target = self._mkdtemp(self._cachedir)
         try:
-            self._git(['clone', '--mirror', '-n', repourl, path])
+            self._git(['clone', '--mirror', '-n', repourl, target])
         except cliapp.AppException, e:
             errors.append('Unable to clone from %s to %s: %s' %
-                          (repourl, path, e))
+                          (repourl, target, e))
+            if self.fs.exists(target):
+                self.fs.removedir(target, recursive=True, force=True)
             raise NoRemote(reponame, errors)
 
+        self.fs.rename(target, path)
         return self.get_repo(reponame)
 
     def get_repo(self, reponame):
@@ -279,7 +217,7 @@ class LocalRepoCache(object):
         else:
             repourl = self._resolver.pull_url(reponame)
             path = self._cache_name(repourl)
-            if self._exists(path):
+            if self.fs.exists(path):
                 repo = morphlib.cachedrepo.CachedRepo(self._app, reponame,
                                                       repourl, path)
                 self._cached_repo_objects[reponame] = repo
diff --git a/morphlib/localrepocache_tests.py b/morphlib/localrepocache_tests.py
index 6c5410ce..22b5ea54 100644
--- a/morphlib/localrepocache_tests.py
+++ b/morphlib/localrepocache_tests.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012  Codethink Limited
+# Copyright (C) 2012-2013  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -19,10 +19,17 @@ import urllib2
 import os
 
 import cliapp
+import fs.memoryfs
 
 import morphlib
 
 
+class FakeApplication(object):
+
+    def status(self, msg):
+        pass
+
+
 class LocalRepoCacheTests(unittest.TestCase):
 
     def setUp(self):
@@ -35,35 +42,24 @@ class LocalRepoCacheTests(unittest.TestCase):
         self.tarball_url = '%s%s.tar' % (tarball_base_url, escaped_url)
         self.cachedir = '/cache/dir'
         self.cache_path = '%s/%s' % (self.cachedir, escaped_url)
-        self.cache = set()
         self.remotes = {}
         self.fetched = []
         self.removed = []
         self.lrc = morphlib.localrepocache.LocalRepoCache(
-            object(), self.cachedir, repo_resolver, tarball_base_url)
+            FakeApplication(), self.cachedir, repo_resolver, tarball_base_url)
+        self.lrc.fs = fs.memoryfs.MemoryFS()
         self.lrc._git = self.fake_git
-        self.lrc._exists = self.fake_exists
         self.lrc._fetch = self.not_found
-        self.lrc._mkdir = self.fake_mkdir
-        self.lrc._remove = self.fake_remove
-        self.lrc._runcmd = self.fake_runcmd
-
-    def fake_runcmd(self, args, cwd=None):
-        if args[0:2] == ['tar', 'xf']:
-            self.unpacked_tar = args[2]
-            self.cache.add(cwd)
-        else:
-            raise NotImplementedError()
+        self.lrc._mkdtemp = self.fake_mkdtemp
+        self._mkdtemp_count = 0
 
     def fake_git(self, args, cwd=None):
         if args[0] == 'clone':
             self.assertEqual(len(args), 5)
             remote = args[3]
             local = args[4]
-            if local in self.cache:
-                raise Exception('cloning twice to %s' % local)
             self.remotes['origin'] = {'url': remote, 'updates': 0}
-            self.cache.add(local)
+            self.lrc.fs.makedir(local, recursive=True)
         elif args[0:2] == ['remote', 'set-url']:
             remote = args[2]
             url = args[3]
@@ -79,22 +75,14 @@ class LocalRepoCacheTests(unittest.TestCase):
         else:
             raise NotImplementedError()
 
-    def fake_exists(self, filename):
-        return filename in self.cache
-
-    def fake_mkdir(self, dirname):
-        self.cache.add(dirname)
-
-    def fake_remove(self, filename):
-        self.removed.append(filename)
+    def fake_mkdtemp(self, dirname):
+        thing = "foo"+str(self._mkdtemp_count)
+        self._mkdtemp_count += 1
+        self.lrc.fs.makedir(dirname+"/"+thing)
+        return thing
 
     def not_found(self, url, path):
-        raise urllib2.URLError('Not found')
-
-    def fake_fetch(self, url, path):
-        self.fetched.append(url)
-        self.cache.add(path)
-        return True
+        raise cliapp.AppException('Not found')
 
     def test_has_not_got_shortened_repo_initially(self):
         self.assertFalse(self.lrc.has_repo(self.reponame))
@@ -113,11 +101,11 @@ class LocalRepoCacheTests(unittest.TestCase):
         self.assertTrue(self.lrc.has_repo(self.repourl))
 
     def test_cachedir_does_not_exist_initially(self):
-        self.assertFalse(self.cachedir in self.cache)
+        self.assertFalse(self.lrc.fs.exists(self.cachedir))
 
     def test_creates_cachedir_if_missing(self):
         self.lrc.cache_repo(self.repourl)
-        self.assertTrue(self.cachedir in self.cache)
+        self.assertTrue(self.lrc.fs.exists(self.cachedir))
 
     def test_happily_caches_same_repo_twice(self):
         self.lrc.cache_repo(self.repourl)
@@ -125,6 +113,7 @@ class LocalRepoCacheTests(unittest.TestCase):
 
     def test_fails_to_cache_when_remote_does_not_exist(self):
         def fail(args):
+            self.lrc.fs.makedir(args[4])
             raise cliapp.AppException('')
         self.lrc._git = fail
         self.assertRaises(morphlib.localrepocache.NoRemote,
@@ -135,12 +124,12 @@ class LocalRepoCacheTests(unittest.TestCase):
         self.assertEqual(self.fetched, [])
 
     def test_fetches_tarball_when_it_exists(self):
-        self.lrc._fetch = self.fake_fetch
+        self.lrc._fetch = lambda url, path: self.fetched.append(url)
         self.unpacked_tar = ""
         self.mkdir_path = ""
         self.lrc.cache_repo(self.repourl)
         self.assertEqual(self.fetched, [self.tarball_url])
-        self.assertEqual(self.removed, [self.cache_path + '.tar'])
+        self.assertFalse(self.lrc.fs.exists(self.cache_path + '.tar'))
         self.assertEqual(self.remotes['origin']['url'], self.repourl)
 
     def test_gets_cached_shortened_repo(self):
@@ -165,6 +154,7 @@ class LocalRepoCacheTests(unittest.TestCase):
         self.assertTrue(self.repourl in str(e))
 
     def test_avoids_caching_local_repo(self):
+        self.lrc.fs.makedir('/local/repo', recursive=True)
         self.lrc.cache_repo('file:///local/repo')
         cached = self.lrc.get_repo('file:///local/repo')
         assert cached.path == '/local/repo'
-- 
cgit v1.2.1


From fbc2b8237a46d417f18be8938a1653945e189c54 Mon Sep 17 00:00:00 2001
From: Daniel Firth <dan.firth@codethink.co.uk>
Date: Fri, 20 Dec 2013 15:47:10 +0000
Subject: Replaces Tempdir with fs.tempfs.TempFS

---
 morphlib/__init__.py                           |  1 -
 morphlib/cachedrepo_tests.py                   | 65 +++++++++++++-------------
 morphlib/plugins/artifact_inspection_plugin.py | 31 ++++++------
 morphlib/tempdir.py                            | 48 -------------------
 morphlib/tempdir_tests.py                      | 57 ----------------------
 5 files changed, 50 insertions(+), 152 deletions(-)
 delete mode 100644 morphlib/tempdir.py
 delete mode 100644 morphlib/tempdir_tests.py

diff --git a/morphlib/__init__.py b/morphlib/__init__.py
index 8f39cb30..fcfcee54 100644
--- a/morphlib/__init__.py
+++ b/morphlib/__init__.py
@@ -82,7 +82,6 @@ import stagingarea
 import stopwatch
 import sysbranchdir
 import systemmetadatadir
-import tempdir
 import util
 import workspace
 
diff --git a/morphlib/cachedrepo_tests.py b/morphlib/cachedrepo_tests.py
index b1825eba..8afe2063 100644
--- a/morphlib/cachedrepo_tests.py
+++ b/morphlib/cachedrepo_tests.py
@@ -18,12 +18,11 @@ import logging
 import os
 import unittest
 
+import fs.tempfs
 import cliapp
 
 import morphlib
 
-from morphlib import cachedrepo
-
 
 class CachedRepoTests(unittest.TestCase):
 
@@ -114,7 +113,7 @@ class CachedRepoTests(unittest.TestCase):
         self.repo_name = 'foo'
         self.repo_url = 'git://foo.bar/foo.git'
         self.repo_path = '/tmp/foo'
-        self.repo = cachedrepo.CachedRepo(
+        self.repo = morphlib.cachedrepo.CachedRepo(
             object(), self.repo_name, self.repo_url, self.repo_path)
         self.repo._rev_parse = self.rev_parse
         self.repo._show_tree_hash = self.show_tree_hash
@@ -123,10 +122,7 @@ class CachedRepoTests(unittest.TestCase):
         self.repo._checkout_ref = self.checkout_ref
         self.repo._ls_tree = self.ls_tree
         self.repo._clone_into = self.clone_into
-        self.tempdir = morphlib.tempdir.Tempdir()
-
-    def tearDown(self):
-        self.tempdir.remove()
+        self.tempfs = fs.tempfs.TempFS()
 
     def test_constructor_sets_name_and_url_and_path(self):
         self.assertEqual(self.repo.original_name, self.repo_name)
@@ -150,7 +146,7 @@ class CachedRepoTests(unittest.TestCase):
         self.assertEqual(tree, 'ffffffffffffffffffffffffffffffffffffffff')
 
     def test_fail_resolving_invalid_named_ref(self):
-        self.assertRaises(cachedrepo.InvalidReferenceError,
+        self.assertRaises(morphlib.cachedrepo.InvalidReferenceError,
                           self.repo.resolve_ref, 'foo/bar')
 
     def test_resolve_sha1_ref(self):
@@ -160,7 +156,7 @@ class CachedRepoTests(unittest.TestCase):
         self.assertEqual(tree, 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee')
 
     def test_fail_resolving_an_invalid_sha1_ref(self):
-        self.assertRaises(cachedrepo.InvalidReferenceError,
+        self.assertRaises(morphlib.cachedrepo.InvalidReferenceError,
                           self.repo.resolve_ref,
                           self.bad_sha1_known_to_rev_parse)
 
@@ -170,9 +166,10 @@ class CachedRepoTests(unittest.TestCase):
         self.assertEqual(data, self.EXAMPLE_MORPH)
 
     def test_fail_cat_file_in_invalid_ref(self):
-        self.assertRaises(cachedrepo.InvalidReferenceError, self.repo.cat,
-                          '079bbfd447c8534e464ce5d40b80114c2022ebf4',
-                          'doesnt-matter-whether-this-file-exists')
+        self.assertRaises(
+            morphlib.cachedrepo.InvalidReferenceError, self.repo.cat,
+            '079bbfd447c8534e464ce5d40b80114c2022ebf4',
+            'doesnt-matter-whether-this-file-exists')
 
     def test_fail_cat_non_existent_file_in_existing_ref(self):
         self.assertRaises(IOError, self.repo.cat,
@@ -180,37 +177,40 @@ class CachedRepoTests(unittest.TestCase):
                           'file-that-does-not-exist')
 
     def test_fail_cat_non_existent_file_in_invalid_ref(self):
-        self.assertRaises(cachedrepo.InvalidReferenceError, self.repo.cat,
-                          '079bbfd447c8534e464ce5d40b80114c2022ebf4',
-                          'file-that-does-not-exist')
+        self.assertRaises(
+            morphlib.cachedrepo.InvalidReferenceError, self.repo.cat,
+            '079bbfd447c8534e464ce5d40b80114c2022ebf4',
+            'file-that-does-not-exist')
 
     def test_fail_because_cat_in_named_ref_is_not_allowed(self):
-        self.assertRaises(cachedrepo.UnresolvedNamedReferenceError,
+        self.assertRaises(morphlib.cachedrepo.UnresolvedNamedReferenceError,
                           self.repo.cat, 'master', 'doesnt-matter')
 
     def test_fail_clone_checkout_into_existing_directory(self):
-        self.assertRaises(cachedrepo.CheckoutDirectoryExistsError,
+        self.assertRaises(morphlib.cachedrepo.CheckoutDirectoryExistsError,
                           self.repo.clone_checkout,
                           'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9',
-                          self.tempdir.dirname)
+                          self.tempfs.root_path)
 
     def test_fail_checkout_due_to_clone_error(self):
-        self.assertRaises(cachedrepo.CloneError, self.repo.clone_checkout,
-                          'a4da32f5a81c8bc6d660404724cedc3bc0914a75',
-                          self.tempdir.join('failed-checkout'))
+        self.assertRaises(
+            morphlib.cachedrepo.CloneError, self.repo.clone_checkout,
+            'a4da32f5a81c8bc6d660404724cedc3bc0914a75',
+            self.tempfs.getsyspath('failed-checkout'))
 
     def test_fail_checkout_due_to_copy_error(self):
-        self.assertRaises(cachedrepo.CopyError, self.repo.checkout,
+        self.assertRaises(morphlib.cachedrepo.CopyError, self.repo.checkout,
                           'a4da32f5a81c8bc6d660404724cedc3bc0914a75',
-                          self.tempdir.join('failed-checkout'))
+                          self.tempfs.getsyspath('failed-checkout'))
 
     def test_fail_checkout_from_invalid_ref(self):
-        self.assertRaises(cachedrepo.CheckoutError, self.repo.checkout,
-                          '079bbfd447c8534e464ce5d40b80114c2022ebf4',
-                          self.tempdir.join('checkout-from-invalid-ref'))
+        self.assertRaises(
+            morphlib.cachedrepo.CheckoutError, self.repo.checkout,
+            '079bbfd447c8534e464ce5d40b80114c2022ebf4',
+            self.tempfs.getsyspath('checkout-from-invalid-ref'))
 
     def test_checkout_from_existing_ref_into_new_directory(self):
-        unpack_dir = self.tempdir.join('unpack-dir')
+        unpack_dir = self.tempfs.getsyspath('unpack-dir')
         self.repo.checkout('e28a23812eadf2fce6583b8819b9c5dbd36b9fb9',
                            unpack_dir)
         self.assertTrue(os.path.exists(unpack_dir))
@@ -227,11 +227,12 @@ class CachedRepoTests(unittest.TestCase):
         self.assertEqual(data, ['foo.morph'])
 
     def test_fail_ls_tree_in_invalid_ref(self):
-        self.assertRaises(cachedrepo.InvalidReferenceError, self.repo.ls_tree,
-                          '079bbfd447c8534e464ce5d40b80114c2022ebf4')
+        self.assertRaises(
+            morphlib.cachedrepo.InvalidReferenceError, self.repo.ls_tree,
+            '079bbfd447c8534e464ce5d40b80114c2022ebf4')
 
     def test_fail_because_ls_tree_in_named_ref_is_not_allowed(self):
-        self.assertRaises(cachedrepo.UnresolvedNamedReferenceError,
+        self.assertRaises(morphlib.cachedrepo.UnresolvedNamedReferenceError,
                           self.repo.ls_tree, 'master')
 
     def test_successful_update(self):
@@ -240,10 +241,10 @@ class CachedRepoTests(unittest.TestCase):
 
     def test_failing_update(self):
         self.repo._update = self.update_with_failure
-        self.assertRaises(cachedrepo.UpdateError, self.repo.update)
+        self.assertRaises(morphlib.cachedrepo.UpdateError, self.repo.update)
 
     def test_no_update_if_local(self):
-        self.repo = cachedrepo.CachedRepo(
+        self.repo = morphlib.cachedrepo.CachedRepo(
             object(), 'local:repo', 'file:///local/repo/', '/local/repo/')
         self.repo._update = self.update_with_failure
         self.repo.update()
diff --git a/morphlib/plugins/artifact_inspection_plugin.py b/morphlib/plugins/artifact_inspection_plugin.py
index 9181d488..6eeece77 100644
--- a/morphlib/plugins/artifact_inspection_plugin.py
+++ b/morphlib/plugins/artifact_inspection_plugin.py
@@ -19,6 +19,9 @@ import glob
 import json
 import os
 import re
+import contextlib
+
+import fs.tempfs
 
 import morphlib
 
@@ -109,25 +112,25 @@ class AutotoolsVersionGuesser(ProjectVersionGuesser):
         return None
 
     def _check_autoconf_package_version(self, repo, ref, filename, data):
-        tempdir = morphlib.tempdir.Tempdir(self.app.settings['tempdir'])
-        with open(tempdir.join(filename), 'w') as f:
-            f.write(data)
-        exit_code, output, errors = self.app.runcmd_unchecked(
+        with contextlib.closing(fs.tempfs.TempFS(
+                temp_dir=self.app.settings['tempdir'])) as tempdir:
+            with open(tempdir.getsyspath(filename), 'w') as f:
+                f.write(data)
+            exit_code, output, errors = self.app.runcmd_unchecked(
                 ['autoconf', filename],
                 ['grep', '^PACKAGE_VERSION='],
                 ['cut', '-d=', '-f2'],
                 ['sed', "s/'//g"],
-                cwd=tempdir.dirname)
-        tempdir.remove()
-        version = None
-        if output:
-            output = output.strip()
-            if output and output[0].isdigit():
-                version = output
-        if exit_code != 0:
-            self.app.status(
+                cwd=tempdir.root_path)
+            version = None
+            if output:
+                output = output.strip()
+                if output and output[0].isdigit():
+                    version = output
+            if exit_code != 0:
+                self.app.status(
                     msg='%(repo)s: Failed to detect version from '
-                        '%(ref)s:%(filename)s',
+                    '%(ref)s:%(filename)s',
                     repo=repo, ref=ref, filename=filename, chatty=True)
         return version
 
diff --git a/morphlib/tempdir.py b/morphlib/tempdir.py
deleted file mode 100644
index bfbabded..00000000
--- a/morphlib/tempdir.py
+++ /dev/null
@@ -1,48 +0,0 @@
-# Copyright (C) 2011-2012  Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import logging
-import os
-import shutil
-import tempfile
-
-
-class Tempdir(object):
-
-    '''Temporary file handling for morph.'''
-
-    def __init__(self, parent=None):
-        self.dirname = tempfile.mkdtemp(dir=parent)
-        logging.debug('Created temporary directory %s' % self.dirname)
-
-    def remove(self):
-        '''Remove the temporary directory.'''
-        logging.debug('Removing temporary directory %s' % self.dirname)
-        shutil.rmtree(self.dirname)
-        self.dirname = None
-
-    def join(self, relative):
-        '''Return full path to file in temporary directory.
-
-        The relative path is given appended to the name of the
-        temporary directory. If the relative path is actually absolute,
-        it is forced to become relative.
-
-        The returned path is normalized.
-
-        '''
-
-        return os.path.normpath(os.path.join(self.dirname, './' + relative))
diff --git a/morphlib/tempdir_tests.py b/morphlib/tempdir_tests.py
deleted file mode 100644
index 64ed214a..00000000
--- a/morphlib/tempdir_tests.py
+++ /dev/null
@@ -1,57 +0,0 @@
-# Copyright (C) 2011-2012  Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import os
-import shutil
-import unittest
-
-import morphlib
-
-
-class TempdirTests(unittest.TestCase):
-
-    def setUp(self):
-        self.parent = os.path.abspath('unittest-tempdir')
-        os.mkdir(self.parent)
-        self.tempdir = morphlib.tempdir.Tempdir(parent=self.parent)
-
-    def tearDown(self):
-        shutil.rmtree(self.parent)
-
-    def test_creates_the_directory(self):
-        self.assert_(os.path.isdir(self.tempdir.dirname))
-
-    def test_creates_subdirectory_of_parent(self):
-        self.assert_(self.tempdir.dirname.startswith(self.parent + '/'))
-
-    def test_uses_default_if_parent_not_specified(self):
-        t = morphlib.tempdir.Tempdir()
-        shutil.rmtree(t.dirname)
-        self.assertNotEqual(t.dirname, None)
-
-    def test_removes_itself(self):
-        dirname = self.tempdir.dirname
-        self.tempdir.remove()
-        self.assertEqual(self.tempdir.dirname, None)
-        self.assertFalse(os.path.exists(dirname))
-
-    def test_joins_filename(self):
-        self.assertEqual(self.tempdir.join('foo'),
-                         os.path.join(self.tempdir.dirname, 'foo'))
-
-    def test_joins_absolute_filename(self):
-        self.assertEqual(self.tempdir.join('/foo'),
-                         os.path.join(self.tempdir.dirname, 'foo'))
-- 
cgit v1.2.1


From 973f61504ec71e0ec925b592f12825aa0c9ab9d9 Mon Sep 17 00:00:00 2001
From: Daniel Firth <dan.firth@codethink.co.uk>
Date: Tue, 10 Dec 2013 17:14:04 +0000
Subject: Removed unused CacheDir class

---
 morphlib/__init__.py       |   1 -
 morphlib/cachedir.py       |  86 ------------------------------
 morphlib/cachedir_tests.py | 128 ---------------------------------------------
 3 files changed, 215 deletions(-)
 delete mode 100644 morphlib/cachedir.py
 delete mode 100644 morphlib/cachedir_tests.py

diff --git a/morphlib/__init__.py b/morphlib/__init__.py
index fcfcee54..67fb944d 100644
--- a/morphlib/__init__.py
+++ b/morphlib/__init__.py
@@ -55,7 +55,6 @@ import buildcommand
 import buildenvironment
 import buildsystem
 import builder2
-import cachedir
 import cachedrepo
 import cachekeycomputer
 import extractedtarball
diff --git a/morphlib/cachedir.py b/morphlib/cachedir.py
deleted file mode 100644
index de1b87ff..00000000
--- a/morphlib/cachedir.py
+++ /dev/null
@@ -1,86 +0,0 @@
-# Copyright (C) 2011-2012  Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import hashlib
-import os
-
-import morphlib
-
-
-class CacheDir(object):
-
-    '''Manage Baserock cached binaries.'''
-
-    def __init__(self, dirname):
-        self.dirname = os.path.abspath(dirname)
-
-    def key(self, dict_key):
-        '''Create a string key from a dictionary key.
-
-        The string key can be used as a filename, or as part of one.
-        The dictionary key is a dict that maps any set of strings to
-        another set of strings.
-
-        The same string key is guaranteed to be returned for a given
-        dictionary key. It is highly unlikely that two different dictionary
-        keys result in the same string key.
-
-        '''
-
-        data = ''.join(key + value for key, value in dict_key.iteritems())
-        return hashlib.sha256(data).hexdigest()
-
-    def name(self, dict_key):
-        '''Return a filename for an object described by dictionary key.
-
-        It is the caller's responsibility to set the fields in the
-        dictionary key suitably. For example, if there is a field
-        specifying a commit id, it should be the full git SHA-1
-        identifier, not something ephemeral like HEAD.
-
-        If the field 'kind' has a value, it is used as a suffix for
-        the filename.
-
-        '''
-
-        key = self.key(dict_key)
-        if 'kind' in dict_key and dict_key['kind']:
-            suffix = '.%s' % dict_key['kind']
-        else:
-            suffix = ''
-
-        return os.path.join(self.dirname, key + suffix)
-
-    def open(self, relative_name_or_cache_key, suffix='', **kwargs):
-        '''Open a file for writing in the cache.
-
-        The file will be written with a temporary name, and renamed to
-        the final name when the file is closed. Additionally, if the
-        caller decides, mid-writing, that they don't want to write the
-        file after all (e.g., a log file), then the ``abort`` method
-        in the returned file handle can be called to remove the
-        temporary file.
-
-        '''
-
-        if type(relative_name_or_cache_key) is dict:
-            path = self.name(relative_name_or_cache_key)
-        else:
-            path = os.path.join(self.dirname, relative_name_or_cache_key)
-        path += suffix
-        if 'mode' not in kwargs:
-            kwargs['mode'] = 'w'
-        return morphlib.savefile.SaveFile(path, **kwargs)
diff --git a/morphlib/cachedir_tests.py b/morphlib/cachedir_tests.py
deleted file mode 100644
index f02f80e1..00000000
--- a/morphlib/cachedir_tests.py
+++ /dev/null
@@ -1,128 +0,0 @@
-# Copyright (C) 2011-2012  Codethink Limited
-#
-# This program is free software; you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
-# the Free Software Foundation; version 2 of the License.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License along
-# with this program; if not, write to the Free Software Foundation, Inc.,
-# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
-
-
-import os
-import shutil
-import tempfile
-import unittest
-
-import morphlib
-
-
-class CacheDirTests(unittest.TestCase):
-
-    def cat(self, relative_name):
-        with open(os.path.join(self.cachedir.dirname, relative_name)) as f:
-            return f.read()
-
-    def setUp(self):
-        self.dirname = tempfile.mkdtemp()
-        self.cachedir = morphlib.cachedir.CacheDir(self.dirname)
-
-    def tearDown(self):
-        shutil.rmtree(self.dirname)
-
-    def test_sets_dirname_attribute(self):
-        self.assertEqual(self.cachedir.dirname, os.path.abspath(self.dirname))
-
-    def test_generates_string_key_for_arbitrary_dict_key(self):
-        key = self.cachedir.key({
-            'foo': 'bar',
-            'xyzzy': 'plugh',
-        })
-        self.assertEqual(type(key), str)
-        self.assertNotEqual(key, '')
-
-    def test_generates_same_string_key_twice(self):
-        dict_key = {
-            'foo': 'bar',
-            'xyzzy': 'plugh',
-        }
-        self.assertEqual(self.cachedir.key(dict_key),
-                         self.cachedir.key(dict_key))
-
-    def test_generates_different_string_keys(self):
-        dict_key_1 = {
-            'foo': 'bar',
-            'xyzzy': 'plugh',
-        }
-        dict_key_2 = {
-            'foo': 'foobar',
-            'xyzzy': 'stevenage',
-        }
-        self.assertNotEqual(self.cachedir.key(dict_key_1),
-                            self.cachedir.key(dict_key_2))
-
-    def test_returns_a_chunk_pathname_in_cache_directory(self):
-        dict_key = {
-            'kind': 'chunk',
-            'ref': 'DEADBEEF',
-            'repo': 'git://git.baserock.org/hello/',
-            'arch': 'armv7',
-        }
-        pathname = self.cachedir.name(dict_key)
-        self.assert_(pathname.startswith(self.cachedir.dirname + '/'))
-        self.assert_(pathname.endswith('.chunk'))
-
-    def test_returns_a_stratum_pathname_in_cache_directory(self):
-        dict_key = {
-            'kind': 'stratum',
-            'ref': 'DEADBEEF',
-            'repo': 'git://git.baserock.org/hello/',
-            'arch': 'armv7',
-        }
-        pathname = self.cachedir.name(dict_key)
-        self.assert_(pathname.startswith(self.cachedir.dirname + '/'))
-        self.assert_(pathname.endswith('.stratum'))
-
-    def test_returns_a_valid_pathname_in_cache_directory(self):
-        dict_key = {
-            'ref': 'DEADBEEF',
-            'repo': 'git://git.baserock.org/hello/',
-            'arch': 'armv7',
-        }
-        pathname = self.cachedir.name(dict_key)
-        self.assert_(pathname.startswith(self.cachedir.dirname + '/'))
-
-    def test_allows_file_to_be_written_via_basename(self):
-        f = self.cachedir.open('foo')
-        f.write('bar')
-        f.close()
-        self.assertEqual(self.cat('foo'), 'bar')
-
-    def test_allows_file_to_be_written_via_basename_and_suffix(self):
-        f = self.cachedir.open('foo', '.blip')
-        f.write('bar')
-        f.close()
-        self.assertEqual(self.cat('foo.blip'), 'bar')
-
-    def test_allows_file_to_be_written_via_dict_key(self):
-        dict_key = {
-            'kind': 'chunk',
-            'meh': 'moo',
-        }
-        name = self.cachedir.name(dict_key)
-        f = self.cachedir.open(dict_key)
-        f.write('bar')
-        f.close()
-        self.assertEqual(self.cat(name), 'bar')
-
-    def test_allows_file_to_be_aborted(self):
-        f = self.cachedir.open('foo')
-        f.write('bar')
-        f.abort()
-        pathname = os.path.join(self.cachedir.dirname, 'foo')
-        self.assertFalse(os.path.exists(pathname))
-- 
cgit v1.2.1


From ce99ab4010dee8f4a60844c004b2bddbcdfe5dec Mon Sep 17 00:00:00 2001
From: Daniel Firth <dan.firth@codethink.co.uk>
Date: Fri, 20 Dec 2013 15:47:25 +0000
Subject: LocalArtifactCache now takes a an FS object

---
 morphlib/localartifactcache.py       | 48 ++++++++++++++++--------------------
 morphlib/localartifactcache_tests.py | 34 +++++++++----------------
 morphlib/plugins/gc_plugin.py        |  3 ++-
 morphlib/util.py                     |  8 +++---
 4 files changed, 40 insertions(+), 53 deletions(-)

diff --git a/morphlib/localartifactcache.py b/morphlib/localartifactcache.py
index 76d085d1..341bbb56 100644
--- a/morphlib/localartifactcache.py
+++ b/morphlib/localartifactcache.py
@@ -16,6 +16,7 @@
 
 import collections
 import os
+import time
 
 import morphlib
 
@@ -44,8 +45,8 @@ class LocalArtifactCache(object):
        sense to put the complication there.
        '''
 
-    def __init__(self, cachedir):
-        self.cachedir = cachedir
+    def __init__(self, cachefs):
+        self.cachefs = cachefs
 
     def put(self, artifact):
         filename = self.artifact_filename(artifact)
@@ -93,28 +94,23 @@ class LocalArtifactCache(object):
         return open(filename)
 
     def artifact_filename(self, artifact):
-        basename = artifact.basename()
-        return os.path.join(self.cachedir, basename)
+        return self.cachefs.getsyspath(artifact.basename())
 
     def _artifact_metadata_filename(self, artifact, name):
-        basename = artifact.metadata_basename(name)
-        return os.path.join(self.cachedir, basename)
+        return self.cachefs.getsyspath(artifact.metadata_basename(name))
 
     def _source_metadata_filename(self, source, cachekey, name):
-        basename = '%s.%s' % (cachekey, name)
-        return os.path.join(self.cachedir, basename)
+        return self.cachefs.getsyspath('%s.%s' % (cachekey, name))
 
     def clear(self):
         '''Clear everything from the artifact cache directory.
         
         After calling this, the artifact cache will be entirely empty.
         Caveat caller.
-        
-        '''
 
-        for dirname, subdirs, basenames in os.walk(self.cachedir):
-            for basename in basenames:
-                os.remove(os.path.join(dirname, basename))
+         '''
+        for filename in self.cachefs.walkfiles():
+            self.cachefs.remove(filename)
 
     def list_contents(self):
         '''Return the set of sources cached and related information.
@@ -124,22 +120,20 @@ class LocalArtifactCache(object):
         '''
         CacheInfo = collections.namedtuple('CacheInfo', ('artifacts', 'mtime'))
         contents = collections.defaultdict(lambda: CacheInfo(set(), 0))
-        for dirpath, dirnames, filenames in os.walk(self.cachedir):
-            for filename in filenames:
-                cachekey = filename[:63]
-                artifact = filename[65:]
-                artifacts, max_mtime = contents[cachekey]
-                artifacts.add(artifact)
-                this_mtime = os.stat(os.path.join(dirpath, filename)).st_mtime
-                contents[cachekey] = CacheInfo(artifacts,
-                                               max(max_mtime, this_mtime))
+        for filename in self.cachefs.walkfiles():
+            cachekey = filename[:63]
+            artifact = filename[65:]
+            artifacts, max_mtime = contents[cachekey]
+            artifacts.add(artifact)
+            art_info = self.cachefs.getinfo(filename)
+            time_t = art_info['modified_time'].timetuple()
+            contents[cachekey] = CacheInfo(artifacts,
+                                           max(max_mtime, time.mktime(time_t)))
         return ((cache_key, info.artifacts, info.mtime)
                 for cache_key, info in contents.iteritems())
 
-
     def remove(self, cachekey):
         '''Remove all artifacts associated with the given cachekey.'''
-        for dirpath, dirnames, filenames in os.walk(self.cachedir):
-            for filename in filenames:
-                if filename.startswith(cachekey):
-                    os.remove(os.path.join(dirpath, filename))
+        for filename in (x for x in self.cachefs.walkfiles()
+                         if x.startswith(cachekey)):
+            self.cachefs.remove(filename)
diff --git a/morphlib/localartifactcache_tests.py b/morphlib/localartifactcache_tests.py
index 082b926a..d7743359 100644
--- a/morphlib/localartifactcache_tests.py
+++ b/morphlib/localartifactcache_tests.py
@@ -17,13 +17,15 @@
 import unittest
 import os
 
+import fs.tempfs
+
 import morphlib
 
 
 class LocalArtifactCacheTests(unittest.TestCase):
 
     def setUp(self):
-        self.tempdir = morphlib.tempdir.Tempdir()
+        self.tempfs = fs.tempfs.TempFS()
 
         morph = morphlib.morph2.Morphology(
             '''
@@ -52,20 +54,14 @@ class LocalArtifactCacheTests(unittest.TestCase):
             self.source, 'chunk-devel')
         self.devel_artifact.cache_key = '0'*64
 
-    def tearDown(self):
-        self.tempdir.remove()
-
     def test_artifact_filename(self):
-        cache = morphlib.localartifactcache.LocalArtifactCache(
-            self.tempdir.dirname)
+        cache = morphlib.localartifactcache.LocalArtifactCache(self.tempfs)
         filename = cache.artifact_filename(self.devel_artifact)
-        expected_name = os.path.join(self.tempdir.dirname,
-                                     self.devel_artifact.basename())
+        expected_name = self.tempfs.getsyspath(self.devel_artifact.basename())
         self.assertEqual(filename, expected_name)
 
     def test_put_artifacts_and_check_whether_the_cache_has_them(self):
-        cache = morphlib.localartifactcache.LocalArtifactCache(
-            self.tempdir.dirname)
+        cache = morphlib.localartifactcache.LocalArtifactCache(self.tempfs)
 
         handle = cache.put(self.runtime_artifact)
         handle.write('runtime')
@@ -81,8 +77,7 @@ class LocalArtifactCacheTests(unittest.TestCase):
         self.assertTrue(cache.has(self.devel_artifact))
 
     def test_put_artifacts_and_get_them_afterwards(self):
-        cache = morphlib.localartifactcache.LocalArtifactCache(
-            self.tempdir.dirname)
+        cache = morphlib.localartifactcache.LocalArtifactCache(self.tempfs)
 
         handle = cache.put(self.runtime_artifact)
         handle.write('runtime')
@@ -111,8 +106,7 @@ class LocalArtifactCacheTests(unittest.TestCase):
         self.assertEqual(stored_data, 'devel')
 
     def test_put_check_and_get_artifact_metadata(self):
-        cache = morphlib.localartifactcache.LocalArtifactCache(
-            self.tempdir.dirname)
+        cache = morphlib.localartifactcache.LocalArtifactCache(self.tempfs)
 
         handle = cache.put_artifact_metadata(self.runtime_artifact, 'log')
         handle.write('log line 1\nlog line 2\n')
@@ -128,8 +122,7 @@ class LocalArtifactCacheTests(unittest.TestCase):
         self.assertEqual(stored_metadata, 'log line 1\nlog line 2\n')
 
     def test_put_check_and_get_source_metadata(self):
-        cache = morphlib.localartifactcache.LocalArtifactCache(
-            self.tempdir.dirname)
+        cache = morphlib.localartifactcache.LocalArtifactCache(self.tempfs)
 
         handle = cache.put_source_metadata(self.source, 'mycachekey', 'log')
         handle.write('source log line 1\nsource log line 2\n')
@@ -146,8 +139,7 @@ class LocalArtifactCacheTests(unittest.TestCase):
                          'source log line 1\nsource log line 2\n')
 
     def test_clears_artifact_cache(self):
-        cache = morphlib.localartifactcache.LocalArtifactCache(
-            self.tempdir.dirname)
+        cache = morphlib.localartifactcache.LocalArtifactCache(self.tempfs)
 
         handle = cache.put(self.runtime_artifact)
         handle.write('runtime')
@@ -158,8 +150,7 @@ class LocalArtifactCacheTests(unittest.TestCase):
         self.assertFalse(cache.has(self.runtime_artifact))
 
     def test_put_artifacts_and_list_them_afterwards(self):
-        cache = morphlib.localartifactcache.LocalArtifactCache(
-            self.tempdir.dirname)
+        cache = morphlib.localartifactcache.LocalArtifactCache(self.tempfs)
 
         handle = cache.put(self.runtime_artifact)
         handle.write('runtime')
@@ -174,8 +165,7 @@ class LocalArtifactCacheTests(unittest.TestCase):
         self.assertTrue(len(list(cache.list_contents())) == 1)
 
     def test_put_artifacts_and_remove_them_afterwards(self):
-        cache = morphlib.localartifactcache.LocalArtifactCache(
-            self.tempdir.dirname)
+        cache = morphlib.localartifactcache.LocalArtifactCache(self.tempfs)
 
         handle = cache.put(self.runtime_artifact)
         handle.write('runtime')
diff --git a/morphlib/plugins/gc_plugin.py b/morphlib/plugins/gc_plugin.py
index 1adabe78..abfa1a30 100644
--- a/morphlib/plugins/gc_plugin.py
+++ b/morphlib/plugins/gc_plugin.py
@@ -19,6 +19,7 @@ import os
 import shutil
 import time
 
+import fs.osfs
 import cliapp
 
 import morphlib
@@ -114,7 +115,7 @@ class GCPlugin(cliapp.Plugin):
                             chatty=True)
             return
         lac = morphlib.localartifactcache.LocalArtifactCache(
-            os.path.join(cache_path, 'artifacts'))
+            fs.osfs.OSFS(os.path.join(cache_path, 'artifacts')))
         max_age, min_age = self.calculate_delete_range()
         logging.debug('Must remove artifacts older than timestamp %d'
                       % max_age)
diff --git a/morphlib/util.py b/morphlib/util.py
index 90ad425e..68aed608 100644
--- a/morphlib/util.py
+++ b/morphlib/util.py
@@ -17,6 +17,8 @@ import itertools
 import os
 import re
 
+import fs.osfs
+
 import morphlib
 
 '''Utility functions for morph.'''
@@ -118,13 +120,13 @@ def new_artifact_caches(settings):  # pragma: no cover
     if not os.path.exists(artifact_cachedir):
         os.mkdir(artifact_cachedir)
 
-    lac = morphlib.localartifactcache.LocalArtifactCache(artifact_cachedir)
+    lac = morphlib.localartifactcache.LocalArtifactCache(
+            fs.osfs.OSFS(artifact_cachedir))
 
     rac_url = get_artifact_cache_server(settings)
+    rac = None
     if rac_url:
         rac = morphlib.remoteartifactcache.RemoteArtifactCache(rac_url)
-    else:
-        rac = None
     return lac, rac
 
 
-- 
cgit v1.2.1