Rewrite tests for remoteartifactcache module

The old test suite was closely tied with the implementation of the
module, which makes it very hard to rewrite any of the module. The new
test suite actually sets up a remote HTTP server and exercises the real
code paths.

I added a new 'testutils' module, because there seem to be quite a few
mock classes duplicated in different test modules. Better to collect
them when they are really needed.

5 files changed, 198 insertions, 231 deletions

diff --git a/morphlib/builder_tests.py b/morphlib/builder_tests.py
index 0cc90819..0067022f 100644
--- a/morphlib/builder_tests.py
+++ b/morphlib/builder_tests.py
@@ -15,12 +15,12 @@
 
 
 import json
-import os
 import StringIO
 import unittest
 
 import morphlib
 import morphlib.gitdir_tests
+import morphlib.testutils
 
 
 class FakeBuildSystem(object):
@@ -78,64 +78,6 @@ class FakeBuildEnv(object):
         }
 
 
-class FakeFileHandle(object):
-
-    def __init__(self, cache, key):
-        self._string = ""
-        self._cache = cache
-        self._key = key
-
-    def __enter__(self):
-        return self
-
-    def _writeback(self):
-        self._cache._cached[self._key] = self._string
-
-    def __exit__(self, type, value, traceback):
-        self._writeback()
-
-    def close(self):
-        self._writeback()
-
-    def write(self, string):
-        self._string += string
-
-
-class FakeArtifactCache(object):
-
-    def __init__(self):
-        self._cached = {}
-
-    def put(self, artifact):
-        return FakeFileHandle(self, (artifact.cache_key, artifact.name))
-
-    def put_artifact_metadata(self, artifact, name):
-        return FakeFileHandle(self, (artifact.cache_key, artifact.name, name))
-
-    def put_source_metadata(self, source, cachekey, name):
-        return FakeFileHandle(self, (cachekey, name))
-
-    def get(self, artifact):
-        return StringIO.StringIO(
-            self._cached[(artifact.cache_key, artifact.name)])
-
-    def get_artifact_metadata(self, artifact, name):
-        return StringIO.StringIO(
-            self._cached[(artifact.cache_key, artifact.name, name)])
-
-    def get_source_metadata(self, source, cachekey, name):
-        return StringIO.StringIO(self._cached[(cachekey, name)])
-
-    def has(self, artifact):
-        return (artifact.cache_key, artifact.name) in self._cached
-
-    def has_artifact_metadata(self, artifact, name):
-        return (artifact.cache_key, artifact.name, name) in self._cached
-
-    def has_source_metadata(self, source, cachekey, name):
-        return (cachekey, name) in self._cached
-
-
 class BuilderBaseTests(unittest.TestCase):
 
     def fake_runcmd(self, argv, *args, **kwargs):
@@ -151,7 +93,7 @@ class BuilderBaseTests(unittest.TestCase):
         self.commands_run = []
         self.app = FakeApp(self.fake_runcmd)
         self.staging_area = FakeStagingArea(self.fake_runcmd, FakeBuildEnv())
-        self.artifact_cache = FakeArtifactCache()
+        self.artifact_cache = morphlib.testutils.FakeLocalArtifactCache()
         self.artifact = FakeArtifact('le-artifact')
         self.repo_cache = None
         self.build_env = FakeBuildEnv()
@@ -187,33 +129,6 @@ class BuilderBaseTests(unittest.TestCase):
         self.assertEqual(sorted(events),
                          sorted(meta['build-times'].keys()))
 
-    def test_downloads_depends(self):
-        lac = FakeArtifactCache()
-        rac = FakeArtifactCache()
-        afacts = [FakeArtifact(name) for name in ('a', 'b', 'c')]
-        for a in afacts:
-            fh = rac.put(a)
-            fh.write(a.name)
-            fh.close()
-        morphlib.builder.download_depends(afacts, lac, rac)
-        self.assertTrue(all(lac.has(a) for a in afacts))
-
-    def test_downloads_depends_metadata(self):
-        lac = FakeArtifactCache()
-        rac = FakeArtifactCache()
-        afacts = [FakeArtifact(name) for name in ('a', 'b', 'c')]
-        for a in afacts:
-            fh = rac.put(a)
-            fh.write(a.name)
-            fh.close()
-            fh = rac.put_artifact_metadata(a, 'meta')
-            fh.write('metadata')
-            fh.close()
-        morphlib.builder.download_depends(afacts, lac, rac, ('meta',))
-        self.assertTrue(all(lac.has(a) for a in afacts))
-        self.assertTrue(all(lac.has_artifact_metadata(a, 'meta')
-                            for a in afacts))
-
 
 class ChunkBuilderTests(unittest.TestCase):
 
diff --git a/morphlib/remoteartifactcache.py b/morphlib/remoteartifactcache.py
index 45933d10..c5bf9a48 100644
--- a/morphlib/remoteartifactcache.py
+++ b/morphlib/remoteartifactcache.py
@@ -22,7 +22,7 @@ import urllib2
 import urlparse
 
 
-class HeadRequest(urllib2.Request):  # pragma: no cover
+class HeadRequest(urllib2.Request):
 
     def get_method(self):
         return 'HEAD'
@@ -51,7 +51,7 @@ class RemoteArtifactCache(object):
         filename = '%s.%s' % (cachekey, name)
         return self._has_file(filename)
 
-    def _has_file(self, filename):  # pragma: no cover
+    def _has_file(self, filename):
         url = self._request_url(filename)
         logging.debug('RemoteArtifactCache._has_file: url=%s' % url)
         request = HeadRequest(url)
@@ -61,7 +61,7 @@ class RemoteArtifactCache(object):
         except (urllib2.HTTPError, urllib2.URLError):
             return False
 
-    def _request_url(self, filename):  # pragma: no cover
+    def _request_url(self, filename):
         server_url = self.server_url
         if not server_url.endswith('/'):
             server_url += '/'
@@ -79,7 +79,7 @@ class RemoteArtifactCache(object):
         try:
             remote_file = urllib2.urlopen(remote_url)
             shutil.copyfileobj(remote_file, local_file)
-        except (urllib.HTTPError, urllib.URLError) as e:
+        except (urllib2.HTTPError, urllib2.URLError) as e:
             logging.debug(str(e))
             raise GetError(self, remote_filename, e)
 
@@ -113,7 +113,9 @@ class RemoteArtifactCache(object):
             to_fetch.append(
                 (artifact.basename(), lac.put(artifact)))
 
-        if artifact.source.morphology.needs_artifact_metadata_cached:
+        needs_artifact_meta = \
+            artifact.source.morphology.needs_artifact_metadata_cached
+        if needs_artifact_meta:  # pragma: no cover
             if not lac.has_artifact_metadata(artifact, 'meta'):
                 to_fetch.append((
                     artifact.metadata_basename(artifact, 'meta'),
diff --git a/morphlib/remoteartifactcache_tests.py b/morphlib/remoteartifactcache_tests.py
index 788882c2..f6d19ae7 100644
--- a/morphlib/remoteartifactcache_tests.py
+++ b/morphlib/remoteartifactcache_tests.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2012-2014  Codethink Limited
+# Copyright (C) 2012-2015  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -14,151 +14,142 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 
-import StringIO
+import BaseHTTPServer
+import contextlib
+import os
+import socket
+import SocketServer
+import threading
 import unittest
-import urllib2
+import urlparse
 
 import morphlib
+import morphlib.testutils
+
+
+def artifact_files_for_chunk_source(source):
+    '''Determine what filenames would exist for all artifacts of one source.'''
+
+    existing_files = set(['%s.meta' % source.cache_key])
+    for a in source.artifacts.itervalues():
+        existing_files.add(a.basename())
+        existing_files.add(a.metadata_basename('meta'))
+    return existing_files
+
+
+
+@contextlib.contextmanager
+def http_server(request_handler):
+    server = SocketServer.TCPServer(('', 0), request_handler)
+    thread = threading.Thread(target=server.serve_forever)
+    thread.setDaemon(True)
+    thread.start()
+
+    url = 'http://%s:%s' % (socket.gethostname(), server.server_address[1])
+    yield url
+
+    server.shutdown()
+
+
+def artifact_cache_server(valid_filenames):
+    '''Run a basic artifact cache server.
+
+    It implements the 'artifacts' request only. Dummy content will be returned
+    for any file listed in 'valid_filenames'.  Anything else will get a 404
+    error.
+
+    '''
+
+    class Handler(BaseHTTPServer.BaseHTTPRequestHandler):
+        dummy_content = 'I am an artifact.\n'
+
+        def log_message(self, *args, **kwargs):
+            # This is overridden to avoid dumping logs on stdout.
+            pass
+
+        def artifacts(self, query, send_body=True):
+            '''Return a cached artifact, or 404.'''
+            filename = urlparse.parse_qs(query)['filename'][0]
+
+            if filename in valid_filenames:
+                self.send_response(200)
+                self.end_headers()
+                if send_body:
+                    self.wfile.write(self.dummy_content)
+            else:
+                self.send_response(404)
+
+        def do_GET(self, send_body=True):
+            parts = urlparse.urlsplit(self.path)
+            if parts.path == '/1.0/artifacts':
+                self.artifacts(parts.query, send_body=send_body)
+            else:
+                self.send_response(404)
+
+        def do_HEAD(self):
+            return self.do_GET(send_body=False)
+
+    return http_server(Handler)
 
 
 class RemoteArtifactCacheTests(unittest.TestCase):
+    def fake_status_cb(*args, **kwargs):
+        pass
+
+    def test_artifacts_are_cached(self):
+        '''Exercise fetching of artifacts that are cached.'''
+
+        chunk = morphlib.testutils.example_chunk()
+        valid_files = artifact_files_for_chunk_source(chunk)
+
+        with artifact_cache_server(valid_files) as url:
+            rac = morphlib.remoteartifactcache.RemoteArtifactCache(url)
+            an_artifact = chunk.artifacts.values()[0]
+
+            self.assertTrue(
+                rac.has(an_artifact))
+
+            self.assertTrue(
+                rac.has_artifact_metadata(an_artifact, 'meta'))
+
+            self.assertTrue(
+                rac.has_source_metadata(chunk, chunk.cache_key, 'meta'))
+
+            lac = morphlib.testutils.FakeLocalArtifactCache()
+            self.assertFalse(lac.has(an_artifact))
 
-    def setUp(self):
-        loader = morphlib.morphloader.MorphologyLoader()
-        morph = loader.load_from_string(
-            '''
-                name: chunk
-                kind: chunk
-                products:
-                    - artifact: chunk-runtime
-                      include:
-                          - usr/bin
-                          - usr/sbin
-                          - usr/lib
-                          - usr/libexec
-                    - artifact: chunk-devel
-                      include:
-                          - usr/include
-                    - artifact: chunk-doc
-                      include:
-                          - usr/share/doc
-            ''')
-        sources = morphlib.source.make_sources('repo', 'original/ref',
-                                               'chunk.morph', 'sha1',
-                                               'tree', morph)
-        self.source, = sources
-        self.runtime_artifact = morphlib.artifact.Artifact(
-            self.source, 'chunk-runtime')
-        self.runtime_artifact.cache_key = 'CHUNK'
-        self.devel_artifact = morphlib.artifact.Artifact(
-            self.source, 'chunk-devel')
-        self.devel_artifact.cache_key = 'CHUNK'
-        self.doc_artifact = morphlib.artifact.Artifact(
-            self.source, 'chunk-doc')
-        self.doc_artifact.cache_key = 'CHUNK'
-
-        self.existing_files = set([
-            self.runtime_artifact.basename(),
-            self.devel_artifact.basename(),
-            self.runtime_artifact.metadata_basename('meta'),
-            '%s.%s' % (self.runtime_artifact.cache_key, 'meta'),
-        ])
-
-        self.server_url = 'http://foo.bar:8080'
-        self.cache = morphlib.remoteartifactcache.RemoteArtifactCache(
-            self.server_url)
-        self.cache._has_file = self._has_file
-        self.cache._get_file = self._get_file
-
-    def _has_file(self, filename):
-        return filename in self.existing_files
-
-    def _get_file(self, filename):
-        if filename in self.existing_files:
-            return StringIO.StringIO('%s' % filename)
-        else:
-            raise urllib2.URLError('foo')
-
-    def test_sets_server_url(self):
-        self.assertEqual(self.cache.server_url, self.server_url)
-
-    def test_has_existing_artifact(self):
-        self.assertTrue(self.cache.has(self.runtime_artifact))
-
-    def test_has_a_different_existing_artifact(self):
-        self.assertTrue(self.cache.has(self.devel_artifact))
-
-    def test_does_not_have_a_non_existent_artifact(self):
-        self.assertFalse(self.cache.has(self.doc_artifact))
-
-    def test_has_existing_artifact_metadata(self):
-        self.assertTrue(self.cache.has_artifact_metadata(
-            self.runtime_artifact, 'meta'))
-
-    def test_does_not_have_non_existent_artifact_metadata(self):
-        self.assertFalse(self.cache.has_artifact_metadata(
-            self.runtime_artifact, 'non-existent-meta'))
-
-    def test_has_existing_source_metadata(self):
-        self.assertTrue(self.cache.has_source_metadata(
-            self.runtime_artifact.source,
-            self.runtime_artifact.cache_key,
-            'meta'))
-
-    def test_does_not_have_non_existent_source_metadata(self):
-        self.assertFalse(self.cache.has_source_metadata(
-            self.runtime_artifact.source,
-            self.runtime_artifact.cache_key,
-            'non-existent-meta'))
-
-    def test_get_existing_artifact(self):
-        handle = self.cache.get(self.runtime_artifact)
-        data = handle.read()
-        self.assertEqual(data, self.runtime_artifact.basename())
-
-    def test_get_a_different_existing_artifact(self):
-        handle = self.cache.get(self.devel_artifact)
-        data = handle.read()
-        self.assertEqual(data, self.devel_artifact.basename())
-
-    def test_fails_to_get_a_non_existent_artifact(self):
-        self.assertRaises(morphlib.remoteartifactcache.GetError,
-                          self.cache.get, self.doc_artifact,
-                          log=lambda *args: None)
-
-    def test_get_existing_artifact_metadata(self):
-        handle = self.cache.get_artifact_metadata(
-            self.runtime_artifact, 'meta')
-        data = handle.read()
-        self.assertEqual(
-            data, '%s.%s' % (self.runtime_artifact.basename(), 'meta'))
-
-    def test_fails_to_get_non_existent_artifact_metadata(self):
-        self.assertRaises(
-            morphlib.remoteartifactcache.GetArtifactMetadataError,
-            self.cache.get_artifact_metadata,
-            self.runtime_artifact,
-            'non-existent-meta',
-            log=lambda *args: None)
-
-    def test_get_existing_source_metadata(self):
-        handle = self.cache.get_source_metadata(
-            self.runtime_artifact.source,
-            self.runtime_artifact.cache_key,
-            'meta')
-        data = handle.read()
-        self.assertEqual(
-            data, '%s.%s' % (self.runtime_artifact.cache_key, 'meta'))
-
-    def test_fails_to_get_non_existent_source_metadata(self):
-        self.assertRaises(
-            morphlib.remoteartifactcache.GetSourceMetadataError,
-            self.cache.get_source_metadata,
-            self.runtime_artifact.source,
-            self.runtime_artifact.cache_key,
-            'non-existent-meta')
+            rac.get_artifacts(
+                chunk.artifacts.values(), lac, self.fake_status_cb)
+            self.assertTrue(lac.has(an_artifact))
+
+    def test_artifacts_are_not_cached(self):
+        '''Exercise trying to fetch artifacts that the cache doesn't have.'''
+
+        chunk = morphlib.testutils.example_chunk()
+        valid_files = []
+
+        with artifact_cache_server(valid_files) as url:
+            rac = morphlib.remoteartifactcache.RemoteArtifactCache(url)
+            an_artifact = chunk.artifacts.values()[0]
+
+            self.assertFalse(
+                rac.has(an_artifact))
+
+            self.assertFalse(
+                rac.has_artifact_metadata(an_artifact, 'non-existent-meta'))
+
+            self.assertFalse(
+                rac.has_source_metadata(chunk, chunk.cache_key, 'meta'))
+
+            lac = morphlib.testutils.FakeLocalArtifactCache()
+            self.assertRaises(
+                morphlib.remoteartifactcache.GetError, rac.get_artifacts,
+                chunk.artifacts.values(), lac, self.fake_status_cb)
 
     def test_escapes_pluses_in_request_urls(self):
-        returned_url = self.cache._request_url('gtk+')
-        correct_url = '%s/1.0/artifacts?filename=gtk%%2B' % self.server_url
+        rac = morphlib.remoteartifactcache.RemoteArtifactCache(
+            'http://example.com')
+
+        returned_url = rac._request_url('gtk+')
+        correct_url = 'http://example.com/1.0/artifacts?filename=gtk%2B'
         self.assertEqual(returned_url, correct_url)
diff --git a/morphlib/testutils.py b/morphlib/testutils.py
new file mode 100644
index 00000000..3678fc50
--- /dev/null
+++ b/morphlib/testutils.py
@@ -0,0 +1,56 @@
+# Copyright (C) 2012-2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+'''Collected utilities for use in automated tests of Morph.'''
+
+
+import fs
+
+import morphlib
+
+
+def example_chunk():
+    '''Returns a chunk source with some artifacts.'''
+
+    loader = morphlib.morphloader.MorphologyLoader()
+    morph = loader.load_from_string(
+        '''
+            name: test
+            kind: chunk
+            products:
+              - artifact: chunk-runtime
+                include:
+                  - usr/bin
+                  - usr/sbin
+                  - usr/lib
+                  - usr/libexec
+              - artifact: chunk-devel
+                include:
+                  - usr/include
+              - artifact: chunk-doc
+                include:
+                  - usr/share/doc
+        ''')
+
+    source = next(morphlib.source.make_sources(
+        'repo', 'original/ref', 'chunk.morph', 'sha1', 'tree', morph))
+    source.cache_key = 'CHUNK'
+    return source
+
+
+class FakeLocalArtifactCache(morphlib.localartifactcache.LocalArtifactCache):
+    def __init__(self):
+        super(FakeLocalArtifactCache, self).__init__(fs.tempfs.TempFS())
diff --git a/without-test-modules b/without-test-modules
index 530deb4f..66f25371 100644
--- a/without-test-modules
+++ b/without-test-modules
@@ -56,3 +56,6 @@ morphlib/buildbranch.py
 # Requires rather a lot of fake data in order to be unit tested; better to
 # leave it to the functional tests.
 morphlib/sourceresolver.py
+
+# These are part of the test suite
+morphlib/testutils.py