Source cache uses new proto methods

This changes the location of source refs from `cache/cas/refs` to
`cache/sources_protos` and fixes tests that looked in the old reference
place. Source cache now uses the new protocol buffer and grpc methods
defined, for storing locally, and pushing and pulling to remotes.

Part of #1038

4 files changed, 27 insertions, 21 deletions

diff --git a/tests/sourcecache/cache.py b/tests/sourcecache/cache.py
index 20faaa64e..793344ef0 100644
--- a/tests/sourcecache/cache.py
+++ b/tests/sourcecache/cache.py
@@ -39,10 +39,10 @@ def test_patch_sources_cached_1(cli, datafiles):
 
     # as we have a local, patch, local config, the first local and patch should
     # be cached together, and the last local on it's own
-    source_dir = os.path.join(project_dir, 'cache', 'cas', 'refs', 'heads', '@sources')
+    source_protos = os.path.join(project_dir, 'cache', 'source_protos')
 
-    assert len(os.listdir(os.path.join(source_dir, 'patch'))) == 1
-    assert len(os.listdir(os.path.join(source_dir, 'local'))) == 2
+    assert len(os.listdir(os.path.join(source_protos, 'patch'))) == 1
+    assert len(os.listdir(os.path.join(source_protos, 'local'))) == 2
 
 
 @pytest.mark.datafiles(DATA_DIR)
@@ -53,9 +53,9 @@ def test_patch_sources_cached_2(cli, datafiles):
     res.assert_success()
 
     # As everything is before the patch it should all be cached together
-    source_dir = os.path.join(project_dir, 'cache', 'cas', 'refs', 'heads', '@sources')
+    source_protos = os.path.join(project_dir, 'cache', 'source_protos')
 
-    assert len(os.listdir(os.path.join(source_dir, 'patch'))) == 1
+    assert len(os.listdir(os.path.join(source_protos, 'patch'))) == 1
 
 
 @pytest.mark.datafiles(DATA_DIR)
@@ -66,9 +66,9 @@ def test_sources_without_patch(cli, datafiles):
     res.assert_success()
 
     # No patches so everything should be cached seperately
-    source_dir = os.path.join(project_dir, 'cache', 'cas', 'refs', 'heads', '@sources')
+    source_protos = os.path.join(project_dir, 'cache', 'source_protos')
 
-    assert len(os.listdir(os.path.join(source_dir, 'local'))) == 3
+    assert len(os.listdir(os.path.join(source_protos, 'local'))) == 3
 
 
 @pytest.mark.datafiles(DATA_DIR)
@@ -105,8 +105,8 @@ def test_source_cache_key(cli, datafiles):
     res.assert_success()
 
     # Should have one source ref
-    patch_refs = os.path.join(project_dir, 'cache', 'cas', 'refs', 'heads', '@sources', 'patch')
-    assert len(os.listdir(patch_refs)) == 1
+    patch_protos = os.path.join(project_dir, 'cache', 'source_protos', 'patch')
+    assert len(os.listdir(patch_protos)) == 1
 
     # modify hello-patch file and check tracking updates refs
     with open(os.path.join(file_path, 'dev-files', 'usr', 'include', 'pony.h'), 'a') as f:
@@ -120,4 +120,4 @@ def test_source_cache_key(cli, datafiles):
     res.assert_success()
 
     # We should have a new source ref
-    assert len(os.listdir(patch_refs)) == 2
+    assert len(os.listdir(patch_protos)) == 2
diff --git a/tests/sourcecache/fetch.py b/tests/sourcecache/fetch.py
index 3fc9d96a6..de8587862 100644
--- a/tests/sourcecache/fetch.py
+++ b/tests/sourcecache/fetch.py
@@ -89,15 +89,22 @@ def test_source_fetch(cli, tmpdir, datafiles):
 
         assert os.listdir(os.path.join(str(tmpdir), 'cache', 'sources', 'git')) != []
 
+        # get root digest of source
+        sourcecache = context.sourcecache
+        digest = sourcecache.export(source)._get_digest()
+
         # Move source in local cas to repo
         shutil.rmtree(os.path.join(str(tmpdir), 'sourceshare', 'repo', 'cas'))
         shutil.move(
+            os.path.join(str(tmpdir), 'cache', 'source_protos'),
+            os.path.join(str(tmpdir), 'sourceshare', 'repo'))
+        shutil.move(
             os.path.join(str(tmpdir), 'cache', 'cas'),
             os.path.join(str(tmpdir), 'sourceshare', 'repo'))
         shutil.rmtree(os.path.join(str(tmpdir), 'cache', 'sources'))
         shutil.rmtree(os.path.join(str(tmpdir), 'cache', 'artifacts'))
 
-        digest = share.cas.resolve_ref(source._get_source_name())
+        # check the share has the object
         assert share.has_object(digest)
 
         state = cli.get_element_state(project_dir, 'fetch.bst')
@@ -163,7 +170,7 @@ def test_fetch_fallback(cli, tmpdir, datafiles):
         res = cli.run(project=project_dir, args=['source', 'fetch', 'fetch.bst'])
         res.assert_success()
         brief_key = source._get_brief_display_key()
-        assert ("Remote ({}) does not have source {} cached"
+        assert ("Remote source service ({}) does not have source {} cached"
                 .format(share.repo, brief_key)) in res.stderr
         assert ("SUCCESS Fetching from {}"
                 .format(repo.source_config(ref=ref)['url'])) in res.stderr
@@ -219,5 +226,5 @@ def test_pull_fail(cli, tmpdir, datafiles):
         res = cli.run(project=project_dir, args=['build', 'push.bst'])
         res.assert_main_error(ErrorDomain.STREAM, None)
         res.assert_task_error(ErrorDomain.PLUGIN, None)
-        assert "Remote ({}) does not have source {} cached".format(
+        assert "Remote source service ({}) does not have source {} cached".format(
             share.repo, source._get_brief_display_key()) in res.stderr
diff --git a/tests/sourcecache/push.py b/tests/sourcecache/push.py
index 6282b6e60..23f5f1ca1 100644
--- a/tests/sourcecache/push.py
+++ b/tests/sourcecache/push.py
@@ -95,7 +95,7 @@ def test_source_push(cli, tmpdir, datafiles):
         assert sourcecache.contains(source)
 
         # check that's the remote CAS now has it
-        digest = share.cas.resolve_ref(source._get_source_name())
+        digest = sourcecache.export(source)._get_digest()
         assert share.has_object(digest)
 
 
diff --git a/tests/sourcecache/staging.py b/tests/sourcecache/staging.py
index 9dc431bda..c15bed215 100644
--- a/tests/sourcecache/staging.py
+++ b/tests/sourcecache/staging.py
@@ -78,8 +78,7 @@ def test_source_staged(tmpdir, cli, datafiles):
     assert sourcecache.contains(source)
 
     # Extract the file and check it's the same as the one we imported
-    ref = source._get_source_name()
-    digest = cas.resolve_ref(ref)
+    digest = sourcecache.export(source)._get_digest()
     extractdir = os.path.join(str(tmpdir), "extract")
     cas.checkout(extractdir, digest)
     dir1 = extractdir
@@ -108,6 +107,7 @@ def test_source_fetch(tmpdir, cli, datafiles):
     context.cachedir = cachedir
     context.messenger.set_message_handler(dummy_message_handler)
     cas = context.get_cascache()
+    sourcecache = context.sourcecache
 
     res = cli.run(project=project_dir, args=["source", "fetch", "import-dev.bst"])
     res.assert_success()
@@ -117,8 +117,7 @@ def test_source_fetch(tmpdir, cli, datafiles):
     assert element._source_cached()
 
     # check that the directory structures are idetical
-    ref = source._get_source_name()
-    digest = cas.resolve_ref(ref)
+    digest = sourcecache.export(source)._get_digest()
     extractdir = os.path.join(str(tmpdir), "extract")
     cas.checkout(extractdir, digest)
     dir1 = extractdir
@@ -133,11 +132,11 @@ def test_staged_source_build(tmpdir, datafiles, cli):
     project_dir = os.path.join(datafiles.dirname, datafiles.basename, 'project')
     cachedir = os.path.join(str(tmpdir), 'cache')
     element_path = 'elements'
-    source_refs = os.path.join(str(tmpdir), 'cache', 'cas', 'refs', 'heads', '@sources')
+    source_protos = os.path.join(str(tmpdir), 'cache', 'source_protos')
     source_dir = os.path.join(str(tmpdir), 'cache', 'sources')
 
     cli.configure({
-        'cachedir': os.path.join(str(tmpdir), 'cache')
+        'cachedir': cachedir
     })
 
     create_element_size('target.bst', project_dir, element_path, [], 10000)
@@ -181,7 +180,7 @@ def test_staged_source_build(tmpdir, datafiles, cli):
     assert files == []
 
     # Now remove the source refs and check the state
-    shutil.rmtree(source_refs)
+    shutil.rmtree(source_protos)
     cli.remove_artifact_from_cache(project_dir, 'target.bst')
     states = cli.get_element_states(project_dir, ['target.bst'])
     assert states['target.bst'] == 'fetch needed'