Source cache tests

Few different source cache tests have been to check that sources are
staged into the local CAS and fetched where appropriate

Part of #440

15 files changed, 429 insertions, 0 deletions

diff --git a/tests/sourcecache/__init__.py b/tests/sourcecache/__init__.py
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/sourcecache/__init__.py
diff --git a/tests/sourcecache/config.py b/tests/sourcecache/config.py
new file mode 100644
index 000000000..b5581a7e2
--- /dev/null
+++ b/tests/sourcecache/config.py
@@ -0,0 +1,58 @@
+#
+#  Copyright (C) 2019 Bloomberg Finance L.P.
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Raoul Hidalgo Charman <raoul.hidalgocharman@codethink.co.uk>
+#
+import os
+import pytest
+
+from buildstream import _yaml
+from buildstream._exceptions import ErrorDomain, LoadErrorReason
+
+from buildstream.plugintestutils.runcli import cli
+
+DATA_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+# Assert that if either the client key or client cert is specified
+# without specifying its counterpart, we get a comprehensive LoadError
+# instead of an unhandled exception.
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize('config_key, config_value', [
+    ('client-cert', 'client.crt'),
+    ('client-key', 'client.key')
+])
+def test_missing_certs(cli, datafiles, config_key, config_value):
+    project = os.path.join(datafiles.dirname, datafiles.basename, 'missing-certs')
+
+    project_conf = {
+        'name': 'test',
+
+        'source-caches': {
+            'url': 'https://cache.example.com:12345',
+            'push': 'true',
+            config_key: config_value
+        }
+    }
+    project_conf_file = os.path.join(project, 'project.conf')
+    _yaml.dump(project_conf, project_conf_file)
+
+    # Use `pull` here to ensure we try to initialize the remotes, triggering the error
+    #
+    # This does not happen for a simple `bst show`.
+    result = cli.run(project=project, args=['source', 'fetch', 'element.bst'])
+    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
diff --git a/tests/sourcecache/missing-certs/certificates/client.crt b/tests/sourcecache/missing-certs/certificates/client.crt
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/sourcecache/missing-certs/certificates/client.crt
diff --git a/tests/sourcecache/missing-certs/certificates/client.key b/tests/sourcecache/missing-certs/certificates/client.key
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/sourcecache/missing-certs/certificates/client.key
diff --git a/tests/sourcecache/missing-certs/element.bst b/tests/sourcecache/missing-certs/element.bst
new file mode 100644
index 000000000..3c29b4ea1
--- /dev/null
+++ b/tests/sourcecache/missing-certs/element.bst
@@ -0,0 +1 @@
+kind: autotools
diff --git a/tests/sourcecache/project/elements/compose-all.bst b/tests/sourcecache/project/elements/compose-all.bst
new file mode 100644
index 000000000..ba47081b3
--- /dev/null
+++ b/tests/sourcecache/project/elements/compose-all.bst
@@ -0,0 +1,12 @@
+kind: compose
+
+depends:
+- filename: import-bin.bst
+  type: build
+- filename: import-dev.bst
+  type: build
+
+config:
+  # Dont try running the sandbox, we dont have a
+  # runtime to run anything in this context.
+  integrate: False
diff --git a/tests/sourcecache/project/elements/import-bin.bst b/tests/sourcecache/project/elements/import-bin.bst
new file mode 100644
index 000000000..a847c0c23
--- /dev/null
+++ b/tests/sourcecache/project/elements/import-bin.bst
@@ -0,0 +1,4 @@
+kind: import
+sources:
+- kind: local
+  path: files/bin-files
diff --git a/tests/sourcecache/project/elements/import-dev.bst b/tests/sourcecache/project/elements/import-dev.bst
new file mode 100644
index 000000000..152a54667
--- /dev/null
+++ b/tests/sourcecache/project/elements/import-dev.bst
@@ -0,0 +1,4 @@
+kind: import
+sources:
+- kind: local
+  path: files/dev-files
diff --git a/tests/sourcecache/project/elements/target.bst b/tests/sourcecache/project/elements/target.bst
new file mode 100644
index 000000000..ba489f1e8
--- /dev/null
+++ b/tests/sourcecache/project/elements/target.bst
@@ -0,0 +1,9 @@
+kind: stack
+description: |
+
+  Main stack target for the bst build test
+
+depends:
+- import-bin.bst
+- import-dev.bst
+- compose-all.bst
diff --git a/tests/sourcecache/project/files/bin-files/usr/bin/hello b/tests/sourcecache/project/files/bin-files/usr/bin/hello
new file mode 100755
index 000000000..f534a4083
--- /dev/null
+++ b/tests/sourcecache/project/files/bin-files/usr/bin/hello
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+echo "Hello !"
diff --git a/tests/sourcecache/project/files/dev-files/usr/include/pony.h b/tests/sourcecache/project/files/dev-files/usr/include/pony.h
new file mode 100644
index 000000000..40bd0c2e7
--- /dev/null
+++ b/tests/sourcecache/project/files/dev-files/usr/include/pony.h
@@ -0,0 +1,12 @@
+#ifndef __PONY_H__
+#define __PONY_H__
+
+#define PONY_BEGIN "Once upon a time, there was a pony."
+#define PONY_END "And they lived happily ever after, the end."
+
+#define MAKE_PONY(story)  \
+  PONY_BEGIN \
+  story \
+  PONY_END
+
+#endif /* __PONY_H__ */
diff --git a/tests/sourcecache/project/project.conf b/tests/sourcecache/project/project.conf
new file mode 100644
index 000000000..854e38693
--- /dev/null
+++ b/tests/sourcecache/project/project.conf
@@ -0,0 +1,4 @@
+# Project config for frontend build test
+name: test
+
+element-path: elements
diff --git a/tests/sourcecache/source-checkout.py b/tests/sourcecache/source-checkout.py
new file mode 100644
index 000000000..f526dc586
--- /dev/null
+++ b/tests/sourcecache/source-checkout.py
@@ -0,0 +1,74 @@
+#
+#  Copyright (C) 2018 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Raoul Hidalgo Charman <raoul.hidalgocharman@codethink.co.uk>
+#
+import os
+import pytest
+import shutil
+
+from buildstream._exceptions import ErrorDomain
+from buildstream.plugintestutils.runcli import cli
+
+from tests.testutils.element_generators import create_element_size
+
+DATA_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_source_checkout(tmpdir, datafiles, cli):
+    project_dir = os.path.join(str(tmpdir), 'project')
+    element_path = 'elements'
+    cache_dir = os.path.join(str(tmpdir), 'cache')
+    source_dir = os.path.join(cache_dir, 'sources')
+
+    cli.configure({
+        'cachedir': cache_dir,
+    })
+    target_dir = os.path.join(str(tmpdir), 'target')
+
+    repo = create_element_size('target.bst', project_dir, element_path, [], 100000)
+
+    # without fetch it should fail
+    res = cli.run(project=project_dir, args=['source', 'checkout', 'target.bst', target_dir])
+    res.assert_main_error(ErrorDomain.PIPELINE, "uncached-sources")
+
+    # fetch and check it works
+    res = cli.run(project=project_dir,
+                  args=['source', 'checkout', '--fetch', 'target.bst',
+                        target_dir])
+    res.assert_success()
+    assert "Fetching from" in res.stderr
+
+    # remove the directory and check source checkout works with sources only in
+    # the CAS
+    shutil.rmtree(repo.repo)
+    shutil.rmtree(target_dir)
+    shutil.rmtree(source_dir)
+
+    res = cli.run(project=project_dir,
+                  args=['source', 'checkout', 'target.bst', target_dir])
+    res.assert_success()
+    assert "Fetching from" not in res.stderr
+
+    # remove the CAS and check it doesn't work again
+    shutil.rmtree(target_dir)
+    shutil.rmtree(os.path.join(cache_dir, 'cas'))
+
+    res = cli.run(project=project_dir,
+                  args=['source', 'checkout', '--fetch', 'target.bst', target_dir])
+    res.assert_task_error(ErrorDomain.PLUGIN, None)
diff --git a/tests/sourcecache/staging.py b/tests/sourcecache/staging.py
new file mode 100644
index 000000000..b62bc3c2f
--- /dev/null
+++ b/tests/sourcecache/staging.py
@@ -0,0 +1,189 @@
+#
+#  Copyright (C) 2019 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Raoul Hidalgo Charman <raoul.hidalgocharman@codethink.co.uk>
+#
+import os
+import shutil
+import pytest
+
+from buildstream._context import Context
+from buildstream._project import Project
+
+from buildstream.plugintestutils.runcli import cli
+from tests.testutils.element_generators import create_element_size
+
+
+DATA_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+def dummy_message_handler(message, context):
+    pass
+
+
+# walk that removes the root directory from roots
+def relative_walk(rootdir):
+    for root, dirnames, filenames in os.walk(rootdir):
+        relative_root = root.split(rootdir)[1]
+        yield (relative_root, dirnames, filenames)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_source_staged(tmpdir, cli, datafiles):
+    project_dir = os.path.join(datafiles.dirname, datafiles.basename, 'project')
+    cachedir = os.path.join(str(tmpdir), 'cache')
+
+    cli.configure({
+        'cachedir': cachedir
+    })
+
+    # set up minimal context
+    context = Context()
+    context.load()
+
+    # load project and sourcecache
+    project = Project(project_dir, context)
+    project.ensure_fully_loaded()
+    context.cachedir = cachedir
+    context.set_message_handler(dummy_message_handler)
+    sourcecache = context.sourcecache
+    cas = context.get_cascache()
+
+    res = cli.run(project=project_dir, args=["build", "import-bin.bst"])
+    res.assert_success()
+
+    # now check that the source is in the refs file, this is pretty messy but
+    # seems to be the only way to get the sources?
+    element = project.load_elements(["import-bin.bst"])[0]
+    source = list(element.sources())[0]
+    assert sourcecache.contains(source)
+    assert element._source_cached()
+
+    # Extract the file and check it's the same as the one we imported
+    ref = source._get_source_name()
+    digest = cas.resolve_ref(ref)
+    extractdir = os.path.join(str(tmpdir), "extract")
+    cas.checkout(extractdir, digest)
+    dir1 = extractdir
+    dir2 = os.path.join(project_dir, "files", "bin-files")
+
+    assert list(relative_walk(dir1)) == list(relative_walk(dir2))
+
+
+# Check sources are staged during a fetch
+@pytest.mark.datafiles(DATA_DIR)
+def test_source_fetch(tmpdir, cli, datafiles):
+    project_dir = os.path.join(datafiles.dirname, datafiles.basename, 'project')
+    cachedir = os.path.join(str(tmpdir), 'cache')
+
+    cli.configure({
+        'cachedir': cachedir
+    })
+
+    # set up minimal context
+    context = Context()
+    context.load()
+
+    # load project and sourcecache
+    project = Project(project_dir, context)
+    project.ensure_fully_loaded()
+    context.cachedir = cachedir
+    context.set_message_handler(dummy_message_handler)
+    cas = context.get_cascache()
+
+    res = cli.run(project=project_dir, args=["source", "fetch", "import-dev.bst"])
+    res.assert_success()
+
+    element = project.load_elements(["import-dev.bst"])[0]
+    source = list(element.sources())[0]
+    assert element._source_cached()
+
+    # check that the directory structures are idetical
+    ref = source._get_source_name()
+    digest = cas.resolve_ref(ref)
+    extractdir = os.path.join(str(tmpdir), "extract")
+    cas.checkout(extractdir, digest)
+    dir1 = extractdir
+    dir2 = os.path.join(project_dir, "files", "dev-files")
+
+    assert list(relative_walk(dir1)) == list(relative_walk(dir2))
+
+
+# Check that with sources only in the CAS build successfully completes
+@pytest.mark.datafiles(DATA_DIR)
+def test_staged_source_build(tmpdir, datafiles, cli):
+    project_dir = os.path.join(datafiles.dirname, datafiles.basename, 'project')
+    cachedir = os.path.join(str(tmpdir), 'cache')
+    element_path = 'elements'
+    source_refs = os.path.join(str(tmpdir), 'cache', 'cas', 'refs', 'heads', '@sources')
+    source_dir = os.path.join(str(tmpdir), 'cache', 'sources')
+
+    cli.configure({
+        'cachedir': os.path.join(str(tmpdir), 'cache')
+    })
+
+    create_element_size('target.bst', project_dir, element_path, [], 10000)
+
+    # get the source object
+    context = Context()
+    context.load()
+    project = Project(project_dir, context)
+    project.ensure_fully_loaded()
+    context.cachedir = cachedir
+    context.set_message_handler(dummy_message_handler)
+
+    element = project.load_elements(["import-dev.bst"])[0]
+    source = list(element.sources())[0]
+
+    # check consistency of the source
+    assert not element._source_cached()
+
+    res = cli.run(project=project_dir, args=['build', 'target.bst'])
+    res.assert_success()
+
+    # delete artifacts check state is buildable
+    cli.remove_artifact_from_cache(project_dir, 'target.bst')
+    states = cli.get_element_states(project_dir, ['target.bst'])
+    assert states['target.bst'] == 'buildable'
+
+    # delete source dir and check that state is still buildable
+    shutil.rmtree(source_dir)
+    states = cli.get_element_states(project_dir, ['target.bst'])
+    assert states['target.bst'] == 'buildable'
+
+    # build and check that no fetching was done.
+    res = cli.run(project=project_dir, args=['build', 'target.bst'])
+    res.assert_success()
+    assert 'Fetching from' not in res.stderr
+
+    # assert the source directory is still empty (though there may be
+    # directories from staging etc.)
+    files = []
+    for _, _, filename in os.walk(source_dir):
+        files.extend(filename)
+    assert files == []
+
+    # Now remove the source refs and check the state
+    shutil.rmtree(source_refs)
+    cli.remove_artifact_from_cache(project_dir, 'target.bst')
+    states = cli.get_element_states(project_dir, ['target.bst'])
+    assert states['target.bst'] == 'fetch needed'
+
+    # Check that it now fetches from when building the target
+    res = cli.run(project=project_dir, args=['build', 'target.bst'])
+    res.assert_success()
+    assert 'Fetching from' in res.stderr
diff --git a/tests/sourcecache/workspace.py b/tests/sourcecache/workspace.py
new file mode 100644
index 000000000..440ca81b8
--- /dev/null
+++ b/tests/sourcecache/workspace.py
@@ -0,0 +1,59 @@
+#
+#  Copyright (C) 2019 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Raoul Hidalgo Charman <raoul.hidalgocharman@codethink.co.uk>
+#
+import os
+import pytest
+import shutil
+
+from buildstream.plugintestutils.runcli import cli
+
+from tests.testutils.element_generators import create_element_size
+
+
+DATA_DIR = os.path.dirname(os.path.realpath(__file__))
+
+
+# Test that when we have sources only in the local CAS buildstream fetches them
+# for opening a workspace
+@pytest.mark.datafiles(DATA_DIR)
+def test_workspace_source_fetch(tmpdir, datafiles, cli):
+    project_dir = os.path.join(str(tmpdir), 'project')
+    element_path = 'elements'
+    source_dir = os.path.join(str(tmpdir), 'cache', 'sources')
+    workspace = os.path.join(cli.directory, 'workspace')
+
+    cli.configure({
+        'cachedir': os.path.join(str(tmpdir), 'cache')
+    })
+
+    create_element_size('target.bst', project_dir, element_path, [], 10000)
+    res = cli.run(project=project_dir, args=['build', 'target.bst'])
+    res.assert_success()
+    assert 'Fetching from' in res.stderr
+
+    # remove the original sources
+    shutil.rmtree(source_dir)
+
+    # Open a workspace and check that fetches the original sources
+    res = cli.run(project=project_dir,
+                  args=['workspace', 'open', 'target.bst', '--directory', workspace])
+    res.assert_success()
+    assert 'Fetching from' in res.stderr
+
+    assert os.listdir(workspace) != []