tests/cachekey: Adding Cache Key tests

This test should contain an entry for every element and source plugin
in buildstream. Further, an entry should exist for every feature of
every plugin which may effect cache key calculation.

MANIFEST.in: Adding .expected files used in this test to the
             generated dist tarballs.

41 files changed, 340 insertions, 0 deletions

diff --git a/tests/cachekey/cachekey.py b/tests/cachekey/cachekey.py
new file mode 100644
index 000000000..d6f840dbc
--- /dev/null
+++ b/tests/cachekey/cachekey.py
@@ -0,0 +1,171 @@
+# Cache Key Test Instructions
+#
+# Adding Tests
+# ~~~~~~~~~~~~
+# Cache key tests are bst element files created created in such a way
+# to exercise a feature which would cause the cache key for an element
+# or source to be calculated differently.
+#
+# Adding tests is a matter to adding files to the project found in the
+# 'project' subdirectory of this test case. Any files should be depended
+# on by the main `target.bst` in the toplevel of the project.
+#
+# One test is comprised of one `<element-name>.bst` file and one
+# '<element-name>.expected' file in the same directory, containing the
+# expected cache key.
+#
+# Running the cache key test once will reveal what the new element's
+# cache key should be and will also cause the depending elements to
+# change cache keys.
+#
+#
+# Updating tests
+# ~~~~~~~~~~~~~~
+# When a test fails it will come with a summary of which cache keys
+# in the test project have mismatched.
+#
+# Also, in the case that the tests have changed or the artifact
+# versions have changed in some way and the test needs to be
+# updated; the expected cache keys for the given run are dumped to
+# '<element-name>.actual' files beside the corresponding
+# '<element-name>.expected' files they mismatched with, all inside
+# a temporary test directory.
+#
+# One can now easily copy over the .actual files from a failed
+# run over to the corresponding .expected source files and commit
+# the result.
+#
+
+import os
+from collections import OrderedDict
+from click.testing import CliRunner
+import pytest
+
+# Import the main cli entrypoint
+from buildstream._frontend.main import cli
+
+
+##############################################
+#                Some Helpers                #
+##############################################
+
+# Get whole filename in the temp project with
+# the option of changing the .bst suffix to something else
+#
+def element_filename(project_dir, element_name, alt_suffix=None):
+
+    if alt_suffix:
+
+        # Just in case...
+        assert(element_name.endswith('.bst'))
+
+        # Chop off the 'bst' in '.bst' and add the new suffix
+        element_name = element_name[:-3]
+        element_name = element_name + alt_suffix
+
+    return os.path.join(project_dir, element_name)
+
+
+# Returns an OrderedDict of element names
+# and their cache keys
+#
+def parse_output_keys(output):
+    actual_keys = OrderedDict()
+    lines = output.splitlines()
+    for line in lines:
+        split = line.split("::")
+        name = split[0]
+        key = split[1]
+        actual_keys[name] = key
+
+    return actual_keys
+
+
+# Returns an OrderedDict of element names
+# and their cache keys
+#
+def load_expected_keys(project_dir, actual_keys):
+
+    expected_keys = OrderedDict()
+    for element_name in actual_keys:
+        expected = element_filename(project_dir, element_name, 'expected')
+        try:
+            with open(expected, 'r') as f:
+                expected_key = f.read()
+                expected_key = expected_key.strip()
+        except FileNotFoundError as e:
+            raise Exception("Cache key test needs update, " +
+                            "expected file {} not found.\n".format(expected) +
+                            "Hint: Actual key for element {} is: {}".format(
+                                element_name,
+                                actual_keys[element_name]))
+
+        expected_keys[element_name] = expected_key
+
+    return expected_keys
+
+
+def assert_cache_keys(project_dir, output):
+
+    # Read in the expected keys from the cache key test directory
+    # and parse the actual keys from the `bst show` output
+    #
+    actual_keys = parse_output_keys(output)
+    expected_keys = load_expected_keys(project_dir, actual_keys)
+    mismatches = []
+
+    for element_name in actual_keys:
+        if actual_keys[element_name] != expected_keys[element_name]:
+            mismatches.append(element_name)
+
+    if mismatches:
+        info = ""
+        for element_name in mismatches:
+            info += "  Element: {}\n".format(element_name) + \
+                    "    Expected: {}\n".format(expected_keys[element_name]) + \
+                    "    Actual: {}\n".format(actual_keys[element_name])
+
+            # Write out the keys into files beside the project
+            # in the temp directory so that we can easily update
+            # the test when the artifact version changes.
+            filename = element_filename(project_dir, element_name, "actual")
+            with open(filename, "w") as f:
+                f.write(actual_keys[element_name])
+
+        raise AssertionError("Cache key mismatches occurred:\n{}\n".format(info) +
+                             "New cache keys have been stored beside the " +
+                             "expected ones at: {}".format(project_dir))
+
+
+##############################################
+#             Test Entry Point               #
+##############################################
+
+# Project directory
+DATA_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    "project",
+)
+
+
+@pytest.fixture(scope="module")
+def runner():
+    return CliRunner()
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_cache_key(datafiles, runner):
+    project_dir = os.path.join(datafiles.dirname, datafiles.basename)
+    result = runner.invoke(cli, [
+        '--no-verbose',
+        '--directory', project_dir,
+        'show',
+        '--format', '%{name}::%{full-key}',
+        'target.bst'
+    ])
+
+    if result.exit_code != 0:
+        raise AssertionError("BuildStream exited with code {} and output:\n{}"
+                             .format(result.exit_code, result.output))
+
+    assert_cache_keys(project_dir, result.output)
diff --git a/tests/cachekey/project/elements/build1.bst b/tests/cachekey/project/elements/build1.bst
new file mode 100644
index 000000000..3675b24cc
--- /dev/null
+++ b/tests/cachekey/project/elements/build1.bst
@@ -0,0 +1,14 @@
+# The cache key calculation algorithm is the same
+# for all build elements, better to only have one test
+# for it and without too much noise from changes which
+# would cause is to change the test case.
+#
+# Lets stick with manual, the most basic build element.
+kind: manual
+sources:
+- kind: local
+  path: files/local
+
+config:
+  build-commands:
+  - make
diff --git a/tests/cachekey/project/elements/build1.expected b/tests/cachekey/project/elements/build1.expected
new file mode 100644
index 000000000..bf175041f
--- /dev/null
+++ b/tests/cachekey/project/elements/build1.expected
@@ -0,0 +1 @@
+af3b9a7b0c98aa8e2cea8894cabec5b0be93ab80fb23a91241caea4f4b3e42f5
diff --git a/tests/cachekey/project/elements/compose1.bst b/tests/cachekey/project/elements/compose1.bst
new file mode 100644
index 000000000..34eeaa10a
--- /dev/null
+++ b/tests/cachekey/project/elements/compose1.bst
@@ -0,0 +1,4 @@
+kind: compose
+depends:
+- filename: elements/import1.bst
+  type: build
diff --git a/tests/cachekey/project/elements/compose1.expected b/tests/cachekey/project/elements/compose1.expected
new file mode 100644
index 000000000..aae0f2a69
--- /dev/null
+++ b/tests/cachekey/project/elements/compose1.expected
@@ -0,0 +1 @@
+361b2efdbd40b33d1cf0d6c49b138ff748e02c809b6b8c545502f26e393c51ec
diff --git a/tests/cachekey/project/elements/compose2.bst b/tests/cachekey/project/elements/compose2.bst
new file mode 100644
index 000000000..656a06455
--- /dev/null
+++ b/tests/cachekey/project/elements/compose2.bst
@@ -0,0 +1,7 @@
+kind: compose
+depends:
+- filename: elements/import1.bst
+  type: build
+
+config:
+  integrate: False
diff --git a/tests/cachekey/project/elements/compose2.expected b/tests/cachekey/project/elements/compose2.expected
new file mode 100644
index 000000000..09fc2994b
--- /dev/null
+++ b/tests/cachekey/project/elements/compose2.expected
@@ -0,0 +1 @@
+1154eed4a7c4d85cf7a86bd3321b4f888d808759d41bbb94becca716bc62b31d
diff --git a/tests/cachekey/project/elements/compose3.bst b/tests/cachekey/project/elements/compose3.bst
new file mode 100644
index 000000000..a4aae1b90
--- /dev/null
+++ b/tests/cachekey/project/elements/compose3.bst
@@ -0,0 +1,7 @@
+kind: compose
+depends:
+- filename: elements/import1.bst
+  type: build
+
+config:
+  include-orphans: False
diff --git a/tests/cachekey/project/elements/compose3.expected b/tests/cachekey/project/elements/compose3.expected
new file mode 100644
index 000000000..8b4d3b3bc
--- /dev/null
+++ b/tests/cachekey/project/elements/compose3.expected
@@ -0,0 +1 @@
+e724d4075b947c757d41449f4744b148f8e6e0f9b75a74a6a6ab8629c4ddd856
diff --git a/tests/cachekey/project/elements/compose4.bst b/tests/cachekey/project/elements/compose4.bst
new file mode 100644
index 000000000..41288b604
--- /dev/null
+++ b/tests/cachekey/project/elements/compose4.bst
@@ -0,0 +1,9 @@
+kind: compose
+depends:
+- filename: elements/import1.bst
+  type: build
+
+config:
+  include:
+  - runtime
+  - devel
diff --git a/tests/cachekey/project/elements/compose4.expected b/tests/cachekey/project/elements/compose4.expected
new file mode 100644
index 000000000..0a97cef45
--- /dev/null
+++ b/tests/cachekey/project/elements/compose4.expected
@@ -0,0 +1 @@
+95b15616dc332b27f34cd46157638f897dde68c428f1675cc1e44c7f5857181d
diff --git a/tests/cachekey/project/elements/compose5.bst b/tests/cachekey/project/elements/compose5.bst
new file mode 100644
index 000000000..46832c36b
--- /dev/null
+++ b/tests/cachekey/project/elements/compose5.bst
@@ -0,0 +1,8 @@
+kind: compose
+depends:
+- filename: elements/import1.bst
+  type: build
+
+config:
+  exclude:
+  - debug
diff --git a/tests/cachekey/project/elements/compose5.expected b/tests/cachekey/project/elements/compose5.expected
new file mode 100644
index 000000000..c2240dd58
--- /dev/null
+++ b/tests/cachekey/project/elements/compose5.expected
@@ -0,0 +1 @@
+1b67cbf1ddd29bfbc7744f10b4298516dda9860f9f8c39c642fc58a1a7c6d592
diff --git a/tests/cachekey/project/elements/import1.bst b/tests/cachekey/project/elements/import1.bst
new file mode 100644
index 000000000..983d2530f
--- /dev/null
+++ b/tests/cachekey/project/elements/import1.bst
@@ -0,0 +1,4 @@
+kind: import
+sources:
+- kind: local
+  path: files/local
diff --git a/tests/cachekey/project/elements/import1.expected b/tests/cachekey/project/elements/import1.expected
new file mode 100644
index 000000000..a0f745efd
--- /dev/null
+++ b/tests/cachekey/project/elements/import1.expected
@@ -0,0 +1 @@
+8638c5c22601307520aa035e91097a1035c5d0805dc14d188fabd5995a41cecf
diff --git a/tests/cachekey/project/elements/import2.bst b/tests/cachekey/project/elements/import2.bst
new file mode 100644
index 000000000..4981862ed
--- /dev/null
+++ b/tests/cachekey/project/elements/import2.bst
@@ -0,0 +1,7 @@
+kind: import
+sources:
+- kind: local
+  path: files/local
+
+config:
+  source: /usr/bin
diff --git a/tests/cachekey/project/elements/import2.expected b/tests/cachekey/project/elements/import2.expected
new file mode 100644
index 000000000..3dc01aead
--- /dev/null
+++ b/tests/cachekey/project/elements/import2.expected
@@ -0,0 +1 @@
+232d1b47a2a04827e7aaf5eb67b10570f014ce0151b8cd13ab764a1d8eb12869
diff --git a/tests/cachekey/project/elements/import3.bst b/tests/cachekey/project/elements/import3.bst
new file mode 100644
index 000000000..94503805f
--- /dev/null
+++ b/tests/cachekey/project/elements/import3.bst
@@ -0,0 +1,7 @@
+kind: import
+sources:
+- kind: local
+  path: files/local
+
+config:
+  target: /opt
diff --git a/tests/cachekey/project/elements/import3.expected b/tests/cachekey/project/elements/import3.expected
new file mode 100644
index 000000000..ebab22a00
--- /dev/null
+++ b/tests/cachekey/project/elements/import3.expected
@@ -0,0 +1 @@
+71cc75f1e43be236ad274c3c55d82971437f1e1ca58d8955b2dfc3892a1157d5
diff --git a/tests/cachekey/project/elements/script1.bst b/tests/cachekey/project/elements/script1.bst
new file mode 100644
index 000000000..104688449
--- /dev/null
+++ b/tests/cachekey/project/elements/script1.bst
@@ -0,0 +1,8 @@
+kind: script
+depends:
+- filename: elements/import1.bst
+  type: build
+
+config:
+  commands:
+  - echo "Hello World !"
diff --git a/tests/cachekey/project/elements/script1.expected b/tests/cachekey/project/elements/script1.expected
new file mode 100644
index 000000000..138ad193a
--- /dev/null
+++ b/tests/cachekey/project/elements/script1.expected
@@ -0,0 +1 @@
+12cd69d517a76e823e0c7a2b9e3da1aed1222c183249edcb97b627b350993c36
diff --git a/tests/cachekey/project/files/local/usr/bin/hello b/tests/cachekey/project/files/local/usr/bin/hello
new file mode 100755
index 000000000..f534a4083
--- /dev/null
+++ b/tests/cachekey/project/files/local/usr/bin/hello
@@ -0,0 +1,3 @@
+#!/bin/bash
+
+echo "Hello !"
diff --git a/tests/cachekey/project/project.conf b/tests/cachekey/project/project.conf
new file mode 100644
index 000000000..49726cdef
--- /dev/null
+++ b/tests/cachekey/project/project.conf
@@ -0,0 +1,2 @@
+# Project config for cache key test
+name: cachekey
diff --git a/tests/cachekey/project/sources/bzr1.bst b/tests/cachekey/project/sources/bzr1.bst
new file mode 100644
index 000000000..4798d8add
--- /dev/null
+++ b/tests/cachekey/project/sources/bzr1.bst
@@ -0,0 +1,6 @@
+kind: import
+sources:
+- kind: bzr
+  url: https://launchpad.net/bzr
+  track: trunk
+  ref: 6622
diff --git a/tests/cachekey/project/sources/bzr1.expected b/tests/cachekey/project/sources/bzr1.expected
new file mode 100644
index 000000000..b7fce4bb3
--- /dev/null
+++ b/tests/cachekey/project/sources/bzr1.expected
@@ -0,0 +1 @@
+b42fa20fb19fe553c8aae85fac271c2028779d278aa5c741da3d76dbc1b8e75d
diff --git a/tests/cachekey/project/sources/git1.bst b/tests/cachekey/project/sources/git1.bst
new file mode 100644
index 000000000..8763f648d
--- /dev/null
+++ b/tests/cachekey/project/sources/git1.bst
@@ -0,0 +1,5 @@
+kind: import
+sources:
+- kind: git
+  url: https://example.com/git/repo.git
+  ref: 6ac68af3e80b7b17c23a3c65233043550a7fa685
diff --git a/tests/cachekey/project/sources/git1.expected b/tests/cachekey/project/sources/git1.expected
new file mode 100644
index 000000000..7c308911a
--- /dev/null
+++ b/tests/cachekey/project/sources/git1.expected
@@ -0,0 +1 @@
+d3d9fce37d51387adfb93717a2d6ac9fa82de4a0454d40dbcae6683d7e90f9fb
diff --git a/tests/cachekey/project/sources/git2.bst b/tests/cachekey/project/sources/git2.bst
new file mode 100644
index 000000000..130ccf33a
--- /dev/null
+++ b/tests/cachekey/project/sources/git2.bst
@@ -0,0 +1,9 @@
+kind: import
+sources:
+- kind: git
+  url: https://example.com/git/repo.git
+  ref: 6ac68af3e80b7b17c23a3c65233043550a7fa685
+
+  submodules:
+    plugins/foo:
+      url: upstream:foo.git
diff --git a/tests/cachekey/project/sources/git2.expected b/tests/cachekey/project/sources/git2.expected
new file mode 100644
index 000000000..3e57d1fc8
--- /dev/null
+++ b/tests/cachekey/project/sources/git2.expected
@@ -0,0 +1 @@
+eb87a91092d9edc4eabd0839adeeb08fb4ca7a46bc4e3c6f8b822b88174fa3a7
diff --git a/tests/cachekey/project/sources/local1.bst b/tests/cachekey/project/sources/local1.bst
new file mode 100644
index 000000000..983d2530f
--- /dev/null
+++ b/tests/cachekey/project/sources/local1.bst
@@ -0,0 +1,4 @@
+kind: import
+sources:
+- kind: local
+  path: files/local
diff --git a/tests/cachekey/project/sources/local1.expected b/tests/cachekey/project/sources/local1.expected
new file mode 100644
index 000000000..a0f745efd
--- /dev/null
+++ b/tests/cachekey/project/sources/local1.expected
@@ -0,0 +1 @@
+8638c5c22601307520aa035e91097a1035c5d0805dc14d188fabd5995a41cecf
diff --git a/tests/cachekey/project/sources/local2.bst b/tests/cachekey/project/sources/local2.bst
new file mode 100644
index 000000000..a7b34fc54
--- /dev/null
+++ b/tests/cachekey/project/sources/local2.bst
@@ -0,0 +1,5 @@
+kind: import
+sources:
+- kind: local
+  path: files/local
+  directory: opt
diff --git a/tests/cachekey/project/sources/local2.expected b/tests/cachekey/project/sources/local2.expected
new file mode 100644
index 000000000..dd2433413
--- /dev/null
+++ b/tests/cachekey/project/sources/local2.expected
@@ -0,0 +1 @@
+1c7179f249836d755086fce01936057500960102056ba755ce79603a171bfaba
diff --git a/tests/cachekey/project/sources/ostree1.bst b/tests/cachekey/project/sources/ostree1.bst
new file mode 100644
index 000000000..766fdecb0
--- /dev/null
+++ b/tests/cachekey/project/sources/ostree1.bst
@@ -0,0 +1,6 @@
+kind: import
+sources:
+- kind: ostree
+  url: https://example.com/repo
+  ref: ccc885b96749f9d1774c7fa0c3262a9a3694e2d2643d1f8d420f5d23adf5db48
+  track: testing/x86_64
diff --git a/tests/cachekey/project/sources/ostree1.expected b/tests/cachekey/project/sources/ostree1.expected
new file mode 100644
index 000000000..a67a7679f
--- /dev/null
+++ b/tests/cachekey/project/sources/ostree1.expected
@@ -0,0 +1 @@
+ebb67e2c6a71e0988fc904d953571c39368a5eb4d0f4ddbac792d117ac654e90
diff --git a/tests/cachekey/project/sources/tar1.bst b/tests/cachekey/project/sources/tar1.bst
new file mode 100644
index 000000000..1dbf27cd8
--- /dev/null
+++ b/tests/cachekey/project/sources/tar1.bst
@@ -0,0 +1,5 @@
+kind: import
+sources:
+- kind: tar
+  url: https://example.com/releases/1.4/foo-1.4.5.tar.gz
+  ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b
diff --git a/tests/cachekey/project/sources/tar1.expected b/tests/cachekey/project/sources/tar1.expected
new file mode 100644
index 000000000..331c88a22
--- /dev/null
+++ b/tests/cachekey/project/sources/tar1.expected
@@ -0,0 +1 @@
+d8923fbbd18be7761f7513df5d3c2fc08a928fc0738ff93e0223bbece564bfc5
diff --git a/tests/cachekey/project/sources/tar2.bst b/tests/cachekey/project/sources/tar2.bst
new file mode 100644
index 000000000..e357be392
--- /dev/null
+++ b/tests/cachekey/project/sources/tar2.bst
@@ -0,0 +1,6 @@
+kind: import
+sources:
+- kind: tar
+  url: https://example.com/releases/1.4/foo-1.4.5.tar.gz
+  ref: 6c9f6f68a131ec6381da82f2bff978083ed7f4f7991d931bfa767b7965ebc94b
+  base-dir: src
diff --git a/tests/cachekey/project/sources/tar2.expected b/tests/cachekey/project/sources/tar2.expected
new file mode 100644
index 000000000..64f565b62
--- /dev/null
+++ b/tests/cachekey/project/sources/tar2.expected
@@ -0,0 +1 @@
+5defafe194db9d1c3913c3ba0bdaaf18deafab38633a420c91496bed35063516
diff --git a/tests/cachekey/project/target.bst b/tests/cachekey/project/target.bst
new file mode 100644
index 000000000..27f68ef67
--- /dev/null
+++ b/tests/cachekey/project/target.bst
@@ -0,0 +1,24 @@
+kind: stack
+description: |
+
+  This is the main entry point including cases in the cache key test.
+
+depends:
+- sources/bzr1.bst
+- sources/git1.bst
+- sources/git2.bst
+- sources/local1.bst
+- sources/local2.bst
+- sources/ostree1.bst
+- sources/tar1.bst
+- sources/tar2.bst
+- elements/build1.bst
+- elements/compose1.bst
+- elements/compose2.bst
+- elements/compose3.bst
+- elements/compose4.bst
+- elements/compose5.bst
+- elements/import1.bst
+- elements/import2.bst
+- elements/import3.bst
+- elements/script1.bst
diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected
new file mode 100644
index 000000000..70883c978
--- /dev/null
+++ b/tests/cachekey/project/target.expected
@@ -0,0 +1 @@
+1c5620af1f5097f98aae8b71059e461010cf61ea0e35de8d8f9e32f840ea165e