testing.py: Add a new `check_cache_key_stability` helperbschubert/cache-key-helper

This allows plugin authors to implement cache keys tests
more easily

6 files changed, 243 insertions, 179 deletions

diff --git a/doc/source/hacking/writing_plugins.rst b/doc/source/hacking/writing_plugins.rst
index 8425425bf..d8462d878 100644
--- a/doc/source/hacking/writing_plugins.rst
+++ b/doc/source/hacking/writing_plugins.rst
@@ -47,6 +47,6 @@ This test ensures that cache keys do not unexpectedly change or become incompati
 due to code changes. As such, the cache key test should have full coverage of every
 YAML configuration which can possibly affect cache key outcome at all times.
 
-See the ``tests/cachekey/update.py`` file for instructions on running the updater,
+See the ``src/buildstream/testing/_update_cachekeys.py`` file for instructions on running the updater,
 you need to run the updater to generate the ``.expected`` files and add the new
 ``.expected`` files in the same commit which extends the cache key test.
diff --git a/src/buildstream/testing/__init__.py b/src/buildstream/testing/__init__.py
index dafc3a9fe..f09c5bda1 100644
--- a/src/buildstream/testing/__init__.py
+++ b/src/buildstream/testing/__init__.py
@@ -27,6 +27,14 @@ from ._yaml import generate_project, generate_element
 from .repo import Repo
 from .runcli import cli, cli_integration, cli_remote_execution
 from .integration import integration_cache
+from ._cachekeys import check_cache_key_stability
+
+__all__ = [
+    "check_cache_key_stability",
+    "create_repo",
+    "register_repo_kind",
+    "sourcetests_collection_hook",
+]
 
 # To make use of these test utilities it is necessary to have pytest
 # available. However, we don't want to have a hard dependency on
diff --git a/src/buildstream/testing/_cachekeys.py b/src/buildstream/testing/_cachekeys.py
new file mode 100644
index 000000000..a7ea5ad7e
--- /dev/null
+++ b/src/buildstream/testing/_cachekeys.py
@@ -0,0 +1,148 @@
+#
+#  Copyright (C) 2020 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+from collections import OrderedDict
+
+from .runcli import Cli
+
+
+def check_cache_key_stability(project_path: os.PathLike, cli: Cli) -> None:
+    """
+    Check that the cache key of various elements has not changed.
+
+    This ensures that elements do not break cache keys unexpectedly.
+
+    The format of the project is expected to be:
+
+    .. code-block::
+
+        ./
+        ./project.conf
+        ./elem1.bst
+        ./elem1.expected
+        ./elem2.bst
+        ./elem2.expected
+        # Or in sub-directories
+        ./mydir/elem3.bst
+        ./mydir/elem3.expected
+
+    The ``.expected`` file should contain the expected cache key.
+
+    In order to automatically created the ``.expected`` files, or updated them,
+    you can run ``python3 -m buildstream.testing._update_cachekeys`` in the
+    project's directory.
+
+    :param project_path: Path to a project
+    :param cli: a `cli` object as provided by the fixture :func:`buildstream.testing.runcli.cli`
+    """
+    result = cli.run(
+        project=project_path, silent=True, args=["show", "--format", "%{name}::%{full-key}", "target.bst"]
+    )
+    result.assert_success()
+    _assert_cache_keys(project_path, result.output)
+
+
+###############################
+## Internal Helper functions ##
+###############################
+# Those functions are for internal use only and are not part of the public API
+
+
+def _element_filename(project_dir, element_name, alt_suffix=None):
+    # Get whole filename in the temp project with
+    # the option of changing the .bst suffix to something else
+    #
+    if alt_suffix:
+
+        # Just in case...
+        assert element_name.endswith(".bst")
+
+        # Chop off the 'bst' in '.bst' and add the new suffix
+        element_name = element_name[:-3]
+        element_name = element_name + alt_suffix
+
+    return os.path.join(project_dir, element_name)
+
+
+def _parse_output_keys(output):
+    # Returns an OrderedDict of element names
+    # and their cache keys
+    #
+    actual_keys = OrderedDict()
+    lines = output.splitlines()
+    for line in lines:
+        split = line.split("::")
+        name = split[0]
+        key = split[1]
+        actual_keys[name] = key
+
+    return actual_keys
+
+
+def _load_expected_keys(project_dir, actual_keys, raise_error=True):
+    # Returns an OrderedDict of element names
+    # and their cache keys
+    #
+    expected_keys = OrderedDict()
+    for element_name in actual_keys:
+        expected = _element_filename(project_dir, element_name, "expected")
+        try:
+            with open(expected, "r") as f:
+                expected_key = f.read()
+                expected_key = expected_key.strip()
+        except FileNotFoundError:
+            expected_key = None
+            if raise_error:
+                raise Exception(
+                    "Cache key test needs update, "
+                    + "expected file {} not found.\n\n".format(expected)
+                    + "Use python3 -m buildstream.testing._update_cachekeys in the"
+                    + " project's directory to automatically update this test case"
+                )
+
+        expected_keys[element_name] = expected_key
+
+    return expected_keys
+
+
+def _assert_cache_keys(project_dir, output):
+    # Read in the expected keys from the cache key test directory
+    # and parse the actual keys from the `bst show` output
+    #
+    actual_keys = _parse_output_keys(output)
+    expected_keys = _load_expected_keys(project_dir, actual_keys)
+    mismatches = []
+
+    for element_name in actual_keys:
+        if actual_keys[element_name] != expected_keys[element_name]:
+            mismatches.append(element_name)
+
+    if mismatches:
+        info = ""
+        for element_name in mismatches:
+            info += (
+                "  Element: {}\n".format(element_name)
+                + "    Expected: {}\n".format(expected_keys[element_name])
+                + "    Actual: {}\n".format(actual_keys[element_name])
+            )
+
+        raise AssertionError(
+            "Cache key mismatches occurred:\n{}\n".format(info)
+            + "Use python3 -m buildstream.testing._update_cachekeys  in the project's"
+            + "directory to automatically update this test case"
+        )
diff --git a/src/buildstream/testing/_update_cachekeys.py b/src/buildstream/testing/_update_cachekeys.py
new file mode 100755
index 000000000..219e17f8b
--- /dev/null
+++ b/src/buildstream/testing/_update_cachekeys.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python3
+#
+#  Copyright (C) 2019 Codethink Limited
+#  Copyright (C) 2020 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+
+#
+# Automatically create or update the .expected files in the
+# cache key test directory.
+#
+# Simply run without any arguments, from the directory containing the project, e.g.:
+#
+#   python3 -m buildstream.testing._update_cachekeys
+#
+# After this, add any files which were newly created and commit
+# the result in order to adjust the cache key test to changed
+# keys.
+#
+import os
+import tempfile
+from unittest import mock
+
+from buildstream.testing._cachekeys import _element_filename, _parse_output_keys, _load_expected_keys
+from buildstream.testing.runcli import Cli
+
+
+def write_expected_key(project_dir, element_name, actual_key):
+    expected_file = _element_filename(project_dir, element_name, "expected")
+    with open(expected_file, "w") as f:
+        f.write(actual_key)
+
+
+def update_keys():
+    project_dir = os.getcwd()
+
+    with tempfile.TemporaryDirectory(dir=project_dir) as cache_dir:
+        # Run bst show
+        cli = Cli(cache_dir, verbose=True)
+        result = cli.run(
+            project=project_dir, silent=True, args=["--no-colors", "show", "--format", "%{name}::%{full-key}"],
+        )
+
+        # Load the actual keys, and the expected ones if they exist
+        if not result.output:
+            print("No results from parsing {}".format(project_dir))
+            return
+
+        actual_keys = _parse_output_keys(result.output)
+        expected_keys = _load_expected_keys(project_dir, actual_keys, raise_error=False)
+
+        for element_name in actual_keys:
+            expected = _element_filename(project_dir, element_name, "expected")
+
+            if actual_keys[element_name] != expected_keys[element_name]:
+                if not expected_keys[element_name]:
+                    print("Creating new expected file: {}".format(expected))
+                else:
+                    print("Updating expected file: {}".format(expected))
+
+                write_expected_key(project_dir, element_name, actual_keys[element_name])
+
+
+if __name__ == "__main__":
+    #  patch the environment BST_TEST_SUITE value to something if it's not
+    #  present. This avoids an exception thrown at the cli level
+    bst = "BST_TEST_SUITE"
+    mock_bst = os.environ.get(bst, "True")
+    with mock.patch.dict(os.environ, {**os.environ, bst: mock_bst}):
+        update_keys()
diff --git a/tests/cachekey/cachekey.py b/tests/cachekey/cachekey.py
index 08870f2a0..951972206 100644
--- a/tests/cachekey/cachekey.py
+++ b/tests/cachekey/cachekey.py
@@ -44,108 +44,13 @@ import os
 
 import pytest
 
+from buildstream.testing._cachekeys import check_cache_key_stability, _parse_output_keys
 from buildstream.testing.runcli import cli  # pylint: disable=unused-import
 from buildstream.testing._utils.site import HAVE_BZR, HAVE_GIT, IS_LINUX, MACHINE_ARCH
 from buildstream.plugin import CoreWarnings
 from buildstream import _yaml
 
 
-##############################################
-#                Some Helpers                #
-##############################################
-
-# Get whole filename in the temp project with
-# the option of changing the .bst suffix to something else
-#
-def element_filename(project_dir, element_name, alt_suffix=None):
-
-    if alt_suffix:
-
-        # Just in case...
-        assert element_name.endswith(".bst")
-
-        # Chop off the 'bst' in '.bst' and add the new suffix
-        element_name = element_name[:-3]
-        element_name = element_name + alt_suffix
-
-    return os.path.join(project_dir, element_name)
-
-
-# Returns an OrderedDict of element names
-# and their cache keys
-#
-def parse_output_keys(output):
-    actual_keys = OrderedDict()
-    lines = output.splitlines()
-    for line in lines:
-        split = line.split("::")
-        name = split[0]
-        key = split[1]
-        actual_keys[name] = key
-
-    return actual_keys
-
-
-# Returns an OrderedDict of element names
-# and their cache keys
-#
-def load_expected_keys(project_dir, actual_keys, raise_error=True):
-
-    expected_keys = OrderedDict()
-    for element_name in actual_keys:
-        expected = element_filename(project_dir, element_name, "expected")
-        try:
-            with open(expected, "r") as f:
-                expected_key = f.read()
-                expected_key = expected_key.strip()
-        except FileNotFoundError:
-            expected_key = None
-            if raise_error:
-                raise Exception(
-                    "Cache key test needs update, "
-                    + "expected file {} not found.\n\n".format(expected)
-                    + "Use tests/cachekey/update.py to automatically "
-                    + "update this test case"
-                )
-
-        expected_keys[element_name] = expected_key
-
-    return expected_keys
-
-
-def assert_cache_keys(project_dir, output):
-
-    # Read in the expected keys from the cache key test directory
-    # and parse the actual keys from the `bst show` output
-    #
-    actual_keys = parse_output_keys(output)
-    expected_keys = load_expected_keys(project_dir, actual_keys)
-    mismatches = []
-
-    for element_name in actual_keys:
-        if actual_keys[element_name] != expected_keys[element_name]:
-            mismatches.append(element_name)
-
-    if mismatches:
-        info = ""
-        for element_name in mismatches:
-            info += (
-                "  Element: {}\n".format(element_name)
-                + "    Expected: {}\n".format(expected_keys[element_name])
-                + "    Actual: {}\n".format(actual_keys[element_name])
-            )
-
-        raise AssertionError(
-            "Cache key mismatches occurred:\n{}\n".format(info)
-            + "Use tests/cachekey/update.py to automatically "
-            + "update this test case"
-        )
-
-
-##############################################
-#             Test Entry Point               #
-##############################################
-
 # Project directory
 DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
 
@@ -172,9 +77,7 @@ def test_cache_key(datafiles, cli):
     # https://github.com/omarkohl/pytest-datafiles/issues/11
     os.chmod(goodbye_link, 0o755)
 
-    result = cli.run(project=project, silent=True, args=["show", "--format", "%{name}::%{full-key}", "target.bst"])
-    result.assert_success()
-    assert_cache_keys(project, result.output)
+    check_cache_key_stability(project, cli)
 
 
 @pytest.mark.datafiles(DATA_DIR)
@@ -228,15 +131,15 @@ def test_keys_stable_over_targets(cli, datafiles):
     project = str(datafiles)
     full_graph_result = cli.run(project=project, args=["show", "--format", "%{name}::%{full-key}", root_element])
     full_graph_result.assert_success()
-    all_cache_keys = parse_output_keys(full_graph_result.output)
+    all_cache_keys = _parse_output_keys(full_graph_result.output)
 
     ordering1_result = cli.run(project=project, args=["show", "--format", "%{name}::%{full-key}", target1, target2])
     ordering1_result.assert_success()
-    ordering1_cache_keys = parse_output_keys(ordering1_result.output)
+    ordering1_cache_keys = _parse_output_keys(ordering1_result.output)
 
     ordering2_result = cli.run(project=project, args=["show", "--format", "%{name}::%{full-key}", target2, target1])
     ordering2_result.assert_success()
-    ordering2_cache_keys = parse_output_keys(ordering2_result.output)
+    ordering2_cache_keys = _parse_output_keys(ordering2_result.output)
 
     elements = ordering1_cache_keys.keys()
 
diff --git a/tests/cachekey/update.py b/tests/cachekey/update.py
deleted file mode 100755
index 49af39fe1..000000000
--- a/tests/cachekey/update.py
+++ /dev/null
@@ -1,76 +0,0 @@
-#!/usr/bin/env python3
-#
-# Automatically create or update the .expected files in the
-# cache key test directory.
-#
-# Simply run without any arguments, from anywhere, e.g.:
-#
-#   PYTHONPATH=. ./tests/cachekey/update.py
-#
-# After this, add any files which were newly created and commit
-# the result in order to adjust the cache key test to changed
-# keys.
-#
-import os
-import tempfile
-from unittest import mock
-from buildstream.testing.runcli import Cli
-
-# This weird try / except is needed, because this will be imported differently
-# when pytest runner imports them vs when you run the updater directly from
-# this directory.
-try:
-    from cachekey import element_filename, parse_output_keys, load_expected_keys
-except ImportError:
-    from .cachekey import element_filename, parse_output_keys, load_expected_keys
-
-# Project directory
-PROJECT_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project",)
-
-
-def write_expected_key(element_name, actual_key):
-    expected_file = element_filename(PROJECT_DIR, element_name, "expected")
-    with open(expected_file, "w") as f:
-        f.write(actual_key)
-
-
-def update_keys():
-
-    with tempfile.TemporaryDirectory(dir=PROJECT_DIR) as tmpdir:
-        directory = os.path.join(tmpdir, "cache")
-        os.makedirs(directory)
-        cli = Cli(directory, verbose=True)
-
-        # Run bst show
-        result = cli.run(
-            project=PROJECT_DIR,
-            silent=True,
-            args=["--no-colors", "show", "--format", "%{name}::%{full-key}", "target.bst"],
-        )
-
-        # Load the actual keys, and the expected ones if they exist
-        if not result.output:
-            print("No results from parsing {}:target.bst".format(PROJECT_DIR))
-            return
-        actual_keys = parse_output_keys(result.output)
-        expected_keys = load_expected_keys(PROJECT_DIR, actual_keys, raise_error=False)
-
-        for element_name in actual_keys:
-            expected = element_filename(PROJECT_DIR, element_name, "expected")
-
-            if actual_keys[element_name] != expected_keys[element_name]:
-                if not expected_keys[element_name]:
-                    print("Creating new expected file: {}".format(expected))
-                else:
-                    print("Updating expected file: {}".format(expected))
-
-                write_expected_key(element_name, actual_keys[element_name])
-
-
-if __name__ == "__main__":
-    #  patch the environment BST_TEST_SUITE value to something if it's not
-    #  present. This avoids an exception thrown at the cli level
-    bst = "BST_TEST_SUITE"
-    mock_bst = os.environ.get(bst, "True")
-    with mock.patch.dict(os.environ, {**os.environ, bst: mock_bst}):
-        update_keys()