tests: Add tests for configurable warnings

This adds multiple tests for custom plugin warnings and core warnings,
providing checks for both cases which should cause warnings and errors
when configured as fatal.

Also adds tests for cache key calculations.

9 files changed, 201 insertions, 2 deletions

diff --git a/tests/cachekey/cachekey.py b/tests/cachekey/cachekey.py
index 36562fbf3..21beef8fb 100644
--- a/tests/cachekey/cachekey.py
+++ b/tests/cachekey/cachekey.py
@@ -37,7 +37,8 @@
 #
 from tests.testutils.runcli import cli
 from tests.testutils.site import HAVE_BZR, HAVE_GIT, HAVE_OSTREE, IS_LINUX
-
+from buildstream.plugin import CoreWarnings
+from buildstream import _yaml
 import os
 from collections import OrderedDict
 import pytest
@@ -128,7 +129,6 @@ def assert_cache_keys(project_dir, output):
                              "Use tests/cachekey/update.py to automatically " +
                              "update this test case")
 
-
 ##############################################
 #             Test Entry Point               #
 ##############################################
@@ -167,3 +167,47 @@ def test_cache_key(datafiles, cli):
     ])
     result.assert_success()
     assert_cache_keys(project, result.output)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("first_warnings, second_warnings, identical_keys", [
+    [[], [], True],
+    [[], [CoreWarnings.REF_NOT_IN_TRACK], False],
+    [[CoreWarnings.REF_NOT_IN_TRACK], [], False],
+    [[CoreWarnings.REF_NOT_IN_TRACK], [CoreWarnings.REF_NOT_IN_TRACK], True],
+    [[CoreWarnings.REF_NOT_IN_TRACK, CoreWarnings.OVERLAPS],
+        [CoreWarnings.OVERLAPS, CoreWarnings.REF_NOT_IN_TRACK], True],
+])
+def test_cache_key_fatal_warnings(cli, tmpdir, first_warnings, second_warnings, identical_keys):
+
+    # Builds project, Runs bst show, gathers cache keys
+    def run_get_cache_key(project_name, warnings):
+        config = {
+            'name': project_name,
+            'element-path': 'elements',
+            'fatal-warnings': warnings
+        }
+
+        project_dir = tmpdir.mkdir(project_name)
+        project_config_file = str(project_dir.join('project.conf'))
+        _yaml.dump(_yaml.node_sanitize(config), filename=project_config_file)
+
+        elem_dir = project_dir.mkdir('elements')
+        element_file = str(elem_dir.join('stack.bst'))
+        _yaml.dump({'kind': 'stack'}, filename=element_file)
+
+        result = cli.run(project=str(project_dir), args=[
+            'show',
+            '--format', '%{name}::%{full-key}',
+            'stack.bst'
+        ])
+        return result.output
+
+    # Returns true if all keys are identical
+    def compare_cache_keys(first_keys, second_keys):
+        return not any((x != y for x, y in zip(first_keys, second_keys)))
+
+    first_keys = run_get_cache_key("first", first_warnings)
+    second_keys = run_get_cache_key("second", second_warnings)
+
+    assert compare_cache_keys(first_keys, second_keys) == identical_keys
diff --git a/tests/frontend/configurable_warnings.py b/tests/frontend/configurable_warnings.py
new file mode 100644
index 000000000..e8a7b6ac8
--- /dev/null
+++ b/tests/frontend/configurable_warnings.py
@@ -0,0 +1,63 @@
+import pytest
+import os
+
+from buildstream.plugin import CoreWarnings
+from buildstream._exceptions import ErrorDomain, LoadErrorReason
+from buildstream import _yaml
+from tests.testutils.runcli import cli
+
+TOP_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    "configuredwarning"
+)
+
+
+def get_project(fatal_warnings):
+    return {
+        "name": "test",
+        "element-path": "elements",
+        "plugins": [
+            {
+                "origin": "local",
+                "path": "plugins",
+                "elements": {
+                    "warninga": 0,
+                    "warningb": 0,
+                    "corewarn": 0,
+                }
+            }
+        ],
+        "fatal-warnings": fatal_warnings
+    }
+
+
+def build_project(datafiles, fatal_warnings):
+    project_path = os.path.join(datafiles.dirname, datafiles.basename)
+
+    project = get_project(fatal_warnings)
+
+    _yaml.dump(project, os.path.join(project_path, "project.conf"))
+
+    return project_path
+
+
+@pytest.mark.datafiles(TOP_DIR)
+@pytest.mark.parametrize("element_name, fatal_warnings, expect_fatal, error_domain", [
+    ("corewarn.bst", [CoreWarnings.OVERLAPS], True, ErrorDomain.STREAM),
+    ("warninga.bst", ["warninga:warning-a"], True, ErrorDomain.STREAM),
+    ("warningb.bst", ["warningb:warning-b"], True, ErrorDomain.STREAM),
+    ("corewarn.bst", [], False, None),
+    ("warninga.bst", [], False, None),
+    ("warningb.bst", [], False, None),
+    ("warninga.bst", [CoreWarnings.OVERLAPS], False, None),
+    ("warningb.bst", [CoreWarnings.OVERLAPS], False, None),
+])
+def test_fatal_warnings(cli, datafiles, element_name,
+                        fatal_warnings, expect_fatal, error_domain):
+    project_path = build_project(datafiles, fatal_warnings)
+
+    result = cli.run(project=project_path, args=["build", element_name])
+    if expect_fatal:
+        result.assert_main_error(error_domain, None, "Expected fatal execution")
+    else:
+        result.assert_success("Unexpected fatal execution")
diff --git a/tests/frontend/configuredwarning/elements/corewarn.bst b/tests/frontend/configuredwarning/elements/corewarn.bst
new file mode 100644
index 000000000..6eac4e07e
--- /dev/null
+++ b/tests/frontend/configuredwarning/elements/corewarn.bst
@@ -0,0 +1 @@
+kind: corewarn
\ No newline at end of file
diff --git a/tests/frontend/configuredwarning/elements/warninga.bst b/tests/frontend/configuredwarning/elements/warninga.bst
new file mode 100644
index 000000000..43c3458a8
--- /dev/null
+++ b/tests/frontend/configuredwarning/elements/warninga.bst
@@ -0,0 +1 @@
+kind: warninga
diff --git a/tests/frontend/configuredwarning/elements/warningb.bst b/tests/frontend/configuredwarning/elements/warningb.bst
new file mode 100644
index 000000000..2def6c5a4
--- /dev/null
+++ b/tests/frontend/configuredwarning/elements/warningb.bst
@@ -0,0 +1 @@
+kind: warningb
diff --git a/tests/frontend/configuredwarning/plugins/corewarn.py b/tests/frontend/configuredwarning/plugins/corewarn.py
new file mode 100644
index 000000000..1f263a0ce
--- /dev/null
+++ b/tests/frontend/configuredwarning/plugins/corewarn.py
@@ -0,0 +1,27 @@
+from buildstream import Element
+from buildstream.plugin import CoreWarnings
+
+
+class CoreWarn(Element):
+    def configure(self, node):
+        pass
+
+    def preflight(self):
+        pass
+
+    def get_unique_key(self):
+        pass
+
+    def configure_sandbox(self, sandbox):
+        pass
+
+    def stage(self, sandbox):
+        pass
+
+    def assemble(self, sandbox):
+        self.warn("Testing: CoreWarning produced during assemble",
+                  warning_token=CoreWarnings.OVERLAPS)
+
+
+def setup():
+    return CoreWarn
diff --git a/tests/frontend/configuredwarning/plugins/warninga.py b/tests/frontend/configuredwarning/plugins/warninga.py
new file mode 100644
index 000000000..9fd8dc61b
--- /dev/null
+++ b/tests/frontend/configuredwarning/plugins/warninga.py
@@ -0,0 +1,27 @@
+from buildstream import Element
+
+WARNING_A = "warning-a"
+
+
+class WarningA(Element):
+    def configure(self, node):
+        pass
+
+    def preflight(self):
+        pass
+
+    def get_unique_key(self):
+        pass
+
+    def configure_sandbox(self, sandbox):
+        pass
+
+    def stage(self, sandbox):
+        pass
+
+    def assemble(self, sandbox):
+        self.warn("Testing: warning-a produced during assemble", warning_token=WARNING_A)
+
+
+def setup():
+    return WarningA
diff --git a/tests/frontend/configuredwarning/plugins/warningb.py b/tests/frontend/configuredwarning/plugins/warningb.py
new file mode 100644
index 000000000..64d25ef39
--- /dev/null
+++ b/tests/frontend/configuredwarning/plugins/warningb.py
@@ -0,0 +1,27 @@
+from buildstream import Element
+
+WARNING_B = "warning-b"
+
+
+class WarningB(Element):
+    def configure(self, node):
+        pass
+
+    def preflight(self):
+        pass
+
+    def get_unique_key(self):
+        pass
+
+    def configure_sandbox(self, sandbox):
+        pass
+
+    def stage(self, sandbox):
+        pass
+
+    def assemble(self, sandbox):
+        self.warn("Testing: warning-b produced during assemble", warning_token=WARNING_B)
+
+
+def setup():
+    return WarningB
diff --git a/tests/frontend/configuredwarning/project.conf b/tests/frontend/configuredwarning/project.conf
new file mode 100644
index 000000000..c73d217b8
--- /dev/null
+++ b/tests/frontend/configuredwarning/project.conf
@@ -0,0 +1,8 @@
+name: test
+element-path: elements
+plugins:
+- origin: local
+  path: element_plugins
+  elements:
+    warninga: 0
+    warningb: 0