Merge branch 'Qinusty/526-fail-on-warnings' into 'master'

Configurable Warnings

Closes #526

See merge request BuildStream/buildstream!627

44 files changed, 398 insertions, 83 deletions

diff --git a/buildstream/_project.py b/buildstream/_project.py
index 951662e2b..702fd81f4 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -31,6 +31,7 @@ from ._options import OptionPool
 from ._artifactcache import ArtifactCache
 from ._elementfactory import ElementFactory
 from ._sourcefactory import SourceFactory
+from .plugin import CoreWarnings
 from ._projectrefs import ProjectRefs, ProjectRefStorage
 from ._versions import BST_FORMAT_VERSION
 from ._loader import Loader
@@ -105,7 +106,7 @@ class Project():
         self.first_pass_config = ProjectConfig()
 
         self.junction = junction                 # The junction Element object, if this is a subproject
-        self.fail_on_overlap = False             # Whether overlaps are treated as errors
+
         self.ref_storage = None                  # ProjectRefStorage setting
         self.base_environment = {}               # The base set of environment variables
         self.base_env_nocache = None             # The base nocache mask (list) for the environment
@@ -120,6 +121,8 @@ class Project():
         self._cli_options = cli_options
         self._cache_key = None
 
+        self._fatal_warnings = []             # A list of warnings which should trigger an error
+
         self._shell_command = []      # The default interactive shell command
         self._shell_environment = {}  # Statically set environment vars
         self._shell_host_files = []   # A list of HostMount objects
@@ -456,7 +459,7 @@ class Project():
             'split-rules', 'elements', 'plugins',
             'aliases', 'name',
             'artifacts', 'options',
-            'fail-on-overlap', 'shell',
+            'fail-on-overlap', 'shell', 'fatal-warnings',
             'ref-storage', 'sandbox', 'mirrors'
         ])
 
@@ -478,8 +481,25 @@ class Project():
         # Load project split rules
         self._splits = _yaml.node_get(config, Mapping, 'split-rules')
 
-        # Fail on overlap
-        self.fail_on_overlap = _yaml.node_get(config, bool, 'fail-on-overlap')
+        # Fatal warnings
+        self._fatal_warnings = _yaml.node_get(config, list, 'fatal-warnings', default_value=[])
+
+        # Support backwards compatibility for fail-on-overlap
+        fail_on_overlap = _yaml.node_get(config, bool, 'fail-on-overlap', default_value=None)
+
+        if (CoreWarnings.OVERLAPS not in self._fatal_warnings) and fail_on_overlap:
+            self._fatal_warnings.append(CoreWarnings.OVERLAPS)
+
+        # Deprecation check
+        if fail_on_overlap is not None:
+            self._context.message(
+                Message(
+                    None,
+                    MessageType.WARN,
+                    "Use of fail-on-overlap within project.conf " +
+                    "is deprecated. Consider using fatal-warnings instead."
+                )
+            )
 
         # Load project.refs if it exists, this may be ignored.
         if self.ref_storage == ProjectRefStorage.PROJECT_REFS:
@@ -712,3 +732,17 @@ class Project():
                 # paths are passed in relative to the project, but must be absolute
                 origin_dict['path'] = os.path.join(self.directory, path)
             destination.append(origin_dict)
+
+    # _warning_is_fatal():
+    #
+    # Returns true if the warning in question should be considered fatal based on
+    # the project configuration.
+    #
+    # Args:
+    #   warning_str (str): The warning configuration string to check against
+    #
+    # Returns:
+    #    (bool): True if the warning should be considered fatal and cause an error.
+    #
+    def _warning_is_fatal(self, warning_str):
+        return warning_str in self._fatal_warnings
diff --git a/buildstream/_versions.py b/buildstream/_versions.py
index 3f32847ae..a722e11a5 100644
--- a/buildstream/_versions.py
+++ b/buildstream/_versions.py
@@ -23,7 +23,7 @@
 # This version is bumped whenever enhancements are made
 # to the `project.conf` format or the core element format.
 #
-BST_FORMAT_VERSION = 14
+BST_FORMAT_VERSION = 15
 
 
 # The base BuildStream artifact version
@@ -33,4 +33,4 @@ BST_FORMAT_VERSION = 14
 # or if buildstream was changed in a way which can cause
 # the same cache key to produce something that is no longer
 # the same.
-BST_CORE_ARTIFACT_VERSION = 4
+BST_CORE_ARTIFACT_VERSION = 5
diff --git a/buildstream/data/projectconfig.yaml b/buildstream/data/projectconfig.yaml
index b4ad2dcb9..686040ee0 100644
--- a/buildstream/data/projectconfig.yaml
+++ b/buildstream/data/projectconfig.yaml
@@ -13,10 +13,6 @@ element-path: .
 # Store source references in element files
 ref-storage: inline
 
-# Overlaps are just warnings
-fail-on-overlap: False
-
-
 # Variable Configuration
 #
 variables:
diff --git a/buildstream/element.py b/buildstream/element.py
index a099e934b..bb205c777 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -94,6 +94,7 @@ from . import _cachekey
 from . import _signals
 from . import _site
 from ._platform import Platform
+from .plugin import CoreWarnings
 from .sandbox._config import SandboxConfig
 
 from .storage.directory import Directory
@@ -746,32 +747,23 @@ class Element(Plugin):
                 ignored[dep.name] = result.ignored
 
         if overlaps:
-            overlap_error = overlap_warning = False
-            error_detail = warning_detail = "Staged files overwrite existing files in staging area:\n"
+            overlap_warning = False
+            warning_detail = "Staged files overwrite existing files in staging area:\n"
             for f, elements in overlaps.items():
-                overlap_error_elements = []
                 overlap_warning_elements = []
                 # The bottom item overlaps nothing
                 overlapping_elements = elements[1:]
                 for elm in overlapping_elements:
                     element = self.search(scope, elm)
-                    element_project = element._get_project()
                     if not element.__file_is_whitelisted(f):
-                        if element_project.fail_on_overlap:
-                            overlap_error_elements.append(elm)
-                            overlap_error = True
-                        else:
-                            overlap_warning_elements.append(elm)
-                            overlap_warning = True
+                        overlap_warning_elements.append(elm)
+                        overlap_warning = True
 
                 warning_detail += _overlap_error_detail(f, overlap_warning_elements, elements)
-                error_detail += _overlap_error_detail(f, overlap_error_elements, elements)
 
             if overlap_warning:
-                self.warn("Non-whitelisted overlaps detected", detail=warning_detail)
-            if overlap_error:
-                raise ElementError("Non-whitelisted overlaps detected and fail-on-overlaps is set",
-                                   detail=error_detail, reason="overlap-error")
+                self.warn("Non-whitelisted overlaps detected", detail=warning_detail,
+                          warning_token=CoreWarnings.OVERLAPS)
 
         if ignored:
             detail = "Not staging files which would replace non-empty directories:\n"
@@ -2054,9 +2046,7 @@ class Element(Plugin):
                 'cache': type(self.__artifacts).__name__
             }
 
-            # fail-on-overlap setting cannot affect elements without dependencies
-            if project.fail_on_overlap and dependencies:
-                self.__cache_key_dict['fail-on-overlap'] = True
+            self.__cache_key_dict['fatal-warnings'] = sorted(project._fatal_warnings)
 
         cache_key_dict = self.__cache_key_dict.copy()
         cache_key_dict['dependencies'] = dependencies
diff --git a/buildstream/plugin.py b/buildstream/plugin.py
index 836b60834..4514ee355 100644
--- a/buildstream/plugin.py
+++ b/buildstream/plugin.py
@@ -47,6 +47,23 @@ it is mandatory to implement the following abstract methods:
   Once all configuration has been loaded and preflight checks have passed,
   this method is used to inform the core of a plugin's unique configuration.
 
+Configurable Warnings
+---------------------
+Warnings raised through calling :func:`Plugin.warn() <buildstream.plugin.Plugin.warn>` can provide an optional
+parameter ``warning_token``, this will raise a :class:`PluginError` if the warning is configured as fatal within
+the project configuration.
+
+Configurable warnings will be prefixed with :func:`Plugin.get_kind() <buildstream.plugin.Plugin.get_kind>`
+within buildstream and must be prefixed as such in project configurations. For more detail on project configuration
+see :ref:`Configurable Warnings <configurable_warnings>`.
+
+It is important to document these warnings in your plugin documentation to allow users to make full use of them
+while configuring their projects.
+
+Example
+~~~~~~~
+If the :class:`git <buildstream.plugins.sources.git.GitSource>` plugin uses the warning ``"inconsistent-submodule"``
+then it could be referenced in project configuration as ``"git:inconsistent-submodule"``.
 
 Plugin Structure
 ----------------
@@ -166,7 +183,6 @@ class Plugin():
         # Infer the kind identifier
         modulename = type(self).__module__
         self.__kind = modulename.split('.')[-1]
-
         self.debug("Created: {}".format(self))
 
     def __del__(self):
@@ -473,14 +489,28 @@ class Plugin():
         """
         self.__message(MessageType.INFO, brief, detail=detail)
 
-    def warn(self, brief, *, detail=None):
-        """Print a warning message
+    def warn(self, brief, *, detail=None, warning_token=None):
+        """Print a warning message, checks warning_token against project configuration
 
         Args:
            brief (str): The brief message
            detail (str): An optional detailed message, can be multiline output
+           warning_token (str): An optional configurable warning assosciated with this warning,
+                                this will cause PluginError to be raised if this warning is configured as fatal.
+                                (*Since 1.4*)
+
+        Raises:
+           (:class:`.PluginError`): When warning_token is considered fatal by the project configuration
         """
-        self.__message(MessageType.WARN, brief, detail=detail)
+        if warning_token:
+            warning_token = _prefix_warning(self, warning_token)
+            brief = "[{}]: {}".format(warning_token, brief)
+            project = self._get_project()
+
+            if project._warning_is_fatal(warning_token):
+                raise PluginError(message="{}\n{}".format(brief, detail), reason=warning_token)
+
+        self.__message(MessageType.WARN, brief=brief, detail=detail)
 
     def log(self, brief, *, detail=None):
         """Log a message into the plugin's log file
@@ -709,6 +739,32 @@ class Plugin():
             return self.name
 
 
+class CoreWarnings():
+    """CoreWarnings()
+
+    Some common warnings which are raised by core functionalities within BuildStream are found in this class.
+    """
+
+    OVERLAPS = "overlaps"
+    """
+    This warning will be produced when buildstream detects an overlap on an element
+        which is not whitelisted. See :ref:`Overlap Whitelist <public_overlap_whitelist>`
+    """
+
+    REF_NOT_IN_TRACK = "ref-not-in-track"
+    """
+    This warning will be produced when a source is configured with a reference
+    which is found to be invalid based on the configured track
+    """
+
+
+__CORE_WARNINGS = [
+    value
+    for name, value in CoreWarnings.__dict__.items()
+    if not name.startswith("__")
+]
+
+
 # Hold on to a lookup table by counter of all instantiated plugins.
 # We use this to send the id back from child processes so we can lookup
 # corresponding element/source in the master process.
@@ -739,6 +795,23 @@ def _plugin_lookup(unique_id):
     return __PLUGINS_TABLE[unique_id]
 
 
+# _prefix_warning():
+#
+# Prefix a warning with the plugin kind. CoreWarnings are not prefixed.
+#
+# Args:
+#   plugin (Plugin): The plugin which raised the warning
+#   warning (str): The warning to prefix
+#
+# Returns:
+#    (str): A prefixed warning
+#
+def _prefix_warning(plugin, warning):
+    if any((warning is core_warning for core_warning in __CORE_WARNINGS)):
+        return warning
+    return "{}:{}".format(plugin.get_kind(), warning)
+
+
 # No need for unregister, WeakValueDictionary() will remove entries
 # in itself when the referenced plugins are garbage collected.
 def _plugin_register(plugin):
diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py
index cf2f649db..855e766d2 100644
--- a/buildstream/plugins/sources/git.py
+++ b/buildstream/plugins/sources/git.py
@@ -68,6 +68,12 @@ git - stage files from a git repository
        url: upstream:baz.git
        checkout: False
 
+**Configurable Warnings:**
+
+This plugin provides the following configurable warnings:
+
+- 'git:inconsistent-submodule' - A submodule was found to be missing from the underlying git repository.
+
 """
 
 import os
@@ -84,6 +90,9 @@ from buildstream import utils
 
 GIT_MODULES = '.gitmodules'
 
+# Warnings
+INCONSISTENT_SUBMODULE = "inconsistent-submodules"
+
 
 # Because of handling of submodules, we maintain a GitMirror
 # for the primary git source and also for each submodule it
@@ -283,7 +292,7 @@ class GitMirror(SourceFetcher):
                      "underlying git repository with `git submodule add`."
 
             self.source.warn("{}: Ignoring inconsistent submodule '{}'"
-                             .format(self.source, submodule), detail=detail)
+                             .format(self.source, submodule), detail=detail, warning_token=INCONSISTENT_SUBMODULE)
 
             return None
 
diff --git a/doc/source/format_project.rst b/doc/source/format_project.rst
index cf0970f77..b43e67005 100644
--- a/doc/source/format_project.rst
+++ b/doc/source/format_project.rst
@@ -126,22 +126,29 @@ following to your ``project.conf``:
    The ``ref-storage`` configuration is available since :ref:`format version 8 <project_format_version>`
 
 
-Fail on overlaps
-~~~~~~~~~~~~~~~~
-When multiple elements are staged, there's a possibility that different
-elements will try and stage different versions of the same file.
+.. _configurable_warnings:
 
-When ``fail-on-overlap`` is true, if an overlap is detected
-that hasn't been allowed by the element's
-:ref:`overlap whitelist<public_overlap_whitelist>`,
-then an error will be raised and the build will fail.
+Configurable Warnings
+~~~~~~~~~~~~~~~~~~~~~
+Warnings can be configured as fatal using the ``fatal-warnings`` configuration item.
+When a warning is configured as fatal, where a warning would usually be thrown instead an error will be thrown
+causing the build to fail.
 
-otherwise, a warning will be raised indicating which files had overlaps,
-and the order that the elements were overlapped.
+Individual warnings can be configured as fatal by setting ``fatal-warnings`` to a list of warnings.
 
 .. code:: yaml
 
-  fail-on-overlap: true
+  fatal-warnings:
+  - overlaps
+  - ref-not-in-track
+  - <plugin>:<warning>
+
+BuildStream provides a collection of :class:`Core Warnings <buildstream.plugin.CoreWarnings>` which may be raised
+by a variety of plugins. Other configurable warnings are plugin specific and should be noted within their individual documentation.
+
+.. note::
+
+  The ``fatal-warnings`` configuration is available since :ref:`format version 14 <project_format_version>`
 
 
 .. _project_source_aliases:
diff --git a/tests/cachekey/cachekey.py b/tests/cachekey/cachekey.py
index 36562fbf3..21beef8fb 100644
--- a/tests/cachekey/cachekey.py
+++ b/tests/cachekey/cachekey.py
@@ -37,7 +37,8 @@
 #
 from tests.testutils.runcli import cli
 from tests.testutils.site import HAVE_BZR, HAVE_GIT, HAVE_OSTREE, IS_LINUX
-
+from buildstream.plugin import CoreWarnings
+from buildstream import _yaml
 import os
 from collections import OrderedDict
 import pytest
@@ -128,7 +129,6 @@ def assert_cache_keys(project_dir, output):
                              "Use tests/cachekey/update.py to automatically " +
                              "update this test case")
 
-
 ##############################################
 #             Test Entry Point               #
 ##############################################
@@ -167,3 +167,47 @@ def test_cache_key(datafiles, cli):
     ])
     result.assert_success()
     assert_cache_keys(project, result.output)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("first_warnings, second_warnings, identical_keys", [
+    [[], [], True],
+    [[], [CoreWarnings.REF_NOT_IN_TRACK], False],
+    [[CoreWarnings.REF_NOT_IN_TRACK], [], False],
+    [[CoreWarnings.REF_NOT_IN_TRACK], [CoreWarnings.REF_NOT_IN_TRACK], True],
+    [[CoreWarnings.REF_NOT_IN_TRACK, CoreWarnings.OVERLAPS],
+        [CoreWarnings.OVERLAPS, CoreWarnings.REF_NOT_IN_TRACK], True],
+])
+def test_cache_key_fatal_warnings(cli, tmpdir, first_warnings, second_warnings, identical_keys):
+
+    # Builds project, Runs bst show, gathers cache keys
+    def run_get_cache_key(project_name, warnings):
+        config = {
+            'name': project_name,
+            'element-path': 'elements',
+            'fatal-warnings': warnings
+        }
+
+        project_dir = tmpdir.mkdir(project_name)
+        project_config_file = str(project_dir.join('project.conf'))
+        _yaml.dump(_yaml.node_sanitize(config), filename=project_config_file)
+
+        elem_dir = project_dir.mkdir('elements')
+        element_file = str(elem_dir.join('stack.bst'))
+        _yaml.dump({'kind': 'stack'}, filename=element_file)
+
+        result = cli.run(project=str(project_dir), args=[
+            'show',
+            '--format', '%{name}::%{full-key}',
+            'stack.bst'
+        ])
+        return result.output
+
+    # Returns true if all keys are identical
+    def compare_cache_keys(first_keys, second_keys):
+        return not any((x != y for x, y in zip(first_keys, second_keys)))
+
+    first_keys = run_get_cache_key("first", first_warnings)
+    second_keys = run_get_cache_key("second", second_warnings)
+
+    assert compare_cache_keys(first_keys, second_keys) == identical_keys
diff --git a/tests/cachekey/project/elements/build1.expected b/tests/cachekey/project/elements/build1.expected
index fe515806d..1b41b5e06 100644
--- a/tests/cachekey/project/elements/build1.expected
+++ b/tests/cachekey/project/elements/build1.expected
@@ -1 +1 @@
-594be3eb2a211f706557b156ec4b0ffb3ca256af35bcd0116b97fdb8c942d1c5
\ No newline at end of file
+90fa9c1b5334aac3bfd8956d92af0a91e2f2bbcbcac73861957165e577bc8768
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/build2.expected b/tests/cachekey/project/elements/build2.expected
index f28fe802c..83c2bb1b3 100644
--- a/tests/cachekey/project/elements/build2.expected
+++ b/tests/cachekey/project/elements/build2.expected
@@ -1 +1 @@
-5038d37bf1714180d160271e688ec1715d69666ca266ed5b1d880abdee36b310
\ No newline at end of file
+a62c29fe4e05820412e391430f61aa88a1a82e138de8cac726a9dc4fcd7ed8b9
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose1.expected b/tests/cachekey/project/elements/compose1.expected
index ec0b94943..ee82ebb6a 100644
--- a/tests/cachekey/project/elements/compose1.expected
+++ b/tests/cachekey/project/elements/compose1.expected
@@ -1 +1 @@
-e050172c2d445849a2716720814501f0e294b81d28be8cd911ee294291ec60d0
\ No newline at end of file
+4e4c719242aa45fed398cc2fb8936195a1fcae9326d808de7fee340ae48862ea
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose2.expected b/tests/cachekey/project/elements/compose2.expected
index fd738fb74..f7b2d396b 100644
--- a/tests/cachekey/project/elements/compose2.expected
+++ b/tests/cachekey/project/elements/compose2.expected
@@ -1 +1 @@
-ad985f243163aab22576fea63a0b89f7560e361326cb041872c568c5feaabf5c
\ No newline at end of file
+2fc1dd398f6c6f6e1d7ca48d88557e133d2130278882e14cd1105b15a600cd7a
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose3.expected b/tests/cachekey/project/elements/compose3.expected
index 24899fb04..e6339b175 100644
--- a/tests/cachekey/project/elements/compose3.expected
+++ b/tests/cachekey/project/elements/compose3.expected
@@ -1 +1 @@
-ec0fec2c821eb34286e9799a6e8d1f5587f161d1d653bd1dbe385602340d86ae
\ No newline at end of file
+4d7c9e2e1e8cfcc4b300a11693767f41f22c7829db9063dec10856328d03ccc3
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose4.expected b/tests/cachekey/project/elements/compose4.expected
index 8f7730a28..de299e892 100644
--- a/tests/cachekey/project/elements/compose4.expected
+++ b/tests/cachekey/project/elements/compose4.expected
@@ -1 +1 @@
-7cfe8f6161c00d8cf10114e7458f0b97eb003a41504ae301b24f45d48d42155b
\ No newline at end of file
+cad8f3b622f4a906f9fc3f5187a7703e2b17dfc550dd5a07479ca3ebffbd5c86
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose5.expected b/tests/cachekey/project/elements/compose5.expected
index 9653745ae..d7c4c3ef8 100644
--- a/tests/cachekey/project/elements/compose5.expected
+++ b/tests/cachekey/project/elements/compose5.expected
@@ -1 +1 @@
-7f016c3165f2de9161312b74f29513dff7dfdcba5ff8c6897beb5b123eaafd3d
\ No newline at end of file
+4fd21699827aa16da8d7a1525020f9fd45422f0431749510947ff472d76c1802
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import1.expected b/tests/cachekey/project/elements/import1.expected
index f28d7c036..bc9c1c93f 100644
--- a/tests/cachekey/project/elements/import1.expected
+++ b/tests/cachekey/project/elements/import1.expected
@@ -1 +1 @@
-41ce5a640fdfd7b6ce8a2c3fa1dde7983bc4df0e4c3ca926670118bae3c051fe
\ No newline at end of file
+aa443ea4607d7dd5a0c99646a1b827e3165862772fc1b26e20195aadd2ab8885
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import2.expected b/tests/cachekey/project/elements/import2.expected
index 3ec64e941..1013f5826 100644
--- a/tests/cachekey/project/elements/import2.expected
+++ b/tests/cachekey/project/elements/import2.expected
@@ -1 +1 @@
-d657db503460486a6de80d87f15c6b1fa84b0c4dacabed374acdc70172d4761d
\ No newline at end of file
+18ea6bbb968ca6945c8c2941650f447b8df58673be7270c967c8152730eff036
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import3.expected b/tests/cachekey/project/elements/import3.expected
index 09c525c75..9a8517728 100644
--- a/tests/cachekey/project/elements/import3.expected
+++ b/tests/cachekey/project/elements/import3.expected
@@ -1 +1 @@
-8de0293a6231dc661cf7229aa5a2f25abdf9a6d38ff70bd6f2562dae51ff05d3
\ No newline at end of file
+34ce4816b0307f0691302460367ab24b4e1f86e61c0e307e68bcd6833946c1f1
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/script1.expected b/tests/cachekey/project/elements/script1.expected
index b81856cd6..e36a3df70 100644
--- a/tests/cachekey/project/elements/script1.expected
+++ b/tests/cachekey/project/elements/script1.expected
@@ -1 +1 @@
-d0a6b7d29226b083c404d76d9551a0eee98753058580cd62901f8dfac06ca08d
\ No newline at end of file
+c48922b3d80d36e6d30bed7581aa1473a5e463321b3a19606b603c055d2b4be4
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/bzr1.expected b/tests/cachekey/project/sources/bzr1.expected
index 9b29c17fe..fecc86c74 100644
--- a/tests/cachekey/project/sources/bzr1.expected
+++ b/tests/cachekey/project/sources/bzr1.expected
@@ -1 +1 @@
-51415ebc7d72315c5c7704759025d6a9237e786bfe9c2bda8c51e15840c3470a
\ No newline at end of file
+ee271c8f469cd33330229d8dcc44e26f3480a9f47b55db46f42d1a396a94609e
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git1.expected b/tests/cachekey/project/sources/git1.expected
index 858cc2b7d..eab44b627 100644
--- a/tests/cachekey/project/sources/git1.expected
+++ b/tests/cachekey/project/sources/git1.expected
@@ -1 +1 @@
-ef9bd728a328301e0b819be7109761aacfb4c87092904306d4117c86f30478a4
\ No newline at end of file
+39fdc83c2760589c2577fb859cc617a8fdd7ac4cf113f9d4e5c723d70cae3c09
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git2.expected b/tests/cachekey/project/sources/git2.expected
index 31d291b6c..868439c9b 100644
--- a/tests/cachekey/project/sources/git2.expected
+++ b/tests/cachekey/project/sources/git2.expected
@@ -1 +1 @@
-a818930895e164bd342ab786061f4d521b27a4470791f55cc28732fdf92794de
\ No newline at end of file
+8bac8c7d3b8bbd264083db8e6f3aa8894625af5396bbe62589d1ab726a87cccd
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local1.expected b/tests/cachekey/project/sources/local1.expected
index f28d7c036..bc9c1c93f 100644
--- a/tests/cachekey/project/sources/local1.expected
+++ b/tests/cachekey/project/sources/local1.expected
@@ -1 +1 @@
-41ce5a640fdfd7b6ce8a2c3fa1dde7983bc4df0e4c3ca926670118bae3c051fe
\ No newline at end of file
+aa443ea4607d7dd5a0c99646a1b827e3165862772fc1b26e20195aadd2ab8885
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local2.expected b/tests/cachekey/project/sources/local2.expected
index f1954ac6b..610d3fe5d 100644
--- a/tests/cachekey/project/sources/local2.expected
+++ b/tests/cachekey/project/sources/local2.expected
@@ -1 +1 @@
-bf3ceaa62d472c10ce1e991e5b07a81ddb5206e043d39e60167292910e6bd31e
\ No newline at end of file
+51296c83a0d5989a67f40391afcbf420cbbd76c1e6c07aa43fe2aef2e88941e3
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/ostree1.expected b/tests/cachekey/project/sources/ostree1.expected
index 6e0aa34ff..891bea41b 100644
--- a/tests/cachekey/project/sources/ostree1.expected
+++ b/tests/cachekey/project/sources/ostree1.expected
@@ -1 +1 @@
-c9dcee5ad0822df19984ba68e2a2068266427ee583e3dd3265f85f0515cf7510
\ No newline at end of file
+cd8b506c38c116d6bea8999720a82afb8844453d5ad05385302eabc7d858859c
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch1.expected b/tests/cachekey/project/sources/patch1.expected
index ec306edfd..76c2a2a91 100644
--- a/tests/cachekey/project/sources/patch1.expected
+++ b/tests/cachekey/project/sources/patch1.expected
@@ -1 +1 @@
-96f2cf27bb2145290fc85b8cbeb7193738a6d22e1328c384bca7ba1e8754d0fd
\ No newline at end of file
+bf2fe787df6f263cfd7dbd4aa91909af4186e252da722c3d2e2383533fbc7057
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch2.expected b/tests/cachekey/project/sources/patch2.expected
index 57ee7f0a4..ea97daf87 100644
--- a/tests/cachekey/project/sources/patch2.expected
+++ b/tests/cachekey/project/sources/patch2.expected
@@ -1 +1 @@
-bc7cca687951fbaecb365f39fc9d9d7b0fd509b861c77de71f3639e6f49e25fd
\ No newline at end of file
+8f040542ebb9b1c690af99d2db4ffc0a54cb13868a364af4771d19615b9e3b02
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch3.expected b/tests/cachekey/project/sources/patch3.expected
index 62b7b3bd7..96e7d4a13 100644
--- a/tests/cachekey/project/sources/patch3.expected
+++ b/tests/cachekey/project/sources/patch3.expected
@@ -1 +1 @@
-411a041ef82eb9021b308dbc873328e4cc8774c9b8aa0901ff865764bdf82c51
\ No newline at end of file
+fb3985a6527f2498974ffa418cc6832b716d9862c713d7b8dc1c22df45857ee5
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote1.expected b/tests/cachekey/project/sources/remote1.expected
index 0dfd062aa..84b807b42 100644
--- a/tests/cachekey/project/sources/remote1.expected
+++ b/tests/cachekey/project/sources/remote1.expected
@@ -1 +1 @@
-f51945f0a4fae358c4587669a725417d969f04ee2c3e5c3440e07492b1c6de77
\ No newline at end of file
+1fe04362ce6b1e65a0907749a8b11dd2838b2505d2f0c7fee01c005bd43cd63a
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote2.expected b/tests/cachekey/project/sources/remote2.expected
index 071c443d6..31e849a82 100644
--- a/tests/cachekey/project/sources/remote2.expected
+++ b/tests/cachekey/project/sources/remote2.expected
@@ -1 +1 @@
-f8e6e83e39edc14225e8680852fa1fdb1fc093bdd552e99911006b4329f43293
\ No newline at end of file
+105c814f5c88c72e2681a39e1b01a0f2009342afa2b1c1a21c1cc4a664eced29
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar1.expected b/tests/cachekey/project/sources/tar1.expected
index 07f541a7f..7df0789f7 100644
--- a/tests/cachekey/project/sources/tar1.expected
+++ b/tests/cachekey/project/sources/tar1.expected
@@ -1 +1 @@
-a907dfad1c12f6303e7ed20896e49ba3fd5ef566777e7b47098116ec03e0e1f4
\ No newline at end of file
+29331729ccb0f67507f9b1a315410d794d62bda6d16ee1fabd927a39808265a7
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar2.expected b/tests/cachekey/project/sources/tar2.expected
index 3da7239f3..044625e7d 100644
--- a/tests/cachekey/project/sources/tar2.expected
+++ b/tests/cachekey/project/sources/tar2.expected
@@ -1 +1 @@
-99865afccb0926ba5bbaa24e0ded7d8353b56fe499511ad6a809580d17abd80e
\ No newline at end of file
+37e135a6a6270245ef0fcfda96cada821095f819b57e701f635e83a6d47b83a9
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/zip1.expected b/tests/cachekey/project/sources/zip1.expected
index e15ed0ac2..c4aca98c4 100644
--- a/tests/cachekey/project/sources/zip1.expected
+++ b/tests/cachekey/project/sources/zip1.expected
@@ -1 +1 @@
-23ac1cc41c6a72214b3e59664fe0ef85d909013befd8afde13cf8877510579e4
\ No newline at end of file
+5834df0bd373aebd3e74fe57534dfbefbad02a0bfc391ea9b67a6c7c63823ba0
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/zip2.expected b/tests/cachekey/project/sources/zip2.expected
index 5208381c1..220f321d9 100644
--- a/tests/cachekey/project/sources/zip2.expected
+++ b/tests/cachekey/project/sources/zip2.expected
@@ -1 +1 @@
-be26d9222bf53589686861ce21391548dd5d6284fdd003ff8a7e39601d6e8bef
\ No newline at end of file
+7f8bb32b8fd8526c1909fbb239706abd7d1ab96911f17eb9f6e96a6f55833c04
\ No newline at end of file
diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected
index 232d71098..d6d99895a 100644
--- a/tests/cachekey/project/target.expected
+++ b/tests/cachekey/project/target.expected
@@ -1 +1 @@
-b11fe7a9883cee2846a6148703b53785dd031482b359ca56d1ff22b08a1b5cd2
\ No newline at end of file
+e045df890262f7b7c663e64ad0bfba428d9d80ec514df3ddb13313d4ef669b73
\ No newline at end of file
diff --git a/tests/frontend/configurable_warnings.py b/tests/frontend/configurable_warnings.py
new file mode 100644
index 000000000..e8a7b6ac8
--- /dev/null
+++ b/tests/frontend/configurable_warnings.py
@@ -0,0 +1,63 @@
+import pytest
+import os
+
+from buildstream.plugin import CoreWarnings
+from buildstream._exceptions import ErrorDomain, LoadErrorReason
+from buildstream import _yaml
+from tests.testutils.runcli import cli
+
+TOP_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    "configuredwarning"
+)
+
+
+def get_project(fatal_warnings):
+    return {
+        "name": "test",
+        "element-path": "elements",
+        "plugins": [
+            {
+                "origin": "local",
+                "path": "plugins",
+                "elements": {
+                    "warninga": 0,
+                    "warningb": 0,
+                    "corewarn": 0,
+                }
+            }
+        ],
+        "fatal-warnings": fatal_warnings
+    }
+
+
+def build_project(datafiles, fatal_warnings):
+    project_path = os.path.join(datafiles.dirname, datafiles.basename)
+
+    project = get_project(fatal_warnings)
+
+    _yaml.dump(project, os.path.join(project_path, "project.conf"))
+
+    return project_path
+
+
+@pytest.mark.datafiles(TOP_DIR)
+@pytest.mark.parametrize("element_name, fatal_warnings, expect_fatal, error_domain", [
+    ("corewarn.bst", [CoreWarnings.OVERLAPS], True, ErrorDomain.STREAM),
+    ("warninga.bst", ["warninga:warning-a"], True, ErrorDomain.STREAM),
+    ("warningb.bst", ["warningb:warning-b"], True, ErrorDomain.STREAM),
+    ("corewarn.bst", [], False, None),
+    ("warninga.bst", [], False, None),
+    ("warningb.bst", [], False, None),
+    ("warninga.bst", [CoreWarnings.OVERLAPS], False, None),
+    ("warningb.bst", [CoreWarnings.OVERLAPS], False, None),
+])
+def test_fatal_warnings(cli, datafiles, element_name,
+                        fatal_warnings, expect_fatal, error_domain):
+    project_path = build_project(datafiles, fatal_warnings)
+
+    result = cli.run(project=project_path, args=["build", element_name])
+    if expect_fatal:
+        result.assert_main_error(error_domain, None, "Expected fatal execution")
+    else:
+        result.assert_success("Unexpected fatal execution")
diff --git a/tests/frontend/configuredwarning/elements/corewarn.bst b/tests/frontend/configuredwarning/elements/corewarn.bst
new file mode 100644
index 000000000..6eac4e07e
--- /dev/null
+++ b/tests/frontend/configuredwarning/elements/corewarn.bst
@@ -0,0 +1 @@
+kind: corewarn
\ No newline at end of file
diff --git a/tests/frontend/configuredwarning/elements/warninga.bst b/tests/frontend/configuredwarning/elements/warninga.bst
new file mode 100644
index 000000000..43c3458a8
--- /dev/null
+++ b/tests/frontend/configuredwarning/elements/warninga.bst
@@ -0,0 +1 @@
+kind: warninga
diff --git a/tests/frontend/configuredwarning/elements/warningb.bst b/tests/frontend/configuredwarning/elements/warningb.bst
new file mode 100644
index 000000000..2def6c5a4
--- /dev/null
+++ b/tests/frontend/configuredwarning/elements/warningb.bst
@@ -0,0 +1 @@
+kind: warningb
diff --git a/tests/frontend/configuredwarning/plugins/corewarn.py b/tests/frontend/configuredwarning/plugins/corewarn.py
new file mode 100644
index 000000000..1f263a0ce
--- /dev/null
+++ b/tests/frontend/configuredwarning/plugins/corewarn.py
@@ -0,0 +1,27 @@
+from buildstream import Element
+from buildstream.plugin import CoreWarnings
+
+
+class CoreWarn(Element):
+    def configure(self, node):
+        pass
+
+    def preflight(self):
+        pass
+
+    def get_unique_key(self):
+        pass
+
+    def configure_sandbox(self, sandbox):
+        pass
+
+    def stage(self, sandbox):
+        pass
+
+    def assemble(self, sandbox):
+        self.warn("Testing: CoreWarning produced during assemble",
+                  warning_token=CoreWarnings.OVERLAPS)
+
+
+def setup():
+    return CoreWarn
diff --git a/tests/frontend/configuredwarning/plugins/warninga.py b/tests/frontend/configuredwarning/plugins/warninga.py
new file mode 100644
index 000000000..9fd8dc61b
--- /dev/null
+++ b/tests/frontend/configuredwarning/plugins/warninga.py
@@ -0,0 +1,27 @@
+from buildstream import Element
+
+WARNING_A = "warning-a"
+
+
+class WarningA(Element):
+    def configure(self, node):
+        pass
+
+    def preflight(self):
+        pass
+
+    def get_unique_key(self):
+        pass
+
+    def configure_sandbox(self, sandbox):
+        pass
+
+    def stage(self, sandbox):
+        pass
+
+    def assemble(self, sandbox):
+        self.warn("Testing: warning-a produced during assemble", warning_token=WARNING_A)
+
+
+def setup():
+    return WarningA
diff --git a/tests/frontend/configuredwarning/plugins/warningb.py b/tests/frontend/configuredwarning/plugins/warningb.py
new file mode 100644
index 000000000..64d25ef39
--- /dev/null
+++ b/tests/frontend/configuredwarning/plugins/warningb.py
@@ -0,0 +1,27 @@
+from buildstream import Element
+
+WARNING_B = "warning-b"
+
+
+class WarningB(Element):
+    def configure(self, node):
+        pass
+
+    def preflight(self):
+        pass
+
+    def get_unique_key(self):
+        pass
+
+    def configure_sandbox(self, sandbox):
+        pass
+
+    def stage(self, sandbox):
+        pass
+
+    def assemble(self, sandbox):
+        self.warn("Testing: warning-b produced during assemble", warning_token=WARNING_B)
+
+
+def setup():
+    return WarningB
diff --git a/tests/frontend/configuredwarning/project.conf b/tests/frontend/configuredwarning/project.conf
new file mode 100644
index 000000000..c73d217b8
--- /dev/null
+++ b/tests/frontend/configuredwarning/project.conf
@@ -0,0 +1,8 @@
+name: test
+element-path: elements
+plugins:
+- origin: local
+  path: element_plugins
+  elements:
+    warninga: 0
+    warningb: 0
diff --git a/tests/frontend/overlaps.py b/tests/frontend/overlaps.py
index 36e15acd7..27be8deac 100644
--- a/tests/frontend/overlaps.py
+++ b/tests/frontend/overlaps.py
@@ -3,6 +3,7 @@ import pytest
 from tests.testutils.runcli import cli
 from buildstream._exceptions import ErrorDomain
 from buildstream import _yaml
+from buildstream.plugin import CoreWarnings
 
 # Project directory
 DATA_DIR = os.path.join(
@@ -16,30 +17,35 @@ project_template = {
 }
 
 
-def gen_project(project_dir, fail_on_overlap):
+def gen_project(project_dir, fail_on_overlap, use_fatal_warnings=True):
     template = dict(project_template)
-    template["fail-on-overlap"] = fail_on_overlap
+    if use_fatal_warnings:
+        template["fatal-warnings"] = [CoreWarnings.OVERLAPS] if fail_on_overlap else []
+    else:
+        template["fail-on-overlap"] = fail_on_overlap
     projectfile = os.path.join(project_dir, "project.conf")
     _yaml.dump(template, projectfile)
 
 
 @pytest.mark.datafiles(DATA_DIR)
-def test_overlaps(cli, datafiles):
+@pytest.mark.parametrize("use_fatal_warnings", [True, False])
+def test_overlaps(cli, datafiles, use_fatal_warnings):
     project_dir = str(datafiles)
-    gen_project(project_dir, False)
+    gen_project(project_dir, False, use_fatal_warnings)
     result = cli.run(project=project_dir, silent=True, args=[
         'build', 'collect.bst'])
     result.assert_success()
 
 
 @pytest.mark.datafiles(DATA_DIR)
-def test_overlaps_error(cli, datafiles):
+@pytest.mark.parametrize("use_fatal_warnings", [True, False])
+def test_overlaps_error(cli, datafiles, use_fatal_warnings):
     project_dir = str(datafiles)
-    gen_project(project_dir, True)
+    gen_project(project_dir, True, use_fatal_warnings)
     result = cli.run(project=project_dir, silent=True, args=[
         'build', 'collect.bst'])
     result.assert_main_error(ErrorDomain.STREAM, None)
-    result.assert_task_error(ErrorDomain.ELEMENT, "overlap-error")
+    result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.OVERLAPS)
 
 
 @pytest.mark.datafiles(DATA_DIR)
@@ -70,15 +76,16 @@ def test_overlaps_whitelist_on_overlapper(cli, datafiles):
     result = cli.run(project=project_dir, silent=True, args=[
         'build', 'collect-partially-whitelisted.bst'])
     result.assert_main_error(ErrorDomain.STREAM, None)
-    result.assert_task_error(ErrorDomain.ELEMENT, "overlap-error")
+    result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.OVERLAPS)
 
 
 @pytest.mark.datafiles(DATA_DIR)
-def test_overlaps_script(cli, datafiles):
+@pytest.mark.parametrize("use_fatal_warnings", [True, False])
+def test_overlaps_script(cli, datafiles, use_fatal_warnings):
     # Test overlaps with script element to test
     # Element.stage_dependency_artifacts() with Scope.RUN
     project_dir = str(datafiles)
-    gen_project(project_dir, False)
+    gen_project(project_dir, False, use_fatal_warnings)
     result = cli.run(project=project_dir, silent=True, args=[
         'build', 'script.bst'])
     result.assert_success()