diff options
| author | Tristan Van Berkom <tristan.van.berkom@gmail.com> | 2019-08-03 23:01:09 +0000 |
|---|---|---|
| committer | Tristan Van Berkom <tristan.van.berkom@gmail.com> | 2019-08-03 23:01:09 +0000 |
| commit | 967b34ab6c55f9e985e9216d4a8536c1f4c89007 (patch) | |
| tree | 809d02453539d2a906673f0e090fb4cafc58435c | |
| parent | 78caec9604729d9dcd1d7108166fb0af6ecb1867 (diff) | |
| parent | acd2e88d5902a1f3a82c481626d91efacc030fd0 (diff) | |
| download | buildstream-967b34ab6c55f9e985e9216d4a8536c1f4c89007.tar.gz | |
Merge branch 'tristan/bst-1/fatal-warnings' into 'bst-1'
Fatal warnings support (bst 1)
See merge request BuildStream/buildstream!1512
44 files changed, 443 insertions, 90 deletions
@@ -9,6 +9,21 @@ buildstream 1.3.2 o Elements may now specify cross-junction dependencies as simple strings using the format '{junction-name}:{element-name}'. + o New `fatal-warnings` has been added to the project.conf format, allowing + projects to specify which warnings they want to consider as fatal. + + Support for the following warnings is included: + + o overlaps: When staged artifact files overlap + (deprecates: 'fail-on-overlap') + + o ref-not-in-track: When the source implementation finds that + the ref is out of bounds for the tracking config + + o git:inconsistent-submodule: A .gitmodules file is present but the + submodule was never added to the repo. + + ================= buildstream 1.3.1 ================= @@ -23,13 +38,6 @@ buildstream 1.3.1 o Add new `pip` source plugin for downloading python packages using pip, based on requirements files from previous sources. - o Elements may now specify 'build-depends' and 'runtime-depends' fields - to avoid having to specify the dependency type for every entry in - 'depends'. - - o Elements may now specify cross-junction dependencies as simple strings - using the format '{junction-name}:{element-name}'. - ================= buildstream 1.2.8 ================= diff --git a/buildstream/_project.py b/buildstream/_project.py index 1660c4770..6e4f09c68 100644 --- a/buildstream/_project.py +++ b/buildstream/_project.py @@ -31,6 +31,7 @@ from ._options import OptionPool from ._artifactcache import ArtifactCache from ._elementfactory import ElementFactory from ._sourcefactory import SourceFactory +from .plugin import CoreWarnings from ._projectrefs import ProjectRefs, ProjectRefStorage from ._versions import BST_FORMAT_VERSION from ._loader import Loader @@ -105,7 +106,7 @@ class Project(): self.first_pass_config = ProjectConfig() self.junction = junction # The junction Element object, if this is a subproject - self.fail_on_overlap = False # Whether overlaps are treated as errors + self.ref_storage = None # ProjectRefStorage setting self.base_environment = {} # The base set of environment variables self.base_env_nocache = None # The base nocache mask (list) for the environment @@ -120,6 +121,8 @@ class Project(): self._cli_options = cli_options self._cache_key = None + self._fatal_warnings = [] # A list of warnings which should trigger an error + self._shell_command = [] # The default interactive shell command self._shell_environment = {} # Statically set environment vars self._shell_host_files = [] # A list of HostMount objects @@ -470,7 +473,7 @@ class Project(): 'split-rules', 'elements', 'plugins', 'aliases', 'name', 'artifacts', 'options', - 'fail-on-overlap', 'shell', + 'fail-on-overlap', 'shell', 'fatal-warnings', 'ref-storage', 'sandbox', 'mirrors' ]) @@ -492,8 +495,25 @@ class Project(): # Load project split rules self._splits = _yaml.node_get(config, Mapping, 'split-rules') - # Fail on overlap - self.fail_on_overlap = _yaml.node_get(config, bool, 'fail-on-overlap') + # Fatal warnings + self._fatal_warnings = _yaml.node_get(config, list, 'fatal-warnings', default_value=[]) + + # Support backwards compatibility for fail-on-overlap + fail_on_overlap = _yaml.node_get(config, bool, 'fail-on-overlap', default_value=None) + + if (CoreWarnings.OVERLAPS not in self._fatal_warnings) and fail_on_overlap: + self._fatal_warnings.append(CoreWarnings.OVERLAPS) + + # Deprecation check + if fail_on_overlap is not None: + self._context.message( + Message( + None, + MessageType.WARN, + "Use of fail-on-overlap within project.conf " + + "is deprecated. Consider using fatal-warnings instead." + ) + ) # Load project.refs if it exists, this may be ignored. if self.ref_storage == ProjectRefStorage.PROJECT_REFS: @@ -734,3 +754,17 @@ class Project(): # paths are passed in relative to the project, but must be absolute origin_dict['path'] = os.path.join(self.directory, path) destination.append(origin_dict) + + # _warning_is_fatal(): + # + # Returns true if the warning in question should be considered fatal based on + # the project configuration. + # + # Args: + # warning_str (str): The warning configuration string to check against + # + # Returns: + # (bool): True if the warning should be considered fatal and cause an error. + # + def _warning_is_fatal(self, warning_str): + return warning_str in self._fatal_warnings diff --git a/buildstream/_versions.py b/buildstream/_versions.py index e0ec4cdec..e4d503929 100644 --- a/buildstream/_versions.py +++ b/buildstream/_versions.py @@ -23,7 +23,7 @@ # This version is bumped whenever enhancements are made # to the `project.conf` format or the core element format. # -BST_FORMAT_VERSION = 15 +BST_FORMAT_VERSION = 16 # The base BuildStream artifact version @@ -34,4 +34,4 @@ BST_FORMAT_VERSION = 15 # the same cache key to produce something that is no longer # the same. -BST_CORE_ARTIFACT_VERSION = ('bst-1.2', 3) +BST_CORE_ARTIFACT_VERSION = ('bst-1.2', 4) diff --git a/buildstream/data/projectconfig.yaml b/buildstream/data/projectconfig.yaml index 247a4536a..eca6e57ad 100644 --- a/buildstream/data/projectconfig.yaml +++ b/buildstream/data/projectconfig.yaml @@ -13,10 +13,6 @@ element-path: . # Store source references in element files ref-storage: inline -# Overlaps are just warnings -fail-on-overlap: False - - # Variable Configuration # variables: diff --git a/buildstream/element.py b/buildstream/element.py index 2fc56d947..370eb7547 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -96,6 +96,7 @@ from . import _cachekey from . import _signals from . import _site from ._platform import Platform +from .plugin import CoreWarnings from .sandbox._config import SandboxConfig @@ -768,31 +769,23 @@ class Element(Plugin): ignored[dep.name] = result.ignored if overlaps: - overlap_error = overlap_warning = False - error_detail = warning_detail = "Staged files overwrite existing files in staging area:\n" + overlap_warning = False + warning_detail = "Staged files overwrite existing files in staging area:\n" for f, elements in overlaps.items(): - overlap_error_elements = [] overlap_warning_elements = [] # The bottom item overlaps nothing overlapping_elements = elements[1:] for elm in overlapping_elements: element = self.search(scope, elm) if not element.__file_is_whitelisted(f): - if project.fail_on_overlap: - overlap_error_elements.append(elm) - overlap_error = True - else: - overlap_warning_elements.append(elm) - overlap_warning = True + overlap_warning_elements.append(elm) + overlap_warning = True warning_detail += _overlap_error_detail(f, overlap_warning_elements, elements) - error_detail += _overlap_error_detail(f, overlap_error_elements, elements) if overlap_warning: - self.warn("Non-whitelisted overlaps detected", detail=warning_detail) - if overlap_error: - raise ElementError("Non-whitelisted overlaps detected and fail-on-overlaps is set", - detail=error_detail, reason="overlap-error") + self.warn("Non-whitelisted overlaps detected", detail=warning_detail, + warning_token=CoreWarnings.OVERLAPS) if ignored: detail = "Not staging files which would replace non-empty directories:\n" @@ -2012,9 +2005,7 @@ class Element(Plugin): 'cache': type(self.__artifacts).__name__ } - # fail-on-overlap setting cannot affect elements without dependencies - if project.fail_on_overlap and dependencies: - self.__cache_key_dict['fail-on-overlap'] = True + self.__cache_key_dict['fatal-warnings'] = sorted(project._fatal_warnings) cache_key_dict = self.__cache_key_dict.copy() cache_key_dict['dependencies'] = dependencies diff --git a/buildstream/plugin.py b/buildstream/plugin.py index fbcf249bb..8fbac365a 100644 --- a/buildstream/plugin.py +++ b/buildstream/plugin.py @@ -47,6 +47,23 @@ it is mandatory to implement the following abstract methods: Once all configuration has been loaded and preflight checks have passed, this method is used to inform the core of a plugin's unique configuration. +Configurable Warnings +--------------------- +Warnings raised through calling :func:`Plugin.warn() <buildstream.plugin.Plugin.warn>` can provide an optional +parameter ``warning_token``, this will raise a :class:`PluginError` if the warning is configured as fatal within +the project configuration. + +Configurable warnings will be prefixed with :func:`Plugin.get_kind() <buildstream.plugin.Plugin.get_kind>` +within buildstream and must be prefixed as such in project configurations. For more detail on project configuration +see :ref:`Configurable Warnings <configurable_warnings>`. + +It is important to document these warnings in your plugin documentation to allow users to make full use of them +while configuring their projects. + +Example +~~~~~~~ +If the :class:`git <buildstream.plugins.sources.git.GitSource>` plugin uses the warning ``"inconsistent-submodule"`` +then it could be referenced in project configuration as ``"git:inconsistent-submodule"``. Plugin Structure ---------------- @@ -206,7 +223,6 @@ class Plugin(): # Infer the kind identifier modulename = type(self).__module__ self.__kind = modulename.split('.')[-1] - self.debug("Created: {}".format(self)) def __del__(self): @@ -513,14 +529,29 @@ class Plugin(): """ self.__message(MessageType.INFO, brief, detail=detail) - def warn(self, brief, *, detail=None): - """Print a warning message + def warn(self, brief, *, detail=None, warning_token=None): + """Print a warning message, checks warning_token against project configuration Args: brief (str): The brief message detail (str): An optional detailed message, can be multiline output + warning_token (str): An optional configurable warning assosciated with this warning, + this will cause PluginError to be raised if this warning is configured as fatal. + (*Since 1.4*) + + Raises: + (:class:`.PluginError`): When warning_token is considered fatal by the project configuration """ - self.__message(MessageType.WARN, brief, detail=detail) + if warning_token: + warning_token = _prefix_warning(self, warning_token) + brief = "[{}]: {}".format(warning_token, brief) + project = self._get_project() + + if project._warning_is_fatal(warning_token): + detail = detail if detail else "" + raise PluginError(message="{}\n{}".format(brief, detail), reason=warning_token) + + self.__message(MessageType.WARN, brief=brief, detail=detail) def log(self, brief, *, detail=None): """Log a message into the plugin's log file @@ -784,3 +815,46 @@ class Plugin(): return '{}:{}'.format(project.junction.name, self.name) else: return self.name + + +class CoreWarnings(): + """CoreWarnings() + + Some common warnings which are raised by core functionalities within BuildStream are found in this class. + """ + + OVERLAPS = "overlaps" + """ + This warning will be produced when buildstream detects an overlap on an element + which is not whitelisted. See :ref:`Overlap Whitelist <public_overlap_whitelist>` + """ + + REF_NOT_IN_TRACK = "ref-not-in-track" + """ + This warning will be produced when a source is configured with a reference + which is found to be invalid based on the configured track + """ + + +__CORE_WARNINGS = [ + value + for name, value in CoreWarnings.__dict__.items() + if not name.startswith("__") +] + + +# _prefix_warning(): +# +# Prefix a warning with the plugin kind. CoreWarnings are not prefixed. +# +# Args: +# plugin (Plugin): The plugin which raised the warning +# warning (str): The warning to prefix +# +# Returns: +# (str): A prefixed warning +# +def _prefix_warning(plugin, warning): + if any((warning is core_warning for core_warning in __CORE_WARNINGS)): + return warning + return "{}:{}".format(plugin.get_kind(), warning) diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py index 9ecef445b..041fed123 100644 --- a/buildstream/plugins/sources/git.py +++ b/buildstream/plugins/sources/git.py @@ -68,6 +68,12 @@ git - stage files from a git repository url: upstream:baz.git checkout: False +**Configurable Warnings:** + +This plugin provides the following configurable warnings: + +- 'git:inconsistent-submodule' - A submodule was found to be missing from the underlying git repository. + """ import os @@ -83,6 +89,9 @@ from buildstream import utils GIT_MODULES = '.gitmodules' +# Warnings +INCONSISTENT_SUBMODULE = "inconsistent-submodules" + # Because of handling of submodules, we maintain a GitMirror # for the primary git source and also for each submodule it @@ -291,7 +300,7 @@ class GitMirror(SourceFetcher): "underlying git repository with `git submodule add`." self.source.warn("{}: Ignoring inconsistent submodule '{}'" - .format(self.source, submodule), detail=detail) + .format(self.source, submodule), detail=detail, warning_token=INCONSISTENT_SUBMODULE) return None diff --git a/doc/source/format_project.rst b/doc/source/format_project.rst index cf4425a0f..1d55bd7e9 100644 --- a/doc/source/format_project.rst +++ b/doc/source/format_project.rst @@ -126,22 +126,29 @@ following to your ``project.conf``: The ``ref-storage`` configuration is available since :ref:`format version 8 <project_format_version>` -Fail on overlaps -~~~~~~~~~~~~~~~~ -When multiple elements are staged, there's a possibility that different -elements will try and stage different versions of the same file. +.. _configurable_warnings: -When ``fail-on-overlap`` is true, if an overlap is detected -that hasn't been allowed by the element's -:ref:`overlap whitelist<public_overlap_whitelist>`, -then an error will be raised and the build will fail. +Configurable Warnings +~~~~~~~~~~~~~~~~~~~~~ +Warnings can be configured as fatal using the ``fatal-warnings`` configuration item. +When a warning is configured as fatal, where a warning would usually be thrown instead an error will be thrown +causing the build to fail. -otherwise, a warning will be raised indicating which files had overlaps, -and the order that the elements were overlapped. +Individual warnings can be configured as fatal by setting ``fatal-warnings`` to a list of warnings. .. code:: yaml - fail-on-overlap: true + fatal-warnings: + - overlaps + - ref-not-in-track + - <plugin>:<warning> + +BuildStream provides a collection of :class:`Core Warnings <buildstream.plugin.CoreWarnings>` which may be raised +by a variety of plugins. Other configurable warnings are plugin specific and should be noted within their individual documentation. + +.. note:: + + The ``fatal-warnings`` configuration is available since :ref:`format version 16 <project_format_version>` .. _project_source_aliases: diff --git a/tests/cachekey/cachekey.py b/tests/cachekey/cachekey.py index 36562fbf3..21beef8fb 100644 --- a/tests/cachekey/cachekey.py +++ b/tests/cachekey/cachekey.py @@ -37,7 +37,8 @@ # from tests.testutils.runcli import cli from tests.testutils.site import HAVE_BZR, HAVE_GIT, HAVE_OSTREE, IS_LINUX - +from buildstream.plugin import CoreWarnings +from buildstream import _yaml import os from collections import OrderedDict import pytest @@ -128,7 +129,6 @@ def assert_cache_keys(project_dir, output): "Use tests/cachekey/update.py to automatically " + "update this test case") - ############################################## # Test Entry Point # ############################################## @@ -167,3 +167,47 @@ def test_cache_key(datafiles, cli): ]) result.assert_success() assert_cache_keys(project, result.output) + + +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.parametrize("first_warnings, second_warnings, identical_keys", [ + [[], [], True], + [[], [CoreWarnings.REF_NOT_IN_TRACK], False], + [[CoreWarnings.REF_NOT_IN_TRACK], [], False], + [[CoreWarnings.REF_NOT_IN_TRACK], [CoreWarnings.REF_NOT_IN_TRACK], True], + [[CoreWarnings.REF_NOT_IN_TRACK, CoreWarnings.OVERLAPS], + [CoreWarnings.OVERLAPS, CoreWarnings.REF_NOT_IN_TRACK], True], +]) +def test_cache_key_fatal_warnings(cli, tmpdir, first_warnings, second_warnings, identical_keys): + + # Builds project, Runs bst show, gathers cache keys + def run_get_cache_key(project_name, warnings): + config = { + 'name': project_name, + 'element-path': 'elements', + 'fatal-warnings': warnings + } + + project_dir = tmpdir.mkdir(project_name) + project_config_file = str(project_dir.join('project.conf')) + _yaml.dump(_yaml.node_sanitize(config), filename=project_config_file) + + elem_dir = project_dir.mkdir('elements') + element_file = str(elem_dir.join('stack.bst')) + _yaml.dump({'kind': 'stack'}, filename=element_file) + + result = cli.run(project=str(project_dir), args=[ + 'show', + '--format', '%{name}::%{full-key}', + 'stack.bst' + ]) + return result.output + + # Returns true if all keys are identical + def compare_cache_keys(first_keys, second_keys): + return not any((x != y for x, y in zip(first_keys, second_keys))) + + first_keys = run_get_cache_key("first", first_warnings) + second_keys = run_get_cache_key("second", second_warnings) + + assert compare_cache_keys(first_keys, second_keys) == identical_keys diff --git a/tests/cachekey/project/elements/build1.expected b/tests/cachekey/project/elements/build1.expected index 750d06234..70d29bffd 100644 --- a/tests/cachekey/project/elements/build1.expected +++ b/tests/cachekey/project/elements/build1.expected @@ -1 +1 @@ -afab4c1a67d1e06489083fa1559bda0b2c8df9b7bc239820ed7cdab30c988a4e
\ No newline at end of file +9195e7c5f1326b878275c3953e39f87fa9909b9eecfe1b07cb23dcbf61e52a70
\ No newline at end of file diff --git a/tests/cachekey/project/elements/build2.expected b/tests/cachekey/project/elements/build2.expected index 3b40d2ddf..2401ea01a 100644 --- a/tests/cachekey/project/elements/build2.expected +++ b/tests/cachekey/project/elements/build2.expected @@ -1 +1 @@ -47395a4e6c86372b181ad1fd6443e11b1ab54c480b7be5e5fe816d84eec3b369
\ No newline at end of file +beb17868e92bdf5203d6a9263cffc91c3252f329116afc04855fe8ef529d77c4
\ No newline at end of file diff --git a/tests/cachekey/project/elements/compose1.expected b/tests/cachekey/project/elements/compose1.expected index 7289d9919..71c199104 100644 --- a/tests/cachekey/project/elements/compose1.expected +++ b/tests/cachekey/project/elements/compose1.expected @@ -1 +1 @@ -f8b69ac5ce84a8e8db30f9ae58d7560054d41da311176f74047694ec1203d7e8
\ No newline at end of file +ef47f8a74e6c96f3d8386ee9b4e4999b91e3fa44a063ee480505c4ae3b9af55a
\ No newline at end of file diff --git a/tests/cachekey/project/elements/compose2.expected b/tests/cachekey/project/elements/compose2.expected index f80f6fb1e..08af0b347 100644 --- a/tests/cachekey/project/elements/compose2.expected +++ b/tests/cachekey/project/elements/compose2.expected @@ -1 +1 @@ -4f542af0ebf3136b0affe42cb5574b7cf1034db6fb60d272ab2304e1c99b0d6f
\ No newline at end of file +739450b3cd7b6cd870ef136ef94880f824f381834b1d5a8c7f616a6590eb34ef
\ No newline at end of file diff --git a/tests/cachekey/project/elements/compose3.expected b/tests/cachekey/project/elements/compose3.expected index 7b9d23ed4..e77e7af48 100644 --- a/tests/cachekey/project/elements/compose3.expected +++ b/tests/cachekey/project/elements/compose3.expected @@ -1 +1 @@ -93863a5513f3b59a107a3ba23a6e47b38738e7c99ac462d2379308dab9910d8f
\ No newline at end of file +c250ea677c5be87114ae1199ebd3ee5f0ceefc6e417b09c8be94448a942a15ac
\ No newline at end of file diff --git a/tests/cachekey/project/elements/compose4.expected b/tests/cachekey/project/elements/compose4.expected index 7feb88917..d656e6e20 100644 --- a/tests/cachekey/project/elements/compose4.expected +++ b/tests/cachekey/project/elements/compose4.expected @@ -1 +1 @@ -e66827c4f0beffbb9eff52522539b10945ad108bd5ad722107e5dfbce7d064ef
\ No newline at end of file +729f40643a6b14b08ed70108ccd9efeb2024ccf4fcd7102996857bc4a75c8eb7
\ No newline at end of file diff --git a/tests/cachekey/project/elements/compose5.expected b/tests/cachekey/project/elements/compose5.expected index d99aa675f..d9c69186a 100644 --- a/tests/cachekey/project/elements/compose5.expected +++ b/tests/cachekey/project/elements/compose5.expected @@ -1 +1 @@ -74a1fcb3c1c7829962398cc104a30d52aeb38af9fe8631a6b77112a1fe99b653
\ No newline at end of file +7e59a1292236d81ac1b79d5d8f188c1457488608944842e45b0c2f813bf08fee
\ No newline at end of file diff --git a/tests/cachekey/project/elements/import1.expected b/tests/cachekey/project/elements/import1.expected index b9020e2d0..fdb2911e5 100644 --- a/tests/cachekey/project/elements/import1.expected +++ b/tests/cachekey/project/elements/import1.expected @@ -1 +1 @@ -619692c94c9b499eaa23358b45fd75b4526d9f5b6d3a6061faad9b6726510db3
\ No newline at end of file +92a12959aba0d4123f383e8d285e1989877f04074d248b3d5bae801d14801fa0
\ No newline at end of file diff --git a/tests/cachekey/project/elements/import2.expected b/tests/cachekey/project/elements/import2.expected index 27a365a64..c7c325048 100644 --- a/tests/cachekey/project/elements/import2.expected +++ b/tests/cachekey/project/elements/import2.expected @@ -1 +1 @@ -8e1ee1f99738be5162e97194d9aa72aef0a9d3458218747f9721102f2d7104d7
\ No newline at end of file +f3d07c82b0bea080dc1783a07234910129cc4f95edab6868482395d77503e445
\ No newline at end of file diff --git a/tests/cachekey/project/elements/import3.expected b/tests/cachekey/project/elements/import3.expected index 8e8eed096..4b943859d 100644 --- a/tests/cachekey/project/elements/import3.expected +++ b/tests/cachekey/project/elements/import3.expected @@ -1 +1 @@ -5d9cfb59d10bb98ca17c52cf8862b84a39202b1d83074a8b8dd3da83a0303c19
\ No newline at end of file +647fe25d06131191ff1b2898f6c57f88f206f6e11297fa25f30610b77fc22e6e
\ No newline at end of file diff --git a/tests/cachekey/project/elements/script1.expected b/tests/cachekey/project/elements/script1.expected index 3613c35d7..f7951113c 100644 --- a/tests/cachekey/project/elements/script1.expected +++ b/tests/cachekey/project/elements/script1.expected @@ -1 +1 @@ -7df04616b29ee538ec5e290dcfd7fdfce9cacdbaf224597856272aec3939d5c8
\ No newline at end of file +f9715017e60fe29557e37402e70eed9119f84324f880e5bf6f25bc78a4113647
\ No newline at end of file diff --git a/tests/cachekey/project/sources/bzr1.expected b/tests/cachekey/project/sources/bzr1.expected index cb276c4c2..25cc7c99b 100644 --- a/tests/cachekey/project/sources/bzr1.expected +++ b/tests/cachekey/project/sources/bzr1.expected @@ -1 +1 @@ -a2682d5e230ea207054fef05eecc1bb8ebc856ae12984ca5ab187d648551e917
\ No newline at end of file +edf2530d3395461ad3e6802944bb023699e0b0a2afb0d4b65f5f35c3c5936154
\ No newline at end of file diff --git a/tests/cachekey/project/sources/git1.expected b/tests/cachekey/project/sources/git1.expected index 427db1397..9d6f05770 100644 --- a/tests/cachekey/project/sources/git1.expected +++ b/tests/cachekey/project/sources/git1.expected @@ -1 +1 @@ -572276304251917e3ce611b19a6c95cb984d989274405344d307e463c53c6b41
\ No newline at end of file +74bae7aaf8af955c0a93148830dee097382d806e6aba3a30bdfec62617661b46
\ No newline at end of file diff --git a/tests/cachekey/project/sources/git2.expected b/tests/cachekey/project/sources/git2.expected index a481139d9..24f95c7a8 100644 --- a/tests/cachekey/project/sources/git2.expected +++ b/tests/cachekey/project/sources/git2.expected @@ -1 +1 @@ -e1c86b2e7a5e87f01a7ea10beacff0c9d2771b39e295729e76db42f2133ad478
\ No newline at end of file +46fae4a8d6195aa6680d82824167eaa453f8945625869fb4aa0bb774bdbe8bf5
\ No newline at end of file diff --git a/tests/cachekey/project/sources/local1.expected b/tests/cachekey/project/sources/local1.expected index b9020e2d0..fdb2911e5 100644 --- a/tests/cachekey/project/sources/local1.expected +++ b/tests/cachekey/project/sources/local1.expected @@ -1 +1 @@ -619692c94c9b499eaa23358b45fd75b4526d9f5b6d3a6061faad9b6726510db3
\ No newline at end of file +92a12959aba0d4123f383e8d285e1989877f04074d248b3d5bae801d14801fa0
\ No newline at end of file diff --git a/tests/cachekey/project/sources/local2.expected b/tests/cachekey/project/sources/local2.expected index 7a5a9fcc7..69215eceb 100644 --- a/tests/cachekey/project/sources/local2.expected +++ b/tests/cachekey/project/sources/local2.expected @@ -1 +1 @@ -080e7416809ccc1a433e28263f6d719c2ac18047ea6def5991ef0dbd049a76f7
\ No newline at end of file +68b17676ea7156e3c267d07a93041f375c868c10015e8742de365dbe561d810e
\ No newline at end of file diff --git a/tests/cachekey/project/sources/ostree1.expected b/tests/cachekey/project/sources/ostree1.expected index b318f5051..eb4c8ddd5 100644 --- a/tests/cachekey/project/sources/ostree1.expected +++ b/tests/cachekey/project/sources/ostree1.expected @@ -1 +1 @@ -9ea532e5911bae30f07910a5da05190c74477e4b5038e8aa43e0178c48d85f92
\ No newline at end of file +f11dc9d54b07608742a6204adbf6dfa74f737b023c4519cc301b5790e3dcf16c
\ No newline at end of file diff --git a/tests/cachekey/project/sources/patch1.expected b/tests/cachekey/project/sources/patch1.expected index 8887de99d..3e00ad4e3 100644 --- a/tests/cachekey/project/sources/patch1.expected +++ b/tests/cachekey/project/sources/patch1.expected @@ -1 +1 @@ -11cab57689dcbb0afd4ee70d589d41d641e1f103427df51fd1d944ec66edc21b
\ No newline at end of file +5982cee04b8a10d4fec13fc6463b8be664323c1c44b802146f483f9dd53ca5f6
\ No newline at end of file diff --git a/tests/cachekey/project/sources/patch2.expected b/tests/cachekey/project/sources/patch2.expected index 196bfedf4..256470fd6 100644 --- a/tests/cachekey/project/sources/patch2.expected +++ b/tests/cachekey/project/sources/patch2.expected @@ -1 +1 @@ -4ab69bc0ecbe926e5659a11bb3b82ac0e5155f1571923b1a57668ce93f27cb46
\ No newline at end of file +95188e54e87a869450049f5b7e865e3f3762d4d387b930c39199ef3b7864fad8
\ No newline at end of file diff --git a/tests/cachekey/project/sources/patch3.expected b/tests/cachekey/project/sources/patch3.expected index abac783cd..351f35ec8 100644 --- a/tests/cachekey/project/sources/patch3.expected +++ b/tests/cachekey/project/sources/patch3.expected @@ -1 +1 @@ -d2d943fa7e0bc7188eaa461d9b92c23aee43361c279106d92dd5f2260ebf8110
\ No newline at end of file +43c215e5be4402eff02f23cd5d1b83659a942b17a56a4c373c36333fcc07fd76
\ No newline at end of file diff --git a/tests/cachekey/project/sources/pip1.expected b/tests/cachekey/project/sources/pip1.expected index 7ab6fd13f..7f0cd5d31 100644 --- a/tests/cachekey/project/sources/pip1.expected +++ b/tests/cachekey/project/sources/pip1.expected @@ -1 +1 @@ -a36bfabe4365076681469b8b6f580d1adb7da5d88a69c168a7bb831fa15651a7
\ No newline at end of file +77241a36a6dc2b89b99e8e8dc52dd057f6f78319a7ff913291aa44419b6afab9
\ No newline at end of file diff --git a/tests/cachekey/project/sources/tar1.expected b/tests/cachekey/project/sources/tar1.expected index 0e657b4a7..1c892d4a7 100644 --- a/tests/cachekey/project/sources/tar1.expected +++ b/tests/cachekey/project/sources/tar1.expected @@ -1 +1 @@ -a284396dae1e98302c98b150d1bb04b576d3039bea138fe0244c3cde6d5ccea5
\ No newline at end of file +2a75001c338b720fac8d5d30933c9ffa14f7e9ef1ebd22c9cf67b6dfc7e94dae
\ No newline at end of file diff --git a/tests/cachekey/project/sources/tar2.expected b/tests/cachekey/project/sources/tar2.expected index 720ea6cfd..4eca37ece 100644 --- a/tests/cachekey/project/sources/tar2.expected +++ b/tests/cachekey/project/sources/tar2.expected @@ -1 +1 @@ -db0d89c04aa964931ee65c456ae91aa2cb784c189f03f2ad36c3ba91381c2005
\ No newline at end of file +0056ada224a0e4c25be48c2b0940b7431af8c79b56cbd5556b7e96c9840060e2
\ No newline at end of file diff --git a/tests/cachekey/project/sources/zip1.expected b/tests/cachekey/project/sources/zip1.expected index 799277681..101b3f704 100644 --- a/tests/cachekey/project/sources/zip1.expected +++ b/tests/cachekey/project/sources/zip1.expected @@ -1 +1 @@ -d106ec29cbd3d04f54d6416edb2300d3e7acb5cb21b231efa6e9f251384d7bc0
\ No newline at end of file +74daf0d1bf5cd07ba420fc059addf2b6e3cb58596b61ebead5ec15830ec8dc63
\ No newline at end of file diff --git a/tests/cachekey/project/sources/zip2.expected b/tests/cachekey/project/sources/zip2.expected index 82ea3c545..5bcc7f2b8 100644 --- a/tests/cachekey/project/sources/zip2.expected +++ b/tests/cachekey/project/sources/zip2.expected @@ -1 +1 @@ -5085978ab2f7f228a6d58ddad945d146d353be6f313e9e13981b7f3a88819d72
\ No newline at end of file +7ebc6f246a7cba793116c7c1ebeba02eab3abadd527f6b3a239f025380f3682b
\ No newline at end of file diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected index 36b583e1d..acf8ab631 100644 --- a/tests/cachekey/project/target.expected +++ b/tests/cachekey/project/target.expected @@ -1 +1 @@ -7534baaacec89d9583a09aa016979c182b5c22f946100050ee5fb44a07ab554d +1d3465903d45365032bff340ec8a8bf0e7d69f4332bb3c525d228a04009dcff7
\ No newline at end of file diff --git a/tests/frontend/configurable_warnings.py b/tests/frontend/configurable_warnings.py new file mode 100644 index 000000000..e8a7b6ac8 --- /dev/null +++ b/tests/frontend/configurable_warnings.py @@ -0,0 +1,63 @@ +import pytest +import os + +from buildstream.plugin import CoreWarnings +from buildstream._exceptions import ErrorDomain, LoadErrorReason +from buildstream import _yaml +from tests.testutils.runcli import cli + +TOP_DIR = os.path.join( + os.path.dirname(os.path.realpath(__file__)), + "configuredwarning" +) + + +def get_project(fatal_warnings): + return { + "name": "test", + "element-path": "elements", + "plugins": [ + { + "origin": "local", + "path": "plugins", + "elements": { + "warninga": 0, + "warningb": 0, + "corewarn": 0, + } + } + ], + "fatal-warnings": fatal_warnings + } + + +def build_project(datafiles, fatal_warnings): + project_path = os.path.join(datafiles.dirname, datafiles.basename) + + project = get_project(fatal_warnings) + + _yaml.dump(project, os.path.join(project_path, "project.conf")) + + return project_path + + +@pytest.mark.datafiles(TOP_DIR) +@pytest.mark.parametrize("element_name, fatal_warnings, expect_fatal, error_domain", [ + ("corewarn.bst", [CoreWarnings.OVERLAPS], True, ErrorDomain.STREAM), + ("warninga.bst", ["warninga:warning-a"], True, ErrorDomain.STREAM), + ("warningb.bst", ["warningb:warning-b"], True, ErrorDomain.STREAM), + ("corewarn.bst", [], False, None), + ("warninga.bst", [], False, None), + ("warningb.bst", [], False, None), + ("warninga.bst", [CoreWarnings.OVERLAPS], False, None), + ("warningb.bst", [CoreWarnings.OVERLAPS], False, None), +]) +def test_fatal_warnings(cli, datafiles, element_name, + fatal_warnings, expect_fatal, error_domain): + project_path = build_project(datafiles, fatal_warnings) + + result = cli.run(project=project_path, args=["build", element_name]) + if expect_fatal: + result.assert_main_error(error_domain, None, "Expected fatal execution") + else: + result.assert_success("Unexpected fatal execution") diff --git a/tests/frontend/configuredwarning/elements/corewarn.bst b/tests/frontend/configuredwarning/elements/corewarn.bst new file mode 100644 index 000000000..6eac4e07e --- /dev/null +++ b/tests/frontend/configuredwarning/elements/corewarn.bst @@ -0,0 +1 @@ +kind: corewarn
\ No newline at end of file diff --git a/tests/frontend/configuredwarning/elements/warninga.bst b/tests/frontend/configuredwarning/elements/warninga.bst new file mode 100644 index 000000000..43c3458a8 --- /dev/null +++ b/tests/frontend/configuredwarning/elements/warninga.bst @@ -0,0 +1 @@ +kind: warninga diff --git a/tests/frontend/configuredwarning/elements/warningb.bst b/tests/frontend/configuredwarning/elements/warningb.bst new file mode 100644 index 000000000..2def6c5a4 --- /dev/null +++ b/tests/frontend/configuredwarning/elements/warningb.bst @@ -0,0 +1 @@ +kind: warningb diff --git a/tests/frontend/configuredwarning/plugins/corewarn.py b/tests/frontend/configuredwarning/plugins/corewarn.py new file mode 100644 index 000000000..bb744b0dd --- /dev/null +++ b/tests/frontend/configuredwarning/plugins/corewarn.py @@ -0,0 +1,37 @@ +import os + +from buildstream import Element +from buildstream.plugin import CoreWarnings + + +class CoreWarn(Element): + def configure(self, node): + pass + + def preflight(self): + pass + + def get_unique_key(self): + pass + + def configure_sandbox(self, sandbox): + sandbox.mark_directory(self.get_variable('install-root')) + + def stage(self, sandbox): + pass + + def assemble(self, sandbox): + self.warn("Testing: CoreWarning produced during assemble", + warning_token=CoreWarnings.OVERLAPS) + + # Return an arbitrary existing directory in the sandbox + # + rootdir = sandbox.get_directory() + install_root = self.get_variable('install-root') + outputdir = os.path.join(rootdir, install_root.lstrip(os.sep)) + os.makedirs(outputdir, exist_ok=True) + return install_root + + +def setup(): + return CoreWarn diff --git a/tests/frontend/configuredwarning/plugins/warninga.py b/tests/frontend/configuredwarning/plugins/warninga.py new file mode 100644 index 000000000..09c9ac3d9 --- /dev/null +++ b/tests/frontend/configuredwarning/plugins/warninga.py @@ -0,0 +1,36 @@ +import os +from buildstream import Element + +WARNING_A = "warning-a" + + +class WarningA(Element): + def configure(self, node): + pass + + def preflight(self): + pass + + def get_unique_key(self): + pass + + def configure_sandbox(self, sandbox): + pass + + def stage(self, sandbox): + pass + + def assemble(self, sandbox): + self.warn("Testing: warning-a produced during assemble", warning_token=WARNING_A) + + # Return an arbitrary existing directory in the sandbox + # + rootdir = sandbox.get_directory() + install_root = self.get_variable('install-root') + outputdir = os.path.join(rootdir, install_root.lstrip(os.sep)) + os.makedirs(outputdir, exist_ok=True) + return install_root + + +def setup(): + return WarningA diff --git a/tests/frontend/configuredwarning/plugins/warningb.py b/tests/frontend/configuredwarning/plugins/warningb.py new file mode 100644 index 000000000..1859e7dee --- /dev/null +++ b/tests/frontend/configuredwarning/plugins/warningb.py @@ -0,0 +1,36 @@ +import os +from buildstream import Element + +WARNING_B = "warning-b" + + +class WarningB(Element): + def configure(self, node): + pass + + def preflight(self): + pass + + def get_unique_key(self): + pass + + def configure_sandbox(self, sandbox): + pass + + def stage(self, sandbox): + pass + + def assemble(self, sandbox): + self.warn("Testing: warning-b produced during assemble", warning_token=WARNING_B) + + # Return an arbitrary existing directory in the sandbox + # + rootdir = sandbox.get_directory() + install_root = self.get_variable('install-root') + outputdir = os.path.join(rootdir, install_root.lstrip(os.sep)) + os.makedirs(outputdir, exist_ok=True) + return install_root + + +def setup(): + return WarningB diff --git a/tests/frontend/configuredwarning/project.conf b/tests/frontend/configuredwarning/project.conf new file mode 100644 index 000000000..c73d217b8 --- /dev/null +++ b/tests/frontend/configuredwarning/project.conf @@ -0,0 +1,8 @@ +name: test +element-path: elements +plugins: +- origin: local + path: element_plugins + elements: + warninga: 0 + warningb: 0 diff --git a/tests/frontend/overlaps.py b/tests/frontend/overlaps.py index 7c86445f6..99c312014 100644 --- a/tests/frontend/overlaps.py +++ b/tests/frontend/overlaps.py @@ -4,6 +4,7 @@ from tests.testutils.runcli import cli from tests.testutils import generate_junction from buildstream._exceptions import ErrorDomain from buildstream import _yaml +from buildstream.plugin import CoreWarnings # Project directory DATA_DIR = os.path.join( @@ -12,32 +13,37 @@ DATA_DIR = os.path.join( ) -def gen_project(project_dir, fail_on_overlap, project_name="test"): +def gen_project(project_dir, fail_on_overlap, use_fatal_warnings=True, project_name="test"): template = { "name": project_name, - "fail-on-overlap": fail_on_overlap } + if use_fatal_warnings: + template["fatal-warnings"] = [CoreWarnings.OVERLAPS] if fail_on_overlap else [] + else: + template["fail-on-overlap"] = fail_on_overlap projectfile = os.path.join(project_dir, "project.conf") _yaml.dump(template, projectfile) @pytest.mark.datafiles(DATA_DIR) -def test_overlaps(cli, datafiles): +@pytest.mark.parametrize("use_fatal_warnings", [True, False]) +def test_overlaps(cli, datafiles, use_fatal_warnings): project_dir = str(datafiles) - gen_project(project_dir, False) + gen_project(project_dir, False, use_fatal_warnings) result = cli.run(project=project_dir, silent=True, args=[ 'build', 'collect.bst']) result.assert_success() @pytest.mark.datafiles(DATA_DIR) -def test_overlaps_error(cli, datafiles): +@pytest.mark.parametrize("use_fatal_warnings", [True, False]) +def test_overlaps_error(cli, datafiles, use_fatal_warnings): project_dir = str(datafiles) - gen_project(project_dir, True) + gen_project(project_dir, True, use_fatal_warnings) result = cli.run(project=project_dir, silent=True, args=[ 'build', 'collect.bst']) result.assert_main_error(ErrorDomain.STREAM, None) - result.assert_task_error(ErrorDomain.ELEMENT, "overlap-error") + result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.OVERLAPS) @pytest.mark.datafiles(DATA_DIR) @@ -68,15 +74,16 @@ def test_overlaps_whitelist_on_overlapper(cli, datafiles): result = cli.run(project=project_dir, silent=True, args=[ 'build', 'collect-partially-whitelisted.bst']) result.assert_main_error(ErrorDomain.STREAM, None) - result.assert_task_error(ErrorDomain.ELEMENT, "overlap-error") + result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.OVERLAPS) @pytest.mark.datafiles(DATA_DIR) -def test_overlaps_script(cli, datafiles): +@pytest.mark.parametrize("use_fatal_warnings", [True, False]) +def test_overlaps_script(cli, datafiles, use_fatal_warnings): # Test overlaps with script element to test # Element.stage_dependency_artifacts() with Scope.RUN project_dir = str(datafiles) - gen_project(project_dir, False) + gen_project(project_dir, False, use_fatal_warnings) result = cli.run(project=project_dir, silent=True, args=[ 'build', 'script.bst']) result.assert_success() @@ -103,7 +110,7 @@ def test_overlap_subproject(cli, tmpdir, datafiles, project_policy, subproject_p result = cli.run(project=project_dir, silent=True, args=['build', 'sub-collect.bst']) if project_policy == 'fail': result.assert_main_error(ErrorDomain.STREAM, None) - result.assert_task_error(ErrorDomain.ELEMENT, "overlap-error") + result.assert_task_error(ErrorDomain.PLUGIN, CoreWarnings.OVERLAPS) else: result.assert_success() - assert "WARNING Non-whitelisted overlaps detected" in result.stderr + assert "WARNING [overlaps]" in result.stderr |