tests/plugins/loading.py: New test replaces removed internal test

This test tests some of the basic failure modes of plugin loading,
which used to be written as internal tests but is not implemented
as a proper end-to-end test.

This commit also adds some machine readable reason codes to
pluginfactory.py so that we can assert the errors more specifically.

7 files changed, 163 insertions, 4 deletions

diff --git a/src/buildstream/_pluginfactory/pluginfactory.py b/src/buildstream/_pluginfactory/pluginfactory.py
index c42b0a3d2..7e5ae2db0 100644
--- a/src/buildstream/_pluginfactory/pluginfactory.py
+++ b/src/buildstream/_pluginfactory/pluginfactory.py
@@ -240,11 +240,13 @@ class PluginFactory:
             plugin_type = plugin.setup()
         except AttributeError as e:
             raise PluginError(
-                "{} plugin '{}' did not provide a setup() function".format(self._base_type.__name__, kind)
+                "{} plugin '{}' did not provide a setup() function".format(self._base_type.__name__, kind),
+                reason="missing-setup-function",
             ) from e
         except TypeError as e:
             raise PluginError(
-                "setup symbol in {} plugin '{}' is not a function".format(self._base_type.__name__, kind)
+                "setup symbol in {} plugin '{}' is not a function".format(self._base_type.__name__, kind),
+                reason="setup-is-not-function",
             ) from e
 
         self._assert_plugin(kind, plugin_type)
@@ -261,11 +263,13 @@ class PluginFactory:
                 raise PluginError(
                     "{} plugin '{}' returned type '{}', which is not a subclass of {}".format(
                         self._base_type.__name__, kind, plugin_type.__name__, self._base_type.__name__
-                    )
+                    ),
+                    reason="setup-returns-bad-type",
                 )
         except TypeError as e:
             raise PluginError(
                 "{} plugin '{}' returned something that is not a type (expected subclass of {})".format(
                     self._base_type.__name__, kind, self._base_type.__name__
-                )
+                ),
+                reason="setup-returns-not-type",
             ) from e
diff --git a/tests/plugins/loading.py b/tests/plugins/loading.py
new file mode 100644
index 000000000..1c8a321d5
--- /dev/null
+++ b/tests/plugins/loading.py
@@ -0,0 +1,110 @@
+# Pylint doesn't play well with fixtures and dependency injection from pytest
+# pylint: disable=redefined-outer-name
+
+#
+# This test case tests the failure modes of loading a plugin
+# after it has already been discovered via it's origin.
+#
+
+import os
+import pytest
+
+from buildstream.exceptions import ErrorDomain
+from buildstream.testing import cli  # pylint: disable=unused-import
+from buildstream import _yaml
+
+
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "loading")
+
+
+def update_project(project_path, updated_configuration):
+    project_conf_path = os.path.join(project_path, "project.conf")
+    project_conf = _yaml.roundtrip_load(project_conf_path)
+
+    project_conf.update(updated_configuration)
+
+    _yaml.roundtrip_dump(project_conf, project_conf_path)
+
+
+# Sets up the element.bst file so that it requires a source
+# or element plugin.
+#
+def setup_element(project_path, plugin_type, plugin_name):
+    element_dir = os.path.join(project_path, "elements")
+    element_path = os.path.join(element_dir, "element.bst")
+    os.makedirs(element_dir, exist_ok=True)
+
+    if plugin_type == "elements":
+        element = {"kind": plugin_name}
+    else:
+        element = {"kind": "manual", "sources": [{"kind": plugin_name}]}
+
+    _yaml.roundtrip_dump(element, element_path)
+
+
+####################################################
+#                     Tests                        #
+####################################################
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("plugin_type", [("elements"), ("sources")])
+def test_nosetup(cli, datafiles, plugin_type):
+    project = str(datafiles)
+
+    update_project(project, {"plugins": [{"origin": "local", "path": "plugins/nosetup", plugin_type: ["nosetup"]}]})
+    setup_element(project, plugin_type, "nosetup")
+
+    result = cli.run(project=project, args=["show", "element.bst"])
+    result.assert_main_error(ErrorDomain.PLUGIN, "missing-setup-function")
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("plugin_type", [("elements"), ("sources")])
+def test_setup_not_function(cli, datafiles, plugin_type):
+    project = str(datafiles)
+
+    update_project(
+        project,
+        {"plugins": [{"origin": "local", "path": "plugins/setupnotfunction", plugin_type: ["setupnotfunction"]}]},
+    )
+    setup_element(project, plugin_type, "setupnotfunction")
+
+    result = cli.run(project=project, args=["show", "element.bst"])
+    result.assert_main_error(ErrorDomain.PLUGIN, "setup-is-not-function")
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("plugin_type", [("elements"), ("sources")])
+def test_setup_returns_not_type(cli, datafiles, plugin_type):
+    project = str(datafiles)
+
+    update_project(
+        project,
+        {
+            "plugins": [
+                {"origin": "local", "path": "plugins/setupreturnsnottype", plugin_type: ["setupreturnsnottype"]}
+            ]
+        },
+    )
+    setup_element(project, plugin_type, "setupreturnsnottype")
+
+    result = cli.run(project=project, args=["show", "element.bst"])
+    result.assert_main_error(ErrorDomain.PLUGIN, "setup-returns-not-type")
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("plugin_type", [("elements"), ("sources")])
+def test_setup_returns_bad_type(cli, datafiles, plugin_type):
+    project = str(datafiles)
+
+    update_project(
+        project,
+        {
+            "plugins": [
+                {"origin": "local", "path": "plugins/setupreturnsbadtype", plugin_type: ["setupreturnsbadtype"]}
+            ]
+        },
+    )
+    setup_element(project, plugin_type, "setupreturnsbadtype")
+
+    result = cli.run(project=project, args=["show", "element.bst"])
+    result.assert_main_error(ErrorDomain.PLUGIN, "setup-returns-bad-type")
diff --git a/tests/plugins/loading/plugins/nosetup/nosetup.py b/tests/plugins/loading/plugins/nosetup/nosetup.py
new file mode 100644
index 000000000..0b5a4fa7e
--- /dev/null
+++ b/tests/plugins/loading/plugins/nosetup/nosetup.py
@@ -0,0 +1,8 @@
+# A plugin is supposed to define a setup function
+# which returns the type that the plugin provides
+#
+# This plugin fails to do so
+
+
+def useless():
+    print("Hello World")
diff --git a/tests/plugins/loading/plugins/setupnotfunction/setupnotfunction.py b/tests/plugins/loading/plugins/setupnotfunction/setupnotfunction.py
new file mode 100644
index 000000000..1272f4ee0
--- /dev/null
+++ b/tests/plugins/loading/plugins/setupnotfunction/setupnotfunction.py
@@ -0,0 +1,6 @@
+# A plugin's setup symbol is supposed to be a function
+# which returns the plugin type.
+#
+# This plugin's setup symbol is not such a function.
+
+setup = 9
diff --git a/tests/plugins/loading/plugins/setupreturnsbadtype/setupreturnsbadtype.py b/tests/plugins/loading/plugins/setupreturnsbadtype/setupreturnsbadtype.py
new file mode 100644
index 000000000..25795c06b
--- /dev/null
+++ b/tests/plugins/loading/plugins/setupreturnsbadtype/setupreturnsbadtype.py
@@ -0,0 +1,15 @@
+# A plugin's setup symbol is supposed to be a function
+# which returns the plugin type, which should be a subclass
+# of Source or Element depending on the plugin type.
+#
+# This plugin's setup function returns a different kind
+# of type.
+
+
+class Pony:
+    def __init__(self):
+        self.pony = 12
+
+
+def setup():
+    return Pony
diff --git a/tests/plugins/loading/plugins/setupreturnsnottype/setupreturnsnottype.py b/tests/plugins/loading/plugins/setupreturnsnottype/setupreturnsnottype.py
new file mode 100644
index 000000000..d47367ea9
--- /dev/null
+++ b/tests/plugins/loading/plugins/setupreturnsnottype/setupreturnsnottype.py
@@ -0,0 +1,8 @@
+# A plugin's setup symbol is supposed to be a function
+# which returns the plugin type.
+#
+# This plugin's setup function returns a number instead
+
+
+def setup():
+    return 5
diff --git a/tests/plugins/loading/project.conf b/tests/plugins/loading/project.conf
new file mode 100644
index 000000000..738051af9
--- /dev/null
+++ b/tests/plugins/loading/project.conf
@@ -0,0 +1,8 @@
+# This project.conf gets rewritten for each plugin loading test
+name: test
+
+# Required BuildStream version
+min-version: 2.0
+
+# Subdirectory where elements are stored
+element-path: elements