Plugin loading refactor, removing all versioning

Plugin format versioning was decided to be removed for local
plugins and any plugins for which we do not load an explicitly
provided plugin. For pip, this will be handled with a standard
distutils/setuptools approach, allowing users to specify pip
style version boundaries in the plugin origins.

This patch refactors plugin loading so that all related code
goes into the private _pluginfactory module, a new small
PluginOrigin type was added to better manipulate loaded
origins.

Test cases have been removed and will be readded in a following
commit, adjusted to new expectations.

5 files changed, 538 insertions, 0 deletions

diff --git a/src/buildstream/_pluginfactory/__init__.py b/src/buildstream/_pluginfactory/__init__.py
new file mode 100644
index 000000000..fe69b6e77
--- /dev/null
+++ b/src/buildstream/_pluginfactory/__init__.py
@@ -0,0 +1,20 @@
+#
+#  Copyright (C) 2020 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from .pluginorigin import PluginOrigin, PluginOriginType
+from .sourcefactory import SourceFactory
+from .elementfactory import ElementFactory
diff --git a/src/buildstream/_pluginfactory/elementfactory.py b/src/buildstream/_pluginfactory/elementfactory.py
new file mode 100644
index 000000000..8879a4173
--- /dev/null
+++ b/src/buildstream/_pluginfactory/elementfactory.py
@@ -0,0 +1,58 @@
+#
+#  Copyright (C) 2016 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+from .. import _site
+from ..element import Element
+
+from .pluginfactory import PluginFactory
+
+
+# A ElementFactory creates Element instances
+# in the context of a given factory
+#
+# Args:
+#     plugin_base (PluginBase): The main PluginBase object to work with
+#
+class ElementFactory(PluginFactory):
+    def __init__(self, plugin_base):
+
+        super().__init__(
+            plugin_base, Element, [_site.element_plugins], "buildstream.plugins.elements",
+        )
+
+    # create():
+    #
+    # Create an Element object, the pipeline uses this to create Element
+    # objects on demand for a given pipeline.
+    #
+    # Args:
+    #    context (object): The Context object for processing
+    #    project (object): The project object
+    #    meta (object): The loaded MetaElement
+    #
+    # Returns: A newly created Element object of the appropriate kind
+    #
+    # Raises:
+    #    PluginError (if the kind lookup failed)
+    #    LoadError (if the element itself took issue with the config)
+    #
+    def create(self, context, project, meta):
+        element_type, default_config = self.lookup(meta.kind)
+        element = element_type(context, project, meta, default_config)
+        return element
diff --git a/src/buildstream/_pluginfactory/pluginfactory.py b/src/buildstream/_pluginfactory/pluginfactory.py
new file mode 100644
index 000000000..c42b0a3d2
--- /dev/null
+++ b/src/buildstream/_pluginfactory/pluginfactory.py
@@ -0,0 +1,271 @@
+#
+#  Copyright (C) 2016 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+import os
+import inspect
+
+from .._exceptions import PluginError
+
+from .pluginorigin import PluginOrigin, PluginOriginType
+
+
+# A Context for loading plugin types
+#
+# Args:
+#     plugin_base (PluginBase): The main PluginBase object to work with
+#     base_type (type):         A base object type for this context
+#     site_plugin_path (str):   Path to where buildstream keeps plugins
+#     entrypoint_group (str):   Name of the entry point group that provides plugins
+#
+# Since multiple pipelines can be processed recursively
+# within the same interpretor, it's important that we have
+# one context associated to the processing of a given pipeline,
+# this way sources and element types which are particular to
+# a given BuildStream project are isolated to their respective
+# Pipelines.
+#
+class PluginFactory:
+    def __init__(self, plugin_base, base_type, site_plugin_path, entrypoint_group):
+
+        # The plugin kinds which were loaded
+        self.loaded_dependencies = []
+
+        #
+        # Private members
+        #
+
+        # For pickling across processes, make sure this context has a unique
+        # identifier, which we prepend to the identifier of each PluginSource.
+        # This keeps plugins loaded during the first and second pass distinct
+        # from eachother.
+        self._identifier = str(id(self))
+
+        self._base_type = base_type  # The base class plugins derive from
+        self._types = {}  # Plugin type lookup table by kind
+        self._origins = {}  # PluginOrigin lookup table by kind
+
+        # The PluginSource object
+        self._plugin_base = plugin_base
+        self._site_plugin_path = site_plugin_path
+        self._entrypoint_group = entrypoint_group
+        self._alternate_sources = {}
+
+        self._init_site_source()
+
+    def _init_site_source(self):
+        self._site_source = self._plugin_base.make_plugin_source(
+            searchpath=self._site_plugin_path, identifier=self._identifier + "site",
+        )
+
+    def __getstate__(self):
+        state = self.__dict__.copy()
+
+        # PluginSource is not a picklable type, so we must reconstruct this one
+        # as best we can when unpickling.
+        #
+        # Since the values of `_types` depend on the PluginSource, we must also
+        # get rid of those. It is only a cache - we will automatically recreate
+        # them on demand.
+        #
+        # Similarly we must clear out the `_alternate_sources` cache.
+        #
+        # Note that this method of referring to members is error-prone in that
+        # a later 'search and replace' renaming might miss these. Guard against
+        # this by making sure we are not creating new members, only clearing
+        # existing ones.
+        #
+        del state["_site_source"]
+        assert "_types" in state
+        state["_types"] = {}
+        assert "_alternate_sources" in state
+        state["_alternate_sources"] = {}
+
+        return state
+
+    def __setstate__(self, state):
+        self.__dict__.update(state)
+
+        # Note that in order to enable plugins to be unpickled along with this
+        # PluginSource, we would also have to set and restore the 'identifier'
+        # of the PluginSource. We would also have to recreate `_types` as it
+        # was before unpickling them. We are not using this method in
+        # BuildStream, so the identifier is not restored here.
+        self._init_site_source()
+
+    # lookup():
+    #
+    # Fetches a type loaded from a plugin in this plugin context
+    #
+    # Args:
+    #     kind (str): The kind of Plugin to create
+    #
+    # Returns: the type associated with the given kind
+    #
+    # Raises: PluginError
+    #
+    def lookup(self, kind):
+        return self._ensure_plugin(kind)
+
+    # register_plugin_origin():
+    #
+    # Registers the PluginOrigin to use for the given plugin kind
+    #
+    # Args:
+    #    kind (str): The kind identifier of the Plugin
+    #    origin (PluginOrigin): The PluginOrigin providing the plugin
+    #
+    def register_plugin_origin(self, kind: str, origin: PluginOrigin):
+        if kind in self._origins:
+            raise PluginError(
+                "More than one {} plugin registered as kind '{}'".format(self._base_type.__name__, kind),
+                reason="duplicate-plugin",
+            )
+
+        self._origins[kind] = origin
+
+    # all_loaded_plugins():
+    #
+    # Returns: an iterable over all the loaded plugins.
+    #
+    def all_loaded_plugins(self):
+        return self._types.values()
+
+    def _get_local_plugin_source(self, path):
+        if ("local", path) not in self._alternate_sources:
+            # key by a tuple to avoid collision
+            source = self._plugin_base.make_plugin_source(searchpath=[path], identifier=self._identifier + path,)
+            # Ensure that sources never get garbage collected,
+            # as they'll take the plugins with them.
+            self._alternate_sources[("local", path)] = source
+        else:
+            source = self._alternate_sources[("local", path)]
+        return source
+
+    def _get_pip_plugin_source(self, package_name, kind):
+        defaults = None
+        if ("pip", package_name) not in self._alternate_sources:
+            import pkg_resources
+
+            # key by a tuple to avoid collision
+            try:
+                package = pkg_resources.get_entry_info(package_name, self._entrypoint_group, kind)
+            except pkg_resources.DistributionNotFound as e:
+                raise PluginError("Failed to load {} plugin '{}': {}".format(self._base_type.__name__, kind, e)) from e
+
+            if package is None:
+                raise PluginError("Pip package {} does not contain a plugin named '{}'".format(package_name, kind))
+
+            location = package.dist.get_resource_filename(
+                pkg_resources._manager, package.module_name.replace(".", os.sep) + ".py"
+            )
+
+            # Also load the defaults - required since setuptools
+            # may need to extract the file.
+            try:
+                defaults = package.dist.get_resource_filename(
+                    pkg_resources._manager, package.module_name.replace(".", os.sep) + ".yaml"
+                )
+            except KeyError:
+                # The plugin didn't have an accompanying YAML file
+                defaults = None
+
+            source = self._plugin_base.make_plugin_source(
+                searchpath=[os.path.dirname(location)], identifier=self._identifier + os.path.dirname(location),
+            )
+            self._alternate_sources[("pip", package_name)] = source
+
+        else:
+            source = self._alternate_sources[("pip", package_name)]
+
+        return source, defaults
+
+    def _ensure_plugin(self, kind):
+
+        if kind not in self._types:
+            source = None
+            defaults = None
+
+            origin = self._origins.get(kind, None)
+            if origin:
+                # Try getting the plugin source from a registered origin
+                if origin.origin_type == PluginOriginType.LOCAL:
+                    source = self._get_local_plugin_source(origin.path)
+                elif origin.origin_type == PluginOriginType.PIP:
+                    source, defaults = self._get_pip_plugin_source(origin.package_name, kind)
+                else:
+                    assert False, "Encountered invalid plugin origin type"
+            else:
+                # Try getting it from the core plugins
+                if kind not in self._site_source.list_plugins():
+                    raise PluginError("No {} type registered for kind '{}'".format(self._base_type.__name__, kind))
+
+                source = self._site_source
+
+            self._types[kind] = self._load_plugin(source, kind, defaults)
+            self.loaded_dependencies.append(kind)
+
+        return self._types[kind]
+
+    def _load_plugin(self, source, kind, defaults):
+
+        try:
+            plugin = source.load_plugin(kind)
+
+            if not defaults:
+                plugin_file = inspect.getfile(plugin)
+                plugin_dir = os.path.dirname(plugin_file)
+                plugin_conf_name = "{}.yaml".format(kind)
+                defaults = os.path.join(plugin_dir, plugin_conf_name)
+
+        except ImportError as e:
+            raise PluginError("Failed to load {} plugin '{}': {}".format(self._base_type.__name__, kind, e)) from e
+
+        try:
+            plugin_type = plugin.setup()
+        except AttributeError as e:
+            raise PluginError(
+                "{} plugin '{}' did not provide a setup() function".format(self._base_type.__name__, kind)
+            ) from e
+        except TypeError as e:
+            raise PluginError(
+                "setup symbol in {} plugin '{}' is not a function".format(self._base_type.__name__, kind)
+            ) from e
+
+        self._assert_plugin(kind, plugin_type)
+        return (plugin_type, defaults)
+
+    def _assert_plugin(self, kind, plugin_type):
+        if kind in self._types:
+            raise PluginError(
+                "Tried to register {} plugin for existing kind '{}' "
+                "(already registered {})".format(self._base_type.__name__, kind, self._types[kind].__name__)
+            )
+        try:
+            if not issubclass(plugin_type, self._base_type):
+                raise PluginError(
+                    "{} plugin '{}' returned type '{}', which is not a subclass of {}".format(
+                        self._base_type.__name__, kind, plugin_type.__name__, self._base_type.__name__
+                    )
+                )
+        except TypeError as e:
+            raise PluginError(
+                "{} plugin '{}' returned something that is not a type (expected subclass of {})".format(
+                    self._base_type.__name__, kind, self._base_type.__name__
+                )
+            ) from e
diff --git a/src/buildstream/_pluginfactory/pluginorigin.py b/src/buildstream/_pluginfactory/pluginorigin.py
new file mode 100644
index 000000000..50852711b
--- /dev/null
+++ b/src/buildstream/_pluginfactory/pluginorigin.py
@@ -0,0 +1,130 @@
+#
+#  Copyright (C) 2020 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+
+from ..types import FastEnum
+
+
+# PluginOriginType:
+#
+# An enumeration depicting the type of plugin origin
+#
+class PluginOriginType(FastEnum):
+    LOCAL = "local"
+    PIP = "pip"
+
+
+# PluginOrigin
+#
+# Base class holding common properties of all origins.
+#
+class PluginOrigin:
+
+    # Common fields valid for all plugin origins
+    _COMMON_CONFIG_KEYS = ["origin", "sources", "elements"]
+
+    def __init__(self, origin_type):
+
+        # Public
+        self.origin_type = origin_type
+        self.elements = []
+        self.sources = []
+
+        # Private
+        self._project = None
+        self._kinds = {}
+
+    # new_from_node()
+    #
+    # Load a PluginOrigin from the YAML in project.conf
+    #
+    # Args:
+    #    project (Project): The project from whence this origin is loaded
+    #    origin_node (MappingNode): The node defining this origin
+    #
+    # Returns:
+    #    (PluginOrigin): The newly created PluginOrigin
+    #
+    @classmethod
+    def new_from_node(cls, project, origin_node):
+
+        origin_type = origin_node.get_enum("origin", PluginOriginType)
+
+        if origin_type == PluginOriginType.LOCAL:
+            origin = PluginOriginLocal()
+        elif origin_type == PluginOriginType.PIP:
+            origin = PluginOriginPip()
+
+        origin._project = project
+        origin._load(origin_node)
+
+        origin.elements = origin_node.get_str_list("elements", [])
+        origin.sources = origin_node.get_str_list("sources", [])
+
+        return origin
+
+    # _load()
+    #
+    # Abstract method for loading data from the origin node, this
+    # method should not load the source and element lists.
+    #
+    # Args:
+    #    origin_node (MappingNode): The node defining this origin
+    #
+    def _load(self, origin_node):
+        pass
+
+
+# PluginOriginLocal
+#
+# PluginOrigin for local plugins
+#
+class PluginOriginLocal(PluginOrigin):
+    def __init__(self):
+        super().__init__(PluginOriginType.LOCAL)
+
+        # An absolute path to where the plugin can be found
+        #
+        self.path = None
+
+    def _load(self, origin_node):
+
+        origin_node.validate_keys(["path", *PluginOrigin._COMMON_CONFIG_KEYS])
+
+        path_node = origin_node.get_scalar("path")
+        path = self._project.get_path_from_node(path_node, check_is_dir=True)
+
+        self.path = os.path.join(self._project.directory, path)
+
+
+# PluginOriginPip
+#
+# PluginOrigin for pip plugins
+#
+class PluginOriginPip(PluginOrigin):
+    def __init__(self):
+        super().__init__(PluginOriginType.PIP)
+
+        # The pip package name to extract plugins from
+        #
+        self.package_name = None
+
+    def _load(self, origin_node):
+
+        origin_node.validate_keys(["package-name", *PluginOrigin._COMMON_CONFIG_KEYS])
+        self.package_name = origin_node.get_str("package-name")
diff --git a/src/buildstream/_pluginfactory/sourcefactory.py b/src/buildstream/_pluginfactory/sourcefactory.py
new file mode 100644
index 000000000..9f6a09784
--- /dev/null
+++ b/src/buildstream/_pluginfactory/sourcefactory.py
@@ -0,0 +1,59 @@
+#
+#  Copyright (C) 2016 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+
+from .. import _site
+from ..source import Source
+
+from .pluginfactory import PluginFactory
+
+
+# A SourceFactory creates Source instances
+# in the context of a given factory
+#
+# Args:
+#     plugin_base (PluginBase): The main PluginBase object to work with
+#
+class SourceFactory(PluginFactory):
+    def __init__(self, plugin_base):
+
+        super().__init__(
+            plugin_base, Source, [_site.source_plugins], "buildstream.plugins.sources",
+        )
+
+    # create():
+    #
+    # Create a Source object, the pipeline uses this to create Source
+    # objects on demand for a given pipeline.
+    #
+    # Args:
+    #    context (object): The Context object for processing
+    #    project (object): The project object
+    #    meta (object): The loaded MetaSource
+    #
+    # Returns:
+    #    A newly created Source object of the appropriate kind
+    #
+    # Raises:
+    #    PluginError (if the kind lookup failed)
+    #    LoadError (if the source itself took issue with the config)
+    #
+    def create(self, context, project, meta):
+        source_type, _ = self.lookup(meta.kind)
+        source = source_type(context, project, meta)
+        return source