_project.py, _artifactproject.py: Adding ArtifactProject

The Project's class initializer is now refactored such that loading
a project.conf is made optional. The initializer is now well sorted
with public members showing up before private members, followed by
the initialization body. pep484 type hints are now employed aggressively
for all project instance members.

The added ArtifactProject is added to serve as the data model counterpart
of the ArtifactElement, ensuring that we never mistakenly use locally
loaded project data in ArtifactElement instances.

Consequently, the Project.sandbox and Project.splits variables are
properly made public by this commit, as these are simply loaded from
the project config and accessed elsewhere by Element; Element is updated
to access these public members by their new public names.

3 files changed, 181 insertions, 67 deletions

diff --git a/src/buildstream/_artifactproject.py b/src/buildstream/_artifactproject.py
new file mode 100644
index 000000000..b8153b06d
--- /dev/null
+++ b/src/buildstream/_artifactproject.py
@@ -0,0 +1,88 @@
+#
+#  Copyright (C) 2020 Codethink Limited
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+#  Authors:
+#        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
+#
+from contextlib import suppress
+from typing import TYPE_CHECKING
+
+from ._project import Project
+from ._context import Context
+from ._loader import Loader
+
+if TYPE_CHECKING:
+    from typing import Dict
+
+
+# ArtifactProject()
+#
+# A project instance to be used as the project for an ArtifactElement.
+#
+# This is basically a simplified Project implementation which ensures that
+# we do not accidentally infer any data from a possibly present local project
+# when processing an ArtifactElement.
+#
+# Args:
+#    project_name: The name of this project
+#
+class ArtifactProject(Project):
+
+    __loaded_artifact_projects = {}  # type: Dict[str, ArtifactProject]
+
+    def __init__(self, project_name: str, context: Context):
+
+        #
+        # Chain up to the Project constructor, and allow it to initialize
+        # without loading anything
+        #
+        super().__init__(None, context, search_for_project=False)
+
+        # Fill in some necessities
+        #
+        self.name = project_name
+        self.element_path = ""  # This needs to be set to avoid Loader crashes
+        self.loader = Loader(self)
+
+    # get_artifact_project():
+    #
+    # Gets a reference to an ArtifactProject for the given
+    # project name, possibly instantiating one if needed.
+    #
+    # Args:
+    #    project_name: The project name
+    #    context: The Context
+    #
+    # Returns:
+    #    An ArtifactProject with the given project_name
+    #
+    @classmethod
+    def get_artifact_project(cls, project_name: str, context: Context) -> "ArtifactProject":
+        with suppress(KeyError):
+            return cls.__loaded_artifact_projects[project_name]
+
+        project = cls(project_name, context)
+        cls.__loaded_artifact_projects[project_name] = project
+        return project
+
+    # clear_project_cache():
+    #
+    # Clears the cache of loaded projects, this can be called directly
+    # after completing a full load.
+    #
+    @classmethod
+    def clear_project_cache(cls):
+        cls.__loaded_artifact_projects = {}
diff --git a/src/buildstream/_project.py b/src/buildstream/_project.py
index fd4a44732..2534e0209 100644
--- a/src/buildstream/_project.py
+++ b/src/buildstream/_project.py
@@ -18,6 +18,8 @@
 #        Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
 #        Tiago Gomes <tiago.gomes@codethink.co.uk>
 
+from typing import TYPE_CHECKING, Optional, Dict, Union, List
+
 import os
 import sys
 from collections import OrderedDict
@@ -44,6 +46,10 @@ from ._message import Message, MessageType
 from ._includes import Includes
 from ._workspaces import WORKSPACE_PROJECT_FILE
 
+if TYPE_CHECKING:
+    from .node import ProvenanceInformation, MappingNode
+    from ._context import Context
+    from ._remote import RemoteSpec
 
 # Project Configuration file
 _PROJECT_CONF_FILE = "project.conf"
@@ -86,95 +92,115 @@ class ProjectConfig:
 #
 # The Project Configuration
 #
+# Args:
+#    directory: The project directory, or None for dummy ArtifactProjects
+#    context: The invocation context
+#    junction: The junction Element causing this project to be loaded
+#    cli_options: The project options specified on the command line
+#    default_mirror: The default mirror specified on the command line
+#    parent_loader: The parent loader
+#    provenance_node: The YAML provenance causing this project to be loaded
+#    search_for_project: Whether to search for a project directory, e.g. from workspace metadata or parent directories
+#    load_project: Whether to attempt to load a project.conf
+#
 class Project:
     def __init__(
         self,
-        directory,
-        context,
+        directory: Optional[str],
+        context: "Context",
         *,
-        junction=None,
-        cli_options=None,
-        default_mirror=None,
-        parent_loader=None,
-        provenance_node=None,
-        search_for_project=True,
+        junction: Optional[object] = None,
+        cli_options: Optional[Dict[str, str]] = None,
+        default_mirror: Optional[str] = None,
+        parent_loader: Optional[Loader] = None,
+        provenance_node: Optional["ProvenanceInformation"] = None,
+        search_for_project: bool = True,
+        load_project: bool = True,
     ):
+        #
+        # Public members
+        #
+        self.name: Optional[str] = None  # The project name
+        self.directory: Optional[str] = directory  # The project directory
+        self.element_path: Optional[str] = None  # The project relative element path
 
-        # The project name
-        self.name = None
-
-        self._context = context  # The invocation Context, a private member
-
-        # Create the LoadContext here if we are the toplevel project.
-        if parent_loader:
-            self.load_context = parent_loader.load_context
-        else:
-            self.load_context = LoadContext(self._context)
-
-        if search_for_project:
-            self.directory, self._invoked_from_workspace_element = self._find_project_dir(directory)
-        else:
-            self.directory = directory
-            self._invoked_from_workspace_element = None
-
-        self._absolute_directory_path = Path(self.directory).resolve()
-
-        # Absolute path to where elements are loaded from within the project
-        self.element_path = None
+        self.load_context: LoadContext  # The LoadContext
+        self.loader: Optional[Loader] = None  # The loader associated to this project
+        self.junction: Optional[object] = junction  # The junction Element object, if this is a subproject
 
-        # ProjectRefs for the main refs and also for junctions
-        self.refs = ProjectRefs(self.directory, "project.refs")
-        self.junction_refs = ProjectRefs(self.directory, "junction.refs")
+        self.ref_storage: Optional[ProjectRefStorage] = None  # Where to store source refs
+        self.refs: Optional[ProjectRefs] = None
+        self.junction_refs: Optional[ProjectRefs] = None
 
-        self.config = ProjectConfig()
-        self.first_pass_config = ProjectConfig()
+        self.config: ProjectConfig = ProjectConfig()
+        self.first_pass_config: ProjectConfig = ProjectConfig()
 
-        self.junction = junction  # The junction Element object, if this is a subproject
+        self.base_environment: Union["MappingNode", Dict[str, str]] = {}  # The base set of environment variables
+        self.base_env_nocache: List[str] = []  # The base nocache mask (list) for the environment
 
-        self.ref_storage = None  # ProjectRefStorage setting
-        self.base_environment = {}  # The base set of environment variables
-        self.base_env_nocache = None  # The base nocache mask (list) for the environment
+        # Remote specs for communicating with remote services
+        self.artifact_cache_specs: List["RemoteSpec"] = []  # Artifact caches
+        self.source_cache_specs: List["RemoteSpec"] = []  # Source caches
+        self.remote_execution_specs: List["RemoteSpec"] = []  # Remote execution services
 
-        self.artifact_cache_specs = None
-        self.source_cache_specs = None
-        self.remote_execution_specs = None
+        self.element_factory: Optional[ElementFactory] = None  # ElementFactory for loading elements
+        self.source_factory: Optional[SourceFactory] = None  # SourceFactory for loading sources
 
-        self.element_factory = None  # ElementFactory for loading elements
-        self.source_factory = None  # SourceFactory for loading sources
+        self.sandbox: Optional["MappingNode"] = None
+        self.splits: Optional["MappingNode"] = None
 
         #
-        # Private Members
+        # Private members
         #
-        self._default_targets = None  # Default target elements
-        self._default_mirror = default_mirror  # The name of the preferred mirror.
+        self._context: "Context" = context  # The invocation Context
+        self._invoked_from_workspace_element: Optional[str] = None
+        self._absolute_directory_path: Optional[Path] = None
 
-        self._cli_options = cli_options
+        self._default_targets: Optional[List[str]] = None  # Default target elements
+        self._default_mirror: Optional[str] = default_mirror  # The name of the preferred mirror.
+        self._cli_options: Optional[Dict[str, str]] = cli_options
 
-        self._fatal_warnings = []  # A list of warnings which should trigger an error
-
-        self._shell_command = []  # The default interactive shell command
-        self._shell_environment = {}  # Statically set environment vars
-        self._shell_host_files = []  # A list of HostMount objects
-        self._sandbox = None
-        self._splits = None
+        self._fatal_warnings: List[str] = []  # A list of warnings which should trigger an error
+        self._shell_command: List[str] = []  # The default interactive shell command
+        self._shell_environment: Dict[str, str] = {}  # Statically set environment vars
+        self._shell_host_files: List[str] = []  # A list of HostMount objects
 
         # This is a lookup table of lists indexed by project,
         # the child dictionaries are lists of ScalarNodes indicating
         # junction names
-        self._junction_duplicates = {}
+        self._junction_duplicates: Dict[str, List[str]] = {}
 
         # A list of project relative junctions to consider as 'internal',
         # stored as ScalarNodes.
-        self._junction_internal = []
+        self._junction_internal: List[str] = []
 
-        self._context.add_project(self)
+        self._partially_loaded: bool = False
+        self._fully_loaded: bool = False
+        self._project_includes: Optional[Includes] = None
 
-        self._partially_loaded = False
-        self._fully_loaded = False
-        self._project_includes = None
+        #
+        # Initialization body
+        #
+        if parent_loader:
+            self.load_context = parent_loader.load_context
+        else:
+            self.load_context = LoadContext(self._context)
 
-        with PROFILER.profile(Topics.LOAD_PROJECT, self.directory.replace(os.sep, "-")):
-            self._load(parent_loader=parent_loader, provenance_node=provenance_node)
+        if search_for_project:
+            self.directory, self._invoked_from_workspace_element = self._find_project_dir(directory)
+
+        if self.directory:
+            self._absolute_directory_path = Path(self.directory).resolve()
+            self.refs = ProjectRefs(self.directory, "project.refs")
+            self.junction_refs = ProjectRefs(self.directory, "junction.refs")
+
+        self._context.add_project(self)
+
+        if self.directory and load_project:
+            with PROFILER.profile(Topics.LOAD_PROJECT, self.directory.replace(os.sep, "-")):
+                self._load(parent_loader=parent_loader, provenance_node=provenance_node)
+        else:
+            self._fully_loaded = True
 
         self._partially_loaded = True
 
@@ -873,10 +899,10 @@ class Project:
         self.base_env_nocache = config.get_str_list("environment-nocache")
 
         # Load sandbox configuration
-        self._sandbox = config.get_mapping("sandbox")
+        self.sandbox = config.get_mapping("sandbox")
 
         # Load project split rules
-        self._splits = config.get_mapping("split-rules")
+        self.splits = config.get_mapping("split-rules")
 
         # Support backwards compatibility for fail-on-overlap
         fail_on_overlap = config.get_scalar("fail-on-overlap", None)
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 3ac201d66..c453e3c88 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -2832,9 +2832,9 @@ class Element(Plugin):
         if first_pass:
             splits = element_splits.clone()
         else:
-            assert project._splits is not None
+            assert project.splits is not None
 
-            splits = project._splits.clone()
+            splits = project.splits.clone()
             # Extend project wide split rules with any split rules defined by the element
             element_splits._composite(splits)
 
@@ -2967,7 +2967,7 @@ class Element(Plugin):
         if load_element.first_pass:
             sandbox_config = Node.from_dict({})
         else:
-            sandbox_config = project._sandbox.clone()
+            sandbox_config = project.sandbox.clone()
 
         # The default config is already composited with the project overrides
         sandbox_defaults = cls.__defaults.get_mapping(Symbol.SANDBOX, default={})