From b99451672bb4e9b6bb5c1d0583bc2608c56a3934 Mon Sep 17 00:00:00 2001 From: Darius Makovsky Date: Mon, 19 Aug 2019 13:20:38 +0100 Subject: Remove workspace mounting for sandboxes Workspaces will be handled via the workspace source plugin methods. This does not currently support reflecting build artifacts in the open workspace. tests: Mark incremental workspace builds as xfail (strict) --- src/buildstream/_loader/loader.py | 2 +- src/buildstream/_stream.py | 2 +- src/buildstream/element.py | 47 ++++++------------------------ src/buildstream/plugins/elements/import.py | 2 +- tests/integration/workspace.py | 2 ++ 5 files changed, 14 insertions(+), 41 deletions(-) diff --git a/src/buildstream/_loader/loader.py b/src/buildstream/_loader/loader.py index 6a8db892f..75db90d15 100644 --- a/src/buildstream/_loader/loader.py +++ b/src/buildstream/_loader/loader.py @@ -641,7 +641,7 @@ class Loader(): filename, element._get_cache_key()) if not os.path.exists(basedir): os.makedirs(basedir, exist_ok=True) - element._stage_sources_at(basedir, mount_workspaces=False) + element._stage_sources_at(basedir) # Load the project project_dir = os.path.join(basedir, element.path) diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py index 83db24d94..75b3dd84e 100644 --- a/src/buildstream/_stream.py +++ b/src/buildstream/_stream.py @@ -1528,7 +1528,7 @@ class Stream(): element_source_dir = self._get_element_dirname(directory, element) if list(element.sources()): os.makedirs(element_source_dir) - element._stage_sources_at(element_source_dir, mount_workspaces=False) + element._stage_sources_at(element_source_dir) # Create a tarball from the content of directory def _create_tarball(self, directory, tar_name, compression): diff --git a/src/buildstream/element.py b/src/buildstream/element.py index 0ad2da17f..4ebb17d09 100644 --- a/src/buildstream/element.py +++ b/src/buildstream/element.py @@ -92,7 +92,7 @@ from ._variables import Variables from ._versions import BST_CORE_ARTIFACT_VERSION from ._exceptions import BstError, LoadError, LoadErrorReason, ImplError, \ ErrorDomain, SourceCacheError -from .utils import FileListResult, UtilError +from .utils import FileListResult from . import utils from . import _cachekey from . import _signals @@ -1449,21 +1449,18 @@ class Element(Plugin): # Args: # sandbox (:class:`.Sandbox`): The build sandbox # directory (str): An absolute path to stage the sources at - # mount_workspaces (bool): mount workspaces if True, copy otherwise # - def _stage_sources_in_sandbox(self, sandbox, directory, mount_workspaces=True): + def _stage_sources_in_sandbox(self, sandbox, directory): # Only artifact caches that implement diff() are allowed to # perform incremental builds. - if mount_workspaces and self.__can_build_incrementally(): - workspace = self._get_workspace() + if self.__can_build_incrementally(): sandbox.mark_directory(directory) - sandbox._set_mount_source(directory, workspace.get_absolute_path()) # Stage all sources that need to be copied sandbox_vroot = sandbox.get_virtual_directory() host_vdirectory = sandbox_vroot.descend(*directory.lstrip(os.sep).split(os.sep), create=True) - self._stage_sources_at(host_vdirectory, mount_workspaces=mount_workspaces, usebuildtree=sandbox._usebuildtree) + self._stage_sources_at(host_vdirectory, usebuildtree=sandbox._usebuildtree) # _stage_sources_at(): # @@ -1471,10 +1468,9 @@ class Element(Plugin): # # Args: # vdirectory (:class:`.storage.Directory`): A virtual directory object to stage sources into. - # mount_workspaces (bool): mount workspaces if True, copy otherwise # usebuildtree (bool): use a the elements build tree as its source. # - def _stage_sources_at(self, vdirectory, mount_workspaces=True, usebuildtree=False): + def _stage_sources_at(self, vdirectory, usebuildtree=False): context = self._get_context() @@ -1490,24 +1486,16 @@ class Element(Plugin): if not vdirectory.is_empty(): raise ElementError("Staging directory '{}' is not empty".format(vdirectory)) - workspace = self._get_workspace() - if workspace: - # If mount_workspaces is set and we're doing incremental builds, - # the workspace is already mounted into the sandbox. - if not (mount_workspaces and self.__can_build_incrementally()): - with self.timed_activity("Staging local files at {}" - .format(workspace.get_absolute_path())): - workspace.stage(import_dir) - # Check if we have a cached buildtree to use - elif usebuildtree: + if usebuildtree: import_dir = self.__artifact.get_buildtree() if import_dir.is_empty(): detail = "Element type either does not expect a buildtree or it was explictily cached without one." self.warn("WARNING: {} Artifact contains an empty buildtree".format(self.name), detail=detail) - # No workspace or cached buildtree, stage source from source cache + # No cached buildtree, stage source from source cache else: + # Assert sources are cached assert self._source_cached() @@ -1522,6 +1510,7 @@ class Element(Plugin): for source in self.__sources[last_required_previous_ix:]: source_dir = sourcecache.export(source) import_dir.import_files(source_dir) + except SourceCacheError as e: raise ElementError("Error trying to export source for {}: {}" .format(self.name, e)) @@ -1733,24 +1722,6 @@ class Element(Plugin): # Shelling into a sandbox is useful to debug this error e.sandbox = True - # If there is a workspace open on this element, it will have - # been mounted for sandbox invocations instead of being staged. - # - # In order to preserve the correct failure state, we need to - # copy over the workspace files into the appropriate directory - # in the sandbox. - # - workspace = self._get_workspace() - if workspace and self.__staged_sources_directory: - sandbox_vroot = sandbox.get_virtual_directory() - path_components = self.__staged_sources_directory.lstrip(os.sep).split(os.sep) - sandbox_vpath = sandbox_vroot.descend(*path_components) - try: - sandbox_vpath.import_files(workspace.get_absolute_path()) - except UtilError as e2: - self.warn("Failed to preserve workspace state for failed build sysroot: {}" - .format(e2)) - self.__set_build_result(success=False, description=str(e), detail=e.detail) self._cache_artifact(rootdir, sandbox, e.collect) diff --git a/src/buildstream/plugins/elements/import.py b/src/buildstream/plugins/elements/import.py index 6ae8cef46..3a15aa7d3 100644 --- a/src/buildstream/plugins/elements/import.py +++ b/src/buildstream/plugins/elements/import.py @@ -75,7 +75,7 @@ class ImportElement(Element): # Stage sources into the input directory # Do not mount workspaces as the files are copied from outside the sandbox - self._stage_sources_in_sandbox(sandbox, 'input', mount_workspaces=False) + self._stage_sources_in_sandbox(sandbox, 'input') rootdir = sandbox.get_virtual_directory() inputdir = rootdir.descend('input') diff --git a/tests/integration/workspace.py b/tests/integration/workspace.py index 9e74e574f..721ec4a53 100644 --- a/tests/integration/workspace.py +++ b/tests/integration/workspace.py @@ -53,6 +53,7 @@ def test_workspace_mount_on_read_only_directory(cli, datafiles): @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') @pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True) +@pytest.mark.xfail(reason="Incremental builds are currently incompatible with workspace source plugin.", strict=True) def test_workspace_commanddir(cli, datafiles): project = str(datafiles) workspace = os.path.join(cli.directory, 'workspace') @@ -256,6 +257,7 @@ def test_updated_dependency_nested(cli, datafiles): @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') @pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True) +@pytest.mark.xfail(reason="Incremental builds are currently incompatible with workspace source plugin.", strict=True) def test_incremental_configure_commands_run_only_once(cli, datafiles): project = str(datafiles) workspace = os.path.join(cli.directory, 'workspace') -- cgit v1.2.1