From 352f4ad9a360ca4f8f3a0b7a68106d0131eeb374 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Wed, 6 Feb 2019 17:02:25 +0000 Subject: loader.py: Stage junctions into .bst instead of a tmpdir --- buildstream/_loader/loader.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py index fc946d50b..d5dea7034 100644 --- a/buildstream/_loader/loader.py +++ b/buildstream/_loader/loader.py @@ -546,10 +546,13 @@ class Loader(): tempdir = None else: # Stage sources - os.makedirs(self._context.builddir, exist_ok=True) - basedir = tempfile.mkdtemp(prefix="{}-".format(element.normal_name), dir=self._context.builddir) - element._stage_sources_at(basedir, mount_workspaces=False) - tempdir = basedir + element._update_state() + basedir = os.path.join(self.project.directory, ".bst", "staged-junctions", + filename, element._get_cache_key()) + if not os.path.exists(basedir): + os.makedirs(basedir, exist_ok=True) + element._stage_sources_at(basedir, mount_workspaces=False) + tempdir = None # Load the project project_dir = os.path.join(basedir, element.path) -- cgit v1.2.1 From 62396af999bc19228035cdc08ec4d3de904d31a2 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Fri, 8 Feb 2019 11:49:51 +0000 Subject: loader.py: Load junction from workspace if one's open --- buildstream/_loader/loader.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py index d5dea7034..238bd9e9d 100644 --- a/buildstream/_loader/loader.py +++ b/buildstream/_loader/loader.py @@ -540,7 +540,12 @@ class Loader(): "Subproject has no ref for junction: {}".format(filename), detail=detail) - if len(sources) == 1 and sources[0]._get_local_path(): + workspace = element._get_workspace() + if workspace: + # If a workspace is open, load it from there instead + basedir = workspace.get_absolute_path() + tempdir = None + elif len(sources) == 1 and sources[0]._get_local_path(): # Optimization for junctions with a single local source basedir = sources[0]._get_local_path() tempdir = None -- cgit v1.2.1 From f1e9cb66f87e1e5a4fb83293bd7b232cd07bf602 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Fri, 8 Feb 2019 13:24:14 +0000 Subject: Remove cleanup functionality from loader Since junction check-outs are now persistent, there is no need to clean up temporary directories once finished. Now, junction checkouts are stored within the top-level project dir and will be cleaned-up when that directory is removed. This involves changes in: * loader.py: Remove cleanup logic and passing-around of tmpdir. * _project.py: Remove passing around tmpdir and calling loader's cleanup. --- buildstream/_loader/loader.py | 36 ++---------------------------------- buildstream/_project.py | 11 ++++------- 2 files changed, 6 insertions(+), 41 deletions(-) diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py index 238bd9e9d..d0cfa8522 100644 --- a/buildstream/_loader/loader.py +++ b/buildstream/_loader/loader.py @@ -20,8 +20,6 @@ import os from functools import cmp_to_key from collections.abc import Mapping -import tempfile -import shutil from .._exceptions import LoadError, LoadErrorReason from .. import Consistency @@ -49,12 +47,10 @@ from .._message import Message, MessageType # context (Context): The Context object # project (Project): The toplevel Project object # parent (Loader): A parent Loader object, in the case this is a junctioned Loader -# tempdir (str): A directory to cleanup with the Loader, given to the loader by a parent -# loader in the case that this loader is a subproject loader. # class Loader(): - def __init__(self, context, project, *, parent=None, tempdir=None): + def __init__(self, context, project, *, parent=None): # Ensure we have an absolute path for the base directory basedir = project.element_path @@ -73,7 +69,6 @@ class Loader(): self._options = project.options # Project options (OptionPool) self._basedir = basedir # Base project directory self._first_pass_options = project.first_pass_config.options # Project options (OptionPool) - self._tempdir = tempdir # A directory to cleanup self._parent = parent # The parent loader self._meta_elements = {} # Dict of resolved meta elements by name @@ -159,30 +154,6 @@ class Loader(): return ret - # cleanup(): - # - # Remove temporary checkout directories of subprojects - # - def cleanup(self): - if self._parent and not self._tempdir: - # already done - return - - # recurse - for loader in self._loaders.values(): - # value may be None with nested junctions without overrides - if loader is not None: - loader.cleanup() - - if not self._parent: - # basedir of top-level loader is never a temporary directory - return - - # safe guard to not accidentally delete directories outside builddir - if self._tempdir.startswith(self._context.builddir + os.sep): - if os.path.exists(self._tempdir): - shutil.rmtree(self._tempdir) - ########################################### # Private Methods # ########################################### @@ -544,11 +515,9 @@ class Loader(): if workspace: # If a workspace is open, load it from there instead basedir = workspace.get_absolute_path() - tempdir = None elif len(sources) == 1 and sources[0]._get_local_path(): # Optimization for junctions with a single local source basedir = sources[0]._get_local_path() - tempdir = None else: # Stage sources element._update_state() @@ -557,14 +526,13 @@ class Loader(): if not os.path.exists(basedir): os.makedirs(basedir, exist_ok=True) element._stage_sources_at(basedir, mount_workspaces=False) - tempdir = None # Load the project project_dir = os.path.join(basedir, element.path) try: from .._project import Project project = Project(project_dir, self._context, junction=element, - parent_loader=self, tempdir=tempdir) + parent_loader=self) except LoadError as e: if e.reason == LoadErrorReason.MISSING_PROJECT_CONF: raise LoadError(reason=LoadErrorReason.INVALID_JUNCTION, diff --git a/buildstream/_project.py b/buildstream/_project.py index 5f4389edb..51cdd5e2b 100644 --- a/buildstream/_project.py +++ b/buildstream/_project.py @@ -91,7 +91,7 @@ class ProjectConfig: class Project(): def __init__(self, directory, context, *, junction=None, cli_options=None, - default_mirror=None, parent_loader=None, tempdir=None): + default_mirror=None, parent_loader=None): # The project name self.name = None @@ -147,7 +147,7 @@ class Project(): self._project_includes = None profile_start(Topics.LOAD_PROJECT, self.directory.replace(os.sep, '-')) - self._load(parent_loader=parent_loader, tempdir=tempdir) + self._load(parent_loader=parent_loader) profile_end(Topics.LOAD_PROJECT, self.directory.replace(os.sep, '-')) self._partially_loaded = True @@ -389,8 +389,6 @@ class Project(): # Cleans up resources used loading elements # def cleanup(self): - self.loader.cleanup() - # Reset the element loader state Element._reset_load_state() @@ -439,7 +437,7 @@ class Project(): # # Raises: LoadError if there was a problem with the project.conf # - def _load(self, parent_loader=None, tempdir=None): + def _load(self, parent_loader=None): # Load builtin default projectfile = os.path.join(self.directory, _PROJECT_CONF_FILE) @@ -505,8 +503,7 @@ class Project(): self._fatal_warnings = _yaml.node_get(pre_config_node, list, 'fatal-warnings', default_value=[]) self.loader = Loader(self._context, self, - parent=parent_loader, - tempdir=tempdir) + parent=parent_loader) self._project_includes = Includes(self.loader, copy_tree=False) -- cgit v1.2.1