From 7892287a53c5ec3259fcd6f14736805b26b285b8 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Wed, 31 Oct 2018 13:55:55 +0000 Subject: utils.py: Add a helper for searching upwards for files i.e. with a given directory and filename, check parent directories until either a directory with the filename is found, or you reach the root of the filesystem. This is a part of #222 --- buildstream/utils.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/buildstream/utils.py b/buildstream/utils.py index 2fe9ab7dc..4da29c259 100644 --- a/buildstream/utils.py +++ b/buildstream/utils.py @@ -1259,3 +1259,34 @@ def _message_digest(message_buffer): digest.hash = sha.hexdigest() digest.size_bytes = len(message_buffer) return digest + + +# _search_upward_for_files() +# +# Searches upwards (from directory, then directory's parent directory...) +# for any of the files listed in `filenames`. +# +# If multiple filenames are specified, and present in the same directory, +# the first filename in the list will be returned. +# +# Args: +# directory (str): The directory to begin searching for files from +# filenames (list of str): The names of files to search for +# +# Returns: +# (str): The directory a file was found in, or None +# (str): The name of the first file that was found in that directory, or None +# +def _search_upward_for_files(directory, filenames): + directory = os.path.abspath(directory) + while True: + for filename in filenames: + file_path = os.path.join(directory, filename) + if os.path.isfile(file_path): + return directory, filename + + parent_dir = os.path.dirname(directory) + if directory == parent_dir: + # i.e. we've reached the root of the filesystem + return None, None + directory = parent_dir -- cgit v1.2.1 From 67c7a58d0a2c3287cba128ef1f4babc57541439e Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Thu, 25 Oct 2018 17:35:25 +0100 Subject: Create and store data inside projects when opening workspaces Changes to _context.py: * Context has been extended to contain a WorkspaceProjectCache, as there are times when we want to use it before a Workspaces can be initialised (looking up a WorkspaceProject to find the directory that the project is in) Changes to _stream.py: * Removed staging the elements from workspace_open() and workspace_reset() Changes in _workspaces.py: * A new WorkspaceProject contains all the information needed to refer back to a project from its workspace (currently this is the project path and the element used to create this workspace) * This is stored within a new WorkspaceProjectCache object, which keeps WorkspaceProjects around so they don't need to be loaded from disk repeatedly. * Workspaces has been extended to contain the WorkspaceProjectCache, and will use it when opening and closing workspaces. * Workspaces.create_workspace has been extended to handle the staging of the element into the workspace, in addition to creating the equivalent WorkspaceProject file. This is a part of #222 --- buildstream/_context.py | 15 ++- buildstream/_stream.py | 15 +-- buildstream/_workspaces.py | 242 +++++++++++++++++++++++++++++++++++++++++++-- 3 files changed, 249 insertions(+), 23 deletions(-) diff --git a/buildstream/_context.py b/buildstream/_context.py index 7ca60e7aa..55d0fd489 100644 --- a/buildstream/_context.py +++ b/buildstream/_context.py @@ -32,7 +32,7 @@ from ._message import Message, MessageType from ._profile import Topics, profile_start, profile_end from ._artifactcache import ArtifactCache from ._artifactcache.cascache import CASCache -from ._workspaces import Workspaces +from ._workspaces import Workspaces, WorkspaceProjectCache from .plugin import _plugin_lookup @@ -140,6 +140,7 @@ class Context(): self._projects = [] self._project_overrides = {} self._workspaces = None + self._workspace_project_cache = WorkspaceProjectCache() self._log_handle = None self._log_filename = None self._cascache = None @@ -285,7 +286,7 @@ class Context(): # def add_project(self, project): if not self._projects: - self._workspaces = Workspaces(project) + self._workspaces = Workspaces(project, self._workspace_project_cache) self._projects.append(project) # get_projects(): @@ -312,6 +313,16 @@ class Context(): def get_workspaces(self): return self._workspaces + # get_workspace_project_cache(): + # + # Return the WorkspaceProjectCache object used for this BuildStream invocation + # + # Returns: + # (WorkspaceProjectCache): The WorkspaceProjectCache object + # + def get_workspace_project_cache(self): + return self._workspace_project_cache + # get_overrides(): # # Fetch the override dictionary for the active project. This returns diff --git a/buildstream/_stream.py b/buildstream/_stream.py index 6f298c259..3256003c8 100644 --- a/buildstream/_stream.py +++ b/buildstream/_stream.py @@ -581,15 +581,7 @@ class Stream(): todo_elements = "\nDid not try to create workspaces for " + todo_elements raise StreamError("Failed to create workspace directory: {}".format(e) + todo_elements) from e - workspaces.create_workspace(target._get_full_name(), directory) - - if not no_checkout: - with target.timed_activity("Staging sources to {}".format(directory)): - target._open_workspace() - - # Saving the workspace once it is set up means that if the next workspace fails to be created before - # the configuration gets saved. The successfully created workspace still gets saved. - workspaces.save_config() + workspaces.create_workspace(target, directory, checkout=not no_checkout) self._message(MessageType.INFO, "Created a workspace for element: {}" .format(target._get_full_name())) @@ -672,10 +664,7 @@ class Stream(): .format(workspace_path, e)) from e workspaces.delete_workspace(element._get_full_name()) - workspaces.create_workspace(element._get_full_name(), workspace_path) - - with element.timed_activity("Staging sources to {}".format(workspace_path)): - element._open_workspace() + workspaces.create_workspace(element, workspace_path, checkout=True) self._message(MessageType.INFO, "Reset workspace for {} at: {}".format(element.name, diff --git a/buildstream/_workspaces.py b/buildstream/_workspaces.py index 468073f05..466c55ce7 100644 --- a/buildstream/_workspaces.py +++ b/buildstream/_workspaces.py @@ -25,6 +25,202 @@ from ._exceptions import LoadError, LoadErrorReason BST_WORKSPACE_FORMAT_VERSION = 3 +BST_WORKSPACE_PROJECT_FORMAT_VERSION = 1 +WORKSPACE_PROJECT_FILE = ".bstproject.yaml" + + +# WorkspaceProject() +# +# An object to contain various helper functions and data required for +# referring from a workspace back to buildstream. +# +# Args: +# directory (str): The directory that the workspace exists in. +# +class WorkspaceProject(): + def __init__(self, directory): + self._projects = [] + self._directory = directory + + # get_default_project_path() + # + # Retrieves the default path to a project. + # + # Returns: + # (str): The path to a project + # + def get_default_project_path(self): + return self._projects[0]['project-path'] + + # get_default_element() + # + # Retrieves the name of the element that owns this workspace. + # + # Returns: + # (str): The name of an element + # + def get_default_element(self): + return self._projects[0]['element-name'] + + # to_dict() + # + # Turn the members data into a dict for serialization purposes + # + # Returns: + # (dict): A dict representation of the WorkspaceProject + # + def to_dict(self): + ret = { + 'projects': self._projects, + 'format-version': BST_WORKSPACE_PROJECT_FORMAT_VERSION, + } + return ret + + # from_dict() + # + # Loads a new WorkspaceProject from a simple dictionary + # + # Args: + # directory (str): The directory that the workspace exists in + # dictionary (dict): The dict to generate a WorkspaceProject from + # + # Returns: + # (WorkspaceProject): A newly instantiated WorkspaceProject + # + @classmethod + def from_dict(cls, directory, dictionary): + # Only know how to handle one format-version at the moment. + format_version = int(dictionary['format-version']) + assert format_version == BST_WORKSPACE_PROJECT_FORMAT_VERSION, \ + "Format version {} not found in {}".format(BST_WORKSPACE_PROJECT_FORMAT_VERSION, dictionary) + + workspace_project = cls(directory) + for item in dictionary['projects']: + workspace_project.add_project(item['project-path'], item['element-name']) + + return workspace_project + + # load() + # + # Loads the WorkspaceProject for a given directory. + # + # Args: + # directory (str): The directory + # Returns: + # (WorkspaceProject): The created WorkspaceProject, if in a workspace, or + # (NoneType): None, if the directory is not inside a workspace. + # + @classmethod + def load(cls, directory): + workspace_file = os.path.join(directory, WORKSPACE_PROJECT_FILE) + if os.path.exists(workspace_file): + data_dict = _yaml.load(workspace_file) + return cls.from_dict(directory, data_dict) + else: + return None + + # write() + # + # Writes the WorkspaceProject to disk + # + def write(self): + os.makedirs(self._directory, exist_ok=True) + _yaml.dump(self.to_dict(), self.get_filename()) + + # get_filename() + # + # Returns the full path to the workspace local project file + # + def get_filename(self): + return os.path.join(self._directory, WORKSPACE_PROJECT_FILE) + + # add_project() + # + # Adds an entry containing the project's path and element's name. + # + # Args: + # project_path (str): The path to the project that opened the workspace. + # element_name (str): The name of the element that the workspace belongs to. + # + def add_project(self, project_path, element_name): + assert (project_path and element_name) + self._projects.append({'project-path': project_path, 'element-name': element_name}) + + +# WorkspaceProjectCache() +# +# A class to manage workspace project data for multiple workspaces. +# +class WorkspaceProjectCache(): + def __init__(self): + self._projects = {} # Mapping of a workspace directory to its WorkspaceProject + + # get() + # + # Returns a WorkspaceProject for a given directory, retrieving from the cache if + # present. + # + # Args: + # directory (str): The directory to search for a WorkspaceProject. + # + # Returns: + # (WorkspaceProject): The WorkspaceProject that was found for that directory. + # or (NoneType): None, if no WorkspaceProject can be found. + # + def get(self, directory): + try: + workspace_project = self._projects[directory] + except KeyError: + workspace_project = WorkspaceProject.load(directory) + if workspace_project: + self._projects[directory] = workspace_project + + return workspace_project + + # add() + # + # Adds the project path and element name to the WorkspaceProject that exists + # for that directory + # + # Args: + # directory (str): The directory to search for a WorkspaceProject. + # project_path (str): The path to the project that refers to this workspace + # element_name (str): The element in the project that was refers to this workspace + # + # Returns: + # (WorkspaceProject): The WorkspaceProject that was found for that directory. + # + def add(self, directory, project_path, element_name): + workspace_project = self.get(directory) + if not workspace_project: + workspace_project = WorkspaceProject(directory) + self._projects[directory] = workspace_project + + workspace_project.add_project(project_path, element_name) + return workspace_project + + # remove() + # + # Removes the project path and element name from the WorkspaceProject that exists + # for that directory. + # + # NOTE: This currently just deletes the file, but with support for multiple + # projects opening the same workspace, this will involve decreasing the count + # and deleting the file if there are no more projects. + # + # Args: + # directory (str): The directory to search for a WorkspaceProject. + # + def remove(self, directory): + workspace_project = self.get(directory) + if not workspace_project: + raise LoadError(LoadErrorReason.MISSING_FILE, + "Failed to find a {} file to remove".format(WORKSPACE_PROJECT_FILE)) + path = workspace_project.get_filename() + try: + os.unlink(path) + except FileNotFoundError: + pass # Workspace() @@ -199,12 +395,14 @@ class Workspace(): # # Args: # toplevel_project (Project): Top project used to resolve paths. +# workspace_project_cache (WorkspaceProjectCache): The cache of WorkspaceProjects # class Workspaces(): - def __init__(self, toplevel_project): + def __init__(self, toplevel_project, workspace_project_cache): self._toplevel_project = toplevel_project self._bst_directory = os.path.join(toplevel_project.directory, ".bst") self._workspaces = self._load_config() + self._workspace_project_cache = workspace_project_cache # list() # @@ -219,19 +417,36 @@ class Workspaces(): # create_workspace() # - # Create a workspace in the given path for the given element. + # Create a workspace in the given path for the given element, and potentially + # checks-out the target into it. # # Args: - # element_name (str) - The element name to create a workspace for + # target (Element) - The element to create a workspace for # path (str) - The path in which the workspace should be kept + # checkout (bool): Whether to check-out the element's sources into the directory # - def create_workspace(self, element_name, path): - if path.startswith(self._toplevel_project.directory): - path = os.path.relpath(path, self._toplevel_project.directory) + def create_workspace(self, target, path, *, checkout): + element_name = target._get_full_name() + project_dir = self._toplevel_project.directory + if path.startswith(project_dir): + workspace_path = os.path.relpath(path, project_dir) + else: + workspace_path = path - self._workspaces[element_name] = Workspace(self._toplevel_project, path=path) + self._workspaces[element_name] = Workspace(self._toplevel_project, path=workspace_path) - return self._workspaces[element_name] + if checkout: + with target.timed_activity("Staging sources to {}".format(path)): + target._open_workspace() + + workspace_project = self._workspace_project_cache.add(path, project_dir, element_name) + project_file_path = workspace_project.get_filename() + + if os.path.exists(project_file_path): + target.warn("{} was staged from this element's sources".format(WORKSPACE_PROJECT_FILE)) + workspace_project.write() + + self.save_config() # get_workspace() # @@ -280,8 +495,19 @@ class Workspaces(): # element_name (str) - The element name whose workspace to delete # def delete_workspace(self, element_name): + workspace = self.get_workspace(element_name) del self._workspaces[element_name] + # Remove from the cache if it exists + try: + self._workspace_project_cache.remove(workspace.get_absolute_path()) + except LoadError as e: + # We might be closing a workspace with a deleted directory + if e.reason == LoadErrorReason.MISSING_FILE: + pass + else: + raise + # save_config() # # Dump the current workspace element to the project configuration -- cgit v1.2.1 From 64836b18c9f72889d94d2fb01fda4ecc80d0816e Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Tue, 30 Oct 2018 16:08:23 +0000 Subject: _project.py: Rename _ensure_project_dir to _find_project_dir This is a part of #222 --- buildstream/_project.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/buildstream/_project.py b/buildstream/_project.py index 386036488..a8962a022 100644 --- a/buildstream/_project.py +++ b/buildstream/_project.py @@ -96,7 +96,7 @@ class Project(): self.name = None # The project directory - self.directory = self._ensure_project_dir(directory) + self.directory = self._find_project_dir(directory) # Absolute path to where elements are loaded from within the project self.element_path = None @@ -650,7 +650,7 @@ class Project(): # Source url aliases output._aliases = _yaml.node_get(config, Mapping, 'aliases', default_value={}) - # _ensure_project_dir() + # _find_project_dir() # # Returns path of the project directory, if a configuration file is found # in given directory or any of its parent directories. @@ -661,7 +661,7 @@ class Project(): # Raises: # LoadError if project.conf is not found # - def _ensure_project_dir(self, directory): + def _find_project_dir(self, directory): directory = os.path.abspath(directory) while not os.path.isfile(os.path.join(directory, _PROJECT_CONF_FILE)): parent_dir = os.path.dirname(directory) -- cgit v1.2.1 From 7deeb2c3cada7b3e779f69c5fd27069e2236f7ab Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Wed, 31 Oct 2018 13:57:13 +0000 Subject: cli.py: Use utils' search upwards helper when searching for project.conf This is a part of #222 --- buildstream/_frontend/cli.py | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py index ef0dadb13..b2ad1e8a4 100644 --- a/buildstream/_frontend/cli.py +++ b/buildstream/_frontend/cli.py @@ -59,18 +59,9 @@ def complete_target(args, incomplete): :return: all the possible user-specified completions for the param """ + from .. import utils project_conf = 'project.conf' - def ensure_project_dir(directory): - directory = os.path.abspath(directory) - while not os.path.isfile(os.path.join(directory, project_conf)): - parent_dir = os.path.dirname(directory) - if directory == parent_dir: - break - directory = parent_dir - - return directory - # First resolve the directory, in case there is an # active --directory/-C option # @@ -89,7 +80,7 @@ def complete_target(args, incomplete): else: # Check if this directory or any of its parent directories # contain a project config file - base_directory = ensure_project_dir(base_directory) + base_directory, _ = utils._search_upward_for_files(base_directory, [project_conf]) # Now parse the project.conf just to find the element path, # this is unfortunately a bit heavy. -- cgit v1.2.1 From 496f0ab74c7955fb71754babc9561a16f8dfea1c Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Wed, 5 Dec 2018 11:10:53 +0000 Subject: _project.py: Find project from workspace if outside of a project This is a part of #222 --- buildstream/_project.py | 48 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/buildstream/_project.py b/buildstream/_project.py index a8962a022..90ed58e14 100644 --- a/buildstream/_project.py +++ b/buildstream/_project.py @@ -41,6 +41,7 @@ from .element import Element from ._message import Message, MessageType from ._includes import Includes from ._platform import Platform +from ._workspaces import WORKSPACE_PROJECT_FILE # Project Configuration file @@ -95,8 +96,10 @@ class Project(): # The project name self.name = None - # The project directory - self.directory = self._find_project_dir(directory) + self._context = context # The invocation Context, a private member + + # The project directory, and whether the element whose workspace it was invoked from + self.directory, self._invoked_from_workspace_element = self._find_project_dir(directory) # Absolute path to where elements are loaded from within the project self.element_path = None @@ -117,7 +120,6 @@ class Project(): # # Private Members # - self._context = context # The invocation Context self._default_mirror = default_mirror # The name of the preferred mirror. @@ -371,6 +373,14 @@ class Project(): self._load_second_pass() + # invoked_from_workspace_element() + # + # Returns the element whose workspace was used to invoke buildstream + # if buildstream was invoked from an external workspace + # + def invoked_from_workspace_element(self): + return self._invoked_from_workspace_element + # cleanup() # # Cleans up resources used loading elements @@ -661,18 +671,30 @@ class Project(): # Raises: # LoadError if project.conf is not found # + # Returns: + # (str) - the directory that contains the project, and + # (str) - the name of the element required to find the project, or None + # def _find_project_dir(self, directory): - directory = os.path.abspath(directory) - while not os.path.isfile(os.path.join(directory, _PROJECT_CONF_FILE)): - parent_dir = os.path.dirname(directory) - if directory == parent_dir: - raise LoadError( - LoadErrorReason.MISSING_PROJECT_CONF, - '{} not found in current directory or any of its parent directories' - .format(_PROJECT_CONF_FILE)) - directory = parent_dir + workspace_element = None + found_directory, filename = utils._search_upward_for_files( + directory, [_PROJECT_CONF_FILE, WORKSPACE_PROJECT_FILE] + ) + if filename == _PROJECT_CONF_FILE: + project_directory = found_directory + elif filename == WORKSPACE_PROJECT_FILE: + workspace_project_cache = self._context.get_workspace_project_cache() + workspace_project = workspace_project_cache.get(found_directory) + if workspace_project: + project_directory = workspace_project.get_default_project_path() + workspace_element = workspace_project.get_default_element() + else: + raise LoadError( + LoadErrorReason.MISSING_PROJECT_CONF, + '{} not found in current directory or any of its parent directories' + .format(_PROJECT_CONF_FILE)) - return directory + return project_directory, workspace_element def _load_plugin_factories(self, config, output): plugin_source_origins = [] # Origins of custom sources -- cgit v1.2.1 From 921f2bcbd6bc050c77a6099d9b90705953f93622 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Tue, 6 Nov 2018 17:33:03 +0000 Subject: _workspaces.py: Do not include .bstproject.yaml in the cache key This is a part of #222 --- buildstream/_workspaces.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/buildstream/_workspaces.py b/buildstream/_workspaces.py index 466c55ce7..24a3cc8d3 100644 --- a/buildstream/_workspaces.py +++ b/buildstream/_workspaces.py @@ -370,10 +370,15 @@ class Workspace(): if recalculate or self._key is None: fullpath = self.get_absolute_path() + excluded_files = (WORKSPACE_PROJECT_FILE,) + # Get a list of tuples of the the project relative paths and fullpaths if os.path.isdir(fullpath): filelist = utils.list_relative_paths(fullpath) - filelist = [(relpath, os.path.join(fullpath, relpath)) for relpath in filelist] + filelist = [ + (relpath, os.path.join(fullpath, relpath)) for relpath in filelist + if relpath not in excluded_files + ] else: filelist = [(self.get_absolute_path(), fullpath)] -- cgit v1.2.1 From f145a3e4c66017ab5b2de492c4d6de538d4ec224 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Thu, 8 Nov 2018 18:16:20 +0000 Subject: cli: Interactively warn if the user is trying to close the workspace they're using to load the project This involves changes in: * _stream.py: * Add the helper Stream.workspace_is_required() * userconfig.yaml: * Add a default value for prompt.really-workspace-close-project-inaccessible * _context.py: * Load the prompt 'really-workspace-close-project-inaccessible' from user config. * cli.py: * If buildstream is invoked interactively, prompt the user to confirm that they want to close the workspace they're using to load this project. This is a part of #222 --- buildstream/_context.py | 7 +++++++ buildstream/_frontend/cli.py | 10 +++++++++- buildstream/_stream.py | 14 ++++++++++++++ buildstream/data/userconfig.yaml | 8 ++++++++ 4 files changed, 38 insertions(+), 1 deletion(-) diff --git a/buildstream/_context.py b/buildstream/_context.py index 55d0fd489..5211fe54d 100644 --- a/buildstream/_context.py +++ b/buildstream/_context.py @@ -122,6 +122,10 @@ class Context(): # remove a workspace directory. self.prompt_workspace_close_remove_dir = None + # Boolean, whether we double-check with the user that they meant to + # close the workspace when they're using it to access the project. + self.prompt_workspace_close_project_inaccessible = None + # Boolean, whether we double-check with the user that they meant to do # a hard reset of a workspace, potentially losing changes. self.prompt_workspace_reset_hard = None @@ -251,12 +255,15 @@ class Context(): defaults, Mapping, 'prompt') _yaml.node_validate(prompt, [ 'auto-init', 'really-workspace-close-remove-dir', + 'really-workspace-close-project-inaccessible', 'really-workspace-reset-hard', ]) self.prompt_auto_init = _node_get_option_str( prompt, 'auto-init', ['ask', 'no']) == 'ask' self.prompt_workspace_close_remove_dir = _node_get_option_str( prompt, 'really-workspace-close-remove-dir', ['ask', 'yes']) == 'ask' + self.prompt_workspace_close_project_inaccessible = _node_get_option_str( + prompt, 'really-workspace-close-project-inaccessible', ['ask', 'yes']) == 'ask' self.prompt_workspace_reset_hard = _node_get_option_str( prompt, 'really-workspace-reset-hard', ['ask', 'yes']) == 'ask' diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py index b2ad1e8a4..e1fab38a0 100644 --- a/buildstream/_frontend/cli.py +++ b/buildstream/_frontend/cli.py @@ -763,11 +763,19 @@ def workspace_close(app, remove_dir, all_, elements): elements = app.stream.redirect_element_names(elements) - # Check that the workspaces in question exist + # Check that the workspaces in question exist, and that it's safe to + # remove them. nonexisting = [] for element_name in elements: if not app.stream.workspace_exists(element_name): nonexisting.append(element_name) + if (app.stream.workspace_is_required(element_name) and app.interactive and + app.context.prompt_workspace_close_project_inaccessible): + click.echo("Removing '{}' will prevent you from running " + "BuildStream commands from the current directory".format(element_name)) + if not click.confirm('Are you sure you want to close this workspace?'): + click.echo('Aborting', err=True) + sys.exit(-1) if nonexisting: raise AppError("Workspace does not exist", detail="\n".join(nonexisting)) diff --git a/buildstream/_stream.py b/buildstream/_stream.py index 3256003c8..6d4af1e79 100644 --- a/buildstream/_stream.py +++ b/buildstream/_stream.py @@ -696,6 +696,20 @@ class Stream(): return False + # workspace_is_required() + # + # Checks whether the workspace belonging to element_name is required to + # load the project + # + # Args: + # element_name (str): The element whose workspace may be required + # + # Returns: + # (bool): True if the workspace is required + def workspace_is_required(self, element_name): + invoked_elm = self._project.invoked_from_workspace_element() + return invoked_elm == element_name + # workspace_list # # Serializes the workspaces and dumps them in YAML to stdout. diff --git a/buildstream/data/userconfig.yaml b/buildstream/data/userconfig.yaml index e81961c53..4429d1f6d 100644 --- a/buildstream/data/userconfig.yaml +++ b/buildstream/data/userconfig.yaml @@ -128,6 +128,14 @@ prompt: # really-workspace-close-remove-dir: ask + # Whether to really proceed with 'bst workspace close' when doing so would + # stop them from running bst commands in this workspace. + # + # ask - Ask the user if they are sure. + # yes - Always close, without asking. + # + really-workspace-close-project-inaccessible: ask + # Whether to really proceed with 'bst workspace reset' doing a hard reset of # a workspace, potentially losing changes. # -- cgit v1.2.1 From 494cb7c65b62b915fe036d8ff39b522db0d1703c Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Thu, 1 Nov 2018 18:22:41 +0000 Subject: tests: Test bst commands from an external workspace This is a part of #222 --- tests/frontend/cross_junction_workspace.py | 20 ++++ tests/frontend/workspace.py | 147 ++++++++++++++++++++++++++++- tests/integration/shell.py | 26 +++++ 3 files changed, 188 insertions(+), 5 deletions(-) diff --git a/tests/frontend/cross_junction_workspace.py b/tests/frontend/cross_junction_workspace.py index fb2b34c43..ad2a62626 100644 --- a/tests/frontend/cross_junction_workspace.py +++ b/tests/frontend/cross_junction_workspace.py @@ -115,3 +115,23 @@ def test_close_all_cross_junction(cli, tmpdir): assert isinstance(loaded.get('workspaces'), list) workspaces = loaded['workspaces'] assert len(workspaces) == 0 + + +def test_subdir_command_cross_junction(cli, tmpdir): + # i.e. commands can be run successfully from a subdirectory of the + # junction's workspace, in case project loading logic has gone wrong + project = prepare_junction_project(cli, tmpdir) + workspace = os.path.join(str(tmpdir), 'workspace') + junction_element = 'sub.bst' + + # Open the junction as a workspace + args = ['workspace', 'open', '--directory', workspace, junction_element] + result = cli.run(project=project, args=args) + result.assert_success() + + # Run commands from a subdirectory of the workspace + newdir = os.path.join(str(workspace), "newdir") + element_name = 'data.bst' + os.makedirs(newdir) + result = cli.run(project=str(workspace), args=['-C', newdir, 'show', element_name]) + result.assert_success() diff --git a/tests/frontend/workspace.py b/tests/frontend/workspace.py index bc928ad51..baca869bf 100644 --- a/tests/frontend/workspace.py +++ b/tests/frontend/workspace.py @@ -31,6 +31,7 @@ import shutil import subprocess from ruamel.yaml.comments import CommentedSet from tests.testutils import cli, create_repo, ALL_REPO_KINDS, wait_for_cache_granularity +from tests.testutils import create_artifact_share from buildstream import _yaml from buildstream._exceptions import ErrorDomain, LoadError, LoadErrorReason @@ -615,9 +616,12 @@ def test_list(cli, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) @pytest.mark.parametrize("kind", repo_kinds) @pytest.mark.parametrize("strict", [("strict"), ("non-strict")]) -def test_build(cli, tmpdir, datafiles, kind, strict): +@pytest.mark.parametrize("call_from", [("project"), ("workspace")]) +def test_build(cli, tmpdir_factory, datafiles, kind, strict, call_from): + tmpdir = tmpdir_factory.mktemp('') element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, kind, False) checkout = os.path.join(str(tmpdir), 'checkout') + args_pre = ['-C', workspace] if call_from == "workspace" else [] # Modify workspace shutil.rmtree(os.path.join(workspace, 'usr', 'bin')) @@ -640,15 +644,14 @@ def test_build(cli, tmpdir, datafiles, kind, strict): # Build modified workspace assert cli.get_element_state(project, element_name) == 'buildable' assert cli.get_element_key(project, element_name) == "{:?<64}".format('') - result = cli.run(project=project, args=['build', element_name]) + result = cli.run(project=project, args=args_pre + ['build', element_name]) result.assert_success() assert cli.get_element_state(project, element_name) == 'cached' assert cli.get_element_key(project, element_name) != "{:?<64}".format('') # Checkout the result - result = cli.run(project=project, args=[ - 'checkout', element_name, checkout - ]) + result = cli.run(project=project, + args=args_pre + ['checkout', element_name, checkout]) result.assert_success() # Check that the pony.conf from the modified workspace exists @@ -1055,3 +1058,137 @@ def test_multiple_failed_builds(cli, tmpdir, datafiles): result = cli.run(project=project, args=["build", element_name]) assert "BUG" not in result.stderr assert cli.get_element_state(project, element_name) != "cached" + + +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.parametrize('subdir', [True, False], ids=["subdir", "no-subdir"]) +def test_external_fetch(cli, datafiles, tmpdir_factory, subdir): + # Fetching from a workspace outside a project doesn't fail horribly + tmpdir = tmpdir_factory.mktemp('') + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False) + + if subdir: + call_dir = os.path.join(workspace, 'usr') + else: + call_dir = workspace + + result = cli.run(project=project, args=['-C', call_dir, 'fetch', element_name]) + result.assert_success() + + # We already fetched it by opening the workspace, but we're also checking + # `bst show` works here + assert cli.get_element_state(project, element_name) == 'buildable' + + +@pytest.mark.datafiles(DATA_DIR) +def test_external_push_pull(cli, datafiles, tmpdir_factory): + # Pushing and pulling to/from an artifact cache works from an external workspace + tmpdir = tmpdir_factory.mktemp('') + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False) + + with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + result = cli.run(project=project, args=['-C', workspace, 'build', element_name]) + result.assert_success() + + cli.configure({ + 'artifacts': {'url': share.repo, 'push': True} + }) + + result = cli.run(project=project, args=['-C', workspace, 'push', element_name]) + result.assert_success() + + result = cli.run(project=project, args=['-C', workspace, 'pull', '--deps', 'all', element_name]) + result.assert_success() + + +@pytest.mark.datafiles(DATA_DIR) +def test_external_track(cli, datafiles, tmpdir_factory): + # Tracking does not get horribly confused + tmpdir = tmpdir_factory.mktemp('') + element_name, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", True) + + # The workspace is necessarily already tracked, so we only care that + # there's no weird errors. + result = cli.run(project=project, args=['-C', workspace, 'track', element_name]) + result.assert_success() + + +@pytest.mark.datafiles(DATA_DIR) +def test_external_open_other(cli, datafiles, tmpdir_factory): + # From inside an external workspace, open another workspace + tmpdir1 = tmpdir_factory.mktemp('') + tmpdir2 = tmpdir_factory.mktemp('') + # Making use of the assumption that it's the same project in both invocations of open_workspace + alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha") + beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta") + + # Closing the other element first, because I'm too lazy to create an + # element without opening it + result = cli.run(project=project, args=['workspace', 'close', beta_element]) + result.assert_success() + + result = cli.run(project=project, args=[ + '-C', alpha_workspace, 'workspace', 'open', '--force', '--directory', beta_workspace, beta_element + ]) + result.assert_success() + + +@pytest.mark.datafiles(DATA_DIR) +def test_external_close_other(cli, datafiles, tmpdir_factory): + # From inside an external workspace, close the other workspace + tmpdir1 = tmpdir_factory.mktemp('') + tmpdir2 = tmpdir_factory.mktemp('') + # Making use of the assumption that it's the same project in both invocations of open_workspace + alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha") + beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta") + + result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close', beta_element]) + result.assert_success() + + +@pytest.mark.datafiles(DATA_DIR) +def test_external_close_self(cli, datafiles, tmpdir_factory): + # From inside an external workspace, close it + tmpdir1 = tmpdir_factory.mktemp('') + tmpdir2 = tmpdir_factory.mktemp('') + # Making use of the assumption that it's the same project in both invocations of open_workspace + alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha") + beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta") + + result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'close', alpha_element]) + result.assert_success() + + +@pytest.mark.datafiles(DATA_DIR) +def test_external_reset_other(cli, datafiles, tmpdir_factory): + tmpdir1 = tmpdir_factory.mktemp('') + tmpdir2 = tmpdir_factory.mktemp('') + # Making use of the assumption that it's the same project in both invocations of open_workspace + alpha_element, project, alpha_workspace = open_workspace(cli, tmpdir1, datafiles, "git", False, suffix="-alpha") + beta_element, _, beta_workspace = open_workspace(cli, tmpdir2, datafiles, "git", False, suffix="-beta") + + result = cli.run(project=project, args=['-C', alpha_workspace, 'workspace', 'reset', beta_element]) + result.assert_success() + + +@pytest.mark.datafiles(DATA_DIR) +def test_external_reset_self(cli, datafiles, tmpdir): + element, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False) + + # Command succeeds + result = cli.run(project=project, args=['-C', workspace, 'workspace', 'reset', element]) + result.assert_success() + + # Successive commands still work (i.e. .bstproject.yaml hasn't been deleted) + result = cli.run(project=project, args=['-C', workspace, 'workspace', 'list']) + result.assert_success() + + +@pytest.mark.datafiles(DATA_DIR) +def test_external_list(cli, datafiles, tmpdir_factory): + tmpdir = tmpdir_factory.mktemp('') + # Making use of the assumption that it's the same project in both invocations of open_workspace + element, project, workspace = open_workspace(cli, tmpdir, datafiles, "git", False) + + result = cli.run(project=project, args=['-C', workspace, 'workspace', 'list']) + result.assert_success() diff --git a/tests/integration/shell.py b/tests/integration/shell.py index 68535bfdc..1cec21818 100644 --- a/tests/integration/shell.py +++ b/tests/integration/shell.py @@ -353,3 +353,29 @@ def test_integration_devices(cli, tmpdir, datafiles): result = execute_shell(cli, project, ["true"], element=element_name) assert result.exit_code == 0 + + +# Test that a shell can be opened from an external workspace +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.parametrize("build_shell", [("build"), ("nobuild")]) +@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux') +def test_integration_external_workspace(cli, tmpdir_factory, datafiles, build_shell): + tmpdir = tmpdir_factory.mktemp("") + project = os.path.join(datafiles.dirname, datafiles.basename) + element_name = 'autotools/amhello.bst' + workspace_dir = os.path.join(str(tmpdir), 'workspace') + + result = cli.run(project=project, args=[ + 'workspace', 'open', '--directory', workspace_dir, element_name + ]) + result.assert_success() + + result = cli.run(project=project, args=['-C', workspace_dir, 'build', element_name]) + result.assert_success() + + command = ['shell'] + if build_shell == 'build': + command.append('--build') + command.extend([element_name, '--', 'true']) + result = cli.run(project=project, cwd=workspace_dir, args=command) + result.assert_success() -- cgit v1.2.1 From fd1c5c5a5bc693a35dff7b27259f87e38b5c2654 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Wed, 21 Nov 2018 15:10:35 +0000 Subject: NEWS: Add an entry for being able to run commands from a workspace This is a part of #222 --- NEWS | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NEWS b/NEWS index b8003ceb6..d11e2c5c4 100644 --- a/NEWS +++ b/NEWS @@ -83,6 +83,10 @@ buildstream 1.3.1 plugin has now a tag tracking feature instead. This can be enabled by setting 'track-tags'. + o Opening a workspace now creates a .bstproject.yaml file that allows buildstream + commands to be run from a workspace that is not inside a project. + + ================= buildstream 1.1.5 ================= -- cgit v1.2.1