_frontend: Fixed yesterdays build breakage.

The click completions code was written based on my branch
of click master, which I had been running locally.

Yesterday it worked but only against master, this patch
adds some extra customizations so that we can handle specific
arguments (like bst file targets) specially, and now it works
with stable releases of click, which means buildstream is
no longer broken also.

2 files changed, 115 insertions, 96 deletions

diff --git a/buildstream/_frontend/complete.py b/buildstream/_frontend/complete.py
index 11a2dfecf..fef6a624f 100644
--- a/buildstream/_frontend/complete.py
+++ b/buildstream/_frontend/complete.py
@@ -27,9 +27,6 @@ import click
 from click.parser import split_arg_string
 from click.core import MultiCommand, Option, Argument
 
-from .. import _yaml
-from .. import LoadError
-
 
 WORDBREAK = '='
 
@@ -45,7 +42,13 @@ COMPLETION_SCRIPT = '''
 complete -F %(complete_func)s -o nospace %(script_names)s
 '''
 
-_invalid_ident_char_re = re.compile(r'[^a-zA-Z0-9_]')
+
+# An exception for our custom completion handler to
+# indicate that it does not want to handle completion
+# for this parameter
+#
+class CompleteUnhandled(Exception):
+    pass
 
 
 def complete_path(path_type, incomplete, base_directory='.'):
@@ -69,6 +72,9 @@ def complete_path(path_type, incomplete, base_directory='.'):
         else:
             base_path = os.path.join(base_directory, base_path)
 
+    elif os.path.isdir(incomplete):
+        base_path = incomplete
+
     try:
         if base_path:
             if os.path.isdir(base_path):
@@ -80,12 +86,27 @@ def complete_path(path_type, incomplete, base_directory='.'):
         # ignore this and avoid a stack trace
         pass
 
-    return [
+    base_directory_slash = base_directory + os.sep
+    base_directory_len = len(base_directory) + len(os.sep)
+
+    def fix_path(path):
         # Append slashes to any entries which are directories, or
         # spaces for other files since they cannot be further completed
-        e + os.path.sep if os.path.isdir(e) else e + " "
+        if os.path.isdir(path) and not path.endswith(os.sep):
+            path = path + os.sep
+        else:
+            path = path + " "
+
+        # Remove the artificial leading path portion which
+        # may have been prepended for search purposes.
+        if path.startswith(base_directory_slash):
+            path = path[base_directory_len:]
 
-        for e in sorted(entries)
+        return path
+
+    return [
+        # Return an appropriate path for each entry
+        fix_path(e) for e in sorted(entries)
 
         # Filter out non directory elements when searching for a directory,
         # the opposite is fine, however.
@@ -189,7 +210,7 @@ def is_incomplete_argument(current_params, cmd_param):
     return False
 
 
-def get_user_autocompletions(ctx, args, incomplete, cmd_param):
+def get_user_autocompletions(ctx, args, incomplete, cmd_param, override):
     """
     :param ctx: context associated with the parsed command
     :param args: full list of args
@@ -197,15 +218,18 @@ def get_user_autocompletions(ctx, args, incomplete, cmd_param):
     :param cmd_param: command definition
     :return: all the possible user-specified completions for the param
     """
-    if cmd_param.autocompletion is not None:
-        return cmd_param.autocompletion(ctx=ctx,
-                                        args=args,
-                                        incomplete=incomplete)
-    else:
+
+    # Use the type specific default completions unless it was overridden
+    try:
+        return override(cmd_param=cmd_param,
+                        ctx=ctx,
+                        args=args,
+                        incomplete=incomplete)
+    except CompleteUnhandled:
         return get_param_type_completion(cmd_param.type, incomplete) or []
 
 
-def get_choices(cli, prog_name, args, incomplete):
+def get_choices(cli, prog_name, args, incomplete, override):
     """
     :param cli: command definition
     :param prog_name: the program that is running
@@ -241,14 +265,14 @@ def get_choices(cli, prog_name, args, incomplete):
         # completion for option values by choices
         for cmd_param in ctx.command.params:
             if isinstance(cmd_param, Option) and is_incomplete_option(all_args, cmd_param):
-                choices.extend(get_user_autocompletions(ctx, all_args, incomplete, cmd_param))
+                choices.extend(get_user_autocompletions(ctx, all_args, incomplete, cmd_param, override))
                 found_param = True
                 break
     if not found_param:
         # completion for argument values by choices
         for cmd_param in ctx.command.params:
             if isinstance(cmd_param, Argument) and is_incomplete_argument(ctx.params, cmd_param):
-                choices.extend(get_user_autocompletions(ctx, all_args, incomplete, cmd_param))
+                choices.extend(get_user_autocompletions(ctx, all_args, incomplete, cmd_param, override))
                 found_param = True
                 break
 
@@ -267,7 +291,7 @@ def get_choices(cli, prog_name, args, incomplete):
             yield item
 
 
-def do_complete(cli, prog_name):
+def do_complete(cli, prog_name, override):
     cwords = split_arg_string(os.environ['COMP_WORDS'])
     cword = int(os.environ['COMP_CWORD'])
     args = cwords[1:cword]
@@ -276,16 +300,16 @@ def do_complete(cli, prog_name):
     except IndexError:
         incomplete = ''
 
-    for item in get_choices(cli, prog_name, args, incomplete):
+    for item in get_choices(cli, prog_name, args, incomplete, override):
         click.echo(item)
 
 
-def bashcomplete(cli, prog_name, complete_instr):
+def bashcomplete(cli, prog_name, complete_instr, override):
     if complete_instr == 'source':
         click.echo(get_completion_script(prog_name, '_BST_COMPLETION'))
         return True
     elif complete_instr == 'complete':
-        do_complete(cli, prog_name)
+        do_complete(cli, prog_name, override)
         return True
     return False
 
@@ -301,56 +325,9 @@ def fast_exit(code):
 
 # Main function called from main.py at startup here
 #
-def main_bashcomplete(cmd, prog_name):
+def main_bashcomplete(cmd, prog_name, override):
     """Internal handler for the bash completion support."""
     complete_instr = os.environ.get('_BST_COMPLETION')
 
-    if complete_instr and bashcomplete(cmd, prog_name, complete_instr):
+    if complete_instr and bashcomplete(cmd, prog_name, complete_instr, override):
         fast_exit(1)
-
-
-# Special completion for completing the target options
-def complete_target(ctx, args, incomplete):
-    app = ctx.obj
-
-    # First resolve the directory, in case there is an
-    # active --directory/-C option
-    #
-    base_directory = '.'
-    idx = -1
-    try:
-        idx = args.index('-C')
-    except ValueError:
-        try:
-            idx = args.index('--directory')
-        except ValueError:
-            pass
-
-    if idx >= 0 and len(args) > idx + 1:
-        base_directory = args[idx + 1]
-
-    # Now parse the project.conf just to find the element path,
-    # this is unfortunately a bit heavy.
-    project_file = os.path.join(base_directory, 'project.conf')
-    element_directory = None
-    try:
-        project = _yaml.load(project_file)
-        element_directory = project['element-path']
-    except LoadError:
-        # If there is no project directory in context, just dont
-        # even bother trying to complete anything.
-        return []
-
-    # If a project was loaded, use it's element-path to
-    # adjust our completion's base directory
-    if element_directory:
-        base_directory = os.path.join(base_directory, element_directory)
-
-    entries = complete_path("File", incomplete, base_directory=base_directory)
-    prefix_len = len(base_directory)
-    filtered = [
-        e[prefix_len + 1:] if base_directory in e else e
-        for e in entries
-    ]
-
-    return filtered
diff --git a/buildstream/_frontend/main.py b/buildstream/_frontend/main.py
index df4336f6b..9dff2605e 100644
--- a/buildstream/_frontend/main.py
+++ b/buildstream/_frontend/main.py
@@ -25,7 +25,7 @@ from contextlib import contextmanager
 from blessings import Terminal
 
 # Import buildstream public symbols
-from .. import Context, Project, Scope, Consistency
+from .. import Context, Project, Scope, Consistency, LoadError
 
 # Import various buildstream internals
 from ..exceptions import _BstError
@@ -33,10 +33,11 @@ from .._message import MessageType, unconditional_messages
 from .._pipeline import Pipeline, PipelineError
 from .._scheduler import Scheduler
 from .._profile import Topics, profile_start, profile_end
+from .. import _yaml
 
 # Import frontend assets
 from . import Profile, LogLine, Status
-from .complete import main_bashcomplete, complete_target
+from .complete import main_bashcomplete, complete_path, CompleteUnhandled
 
 # Some globals resolved for default arguments in the cli
 build_stream_version = pkg_resources.require("buildstream")[0].version
@@ -46,13 +47,66 @@ _, _, _, _, host_machine = os.uname()
 ##################################################################
 #            Override of click's main entry point                #
 ##################################################################
+
+# Special completion for completing the bst elements in a project dir
+def complete_target(ctx, args, incomplete):
+    app = ctx.obj
+
+    # First resolve the directory, in case there is an
+    # active --directory/-C option
+    #
+    base_directory = '.'
+    idx = -1
+    try:
+        idx = args.index('-C')
+    except ValueError:
+        try:
+            idx = args.index('--directory')
+        except ValueError:
+            pass
+
+    if idx >= 0 and len(args) > idx + 1:
+        base_directory = args[idx + 1]
+
+    # Now parse the project.conf just to find the element path,
+    # this is unfortunately a bit heavy.
+    project_file = os.path.join(base_directory, 'project.conf')
+    element_directory = None
+    try:
+        project = _yaml.load(project_file)
+        element_directory = project['element-path']
+    except LoadError:
+        # If there is no project directory in context, just dont
+        # even bother trying to complete anything.
+        return []
+
+    # If a project was loaded, use it's element-path to
+    # adjust our completion's base directory
+    if element_directory:
+        base_directory = os.path.join(base_directory, element_directory)
+
+    return complete_path("File", incomplete, base_directory=base_directory)
+
+
+def override_completions(cmd_param, ctx, args, incomplete):
+
+    # We can't easily extend click's data structures without
+    # modifying click itself, so just do some weak special casing
+    # right here and select which parameters we want to handle specially.
+    if isinstance(cmd_param.type, click.Path) and \
+       (cmd_param.name == 'target' or cmd_param.name == 'element'):
+        return complete_target(ctx, args, incomplete)
+
+    raise CompleteUnhandled()
+
+
 def override_main(self, args=None, prog_name=None, complete_var=None,
                   standalone_mode=True, **extra):
 
     # Hook for the Bash completion.  This only activates if the Bash
     # completion is actually enabled, otherwise this is quite a fast
     # noop.
-    main_bashcomplete(self, prog_name)
+    main_bashcomplete(self, prog_name, override_completions)
 
     original_main(self, args=args, prog_name=prog_name, complete_var=None,
                   standalone_mode=standalone_mode, **extra)
@@ -130,8 +184,7 @@ def cli(context, **kwargs):
 @click.option('--variant',
               help='A variant of the specified target')
 @click.argument('target',
-                type=click.Path(dir_okay=False, readable=True),
-                autocompletion=complete_target)
+                type=click.Path(dir_okay=False, readable=True))
 @click.pass_obj
 def build(app, target, variant, all, track):
     """Build elements in a pipeline"""
@@ -162,8 +215,7 @@ def build(app, target, variant, all, track):
 @click.option('--variant',
               help='A variant of the specified target')
 @click.argument('target',
-                type=click.Path(dir_okay=False, readable=True),
-                autocompletion=complete_target)
+                type=click.Path(dir_okay=False, readable=True))
 @click.pass_obj
 def fetch(app, target, variant, deps, track, except_):
     """Fetch sources required to build the pipeline
@@ -205,8 +257,7 @@ def fetch(app, target, variant, deps, track, except_):
 @click.option('--variant',
               help='A variant of the specified target')
 @click.argument('target',
-                type=click.Path(dir_okay=False, readable=True),
-                autocompletion=complete_target)
+                type=click.Path(dir_okay=False, readable=True))
 @click.pass_obj
 def track(app, target, variant, deps, except_):
     """Consults the specified tracking branches for new versions available
@@ -244,8 +295,7 @@ def track(app, target, variant, deps, except_):
               type=click.Choice(['none', 'all']),
               help='The dependencies to fetch (default: none)')
 @click.argument('target',
-                type=click.Path(dir_okay=False, readable=True),
-                autocompletion=complete_target)
+                type=click.Path(dir_okay=False, readable=True))
 @click.pass_obj
 def pull(app, target, variant, deps):
     """Pull a built artifact from the configured remote artifact cache.
@@ -277,8 +327,7 @@ def pull(app, target, variant, deps):
               type=click.Choice(['none', 'all']),
               help='The dependencies to fetch (default: none)')
 @click.argument('target',
-                type=click.Path(dir_okay=False, readable=True),
-                autocompletion=complete_target)
+                type=click.Path(dir_okay=False, readable=True))
 @click.pass_obj
 def push(app, target, variant, deps):
     """Push a built artifact to the configured remote artifact cache.
@@ -318,8 +367,7 @@ def push(app, target, variant, deps):
 @click.option('--variant',
               help='A variant of the specified target')
 @click.argument('target',
-                type=click.Path(dir_okay=False, readable=True),
-                autocompletion=complete_target)
+                type=click.Path(dir_okay=False, readable=True))
 @click.pass_obj
 def show(app, target, variant, deps, except_, order, format):
     """Show elements in the pipeline
@@ -401,8 +449,7 @@ def show(app, target, variant, deps, except_, order, format):
 @click.option('--variant',
               help='A variant of the specified target')
 @click.argument('target',
-                type=click.Path(dir_okay=False, readable=True),
-                autocompletion=complete_target)
+                type=click.Path(dir_okay=False, readable=True))
 @click.pass_obj
 def shell(app, target, variant, builddir, scope, command):
     """Shell into an element's sandbox environment
@@ -459,8 +506,7 @@ def shell(app, target, variant, builddir, scope, command):
 @click.option('--variant',
               help='A variant of the specified target')
 @click.argument('target',
-                type=click.Path(dir_okay=False, readable=True),
-                autocompletion=complete_target)
+                type=click.Path(dir_okay=False, readable=True))
 @click.argument('directory', type=click.Path(file_okay=False))
 @click.pass_obj
 def checkout(app, target, variant, directory, force):
@@ -494,8 +540,7 @@ def checkout(app, target, variant, directory, force):
 @click.option('--directory', default=os.getcwd(),
               help="The directory to write the tarball to")
 @click.argument('target',
-                type=click.Path(dir_okay=False, readable=True),
-                autocompletion=complete_target)
+                type=click.Path(dir_okay=False, readable=True))
 @click.pass_obj
 def source_bundle(app, target, variant, force, directory,
                   track, compression, except_):
@@ -537,8 +582,7 @@ def workspace():
 @click.option('--track', default=False, is_flag=True,
               help="Track and fetch new source references before checking out the workspace")
 @click.argument('element',
-                type=click.Path(dir_okay=False, readable=True),
-                autocompletion=complete_target)
+                type=click.Path(dir_okay=False, readable=True))
 @click.argument('directory', type=click.Path(file_okay=False))
 @click.pass_obj
 def workspace_open(app, no_checkout, force, source, variant, track, element, directory):
@@ -565,8 +609,7 @@ def workspace_open(app, no_checkout, force, source, variant, track, element, dir
 @click.option('--variant',
               help='A variant of the specified target')
 @click.argument('element',
-                type=click.Path(dir_okay=False, readable=True),
-                autocompletion=complete_target)
+                type=click.Path(dir_okay=False, readable=True))
 @click.pass_obj
 def workspace_close(app, source, remove_dir, variant, element):
     """Close a workspace"""
@@ -600,8 +643,7 @@ def workspace_close(app, source, remove_dir, variant, element):
               help='A variant of the specified target')
 @click.confirmation_option(prompt='This will remove all your changes, are you sure?')
 @click.argument('element',
-                type=click.Path(dir_okay=False, readable=True),
-                autocompletion=complete_target)
+                type=click.Path(dir_okay=False, readable=True))
 @click.pass_obj
 def workspace_reset(app, source, track, no_checkout, variant, element):
     """Reset a workspace to its original state"""