Add `bst source push/pull` commandstlater/source-pushll

6 files changed, 193 insertions, 1 deletions

diff --git a/src/buildstream/_frontend/cli.py b/src/buildstream/_frontend/cli.py
index 1365cede4..8f0e1fbd0 100644
--- a/src/buildstream/_frontend/cli.py
+++ b/src/buildstream/_frontend/cli.py
@@ -664,6 +664,90 @@ def source():
     """Manipulate sources for an element"""
 
 
+@source.command(name="push", short_help="Push an element's sources")
+@click.option('--deps', '-d', default='none',
+              type=click.Choice(['none', 'all']),
+              help='The dependencies to push (default: none)')
+@click.option('--remote', '-r', default=None,
+              help="The URL of the remote cache (defaults to the first configured cache)")
+@click.argument('elements', nargs=-1,
+                type=click.Path(readable=False))
+@click.pass_obj
+def source_push(app, elements, deps, remote):
+    """Push an element's sources to a remote source cache.
+
+    Specifying no elements will result in pushing the default targets
+    of the project. If no default targets are configured, all project
+    elements will be pushed.
+
+    When this command is executed from a workspace directory, the default
+    is to push the workspace element.
+
+    The default destination is the highest priority configured cache. You can
+    override this by passing a different cache URL with the `--remote` flag.
+
+    Specify `--deps` to control which sources to push:
+
+    \b
+        none:  No dependencies, just the element itself
+        all:   All dependencies
+    """
+    with app.initialized(session_name="Push"):
+        ignore_junction_targets = False
+
+        if not elements:
+            elements = app.project.get_default_targets()
+            # Junction elements cannot be pushed, exclude them from default targets
+            ignore_junction_targets = True
+
+        app.stream.push_sources(elements, selection=deps, remote=remote,
+                                ignore_junction_targets=ignore_junction_targets)
+
+
+@source.command(name="pull", short_help="Pull an element's sources")
+@click.option('--deps', '-d', default='none',
+              type=click.Choice(['none', 'all']),
+              help='The dependency sources to pull (default: none)')
+@click.option('--remote', '-r', default=None,
+              help="The URL of the remote cache (defaults to the first configured cache)")
+@click.argument('elements', nargs=-1,
+                type=click.Path(readable=False))
+@click.pass_obj
+def source_pull(app, elements, deps, remote):
+    """Pull sources from the configured remote source cache.
+
+    Specifying no elements will result in pulling the default targets
+    of the project. If no default targets are configured, all project
+    elements will be pulled.
+
+    When this command is executed from a workspace directory, the default
+    is to pull the workspace element.
+
+    By default the sources will be pulled from one of the configured
+    caches if possible, following the usual priority order. If the
+    `--remote` flag is given, only the specified cache will be
+    queried.
+
+    Specify `--deps` to control which sources to pull:
+
+    \b
+        none:  No dependencies, just the element itself
+        all:   All dependencies
+
+    """
+
+    with app.initialized(session_name="Pull"):
+        ignore_junction_targets = False
+
+        if not elements:
+            elements = app.project.get_default_targets()
+            # Junction elements cannot be pulled, exclude them from default targets
+            ignore_junction_targets = True
+
+        app.stream.pull_sources(elements, selection=deps, remote=remote,
+                                ignore_junction_targets=ignore_junction_targets)
+
+
 ##################################################################
 #                     Source Fetch Command                       #
 ##################################################################
diff --git a/src/buildstream/_scheduler/__init__.py b/src/buildstream/_scheduler/__init__.py
index d2f458fa5..123f3299b 100644
--- a/src/buildstream/_scheduler/__init__.py
+++ b/src/buildstream/_scheduler/__init__.py
@@ -21,6 +21,7 @@ from .queues import Queue, QueueStatus
 
 from .queues.fetchqueue import FetchQueue
 from .queues.sourcepushqueue import SourcePushQueue
+from .queues.sourcepullqueue import SourcePullQueue
 from .queues.trackqueue import TrackQueue
 from .queues.buildqueue import BuildQueue
 from .queues.artifactpushqueue import ArtifactPushQueue
diff --git a/src/buildstream/_scheduler/queues/sourcepullqueue.py b/src/buildstream/_scheduler/queues/sourcepullqueue.py
new file mode 100644
index 000000000..255edf443
--- /dev/null
+++ b/src/buildstream/_scheduler/queues/sourcepullqueue.py
@@ -0,0 +1,43 @@
+#
+#  Copyright (C) 2019 Bloomberg Finance LP
+#
+#  This program is free software; you can redistribute it and/or
+#  modify it under the terms of the GNU Lesser General Public
+#  License as published by the Free Software Foundation; either
+#  version 2 of the License, or (at your option) any later version.
+#
+#  This library is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.	 See the GNU
+#  Lesser General Public License for more details.
+#
+#  You should have received a copy of the GNU Lesser General Public
+#  License along with this library. If not, see <http://www.gnu.org/licenses/>.
+#
+
+from . import Queue, QueueStatus
+from ..resources import ResourceType
+from ..._exceptions import SkipJob
+
+
+# A queue which pushes staged sources
+#
+class SourcePullQueue(Queue):
+
+    action_name = "Src-pull"
+    complete_name = "Sources pulled"
+    resources = [ResourceType.DOWNLOAD, ResourceType.CACHE]
+
+    def get_process_func(self):
+        return SourcePullQueue._pull_or_skip
+
+    def status(self, element):
+        if element._skip_source_pull():
+            return QueueStatus.SKIP
+
+        return QueueStatus.READY
+
+    @staticmethod
+    def _pull_or_skip(element):
+        if not element._source_pull():
+            raise SkipJob(SourcePullQueue.action_name)
diff --git a/src/buildstream/_stream.py b/src/buildstream/_stream.py
index 453670ad1..9f3792520 100644
--- a/src/buildstream/_stream.py
+++ b/src/buildstream/_stream.py
@@ -36,7 +36,7 @@ from ._artifactelement import verify_artifact_ref, ArtifactElement
 from ._exceptions import StreamError, ImplError, BstError, ArtifactElementError, ArtifactError
 from ._message import Message, MessageType
 from ._scheduler import Scheduler, SchedStatus, TrackQueue, FetchQueue, \
-    SourcePushQueue, BuildQueue, PullQueue, ArtifactPushQueue
+    SourcePushQueue, SourcePullQueue, BuildQueue, PullQueue, ArtifactPushQueue
 from ._pipeline import Pipeline, PipelineSelection
 from ._profile import Topics, PROFILER
 from ._state import State
@@ -311,6 +311,52 @@ class Stream():
         self._enqueue_plan(elements)
         self._run()
 
+    def push_sources(self, targets, *,
+                     selection=PipelineSelection.NONE,
+                     ignore_junction_targets=False,
+                     remote=None):
+        use_config = True
+        if remote:
+            use_config = False
+
+        elements, _ = self._load(targets, (),
+                                 selection=selection,
+                                 ignore_junction_targets=ignore_junction_targets,
+                                 source_remote_url=remote,
+                                 use_source_config=use_config)
+
+        if not self._sourcecache.has_push_remotes():
+            raise StreamError("No source caches available for pushing sources")
+
+        self._scheduler.clear_queues()
+        push_queue = SourcePushQueue(self._scheduler)
+        self._add_queue(push_queue)
+        self._enqueue_plan(elements, queue=push_queue)
+        self._run()
+
+    def pull_sources(self, targets, *,
+                     selection=PipelineSelection.NONE,
+                     ignore_junction_targets=False,
+                     remote=None):
+        use_config = True
+        if remote:
+            use_config = False
+
+        elements, _ = self._load(targets, (),
+                                 selection=selection,
+                                 ignore_junction_targets=ignore_junction_targets,
+                                 source_remote_url=remote,
+                                 use_source_config=use_config)
+
+        if not self._sourcecache.has_fetch_remotes():
+            raise StreamError("No source caches available for pulling sources")
+
+        self._scheduler.clear_queues()
+        pull_queue = SourcePullQueue(self._scheduler)
+        self._add_queue(push_queue)
+        self._enqueue_plan(elements, queue=push_queue)
+        self._run()
+
     # fetch()
     #
     # Fetches sources on the pipeline.
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index ca573bee1..0bac84617 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -1836,6 +1836,22 @@ class Element(Plugin):
         # Notify successfull download
         return True
 
+    def _skip_source_pull(self):
+        if not self.__sources or self._get_workspace():
+            return True
+        return self._source_cached() or not self.__sourcecache.has_fetch_remotes(plugin=self)
+
+    def _source_pull(self):
+        if self.__sourcecache.has_fetch_remotes(plugin=self) and not self._source_cached():
+            for source in self.sources():
+                if self.__sourcecache.pull(source):
+                    # Once we find a cache with the source and manage
+                    # to pull from it, we're done
+                    return True
+
+        # If no caches gave us the source, this failed
+        return False
+
     def _skip_source_push(self):
         if not self.__sources or self._get_workspace():
             return True
diff --git a/tests/frontend/completions.py b/tests/frontend/completions.py
index e9fa25b73..c0ea47385 100644
--- a/tests/frontend/completions.py
+++ b/tests/frontend/completions.py
@@ -56,6 +56,8 @@ MAIN_OPTIONS = [
 SOURCE_COMMANDS = [
     'checkout ',
     'fetch ',
+    'pull ',
+    'push ',
     'track ',
 ]