Merge branch '328-support-for-downloading-sources-from-mirrors' into 'master'

Resolve "Support for downloading sources from mirrors"

Closes #328

See merge request BuildStream/buildstream!404

21 files changed, 1168 insertions, 69 deletions

diff --git a/NEWS b/NEWS
index dd60fe62b..2c1d93543 100644
--- a/NEWS
+++ b/NEWS
@@ -5,6 +5,10 @@ buildstream 1.3.1
   o Add a `--tar` option to `bst checkout` which allows a tarball to be
     created from the artifact contents.
 
+  o Fetching and tracking will consult mirrors defined in project config,
+    and the preferred mirror to fetch from can be defined in the command
+    line or user config.
+
 =================
 buildstream 1.1.4
 =================
diff --git a/buildstream/__init__.py b/buildstream/__init__.py
index cf56ecfe1..895adc60f 100644
--- a/buildstream/__init__.py
+++ b/buildstream/__init__.py
@@ -29,7 +29,7 @@ if "_BST_COMPLETION" not in os.environ:
     from .utils import UtilError, ProgramNotFoundError
     from .sandbox import Sandbox, SandboxFlags
     from .plugin import Plugin
-    from .source import Source, SourceError, Consistency
+    from .source import Source, SourceError, Consistency, SourceFetcher
     from .element import Element, ElementError, Scope
     from .buildelement import BuildElement
     from .scriptelement import ScriptElement
diff --git a/buildstream/_context.py b/buildstream/_context.py
index fbb0a9686..7dc68af79 100644
--- a/buildstream/_context.py
+++ b/buildstream/_context.py
@@ -259,7 +259,7 @@ class Context():
         # Shallow validation of overrides, parts of buildstream which rely
         # on the overrides are expected to validate elsewhere.
         for _, overrides in _yaml.node_items(self._project_overrides):
-            _yaml.node_validate(overrides, ['artifacts', 'options', 'strict'])
+            _yaml.node_validate(overrides, ['artifacts', 'options', 'strict', 'default-mirror'])
 
         profile_end(Topics.LOAD_CONTEXT, 'load')
 
diff --git a/buildstream/_frontend/app.py b/buildstream/_frontend/app.py
index 5fae0307b..04a90ea32 100644
--- a/buildstream/_frontend/app.py
+++ b/buildstream/_frontend/app.py
@@ -202,7 +202,8 @@ class App():
         # Load the Project
         #
         try:
-            self.project = Project(directory, self.context, cli_options=self._main_options['option'])
+            self.project = Project(directory, self.context, cli_options=self._main_options['option'],
+                                   default_mirror=self._main_options.get('default_mirror'))
         except LoadError as e:
 
             # Let's automatically start a `bst init` session in this case
diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index bd2ce8a73..20624e2ac 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -217,6 +217,8 @@ def print_version(ctx, param, value):
               help="Elements must be rebuilt when their dependencies have changed")
 @click.option('--option', '-o', type=click.Tuple([str, str]), multiple=True, metavar='OPTION VALUE',
               help="Specify a project option")
+@click.option('--default-mirror', default=None,
+              help="The mirror to fetch from first, before attempting other mirrors")
 @click.pass_context
 def cli(context, **kwargs):
     """Build and manipulate BuildStream projects
diff --git a/buildstream/_loader/loader.py b/buildstream/_loader/loader.py
index 07b0de996..e9b9d95f1 100644
--- a/buildstream/_loader/loader.py
+++ b/buildstream/_loader/loader.py
@@ -513,7 +513,7 @@ class Loader():
                 if self._fetch_subprojects:
                     if ticker:
                         ticker(filename, 'Fetching subproject from {} source'.format(source.get_kind()))
-                    source.fetch()
+                    source._fetch()
                 else:
                     detail = "Try fetching the project with `bst fetch {}`".format(filename)
                     raise LoadError(LoadErrorReason.SUBPROJECT_FETCH_NEEDED,
diff --git a/buildstream/_project.py b/buildstream/_project.py
index 54ec9ee34..1c30fb9bb 100644
--- a/buildstream/_project.py
+++ b/buildstream/_project.py
@@ -19,7 +19,7 @@
 
 import os
 import multiprocessing  # for cpu_count()
-from collections import Mapping
+from collections import Mapping, OrderedDict
 from pluginbase import PluginBase
 from . import utils
 from . import _cachekey
@@ -35,9 +35,6 @@ from ._projectrefs import ProjectRefs, ProjectRefStorage
 from ._versions import BST_FORMAT_VERSION
 
 
-# The separator we use for user specified aliases
-_ALIAS_SEPARATOR = ':'
-
 # Project Configuration file
 _PROJECT_CONF_FILE = 'project.conf'
 
@@ -70,7 +67,7 @@ class HostMount():
 #
 class Project():
 
-    def __init__(self, directory, context, *, junction=None, cli_options=None):
+    def __init__(self, directory, context, *, junction=None, cli_options=None, default_mirror=None):
 
         # The project name
         self.name = None
@@ -94,6 +91,8 @@ class Project():
         self.base_env_nocache = None             # The base nocache mask (list) for the environment
         self.element_overrides = {}              # Element specific configurations
         self.source_overrides = {}               # Source specific configurations
+        self.mirrors = OrderedDict()             # contains dicts of alias-mappings to URIs.
+        self.default_mirror = default_mirror     # The name of the preferred mirror.
 
         #
         # Private Members
@@ -133,8 +132,8 @@ class Project():
     # fully qualified urls based on the shorthand which is allowed
     # to be specified in the YAML
     def translate_url(self, url):
-        if url and _ALIAS_SEPARATOR in url:
-            url_alias, url_body = url.split(_ALIAS_SEPARATOR, 1)
+        if url and utils._ALIAS_SEPARATOR in url:
+            url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1)
             alias_url = self._aliases.get(url_alias)
             if alias_url:
                 url = alias_url + url_body
@@ -202,6 +201,36 @@ class Project():
         self._assert_plugin_format(source, version)
         return source
 
+    # get_alias_uri()
+    #
+    # Returns the URI for a given alias, if it exists
+    #
+    # Args:
+    #    alias (str): The alias.
+    #
+    # Returns:
+    #    str: The URI for the given alias; or None: if there is no URI for
+    #         that alias.
+    def get_alias_uri(self, alias):
+        return self._aliases.get(alias)
+
+    # get_alias_uris()
+    #
+    # Returns a list of every URI to replace an alias with
+    def get_alias_uris(self, alias):
+        if not alias or alias not in self._aliases:
+            return [None]
+
+        mirror_list = []
+        for key, alias_mapping in self.mirrors.items():
+            if alias in alias_mapping:
+                if key == self.default_mirror:
+                    mirror_list = alias_mapping[alias] + mirror_list
+                else:
+                    mirror_list += alias_mapping[alias]
+        mirror_list.append(self._aliases[alias])
+        return mirror_list
+
     # _load():
     #
     # Loads the project configuration file in the project directory.
@@ -249,7 +278,7 @@ class Project():
             'aliases', 'name',
             'artifacts', 'options',
             'fail-on-overlap', 'shell',
-            'ref-storage', 'sandbox'
+            'ref-storage', 'sandbox', 'mirrors',
         ])
 
         # The project name, element path and option declarations
@@ -290,6 +319,10 @@ class Project():
         #
         self.options.process_node(config)
 
+        # Override default_mirror if not set by command-line
+        if not self.default_mirror:
+            self.default_mirror = _yaml.node_get(overrides, str, 'default-mirror', default_value=None)
+
         #
         # Now all YAML composition is done, from here on we just load
         # the values from our loaded configuration dictionary.
@@ -414,6 +447,21 @@ class Project():
 
             self._shell_host_files.append(mount)
 
+        mirrors = _yaml.node_get(config, list, 'mirrors', default_value=[])
+        for mirror in mirrors:
+            allowed_mirror_fields = [
+                'name', 'aliases'
+            ]
+            _yaml.node_validate(mirror, allowed_mirror_fields)
+            mirror_name = _yaml.node_get(mirror, str, 'name')
+            alias_mappings = {}
+            for alias_mapping, uris in _yaml.node_items(mirror['aliases']):
+                assert isinstance(uris, list)
+                alias_mappings[alias_mapping] = list(uris)
+            self.mirrors[mirror_name] = alias_mappings
+            if not self.default_mirror:
+                self.default_mirror = mirror_name
+
     # _assert_plugin_format()
     #
     # Helper to raise a PluginError if the loaded plugin is of a lesser version then
diff --git a/buildstream/_versions.py b/buildstream/_versions.py
index 19699e125..4531d9a72 100644
--- a/buildstream/_versions.py
+++ b/buildstream/_versions.py
@@ -23,7 +23,7 @@
 # This version is bumped whenever enhancements are made
 # to the `project.conf` format or the core element format.
 #
-BST_FORMAT_VERSION = 10
+BST_FORMAT_VERSION = 11
 
 
 # The base BuildStream artifact version
diff --git a/buildstream/plugins/sources/bzr.py b/buildstream/plugins/sources/bzr.py
index 9fd2eb60c..3afd0f04f 100644
--- a/buildstream/plugins/sources/bzr.py
+++ b/buildstream/plugins/sources/bzr.py
@@ -102,7 +102,7 @@ class BzrSource(Source):
     def track(self):
         with self.timed_activity("Tracking {}".format(self.url),
                                  silent_nested=True):
-            self._ensure_mirror()
+            self._ensure_mirror(skip_ref_check=True)
             ret, out = self.check_output([self.host_bzr, "version-info",
                                           "--custom", "--template={revno}",
                                           self._get_branch_dir()],
@@ -214,7 +214,7 @@ class BzrSource(Source):
             yield repodir
             self._atomic_replace_mirrordir(repodir)
 
-    def _ensure_mirror(self):
+    def _ensure_mirror(self, skip_ref_check=False):
         with self._atomic_repodir() as repodir:
             # Initialize repo if no metadata
             bzr_metadata_dir = os.path.join(repodir, ".bzr")
@@ -223,18 +223,21 @@ class BzrSource(Source):
                           fail="Failed to initialize bzr repository")
 
             branch_dir = os.path.join(repodir, self.tracking)
+            branch_url = self.url + "/" + self.tracking
             if not os.path.exists(branch_dir):
                 # `bzr branch` the branch if it doesn't exist
                 # to get the upstream code
-                branch_url = self.url + "/" + self.tracking
                 self.call([self.host_bzr, "branch", branch_url, branch_dir],
                           fail="Failed to branch from {} to {}".format(branch_url, branch_dir))
 
             else:
                 # `bzr pull` the branch if it does exist
                 # to get any changes to the upstream code
-                self.call([self.host_bzr, "pull", "--directory={}".format(branch_dir)],
+                self.call([self.host_bzr, "pull", "--directory={}".format(branch_dir), branch_url],
                           fail="Failed to pull new changes for {}".format(branch_dir))
+        if not skip_ref_check and not self._check_ref():
+            raise SourceError("Failed to ensure ref '{}' was mirrored".format(self.ref),
+                              reason="ref-not-mirrored")
 
 
 def setup():
diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py
index 6762f6025..bccf5d6b7 100644
--- a/buildstream/plugins/sources/git.py
+++ b/buildstream/plugins/sources/git.py
@@ -78,7 +78,7 @@ from io import StringIO
 
 from configparser import RawConfigParser
 
-from buildstream import Source, SourceError, Consistency
+from buildstream import Source, SourceError, Consistency, SourceFetcher
 from buildstream import utils
 
 GIT_MODULES = '.gitmodules'
@@ -88,18 +88,20 @@ GIT_MODULES = '.gitmodules'
 # for the primary git source and also for each submodule it
 # might have at a given time
 #
-class GitMirror():
+class GitMirror(SourceFetcher):
 
     def __init__(self, source, path, url, ref):
 
+        super().__init__()
         self.source = source
         self.path = path
-        self.url = source.translate_url(url)
+        self.url = url
         self.ref = ref
-        self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(self.url))
+        self.mirror = os.path.join(source.get_mirror_directory(), utils.url_directory_name(url))
+        self.mark_download_url(url)
 
     # Ensures that the mirror exists
-    def ensure(self):
+    def ensure(self, alias_override=None):
 
         # Unfortunately, git does not know how to only clone just a specific ref,
         # so we have to download all of those gigs even if we only need a couple
@@ -112,22 +114,47 @@ class GitMirror():
             # system configured tmpdir is not on the same partition.
             #
             with self.source.tempdir() as tmpdir:
-                self.source.call([self.source.host_git, 'clone', '--mirror', '-n', self.url, tmpdir],
-                                 fail="Failed to clone git repository {}".format(self.url),
+                url = self.source.translate_url(self.url, alias_override=alias_override)
+                self.source.call([self.source.host_git, 'clone', '--mirror', '-n', url, tmpdir],
+                                 fail="Failed to clone git repository {}".format(url),
                                  fail_temporarily=True)
 
                 try:
                     shutil.move(tmpdir, self.mirror)
                 except (shutil.Error, OSError) as e:
                     raise SourceError("{}: Failed to move cloned git repository {} from '{}' to '{}'"
-                                      .format(self.source, self.url, tmpdir, self.mirror)) from e
+                                      .format(self.source, url, tmpdir, self.mirror)) from e
+
+    def _fetch(self, alias_override=None):
+        url = self.source.translate_url(self.url, alias_override=alias_override)
+
+        if alias_override:
+            remote_name = utils.url_directory_name(alias_override)
+            _, remotes = self.source.check_output(
+                [self.source.host_git, 'remote'],
+                fail="Failed to retrieve list of remotes in {}".format(self.mirror),
+                cwd=self.mirror
+            )
+            if remote_name not in remotes:
+                self.source.call(
+                    [self.source.host_git, 'remote', 'add', remote_name, url],
+                    fail="Failed to add remote {} with url {}".format(remote_name, url),
+                    cwd=self.mirror
+                )
+        else:
+            remote_name = "origin"
 
-    def fetch(self):
-        self.source.call([self.source.host_git, 'fetch', 'origin', '--prune'],
-                         fail="Failed to fetch from remote git repository: {}".format(self.url),
+        self.source.call([self.source.host_git, 'fetch', remote_name, '--prune'],
+                         fail="Failed to fetch from remote git repository: {}".format(url),
                          fail_temporarily=True,
                          cwd=self.mirror)
 
+    def fetch(self, alias_override=None):
+        self.ensure(alias_override)
+        if not self.has_ref():
+            self._fetch(alias_override)
+        self.assert_ref()
+
     def has_ref(self):
         if not self.ref:
             return False
@@ -171,13 +198,14 @@ class GitMirror():
 
     def init_workspace(self, directory):
         fullpath = os.path.join(directory, self.path)
+        url = self.source.translate_url(self.url)
 
         self.source.call([self.source.host_git, 'clone', '--no-checkout', self.mirror, fullpath],
                          fail="Failed to clone git mirror {} in directory: {}".format(self.mirror, fullpath),
                          fail_temporarily=True)
 
-        self.source.call([self.source.host_git, 'remote', 'set-url', 'origin', self.url],
-                         fail='Failed to add remote origin "{}"'.format(self.url),
+        self.source.call([self.source.host_git, 'remote', 'set-url', 'origin', url],
+                         fail='Failed to add remote origin "{}"'.format(url),
                          cwd=fullpath)
 
         self.source.call([self.source.host_git, 'checkout', '--force', self.ref],
@@ -277,6 +305,8 @@ class GitSource(Source):
                 checkout = self.node_get_member(submodule, bool, 'checkout')
                 self.submodule_checkout_overrides[path] = checkout
 
+        self.mark_download_url(self.original_url)
+
     def preflight(self):
         # Check if git is installed, get the binary at the same time
         self.host_git = utils.get_host_tool('git')
@@ -328,31 +358,13 @@ class GitSource(Source):
                                  .format(self.tracking, self.mirror.url),
                                  silent_nested=True):
             self.mirror.ensure()
-            self.mirror.fetch()
+            self.mirror._fetch()
 
             # Update self.mirror.ref and node.ref from the self.tracking branch
             ret = self.mirror.latest_commit(self.tracking)
 
         return ret
 
-    def fetch(self):
-
-        with self.timed_activity("Fetching {}".format(self.mirror.url), silent_nested=True):
-
-            # Here we are only interested in ensuring that our mirror contains
-            # the self.mirror.ref commit.
-            self.mirror.ensure()
-            if not self.mirror.has_ref():
-                self.mirror.fetch()
-
-            self.mirror.assert_ref()
-
-            # Here after performing any fetches, we need to also ensure that
-            # we've cached the desired refs in our mirrors of submodules.
-            #
-            self.refresh_submodules()
-            self.fetch_submodules()
-
     def init_workspace(self, directory):
         # XXX: may wish to refactor this as some code dupe with stage()
         self.refresh_submodules()
@@ -384,6 +396,10 @@ class GitSource(Source):
                 if checkout:
                     mirror.stage(directory)
 
+    def get_source_fetchers(self):
+        self.refresh_submodules()
+        return [self.mirror] + self.submodules
+
     ###########################################################
     #                     Local Functions                     #
     ###########################################################
@@ -405,6 +421,7 @@ class GitSource(Source):
     # Assumes that we have our mirror and we have the ref which we point to
     #
     def refresh_submodules(self):
+        self.mirror.ensure()
         submodules = []
 
         # XXX Here we should issue a warning if either:
@@ -426,19 +443,6 @@ class GitSource(Source):
 
         self.submodules = submodules
 
-    # Ensures that we have mirrored git repositories for all
-    # the submodules existing at the given commit of the main git source.
-    #
-    # Also ensure that these mirrors have the required commits
-    # referred to at the given commit of the main git source.
-    #
-    def fetch_submodules(self):
-        for mirror in self.submodules:
-            mirror.ensure()
-            if not mirror.has_ref():
-                mirror.fetch()
-                mirror.assert_ref()
-
 
 # Plugin entry point
 def setup():
diff --git a/buildstream/source.py b/buildstream/source.py
index 2db4f8eea..9866c99e9 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -65,6 +65,33 @@ these methods are mandatory to implement.
 
   **Optional**: If left unimplemented, this will default to calling
   :func:`Source.stage() <buildstream.source.Source.stage>`
+
+* :func:`Source.get_source_fetchers() <buildstream.source.Source.get_source_fetchers>`
+
+  Get the objects that are used for fetching.
+
+  **Optional**: This only needs to be implemented for sources that need to
+  download from multiple URLs while fetching (e.g. a git repo and its
+  submodules). For details on how to define a SourceFetcher, see
+  :ref:`SourceFetcher <core_source_fetcher>`.
+
+
+.. _core_source_fetcher:
+
+SourceFetcher - Object for fetching individual URLs
+===================================================
+
+
+Abstract Methods
+----------------
+SourceFetchers expose the following abstract methods. Unless explicitly
+mentioned, these methods are mandatory to implement.
+
+* :func:`SourceFetcher.fetch() <buildstream.source.SourceFetcher.fetch>`
+
+  Fetches the URL associated with this SourceFetcher, optionally taking an
+  alias override.
+
 """
 
 import os
@@ -114,6 +141,63 @@ class SourceError(BstError):
         super().__init__(message, detail=detail, domain=ErrorDomain.SOURCE, reason=reason, temporary=temporary)
 
 
+class SourceFetcher():
+    """SourceFetcher()
+
+    This interface exists so that a source that downloads from multiple
+    places (e.g. a git source with submodules) has a consistent interface for
+    fetching and substituting aliases.
+
+    *Since: 1.4*
+    """
+    def __init__(self):
+        self.__alias = None
+
+    #############################################################
+    #                      Abstract Methods                     #
+    #############################################################
+    def fetch(self, alias_override=None):
+        """Fetch remote sources and mirror them locally, ensuring at least
+        that the specific reference is cached locally.
+
+        Args:
+           alias_override (str): The alias to use instead of the default one
+               defined by the :ref:`aliases <project_source_aliases>` field
+               in the project's config.
+
+        Raises:
+           :class:`.SourceError`
+
+        Implementors should raise :class:`.SourceError` if the there is some
+        network error or if the source reference could not be matched.
+        """
+        raise ImplError("Source fetcher '{}' does not implement fetch()".format(type(self)))
+
+    #############################################################
+    #                       Public Methods                      #
+    #############################################################
+    def mark_download_url(self, url):
+        """Identifies the URL that this SourceFetcher uses to download
+
+        This must be called during the fetcher's initialization
+
+        Args:
+           url (str): The url used to download.
+        """
+        # Not guaranteed to be a valid alias yet.
+        # Ensuring it's a valid alias currently happens in Project.get_alias_uris
+        alias, _ = url.split(utils._ALIAS_SEPARATOR, 1)
+        self.__alias = alias
+
+    #############################################################
+    #            Private Methods used in BuildStream            #
+    #############################################################
+
+    # Returns the alias used by this fetcher
+    def _get_alias(self):
+        return self.__alias
+
+
 class Source(Plugin):
     """Source()
 
@@ -125,7 +209,7 @@ class Source(Plugin):
     __defaults = {}          # The defaults from the project
     __defaults_set = False   # Flag, in case there are not defaults at all
 
-    def __init__(self, context, project, meta):
+    def __init__(self, context, project, meta, *, alias_override=None):
         provenance = _yaml.node_get_provenance(meta.config)
         super().__init__("{}-{}".format(meta.element_name, meta.element_index),
                          context, project, provenance, "source")
@@ -135,6 +219,11 @@ class Source(Plugin):
         self.__element_kind = meta.element_kind         # The kind of the element owning this source
         self.__directory = meta.directory               # Staging relative directory
         self.__consistency = Consistency.INCONSISTENT   # Cached consistency state
+        self.__alias_override = alias_override          # Tuple of alias and its override to use instead
+        self.__expected_alias = None                    # A hacky way to store the first alias used
+
+        # FIXME: Reconstruct a MetaSource from a Source instead of storing it.
+        self.__meta = meta                              # MetaSource stored so we can copy this source later.
 
         # Collect the composited element configuration and
         # ask the element to configure itself.
@@ -284,6 +373,36 @@ class Source(Plugin):
         """
         self.stage(directory)
 
+    def mark_download_url(self, url):
+        """Identifies the URL that this Source uses to download
+
+        This must be called during :func:`~buildstream.plugin.Plugin.configure` if
+        :func:`~buildstream.source.Source.translate_url` is not called.
+
+        Args:
+           url (str): The url used to download
+
+        *Since: 1.4*
+        """
+        alias, _ = url.split(utils._ALIAS_SEPARATOR, 1)
+        self.__expected_alias = alias
+
+    def get_source_fetchers(self):
+        """Get the objects that are used for fetching
+
+        If this source doesn't download from multiple URLs,
+        returning None and falling back on the default behaviour
+        is recommended.
+
+        Returns:
+           list: A list of SourceFetchers. If SourceFetchers are not supported,
+                 this will be an empty list.
+
+        *Since: 1.4*
+        """
+
+        return []
+
     #############################################################
     #                       Public Methods                      #
     #############################################################
@@ -300,18 +419,42 @@ class Source(Plugin):
         os.makedirs(directory, exist_ok=True)
         return directory
 
-    def translate_url(self, url):
+    def translate_url(self, url, *, alias_override=None):
         """Translates the given url which may be specified with an alias
         into a fully qualified url.
 
         Args:
            url (str): A url, which may be using an alias
+           alias_override (str): Optionally, an URI to override the alias with. (*Since: 1.4*)
 
         Returns:
            str: The fully qualified url, with aliases resolved
         """
-        project = self._get_project()
-        return project.translate_url(url)
+        # Alias overriding can happen explicitly (by command-line) or
+        # implicitly (the Source being constructed with an __alias_override).
+        if alias_override or self.__alias_override:
+            url_alias, url_body = url.split(utils._ALIAS_SEPARATOR, 1)
+            if url_alias:
+                if alias_override:
+                    url = alias_override + url_body
+                else:
+                    # Implicit alias overrides may only be done for one
+                    # specific alias, so that sources that fetch from multiple
+                    # URLs and use different aliases default to only overriding
+                    # one alias, rather than getting confused.
+                    override_alias = self.__alias_override[0]
+                    override_url = self.__alias_override[1]
+                    if url_alias == override_alias:
+                        url = override_url + url_body
+            return url
+        else:
+            # Sneakily store the alias if it hasn't already been stored
+            if not self.__expected_alias and url and utils._ALIAS_SEPARATOR in url:
+                url_alias, _ = url.split(utils._ALIAS_SEPARATOR, 1)
+                self.__expected_alias = url_alias
+
+            project = self._get_project()
+            return project.translate_url(url)
 
     def get_project_directory(self):
         """Fetch the project base directory
@@ -375,7 +518,45 @@ class Source(Plugin):
     # Wrapper function around plugin provided fetch method
     #
     def _fetch(self):
-        self.fetch()
+        project = self._get_project()
+        source_fetchers = self.get_source_fetchers()
+        if source_fetchers:
+            for fetcher in source_fetchers:
+                alias = fetcher._get_alias()
+                success = False
+                for uri in project.get_alias_uris(alias):
+                    try:
+                        fetcher.fetch(uri)
+                    # FIXME: Need to consider temporary vs. permanent failures,
+                    #        and how this works with retries.
+                    except BstError as e:
+                        last_error = e
+                        continue
+                    success = True
+                    break
+                if not success:
+                    raise last_error
+        else:
+            alias = self._get_alias()
+            if not project.mirrors or not alias:
+                self.fetch()
+                return
+
+            context = self._get_context()
+            source_kind = type(self)
+            for uri in project.get_alias_uris(alias):
+                new_source = source_kind(context, project, self.__meta,
+                                         alias_override=(alias, uri))
+                new_source._preflight()
+                try:
+                    new_source.fetch()
+                # FIXME: Need to consider temporary vs. permanent failures,
+                #        and how this works with retries.
+                except BstError as e:
+                    last_error = e
+                    continue
+                return
+            raise last_error
 
     # Wrapper for stage() api which gives the source
     # plugin a fully constructed path considering the
@@ -582,7 +763,7 @@ class Source(Plugin):
     # Wrapper for track()
     #
     def _track(self):
-        new_ref = self.track()
+        new_ref = self.__do_track()
         current_ref = self.get_ref()
 
         if new_ref is None:
@@ -594,10 +775,48 @@ class Source(Plugin):
 
         return new_ref
 
+    # Returns the alias if it's defined in the project
+    def _get_alias(self):
+        alias = self.__expected_alias
+        project = self._get_project()
+        if project.get_alias_uri(alias):
+            # The alias must already be defined in the project's aliases
+            # otherwise http://foo gets treated like it contains an alias
+            return alias
+        else:
+            return None
+
     #############################################################
     #                   Local Private Methods                   #
     #############################################################
 
+    # Tries to call track for every mirror, stopping once it succeeds
+    def __do_track(self):
+        project = self._get_project()
+        # If there are no mirrors, or no aliases to replace, there's nothing to do here.
+        alias = self._get_alias()
+        if not project.mirrors or not alias:
+            return self.track()
+
+        context = self._get_context()
+        source_kind = type(self)
+
+        # NOTE: We are assuming here that tracking only requires substituting the
+        #       first alias used
+        for uri in reversed(project.get_alias_uris(alias)):
+            new_source = source_kind(context, project, self.__meta,
+                                     alias_override=(alias, uri))
+            new_source._preflight()
+            try:
+                ref = new_source.track()
+            # FIXME: Need to consider temporary vs. permanent failures,
+            #        and how this works with retries.
+            except BstError as e:
+                last_error = e
+                continue
+            return ref
+        raise last_error
+
     # Ensures a fully constructed path and returns it
     def __ensure_directory(self, directory):
 
diff --git a/buildstream/utils.py b/buildstream/utils.py
index e8270d82f..bfb58c9ef 100644
--- a/buildstream/utils.py
+++ b/buildstream/utils.py
@@ -42,6 +42,10 @@ from . import _signals
 from ._exceptions import BstError, ErrorDomain
 
 
+# The separator we use for user specified aliases
+_ALIAS_SEPARATOR = ':'
+
+
 class UtilError(BstError):
     """Raised by utility functions when system calls fail.
 
diff --git a/doc/source/examples/git-mirror.rst b/doc/source/examples/git-mirror.rst
new file mode 100644
index 000000000..1b05bf607
--- /dev/null
+++ b/doc/source/examples/git-mirror.rst
@@ -0,0 +1,144 @@
+
+
+Creating and using a git mirror
+'''''''''''''''''''''''''''''''
+This is an example of how to create a git mirror using git's
+`git-http-backend <https://git-scm.com/docs/git-http-backend>`_ and
+`lighttpd <https://redmine.lighttpd.net/projects/1/wiki/TutorialConfiguration>`_.
+
+
+Prerequisites
+=============
+You will need git installed, and git-http-backend must be present. It is assumed
+that the git-http-backend binary exists at `/usr/lib/git-core/git-http-backend`.
+
+You will need `lighttpd` installed, and at the bare minimum has the modules
+`mod_alias`, `mod_cgi`, and `mod_setenv`.
+
+I will be using gnome-modulesets as an example, which can be cloned from
+`http://gnome7.codethink.co.uk/gnome-modulesets.git`.
+
+
+Starting a git http server
+==========================
+
+
+1. Set up a directory containing mirrors
+----------------------------------------
+Choose a suitable directory to hold your mirrors, e.g. `/var/www/git`.
+
+Place the git repositories you want to use as mirrors in the mirror dir, e.g.
+``git clone --mirror http://git.gnome.org/browse/yelp-xsl /var/www/git/yelp-xsl.git``.
+
+
+2. Configure lighttpd
+---------------------
+Write out a lighttpd.conf as follows:
+
+::
+
+   server.document-root = "/var/www/git/" 
+   server.port = 3000
+   server.modules = (
+        "mod_alias",
+        "mod_cgi",
+        "mod_setenv",
+   )
+   
+   alias.url += ( "/git" => "/usr/lib/git-core/git-http-backend" )
+   $HTTP["url"] =~ "^/git" {
+        cgi.assign = ("" => "")
+        setenv.add-environment = (
+                "GIT_PROJECT_ROOT" => "/var/www/git",
+                "GIT_HTTP_EXPORT_ALL" => ""
+        )
+   }
+
+.. note::
+
+   If you have your mirrors in another directory, replace /var/www/git/ with that directory.
+
+
+3. Start lighttpd
+-----------------
+lighttpd can be invoked with the command-line ``lighttpd -D -f lighttpd.conf``.
+
+
+4. Test that you can fetch from it
+----------------------------------
+We can then clone the mirrored repo using git via http with
+``git clone http://127.0.0.1:3000/git/yelp-xsl``.
+
+.. note::
+
+   If you have set server.port to something other than the default, you will
+   need to replace the '3000' in the command-line.
+
+
+5. Configure the project to use the mirror
+------------------------------------------
+To add this local http server as a mirror, add the following to the project.conf:
+
+.. code:: yaml
+
+   mirrors:
+   - name: local-mirror
+     aliases:
+       git_gnome_org:
+       - http://127.0.0.1:3000/git/
+
+
+6. Test that the mirror works
+-----------------------------
+We can make buildstream use the mirror by setting the alias to an invalid URL, e.g.
+
+.. code:: yaml
+
+   aliases:
+     git_gnome_org: https://www.example.com/invalid/url/
+
+Now, if you build an element that uses the source you placed in the mirror
+(e.g. ``bst build core-deps/yelp-xsl.bst``), you will see that it uses your mirror.
+
+
+.. _lighttpd_git_tar_conf:
+
+Bonus: lighttpd conf for git and tar
+====================================
+For those who have also used the :ref:`tar-mirror tutorial <using_tar_mirror>`,
+a combined lighttpd.conf is below:
+
+::
+
+   server.document-root = "/var/www/"
+   server.port = 3000
+   server.modules = (
+           "mod_alias",
+           "mod_cgi",
+           "mod_setenv",
+   )
+   
+   alias.url += ( "/git" => "/usr/lib/git-core/git-http-backend" )
+   $HTTP["url"] =~ "^/git" {
+           cgi.assign = ("" => "")
+           setenv.add-environment = (
+                   "GIT_PROJECT_ROOT" => "/var/www/git",
+                   "GIT_HTTP_EXPORT_ALL" => ""
+           )
+   } else $HTTP["url"] =~ "^/tar" {
+           dir-listing.activate = "enable"
+   }
+
+
+Further reading
+===============
+If this mirror isn't being used exclusively in a secure network, it is strongly
+recommended you `use SSL <https://redmine.lighttpd.net/projects/1/wiki/HowToSimpleSSL>`_.
+
+This is the bare minimum required to set up a git mirror. A large, public project
+would prefer to set it up using the
+`git protocol <https://git-scm.com/book/en/v1/Git-on-the-Server-Git-Daemon>`_,
+and a security-conscious project would be configured to use
+`git over SSH <https://git-scm.com/book/en/v1/Git-on-the-Server-Getting-Git-on-a-Server#Small-Setups>`_.
+
+Lighttpd is documented on `its wiki <https://redmine.lighttpd.net/projects/lighttpd/wiki>`_.
diff --git a/doc/source/examples/tar-mirror.rst b/doc/source/examples/tar-mirror.rst
new file mode 100644
index 000000000..7a4a2f51e
--- /dev/null
+++ b/doc/source/examples/tar-mirror.rst
@@ -0,0 +1,103 @@
+
+
+.. _using_tar_mirror:
+
+Creating and using a tar mirror
+'''''''''''''''''''''''''''''''
+This is an example of how to create a tar mirror using 
+`lighttpd <https://redmine.lighttpd.net/projects/1/wiki/TutorialConfiguration>`_.
+
+
+Prerequisites
+=============
+You will need `lighttpd` installed.
+
+
+I will be using gnome-modulesets as an example, which can be cloned from
+`http://gnome7.codethink.co.uk/gnome-modulesets.git`.
+
+
+Starting a tar server
+=====================
+
+
+1. Set up a directory containing mirrors
+----------------------------------------
+Choose a suitable directory to hold your mirrored tar files, e.g. `/var/www/tar`.
+
+Place the tar files you want to use as mirrors in your mirror dir, e.g.
+
+.. code::
+
+   mkdir -p /var/www/tar/gettext
+   wget -O /var/www/tar/gettext/gettext-0.19.8.1.tar.xz https://ftp.gnu.org/gnu/gettext/gettext-0.19.8.1.tar.xz
+
+
+2. Configure lighttpd
+---------------------
+Write out a lighttpd.conf as follows:
+
+::
+
+   server.document-root = "/var/www/tar/" 
+   server.port = 3000
+   
+   dir-listing.activate = "enable"
+
+.. note::
+
+   If you have your mirrors in another directory, replace /var/www/tar/ with that directory.
+
+.. note::
+
+   An example lighttpd.conf that works for both git and tar services is available
+   :ref:`here <lighttpd_git_tar_conf>`
+
+
+3. Start lighttpd
+-----------------
+lighttpd can be invoked with the command-line ``lighttpd -D -f lighttpd.conf``.
+
+
+4. Test that you can fetch from it
+----------------------------------
+We can then download the mirrored file with ``wget 127.0.0.1:3000/tar/gettext/gettext-0.19.8.1.tar.xz``.
+
+.. note::
+
+   If you have set server.port to something other than the default, you will need
+   to replace the '3000' in the command-line.
+
+
+5. Configure the project to use the mirror
+------------------------------------------
+To add this local http server as a mirror, add the following to the project.conf:
+
+.. code:: yaml
+
+   mirrors:
+   - name: local-mirror
+     aliases:
+       ftp_gnu_org:
+       - http://127.0.0.1:3000/tar/
+
+
+6. Test that the mirror works
+-----------------------------
+We can make buildstream use the mirror by setting the alias to an invalid URL, e.g.
+
+.. code:: yaml
+
+   aliases:
+     ftp_gnu_org: https://www.example.com/invalid/url/
+
+Now, if you build an element that uses the source you placed in the mirror
+(e.g. ``bst build core-deps/gettext.bst``), you will see that it uses your mirror.
+
+
+Further reading
+===============
+If this mirror isn't being used exclusively in a secure network, it is strongly
+recommended you `use SSL <https://redmine.lighttpd.net/projects/1/wiki/HowToSimpleSSL>`_.
+
+Lighttpd is documented on `its wiki <https://redmine.lighttpd.net/projects/lighttpd/wiki>`_.
diff --git a/doc/source/format_project.rst b/doc/source/format_project.rst
index 36de41a55..cf0970f77 100644
--- a/doc/source/format_project.rst
+++ b/doc/source/format_project.rst
@@ -198,6 +198,43 @@ You can also specify a list of caches here; earlier entries in the list
 will have higher priority than later ones.
 
 
+.. _project_essentials_mirrors:
+
+Mirrors
+~~~~~~~
+A list of mirrors can be defined that couple a location to a mapping of aliases to a
+list of URIs, e.g.
+
+.. code:: yaml
+
+  mirrors:
+  - name: middle-earth
+    aliases:
+      foo:
+      - http://www.middle-earth.com/foo/1
+      - http://www.middle-earth.com/foo/2
+      bar:
+      - http://www.middle-earth.com/bar/1
+      - http://www.middle-earth.com/bar/2
+  - name: oz
+    aliases:
+      foo:
+      - http://www.oz.com/foo
+      bar:
+      - http://www.oz.com/bar
+
+The order that the mirrors (and the URIs therein) are consulted is in the order
+they are defined when fetching, and in reverse-order when tracking.
+
+A default mirror to consult first can be defined via
+:ref:`user config <config_default_mirror>`, or the command-line argument
+:ref:`--default-mirror <invoking_bst>`.
+
+.. note::
+
+   The ``mirrors`` field is available since :ref:`format version 11 <project_format_version>`
+
+
 .. _project_plugins:
 
 External plugins
diff --git a/doc/source/using_config.rst b/doc/source/using_config.rst
index d02aaf3d4..c707cd04b 100644
--- a/doc/source/using_config.rst
+++ b/doc/source/using_config.rst
@@ -89,6 +89,27 @@ modifying some low level component.
    the ``--strict`` and ``--no-strict`` command line options.
 
 
+.. _config_default_mirror:
+
+Default Mirror
+~~~~~~~~~~~~~~
+When using :ref:`mirrors <project_essentials_mirrors>`, a default mirror can
+be defined to be fetched first.
+The default mirror is defined by its name, e.g.
+
+.. code:: yaml
+
+  projects:
+    project-name:
+      default-mirror: oz
+
+
+.. note::
+
+   It is possible to override this at invocation time using the
+   ``--default-mirror`` command-line option.
+
+
 Default configuration
 ---------------------
 The default BuildStream configuration is specified here for reference:
diff --git a/doc/source/using_examples.rst b/doc/source/using_examples.rst
index aa100d007..622b09e32 100644
--- a/doc/source/using_examples.rst
+++ b/doc/source/using_examples.rst
@@ -10,3 +10,5 @@ maintained and work as expected.
    :maxdepth: 1
 
    examples/flatpak-autotools
+   examples/tar-mirror
+   examples/git-mirror
diff --git a/tests/completions/completions.py b/tests/completions/completions.py
index 7c169c2d2..1ff026ea5 100644
--- a/tests/completions/completions.py
+++ b/tests/completions/completions.py
@@ -27,6 +27,7 @@ MAIN_OPTIONS = [
     "--colors ",
     "--config ",
     "--debug ",
+    "--default-mirror ",
     "--directory ",
     "--error-lines ",
     "--fetchers ",
diff --git a/tests/frontend/mirror.py b/tests/frontend/mirror.py
new file mode 100644
index 000000000..62c796ab8
--- /dev/null
+++ b/tests/frontend/mirror.py
@@ -0,0 +1,402 @@
+import os
+import pytest
+
+from tests.testutils import cli, create_repo, ALL_REPO_KINDS
+
+from buildstream import _yaml
+
+
+# Project directory
+TOP_DIR = os.path.dirname(os.path.realpath(__file__))
+DATA_DIR = os.path.join(TOP_DIR, 'project')
+
+
+def generate_element(output_file):
+    element = {
+        'kind': 'import',
+        'sources': [
+            {
+                'kind': 'fetch_source',
+                "output-text": output_file,
+                "urls": ["foo:repo1", "bar:repo2"],
+                "fetch-succeeds": {
+                    "FOO/repo1": True,
+                    "BAR/repo2": False,
+                    "OOF/repo1": False,
+                    "RAB/repo2": True,
+                    "OFO/repo1": False,
+                    "RBA/repo2": False,
+                    "ooF/repo1": False,
+                    "raB/repo2": False,
+                }
+            }
+        ]
+    }
+    return element
+
+
+def generate_project():
+    project = {
+        'name': 'test',
+        'element-path': 'elements',
+        'aliases': {
+            'foo': 'FOO/',
+            'bar': 'BAR/',
+        },
+        'mirrors': [
+            {
+                'name': 'middle-earth',
+                'aliases': {
+                    'foo': ['OOF/'],
+                    'bar': ['RAB/'],
+                },
+            },
+            {
+                'name': 'arrakis',
+                'aliases': {
+                    'foo': ['OFO/'],
+                    'bar': ['RBA/'],
+                },
+            },
+            {
+                'name': 'oz',
+                'aliases': {
+                    'foo': ['ooF/'],
+                    'bar': ['raB/'],
+                }
+            },
+        ],
+        'plugins': [
+            {
+                'origin': 'local',
+                'path': 'sources',
+                'sources': {
+                    'fetch_source': 0
+                }
+            }
+        ]
+    }
+    return project
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+def test_mirror_fetch(cli, tmpdir, datafiles, kind):
+    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
+    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
+    upstream_repodir = os.path.join(str(tmpdir), 'upstream')
+    mirror_repodir = os.path.join(str(tmpdir), 'mirror')
+    project_dir = os.path.join(str(tmpdir), 'project')
+    os.makedirs(project_dir)
+    element_dir = os.path.join(project_dir, 'elements')
+
+    # Create repo objects of the upstream and mirror
+    upstream_repo = create_repo(kind, upstream_repodir)
+    upstream_ref = upstream_repo.create(bin_files_path)
+    mirror_repo = upstream_repo.copy(mirror_repodir)
+    mirror_ref = upstream_ref
+    upstream_ref = upstream_repo.create(dev_files_path)
+
+    element = {
+        'kind': 'import',
+        'sources': [
+            upstream_repo.source_config(ref=upstream_ref)
+        ]
+    }
+    element_name = 'test.bst'
+    element_path = os.path.join(element_dir, element_name)
+    full_repo = element['sources'][0]['url']
+    upstream_map, repo_name = os.path.split(full_repo)
+    alias = 'foo-' + kind
+    aliased_repo = alias + ':' + repo_name
+    element['sources'][0]['url'] = aliased_repo
+    full_mirror = mirror_repo.source_config()['url']
+    mirror_map, _ = os.path.split(full_mirror)
+    os.makedirs(element_dir)
+    _yaml.dump(element, element_path)
+
+    project = {
+        'name': 'test',
+        'element-path': 'elements',
+        'aliases': {
+            alias: upstream_map + "/"
+        },
+        'mirrors': [
+            {
+                'name': 'middle-earth',
+                'aliases': {
+                    alias: [mirror_map + "/"],
+                },
+            },
+        ]
+    }
+    project_file = os.path.join(project_dir, 'project.conf')
+    _yaml.dump(project, project_file)
+
+    # No obvious ways of checking that the mirror has been fetched
+    # But at least we can be sure it succeeds
+    result = cli.run(project=project_dir, args=['fetch', element_name])
+    result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_mirror_fetch_multi(cli, tmpdir, datafiles):
+    output_file = os.path.join(str(tmpdir), "output.txt")
+    project_dir = str(tmpdir)
+    element_dir = os.path.join(project_dir, 'elements')
+    os.makedirs(element_dir, exist_ok=True)
+    element_name = "test.bst"
+    element_path = os.path.join(element_dir, element_name)
+    element = generate_element(output_file)
+    _yaml.dump(element, element_path)
+
+    project_file = os.path.join(project_dir, 'project.conf')
+    project = generate_project()
+    _yaml.dump(project, project_file)
+
+    result = cli.run(project=project_dir, args=['fetch', element_name])
+    result.assert_success()
+    with open(output_file) as f:
+        contents = f.read()
+        assert "Fetch foo:repo1 succeeded from FOO/repo1" in contents
+        assert "Fetch bar:repo2 succeeded from RAB/repo2" in contents
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_mirror_fetch_default_cmdline(cli, tmpdir, datafiles):
+    output_file = os.path.join(str(tmpdir), "output.txt")
+    project_dir = str(tmpdir)
+    element_dir = os.path.join(project_dir, 'elements')
+    os.makedirs(element_dir, exist_ok=True)
+    element_name = "test.bst"
+    element_path = os.path.join(element_dir, element_name)
+    element = generate_element(output_file)
+    _yaml.dump(element, element_path)
+
+    project_file = os.path.join(project_dir, 'project.conf')
+    project = generate_project()
+    _yaml.dump(project, project_file)
+
+    result = cli.run(project=project_dir, args=['--default-mirror', 'arrakis', 'fetch', element_name])
+    result.assert_success()
+    with open(output_file) as f:
+        contents = f.read()
+        print(contents)
+        # Success if fetching from arrakis' mirror happened before middle-earth's
+        arrakis_str = "OFO/repo1"
+        arrakis_pos = contents.find(arrakis_str)
+        assert arrakis_pos != -1, "'{}' wasn't found".format(arrakis_str)
+        me_str = "OOF/repo1"
+        me_pos = contents.find(me_str)
+        assert me_pos != -1, "'{}' wasn't found".format(me_str)
+        assert arrakis_pos < me_pos, "'{}' wasn't found before '{}'".format(arrakis_str, me_str)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_mirror_fetch_default_userconfig(cli, tmpdir, datafiles):
+    output_file = os.path.join(str(tmpdir), "output.txt")
+    project_dir = str(tmpdir)
+    element_dir = os.path.join(project_dir, 'elements')
+    os.makedirs(element_dir, exist_ok=True)
+    element_name = "test.bst"
+    element_path = os.path.join(element_dir, element_name)
+    element = generate_element(output_file)
+    _yaml.dump(element, element_path)
+
+    project_file = os.path.join(project_dir, 'project.conf')
+    project = generate_project()
+    _yaml.dump(project, project_file)
+
+    userconfig = {
+        'projects': {
+            'test': {
+                'default-mirror': 'oz'
+            }
+        }
+    }
+    cli.configure(userconfig)
+
+    result = cli.run(project=project_dir, args=['fetch', element_name])
+    result.assert_success()
+    with open(output_file) as f:
+        contents = f.read()
+        print(contents)
+        # Success if fetching from Oz' mirror happened before middle-earth's
+        oz_str = "ooF/repo1"
+        oz_pos = contents.find(oz_str)
+        assert oz_pos != -1, "'{}' wasn't found".format(oz_str)
+        me_str = "OOF/repo1"
+        me_pos = contents.find(me_str)
+        assert me_pos != -1, "'{}' wasn't found".format(me_str)
+        assert oz_pos < me_pos, "'{}' wasn't found before '{}'".format(oz_str, me_str)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_mirror_fetch_default_cmdline_overrides_config(cli, tmpdir, datafiles):
+    output_file = os.path.join(str(tmpdir), "output.txt")
+    project_dir = str(tmpdir)
+    element_dir = os.path.join(project_dir, 'elements')
+    os.makedirs(element_dir, exist_ok=True)
+    element_name = "test.bst"
+    element_path = os.path.join(element_dir, element_name)
+    element = generate_element(output_file)
+    _yaml.dump(element, element_path)
+
+    project_file = os.path.join(project_dir, 'project.conf')
+    project = generate_project()
+    _yaml.dump(project, project_file)
+
+    userconfig = {
+        'projects': {
+            'test': {
+                'default-mirror': 'oz'
+            }
+        }
+    }
+    cli.configure(userconfig)
+
+    result = cli.run(project=project_dir, args=['--default-mirror', 'arrakis', 'fetch', element_name])
+    result.assert_success()
+    with open(output_file) as f:
+        contents = f.read()
+        print(contents)
+        # Success if fetching from arrakis' mirror happened before middle-earth's
+        arrakis_str = "OFO/repo1"
+        arrakis_pos = contents.find(arrakis_str)
+        assert arrakis_pos != -1, "'{}' wasn't found".format(arrakis_str)
+        me_str = "OOF/repo1"
+        me_pos = contents.find(me_str)
+        assert me_pos != -1, "'{}' wasn't found".format(me_str)
+        assert arrakis_pos < me_pos, "'{}' wasn't found before '{}'".format(arrakis_str, me_str)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+def test_mirror_track_upstream_present(cli, tmpdir, datafiles, kind):
+    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
+    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
+    upstream_repodir = os.path.join(str(tmpdir), 'upstream')
+    mirror_repodir = os.path.join(str(tmpdir), 'mirror')
+    project_dir = os.path.join(str(tmpdir), 'project')
+    os.makedirs(project_dir)
+    element_dir = os.path.join(project_dir, 'elements')
+
+    # Create repo objects of the upstream and mirror
+    upstream_repo = create_repo(kind, upstream_repodir)
+    upstream_ref = upstream_repo.create(bin_files_path)
+    mirror_repo = upstream_repo.copy(mirror_repodir)
+    mirror_ref = upstream_ref
+    upstream_ref = upstream_repo.create(dev_files_path)
+
+    element = {
+        'kind': 'import',
+        'sources': [
+            upstream_repo.source_config(ref=upstream_ref)
+        ]
+    }
+
+    element['sources'][0]
+    element_name = 'test.bst'
+    element_path = os.path.join(element_dir, element_name)
+    full_repo = element['sources'][0]['url']
+    upstream_map, repo_name = os.path.split(full_repo)
+    alias = 'foo-' + kind
+    aliased_repo = alias + ':' + repo_name
+    element['sources'][0]['url'] = aliased_repo
+    full_mirror = mirror_repo.source_config()['url']
+    mirror_map, _ = os.path.split(full_mirror)
+    os.makedirs(element_dir)
+    _yaml.dump(element, element_path)
+
+    project = {
+        'name': 'test',
+        'element-path': 'elements',
+        'aliases': {
+            alias: upstream_map + "/"
+        },
+        'mirrors': [
+            {
+                'name': 'middle-earth',
+                'aliases': {
+                    alias: [mirror_map + "/"],
+                },
+            },
+        ]
+    }
+    project_file = os.path.join(project_dir, 'project.conf')
+    _yaml.dump(project, project_file)
+
+    result = cli.run(project=project_dir, args=['track', element_name])
+    result.assert_success()
+
+    # Tracking tries upstream first. Check the ref is from upstream.
+    new_element = _yaml.load(element_path)
+    source = new_element['sources'][0]
+    if 'ref' in source:
+        assert source['ref'] == upstream_ref
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind):
+    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
+    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
+    upstream_repodir = os.path.join(str(tmpdir), 'upstream')
+    mirror_repodir = os.path.join(str(tmpdir), 'mirror')
+    project_dir = os.path.join(str(tmpdir), 'project')
+    os.makedirs(project_dir)
+    element_dir = os.path.join(project_dir, 'elements')
+
+    # Create repo objects of the upstream and mirror
+    upstream_repo = create_repo(kind, upstream_repodir)
+    upstream_ref = upstream_repo.create(bin_files_path)
+    mirror_repo = upstream_repo.copy(mirror_repodir)
+    mirror_ref = upstream_ref
+    upstream_ref = upstream_repo.create(dev_files_path)
+
+    element = {
+        'kind': 'import',
+        'sources': [
+            upstream_repo.source_config(ref=upstream_ref)
+        ]
+    }
+
+    element['sources'][0]
+    element_name = 'test.bst'
+    element_path = os.path.join(element_dir, element_name)
+    full_repo = element['sources'][0]['url']
+    upstream_map, repo_name = os.path.split(full_repo)
+    alias = 'foo-' + kind
+    aliased_repo = alias + ':' + repo_name
+    element['sources'][0]['url'] = aliased_repo
+    full_mirror = mirror_repo.source_config()['url']
+    mirror_map, _ = os.path.split(full_mirror)
+    os.makedirs(element_dir)
+    _yaml.dump(element, element_path)
+
+    project = {
+        'name': 'test',
+        'element-path': 'elements',
+        'aliases': {
+            alias: 'http://www.example.com/'
+        },
+        'mirrors': [
+            {
+                'name': 'middle-earth',
+                'aliases': {
+                    alias: [mirror_map + "/"],
+                },
+            },
+        ]
+    }
+    project_file = os.path.join(project_dir, 'project.conf')
+    _yaml.dump(project, project_file)
+
+    result = cli.run(project=project_dir, args=['track', element_name])
+    result.assert_success()
+
+    # Check that tracking fell back to the mirror
+    new_element = _yaml.load(element_path)
+    source = new_element['sources'][0]
+    if 'ref' in source:
+        assert source['ref'] == mirror_ref
diff --git a/tests/frontend/project/sources/fetch_source.py b/tests/frontend/project/sources/fetch_source.py
new file mode 100644
index 000000000..ebd3fe757
--- /dev/null
+++ b/tests/frontend/project/sources/fetch_source.py
@@ -0,0 +1,85 @@
+import os
+import sys
+
+from buildstream import Source, Consistency, SourceError, SourceFetcher
+
+# Expected config
+# sources:
+# - output-text: $FILE
+#   urls:
+#   - foo:bar
+#   - baz:quux
+#   fetch-succeeds:
+#     Foo/bar: true
+#     ooF/bar: false
+
+
+class FetchFetcher(SourceFetcher):
+    def __init__(self, source, url):
+        super().__init__()
+        self.source = source
+        self.original_url = url
+        self.mark_download_url(url)
+
+    def fetch(self, alias_override=None):
+        url = self.source.translate_url(self.original_url, alias_override=alias_override)
+        with open(self.source.output_file, "a") as f:
+            success = url in self.source.fetch_succeeds and self.source.fetch_succeeds[url]
+            message = "Fetch {} {} from {}\n".format(self.original_url,
+                                                     "succeeded" if success else "failed",
+                                                     url)
+            f.write(message)
+            if not success:
+                raise SourceError("Failed to fetch {}".format(url))
+
+
+class FetchSource(Source):
+    # Read config to know which URLs to fetch
+    def configure(self, node):
+        self.original_urls = self.node_get_member(node, list, 'urls')
+        self.fetchers = [FetchFetcher(self, url) for url in self.original_urls]
+        self.output_file = self.node_get_member(node, str, 'output-text')
+        self.fetch_succeeds = {}
+        if 'fetch-succeeds' in node:
+            self.fetch_succeeds = {x[0]: x[1] for x in self.node_items(node['fetch-succeeds'])}
+
+    def get_source_fetchers(self):
+        return self.fetchers
+
+    def preflight(self):
+        output_dir = os.path.dirname(self.output_file)
+        if not os.path.exists(output_dir):
+            raise SourceError("Directory '{}' does not exist".format(output_dir))
+
+    def fetch(self):
+        for fetcher in self.fetchers:
+            fetcher.fetch()
+
+    def get_unique_key(self):
+        return {"urls": self.original_urls, "output_file": self.output_file}
+
+    def get_consistency(self):
+        if not os.path.exists(self.output_file):
+            return Consistency.RESOLVED
+
+        with open(self.output_file, "r") as f:
+            contents = f.read()
+            for url in self.original_urls:
+                if url not in contents:
+                    return Consistency.RESOLVED
+
+        return Consistency.CACHED
+
+    # We dont have a ref, we're a local file...
+    def load_ref(self, node):
+        pass
+
+    def get_ref(self):
+        return None  # pragma: nocover
+
+    def set_ref(self, ref, node):
+        pass  # pragma: nocover
+
+
+def setup():
+    return FetchSource
diff --git a/tests/testutils/repo/repo.py b/tests/testutils/repo/repo.py
index 4c9ee59a9..234aa374c 100644
--- a/tests/testutils/repo/repo.py
+++ b/tests/testutils/repo/repo.py
@@ -22,7 +22,7 @@ class Repo():
         # The directory the actual repo will be stored in
         self.repo = os.path.join(self.directory, subdir)
 
-        os.makedirs(self.repo)
+        os.makedirs(self.repo, exist_ok=True)
 
     # create():
     #
@@ -69,3 +69,22 @@ class Repo():
                 shutil.copytree(src_path, dest_path)
             else:
                 shutil.copy2(src_path, dest_path)
+
+    # copy():
+    #
+    # Creates a copy of this repository in the specified
+    # destination.
+    #
+    # Args:
+    #    dest (str): The destination directory
+    #
+    # Returns:
+    #    (Repo): A Repo object for the new repository.
+    def copy(self, dest):
+        subdir = self.repo[len(self.directory):].lstrip(os.sep)
+        new_dir = os.path.join(dest, subdir)
+        os.makedirs(new_dir, exist_ok=True)
+        self.copy_directory(self.repo, new_dir)
+        repo_type = type(self)
+        new_repo = repo_type(dest, subdir)
+        return new_repo