Merge branch 'valentindavid/fallback_mirror_git-1.2' into 'bst-1.2'

Delay refreshing git submodule until we have a clone

See merge request BuildStream/buildstream!666

4 files changed, 337 insertions, 5 deletions

diff --git a/buildstream/plugins/sources/git.py b/buildstream/plugins/sources/git.py
index 073365bff..b3cf903bf 100644
--- a/buildstream/plugins/sources/git.py
+++ b/buildstream/plugins/sources/git.py
@@ -404,8 +404,10 @@ class GitSource(Source):
                     mirror.stage(directory)
 
     def get_source_fetchers(self):
+        yield self.mirror
         self.refresh_submodules()
-        return [self.mirror] + self.submodules
+        for submodule in self.submodules:
+            yield submodule
 
     ###########################################################
     #                     Local Functions                     #
diff --git a/tests/frontend/mirror.py b/tests/frontend/mirror.py
index f37cc18af..93e270b4d 100644
--- a/tests/frontend/mirror.py
+++ b/tests/frontend/mirror.py
@@ -140,6 +140,67 @@ def test_mirror_fetch(cli, tmpdir, datafiles, kind):
 
 
 @pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+def test_mirror_fetch_upstream_absent(cli, tmpdir, datafiles, kind):
+    if kind == 'ostree':
+        # FIXME: Mirroring fallback fails with ostree
+        pytest.skip("Bug #538 - ostree mirror fallback breaks assertion")
+
+    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
+    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
+    upstream_repodir = os.path.join(str(tmpdir), 'upstream')
+    mirror_repodir = os.path.join(str(tmpdir), 'mirror')
+    project_dir = os.path.join(str(tmpdir), 'project')
+    os.makedirs(project_dir)
+    element_dir = os.path.join(project_dir, 'elements')
+
+    # Create repo objects of the upstream and mirror
+    upstream_repo = create_repo(kind, upstream_repodir)
+    ref = upstream_repo.create(dev_files_path)
+    mirror_repo = upstream_repo.copy(mirror_repodir)
+
+    element = {
+        'kind': 'import',
+        'sources': [
+            upstream_repo.source_config(ref=ref)
+        ]
+    }
+
+    element_name = 'test.bst'
+    element_path = os.path.join(element_dir, element_name)
+    full_repo = element['sources'][0]['url']
+    upstream_map, repo_name = os.path.split(full_repo)
+    alias = 'foo-' + kind
+    aliased_repo = alias + ':' + repo_name
+    element['sources'][0]['url'] = aliased_repo
+    full_mirror = mirror_repo.source_config()['url']
+    mirror_map, _ = os.path.split(full_mirror)
+    os.makedirs(element_dir)
+    _yaml.dump(element, element_path)
+
+    project = {
+        'name': 'test',
+        'element-path': 'elements',
+        'aliases': {
+            alias: 'http://www.example.com/'
+        },
+        'mirrors': [
+            {
+                'name': 'middle-earth',
+                'aliases': {
+                    alias: [mirror_map + "/"],
+                },
+            },
+        ]
+    }
+    project_file = os.path.join(project_dir, 'project.conf')
+    _yaml.dump(project, project_file)
+
+    result = cli.run(project=project_dir, args=['fetch', element_name])
+    result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
 def test_mirror_fetch_multi(cli, tmpdir, datafiles):
     output_file = os.path.join(str(tmpdir), "output.txt")
     project_dir = str(tmpdir)
@@ -405,10 +466,6 @@ def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind):
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
 def test_mirror_from_includes(cli, tmpdir, datafiles, kind):
-    if kind == 'git':
-        # FIXME: Mirroring fallback does not work with git because it tries to
-        # fetch submodules on upstream.
-        pytest.skip("Bug #537 - Mirror fallback does not work for git")
     if kind == 'ostree':
         # FIXME: Mirroring fallback fails with ostree
         pytest.skip("Bug #538 - ostree mirror fallback breaks assertion")
@@ -559,3 +616,276 @@ def test_mirror_junction_from_includes(cli, tmpdir, datafiles, kind):
     os.rename('{}.bak'.format(upstream_repo.repo), upstream_repo.repo)
     result = cli.run(project=project_dir, args=['fetch', element_name])
     result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_mirror_git_submodule_fetch(cli, tmpdir, datafiles):
+    # Test that it behaves as expected with submodules, both defined in config
+    # and discovered when fetching.
+    foo_file = os.path.join(str(datafiles), 'files', 'foo')
+    bar_file = os.path.join(str(datafiles), 'files', 'bar')
+    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
+    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
+    mirror_dir = os.path.join(str(datafiles), 'mirror')
+
+    defined_subrepo = create_repo('git', str(tmpdir), 'defined_subrepo')
+    defined_mirror_ref = defined_subrepo.create(bin_files_path)
+    defined_mirror = defined_subrepo.copy(mirror_dir)
+    defined_subref = defined_subrepo.add_file(foo_file)
+
+    found_subrepo = create_repo('git', str(tmpdir), 'found_subrepo')
+    found_subref = found_subrepo.create(dev_files_path)
+
+    main_repo = create_repo('git', str(tmpdir))
+    main_mirror_ref = main_repo.create(bin_files_path)
+    main_repo.add_submodule('defined', 'file://' + defined_subrepo.repo)
+    main_repo.add_submodule('found', 'file://' + found_subrepo.repo)
+    main_mirror = main_repo.copy(mirror_dir)
+    main_ref = main_repo.add_file(bar_file)
+
+    project_dir = os.path.join(str(tmpdir), 'project')
+    os.makedirs(project_dir)
+    element_dir = os.path.join(project_dir, 'elements')
+    os.makedirs(element_dir)
+    element = {
+        'kind': 'import',
+        'sources': [
+            main_repo.source_config(ref=main_mirror_ref)
+        ]
+    }
+    element_name = 'test.bst'
+    element_path = os.path.join(element_dir, element_name)
+
+    # Alias the main repo
+    full_repo = element['sources'][0]['url']
+    _, repo_name = os.path.split(full_repo)
+    alias = 'foo'
+    aliased_repo = alias + ':' + repo_name
+    element['sources'][0]['url'] = aliased_repo
+
+    # Hide the found subrepo
+    del element['sources'][0]['submodules']['found']
+
+    # Alias the defined subrepo
+    subrepo = element['sources'][0]['submodules']['defined']['url']
+    _, repo_name = os.path.split(subrepo)
+    aliased_repo = alias + ':' + repo_name
+    element['sources'][0]['submodules']['defined']['url'] = aliased_repo
+
+    _yaml.dump(element, element_path)
+
+    full_mirror = main_mirror.source_config()['url']
+    mirror_map, _ = os.path.split(full_mirror)
+    project = {
+        'name': 'test',
+        'element-path': 'elements',
+        'aliases': {
+            alias: 'http://www.example.com/'
+        },
+        'mirrors': [
+            {
+                'name': 'middle-earth',
+                'aliases': {
+                    alias: [mirror_map + "/"],
+                },
+            },
+        ]
+    }
+    project_file = os.path.join(project_dir, 'project.conf')
+    _yaml.dump(project, project_file)
+
+    result = cli.run(project=project_dir, args=['fetch', element_name])
+    result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_mirror_fallback_git_only_submodules(cli, tmpdir, datafiles):
+    # Main repo has no mirror or alias.
+    # One submodule is overridden to use a mirror.
+    # There is another submodules not overriden.
+    # Upstream for overriden submodule is down.
+    #
+    # We expect:
+    #  - overriden submodule is fetched from mirror.
+    #  - other submodule is fetched.
+
+    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
+    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
+
+    upstream_bin_repodir = os.path.join(str(tmpdir), 'bin-upstream')
+    mirror_bin_repodir = os.path.join(str(tmpdir), 'bin-mirror')
+    upstream_bin_repo = create_repo('git', upstream_bin_repodir)
+    upstream_bin_repo.create(bin_files_path)
+    mirror_bin_repo = upstream_bin_repo.copy(mirror_bin_repodir)
+
+    dev_repodir = os.path.join(str(tmpdir), 'dev-upstream')
+    dev_repo = create_repo('git', dev_repodir)
+    dev_repo.create(dev_files_path)
+
+    main_files = os.path.join(str(tmpdir), 'main-files')
+    os.makedirs(main_files)
+    with open(os.path.join(main_files, 'README'), 'w') as f:
+        f.write("TEST\n")
+    main_repodir = os.path.join(str(tmpdir), 'main-upstream')
+    main_repo = create_repo('git', main_repodir)
+    main_repo.create(main_files)
+
+    upstream_url = 'file://{}'.format(upstream_bin_repo.repo)
+    main_repo.add_submodule('bin', url=upstream_url)
+    main_repo.add_submodule('dev', url='file://{}'.format(dev_repo.repo))
+    # Unlist 'dev'.
+    del main_repo.submodules['dev']
+
+    main_ref = main_repo.latest_commit()
+
+    upstream_map, repo_name = os.path.split(upstream_url)
+    alias = 'foo'
+    aliased_repo = '{}:{}'.format(alias, repo_name)
+    main_repo.submodules['bin']['url'] = aliased_repo
+
+    full_mirror = mirror_bin_repo.source_config()['url']
+    mirror_map, _ = os.path.split(full_mirror)
+
+    project_dir = os.path.join(str(tmpdir), 'project')
+    os.makedirs(project_dir)
+    element_dir = os.path.join(project_dir, 'elements')
+
+    element = {
+        'kind': 'import',
+        'sources': [
+            main_repo.source_config(ref=main_ref, checkout_submodules=True)
+        ]
+    }
+    element_name = 'test.bst'
+    element_path = os.path.join(element_dir, element_name)
+    os.makedirs(element_dir)
+    _yaml.dump(element, element_path)
+
+    project = {
+        'name': 'test',
+        'element-path': 'elements',
+        'aliases': {
+            alias: upstream_map + "/"
+        },
+        'mirrors': [
+            {
+                'name': 'middle-earth',
+                'aliases': {
+                    alias: [mirror_map + "/"],
+                }
+            }
+        ]
+    }
+    project_file = os.path.join(project_dir, 'project.conf')
+    _yaml.dump(project, project_file)
+
+    # Now make the upstream unavailable.
+    os.rename(upstream_bin_repo.repo, '{}.bak'.format(upstream_bin_repo.repo))
+    result = cli.run(project=project_dir, args=['fetch', element_name])
+    result.assert_success()
+
+    result = cli.run(project=project_dir, args=['build', element_name])
+    result.assert_success()
+
+    checkout = os.path.join(str(tmpdir), 'checkout')
+    result = cli.run(project=project_dir, args=['checkout', element_name, checkout])
+    result.assert_success()
+
+    assert os.path.exists(os.path.join(checkout, 'bin', 'bin', 'hello'))
+    assert os.path.exists(os.path.join(checkout, 'dev', 'include', 'pony.h'))
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_mirror_fallback_git_with_submodules(cli, tmpdir, datafiles):
+    # Main repo has mirror. But does not list submodules.
+    #
+    # We expect:
+    #  - we will fetch submodules anyway
+
+    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
+    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
+
+    bin_repodir = os.path.join(str(tmpdir), 'bin-repo')
+    bin_repo = create_repo('git', bin_repodir)
+    bin_repo.create(bin_files_path)
+
+    dev_repodir = os.path.join(str(tmpdir), 'dev-repo')
+    dev_repo = create_repo('git', dev_repodir)
+    dev_repo.create(dev_files_path)
+
+    main_files = os.path.join(str(tmpdir), 'main-files')
+    os.makedirs(main_files)
+    with open(os.path.join(main_files, 'README'), 'w') as f:
+        f.write("TEST\n")
+    upstream_main_repodir = os.path.join(str(tmpdir), 'main-upstream')
+    upstream_main_repo = create_repo('git', upstream_main_repodir)
+    upstream_main_repo.create(main_files)
+
+    upstream_main_repo.add_submodule('bin', url='file://{}'.format(bin_repo.repo))
+    upstream_main_repo.add_submodule('dev', url='file://{}'.format(dev_repo.repo))
+    # Unlist submodules.
+    del upstream_main_repo.submodules['bin']
+    del upstream_main_repo.submodules['dev']
+
+    upstream_main_ref = upstream_main_repo.latest_commit()
+
+    mirror_main_repodir = os.path.join(str(tmpdir), 'main-mirror')
+    mirror_main_repo = upstream_main_repo.copy(mirror_main_repodir)
+
+    upstream_url = mirror_main_repo.source_config()['url']
+
+    upstream_map, repo_name = os.path.split(upstream_url)
+    alias = 'foo'
+    aliased_repo = '{}:{}'.format(alias, repo_name)
+
+    full_mirror = mirror_main_repo.source_config()['url']
+    mirror_map, _ = os.path.split(full_mirror)
+
+    project_dir = os.path.join(str(tmpdir), 'project')
+    os.makedirs(project_dir)
+    element_dir = os.path.join(project_dir, 'elements')
+
+    element = {
+        'kind': 'import',
+        'sources': [
+            upstream_main_repo.source_config(ref=upstream_main_ref, checkout_submodules=True)
+        ]
+    }
+    element['sources'][0]['url'] = aliased_repo
+    element_name = 'test.bst'
+    element_path = os.path.join(element_dir, element_name)
+    os.makedirs(element_dir)
+    _yaml.dump(element, element_path)
+
+    project = {
+        'name': 'test',
+        'element-path': 'elements',
+        'aliases': {
+            alias: upstream_map + "/"
+        },
+        'mirrors': [
+            {
+                'name': 'middle-earth',
+                'aliases': {
+                    alias: [mirror_map + "/"],
+                }
+            }
+        ]
+    }
+    project_file = os.path.join(project_dir, 'project.conf')
+    _yaml.dump(project, project_file)
+
+    # Now make the upstream unavailable.
+    os.rename(upstream_main_repo.repo, '{}.bak'.format(upstream_main_repo.repo))
+    result = cli.run(project=project_dir, args=['fetch', element_name])
+    result.assert_success()
+
+    result = cli.run(project=project_dir, args=['build', element_name])
+    result.assert_success()
+
+    checkout = os.path.join(str(tmpdir), 'checkout')
+    result = cli.run(project=project_dir, args=['checkout', element_name, checkout])
+    result.assert_success()
+
+    assert os.path.exists(os.path.join(checkout, 'bin', 'bin', 'hello'))
+    assert os.path.exists(os.path.join(checkout, 'dev', 'include', 'pony.h'))
diff --git a/tests/frontend/project/files/bar b/tests/frontend/project/files/bar
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/frontend/project/files/bar
diff --git a/tests/frontend/project/files/foo b/tests/frontend/project/files/foo
new file mode 100644
index 000000000..e69de29bb
--- /dev/null
+++ b/tests/frontend/project/files/foo