Add support for include '(@)' in project.conf and .bst files

Fixes #331.

51 files changed, 878 insertions, 3 deletions

diff --git a/tests/artifactcache/config.py b/tests/artifactcache/config.py
index 079e511ef..f59474708 100644
--- a/tests/artifactcache/config.py
+++ b/tests/artifactcache/config.py
@@ -98,6 +98,7 @@ def test_artifact_cache_precedence(tmpdir, override_caches, project_caches, user
     context = Context()
     context.load(config=user_config_file)
     project = Project(str(project_dir), context)
+    project.ensure_fully_loaded()
 
     # Use the helper from the artifactcache module to parse our configuration.
     parsed_cache_specs = _configured_remote_artifact_cache_specs(context, project)
diff --git a/tests/format/include.py b/tests/format/include.py
new file mode 100644
index 000000000..36e723ed0
--- /dev/null
+++ b/tests/format/include.py
@@ -0,0 +1,263 @@
+import os
+import pytest
+from buildstream import _yaml
+from buildstream._exceptions import ErrorDomain, LoadErrorReason
+from tests.testutils import cli, generate_junction, create_repo
+
+
+# Project directory
+DATA_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    'include'
+)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_project_file(cli, datafiles):
+    project = os.path.join(str(datafiles), 'file')
+    result = cli.run(project=project, args=[
+        'show',
+        '--deps', 'none',
+        '--format', '%{vars}',
+        'element.bst'])
+    result.assert_success()
+    loaded = _yaml.load_data(result.output)
+    assert loaded['included'] == 'True'
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_junction_file(cli, tmpdir, datafiles):
+    project = os.path.join(str(datafiles), 'junction')
+
+    generate_junction(tmpdir,
+                      os.path.join(project, 'subproject'),
+                      os.path.join(project, 'junction.bst'),
+                      store_ref=True)
+
+    result = cli.run(project=project, args=[
+        'show',
+        '--deps', 'none',
+        '--format', '%{vars}',
+        'element.bst'])
+    result.assert_success()
+    loaded = _yaml.load_data(result.output)
+    assert loaded['included'] == 'True'
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_junction_options(cli, tmpdir, datafiles):
+    project = os.path.join(str(datafiles), 'options')
+
+    result = cli.run(project=project, args=[
+        '-o', 'build_arch', 'x86_64',
+        'show',
+        '--deps', 'none',
+        '--format', '%{vars}',
+        'element.bst'])
+    result.assert_success()
+    loaded = _yaml.load_data(result.output)
+    assert loaded['build_arch'] == 'x86_64'
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_junction_element_partial_project_project(cli, tmpdir, datafiles):
+    """
+    Junction elements never depend on fully include processed project.
+    """
+
+    project = os.path.join(str(datafiles), 'junction')
+
+    subproject_path = os.path.join(project, 'subproject')
+    junction_path = os.path.join(project, 'junction.bst')
+
+    repo = create_repo('git', str(tmpdir))
+
+    ref = repo.create(subproject_path)
+
+    element = {
+        'kind': 'junction',
+        'sources': [
+            repo.source_config(ref=ref)
+        ]
+    }
+    _yaml.dump(element, junction_path)
+
+    result = cli.run(project=project, args=[
+        'show',
+        '--deps', 'none',
+        '--format', '%{vars}',
+        'junction.bst'])
+    result.assert_success()
+    loaded = _yaml.load_data(result.output)
+    assert 'included' not in loaded
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_junction_element_not_partial_project_file(cli, tmpdir, datafiles):
+    """
+    Junction elements never depend on fully include processed project.
+    """
+
+    project = os.path.join(str(datafiles), 'file_with_subproject')
+
+    subproject_path = os.path.join(project, 'subproject')
+    junction_path = os.path.join(project, 'junction.bst')
+
+    repo = create_repo('git', str(tmpdir))
+
+    ref = repo.create(subproject_path)
+
+    element = {
+        'kind': 'junction',
+        'sources': [
+            repo.source_config(ref=ref)
+        ]
+    }
+    _yaml.dump(element, junction_path)
+
+    result = cli.run(project=project, args=[
+        'show',
+        '--deps', 'none',
+        '--format', '%{vars}',
+        'junction.bst'])
+    result.assert_success()
+    loaded = _yaml.load_data(result.output)
+    assert 'included' in loaded
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_element_overrides(cli, tmpdir, datafiles):
+    project = os.path.join(str(datafiles), 'overrides')
+
+    result = cli.run(project=project, args=[
+        'show',
+        '--deps', 'none',
+        '--format', '%{vars}',
+        'element.bst'])
+    result.assert_success()
+    loaded = _yaml.load_data(result.output)
+    assert 'manual_main_override' in loaded
+    assert 'manual_included_override' in loaded
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_element_overrides_composition(cli, tmpdir, datafiles):
+    project = os.path.join(str(datafiles), 'overrides')
+
+    result = cli.run(project=project, args=[
+        'show',
+        '--deps', 'none',
+        '--format', '%{config}',
+        'element.bst'])
+    result.assert_success()
+    loaded = _yaml.load_data(result.output)
+    assert 'build-commands' in loaded
+    assert loaded['build-commands'] == ['first', 'second']
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_element_overrides_sub_include(cli, tmpdir, datafiles):
+    project = os.path.join(str(datafiles), 'sub-include')
+
+    result = cli.run(project=project, args=[
+        'show',
+        '--deps', 'none',
+        '--format', '%{vars}',
+        'element.bst'])
+    result.assert_success()
+    loaded = _yaml.load_data(result.output)
+    assert 'included' in loaded
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_junction_do_not_use_included_overrides(cli, tmpdir, datafiles):
+    project = os.path.join(str(datafiles), 'overrides-junction')
+
+    generate_junction(tmpdir,
+                      os.path.join(project, 'subproject'),
+                      os.path.join(project, 'junction.bst'),
+                      store_ref=True)
+
+    result = cli.run(project=project, args=[
+        'show',
+        '--deps', 'none',
+        '--format', '%{vars}',
+        'junction.bst'])
+    result.assert_success()
+    loaded = _yaml.load_data(result.output)
+    assert 'main_override' in loaded
+    assert 'included_override' not in loaded
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_conditional_in_fragment(cli, tmpdir, datafiles):
+    project = os.path.join(str(datafiles), 'conditional')
+
+    result = cli.run(project=project, args=[
+        '-o', 'build_arch', 'x86_64',
+        'show',
+        '--deps', 'none',
+        '--format', '%{vars}',
+        'element.bst'])
+    result.assert_success()
+    loaded = _yaml.load_data(result.output)
+    assert 'size' in loaded
+    assert loaded['size'] == '8'
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_inner(cli, datafiles):
+    project = os.path.join(str(datafiles), 'inner')
+    result = cli.run(project=project, args=[
+        '-o', 'build_arch', 'x86_64',
+        'show',
+        '--deps', 'none',
+        '--format', '%{vars}',
+        'element.bst'])
+    result.assert_success()
+    loaded = _yaml.load_data(result.output)
+    assert loaded['build_arch'] == 'x86_64'
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_recusive_include(cli, tmpdir, datafiles):
+    project = os.path.join(str(datafiles), 'recursive')
+
+    result = cli.run(project=project, args=[
+        'show',
+        '--deps', 'none',
+        '--format', '%{vars}',
+        'element.bst'])
+    result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.RECURSIVE_INCLUDE)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_local_to_junction(cli, tmpdir, datafiles):
+    project = os.path.join(str(datafiles), 'local_to_junction')
+
+    generate_junction(tmpdir,
+                      os.path.join(project, 'subproject'),
+                      os.path.join(project, 'junction.bst'),
+                      store_ref=True)
+
+    result = cli.run(project=project, args=[
+        'show',
+        '--deps', 'none',
+        '--format', '%{vars}',
+        'element.bst'])
+    result.assert_success()
+    loaded = _yaml.load_data(result.output)
+    assert loaded['included'] == 'True'
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_include_project_file(cli, datafiles):
+    project = os.path.join(str(datafiles), 'string')
+    result = cli.run(project=project, args=[
+        'show',
+        '--deps', 'none',
+        '--format', '%{vars}',
+        'element.bst'])
+    result.assert_success()
+    loaded = _yaml.load_data(result.output)
+    assert loaded['included'] == 'True'
diff --git a/tests/format/include/conditional/element.bst b/tests/format/include/conditional/element.bst
new file mode 100644
index 000000000..4d7f70266
--- /dev/null
+++ b/tests/format/include/conditional/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/conditional/extra_conf.yml b/tests/format/include/conditional/extra_conf.yml
new file mode 100644
index 000000000..dd58c9855
--- /dev/null
+++ b/tests/format/include/conditional/extra_conf.yml
@@ -0,0 +1,6 @@
+variables:
+  (?):
+    - build_arch == "i586":
+        size: "4"
+    - build_arch == "x86_64":
+        size: "8"
diff --git a/tests/format/include/conditional/project.conf b/tests/format/include/conditional/project.conf
new file mode 100644
index 000000000..cb54779d3
--- /dev/null
+++ b/tests/format/include/conditional/project.conf
@@ -0,0 +1,13 @@
+name: test
+
+options:
+  build_arch:
+    type: arch
+    description: Architecture
+    variable: build_arch
+    values:
+      - i586
+      - x86_64
+
+(@):
+  - extra_conf.yml
diff --git a/tests/format/include/file/element.bst b/tests/format/include/file/element.bst
new file mode 100644
index 000000000..4d7f70266
--- /dev/null
+++ b/tests/format/include/file/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/file/extra_conf.yml b/tests/format/include/file/extra_conf.yml
new file mode 100644
index 000000000..404ecd6dd
--- /dev/null
+++ b/tests/format/include/file/extra_conf.yml
@@ -0,0 +1,2 @@
+variables:
+  included: 'True'
diff --git a/tests/format/include/file/project.conf b/tests/format/include/file/project.conf
new file mode 100644
index 000000000..a7791a416
--- /dev/null
+++ b/tests/format/include/file/project.conf
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+  - extra_conf.yml
diff --git a/tests/format/include/file_with_subproject/element.bst b/tests/format/include/file_with_subproject/element.bst
new file mode 100644
index 000000000..4d7f70266
--- /dev/null
+++ b/tests/format/include/file_with_subproject/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/file_with_subproject/extra_conf.yml b/tests/format/include/file_with_subproject/extra_conf.yml
new file mode 100644
index 000000000..404ecd6dd
--- /dev/null
+++ b/tests/format/include/file_with_subproject/extra_conf.yml
@@ -0,0 +1,2 @@
+variables:
+  included: 'True'
diff --git a/tests/format/include/file_with_subproject/project.bst b/tests/format/include/file_with_subproject/project.bst
new file mode 100644
index 000000000..4836c5f8b
--- /dev/null
+++ b/tests/format/include/file_with_subproject/project.bst
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+  - junction.bst:extra_conf.yml
diff --git a/tests/format/include/file_with_subproject/project.conf b/tests/format/include/file_with_subproject/project.conf
new file mode 100644
index 000000000..a7791a416
--- /dev/null
+++ b/tests/format/include/file_with_subproject/project.conf
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+  - extra_conf.yml
diff --git a/tests/format/include/file_with_subproject/subproject/project.conf b/tests/format/include/file_with_subproject/subproject/project.conf
new file mode 100644
index 000000000..7a6655421
--- /dev/null
+++ b/tests/format/include/file_with_subproject/subproject/project.conf
@@ -0,0 +1 @@
+name: test-sub
diff --git a/tests/format/include/inner/element.bst b/tests/format/include/inner/element.bst
new file mode 100644
index 000000000..4d7f70266
--- /dev/null
+++ b/tests/format/include/inner/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/inner/extra_conf.yml b/tests/format/include/inner/extra_conf.yml
new file mode 100644
index 000000000..4c1847b5f
--- /dev/null
+++ b/tests/format/include/inner/extra_conf.yml
@@ -0,0 +1,7 @@
+build_arch:
+  type: arch
+  description: Architecture
+  variable: build_arch
+  values:
+    - i586
+    - x86_64
diff --git a/tests/format/include/inner/project.conf b/tests/format/include/inner/project.conf
new file mode 100644
index 000000000..8bdfc428a
--- /dev/null
+++ b/tests/format/include/inner/project.conf
@@ -0,0 +1,5 @@
+name: test
+
+options:
+  (@):
+    - extra_conf.yml
diff --git a/tests/format/include/junction/element.bst b/tests/format/include/junction/element.bst
new file mode 100644
index 000000000..4d7f70266
--- /dev/null
+++ b/tests/format/include/junction/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/junction/project.conf b/tests/format/include/junction/project.conf
new file mode 100644
index 000000000..4836c5f8b
--- /dev/null
+++ b/tests/format/include/junction/project.conf
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+  - junction.bst:extra_conf.yml
diff --git a/tests/format/include/junction/subproject/extra_conf.yml b/tests/format/include/junction/subproject/extra_conf.yml
new file mode 100644
index 000000000..404ecd6dd
--- /dev/null
+++ b/tests/format/include/junction/subproject/extra_conf.yml
@@ -0,0 +1,2 @@
+variables:
+  included: 'True'
diff --git a/tests/format/include/junction/subproject/project.conf b/tests/format/include/junction/subproject/project.conf
new file mode 100644
index 000000000..7a6655421
--- /dev/null
+++ b/tests/format/include/junction/subproject/project.conf
@@ -0,0 +1 @@
+name: test-sub
diff --git a/tests/format/include/local_to_junction/element.bst b/tests/format/include/local_to_junction/element.bst
new file mode 100644
index 000000000..4d7f70266
--- /dev/null
+++ b/tests/format/include/local_to_junction/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/local_to_junction/project.conf b/tests/format/include/local_to_junction/project.conf
new file mode 100644
index 000000000..4836c5f8b
--- /dev/null
+++ b/tests/format/include/local_to_junction/project.conf
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+  - junction.bst:extra_conf.yml
diff --git a/tests/format/include/local_to_junction/subproject/extra_conf.yml b/tests/format/include/local_to_junction/subproject/extra_conf.yml
new file mode 100644
index 000000000..1c0b8ccdd
--- /dev/null
+++ b/tests/format/include/local_to_junction/subproject/extra_conf.yml
@@ -0,0 +1,2 @@
+(@):
+  - internal.yml
diff --git a/tests/format/include/local_to_junction/subproject/internal.yml b/tests/format/include/local_to_junction/subproject/internal.yml
new file mode 100644
index 000000000..404ecd6dd
--- /dev/null
+++ b/tests/format/include/local_to_junction/subproject/internal.yml
@@ -0,0 +1,2 @@
+variables:
+  included: 'True'
diff --git a/tests/format/include/local_to_junction/subproject/project.conf b/tests/format/include/local_to_junction/subproject/project.conf
new file mode 100644
index 000000000..7a6655421
--- /dev/null
+++ b/tests/format/include/local_to_junction/subproject/project.conf
@@ -0,0 +1 @@
+name: test-sub
diff --git a/tests/format/include/options/element.bst b/tests/format/include/options/element.bst
new file mode 100644
index 000000000..4d7f70266
--- /dev/null
+++ b/tests/format/include/options/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/options/extra_conf.yml b/tests/format/include/options/extra_conf.yml
new file mode 100644
index 000000000..ad1401e0a
--- /dev/null
+++ b/tests/format/include/options/extra_conf.yml
@@ -0,0 +1,8 @@
+options:
+  build_arch:
+    type: arch
+    description: Architecture
+    variable: build_arch
+    values:
+      - i586
+      - x86_64
diff --git a/tests/format/include/options/project.conf b/tests/format/include/options/project.conf
new file mode 100644
index 000000000..a7791a416
--- /dev/null
+++ b/tests/format/include/options/project.conf
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+  - extra_conf.yml
diff --git a/tests/format/include/overrides-junction/element.bst b/tests/format/include/overrides-junction/element.bst
new file mode 100644
index 000000000..4d7f70266
--- /dev/null
+++ b/tests/format/include/overrides-junction/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/overrides-junction/project.conf b/tests/format/include/overrides-junction/project.conf
new file mode 100644
index 000000000..d03bec634
--- /dev/null
+++ b/tests/format/include/overrides-junction/project.conf
@@ -0,0 +1,20 @@
+name: test
+
+elements:
+  junction:
+    variables:
+      main_override: True
+  manual:
+    variables:
+      manual_main_override: True
+    config:
+      build-commands:
+        - "first"
+
+sources:
+  git:
+    variables:
+      from_main: True
+
+(@):
+  - junction.bst:extra_conf.yml
diff --git a/tests/format/include/overrides-junction/subproject/extra_conf.yml b/tests/format/include/overrides-junction/subproject/extra_conf.yml
new file mode 100644
index 000000000..3cd3530c5
--- /dev/null
+++ b/tests/format/include/overrides-junction/subproject/extra_conf.yml
@@ -0,0 +1,16 @@
+elements:
+  junction:
+    variables:
+      included_override: True
+  manual:
+    variables:
+      manual_included_override: True
+    config:
+      build-commands:
+        (>):
+          - "second"
+
+sources:
+  git:
+    variables:
+      from_included: True
diff --git a/tests/format/include/overrides-junction/subproject/project.conf b/tests/format/include/overrides-junction/subproject/project.conf
new file mode 100644
index 000000000..7a6655421
--- /dev/null
+++ b/tests/format/include/overrides-junction/subproject/project.conf
@@ -0,0 +1 @@
+name: test-sub
diff --git a/tests/format/include/overrides/element.bst b/tests/format/include/overrides/element.bst
new file mode 100644
index 000000000..4d7f70266
--- /dev/null
+++ b/tests/format/include/overrides/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/overrides/extra_conf.yml b/tests/format/include/overrides/extra_conf.yml
new file mode 100644
index 000000000..ccb874bd7
--- /dev/null
+++ b/tests/format/include/overrides/extra_conf.yml
@@ -0,0 +1,15 @@
+elements:
+  junction:
+    variables:
+      included_override: True
+  manual:
+    variables:
+      manual_included_override: True
+    config:
+      build-commands:
+        - "ignored"
+
+sources:
+  git:
+    variables:
+      from_included: True
diff --git a/tests/format/include/overrides/extra_conf2.yml b/tests/format/include/overrides/extra_conf2.yml
new file mode 100644
index 000000000..750abd725
--- /dev/null
+++ b/tests/format/include/overrides/extra_conf2.yml
@@ -0,0 +1,5 @@
+elements:
+  manual:
+    config:
+      build-commands:
+        - "first"
diff --git a/tests/format/include/overrides/project.conf b/tests/format/include/overrides/project.conf
new file mode 100644
index 000000000..fa3c75703
--- /dev/null
+++ b/tests/format/include/overrides/project.conf
@@ -0,0 +1,22 @@
+name: test
+
+elements:
+  junction:
+    variables:
+      main_override: True
+  manual:
+    variables:
+      manual_main_override: True
+    config:
+      build-commands:
+        (>):
+          - "second"
+
+sources:
+  git:
+    variables:
+      from_main: True
+
+(@):
+  - extra_conf.yml
+  - extra_conf2.yml
diff --git a/tests/format/include/overrides/subproject/project.conf b/tests/format/include/overrides/subproject/project.conf
new file mode 100644
index 000000000..7a6655421
--- /dev/null
+++ b/tests/format/include/overrides/subproject/project.conf
@@ -0,0 +1 @@
+name: test-sub
diff --git a/tests/format/include/recursive/element.bst b/tests/format/include/recursive/element.bst
new file mode 100644
index 000000000..4d7f70266
--- /dev/null
+++ b/tests/format/include/recursive/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/recursive/extra_conf.yml b/tests/format/include/recursive/extra_conf.yml
new file mode 100644
index 000000000..57db0d3c6
--- /dev/null
+++ b/tests/format/include/recursive/extra_conf.yml
@@ -0,0 +1,2 @@
+(@):
+  - extra_conf2.yml
diff --git a/tests/format/include/recursive/extra_conf2.yml b/tests/format/include/recursive/extra_conf2.yml
new file mode 100644
index 000000000..e8dd5e2ed
--- /dev/null
+++ b/tests/format/include/recursive/extra_conf2.yml
@@ -0,0 +1,2 @@
+(@):
+  - extra_conf.yml
diff --git a/tests/format/include/recursive/project.conf b/tests/format/include/recursive/project.conf
new file mode 100644
index 000000000..a7791a416
--- /dev/null
+++ b/tests/format/include/recursive/project.conf
@@ -0,0 +1,4 @@
+name: test
+
+(@):
+  - extra_conf.yml
diff --git a/tests/format/include/string/element.bst b/tests/format/include/string/element.bst
new file mode 100644
index 000000000..4d7f70266
--- /dev/null
+++ b/tests/format/include/string/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/string/extra_conf.yml b/tests/format/include/string/extra_conf.yml
new file mode 100644
index 000000000..404ecd6dd
--- /dev/null
+++ b/tests/format/include/string/extra_conf.yml
@@ -0,0 +1,2 @@
+variables:
+  included: 'True'
diff --git a/tests/format/include/string/project.conf b/tests/format/include/string/project.conf
new file mode 100644
index 000000000..6ee9988e9
--- /dev/null
+++ b/tests/format/include/string/project.conf
@@ -0,0 +1,3 @@
+name: test
+
+(@): extra_conf.yml
diff --git a/tests/format/include/sub-include/element.bst b/tests/format/include/sub-include/element.bst
new file mode 100644
index 000000000..4d7f70266
--- /dev/null
+++ b/tests/format/include/sub-include/element.bst
@@ -0,0 +1 @@
+kind: manual
diff --git a/tests/format/include/sub-include/manual_conf.yml b/tests/format/include/sub-include/manual_conf.yml
new file mode 100644
index 000000000..9c2c0dd34
--- /dev/null
+++ b/tests/format/include/sub-include/manual_conf.yml
@@ -0,0 +1,2 @@
+variables:
+  included: True
diff --git a/tests/format/include/sub-include/project.conf b/tests/format/include/sub-include/project.conf
new file mode 100644
index 000000000..7f7df84c8
--- /dev/null
+++ b/tests/format/include/sub-include/project.conf
@@ -0,0 +1,6 @@
+name: test
+
+elements:
+  manual:
+    (@):
+      - manual_conf.yml
diff --git a/tests/format/include_composition.py b/tests/format/include_composition.py
new file mode 100644
index 000000000..b73fca392
--- /dev/null
+++ b/tests/format/include_composition.py
@@ -0,0 +1,131 @@
+import os
+from buildstream._context import Context
+from buildstream._project import Project
+from buildstream._includes import Includes
+from buildstream import _yaml
+
+
+def make_includes(basedir):
+    _yaml.dump({'name': 'test'},
+               os.path.join(basedir, 'project.conf'))
+    context = Context()
+    project = Project(basedir, context)
+    loader = project.loader
+    return Includes(loader)
+
+
+def test_main_has_prority(tmpdir):
+    includes = make_includes(str(tmpdir))
+
+    _yaml.dump({'(@)': ['a.yml'],
+                'test': ['main']},
+               str(tmpdir.join('main.yml')))
+
+    main = _yaml.load(str(tmpdir.join('main.yml')))
+
+    _yaml.dump({'test': ['a']},
+               str(tmpdir.join('a.yml')))
+
+    includes.process(main)
+
+    assert main['test'] == ['main']
+
+
+def test_include_cannot_append(tmpdir):
+    includes = make_includes(str(tmpdir))
+
+    _yaml.dump({'(@)': ['a.yml'],
+                'test': ['main']},
+               str(tmpdir.join('main.yml')))
+    main = _yaml.load(str(tmpdir.join('main.yml')))
+
+    _yaml.dump({'test': {'(>)': ['a']}},
+               str(tmpdir.join('a.yml')))
+
+    includes.process(main)
+
+    assert main['test'] == ['main']
+
+
+def test_main_can_append(tmpdir):
+    includes = make_includes(str(tmpdir))
+
+    _yaml.dump({'(@)': ['a.yml'],
+                'test': {'(>)': ['main']}},
+               str(tmpdir.join('main.yml')))
+    main = _yaml.load(str(tmpdir.join('main.yml')))
+
+    _yaml.dump({'test': ['a']},
+               str(tmpdir.join('a.yml')))
+
+    includes.process(main)
+
+    assert main['test'] == ['a', 'main']
+
+
+def test_sibling_cannot_append_backward(tmpdir):
+    includes = make_includes(str(tmpdir))
+
+    _yaml.dump({'(@)': ['a.yml', 'b.yml']},
+               str(tmpdir.join('main.yml')))
+    main = _yaml.load(str(tmpdir.join('main.yml')))
+
+    _yaml.dump({'test': {'(>)': ['a']}},
+               str(tmpdir.join('a.yml')))
+    _yaml.dump({'test': ['b']},
+               str(tmpdir.join('b.yml')))
+
+    includes.process(main)
+
+    assert main['test'] == ['b']
+
+
+def test_sibling_can_append_forward(tmpdir):
+    includes = make_includes(str(tmpdir))
+
+    _yaml.dump({'(@)': ['a.yml', 'b.yml']},
+               str(tmpdir.join('main.yml')))
+    main = _yaml.load(str(tmpdir.join('main.yml')))
+
+    _yaml.dump({'test': ['a']},
+               str(tmpdir.join('a.yml')))
+    _yaml.dump({'test': {'(>)': ['b']}},
+               str(tmpdir.join('b.yml')))
+
+    includes.process(main)
+
+    assert main['test'] == ['a', 'b']
+
+
+def test_lastest_sibling_has_priority(tmpdir):
+    includes = make_includes(str(tmpdir))
+
+    _yaml.dump({'(@)': ['a.yml', 'b.yml']},
+               str(tmpdir.join('main.yml')))
+    main = _yaml.load(str(tmpdir.join('main.yml')))
+
+    _yaml.dump({'test': ['a']},
+               str(tmpdir.join('a.yml')))
+    _yaml.dump({'test': ['b']},
+               str(tmpdir.join('b.yml')))
+
+    includes.process(main)
+
+    assert main['test'] == ['b']
+
+
+def test_main_keeps_keys(tmpdir):
+    includes = make_includes(str(tmpdir))
+
+    _yaml.dump({'(@)': ['a.yml'],
+                'something': 'else'},
+               str(tmpdir.join('main.yml')))
+    main = _yaml.load(str(tmpdir.join('main.yml')))
+
+    _yaml.dump({'test': ['a']},
+               str(tmpdir.join('a.yml')))
+
+    includes.process(main)
+
+    assert main['test'] == ['a']
+    assert main['something'] == 'else'
diff --git a/tests/frontend/mirror.py b/tests/frontend/mirror.py
index 62c796ab8..f37cc18af 100644
--- a/tests/frontend/mirror.py
+++ b/tests/frontend/mirror.py
@@ -1,10 +1,10 @@
 import os
 import pytest
 
-from tests.testutils import cli, create_repo, ALL_REPO_KINDS
+from tests.testutils import cli, create_repo, ALL_REPO_KINDS, generate_junction
 
 from buildstream import _yaml
-
+from buildstream._exceptions import ErrorDomain
 
 # Project directory
 TOP_DIR = os.path.dirname(os.path.realpath(__file__))
@@ -400,3 +400,162 @@ def test_mirror_track_upstream_absent(cli, tmpdir, datafiles, kind):
     source = new_element['sources'][0]
     if 'ref' in source:
         assert source['ref'] == mirror_ref
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+def test_mirror_from_includes(cli, tmpdir, datafiles, kind):
+    if kind == 'git':
+        # FIXME: Mirroring fallback does not work with git because it tries to
+        # fetch submodules on upstream.
+        pytest.skip("Bug #537 - Mirror fallback does not work for git")
+    if kind == 'ostree':
+        # FIXME: Mirroring fallback fails with ostree
+        pytest.skip("Bug #538 - ostree mirror fallback breaks assertion")
+
+    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
+    upstream_repodir = os.path.join(str(tmpdir), 'upstream')
+    mirror_repodir = os.path.join(str(tmpdir), 'mirror')
+    project_dir = os.path.join(str(tmpdir), 'project')
+    os.makedirs(project_dir)
+    element_dir = os.path.join(project_dir, 'elements')
+
+    # Create repo objects of the upstream and mirror
+    upstream_repo = create_repo(kind, upstream_repodir)
+    upstream_ref = upstream_repo.create(bin_files_path)
+    mirror_repo = upstream_repo.copy(mirror_repodir)
+
+    element = {
+        'kind': 'import',
+        'sources': [
+            upstream_repo.source_config(ref=upstream_ref)
+        ]
+    }
+    element_name = 'test.bst'
+    element_path = os.path.join(element_dir, element_name)
+    full_repo = element['sources'][0]['url']
+    upstream_map, repo_name = os.path.split(full_repo)
+    alias = 'foo-' + kind
+    aliased_repo = alias + ':' + repo_name
+    element['sources'][0]['url'] = aliased_repo
+    full_mirror = mirror_repo.source_config()['url']
+    mirror_map, _ = os.path.split(full_mirror)
+    os.makedirs(element_dir)
+    _yaml.dump(element, element_path)
+
+    config_project_dir = str(tmpdir.join('config'))
+    os.makedirs(config_project_dir, exist_ok=True)
+    config_project = {
+        'name': 'config'
+    }
+    _yaml.dump(config_project, os.path.join(config_project_dir, 'project.conf'))
+    extra_mirrors = {
+        'mirrors': [
+            {
+                'name': 'middle-earth',
+                'aliases': {
+                    alias: [mirror_map + "/"],
+                }
+            }
+        ]
+    }
+    _yaml.dump(extra_mirrors, os.path.join(config_project_dir, 'mirrors.yml'))
+    generate_junction(str(tmpdir.join('config_repo')),
+                      config_project_dir,
+                      os.path.join(element_dir, 'config.bst'))
+
+    project = {
+        'name': 'test',
+        'element-path': 'elements',
+        'aliases': {
+            alias: upstream_map + "/"
+        },
+        '(@)': [
+            'config.bst:mirrors.yml'
+        ]
+    }
+    project_file = os.path.join(project_dir, 'project.conf')
+    _yaml.dump(project, project_file)
+
+    # Now make the upstream unavailable.
+    os.rename(upstream_repo.repo, '{}.bak'.format(upstream_repo.repo))
+    result = cli.run(project=project_dir, args=['fetch', element_name])
+    result.assert_success()
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+def test_mirror_junction_from_includes(cli, tmpdir, datafiles, kind):
+    bin_files_path = os.path.join(str(datafiles), 'files', 'bin-files', 'usr')
+    dev_files_path = os.path.join(str(datafiles), 'files', 'dev-files', 'usr')
+    upstream_repodir = os.path.join(str(tmpdir), 'upstream')
+    mirror_repodir = os.path.join(str(tmpdir), 'mirror')
+    project_dir = os.path.join(str(tmpdir), 'project')
+    os.makedirs(project_dir)
+    element_dir = os.path.join(project_dir, 'elements')
+
+    # Create repo objects of the upstream and mirror
+    upstream_repo = create_repo(kind, upstream_repodir)
+    upstream_ref = upstream_repo.create(bin_files_path)
+    mirror_repo = upstream_repo.copy(mirror_repodir)
+
+    element = {
+        'kind': 'junction',
+        'sources': [
+            upstream_repo.source_config(ref=upstream_ref)
+        ]
+    }
+    element_name = 'test.bst'
+    element_path = os.path.join(element_dir, element_name)
+    full_repo = element['sources'][0]['url']
+    upstream_map, repo_name = os.path.split(full_repo)
+    alias = 'foo-' + kind
+    aliased_repo = alias + ':' + repo_name
+    element['sources'][0]['url'] = aliased_repo
+    full_mirror = mirror_repo.source_config()['url']
+    mirror_map, _ = os.path.split(full_mirror)
+    os.makedirs(element_dir)
+    _yaml.dump(element, element_path)
+
+    config_project_dir = str(tmpdir.join('config'))
+    os.makedirs(config_project_dir, exist_ok=True)
+    config_project = {
+        'name': 'config'
+    }
+    _yaml.dump(config_project, os.path.join(config_project_dir, 'project.conf'))
+    extra_mirrors = {
+        'mirrors': [
+            {
+                'name': 'middle-earth',
+                'aliases': {
+                    alias: [mirror_map + "/"],
+                }
+            }
+        ]
+    }
+    _yaml.dump(extra_mirrors, os.path.join(config_project_dir, 'mirrors.yml'))
+    generate_junction(str(tmpdir.join('config_repo')),
+                      config_project_dir,
+                      os.path.join(element_dir, 'config.bst'))
+
+    project = {
+        'name': 'test',
+        'element-path': 'elements',
+        'aliases': {
+            alias: upstream_map + "/"
+        },
+        '(@)': [
+            'config.bst:mirrors.yml'
+        ]
+    }
+    project_file = os.path.join(project_dir, 'project.conf')
+    _yaml.dump(project, project_file)
+
+    # Now make the upstream unavailable.
+    os.rename(upstream_repo.repo, '{}.bak'.format(upstream_repo.repo))
+    result = cli.run(project=project_dir, args=['fetch', element_name])
+    result.assert_main_error(ErrorDomain.STREAM, None)
+    # Now make the upstream available again.
+    os.rename('{}.bak'.format(upstream_repo.repo), upstream_repo.repo)
+    result = cli.run(project=project_dir, args=['fetch', element_name])
+    result.assert_success()
diff --git a/tests/frontend/track.py b/tests/frontend/track.py
index 4e1059824..1cf962f88 100644
--- a/tests/frontend/track.py
+++ b/tests/frontend/track.py
@@ -480,3 +480,135 @@ def test_cross_junction(cli, tmpdir, datafiles, ref_storage, kind):
         assert cli.get_element_state(project, 'junction.bst:import-etc-repo.bst') == 'buildable'
 
         assert os.path.exists(os.path.join(project, 'project.refs'))
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
+@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+def test_track_include(cli, tmpdir, datafiles, ref_storage, kind):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    dev_files_path = os.path.join(project, 'files', 'dev-files')
+    element_path = os.path.join(project, 'elements')
+    element_name = 'track-test-{}.bst'.format(kind)
+
+    configure_project(project, {
+        'ref-storage': ref_storage
+    })
+
+    # Create our repo object of the given source type with
+    # the dev files, and then collect the initial ref.
+    #
+    repo = create_repo(kind, str(tmpdir))
+    ref = repo.create(dev_files_path)
+
+    # Generate the element
+    element = {
+        'kind': 'import',
+        '(@)': ['elements/sources.yml']
+    }
+    sources = {
+        'sources': [
+            repo.source_config()
+        ]
+    }
+
+    _yaml.dump(element, os.path.join(element_path, element_name))
+    _yaml.dump(sources, os.path.join(element_path, 'sources.yml'))
+
+    # Assert that a fetch is needed
+    assert cli.get_element_state(project, element_name) == 'no reference'
+
+    # Now first try to track it
+    result = cli.run(project=project, args=['track', element_name])
+    result.assert_success()
+
+    # And now fetch it: The Source has probably already cached the
+    # latest ref locally, but it is not required to have cached
+    # the associated content of the latest ref at track time, that
+    # is the job of fetch.
+    result = cli.run(project=project, args=['fetch', element_name])
+    result.assert_success()
+
+    # Assert that we are now buildable because the source is
+    # now cached.
+    assert cli.get_element_state(project, element_name) == 'buildable'
+
+    # Assert there was a project.refs created, depending on the configuration
+    if ref_storage == 'project.refs':
+        assert os.path.exists(os.path.join(project, 'project.refs'))
+    else:
+        assert not os.path.exists(os.path.join(project, 'project.refs'))
+        new_sources = _yaml.load(os.path.join(element_path, 'sources.yml'))
+        assert 'sources' in new_sources
+        assert len(new_sources['sources']) == 1
+        assert 'ref' in new_sources['sources'][0]
+        assert ref == new_sources['sources'][0]['ref']
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
+@pytest.mark.parametrize("kind", [(kind) for kind in ALL_REPO_KINDS])
+def test_track_include_junction(cli, tmpdir, datafiles, ref_storage, kind):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    dev_files_path = os.path.join(project, 'files', 'dev-files')
+    element_path = os.path.join(project, 'elements')
+    element_name = 'track-test-{}.bst'.format(kind)
+    subproject_path = os.path.join(project, 'files', 'sub-project')
+    sub_element_path = os.path.join(subproject_path, 'elements')
+    junction_path = os.path.join(element_path, 'junction.bst')
+
+    configure_project(project, {
+        'ref-storage': ref_storage
+    })
+
+    # Create our repo object of the given source type with
+    # the dev files, and then collect the initial ref.
+    #
+    repo = create_repo(kind, str(tmpdir.join('element_repo')))
+    ref = repo.create(dev_files_path)
+
+    # Generate the element
+    element = {
+        'kind': 'import',
+        '(@)': ['junction.bst:elements/sources.yml']
+    }
+    sources = {
+        'sources': [
+            repo.source_config()
+        ]
+    }
+
+    _yaml.dump(element, os.path.join(element_path, element_name))
+    _yaml.dump(sources, os.path.join(sub_element_path, 'sources.yml'))
+
+    generate_junction(str(tmpdir.join('junction_repo')),
+                      subproject_path, junction_path, store_ref=True)
+
+    result = cli.run(project=project, args=['track', 'junction.bst'])
+    result.assert_success()
+
+    # Assert that a fetch is needed
+    assert cli.get_element_state(project, element_name) == 'no reference'
+
+    # Now first try to track it
+    result = cli.run(project=project, args=['track', element_name])
+
+    # Assert there was a project.refs created, depending on the configuration
+    if ref_storage == 'inline':
+        # FIXME: We should expect an error. But only a warning is emitted
+        # result.assert_main_error(ErrorDomain.SOURCE, 'tracking-junction-fragment')
+
+        assert 'junction.bst:elements/sources.yml: Cannot track source in a fragment from a junction' in result.stderr
+    else:
+        assert os.path.exists(os.path.join(project, 'project.refs'))
+
+        # And now fetch it: The Source has probably already cached the
+        # latest ref locally, but it is not required to have cached
+        # the associated content of the latest ref at track time, that
+        # is the job of fetch.
+        result = cli.run(project=project, args=['fetch', element_name])
+        result.assert_success()
+
+        # Assert that we are now buildable because the source is
+        # now cached.
+        assert cli.get_element_state(project, element_name) == 'buildable'
diff --git a/tests/yaml/yaml.py b/tests/yaml/yaml.py
index 3b9f385ed..781763717 100644
--- a/tests/yaml/yaml.py
+++ b/tests/yaml/yaml.py
@@ -33,7 +33,7 @@ def assert_provenance(filename, line, col, node, key=None, indices=[]):
     else:
         assert(isinstance(provenance, _yaml.DictProvenance))
 
-    assert(provenance.filename == filename)
+    assert(provenance.filename.shortname == filename)
     assert(provenance.line == line)
     assert(provenance.col == col)