Fix teststracking-changes

8 files changed, 357 insertions, 2 deletions

diff --git a/tests/completions/completions.py b/tests/completions/completions.py
index d2c631105..8ecf83e4b 100644
--- a/tests/completions/completions.py
+++ b/tests/completions/completions.py
@@ -97,7 +97,8 @@ def test_commands(cli, cmd, word_idx, expected):
     ('bst --l', 1, ['--log-file ']),
 
     # Test that options of subcommands also complete
-    ('bst --no-colors build -', 3, ['--all ', '--track ']),
+    ('bst --no-colors build -', 3, ['--all ', '--track ', '--track-all ',
+                                    '--track-except ', '--track-save ']),
 
     # Test the behavior of completing after an option that has a
     # parameter that cannot be completed, vs an option that has
diff --git a/tests/frontend/buildtrack.py b/tests/frontend/buildtrack.py
new file mode 100644
index 000000000..e91274a29
--- /dev/null
+++ b/tests/frontend/buildtrack.py
@@ -0,0 +1,312 @@
+import os
+import re
+import shutil
+import itertools
+
+import pytest
+from tests.testutils import cli, create_repo
+
+from buildstream import _yaml
+from buildstream._exceptions import LoadError
+
+
+# Project directory
+DATA_DIR = os.path.join(
+    os.path.dirname(os.path.realpath(__file__)),
+    "project",
+)
+
+
+def create_element(repo, name, path, dependencies, ref=None):
+    element = {
+        'kind': 'import',
+        'sources': [
+            repo.source_config(ref=ref)
+        ],
+        'depends': dependencies
+    }
+    _yaml.dump(element, os.path.join(path, name))
+
+
+@pytest.mark.parametrize("save", [([True]), ([False])])
+@pytest.mark.datafiles(os.path.join(DATA_DIR))
+@pytest.mark.parametrize("exceptions,excepted", [
+    # Test with no exceptions
+    ([], []),
+
+    # Test excepting '2.bst'
+    (['2.bst'], ['2.bst', '7.bst']),
+
+    # Test excepting '2.bst' and '3.bst'
+    (['2.bst', '3.bst'], [
+        '2.bst', '3.bst', '4.bst',
+        '5.bst', '6.bst', '7.bst'
+    ])
+])
+@pytest.mark.parametrize("track_targets,tracked", [
+    # Test tracking the main target element
+    (['0.bst'], [
+        '0.bst', '2.bst', '3.bst',
+        '4.bst', '5.bst', '6.bst', '7.bst'
+    ]),
+
+    # Test tracking a child element
+    (['3.bst'], [
+        '3.bst', '4.bst', '5.bst',
+        '6.bst'
+    ]),
+
+    # Test tracking multiple children
+    (['2.bst', '3.bst'], [
+        '2.bst', '3.bst', '4.bst',
+        '5.bst', '6.bst', '7.bst'
+    ])
+])
+def test_build_track(cli, datafiles, tmpdir, track_targets,
+                     exceptions, tracked, excepted, save):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    dev_files_path = os.path.join(project, 'files', 'dev-files')
+    element_path = os.path.join(project, 'elements')
+
+    repo = create_repo('git', str(tmpdir))
+    ref = repo.create(dev_files_path)
+
+    create_elements = {
+        '0.bst': [
+            '2.bst',
+            '3.bst'
+        ],
+        '2.bst': [
+            '3.bst',
+            '7.bst'
+        ],
+        '3.bst': [
+            '4.bst',
+            '5.bst',
+            '6.bst'
+        ],
+        '4.bst': [],
+        '5.bst': [],
+        '6.bst': [
+            '5.bst'
+        ],
+        '7.bst': []
+    }
+    for element, dependencies in create_elements.items():
+        # Test the element inconsistency resolution by ensuring that
+        # only elements that aren't tracked have refs
+        if element in set(tracked) - set(excepted):
+            create_element(repo, element, element_path, dependencies)
+        else:
+            create_element(repo, element, element_path, dependencies, ref=ref)
+
+    args = ['build']
+    if save:
+        args += ['--track-save']
+    args += itertools.chain.from_iterable(zip(itertools.repeat('--track'), track_targets))
+    args += itertools.chain.from_iterable(zip(itertools.repeat('--track-except'), exceptions))
+    args += ['0.bst']
+
+    result = cli.run(project=project, silent=True, args=args)
+    tracked_elements = result.get_tracked_elements()
+
+    assert set(tracked_elements) == set(tracked) - set(excepted)
+
+    for target in set(tracked) - set(excepted):
+        cli.remove_artifact_from_cache(project, target)
+
+        # Delete element sources
+        source_dir = os.path.join(project, 'cache', 'sources')
+        shutil.rmtree(source_dir)
+
+        if not save:
+            assert cli.get_element_state(project, target) == 'no reference'
+        else:
+            assert cli.get_element_state(project, target) == 'fetch needed'
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR))
+@pytest.mark.parametrize("exceptions,excepted", [
+    # Test with no exceptions
+    ([], []),
+
+    # Test excepting '2.bst'
+    (['2.bst'], ['2.bst', '7.bst']),
+
+    # Test excepting '2.bst' and '3.bst'
+    (['2.bst', '3.bst'], [
+        '2.bst', '3.bst', '4.bst',
+        '5.bst', '6.bst', '7.bst'
+    ])
+])
+@pytest.mark.parametrize("track_targets,tracked", [
+    # Test tracking the main target element
+    (['0.bst'], [
+        '0.bst', '2.bst', '3.bst',
+        '4.bst', '5.bst', '6.bst', '7.bst'
+    ]),
+
+    # Test tracking a child element
+    (['3.bst'], [
+        '3.bst', '4.bst', '5.bst',
+        '6.bst'
+    ]),
+
+    # Test tracking multiple children
+    (['2.bst', '3.bst'], [
+        '2.bst', '3.bst', '4.bst',
+        '5.bst', '6.bst', '7.bst'
+    ])
+])
+def test_build_track_update(cli, datafiles, tmpdir, track_targets,
+                            exceptions, tracked, excepted):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    dev_files_path = os.path.join(project, 'files', 'dev-files')
+    element_path = os.path.join(project, 'elements')
+
+    repo = create_repo('git', str(tmpdir))
+    ref = repo.create(dev_files_path)
+
+    create_elements = {
+        '0.bst': [
+            '2.bst',
+            '3.bst'
+        ],
+        '2.bst': [
+            '3.bst',
+            '7.bst'
+        ],
+        '3.bst': [
+            '4.bst',
+            '5.bst',
+            '6.bst'
+        ],
+        '4.bst': [],
+        '5.bst': [],
+        '6.bst': [
+            '5.bst'
+        ],
+        '7.bst': []
+    }
+    for element, dependencies in create_elements.items():
+        # We set a ref for all elements, so that we ensure that we
+        # only track exactly those elements that we want to track,
+        # even if others can be tracked
+        create_element(repo, element, element_path, dependencies, ref=ref)
+        repo.add_commit()
+
+    args = ['build']
+    args += itertools.chain.from_iterable(zip(itertools.repeat('--track'), track_targets))
+    args += itertools.chain.from_iterable(zip(itertools.repeat('--track-except'), exceptions))
+    args += ['0.bst']
+
+    result = cli.run(project=project, silent=True, args=args)
+    tracked_elements = result.get_tracked_elements()
+
+    assert set(tracked_elements) == set(tracked) - set(excepted)
+
+
+@pytest.mark.datafiles(os.path.join(DATA_DIR))
+@pytest.mark.parametrize("track_targets,exceptions", [
+    # Test tracking the main target element, but excepting some of its
+    # children
+    (['0.bst'], ['6.bst']),
+
+    # Test only tracking a child element
+    (['3.bst'], []),
+])
+def test_build_track_inconsistent(cli, datafiles, tmpdir,
+                                  track_targets, exceptions):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    dev_files_path = os.path.join(project, 'files', 'dev-files')
+    element_path = os.path.join(project, 'elements')
+
+    repo = create_repo('git', str(tmpdir))
+    repo.create(dev_files_path)
+
+    create_elements = {
+        '0.bst': [
+            '2.bst',
+            '3.bst'
+        ],
+        '2.bst': [
+            '3.bst',
+            '7.bst'
+        ],
+        '3.bst': [
+            '4.bst',
+            '5.bst',
+            '6.bst'
+        ],
+        '4.bst': [],
+        '5.bst': [],
+        '6.bst': [
+            '5.bst'
+        ],
+        '7.bst': []
+    }
+    for element, dependencies in create_elements.items():
+        # We don't add refs so that all elements *have* to be tracked
+        create_element(repo, element, element_path, dependencies)
+
+    args = ['build']
+    args += itertools.chain.from_iterable(zip(itertools.repeat('--track'), track_targets))
+    args += itertools.chain.from_iterable(zip(itertools.repeat('--track-except'), exceptions))
+    args += ['0.bst']
+
+    result = cli.run(args=args, silent=True)
+
+    assert result.exit_code != 0
+    assert isinstance(result.exception, LoadError)
+
+
+# Assert that if a build element has a dependency in the tracking
+# queue it does not start building before tracking finishes.
+@pytest.mark.datafiles(os.path.join(DATA_DIR))
+@pytest.mark.parametrize("strict", ['--strict', '--no-strict'])
+def test_build_track_track_first(cli, datafiles, tmpdir, strict):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    dev_files_path = os.path.join(project, 'files', 'dev-files')
+    element_path = os.path.join(project, 'elements')
+
+    repo = create_repo('git', str(tmpdir))
+    ref = repo.create(dev_files_path)
+
+    create_elements = {
+        '0.bst': [
+            '1.bst'
+        ],
+        '1.bst': [],
+        '2.bst': [
+            '0.bst'
+        ]
+    }
+    for element, dependencies in create_elements.items():
+        # We set a ref so that 0.bst can already be built even if
+        # 1.bst has not been tracked yet.
+        create_element(repo, element, element_path, dependencies, ref=ref)
+        repo.add_commit()
+
+    # Build 1.bst and 2.bst first so we have an artifact for them
+    args = [strict, 'build', '2.bst']
+    result = cli.run(args=args, project=project, silent=True)
+    assert result.exit_code == 0
+
+    # Test building 0.bst while tracking 1.bst
+    cli.remove_artifact_from_cache(project, '0.bst')
+
+    args = [strict, 'build', '--track', '1.bst', '2.bst']
+    result = cli.run(args=args, project=project, silent=True)
+    assert result.exit_code == 0
+
+    # Assert that 1.bst successfully tracks before 0.bst builds
+    track_messages = re.finditer(r'\[track:1.bst\s*]', result.output)
+    build_0 = re.search(r'\[build:0.bst\s*] START', result.output).start()
+    assert all(track_message.start() < build_0 for track_message in track_messages)
+
+    # Assert that 2.bst is *only* rebuilt if we are in strict mode
+    build_2 = re.search(r'\[build:2.bst\s*] START', result.output)
+    if strict == '--strict':
+        assert build_2 is not None
+    else:
+        assert build_2 is None
diff --git a/tests/pipeline/load.py b/tests/pipeline/load.py
index 80a366a6b..fa70fad4c 100644
--- a/tests/pipeline/load.py
+++ b/tests/pipeline/load.py
@@ -126,6 +126,8 @@ def test_iterate_no_recurse(cli, datafiles, tmpdir):
         'third-level-1.bst',
         'fourth-level-2.bst',
         'third-level-2.bst',
+        'fourth-level-3.bst',
+        'third-level-3.bst',
         'second-level-1.bst',
         'first-level-1.bst',
         'first-level-2.bst',
@@ -137,6 +139,8 @@ def test_iterate_no_recurse(cli, datafiles, tmpdir):
     (['build.bst'], ['third-level-1.bst'], [
         'fourth-level-2.bst',
         'third-level-2.bst',
+        'fourth-level-3.bst',
+        'third-level-3.bst',
         'second-level-1.bst',
         'first-level-1.bst',
         'first-level-2.bst',
diff --git a/tests/pipeline/load/exceptions/fourth-level-3.bst b/tests/pipeline/load/exceptions/fourth-level-3.bst
new file mode 100644
index 000000000..6812e2fa1
--- /dev/null
+++ b/tests/pipeline/load/exceptions/fourth-level-3.bst
@@ -0,0 +1,2 @@
+kind: autotools
+description: Should not be removed
diff --git a/tests/pipeline/load/exceptions/second-level-1.bst b/tests/pipeline/load/exceptions/second-level-1.bst
index bd45ef344..3183e670b 100644
--- a/tests/pipeline/load/exceptions/second-level-1.bst
+++ b/tests/pipeline/load/exceptions/second-level-1.bst
@@ -1,5 +1,6 @@
 kind: autotools
-description: Depends uniquely on one dependency, shares another
+description: Depends uniquely on one dependency, shares another, has another unique nested dependency
 depends:
   - third-level-1.bst
   - third-level-2.bst
+  - third-level-3.bst
diff --git a/tests/pipeline/load/exceptions/third-level-3.bst b/tests/pipeline/load/exceptions/third-level-3.bst
new file mode 100644
index 000000000..56fbe5586
--- /dev/null
+++ b/tests/pipeline/load/exceptions/third-level-3.bst
@@ -0,0 +1,4 @@
+kind: autotools
+description: Should be an explicit dependency, and *not* remove its children
+depends:
+  - fourth-level-3.bst
diff --git a/tests/testutils/repo/git.py b/tests/testutils/repo/git.py
index 46338f445..57a441a55 100644
--- a/tests/testutils/repo/git.py
+++ b/tests/testutils/repo/git.py
@@ -32,6 +32,11 @@ class Git(Repo):
         subprocess.call(['git', 'commit', '-m', 'Initial commit'], env=GIT_ENV, cwd=self.repo)
         return self.latest_commit()
 
+    def add_commit(self):
+        subprocess.call(['git', 'commit', '--allow-empty', '-m', 'Additional commit'],
+                        env=GIT_ENV, cwd=self.repo)
+        return self.latest_commit()
+
     def add_submodule(self, subdir, url):
         self.submodules[subdir] = url
         subprocess.call(['git', 'submodule', 'add', url, subdir], env=GIT_ENV, cwd=self.repo)
diff --git a/tests/testutils/runcli.py b/tests/testutils/runcli.py
index f165232dc..202766fb3 100644
--- a/tests/testutils/runcli.py
+++ b/tests/testutils/runcli.py
@@ -1,5 +1,7 @@
 import os
+import re
 import sys
+import shutil
 import itertools
 import traceback
 from contextlib import contextmanager, ExitStack
@@ -7,6 +9,8 @@ from click.testing import CliRunner
 from ruamel import yaml
 import pytest
 
+from tests.testutils.site import IS_LINUX
+
 # Import the main cli entrypoint
 from buildstream._frontend.main import cli as bst_cli
 from buildstream import _yaml
@@ -24,6 +28,17 @@ class Result():
         self.exception = _get_last_exception()
         self.result = result
 
+    ##################################################################
+    #                         Result parsers                         #
+    ##################################################################
+    def get_tracked_elements(self):
+        tracked = re.findall(r'\[track:(\S+)\s*]',
+                             self.result.output)
+        if tracked is None:
+            return []
+
+        return list(tracked)
+
 
 class Cli():
 
@@ -44,6 +59,17 @@ class Cli():
     def configure(self, config):
         self.config = config
 
+    def remove_artifact_from_cache(self, project, element_name):
+        cache_dir = os.path.join(project, 'cache', 'artifacts')
+
+        if IS_LINUX:
+            cache_dir = os.path.join(cache_dir, 'ostree', 'refs', 'heads')
+        else:
+            cache_dir = os.path.join(cache_dir, 'tar')
+
+        cache_dir = os.path.splitext(os.path.join(cache_dir, 'test', element_name))[0]
+        shutil.rmtree(cache_dir)
+
     # run():
     #
     # Runs buildstream with the given arguments, additionally