From 70183d8274898701f4102a1581262807ca89f9ee Mon Sep 17 00:00:00 2001 From: Tristan Van Berkom Date: Thu, 9 May 2019 17:15:55 +0900 Subject: element.py: Update state on reverse dependencies when strict key is resolved. When calculating the strict key, we do not know for sure yet that the element is required or whether it will ever have a cache key, as we could be tracking elements which will result in resolving an element which is a build-only dependency of the target. This ensures that we process all elements which need to be processed, which is important for cases where we are building in non-strict mode and tracking is also enabled. This fixes issue #1014 --- buildstream/element.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/buildstream/element.py b/buildstream/element.py index cc31d3f02..30bbc141c 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -2952,9 +2952,11 @@ class Element(Plugin): element = queue.pop() old_ready_for_runtime = element.__ready_for_runtime + old_strict_cache_key = element.__strict_cache_key element._update_state() - if element.__ready_for_runtime != old_ready_for_runtime: + if element.__ready_for_runtime != old_ready_for_runtime or \ + element.__strict_cache_key != old_strict_cache_key: for rdep in element.__reverse_dependencies: queue.push(rdep._unique_id, rdep) -- cgit v1.2.1 From 854ba3a141f0351520640bb4ac79a0a2a7260ef4 Mon Sep 17 00:00:00 2001 From: Tristan Van Berkom Date: Wed, 8 May 2019 15:31:05 +0900 Subject: tests/frontend/buildtrack.py: Extending tests to ensure we build after tracking This adds a check to test_build_track() to ensure that the target is cached as a result of building with tracking of selected elements. --- tests/frontend/buildtrack.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/frontend/buildtrack.py b/tests/frontend/buildtrack.py index 404882beb..d12481c6a 100644 --- a/tests/frontend/buildtrack.py +++ b/tests/frontend/buildtrack.py @@ -34,6 +34,7 @@ def create_element(repo, name, path, dependencies, ref=None): @pytest.mark.datafiles(os.path.join(DATA_DIR)) +@pytest.mark.parametrize("strict", [True, False], ids=["strict", "no-strict"]) @pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) @pytest.mark.parametrize("track_targets,exceptions,tracked", [ # Test with no exceptions @@ -51,7 +52,7 @@ def create_element(repo, name, path, dependencies, ref=None): (['3.bst'], ['2.bst', '3.bst'], []), (['2.bst', '3.bst'], ['2.bst', '3.bst'], []) ]) -def test_build_track(cli, datafiles, tmpdir, ref_storage, +def test_build_track(cli, datafiles, tmpdir, ref_storage, strict, track_targets, exceptions, tracked): project = str(datafiles) dev_files_path = os.path.join(project, 'files', 'dev-files') @@ -63,6 +64,13 @@ def test_build_track(cli, datafiles, tmpdir, ref_storage, configure_project(project, { 'ref-storage': ref_storage }) + cli.configure({ + 'projects': { + 'test': { + 'strict': strict + } + } + }) create_elements = { '0.bst': [ @@ -120,8 +128,12 @@ def test_build_track(cli, datafiles, tmpdir, ref_storage, result = cli.run(project=project, silent=True, args=args) result.assert_success() - tracked_elements = result.get_tracked_elements() + # Assert that the main target 0.bst is cached + assert cli.get_element_state(project, '0.bst') == 'cached' + + # Assert that we tracked exactly the elements we expected to + tracked_elements = result.get_tracked_elements() assert set(tracked_elements) == set(tracked) # Delete element sources -- cgit v1.2.1 From 223834422d12136879ea2840bd8c8270da87b657 Mon Sep 17 00:00:00 2001 From: Tristan Van Berkom Date: Wed, 8 May 2019 19:08:45 +0900 Subject: tests/frontend/buildtrack.py: Test that tracking builds in non-strict mode actually build This is a regression test for issue #1014 --- tests/frontend/buildtrack.py | 77 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 77 insertions(+) diff --git a/tests/frontend/buildtrack.py b/tests/frontend/buildtrack.py index d12481c6a..d42b6d1ba 100644 --- a/tests/frontend/buildtrack.py +++ b/tests/frontend/buildtrack.py @@ -12,6 +12,7 @@ from buildstream import _yaml from buildstream.testing import create_repo from buildstream.testing import cli # pylint: disable=unused-import from buildstream._exceptions import ErrorDomain +from tests.testutils import generate_junction from . import configure_project @@ -156,6 +157,82 @@ def test_build_track(cli, datafiles, tmpdir, ref_storage, strict, assert not os.path.exists(os.path.join(project, 'project.refs')) +# This tests a very specific scenario: +# +# o Local cache is empty +# o Strict mode is disabled +# o The build target has only build dependencies +# o The build is launched with --track-all +# +# In this scenario, we have encountered bugs where BuildStream returns +# successfully after tracking completes without ever pulling, fetching or +# building anything. +# +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.parametrize("strict", [True, False], ids=["strict", "no-strict"]) +@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')]) +def test_build_track_all(cli, tmpdir, datafiles, strict, ref_storage): + project = os.path.join(datafiles.dirname, datafiles.basename) + subproject_path = os.path.join(project, 'files', 'sub-project') + subproject_element_path = os.path.join(project, 'files', 'sub-project', 'elements') + junction_path = os.path.join(project, 'elements', 'junction.bst') + element_path = os.path.join(project, 'elements') + dev_files_path = os.path.join(project, 'files', 'dev-files') + + configure_project(project, { + 'ref-storage': ref_storage + }) + cli.configure({ + 'projects': { + 'test': { + 'strict': strict + } + } + }) + + # We need a repo for real trackable elements + repo = create_repo('git', str(tmpdir)) + ref = repo.create(dev_files_path) + + # Create a trackable element to depend on the cross junction element, + # this one has it's ref resolved already + create_element(repo, 'sub-target.bst', subproject_element_path, ['import-etc.bst'], ref=ref) + + # Create a trackable element to depend on the cross junction element + create_element(repo, 'target.bst', element_path, [ + { + 'junction': 'junction.bst', + 'filename': 'sub-target.bst' + } + ]) + + # Create a repo to hold the subproject and generate a junction element for it + generate_junction(tmpdir, subproject_path, junction_path, store_ref=False) + + # Now create a compose element at the top level + element = { + 'kind': 'compose', + 'depends': [ + { + 'filename': 'target.bst', + 'type': 'build' + } + ] + } + _yaml.dump(element, os.path.join(element_path, 'composed.bst')) + + # Track the junction itself first. + result = cli.run(project=project, args=['source', 'track', 'junction.bst']) + result.assert_success() + + # Build it with --track-all + result = cli.run(project=project, silent=True, args=['build', '--track-all', 'composed.bst']) + result.assert_success() + + # Assert that the main target is cached as a result + assert cli.get_element_state(project, 'composed.bst') == 'cached' + + @pytest.mark.datafiles(os.path.join(DATA_DIR)) @pytest.mark.parametrize("track_targets,exceptions,tracked", [ # Test with no exceptions -- cgit v1.2.1