tests/frontend: Refactor tests for default targets

`tests/frontend/buildcheckout.py` is already very crowded with all sorts
of tests. Since this test is not really about testing `bst build` but
rather about testing whether BuildStream can detect the default targets
correctly, move it to a separate test module.

At the same time, also improve the actual tests. Currently the test
compares the number of sub-directories in BuildStream log directory with
the number of elements. Change it such that we directly query
BuildStream about what's been built rather than trying to guess.

While doing so, sanitize its project directory to make it clearer what's
being tested.

25 files changed, 197 insertions, 255 deletions

diff --git a/tests/frontend/buildcheckout.py b/tests/frontend/buildcheckout.py
index 364ff98d1..ffe564e81 100644
--- a/tests/frontend/buildcheckout.py
+++ b/tests/frontend/buildcheckout.py
@@ -4,7 +4,6 @@
 import os
 import tarfile
 import hashlib
-import subprocess
 import re
 
 import pytest
@@ -66,35 +65,6 @@ def test_build_checkout(datafiles, cli, strict, hardlinks):
     assert os.path.exists(filename)
 
 
-@pytest.mark.datafiles(DATA_DIR + "_world")
-def test_build_default_all(datafiles, cli):
-    project = str(datafiles)
-    result = cli.run(project=project, silent=True, args=["build"])
-
-    result.assert_success()
-    target_dir = os.path.join(cli.directory, DATA_DIR + "_world", "elements")
-    output_dir = os.path.join(cli.directory, "logs", "test")
-
-    expected = subprocess.Popen(("ls", target_dir), stdout=subprocess.PIPE)
-    expected = subprocess.check_output(("wc", "-w"), stdin=expected.stdout)
-
-    results = subprocess.Popen(("ls", output_dir), stdout=subprocess.PIPE)
-    results = subprocess.check_output(("wc", "-w"), stdin=results.stdout)
-
-    assert results == expected
-
-
-@pytest.mark.datafiles(DATA_DIR + "_default")
-def test_build_default(cli, datafiles):
-    project = str(datafiles)
-    result = cli.run(project=project, silent=True, args=["build"])
-
-    result.assert_success()
-    results = cli.get_element_state(project, "target2.bst")
-    expected = "cached"
-    assert results == expected
-
-
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.parametrize("strict,hardlinks", [("non-strict", "hardlinks"),])
 def test_build_invalid_suffix(datafiles, cli, strict, hardlinks):
@@ -789,43 +759,6 @@ def test_build_checkout_junction(cli, tmpdir, datafiles):
     assert contents == "animal=Pony\n"
 
 
-# Test that default targets work with projects with junctions
-@pytest.mark.datafiles(DATA_DIR + "_world")
-def test_build_checkout_junction_default_targets(cli, tmpdir, datafiles):
-    project = str(datafiles)
-    subproject_path = os.path.join(project, "files", "sub-project")
-    junction_path = os.path.join(project, "elements", "junction.bst")
-    element_path = os.path.join(project, "elements", "junction-dep.bst")
-    checkout = os.path.join(cli.directory, "checkout")
-
-    # Create a repo to hold the subproject and generate a junction element for it
-    generate_junction(tmpdir, subproject_path, junction_path)
-
-    # Create a stack element to depend on a cross junction element
-    #
-    element = {"kind": "stack", "depends": [{"junction": "junction.bst", "filename": "import-etc.bst"}]}
-    _yaml.roundtrip_dump(element, element_path)
-
-    # Now try to build it, this should automatically result in fetching
-    # the junction itself at load time.
-    result = cli.run(project=project, args=["build"])
-    result.assert_success()
-
-    # Assert that it's cached now
-    assert cli.get_element_state(project, "junction-dep.bst") == "cached"
-
-    # Now check it out
-    result = cli.run(project=project, args=["artifact", "checkout", "junction-dep.bst", "--directory", checkout])
-    result.assert_success()
-
-    # Assert the content of /etc/animal.conf
-    filename = os.path.join(checkout, "etc", "animal.conf")
-    assert os.path.exists(filename)
-    with open(filename, "r") as f:
-        contents = f.read()
-    assert contents == "animal=Pony\n"
-
-
 @pytest.mark.datafiles(DATA_DIR)
 def test_build_checkout_workspaced_junction(cli, tmpdir, datafiles):
     project = str(datafiles)
diff --git a/tests/frontend/project_world/elements/import-bin.bst b/tests/frontend/default-target/elements/dummy_1.bst
index 066a03328..066a03328 100644
--- a/tests/frontend/project_world/elements/import-bin.bst
+++ b/tests/frontend/default-target/elements/dummy_1.bst
diff --git a/tests/frontend/project_world/elements/import-dev.bst b/tests/frontend/default-target/elements/dummy_2.bst
index 066a03328..066a03328 100644
--- a/tests/frontend/project_world/elements/import-dev.bst
+++ b/tests/frontend/default-target/elements/dummy_2.bst
diff --git a/tests/frontend/default-target/elements/dummy_3.bst b/tests/frontend/default-target/elements/dummy_3.bst
new file mode 100644
index 000000000..066a03328
--- /dev/null
+++ b/tests/frontend/default-target/elements/dummy_3.bst
@@ -0,0 +1 @@
+kind: stack
diff --git a/tests/frontend/default-target/elements/dummy_stack.bst b/tests/frontend/default-target/elements/dummy_stack.bst
new file mode 100644
index 000000000..5f921667f
--- /dev/null
+++ b/tests/frontend/default-target/elements/dummy_stack.bst
@@ -0,0 +1,5 @@
+kind: stack
+
+runtime-depends:
+- dummy_1.bst
+- dummy_2.bst
diff --git a/tests/frontend/default-target/files/sub-project/elements/dummy_subproject.bst b/tests/frontend/default-target/files/sub-project/elements/dummy_subproject.bst
new file mode 100644
index 000000000..066a03328
--- /dev/null
+++ b/tests/frontend/default-target/files/sub-project/elements/dummy_subproject.bst
@@ -0,0 +1 @@
+kind: stack
diff --git a/tests/frontend/default-target/files/sub-project/project.conf b/tests/frontend/default-target/files/sub-project/project.conf
new file mode 100644
index 000000000..db76b5267
--- /dev/null
+++ b/tests/frontend/default-target/files/sub-project/project.conf
@@ -0,0 +1,3 @@
+name: test-default-target-subproject
+
+element-path: elements
diff --git a/tests/frontend/default-target/project.conf b/tests/frontend/default-target/project.conf
new file mode 100644
index 000000000..17937ed73
--- /dev/null
+++ b/tests/frontend/default-target/project.conf
@@ -0,0 +1,3 @@
+name: test-default-target
+
+element-path: elements
diff --git a/tests/frontend/default_target.py b/tests/frontend/default_target.py
new file mode 100644
index 000000000..d2a75c961
--- /dev/null
+++ b/tests/frontend/default_target.py
@@ -0,0 +1,184 @@
+# Pylint doesn't play well with fixtures and dependency injection from pytest
+# pylint: disable=redefined-outer-name
+
+import os
+
+import pytest
+
+from buildstream import _yaml
+from buildstream.testing import cli, create_repo  # pylint: disable=unused-import
+from tests.testutils import create_artifact_share
+
+# project directory
+DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "default-target",)
+
+
+###################################
+#      build/show operations      #
+###################################
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("operation,expected_state", [("show", "buildable"), ("build", "cached")])
+def test_no_default(cli, datafiles, operation, expected_state):
+    project = str(datafiles)
+    all_targets = ["dummy_1.bst", "dummy_2.bst", "dummy_3.bst", "dummy_stack.bst"]
+
+    result = cli.run(project=project, args=[operation])
+    result.assert_success()
+
+    states = cli.get_element_states(project, all_targets)
+    assert all(states[e] == expected_state for e in all_targets)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("operation,expected_state", [("show", "buildable"), ("build", "cached")])
+def test_default_target(cli, datafiles, operation, expected_state):
+    project = str(datafiles)
+    project_path = os.path.join(project, "project.conf")
+
+    # First, modify project configuration to set a default target
+    project_conf = {
+        "name": "test-default-target",
+        "element-path": "elements",
+        "defaults": {"targets": ["dummy_stack.bst"]},
+    }
+    _yaml.roundtrip_dump(project_conf, project_path)
+
+    # dummy_stack only depends on dummy_1 and dummy_2, but not dummy_3
+    all_targets = ["dummy_1.bst", "dummy_2.bst", "dummy_stack.bst"]
+
+    result = cli.run(project=project, args=[operation])
+    result.assert_success()
+
+    states = cli.get_element_states(project, all_targets)
+    assert all(states[e] == expected_state for e in all_targets)
+
+    # assert that dummy_3 isn't included in the output
+    assert "dummy_3.bst" not in states
+
+
+@pytest.mark.datafiles(DATA_DIR)
+@pytest.mark.parametrize("operation,expected_state", [("show", "buildable"), ("build", "cached")])
+def test_no_default_with_junction(cli, datafiles, operation, expected_state):
+    project = str(datafiles)
+    junction_path = os.path.join(project, "elements", "junction.bst")
+    target_path = os.path.join(project, "elements", "junction-target.bst")
+
+    # First, create a junction element to refer to the subproject
+    junction_config = {"kind": "junction", "sources": [{"kind": "local", "path": "files/sub-project",}]}
+    _yaml.roundtrip_dump(junction_config, junction_path)
+
+    # Then, create a stack element with dependency on cross junction element
+    target_config = {"kind": "stack", "runtime-depends": ["junction.bst:dummy_subproject.bst"]}
+    _yaml.roundtrip_dump(target_config, target_path)
+
+    # Now try to perform the specified operation.
+    # This should automatically fetch the junction at load time.
+    result = cli.run(project=project, args=[operation])
+    result.assert_success()
+
+    assert cli.get_element_state(project, "junction.bst:dummy_subproject.bst") == expected_state
+    assert cli.get_element_state(project, "junction-target.bst") == expected_state
+
+
+###################################
+#     track/fetch operations      #
+###################################
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_default_target_track(cli, tmpdir, datafiles):
+    project = str(datafiles)
+    project_path = os.path.join(project, "project.conf")
+    target = "track-fetch-test.bst"
+
+    # First, create an element with trackable sources
+    repo = create_repo("git", str(tmpdir))
+    repo.create(project)
+    element_conf = {"kind": "import", "sources": [repo.source_config()]}
+    _yaml.roundtrip_dump(element_conf, os.path.join(project, "elements", target))
+
+    # Then, make it the default target
+    project_conf = {
+        "name": "test-default-target",
+        "element-path": "elements",
+        "defaults": {"targets": [target]},
+    }
+    _yaml.roundtrip_dump(project_conf, project_path)
+
+    # Setup finished. Track it now
+    assert cli.get_element_state(project, target) == "no reference"
+    result = cli.run(project=project, args=["source", "track"])
+    result.assert_success()
+    # Tracking will result in fetching it automatically, so we expect the state
+    # to be buildable.
+    assert cli.get_element_state(project, target) == "buildable"
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_default_target_fetch(cli, tmpdir, datafiles):
+    project = str(datafiles)
+    project_path = os.path.join(project, "project.conf")
+    target = "track-fetch-test.bst"
+
+    # First, create an element with trackable sources
+    repo = create_repo("git", str(tmpdir))
+    ref = repo.create(project)
+    element_conf = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
+    _yaml.roundtrip_dump(element_conf, os.path.join(project, "elements", target))
+
+    # Then, make it the default target
+    project_conf = {
+        "name": "test-default-target",
+        "element-path": "elements",
+        "defaults": {"targets": [target]},
+    }
+    _yaml.roundtrip_dump(project_conf, project_path)
+
+    # Setup finished. Track it now
+    assert cli.get_element_state(project, target) == "fetch needed"
+    result = cli.run(project=project, args=["source", "fetch"])
+    result.assert_success()
+    assert cli.get_element_state(project, target) == "buildable"
+
+
+###################################
+#      pull/push operations      #
+###################################
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_default_target_push_pull(cli, tmpdir, datafiles):
+    project = str(datafiles)
+    project_path = os.path.join(project, "project.conf")
+    target = "dummy_1.bst"
+
+    # Set a default target
+    project_conf = {
+        "name": "test-default-target",
+        "element-path": "elements",
+        "defaults": {"targets": [target]},
+    }
+    _yaml.roundtrip_dump(project_conf, project_path)
+
+    # Build the target
+    result = cli.run(project=project, args=["build"])
+    result.assert_success()
+    assert cli.get_element_state(project, target) == "cached"
+
+    with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
+        # Push the artifacts
+        cli.configure({"artifacts": {"url": share.repo, "push": True}})
+        result = cli.run(project=project, args=["artifact", "push"])
+        result.assert_success()
+
+        # Delete local artifacts
+        # Note that `artifact delete` does not support default targets
+        result = cli.run(project=project, args=["artifact", "delete", target])
+        result.assert_success()
+
+        # Target should be buildable now, and we should be able to pull it
+        assert cli.get_element_state(project, target) == "buildable"
+        result = cli.run(project=project, args=["artifact", "pull"])
+        assert cli.get_element_state(project, target) == "cached"
diff --git a/tests/frontend/fetch.py b/tests/frontend/fetch.py
index 9258fc7d9..8bd54ccc1 100644
--- a/tests/frontend/fetch.py
+++ b/tests/frontend/fetch.py
@@ -4,7 +4,6 @@
 import os
 import pytest
 
-from buildstream.testing import create_repo
 from buildstream.testing import cli  # pylint: disable=unused-import
 from buildstream import _yaml
 from buildstream._exceptions import ErrorDomain, LoadErrorReason
@@ -18,34 +17,6 @@ TOP_DIR = os.path.dirname(os.path.realpath(__file__))
 DATA_DIR = os.path.join(TOP_DIR, "project")
 
 
-@pytest.mark.datafiles(os.path.join(TOP_DIR, "project_world"))
-def test_fetch_default_targets(cli, tmpdir, datafiles):
-    project = str(datafiles)
-    element_path = os.path.join(project, "elements")
-    element_name = "fetch-test.bst"
-
-    # Create our repo object of the given source type with
-    # the bin files, and then collect the initial ref.
-    #
-    repo = create_repo("git", str(tmpdir))
-    ref = repo.create(project)
-
-    # Write out our test target
-    element = {"kind": "import", "sources": [repo.source_config(ref=ref)]}
-    _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
-
-    # Assert that a fetch is needed
-    assert cli.get_element_state(project, element_name) == "fetch needed"
-
-    # Now try to fetch it, using the default target feature
-    result = cli.run(project=project, args=["source", "fetch"])
-    result.assert_success()
-
-    # Assert that we are now buildable because the source is
-    # now cached.
-    assert cli.get_element_state(project, element_name) == "buildable"
-
-
 @pytest.mark.datafiles(os.path.join(TOP_DIR, "consistencyerror"))
 def test_fetch_consistency_error(cli, datafiles):
     project = str(datafiles)
diff --git a/tests/frontend/project_default/elements/target.bst b/tests/frontend/project_default/elements/target.bst
deleted file mode 100644
index d644c89ba..000000000
--- a/tests/frontend/project_default/elements/target.bst
+++ /dev/null
@@ -1,4 +0,0 @@
-kind: stack
-description: |
-
-  Main stack target for the bst build test
diff --git a/tests/frontend/project_default/elements/target2.bst b/tests/frontend/project_default/elements/target2.bst
deleted file mode 100644
index d644c89ba..000000000
--- a/tests/frontend/project_default/elements/target2.bst
+++ /dev/null
@@ -1,4 +0,0 @@
-kind: stack
-description: |
-
-  Main stack target for the bst build test
diff --git a/tests/frontend/project_default/project.conf b/tests/frontend/project_default/project.conf
deleted file mode 100644
index 5987c82f1..000000000
--- a/tests/frontend/project_default/project.conf
+++ /dev/null
@@ -1,11 +0,0 @@
-# Project config for frontend build test
-name: test
-
-element-path: elements
-
-fatal-warnings:
-- bad-element-suffix
-
-defaults:
- targets:
- - target2.bst
diff --git a/tests/frontend/project_world/elements/checkout-deps.bst b/tests/frontend/project_world/elements/checkout-deps.bst
deleted file mode 100644
index e3a548690..000000000
--- a/tests/frontend/project_world/elements/checkout-deps.bst
+++ /dev/null
@@ -1,7 +0,0 @@
-kind: stack
-description: It is important for this element to have both build and runtime dependencies
-depends:
-- filename: import-dev.bst
-  type: build
-- filename: import-bin.bst
-  type: runtime
diff --git a/tests/frontend/project_world/elements/compose-all.bst b/tests/frontend/project_world/elements/compose-all.bst
deleted file mode 100644
index ba47081b3..000000000
--- a/tests/frontend/project_world/elements/compose-all.bst
+++ /dev/null
@@ -1,12 +0,0 @@
-kind: compose
-
-depends:
-- filename: import-bin.bst
-  type: build
-- filename: import-dev.bst
-  type: build
-
-config:
-  # Dont try running the sandbox, we dont have a
-  # runtime to run anything in this context.
-  integrate: False
diff --git a/tests/frontend/project_world/elements/compose-exclude-dev.bst b/tests/frontend/project_world/elements/compose-exclude-dev.bst
deleted file mode 100644
index 75c14378c..000000000
--- a/tests/frontend/project_world/elements/compose-exclude-dev.bst
+++ /dev/null
@@ -1,16 +0,0 @@
-kind: compose
-
-depends:
-- filename: import-bin.bst
-  type: build
-- filename: import-dev.bst
-  type: build
-
-config:
-  # Dont try running the sandbox, we dont have a
-  # runtime to run anything in this context.
-  integrate: False
-
-  # Exclude the dev domain
-  exclude:
-  - devel
diff --git a/tests/frontend/project_world/elements/compose-include-bin.bst b/tests/frontend/project_world/elements/compose-include-bin.bst
deleted file mode 100644
index 9571203c6..000000000
--- a/tests/frontend/project_world/elements/compose-include-bin.bst
+++ /dev/null
@@ -1,16 +0,0 @@
-kind: compose
-
-depends:
-- filename: import-bin.bst
-  type: build
-- filename: import-dev.bst
-  type: build
-
-config:
-  # Dont try running the sandbox, we dont have a
-  # runtime to run anything in this context.
-  integrate: False
-
-  # Only include the runtim
-  include:
-  - runtime
diff --git a/tests/frontend/project_world/elements/rebuild-target.bst b/tests/frontend/project_world/elements/rebuild-target.bst
deleted file mode 100644
index 49a02c217..000000000
--- a/tests/frontend/project_world/elements/rebuild-target.bst
+++ /dev/null
@@ -1,4 +0,0 @@
-kind: compose
-
-build-depends:
-- target.bst
diff --git a/tests/frontend/project_world/elements/target.bst b/tests/frontend/project_world/elements/target.bst
deleted file mode 100644
index b9432fafa..000000000
--- a/tests/frontend/project_world/elements/target.bst
+++ /dev/null
@@ -1,8 +0,0 @@
-kind: stack
-description: |
-
-  Main stack target for the bst build test
-
-depends:
-- import-bin.bst
-- compose-all.bst
diff --git a/tests/frontend/project_world/files/sub-project/elements/import-etc.bst b/tests/frontend/project_world/files/sub-project/elements/import-etc.bst
deleted file mode 100644
index f0171990e..000000000
--- a/tests/frontend/project_world/files/sub-project/elements/import-etc.bst
+++ /dev/null
@@ -1,4 +0,0 @@
-kind: import
-sources:
-- kind: local
-  path: files/etc-files
diff --git a/tests/frontend/project_world/files/sub-project/files/etc-files/etc/animal.conf b/tests/frontend/project_world/files/sub-project/files/etc-files/etc/animal.conf
deleted file mode 100644
index db8c36cba..000000000
--- a/tests/frontend/project_world/files/sub-project/files/etc-files/etc/animal.conf
+++ /dev/null
@@ -1 +0,0 @@
-animal=Pony
diff --git a/tests/frontend/project_world/files/sub-project/project.conf b/tests/frontend/project_world/files/sub-project/project.conf
deleted file mode 100644
index bbb8414a3..000000000
--- a/tests/frontend/project_world/files/sub-project/project.conf
+++ /dev/null
@@ -1,4 +0,0 @@
-# Project config for frontend build test
-name: subtest
-
-element-path: elements
diff --git a/tests/frontend/project_world/project.conf b/tests/frontend/project_world/project.conf
deleted file mode 100644
index a7e4a023c..000000000
--- a/tests/frontend/project_world/project.conf
+++ /dev/null
@@ -1,7 +0,0 @@
-# Project config for frontend build test
-name: test
-
-element-path: elements
-
-fatal-warnings:
-- bad-element-suffix
diff --git a/tests/frontend/pull.py b/tests/frontend/pull.py
index 3e726ffcb..fcfd447b7 100644
--- a/tests/frontend/pull.py
+++ b/tests/frontend/pull.py
@@ -60,54 +60,6 @@ def test_push_pull_all(cli, tmpdir, datafiles):
 
 # Tests that:
 #
-#  * `bst artifact push` (default targets) pushes all built elements to configured 'push' cache
-#  * `bst artifact pull` (default targets) downloads everything from cache after local deletion
-#
-@pytest.mark.datafiles(DATA_DIR + "_world")
-def test_push_pull_default_targets(cli, tmpdir, datafiles):
-    project = str(datafiles)
-
-    with create_artifact_share(os.path.join(str(tmpdir), "artifactshare")) as share:
-
-        # First build the target elements
-        cli.configure({"artifacts": {"url": share.repo}})
-        result = cli.run(project=project, args=["build"])
-        result.assert_success()
-        assert cli.get_element_state(project, "target.bst") == "cached"
-
-        # Push all elements
-        cli.configure({"artifacts": {"url": share.repo, "push": True}})
-        result = cli.run(project=project, args=["artifact", "push"])
-        result.assert_success()
-
-        # Assert that everything is now cached in the remote.
-        all_elements = ["target.bst", "import-bin.bst", "import-dev.bst", "compose-all.bst"]
-        for element_name in all_elements:
-            assert_shared(cli, share, project, element_name)
-
-        # Now we've pushed, delete the user's local artifact cache
-        # directory and try to redownload it from the share
-        #
-        casdir = os.path.join(cli.directory, "cas")
-        shutil.rmtree(casdir)
-        artifactdir = os.path.join(cli.directory, "artifacts")
-        shutil.rmtree(artifactdir)
-
-        # Assert that nothing is cached locally anymore
-        states = cli.get_element_states(project, all_elements)
-        assert not any(states[e] == "cached" for e in all_elements)
-
-        # Now try bst artifact pull
-        result = cli.run(project=project, args=["artifact", "pull"])
-        result.assert_success()
-
-        # And assert that it's again in the local cache, without having built
-        states = cli.get_element_states(project, all_elements)
-        assert not any(states[e] != "cached" for e in all_elements)
-
-
-# Tests that:
-#
 #  * `bst build` pushes all build elements ONLY to configured 'push' cache
 #  * `bst artifact pull` finds artifacts that are available only in the secondary cache
 #
diff --git a/tests/frontend/show.py b/tests/frontend/show.py
index b08670a30..94b94a058 100644
--- a/tests/frontend/show.py
+++ b/tests/frontend/show.py
@@ -42,19 +42,6 @@ def test_show_invalid_element_path(cli, datafiles):
     cli.run(project=project, silent=True, args=["show", "foo.bst"])
 
 
-@pytest.mark.datafiles(os.path.join(DATA_DIR, "project_default"))
-def test_show_default(cli, datafiles):
-    project = str(datafiles)
-    result = cli.run(project=project, silent=True, args=["show"])
-
-    result.assert_success()
-
-    # Get the result output of "[state sha element]" and turn into a list
-    results = result.output.strip().split(" ")
-    expected = "target2.bst"
-    assert results[2] == expected
-
-
 @pytest.mark.datafiles(os.path.join(DATA_DIR, "project_fail"))
 def test_show_fail(cli, datafiles):
     project = str(datafiles)