Change default of cache-buildtrees option to 'auto'juerg/cache-buildtrees

Caching buildtrees may significantly increase CAS disk usage. Change the
default to cache buildtrees only when required for core functionality.

7 files changed, 28 insertions, 27 deletions

diff --git a/NEWS b/NEWS
index 5da4a1322..a19182c09 100644
--- a/NEWS
+++ b/NEWS
@@ -138,7 +138,7 @@ buildstream 1.3.1
     directory, this can now be extended to all or successful artifacts to save on cache
     overheads. The cli main option '--cache-buildtrees' or the user configuration cache
     group option 'cache-buildtrees' can be set as 'always', 'auto' or 'never', with
-    the default being always. Note, as the cache-key for the artifact is independant of
+    the default being 'auto'. Note, as the cache-key for the artifact is independent of
     the cached build tree input it will remain unaltered, however the availbility of the
     build tree content may differ.
 
diff --git a/buildstream/_artifact.py b/buildstream/_artifact.py
index 4d956d8d7..234116fad 100644
--- a/buildstream/_artifact.py
+++ b/buildstream/_artifact.py
@@ -141,13 +141,6 @@ class Artifact():
             filesvdir = assemblevdir.descend("files", create=True)
             filesvdir.import_files(collectvdir)
 
-        # cache_buildtrees defaults to 'always', as such the
-        # default behaviour is to attempt to cache them. If only
-        # caching failed artifact buildtrees, then query the build
-        # result. Element types without a build-root dir will be cached
-        # with an empty buildtreedir regardless of this configuration as
-        # there will not be an applicable sandbox_build_dir.
-
         if sandbox_build_dir:
             buildtreevdir = assemblevdir.descend("buildtree", create=True)
             buildtreevdir.import_files(sandbox_build_dir)
diff --git a/buildstream/data/userconfig.yaml b/buildstream/data/userconfig.yaml
index e70c41312..34fd300d1 100644
--- a/buildstream/data/userconfig.yaml
+++ b/buildstream/data/userconfig.yaml
@@ -47,7 +47,7 @@ cache:
   #            debugging failed builds and may break additional functionality
   #            in future versions.
   #
-  cache-buildtrees: always
+  cache-buildtrees: auto
 
 
 #
diff --git a/buildstream/element.py b/buildstream/element.py
index 45bb983bb..e5bc0792e 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -1724,6 +1724,12 @@ class Element(Plugin):
         cache_buildtrees = context.cache_buildtrees
         build_success = buildresult[0]
 
+        # cache_buildtrees defaults to 'auto', only caching buildtrees
+        # when necessary, which includes failed builds.
+        # If only caching failed artifact buildtrees, then query the build
+        # result. Element types without a build-root dir will be cached
+        # with an empty buildtreedir regardless of this configuration.
+
         if cache_buildtrees == 'always' or (cache_buildtrees == 'auto' and not build_success):
             try:
                 sandbox_build_dir = sandbox_vroot.descend(
diff --git a/tests/integration/artifact.py b/tests/integration/artifact.py
index 98c6ce13f..1f6028e4b 100644
--- a/tests/integration/artifact.py
+++ b/tests/integration/artifact.py
@@ -68,11 +68,11 @@ def test_cache_buildtrees(cli, tmpdir, datafiles):
             finally:
                 utils._force_rmtree(extractdir)
 
-        # Build autotools element with cache-buildtrees set via the
-        # cli. The artifact should be successfully pushed to the share1 remote
+        # Build autotools element with the default behavior of caching buildtrees
+        # only when necessary. The artifact should be successfully pushed to the share1 remote
         # and cached locally with an 'empty' buildtree digest, as it's not a
         # dangling ref
-        result = cli.run(project=project, args=['--cache-buildtrees', 'never', 'build', element_name])
+        result = cli.run(project=project, args=['build', element_name])
         assert result.exit_code == 0
         assert cli.get_element_state(project, element_name) == 'cached'
         assert share1.has_artifact('test', element_name, cli.get_element_key(project, element_name))
@@ -103,13 +103,13 @@ def test_cache_buildtrees(cli, tmpdir, datafiles):
             assert not os.path.isdir(buildtreedir)
         shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
 
-        # Repeat building the artifacts, this time with the default behaviour of caching buildtrees,
-        # as such the buildtree dir should not be empty
+        # Repeat building the artifacts, this time with cache-buildtrees set to
+        # 'always' via the cli, as such the buildtree dir should not be empty
         cli.configure({
             'artifacts': {'url': share2.repo, 'push': True},
             'cachedir': str(tmpdir)
         })
-        result = cli.run(project=project, args=['build', element_name])
+        result = cli.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
         assert result.exit_code == 0
         assert cli.get_element_state(project, element_name) == 'cached'
         assert share2.has_artifact('test', element_name, cli.get_element_key(project, element_name))
@@ -137,7 +137,7 @@ def test_cache_buildtrees(cli, tmpdir, datafiles):
         cli.configure({
             'artifacts': {'url': share3.repo, 'push': True},
             'cachedir': str(tmpdir),
-            'cache': {'cache-buildtrees': 'never'}
+            'cache': {'cache-buildtrees': 'always'}
         })
         result = cli.run(project=project, args=['build', element_name])
         assert result.exit_code == 0
@@ -145,4 +145,5 @@ def test_cache_buildtrees(cli, tmpdir, datafiles):
         cache_key = cli.get_element_key(project, element_name)
         elementdigest = share3.has_artifact('test', element_name, cache_key)
         with cas_extract_buildtree(elementdigest) as buildtreedir:
-            assert not os.path.isdir(buildtreedir)
+            assert os.path.isdir(buildtreedir)
+            assert os.listdir(buildtreedir)
diff --git a/tests/integration/pullbuildtrees.py b/tests/integration/pullbuildtrees.py
index efcf9cf87..6a3b92723 100644
--- a/tests/integration/pullbuildtrees.py
+++ b/tests/integration/pullbuildtrees.py
@@ -48,6 +48,7 @@ def test_pullbuildtrees(cli2, tmpdir, datafiles):
         cli2.configure({
             'artifacts': {'url': share1.repo, 'push': True},
             'cachedir': str(tmpdir),
+            'cache': {'cache-buildtrees': 'always'},
         })
 
         @contextmanager
diff --git a/tests/integration/shellbuildtrees.py b/tests/integration/shellbuildtrees.py
index 8c5411526..fed3c5167 100644
--- a/tests/integration/shellbuildtrees.py
+++ b/tests/integration/shellbuildtrees.py
@@ -26,7 +26,7 @@ def test_buildtree_staged(cli_integration, datafiles):
     project = os.path.join(datafiles.dirname, datafiles.basename)
     element_name = 'build-shell/buildtree.bst'
 
-    res = cli_integration.run(project=project, args=['build', element_name])
+    res = cli_integration.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
     res.assert_success()
 
     res = cli_integration.run(project=project, args=[
@@ -42,7 +42,7 @@ def test_buildtree_staged_forced_true(cli_integration, datafiles):
     project = os.path.join(datafiles.dirname, datafiles.basename)
     element_name = 'build-shell/buildtree.bst'
 
-    res = cli_integration.run(project=project, args=['build', element_name])
+    res = cli_integration.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
     res.assert_success()
 
     res = cli_integration.run(project=project, args=[
@@ -65,7 +65,7 @@ def test_buildtree_staged_warn_empty_cached(cli_integration, tmpdir, datafiles):
         'cachedir': str(tmpdir)
     })
 
-    res = cli_integration.run(project=project, args=['--cache-buildtrees', 'never', 'build', element_name])
+    res = cli_integration.run(project=project, args=['build', element_name])
     res.assert_success()
 
     res = cli_integration.run(project=project, args=[
@@ -82,7 +82,7 @@ def test_buildtree_staged_if_available(cli_integration, datafiles):
     project = os.path.join(datafiles.dirname, datafiles.basename)
     element_name = 'build-shell/buildtree.bst'
 
-    res = cli_integration.run(project=project, args=['build', element_name])
+    res = cli_integration.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
     res.assert_success()
 
     res = cli_integration.run(project=project, args=[
@@ -99,7 +99,7 @@ def test_buildtree_staged_forced_false(cli_integration, datafiles):
     project = os.path.join(datafiles.dirname, datafiles.basename)
     element_name = 'build-shell/buildtree.bst'
 
-    res = cli_integration.run(project=project, args=['build', element_name])
+    res = cli_integration.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
     res.assert_success()
 
     res = cli_integration.run(project=project, args=[
@@ -154,19 +154,19 @@ def test_buildtree_from_failure_option_never(cli_integration, tmpdir, datafiles)
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-def test_buildtree_from_failure_option_auto(cli_integration, tmpdir, datafiles):
+def test_buildtree_from_failure_option_always(cli_integration, tmpdir, datafiles):
 
     project = os.path.join(datafiles.dirname, datafiles.basename)
     element_name = 'build-shell/buildtree-fail.bst'
 
-    # build with  --cache-buildtrees set to 'auto', behaviour should match
+    # build with  --cache-buildtrees set to 'always', behaviour should match
     # default behaviour (which is always) as the buildtree will explicitly have been
     # cached with content.
     cli_integration.configure({
         'cachedir': str(tmpdir)
     })
 
-    res = cli_integration.run(project=project, args=['--cache-buildtrees', 'auto', 'build', element_name])
+    res = cli_integration.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
     res.assert_main_error(ErrorDomain.STREAM, None)
 
     res = cli_integration.run(project=project, args=[
@@ -190,7 +190,7 @@ def test_buildtree_pulled(cli, tmpdir, datafiles):
         cli.configure({
             'artifacts': {'url': share.repo, 'push': True}
         })
-        result = cli.run(project=project, args=['build', element_name])
+        result = cli.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
         result.assert_success()
         assert cli.get_element_state(project, element_name) == 'cached'
 
@@ -222,7 +222,7 @@ def test_buildtree_options(cli, tmpdir, datafiles):
         cli.configure({
             'artifacts': {'url': share.repo, 'push': True}
         })
-        result = cli.run(project=project, args=['build', element_name])
+        result = cli.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name])
         result.assert_success()
         assert cli.get_element_state(project, element_name) == 'cached'
         assert share.has_artifact('test', element_name, cli.get_element_key(project, element_name))