diff options
author | Tom Pollard <tom.pollard@codethink.co.uk> | 2019-02-08 10:51:17 +0000 |
---|---|---|
committer | Tom Pollard <tom.pollard@codethink.co.uk> | 2019-02-13 13:37:32 +0000 |
commit | d20294440860894ae8272b522efe35f5bbbaa258 (patch) | |
tree | 84de0eea5913859fd92048fc0ad0d51955e7dcb1 | |
parent | b546bac10b8c339ae7323282fcbcbc58c16e4ac0 (diff) | |
download | buildstream-tpollard/896.tar.gz |
Provide configuration for the optional creation of buildtreestpollard/896
Artifacts can be cached explicitly with an empty `build tree` when
built via the cli main options or user config for all or only
successful build artifacts. Default behaviour is to still create
and cache all expected buildtrees.
element.py: _cache_artifact() Check if context for cache_buildtrees
has been set to always or failure with a corresponding build
result, if not skip attempting to export the build-root. Element
types without a build-root are cached with an empty buildtree
regardless. Update _stage_sources_at() to warn the user that the
buildtree import is empty.
tests/integration: Add test to artifact.py for the optional caching
of buildtree content from bst build. Rename build-tree.py to
shellbuildtrees.py to reflect included test cases, add test for
empty buildtree warning and failure option.
NEWS: Add entry for new option.
-rw-r--r-- | NEWS | 8 | ||||
-rw-r--r-- | buildstream/element.py | 43 | ||||
-rw-r--r-- | tests/integration/artifact.py | 108 | ||||
-rw-r--r-- | tests/integration/shellbuildtrees.py (renamed from tests/integration/build-tree.py) | 71 |
4 files changed, 216 insertions, 14 deletions
@@ -126,6 +126,14 @@ buildstream 1.3.1 Providing a remote will limit build's pull/push remote actions to the given remote specifically, ignoring those defined via user or project configuration. + o Artifacts can now be cached explicitly with an empty `build tree` when built. + Element types without a build-root were already cached with an empty build tree + directory, this can now be extended to all or successful artifacts to save on cache + overheads. The cli main option '--cache-buildtrees' or the user configuration cache + group option 'cache-buildtrees' can be set as 'always', 'failure' or 'never', with + the default being always. Note, as the cache-key for the artifact is independant of + the cached build tree input it will remain unaltered, however the availbility of the + build tree content may differ. ================= buildstream 1.1.5 diff --git a/buildstream/element.py b/buildstream/element.py index e03f1e171..a3bf59dc6 100644 --- a/buildstream/element.py +++ b/buildstream/element.py @@ -1457,6 +1457,9 @@ class Element(Plugin): elif usebuildtree: artifact_base, _ = self.__extract() import_dir = os.path.join(artifact_base, 'buildtree') + if not os.listdir(import_dir): + detail = "Element type either does not expect a buildtree or it was explictily cached without one." + self.warn("WARNING: {} Artifact contains an empty buildtree".format(self.name), detail=detail) else: # No workspace or cached buildtree, stage source directly for source in self.sources(): @@ -1663,6 +1666,8 @@ class Element(Plugin): # No collect directory existed collectvdir = None + context = self._get_context() + # Create artifact directory structure assembledir = os.path.join(rootdir, 'artifact') filesdir = os.path.join(assembledir, 'files') @@ -1680,20 +1685,30 @@ class Element(Plugin): if collect is not None and collectvdir is not None: collectvdir.export_files(filesdir, can_link=True) - try: - sandbox_vroot = sandbox.get_virtual_directory() - sandbox_build_dir = sandbox_vroot.descend( - self.get_variable('build-root').lstrip(os.sep).split(os.sep)) - # Hard link files from build-root dir to buildtreedir directory - sandbox_build_dir.export_files(buildtreedir) - except VirtualDirectoryError: - # Directory could not be found. Pre-virtual - # directory behaviour was to continue silently - # if the directory could not be found. - pass + cache_buildtrees = context.cache_buildtrees + build_success = self.__build_result[0] + + # cache_buildtrees defaults to 'always', as such the + # default behaviour is to attempt to cache them. If only + # caching failed artifact buildtrees, then query the build + # result. Element types without a build-root dir will be cached + # with an empty buildtreedir regardless of this configuration. + + if cache_buildtrees == 'always' or (cache_buildtrees == 'failure' and not build_success): + try: + sandbox_vroot = sandbox.get_virtual_directory() + sandbox_build_dir = sandbox_vroot.descend( + self.get_variable('build-root').lstrip(os.sep).split(os.sep)) + # Hard link files from build-root dir to buildtreedir directory + sandbox_build_dir.export_files(buildtreedir) + except VirtualDirectoryError: + # Directory could not be found. Pre-virtual + # directory behaviour was to continue silently + # if the directory could not be found. + pass # Copy build log - log_filename = self._get_context().get_log_filename() + log_filename = context.get_log_filename() self._build_log_path = os.path.join(logsdir, 'build.log') if log_filename: shutil.copyfile(log_filename, self._build_log_path) @@ -1834,7 +1849,7 @@ class Element(Plugin): return True # Do not push elements that aren't cached, or that are cached with a dangling buildtree - # artifact unless element type is expected to have an an empty buildtree directory + # ref unless element type is expected to have an an empty buildtree directory if not self._cached_buildtree(): return True @@ -2036,6 +2051,8 @@ class Element(Plugin): # Returns: # (bool): True if artifact cached with buildtree, False if # element not cached or missing expected buildtree. + # Note this only confirms if a buildtree is present, + # not its contents. # def _cached_buildtree(self): context = self._get_context() diff --git a/tests/integration/artifact.py b/tests/integration/artifact.py index 459241209..742c33455 100644 --- a/tests/integration/artifact.py +++ b/tests/integration/artifact.py @@ -20,9 +20,12 @@ import os import pytest +import shutil from buildstream.plugintestutils import cli_integration as cli - +from tests.testutils import create_artifact_share +from tests.testutils.site import HAVE_SANDBOX +from buildstream._exceptions import ErrorDomain pytestmark = pytest.mark.integration @@ -66,3 +69,106 @@ def test_artifact_log(cli, tmpdir, datafiles): assert result.exit_code == 0 # The artifact is cached under both a strong key and a weak key assert (log + log) == result.output + + +# A test to capture the integration of the cachebuildtrees +# behaviour, which by default is to include the buildtree +# content of an element on caching. +@pytest.mark.integration +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +def test_cache_buildtrees(cli, tmpdir, datafiles): + project = os.path.join(datafiles.dirname, datafiles.basename) + element_name = 'autotools/amhello.bst' + + # Create artifact shares for pull & push testing + with create_artifact_share(os.path.join(str(tmpdir), 'share1')) as share1,\ + create_artifact_share(os.path.join(str(tmpdir), 'share2')) as share2,\ + create_artifact_share(os.path.join(str(tmpdir), 'share3')) as share3: + cli.configure({ + 'artifacts': {'url': share1.repo, 'push': True}, + 'artifactdir': os.path.join(str(tmpdir), 'artifacts') + }) + + # Build autotools element with cache-buildtrees set via the + # cli. The artifact should be successfully pushed to the share1 remote + # and cached locally with an 'empty' buildtree digest, as it's not a + # dangling ref + result = cli.run(project=project, args=['--cache-buildtrees', 'never', 'build', element_name]) + assert result.exit_code == 0 + assert cli.get_element_state(project, element_name) == 'cached' + assert share1.has_artifact('test', element_name, cli.get_element_key(project, element_name)) + + # The extracted buildtree dir should be empty, as we set the config + # to not cache buildtrees + cache_key = cli.get_element_key(project, element_name) + elementdigest = share1.has_artifact('test', element_name, cache_key) + buildtreedir = os.path.join(str(tmpdir), 'artifacts', 'extract', 'test', 'autotools-amhello', + elementdigest.hash, 'buildtree') + assert os.path.isdir(buildtreedir) + assert not os.listdir(buildtreedir) + + # Delete the local cached artifacts, and assert the when pulled with --pull-buildtrees + # that is was cached in share1 as expected with an empty buildtree dir + shutil.rmtree(os.path.join(str(tmpdir), 'artifacts')) + assert cli.get_element_state(project, element_name) != 'cached' + result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name]) + assert element_name in result.get_pulled_elements() + assert os.path.isdir(buildtreedir) + assert not os.listdir(buildtreedir) + shutil.rmtree(os.path.join(str(tmpdir), 'artifacts')) + + # Assert that the default behaviour of pull to not include buildtrees on the artifact + # in share1 which was purposely cached with an empty one behaves as expected. As such the + # pulled artifact will have a dangling ref for the buildtree dir, regardless of content, + # leading to no buildtreedir being extracted + result = cli.run(project=project, args=['artifact', 'pull', element_name]) + assert element_name in result.get_pulled_elements() + assert not os.path.isdir(buildtreedir) + shutil.rmtree(os.path.join(str(tmpdir), 'artifacts')) + + # Repeat building the artifacts, this time with the default behaviour of caching buildtrees, + # as such the buildtree dir should not be empty + cli.configure({ + 'artifacts': {'url': share2.repo, 'push': True}, + 'artifactdir': os.path.join(str(tmpdir), 'artifacts') + }) + result = cli.run(project=project, args=['build', element_name]) + assert result.exit_code == 0 + assert cli.get_element_state(project, element_name) == 'cached' + assert share2.has_artifact('test', element_name, cli.get_element_key(project, element_name)) + + # Cache key will be the same however the digest hash will have changed as expected, so reconstruct paths + elementdigest = share2.has_artifact('test', element_name, cache_key) + buildtreedir = os.path.join(str(tmpdir), 'artifacts', 'extract', 'test', 'autotools-amhello', + elementdigest.hash, 'buildtree') + assert os.path.isdir(buildtreedir) + assert os.listdir(buildtreedir) is not None + + # Delete the local cached artifacts, and assert that when pulled with --pull-buildtrees + # that it was cached in share2 as expected with a populated buildtree dir + shutil.rmtree(os.path.join(str(tmpdir), 'artifacts')) + assert cli.get_element_state(project, element_name) != 'cached' + result = cli.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name]) + assert element_name in result.get_pulled_elements() + assert os.path.isdir(buildtreedir) + assert os.listdir(buildtreedir) is not None + shutil.rmtree(os.path.join(str(tmpdir), 'artifacts')) + + # Clarify that the user config option for cache-buildtrees works as the cli + # main option does. Point to share3 which does not have the artifacts cached to force + # a build + cli.configure({ + 'artifacts': {'url': share3.repo, 'push': True}, + 'artifactdir': os.path.join(str(tmpdir), 'artifacts'), + 'cache': {'cache-buildtrees': 'never'} + }) + result = cli.run(project=project, args=['build', element_name]) + assert result.exit_code == 0 + assert cli.get_element_state(project, element_name) == 'cached' + cache_key = cli.get_element_key(project, element_name) + elementdigest = share3.has_artifact('test', element_name, cache_key) + buildtreedir = os.path.join(str(tmpdir), 'artifacts', 'extract', 'test', 'autotools-amhello', + elementdigest.hash, 'buildtree') + assert os.path.isdir(buildtreedir) + assert not os.listdir(buildtreedir) diff --git a/tests/integration/build-tree.py b/tests/integration/shellbuildtrees.py index 98bb5b1e8..4d9d24e26 100644 --- a/tests/integration/build-tree.py +++ b/tests/integration/shellbuildtrees.py @@ -54,6 +54,29 @@ def test_buildtree_staged_forced_true(cli_integration, tmpdir, datafiles): @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +def test_buildtree_staged_warn_empty_cached(cli_integration, tmpdir, datafiles): + # Test that if we stage a cached and empty buildtree, we warn the user. + project = os.path.join(datafiles.dirname, datafiles.basename) + element_name = 'build-shell/buildtree.bst' + + # Switch to a temp artifact cache dir to ensure the artifact is rebuilt, + # caching an empty buildtree + cli_integration.configure({ + 'artifactdir': os.path.join(os.path.join(str(tmpdir), 'artifacts')) + }) + + res = cli_integration.run(project=project, args=['--cache-buildtrees', 'never', 'build', element_name]) + res.assert_success() + + res = cli_integration.run(project=project, args=[ + 'shell', '--build', '--use-buildtree', 'always', element_name, '--', 'cat', 'test' + ]) + res.assert_shell_error() + assert "Artifact contains an empty buildtree" in res.stderr + + +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') def test_buildtree_staged_if_available(cli_integration, tmpdir, datafiles): # Test that a build tree can be correctly detected. project = os.path.join(datafiles.dirname, datafiles.basename) @@ -106,6 +129,54 @@ def test_buildtree_from_failure(cli_integration, tmpdir, datafiles): assert 'Hi' in res.output +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +def test_buildtree_from_failure_option_never(cli_integration, tmpdir, datafiles): + + project = os.path.join(datafiles.dirname, datafiles.basename) + element_name = 'build-shell/buildtree-fail.bst' + + # Switch to a temp artifact cache dir to ensure the artifact is rebuilt, + # caching an empty buildtree + cli_integration.configure({ + 'artifactdir': os.path.join(os.path.join(str(tmpdir), 'artifacts')) + }) + + res = cli_integration.run(project=project, args=['--cache-buildtrees', 'never', 'build', element_name]) + res.assert_main_error(ErrorDomain.STREAM, None) + + res = cli_integration.run(project=project, args=[ + 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test' + ]) + res.assert_shell_error() + assert "Artifact contains an empty buildtree" in res.stderr + + +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +def test_buildtree_from_failure_option_failure(cli_integration, tmpdir, datafiles): + + project = os.path.join(datafiles.dirname, datafiles.basename) + element_name = 'build-shell/buildtree-fail.bst' + + # build with --cache-buildtrees set to 'failure', behaviour should match + # default behaviour (which is always) as the buildtree will explicitly have been + # cached with content. + cli_integration.configure({ + 'artifactdir': os.path.join(os.path.join(str(tmpdir), 'artifacts')) + }) + + res = cli_integration.run(project=project, args=['--cache-buildtrees', 'failure', 'build', element_name]) + res.assert_main_error(ErrorDomain.STREAM, None) + + res = cli_integration.run(project=project, args=[ + 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test' + ]) + res.assert_success() + assert "WARNING: using a buildtree from a failed build" in res.stderr + assert 'Hi' in res.output + + # Check that build shells work when pulled from a remote cache # This is to roughly simulate remote execution @pytest.mark.datafiles(DATA_DIR) |