diff options
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/frontend/buildcheckout.py | 64 | ||||
| -rw-r--r-- | tests/integration/shell.py | 50 | ||||
| -rw-r--r-- | tests/integration/shellbuildtrees.py | 29 | ||||
| -rw-r--r-- | tests/remoteexecution/partial.py | 25 |
4 files changed, 167 insertions, 1 deletions
diff --git a/tests/frontend/buildcheckout.py b/tests/frontend/buildcheckout.py index 556bf811c..97bce91a7 100644 --- a/tests/frontend/buildcheckout.py +++ b/tests/frontend/buildcheckout.py @@ -5,6 +5,7 @@ import os import tarfile import hashlib import subprocess +import re import pytest @@ -12,8 +13,9 @@ from buildstream.testing import cli # pylint: disable=unused-import from buildstream.testing._utils.site import IS_WINDOWS from buildstream import _yaml from buildstream._exceptions import ErrorDomain, LoadErrorReason +from buildstream import utils -from tests.testutils import generate_junction, yaml_file_get_provenance +from tests.testutils import generate_junction, yaml_file_get_provenance, create_artifact_share from . import configure_project @@ -823,3 +825,63 @@ def test_build_junction_transitive_short_notation_with_junction(cli, tmpdir, dat # cross-junction elements is not allowed. result = cli.run(project=project, args=['build', 'junction-dep.bst']) result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) + + +# Should check that after a build we have partial artifacts locally, but should +# then attempt to fetch them when doing a artifact checkout +@pytest.mark.datafiles(DATA_DIR) +def test_partial_artifact_checkout_fetch(cli, datafiles, tmpdir): + project = str(datafiles) + build_elt = 'import-bin.bst' + checkout_dir = os.path.join(str(tmpdir), 'checkout') + + with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + + cli.configure({'artifacts': { + 'url': share.repo, + 'push': True + }}) + + result = cli.run(project=project, args=['build', build_elt]) + result.assert_success() + + # A push artifact cache means we have to pull to push to them, so + # delete some blobs from that CAS such that we have to fetch + digest = utils.sha256sum(os.path.join(project, 'files', 'bin-files', 'usr', 'bin', 'hello')) + objpath = os.path.join(cli.directory, 'cas', 'objects', digest[:2], digest[2:]) + os.unlink(objpath) + + # Verify that the build-only dependency is not (complete) in the local cache + result = cli.run(project=project, args=[ + 'artifact', 'checkout', build_elt, + '--directory', checkout_dir]) + result.assert_main_error(ErrorDomain.STREAM, 'uncached-checkout-attempt') + + # Verify that the pull method fetches relevant artifacts in order to stage + result = cli.run(project=project, args=[ + 'artifact', 'checkout', '--pull', build_elt, + '--directory', checkout_dir]) + result.assert_success() + + # should have pulled whatever was deleted previous + assert 'import-bin.bst' in result.get_pulled_elements() + + +@pytest.mark.datafiles(DATA_DIR) +def test_partial_checkout_fail(tmpdir, datafiles, cli): + project = str(datafiles) + build_elt = 'import-bin.bst' + checkout_dir = os.path.join(str(tmpdir), 'checkout') + + with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + + cli.configure({'artifacts': { + 'url': share.repo, + 'push': True + }}) + + res = cli.run(project=project, args=[ + 'artifact', 'checkout', '--pull', build_elt, '--directory', + checkout_dir]) + res.assert_main_error(ErrorDomain.STREAM, 'uncached-checkout-attempt') + assert re.findall(r'Remote \((\S+)\) does not have artifact (\S+) cached', res.stderr) diff --git a/tests/integration/shell.py b/tests/integration/shell.py index d0c9f1f99..f7de3e462 100644 --- a/tests/integration/shell.py +++ b/tests/integration/shell.py @@ -7,6 +7,10 @@ import pytest from buildstream import _yaml from buildstream.testing import cli_integration as cli # pylint: disable=unused-import from buildstream.testing._utils.site import HAVE_SANDBOX +from buildstream._exceptions import ErrorDomain +from buildstream import utils + +from tests.testutils import create_artifact_share pytestmark = pytest.mark.integration @@ -391,3 +395,49 @@ def test_integration_external_workspace(cli, tmpdir_factory, datafiles, build_sh command.extend([element_name, '--', 'true']) result = cli.run(project=project, cwd=workspace_dir, args=command) result.assert_success() + + +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +def test_integration_partial_artifact(cli, datafiles, tmpdir, integration_cache): + + project = str(datafiles) + element_name = 'autotools/amhello.bst' + + # push to an artifact server so we can pull from it later. + with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + cli.configure({'artifacts': { + 'url': share.repo, + 'push': True + }}) + result = cli.run(project=project, args=['build', element_name]) + result.assert_success() + + # If the build is cached then it might not push to the artifact cache + result = cli.run(project=project, args=['artifact', 'push', element_name]) + result.assert_success() + + result = cli.run(project=project, args=['shell', element_name]) + result.assert_success() + + # do a checkout and get the digest of the hello binary. + result = cli.run(project=project, args=[ + 'artifact', 'checkout', '--deps', 'none', + '--directory', os.path.join(str(tmpdir), 'tmp'), + 'autotools/amhello.bst']) + result.assert_success() + digest = utils.sha256sum(os.path.join(str(tmpdir), 'tmp', 'usr', 'bin', 'hello')) + + # Remove the binary from the CAS + cachedir = cli.config['cachedir'] + objpath = os.path.join(cachedir, 'cas', 'objects', digest[:2], digest[2:]) + os.unlink(objpath) + + # check shell doesn't work + result = cli.run(project=project, args=['shell', element_name, '--', 'hello']) + result.assert_main_error(ErrorDomain.APP, None) + + # check the artifact gets completed with '--pull' specified + result = cli.run(project=project, args=['shell', '--pull', element_name, '--', 'hello']) + result.assert_success() + assert 'autotools/amhello.bst' in result.get_pulled_elements() diff --git a/tests/integration/shellbuildtrees.py b/tests/integration/shellbuildtrees.py index b48f4afe7..a1eecb1eb 100644 --- a/tests/integration/shellbuildtrees.py +++ b/tests/integration/shellbuildtrees.py @@ -305,3 +305,32 @@ def test_buildtree_options(cli, tmpdir, datafiles): assert 'Hi' in res.output assert "buildtree is not cached locally, will attempt to pull from available remotes" in res.stderr assert 'Attempting to fetch missing artifact buildtree' in res.stderr + + +# Tests running pull and pull-buildtree options at the same time. +@pytest.mark.datafiles(DATA_DIR) +@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox') +def test_pull_buildtree_pulled(cli, tmpdir, datafiles): + project = str(datafiles) + element_name = 'build-shell/buildtree.bst' + + with create_artifact_share(os.path.join(str(tmpdir), 'artifactshare')) as share: + # Build the element to push it to cache + cli.configure({ + 'artifacts': {'url': share.repo, 'push': True} + }) + result = cli.run(project=project, args=['--cache-buildtrees', 'always', 'build', element_name]) + result.assert_success() + assert cli.get_element_state(project, element_name) == 'cached' + + # Discard the cache + shutil.rmtree(str(os.path.join(str(tmpdir), 'cache', 'cas'))) + shutil.rmtree(str(os.path.join(str(tmpdir), 'cache', 'artifacts'))) + assert cli.get_element_state(project, element_name) != 'cached' + + # Check it's using the cached build tree + res = cli.run(project=project, args=[ + '--pull-buildtrees', 'shell', '--build', element_name, '--pull', + '--use-buildtree', 'always', '--', 'cat', 'test' + ]) + res.assert_success() diff --git a/tests/remoteexecution/partial.py b/tests/remoteexecution/partial.py index fd7b11cdb..a684bda18 100644 --- a/tests/remoteexecution/partial.py +++ b/tests/remoteexecution/partial.py @@ -8,6 +8,8 @@ from buildstream._exceptions import ErrorDomain from buildstream.testing import cli_remote_execution as cli # pylint: disable=unused-import from buildstream.testing.integration import assert_contains +from tests.testutils.artifactshare import create_artifact_share + pytestmark = pytest.mark.remoteexecution @@ -44,3 +46,26 @@ def test_build_dependency_partial_local_cas(cli, datafiles): result = cli.run(project=project, args=['artifact', 'checkout', builddep_element_name, '--directory', builddep_checkout]) result.assert_main_error(ErrorDomain.STREAM, 'uncached-checkout-attempt') + + +@pytest.mark.datafiles(DATA_DIR) +def test_build_partial_push(cli, tmpdir, datafiles): + project = str(datafiles) + share_dir = os.path.join(str(tmpdir), "artifactshare") + element_name = 'no-runtime-deps.bst' + builddep_element_name = 'autotools/amhello.bst' + + with create_artifact_share(share_dir) as share: + + services = cli.ensure_services() + assert set(services) == set(['action-cache', 'execution', 'storage']) + + cli.config['artifacts'] = { + 'url': share.repo, + 'push': True, + } + + res = cli.run(project=project, args=['build', element_name]) + res.assert_success() + + assert builddep_element_name in res.get_pushed_elements() |