import os import shutil import pytest from tests.testutils import cli_integration as cli, create_artifact_share from tests.testutils.integration import assert_contains from tests.testutils.site import HAVE_BWRAP, IS_LINUX from buildstream._exceptions import ErrorDomain, LoadErrorReason DATA_DIR = os.path.join( os.path.dirname(os.path.realpath(__file__)), "project" ) # Remove artifact cache & set cli.config value of pull-buildtrees # to false, which is the default user context. The cache has to be # cleared as just forcefully removing the refpath leaves dangling objects. def default_state(cli, tmpdir, share): shutil.rmtree(os.path.join(str(tmpdir), 'artifacts')) cli.configure({ 'artifacts': {'url': share.repo, 'push': False}, 'artifactdir': os.path.join(str(tmpdir), 'artifacts'), 'cache': {'pull-buildtrees': False}, }) # A test to capture the integration of the pullbuildtrees # behaviour, which by default is to not include the buildtree # directory of an element. @pytest.mark.integration @pytest.mark.datafiles(DATA_DIR) @pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux') def test_pullbuildtrees(cli, tmpdir, datafiles, integration_cache): project = os.path.join(datafiles.dirname, datafiles.basename) element_name = 'autotools/amhello.bst' # Create artifact shares for pull & push testing with create_artifact_share(os.path.join(str(tmpdir), 'share1')) as share1,\ create_artifact_share(os.path.join(str(tmpdir), 'share2')) as share2,\ create_artifact_share(os.path.join(str(tmpdir), 'share3')) as share3: cli.configure({ 'artifacts': {'url': share1.repo, 'push': True}, 'artifactdir': os.path.join(str(tmpdir), 'artifacts') }) # Build autotools element, checked pushed, delete local result = cli.run(project=project, args=['build', element_name]) assert result.exit_code == 0 assert cli.get_element_state(project, element_name) == 'cached' assert share1.has_artifact('test', element_name, cli.get_element_key(project, element_name)) default_state(cli, tmpdir, share1) # Pull artifact with default config, assert that pulling again # doesn't create a pull job, then assert with buildtrees user # config set creates a pull job. result = cli.run(project=project, args=['pull', element_name]) assert element_name in result.get_pulled_elements() result = cli.run(project=project, args=['pull', element_name]) assert element_name not in result.get_pulled_elements() cli.configure({'cache': {'pull-buildtrees': True}}) result = cli.run(project=project, args=['pull', element_name]) assert element_name in result.get_pulled_elements() default_state(cli, tmpdir, share1) # Pull artifact with default config, then assert that pulling # with buildtrees cli flag set creates a pull job. # Also assert that the buildtree is added to the artifact's # extract dir result = cli.run(project=project, args=['pull', element_name]) assert element_name in result.get_pulled_elements() elementdigest = share1.has_artifact('test', element_name, cli.get_element_key(project, element_name)) buildtreedir = os.path.join(str(tmpdir), 'artifacts', 'extract', 'test', 'autotools-amhello', elementdigest.hash, 'buildtree') assert not os.path.isdir(buildtreedir) result = cli.run(project=project, args=['--pull-buildtrees', 'pull', element_name]) assert element_name in result.get_pulled_elements() assert os.path.isdir(buildtreedir) default_state(cli, tmpdir, share1) # Pull artifact with pullbuildtrees set in user config, then assert # that pulling with the same user config doesn't creates a pull job, # or when buildtrees cli flag is set. cli.configure({'cache': {'pull-buildtrees': True}}) result = cli.run(project=project, args=['pull', element_name]) assert element_name in result.get_pulled_elements() result = cli.run(project=project, args=['pull', element_name]) assert element_name not in result.get_pulled_elements() result = cli.run(project=project, args=['--pull-buildtrees', 'pull', element_name]) assert element_name not in result.get_pulled_elements() default_state(cli, tmpdir, share1) # Pull artifact with default config and buildtrees cli flag set, then assert # that pulling with pullbuildtrees set in user config doesn't create a pull # job. result = cli.run(project=project, args=['--pull-buildtrees', 'pull', element_name]) assert element_name in result.get_pulled_elements() cli.configure({'cache': {'pull-buildtrees': True}}) result = cli.run(project=project, args=['pull', element_name]) assert element_name not in result.get_pulled_elements() default_state(cli, tmpdir, share1) # Assert that a partial build element (not containing a populated buildtree dir) # can't be pushed to an artifact share, then assert that a complete build element # can be. This will attempt a partial pull from share1 and then a partial push # to share2 result = cli.run(project=project, args=['pull', element_name]) assert element_name in result.get_pulled_elements() cli.configure({'artifacts': {'url': share2.repo, 'push': True}}) result = cli.run(project=project, args=['push', element_name]) assert element_name not in result.get_pushed_elements() assert not share2.has_artifact('test', element_name, cli.get_element_key(project, element_name)) # Assert that after pulling the missing buildtree the element artifact can be # successfully pushed to the remote. This will attempt to pull the buildtree # from share1 and then a 'complete' push to share2 cli.configure({'artifacts': {'url': share1.repo, 'push': False}}) result = cli.run(project=project, args=['--pull-buildtrees', 'pull', element_name]) assert element_name in result.get_pulled_elements() cli.configure({'artifacts': {'url': share2.repo, 'push': True}}) result = cli.run(project=project, args=['push', element_name]) assert element_name in result.get_pushed_elements() assert share2.has_artifact('test', element_name, cli.get_element_key(project, element_name)) default_state(cli, tmpdir, share1) # Assert that bst push will automatically attempt to pull a missing buildtree # if pull-buildtrees is set, however as share3 is the only defined remote and is empty, # assert that no element artifact buildtrees are pulled (no available remote buildtree) and thus the # artifact cannot be pushed. result = cli.run(project=project, args=['pull', element_name]) assert element_name in result.get_pulled_elements() cli.configure({'artifacts': {'url': share3.repo, 'push': True}}) result = cli.run(project=project, args=['--pull-buildtrees', 'push', element_name]) assert "Attempting to fetch missing artifact buildtrees" in result.stderr assert element_name not in result.get_pulled_elements() assert not os.path.isdir(buildtreedir) assert element_name not in result.get_pushed_elements() assert not share3.has_artifact('test', element_name, cli.get_element_key(project, element_name)) # Assert that if we add an extra remote that has the buildtree artfact cached, bst push will # automatically attempt to pull it and will be successful, leading to the full artifact being pushed # to the empty share3. This gives the ability to attempt push currently partial artifacts to a remote, # without exlipictly requiring a bst pull. cli.configure({'artifacts': [{'url': share1.repo, 'push': False}, {'url': share3.repo, 'push': True}]}) result = cli.run(project=project, args=['--pull-buildtrees', 'push', element_name]) assert "Attempting to fetch missing artifact buildtrees" in result.stderr assert element_name in result.get_pulled_elements() assert os.path.isdir(buildtreedir) assert element_name in result.get_pushed_elements() assert share3.has_artifact('test', element_name, cli.get_element_key(project, element_name)) # Ensure that only valid pull-buildtrees boolean options make it through the loading # process. @pytest.mark.parametrize("value,success", [ (True, True), (False, True), ("pony", False), ("1", False) ]) @pytest.mark.datafiles(DATA_DIR) def test_invalid_cache_pullbuildtrees(cli, datafiles, tmpdir, value, success): project = os.path.join(datafiles.dirname, datafiles.basename) cli.configure({ 'cache': { 'pull-buildtrees': value, } }) res = cli.run(project=project, args=['workspace', 'list']) if success: res.assert_success() else: res.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.ILLEGAL_COMPOSITE)