1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
|
from contextlib import contextmanager
import os
import pytest
import shutil
import tempfile
from tests.testutils import create_artifact_share
from tests.testutils.site import HAVE_SANDBOX
from buildstream import utils
from buildstream.plugintestutils import cli, cli_integration as cli2
from buildstream._cas import CASCache
from buildstream._exceptions import ErrorDomain, LoadErrorReason
DATA_DIR = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"project"
)
# Remove artifact cache & set cli.config value of pull-buildtrees
# to false, which is the default user context. The cache has to be
# cleared as just forcefully removing the refpath leaves dangling objects.
def default_state(cli, tmpdir, share):
shutil.rmtree(os.path.join(str(tmpdir), 'cas'))
cli.configure({
'artifacts': {'url': share.repo, 'push': False},
'cachedir': str(tmpdir),
'cache': {'pull-buildtrees': False},
})
# A test to capture the integration of the pullbuildtrees
# behaviour, which by default is to not include the buildtree
# directory of an element.
@pytest.mark.integration
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
def test_pullbuildtrees(cli2, tmpdir, datafiles):
project = str(datafiles)
element_name = 'autotools/amhello.bst'
# Create artifact shares for pull & push testing
with create_artifact_share(os.path.join(str(tmpdir), 'share1')) as share1,\
create_artifact_share(os.path.join(str(tmpdir), 'share2')) as share2,\
create_artifact_share(os.path.join(str(tmpdir), 'share3')) as share3:
cli2.configure({
'artifacts': {'url': share1.repo, 'push': True},
'cachedir': str(tmpdir),
'cache': {'cache-buildtrees': 'always'},
})
@contextmanager
def cas_extract_buildtree(digest):
extractdir = tempfile.mkdtemp(prefix="tmp", dir=str(tmpdir))
try:
cas = CASCache(str(tmpdir))
cas.checkout(extractdir, digest)
yield os.path.join(extractdir, 'buildtree')
except FileNotFoundError:
yield None
finally:
utils._force_rmtree(extractdir)
# Build autotools element, checked pushed, delete local
result = cli2.run(project=project, args=['build', element_name])
assert result.exit_code == 0
assert cli2.get_element_state(project, element_name) == 'cached'
assert share1.has_artifact('test', element_name, cli2.get_element_key(project, element_name))
default_state(cli2, tmpdir, share1)
# Pull artifact with default config, assert that pulling again
# doesn't create a pull job, then assert with buildtrees user
# config set creates a pull job.
result = cli2.run(project=project, args=['artifact', 'pull', element_name])
assert element_name in result.get_pulled_elements()
result = cli2.run(project=project, args=['artifact', 'pull', element_name])
assert element_name not in result.get_pulled_elements()
cli2.configure({'cache': {'pull-buildtrees': True}})
result = cli2.run(project=project, args=['artifact', 'pull', element_name])
assert element_name in result.get_pulled_elements()
default_state(cli2, tmpdir, share1)
# Pull artifact with default config, then assert that pulling
# with buildtrees cli flag set creates a pull job.
# Also assert that the buildtree is added to the local CAS.
result = cli2.run(project=project, args=['artifact', 'pull', element_name])
assert element_name in result.get_pulled_elements()
elementdigest = share1.has_artifact('test', element_name, cli2.get_element_key(project, element_name))
with cas_extract_buildtree(elementdigest) as buildtreedir:
assert not buildtreedir
result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
assert element_name in result.get_pulled_elements()
with cas_extract_buildtree(elementdigest) as buildtreedir:
assert os.path.isdir(buildtreedir)
default_state(cli2, tmpdir, share1)
# Pull artifact with pullbuildtrees set in user config, then assert
# that pulling with the same user config doesn't creates a pull job,
# or when buildtrees cli flag is set.
cli2.configure({'cache': {'pull-buildtrees': True}})
result = cli2.run(project=project, args=['artifact', 'pull', element_name])
assert element_name in result.get_pulled_elements()
result = cli2.run(project=project, args=['artifact', 'pull', element_name])
assert element_name not in result.get_pulled_elements()
result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
assert element_name not in result.get_pulled_elements()
default_state(cli2, tmpdir, share1)
# Pull artifact with default config and buildtrees cli flag set, then assert
# that pulling with pullbuildtrees set in user config doesn't create a pull
# job.
result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
assert element_name in result.get_pulled_elements()
cli2.configure({'cache': {'pull-buildtrees': True}})
result = cli2.run(project=project, args=['artifact', 'pull', element_name])
assert element_name not in result.get_pulled_elements()
default_state(cli2, tmpdir, share1)
# Assert that a partial build element (not containing a populated buildtree dir)
# can't be pushed to an artifact share, then assert that a complete build element
# can be. This will attempt a partial pull from share1 and then a partial push
# to share2
result = cli2.run(project=project, args=['artifact', 'pull', element_name])
assert element_name in result.get_pulled_elements()
cli2.configure({'artifacts': {'url': share2.repo, 'push': True}})
result = cli2.run(project=project, args=['artifact', 'push', element_name])
assert element_name not in result.get_pushed_elements()
assert not share2.has_artifact('test', element_name, cli2.get_element_key(project, element_name))
# Assert that after pulling the missing buildtree the element artifact can be
# successfully pushed to the remote. This will attempt to pull the buildtree
# from share1 and then a 'complete' push to share2
cli2.configure({'artifacts': {'url': share1.repo, 'push': False}})
result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'pull', element_name])
assert element_name in result.get_pulled_elements()
cli2.configure({'artifacts': {'url': share2.repo, 'push': True}})
result = cli2.run(project=project, args=['artifact', 'push', element_name])
assert element_name in result.get_pushed_elements()
assert share2.has_artifact('test', element_name, cli2.get_element_key(project, element_name))
default_state(cli2, tmpdir, share1)
# Assert that bst push will automatically attempt to pull a missing buildtree
# if pull-buildtrees is set, however as share3 is the only defined remote and is empty,
# assert that no element artifact buildtrees are pulled (no available remote buildtree) and thus the
# artifact cannot be pushed.
result = cli2.run(project=project, args=['artifact', 'pull', element_name])
assert element_name in result.get_pulled_elements()
cli2.configure({'artifacts': {'url': share3.repo, 'push': True}})
result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'push', element_name])
assert "Attempting to fetch missing artifact buildtrees" in result.stderr
assert element_name not in result.get_pulled_elements()
with cas_extract_buildtree(elementdigest) as buildtreedir:
assert not buildtreedir
assert element_name not in result.get_pushed_elements()
assert not share3.has_artifact('test', element_name, cli2.get_element_key(project, element_name))
# Assert that if we add an extra remote that has the buildtree artfact cached, bst push will
# automatically attempt to pull it and will be successful, leading to the full artifact being pushed
# to the empty share3. This gives the ability to attempt push currently partial artifacts to a remote,
# without exlipictly requiring a bst pull.
cli2.configure({'artifacts': [{'url': share1.repo, 'push': False}, {'url': share3.repo, 'push': True}]})
result = cli2.run(project=project, args=['--pull-buildtrees', 'artifact', 'push', element_name])
assert "Attempting to fetch missing artifact buildtrees" in result.stderr
assert element_name in result.get_pulled_elements()
with cas_extract_buildtree(elementdigest) as buildtreedir:
assert os.path.isdir(buildtreedir)
assert element_name in result.get_pushed_elements()
assert share3.has_artifact('test', element_name, cli2.get_element_key(project, element_name))
# Ensure that only valid pull-buildtrees boolean options make it through the loading
# process.
@pytest.mark.parametrize("value,success", [
(True, True),
(False, True),
("pony", False),
("1", False)
])
@pytest.mark.datafiles(DATA_DIR)
def test_invalid_cache_pullbuildtrees(cli, datafiles, value, success):
project = str(datafiles)
cli.configure({
'cache': {
'pull-buildtrees': value,
}
})
res = cli.run(project=project, args=['workspace', 'list'])
if success:
res.assert_success()
else:
res.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)
|
