1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
|
#!/usr/bin/env python3
#
# Copyright (C) 2016 Codethink Limited
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/>.
#
# Authors:
# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
"""Context for running BuildStream pipelines
The :class:`.Context` object holds all of the user preferences
and context for a given invocation of BuildStream.
This is a collection of data from configuration files and command
line arguments and consists of information such as where to store
logs and artifacts, where to perform builds and cache downloaded sources,
verbosity levels and basically anything pertaining to the context
in which BuildStream was invoked.
"""
import os
import hashlib
import pickle
from collections import deque, Mapping
from . import _site
from . import _yaml
from . import utils
from . import LoadError, LoadErrorReason
from ._profile import Topics, profile_start, profile_end
class Context():
"""Context of how BuildStream was invoked
Args:
host_arch (string): The desired architecture on which to run the build
target_arch (string): The machine on which the results of the build should execute
"""
def __init__(self, host_arch, target_arch=None):
self.config_origin = None
"""Filename indicating which configuration file was used, or None for the defaults"""
self.host_arch = host_arch
"""The desired architecture on which to run the build"""
self.target_arch = target_arch or host_arch
"""The machine on which the results of the build should execute"""
self.strict_build_plan = True
"""Whether elements must be rebuilt when their dependencies have changed"""
self.sourcedir = None
"""The directory where various sources are stored"""
self.builddir = None
"""The directory where build sandboxes will be created"""
self.artifactdir = None
"""The local binary artifact cache directory"""
self.artifact_pull = None
"""The URL from which to download prebuilt artifacts"""
self.artifact_push = None
"""The URL to upload built artifacts to"""
self.artifact_push_port = 22
"""The port number for pushing artifacts over ssh"""
self.logdir = None
"""The directory to store build logs"""
self.log_key_length = 0
"""The abbreviated cache key length to display in the UI"""
self.log_debug = False
"""Whether debug mode is enabled"""
self.log_verbose = False
"""Whether verbose mode is enabled"""
self.log_error_lines = 0
"""Maximum number of lines to print from build logs"""
self.log_message_lines = 0
"""Maximum number of lines to print in the master log for a detailed message"""
self.log_element_format = None
"""Format string for printing the pipeline at startup time"""
self.sched_fetchers = 4
"""Maximum number of fetch or refresh tasks"""
self.sched_builders = 4
"""Maximum number of build tasks"""
self.sched_pushers = 4
"""Maximum number of push tasks"""
self.sched_network_retries = 2
"""Maximum number of retries for network tasks"""
self.sched_error_action = 'continue'
"""What to do when a build fails in non interactive mode"""
# Make sure the XDG vars are set in the environment before loading anything
self._init_xdg()
# Private variables
self._cache_key = None
self._message_handler = None
self._message_depth = deque()
def load(self, config=None):
"""Loads the configuration files
Args:
config (filename): The user specified configuration file, if any
Raises:
:class:`.LoadError`
This will first load the BuildStream default configuration and then
override that configuration with the configuration file indicated
by *config*, if any was specified.
"""
profile_start(Topics.LOAD_CONTEXT, 'load')
# If a specific config file is not specified, default to trying
# a $XDG_CONFIG_HOME/buildstream.conf file
#
if not config:
default_config = os.path.join(os.environ['XDG_CONFIG_HOME'],
'buildstream.conf')
if os.path.exists(default_config):
config = default_config
# Load default config
#
defaults = _yaml.load(_site.default_user_config)
if config:
self.config_origin = os.path.abspath(config)
user_config = _yaml.load(config)
_yaml.composite(defaults, user_config, typesafe=True)
self.strict_build_plan = _yaml.node_get(defaults, bool, 'strict')
for dir in ['sourcedir', 'builddir', 'artifactdir', 'logdir']:
# Allow the ~ tilde expansion and any environment variables in
# path specification in the config files.
#
path = _yaml.node_get(defaults, str, dir)
path = os.path.expanduser(path)
path = os.path.expandvars(path)
setattr(self, dir, path)
# Load artifact share configuration
artifacts = _yaml.node_get(defaults, Mapping, 'artifacts')
self.artifact_pull = _yaml.node_get(artifacts, str, 'pull-url', default_value='') or None
self.artifact_push = _yaml.node_get(artifacts, str, 'push-url', default_value='') or None
self.artifact_push_port = _yaml.node_get(artifacts, int, 'push-port', default_value=22)
# Load logging config
logging = _yaml.node_get(defaults, Mapping, 'logging')
self.log_key_length = _yaml.node_get(logging, int, 'key-length')
self.log_debug = _yaml.node_get(logging, bool, 'debug')
self.log_verbose = _yaml.node_get(logging, bool, 'verbose')
self.log_error_lines = _yaml.node_get(logging, int, 'error-lines')
self.log_message_lines = _yaml.node_get(logging, int, 'message-lines')
self.log_element_format = _yaml.node_get(logging, str, 'element-format')
# Load scheduler config
scheduler = _yaml.node_get(defaults, Mapping, 'scheduler')
self.sched_error_action = _yaml.node_get(scheduler, str, 'on-error')
self.sched_fetchers = _yaml.node_get(scheduler, int, 'fetchers')
self.sched_builders = _yaml.node_get(scheduler, int, 'builders')
self.sched_pushers = _yaml.node_get(scheduler, int, 'pushers')
self.sched_network_retries = _yaml.node_get(scheduler, int, 'network-retries')
profile_end(Topics.LOAD_CONTEXT, 'load')
valid_actions = ['continue', 'quit']
if self.sched_error_action not in valid_actions:
provenance = _yaml.node_get_provenance(scheduler, 'on-error')
raise LoadError(LoadErrorReason.INVALID_DATA,
"{}: on-error should be one of: {}".format(
provenance, ", ".join(valid_actions)))
#############################################################
# Private Methods used in BuildStream #
#############################################################
# _set_message_handler()
#
# Sets the handler for any status messages propagated through
# the context.
#
# The message handler should have the same signature as
# the _message() method
def _set_message_handler(self, handler):
self._message_handler = handler
# _get_cache_key():
#
# Returns the cache key, calculating it if necessary
#
# Returns:
# (str): A hex digest cache key for the Context
#
def _get_cache_key(self):
if self._cache_key is None:
# Anything that alters the build goes into the unique key
self._cache_key = utils._generate_key({
'host-arch': self.host_arch,
'target-arch': self.target_arch
})
return self._cache_key
# _push_message_depth() / _pop_message_depth()
#
# For status messages, send the depth of timed
# activities inside a given task through the message
#
def _push_message_depth(self, silent_nested):
self._message_depth.appendleft(silent_nested)
def _pop_message_depth(self):
assert(self._message_depth)
self._message_depth.popleft()
def _silent_messages(self):
for silent in self._message_depth:
if silent:
return True
return False
# _message():
#
# Proxies a message back to the caller, this is the central
# point through which all messages pass.
#
# Args:
# message: A Message object
#
def _message(self, message):
# Tag message only once
if message.depth is None:
message.depth = len(list(self._message_depth))
# Send it off to the log handler (can be the frontend,
# or it can be the child task which will log and propagate
# to the frontend)
assert(self._message_handler)
self._message_handler(message, context=self)
return
# Force the resolved XDG variables into the environment,
# this is so that they can be used directly to specify
# preferred locations of things from user configuration
# files.
def _init_xdg(self):
if not os.environ.get('XDG_CACHE_HOME'):
os.environ['XDG_CACHE_HOME'] = os.path.expanduser('~/.cache')
if not os.environ.get('XDG_CONFIG_HOME'):
os.environ['XDG_CONFIG_HOME'] = os.path.expanduser('~/.config')
if not os.environ.get('XDG_DATA_HOME'):
os.environ['XDG_DATA_HOME'] = os.path.expanduser('~/.local/share')
|
