1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
|
#
# Copyright (C) 2017 Codethink Limited
# Copyright (C) 2018 Bloomberg Finance LP
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/>.
#
# Authors:
# Tristan Maat <tristan.maat@codethink.co.uk>
import multiprocessing
import os
import platform
import sys
import psutil
from .._exceptions import PlatformError, ImplError, SandboxError
from .. import utils
class Platform:
# Platform()
#
# A class to manage platform-specific details. Currently holds the
# sandbox factory as well as platform helpers.
#
# Args:
# force_sandbox (bool): Force bst to use a particular sandbox
#
def __init__(self, force_sandbox=None):
self.maximize_open_file_limit()
self._local_sandbox = None
self.dummy_reasons = []
self._setup_sandbox(force_sandbox)
def _setup_sandbox(self, force_sandbox):
sandbox_setups = {"dummy": self._setup_dummy_sandbox}
preferred_sandboxes = []
self._try_sandboxes(force_sandbox, sandbox_setups, preferred_sandboxes)
def _try_sandboxes(self, force_sandbox, sandbox_setups, preferred_sandboxes):
# Any sandbox from sandbox_setups can be forced by BST_FORCE_SANDBOX
# But if a specific sandbox is not forced then only `first class` sandbox are tried before
# falling back to the dummy sandbox.
# Where `first_class` sandboxes are those in preferred_sandboxes
if force_sandbox:
try:
sandbox_setups[force_sandbox]()
except KeyError:
raise PlatformError(
"Forced Sandbox is unavailable on this platform: BST_FORCE_SANDBOX"
" is set to {} but it is not available".format(force_sandbox)
)
except SandboxError as Error:
raise PlatformError(
"Forced Sandbox Error: BST_FORCE_SANDBOX"
" is set to {} but cannot be setup".format(force_sandbox),
detail=" and ".join(self.dummy_reasons),
) from Error
else:
for good_sandbox in preferred_sandboxes:
try:
sandbox_setups[good_sandbox]()
return
except SandboxError:
continue
except utils.ProgramNotFoundError:
continue
sandbox_setups["dummy"]()
def _check_sandbox(self, Sandbox):
try:
Sandbox.check_available()
except SandboxError as Error:
self.dummy_reasons += Sandbox._dummy_reasons
raise Error
@classmethod
def create_instance(cls):
# Meant for testing purposes and therefore hidden in the
# deepest corners of the source code. Try not to abuse this,
# please?
if os.getenv("BST_FORCE_SANDBOX"):
force_sandbox = os.getenv("BST_FORCE_SANDBOX")
else:
force_sandbox = None
if os.getenv("BST_FORCE_BACKEND"):
backend = os.getenv("BST_FORCE_BACKEND")
elif sys.platform.startswith("darwin"):
backend = "darwin"
elif sys.platform.startswith("linux"):
backend = "linux"
elif sys.platform == "win32":
backend = "win32"
else:
backend = "fallback"
if backend == "linux":
from .linux import Linux as PlatformImpl # pylint: disable=cyclic-import
elif backend == "darwin":
from .darwin import Darwin as PlatformImpl # pylint: disable=cyclic-import
elif backend == "win32":
from .win32 import Win32 as PlatformImpl # pylint: disable=cyclic-import
elif backend == "fallback":
from .fallback import Fallback as PlatformImpl # pylint: disable=cyclic-import
else:
raise PlatformError("No such platform: '{}'".format(backend))
return PlatformImpl(force_sandbox=force_sandbox)
def get_cpu_count(self, cap=None):
cpu_count = len(psutil.Process().cpu_affinity())
if cap is None:
return cpu_count
else:
return min(cpu_count, cap)
@staticmethod
def get_host_os():
return platform.uname().system
# canonicalize_arch():
#
# This returns the canonical, OS-independent architecture name
# or raises a PlatformError if the architecture is unknown.
#
@staticmethod
def canonicalize_arch(arch):
# Note that these are all expected to be lowercase, as we want a
# case-insensitive lookup. Windows can report its arch in ALLCAPS.
aliases = {
"aarch32": "aarch32",
"aarch64": "aarch64",
"aarch64-be": "aarch64-be",
"amd64": "x86-64",
"arm": "aarch32",
"armv8l": "aarch64",
"armv8b": "aarch64-be",
"i386": "x86-32",
"i486": "x86-32",
"i586": "x86-32",
"i686": "x86-32",
"power-isa-be": "power-isa-be",
"power-isa-le": "power-isa-le",
"ppc64": "power-isa-be",
"ppc64le": "power-isa-le",
"sparc": "sparc-v9",
"sparc64": "sparc-v9",
"sparc-v9": "sparc-v9",
"x86-32": "x86-32",
"x86-64": "x86-64",
}
try:
return aliases[arch.replace("_", "-").lower()]
except KeyError:
raise PlatformError("Unknown architecture: {}".format(arch))
# get_host_arch():
#
# This returns the architecture of the host machine. The possible values
# map from uname -m in order to be a OS independent list.
#
# Returns:
# (string): String representing the architecture
@staticmethod
def get_host_arch():
# get the hardware identifier from uname
uname_machine = platform.uname().machine
return Platform.canonicalize_arch(uname_machine)
# does_multiprocessing_start_require_pickling():
#
# Returns True if the multiprocessing start method will pickle arguments
# to new processes.
#
# Returns:
# (bool): Whether pickling is required or not
#
def does_multiprocessing_start_require_pickling(self):
# Note that if the start method has not been set before now, it will be
# set to the platform default by `get_start_method`.
return multiprocessing.get_start_method() != "fork"
##################################################################
# Sandbox functions #
##################################################################
# create_sandbox():
#
# Create a build sandbox suitable for the environment
#
# Args:
# args (dict): The arguments to pass to the sandbox constructor
# kwargs (file): The keyword arguments to pass to the sandbox constructor
#
# Returns:
# (Sandbox) A sandbox
#
def create_sandbox(self, *args, **kwargs):
raise ImplError("Platform {platform} does not implement create_sandbox()".format(platform=type(self).__name__))
def check_sandbox_config(self, config):
raise ImplError(
"Platform {platform} does not implement check_sandbox_config()".format(platform=type(self).__name__)
)
def maximize_open_file_limit(self):
# Need to set resources for _frontend/app.py as this is dependent on the platform
# SafeHardlinks FUSE needs to hold file descriptors for all processes in the sandbox.
# Avoid hitting the limit too quickly, by increasing it as far as we can.
# Import this late, as it is not available on Windows. Note that we
# could use `psutil.Process().rlimit` instead, but this would introduce
# a dependency on the `prlimit(2)` call, which seems to only be
# available on Linux. For more info:
# https://github.com/giampaolo/psutil/blob/cbf2bafbd33ad21ef63400d94cb313c299e78a45/psutil/_psutil_linux.c#L45
import resource
soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
if soft_limit != hard_limit:
resource.setrlimit(resource.RLIMIT_NOFILE, (hard_limit, hard_limit))
def _setup_dummy_sandbox(self):
raise ImplError(
"Platform {platform} does not implement _setup_dummy_sandbox()".format(platform=type(self).__name__)
)
|
