1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
|
#
# Copyright (C) 2017 Codethink Limited
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library. If not, see <http://www.gnu.org/licenses/>.
#
# Authors:
# Tristan Van Berkom <tristan.vanberkom@codethink.co.uk>
"""
compose - Compose the output of multiple elements
=================================================
This element creates a selective composition of its dependencies.
This is normally used at near the end of a pipeline to prepare
something for later deployment.
Since this element's output includes its dependencies, it may only
depend on elements as `build` type dependencies.
The default configuration and possible options are as such:
.. literalinclude:: ../../../src/buildstream/plugins/elements/compose.yaml
:language: yaml
"""
import os
from buildstream import Element, Scope
# Element implementation for the 'compose' kind.
class ComposeElement(Element):
# pylint: disable=attribute-defined-outside-init
# The compose element's output is its dependencies, so
# we must rebuild if the dependencies change even when
# not in strict build plans.
#
BST_STRICT_REBUILD = True
# Compose artifacts must never have indirect dependencies,
# so runtime dependencies are forbidden.
BST_FORBID_RDEPENDS = True
# This element ignores sources, so we should forbid them from being
# added, to reduce the potential for confusion
BST_FORBID_SOURCES = True
# This plugin has been modified to avoid the use of Sandbox.get_directory
BST_VIRTUAL_DIRECTORY = True
# Compose elements do not run any commands
BST_RUN_COMMANDS = False
def configure(self, node):
node.validate_keys(["integrate", "include", "exclude", "include-orphans"])
# We name this variable 'integration' only to avoid
# collision with the Element.integrate() method.
self.integration = node.get_bool("integrate")
self.include = node.get_str_list("include")
self.exclude = node.get_str_list("exclude")
self.include_orphans = node.get_bool("include-orphans")
def preflight(self):
pass
def get_unique_key(self):
key = {"integrate": self.integration, "include": sorted(self.include), "orphans": self.include_orphans}
if self.exclude:
key["exclude"] = sorted(self.exclude)
return key
def configure_sandbox(self, sandbox):
pass
def stage(self, sandbox):
pass
def assemble(self, sandbox):
require_split = self.include or self.exclude or not self.include_orphans
# Stage deps in the sandbox root
with self.timed_activity("Staging dependencies", silent_nested=True):
self.stage_dependency_artifacts(sandbox, Scope.BUILD)
manifest = set()
if require_split:
with self.timed_activity("Computing split", silent_nested=True):
for dep in self.dependencies(Scope.BUILD):
files = dep.compute_manifest(
include=self.include, exclude=self.exclude, orphans=self.include_orphans
)
manifest.update(files)
# Make a snapshot of all the files.
vbasedir = sandbox.get_virtual_directory()
modified_files = set()
removed_files = set()
added_files = set()
# Run any integration commands provided by the dependencies
# once they are all staged and ready
if self.integration:
with self.timed_activity("Integrating sandbox"):
if require_split:
# Make a snapshot of all the files before integration-commands are run.
snapshot = set(vbasedir.list_relative_paths())
vbasedir.mark_unmodified()
with sandbox.batch(0):
for dep in self.dependencies(Scope.BUILD):
dep.integrate(sandbox)
if require_split:
# Calculate added, modified and removed files
post_integration_snapshot = vbasedir.list_relative_paths()
modified_files = set(vbasedir.list_modified_paths())
basedir_contents = set(post_integration_snapshot)
for path in manifest:
if path in snapshot and path not in basedir_contents:
removed_files.add(path)
for path in basedir_contents:
if path not in snapshot:
added_files.add(path)
self.info(
"Integration modified {}, added {} and removed {} files".format(
len(modified_files), len(added_files), len(removed_files)
)
)
# The remainder of this is expensive, make an early exit if
# we're not being selective about what is to be included.
if not require_split:
return "/"
# Do we want to force include files which were modified by
# the integration commands, even if they were not added ?
#
manifest.update(added_files)
manifest.difference_update(removed_files)
# XXX We should be moving things outside of the build sandbox
# instead of into a subdir. The element assemble() method should
# support this in some way.
#
installdir = vbasedir.descend("buildstream", "install", create=True)
# We already saved the manifest for created files in the integration phase,
# now collect the rest of the manifest.
#
lines = []
if self.include:
lines.append("Including files from domains: " + ", ".join(self.include))
else:
lines.append("Including files from all domains")
if self.exclude:
lines.append("Excluding files from domains: " + ", ".join(self.exclude))
if self.include_orphans:
lines.append("Including orphaned files")
else:
lines.append("Excluding orphaned files")
detail = "\n".join(lines)
def import_filter(path):
return path in manifest
with self.timed_activity("Creating composition", detail=detail, silent_nested=True):
self.info("Composing {} files".format(len(manifest)))
installdir.import_files(vbasedir, filter_callback=import_filter, can_link=True)
# And we're done
return os.path.join(os.sep, "buildstream", "install")
# Plugin entry point
def setup():
return ComposeElement
|
