summaryrefslogtreecommitdiff
path: root/morphlib/plugins/cross-bootstrap_plugin.py
blob: 2b5c8911cf1399e631e23f680b326fcb98b9a67e (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
# Copyright (C) 2013-2014  Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.

import cliapp
import logging
import os.path
import re
import tarfile
import traceback

import morphlib

driver_header = '''#!/bin/sh
echo "Morph native bootstrap script"
echo "Generated by Morph version %s\n"

set -eu

export PATH=$PATH:/tools/bin:/tools/sbin
export SRCDIR=/src

''' % morphlib.__version__

driver_footer = '''

echo "Complete!"
'''

def escape_source_name(source):
    repo_name = source.repo.original_name
    ref = source.original_ref
    source_name = '%s__%s' % (repo_name, ref)
    return re.sub('[:/]', '_', source_name)

# Most of this is ripped from RootfsTarballBuilder, and should be reconciled
# with it.
class BootstrapSystemBuilder(morphlib.builder.BuilderBase):
    '''Build a bootstrap system tarball
    
       The bootstrap system image contains a minimal cross-compiled toolchain
       and a set of extracted sources for the rest of the system, with shell
       scripts to run the required morphology commands. This allows new
       architectures to be bootstrapped without needing to build Python, Git,
       Perl and all of Morph's other dependencies first.
    '''

    def build_and_cache(self):
        with self.build_watch('overall-build'):
            for system_name, artifact in self.source.artifacts.iteritems():
                handle = self.local_artifact_cache.put(artifact)
                fs_root = self.staging_area.destdir(self.source)
                try:
                    self.unpack_binary_chunks(fs_root)
                    self.unpack_sources(fs_root)
                    self.write_build_script(fs_root)
                    self.create_tarball(handle, fs_root, system_name)
                except BaseException, e:
                    logging.error(traceback.format_exc())
                    self.app.status(msg='Error while building bootstrap image',
                                    error=True)
                    handle.abort()
                    raise

                handle.close()

        self.save_build_times()
        return self.source.artifacts.items()

    def unpack_binary_chunks(self, dest):
        cache = self.local_artifact_cache
        for chunk_source in self.source.cross_sources:
            for chunk_artifact in chunk_source.artifacts.itervalues():
                with cache.get(chunk_artifact) as chunk_file:
                    try:
                        morphlib.bins.unpack_binary_from_file(chunk_file, dest)
                    except BaseException, e:
                        self.app.status(
                            msg='Error unpacking binary chunk %(name)s',
                            name=chunk_artifact.name,
                            error=True)
                        raise

    def unpack_sources(self, path):
        # Multiple chunks sources may be built from the same repo ('linux'
        # and 'linux-api-headers' are a good example), so we check out the
        # sources once per repository.
        #
        # It might be neater to build these as "source artifacts" individually,
        # but that would waste huge amounts of space in the artifact cache.
        for s in self.source.native_sources:
            escaped_source = escape_source_name(s)
            source_dir = os.path.join(path, 'src', escaped_source)
            if not os.path.exists(source_dir):
                os.makedirs(source_dir)
                morphlib.builder.extract_sources(
                    self.app, self.repo_cache, s.repo, s.sha1, source_dir)

            name = s.name
            chunk_script = os.path.join(path, 'src', 'build-%s' % name)
            with morphlib.savefile.SaveFile(chunk_script, 'w') as f:
                self.write_chunk_build_script(s, f)
            os.chmod(chunk_script, 0777)

    def write_build_script(self, path):
        '''Output a script to run build on the bootstrap target'''

        driver_script = os.path.join(path, 'native-bootstrap')
        with morphlib.savefile.SaveFile(driver_script, 'w') as f:
            f.write(driver_header)

            f.write('echo Setting up build environment...\n')
            for k,v in self.staging_area.env.iteritems():
                if k != 'PATH':
                    f.write('export %s="%s"\n' % (k, v))

            for s in self.source.native_sources:
                name = s.name
                f.write('\necho Building %s\n' % name)
                f.write('mkdir /%s.inst\n' % name)
                f.write('env DESTDIR=/%s.inst $SRCDIR/build-%s\n'
                        % (name, name))
                f.write('echo Installing %s\n' % name)
                f.write('(cd /%s.inst; find . | cpio -umdp /)\n' % name)
                f.write('if [ -e /sbin/ldconfig ]; then /sbin/ldconfig; fi\n')

            f.write(driver_footer)
        os.chmod(driver_script, 0777)

    def write_chunk_build_script(self, source, f):
        m = source.morphology
        f.write('#!/bin/sh\n')
        f.write('# Build script generated by morph\n')
        f.write('set -e\n')
        f.write('chunk_name=%s\n' % m['name'])

        repo = escape_source_name(source)
        f.write('cp -a $SRCDIR/%s $DESTDIR/$chunk_name.build\n' % repo)
        f.write('cd $DESTDIR/$chunk_name.build\n')
        f.write('export PREFIX=%s\n' % source.prefix)

        bs = morphlib.buildsystem.lookup_build_system(m['build-system'])

        # FIXME: merge some of this with Morphology
        steps = [
            ('pre-configure', False),
            ('configure', False),
            ('post-configure', False),
            ('pre-build', True),
            ('build', True),
            ('post-build', True),
            ('pre-test', False),
            ('test', False),
            ('post-test', False),
            ('pre-install', False),
            ('install', False),
            ('post-install', False),
        ]
        
        for step, in_parallel in steps:
            key = '%s-commands' % step
            cmds = m[key]
            for cmd in cmds:
                f.write('(')
                if in_parallel:
                    max_jobs = m['max-jobs']
                    if max_jobs is None:
                        max_jobs = self.max_jobs
                    f.write('export MAKEFLAGS=-j%s; ' % max_jobs)
                f.write('set -e; %s) || exit 1\n' % cmd)

        f.write('rm -Rf $DESTDIR/$chunk_name.build')

    def create_tarball(self, handle, fs_root, system_name):
        unslashy_root = fs_root[1:]
        def uproot_info(info):
            info.name = os.path.relpath(info.name, unslashy_root)
            if info.islnk():
                info.linkname = os.path.relpath(info.linkname, unslashy_root)
            return info

        tar = tarfile.TarFile.gzopen(fileobj=handle, mode="w",
                                     compresslevel=1,
                                     name=system_name)
        self.app.status(msg='Constructing tarball of root filesystem',
                        chatty=True)
        tar.add(fs_root, recursive=True, filter=uproot_info)
        tar.close()


class CrossBootstrapPlugin(cliapp.Plugin):

    def enable(self):
        self.app.add_subcommand('cross-bootstrap',
                                self.cross_bootstrap,
                                arg_synopsis='TARGET REPO REF SYSTEM-MORPH')

    def disable(self):
        pass

    def cross_bootstrap(self, args):
        '''Cross-bootstrap a system from a different architecture.'''

        # A brief overview of this process: the goal is to native build as much
        # of the system as possible because that's easier, but in order to do
        # so we need at least 'build-essential'. 'morph cross-bootstrap' will
        # cross-build any bootstrap-mode chunks in the given system and
        # will then prepare a large rootfs tarball which, when booted, will
        # build the rest of the chunks in the system using the cross-built
        # build-essential.
        #
        # This approach saves us from having to run Morph, Git, Python, Perl,
        # or anything else complex and difficult to cross-build on the target
        # until it is bootstrapped. The user of this command needs to provide
        # a kernel and handle booting the system themselves (the generated
        # tarball contains a /bin/sh that can be used as 'init'.
        #
        # This function is a variant of the BuildCommand() class in morphlib.

        # To do: make it work on a system branch instead of repo/ref/morph
        # triplet.

        if len(args) < 4:
            raise cliapp.AppException(
                'cross-bootstrap requires 4 arguments: target archicture, and '
                'repo, ref and and name of the system morphology')

        arch = args[0]
        root_repo, ref, system_name = args[1:4]

        if arch not in morphlib.valid_archs:
            raise morphlib.Error('Unsupported architecture "%s"' % arch)

        # Get system artifact

        build_env = morphlib.buildenvironment.BuildEnvironment(
            self.app.settings, arch)
        build_command = morphlib.buildcommand.BuildCommand(self.app, build_env)

        morph_name = morphlib.util.sanitise_morphology_path(system_name)
        srcpool = build_command.create_source_pool(root_repo, ref, morph_name)

        # FIXME: this is a quick fix in order to get it working for
        # Baserock 13 release, it is not a reasonable fix
        def validate(self, root_artifact):
            root_arch = root_artifact.source.morphology['arch']
            target_arch = arch
            if root_arch != target_arch:
                raise morphlib.Error(
                    'Target architecture is %s '
                    'but the system architecture is %s'
                    % (target_arch, root_arch))

        morphlib.buildcommand.BuildCommand._validate_architecture = validate

        system_artifact = build_command.resolve_artifacts(srcpool)

        # Calculate build order
        # This is basically a hacked version of BuildCommand.build_in_order()
        sources = build_command.get_ordered_sources(system_artifact.walk())
        cross_sources = []
        native_sources = []
        for s in sources:
            if s.morphology['kind'] == 'chunk':
                if s.build_mode == 'bootstrap':
                    cross_sources.append(s)
                else:
                    native_sources.append(s)

        if len(cross_sources) == 0:
            raise morphlib.Error(
                'Nothing to cross-compile. Only chunks built in \'bootstrap\' '
                'mode can be cross-compiled.')

        for s in cross_sources:
            build_command.cache_or_build_source(s, build_env)

        for s in native_sources:
            build_command.fetch_sources(s)

        # Install those to the output tarball ...
        self.app.status(msg='Building final bootstrap system image')
        system_artifact.source.cross_sources = cross_sources
        system_artifact.source.native_sources = native_sources
        staging_area = build_command.create_staging_area(
            build_env, use_chroot=False)
        builder = BootstrapSystemBuilder(
            self.app, staging_area, build_command.lac, build_command.rac,
            system_artifact.source, build_command.lrc, 1, False)
        builder.build_and_cache()

        self.app.status(
            msg='Bootstrap tarball for %(name)s is cached at %(cachepath)s',
            name=system_artifact.name,
            cachepath=build_command.lac.artifact_filename(system_artifact))