summaryrefslogtreecommitdiff
path: root/morphlib/builder2.py
blob: a27c35abf44eb5834859c50f1f09bba085b0db55 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
# Copyright (C) 2012  Codethink Limited
# 
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
# 
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
# 
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.


import json
import logging
import os
import shutil
import time

import morphlib


class BuilderBase(object):

    '''Base class for building artifacts.'''

    def __init__(self, staging_area, artifact_cache, artifact, repo_cache,
                 build_env, max_jobs):
        self.staging_area = staging_area
        self.artifact_cache = artifact_cache
        self.artifact = artifact
        self.repo_cache = repo_cache
        self.build_env = build_env
        self.max_jobs = max_jobs
    
    def create_metadata(self, artifact_name):
        '''Create metadata to artifact to allow it to be reproduced later.
        
        The metadata is represented as a dict, which later on will be
        written out as a JSON file.
        
        '''
        
        assert isinstance(self.artifact.source.repo, 
                          morphlib.cachedrepo.CachedRepo)
        meta = {
            'artifact-name': artifact_name,
            'source-name': self.artifact.source.morphology['name'],
            'kind': self.artifact.source.morphology['kind'],
            'description': self.artifact.source.morphology['description'],
            'repo': self.artifact.source.repo.url,
            'original_ref': self.artifact.source.original_ref,
            'sha1': self.artifact.source.sha1,
            'morphology': self.artifact.source.filename,
            'cache-key': self.artifact.cache_key,
            'cache-id': self.artifact.cache_id,
        }
        
        return meta

    # Wrapper around open() to allow it to be overridden by unit tests.
    def _open(self, filename, mode): # pragma: no cover
        dirname = os.path.dirname(filename)
        if not os.path.exists(dirname):
            os.makedirs(dirname)
        return open(filename, mode)

    def write_metadata(self, instdir, artifact_name):
        '''Write the metadata for an artifact.
        
        The file will be located under the ``baserock`` directory under
        instdir, named after ``cache_key`` with ``.meta`` as the suffix.
        It will be in JSON format.
        
        '''
        
        meta = self.create_metadata(artifact_name)

        basename = '%s.meta' % artifact_name
        filename = os.path.join(instdir, 'baserock', basename)

        # Unit tests use StringIO, which in Python 2.6 isn't usable with
        # the "with" statement. So we don't do it with "with".
        f = self._open(filename, 'w')
        f.write(json.dumps(meta, indent=4, sort_keys=True))
        f.close()
        
    def new_artifact(self, artifact_name):
        '''Return an Artifact object for something built from our source.'''
        a = morphlib.artifact.Artifact(self.artifact.source, artifact_name)
        a.cache_key = self.artifact.cache_key
        return a
        
    def runcmd(self, *args, **kwargs):
        kwargs['env'] = self.build_env.env
        return self.staging_area.runcmd(*args, **kwargs)


class ChunkBuilder(BuilderBase):

    '''Build chunk artifacts.'''
    
    def get_commands(self, which, morphology, build_system):
        '''Return the commands to run from a morphology or the build system.'''
        if morphology[which] is None:
            attr = '_'.join(which.split('-'))
            return getattr(build_system, attr)
        else:
            return morphology[which]

    def build_and_cache(self): # pragma: no cover
        builddir = self.staging_area.builddir(self.artifact.source)
        self.get_sources(builddir)
        destdir = self.staging_area.destdir(self.artifact.source)
        self.run_commands(builddir, destdir)
        self.assemble_chunk_artifacts(destdir)

    def get_sources(self, srcdir): # pragma: no cover
        '''Get sources from git to a source directory, for building.'''

        cache_dir = os.path.dirname(self.artifact.source.repo.path)

        def extract_repo(path, sha1, destdir):
            logging.debug('Extracting %s into %s' % (path, destdir))
            if not os.path.exists(destdir):
                os.mkdir(destdir)
            morphlib.git.copy_repository(path, destdir, logging.debug)
            morphlib.git.checkout_ref(destdir, sha1, logging.debug)
            morphlib.git.reset_workdir(destdir, logging.debug)
            submodules = morphlib.git.Submodules(path, sha1)
            try:
                submodules.load()
            except morphlib.git.NoModulesFileError:
                return []
            else:
                tuples = []
                for sub in submodules:
                    cached_repo = self.repo_cache.get_repo(sub.repo)
                    sub_dir = os.path.join(destdir, sub.path)
                    tuples.append((cached_repo.path, sub.commit, sub_dir))
                return tuples

        s = self.artifact.source
        todo = [(s.repo.path, s.sha1, srcdir)]
        while todo:
            path, sha1, srcdir = todo.pop()
            todo += extract_repo(path, sha1, srcdir)
        self.set_mtime_recursively(srcdir)

    def set_mtime_recursively(self, root): # pragma: no cover
        '''Set the mtime for every file in a directory tree to the same.
        
        We do this because git checkout does not set the mtime to anything,
        and some projects (binutils, gperf for example) include formatted
        documentation and try to randomly build things or not because of
        the timestamps. This should help us get more reliable  builds.
        
        '''
        
        now = time.time()
        for dirname, subdirs, basenames in os.walk(root, topdown=False):
            for basename in basenames:
                pathname = os.path.join(dirname, basename)
                # we need the following check to ignore broken symlinks
                if os.path.exists(pathname):
                    os.utime(pathname, (now, now))
            os.utime(dirname, (now, now))


    def run_commands(self, builddir, destdir): # pragma: no cover
        m = self.artifact.source.morphology
        bs = morphlib.buildsystem.lookup_build_system(m['build-system'])

        relative_builddir = self.staging_area.relative(builddir)
        relative_destdir = self.staging_area.relative(destdir)
        self.build_env.env['DESTDIR'] = relative_destdir

        steps = [('configure', False), 
                 ('build', True),
                 ('test', False),
                 ('install', False)]
        for step, in_parallel in steps:
            cmds = self.get_commands('%s-commands' % step, m, bs)
            for cmd in cmds:
                if in_parallel:
                    max_jobs = self.artifact.source.morphology['max-jobs']
                    if max_jobs is None:
                        max_jobs = self.max_jobs
                    self.build_env.env['MAKEFLAGS'] = '-j%s' % max_jobs
                else:
                    self.build_env.env['MAKEFLAGS'] = '-j1'
                self.runcmd(['sh', '-c', cmd], cwd=relative_builddir)

    def assemble_chunk_artifacts(self, destdir): # pragma: no cover
        ex = None # create_chunk doesn't actually use this
        specs = self.artifact.source.morphology['chunks']
        if len(specs) == 0:
            specs = {
                self.artifact.source.morphology['name']: ['.'],
            }
        for artifact_name in specs:
            self.write_metadata(destdir, artifact_name)
            patterns = specs[artifact_name]
            patterns += [r'baserock/%s\.' % artifact_name]

            artifact = self.new_artifact(artifact_name)
            with self.artifact_cache.put(artifact) as f:
                logging.debug('assembling chunk %s' % artifact_name)
                logging.debug('assembling into %s' % f.name)
                morphlib.bins.create_chunk(destdir, f, patterns, ex)

        files = os.listdir(destdir)
        if files:
            raise Exception('DESTDIR %s is not empty: %s' % (destdir, files))


class StratumBuilder(BuilderBase):

    '''Build stratum artifacts.'''

    def build_and_cache(self): # pragma: no cover
        destdir = self.staging_area.destdir(self.artifact.source)

        constituents = [dependency
                        for dependency in self.artifact.dependencies
                        if dependency.source.morphology['kind'] == 'chunk']
        for chunk_artifact in constituents:
            with self.artifact_cache.get(chunk_artifact) as f:
                morphlib.bins.unpack_binary_from_file(f, destdir)

        artifact_name = self.artifact.source.morphology['name']
        self.write_metadata(destdir, artifact_name)
        artifact = self.new_artifact(artifact_name)
        with self.artifact_cache.put(artifact) as f:
            morphlib.bins.create_stratum(destdir, f, None)


class SystemBuilder(BuilderBase): # pragma: no cover

    '''Build system image artifacts.'''

    def build_and_cache(self):
        logging.debug('SystemBuilder.do_build called')
        self.ex = morphlib.execute.Execute(self.staging_area.dirname,
                                           logging.debug)
        
        image_name = os.path.join(self.staging_area.dirname,
                                  '%s.img' % self.artifact.name)
        self._create_image(image_name)
        self._partition_image(image_name)
        self._install_mbr(image_name)
        partition = self._setup_device_mapping(image_name)

        mount_point = None
        try:
            self._create_fs(partition)
            mount_point = self.staging_area.destdir(self.artifact.source)
            self._mount(partition, mount_point)
            factory_path = os.path.join(mount_point, 'factory')
            self._create_subvolume(factory_path)
            self._unpack_strata(factory_path)
            self._create_fstab(factory_path)
            self._create_extlinux_config(factory_path)
            self._create_subvolume_snapshot(
                    mount_point, 'factory', 'factory-run')
            factory_run_path = os.path.join(mount_point, 'factory-run')
            self._install_boot_files(factory_run_path, mount_point)
            self._install_extlinux(mount_point)
            self._unmount(mount_point)
        except BaseException, e:
            logging.error('Got error while system image building, '
                            'unmounting and device unmapping')
            self._unmount(mount_point)
            self._undo_device_mapping(image_name)
            raise

        self._undo_device_mapping(image_name)
        self._move_image_to_cache(image_name)

    def _create_image(self, image_name):
        logging.debug('Creating disk image %s' % image_name)
        morphlib.fsutils.create_image(
            self.ex, image_name, self.artifact.source.morphology['disk-size'])

    def _partition_image(self, image_name):
        logging.debug('Partitioning disk image %s' % image_name)
        morphlib.fsutils.partition_image(self.ex, image_name)

    def _install_mbr(self, image_name):
        logging.debug('Installing mbr on disk image %s' % image_name)
        morphlib.fsutils.install_mbr(self.ex, image_name)

    def _setup_device_mapping(self, image_name):
        logging.debug('Device mapping partitions in %s' % image_name)
        return morphlib.fsutils.setup_device_mapping(self.ex, image_name)

    def _create_fs(self, partition):
        logging.debug('Creating filesystem on %s' % partition)
        morphlib.fsutils.create_fs(self.ex, partition)

    def _mount(self, partition, mount_point):
        logging.debug('Mounting %s on %s' % (partition, mount_point))
        morphlib.fsutils.mount(self.ex, partition, mount_point)

    def _create_subvolume(self, path):
        logging.debug('Creating subvolume %s' % path)
        self.ex.runv(['btrfs', 'subvolume', 'create', path])

    def _unpack_strata(self, path):
        logging.debug('Unpacking strata to %s' % path)
        for stratum_artifact in self.artifact.dependencies:
            with self.artifact_cache.get(stratum_artifact) as f:
                morphlib.bins.unpack_binary_from_file(f, path)
        morphlib.builder.ldconfig(self.ex, path)

    def _create_fstab(self, path):
        logging.debug('Creating fstab in %s' % path)
        fstab = os.path.join(path, 'etc', 'fstab')
        if not os.path.exists(os.path.dirname(fstab)): # FIXME: should exist
            os.makedirs(os.path.dirname(fstab))
        with open(fstab, 'w') as f:
            f.write('proc      /proc proc  defaults          0 0\n')
            f.write('sysfs     /sys  sysfs defaults          0 0\n')
            f.write('/dev/sda1 / btrfs errors=remount-ro 0 1\n')

    def _create_extlinux_config(self, path):
        logging.debug('Creating extlinux.conf in %s' % path)
        config = os.path.join(path, 'extlinux.conf')
        with open(config, 'w') as f:
            f.write('default linux\n')
            f.write('timeout 1\n')
            f.write('label linux\n')
            f.write('kernel /boot/vmlinuz\n')
            f.write('append root=/dev/sda1 rootflags=subvol=factory-run '
                                           'init=/sbin/init quiet rw\n')

    def _create_subvolume_snapshot(self, path, source, target):
        logging.debug('Creating subvolume snapshot %s to %s' % 
                        (source, target))
        self.ex.runv(['btrfs', 'subvolume', 'snapshot', source, target],
                     cwd=path)

    def _install_boot_files(self, sourcefs, targetfs):
        logging.debug('installing boot files into root volume')
        shutil.copy2(os.path.join(sourcefs, 'extlinux.conf'),
                     os.path.join(targetfs, 'extlinux.conf'))
        os.mkdir(os.path.join(targetfs, 'boot'))
        shutil.copy2(os.path.join(sourcefs, 'boot', 'vmlinuz'),
                     os.path.join(targetfs, 'boot', 'vmlinuz'))
        shutil.copy2(os.path.join(sourcefs, 'boot', 'System.map'),
                     os.path.join(targetfs, 'boot', 'System.map'))

    def _install_extlinux(self, path):
        logging.debug('Installing extlinux to %s' % path)
        self.ex.runv(['extlinux', '--install', path])

        # FIXME this hack seems to be necessary to let extlinux finish
        self.ex.runv(['sync'])
        time.sleep(2)

    def _unmount(self, mount_point):
        logging.debug('Unmounting %s' % mount_point)
        if mount_point is not None:
            morphlib.fsutils.unmount(self.ex, mount_point)

    def _undo_device_mapping(self, image_name):
        logging.debug('Undoing device mappings for %s' % image_name)
        morphlib.fsutils.undo_device_mapping(self.ex, image_name)

    def _move_image_to_cache(self, image_name):
        logging.debug('Moving image to cache: %s' % image_name)
        # FIXME: Need to create file directly in cache to avoid costly
        # copying here.
        with self.artifact_cache.put(self.artifact) as outf:
            with open(image_name) as inf:
                while True:
                    data = inf.read(1024**2)
                    if not data:
                        break
                    outf.write(data)


class Builder(object): # pragma: no cover

    '''Helper class to build with the right BuilderBase subclass.'''
    
    classes = {
        'chunk': ChunkBuilder,
        'stratum': StratumBuilder,
        'system': SystemBuilder,
    }

    def __init__(self, staging_area, artifact_cache, repo_cache, build_env, 
                 max_jobs):
        self.staging_area = staging_area
        self.artifact_cache = artifact_cache
        self.repo_cache = repo_cache
        self.build_env = build_env
        self.max_jobs = max_jobs
        
    def build_and_cache(self, artifact):
        kind = artifact.source.morphology['kind']
        o = self.classes[kind](self.staging_area, self.artifact_cache, 
                               artifact, self.repo_cache, self.build_env, 
                               self.max_jobs)
        logging.debug('Builder.build: artifact %s with %s' %
                      (artifact.name, repr(o)))
        o.build_and_cache()
        logging.debug('Builder.build: done')