summaryrefslogtreecommitdiff
path: root/morphlib/stagingarea.py
blob: b676d4dbcb3f1817506794692eaa656c4145916d (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
# Copyright (C) 2012-2014  Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.


import logging
import os
import shutil
import stat
import cliapp
from urlparse import urlparse
import tempfile

import morphlib


class StagingArea(object):

    '''Represent the staging area for building software.

    The staging area is a temporary directory. In normal operation the build
    dependencies of the artifact being built are installed into the staging
    area and then 'chroot' is used to isolate the build processes from the host
    system. Chunks built in 'test' or 'build-essential' mode have an empty
    staging area and are allowed to use the tools of the host.

    '''

    _base_path = ['/sbin', '/usr/sbin', '/bin', '/usr/bin']

    def __init__(self, app, dirname, build_env, use_chroot=True, extra_env={},
                 extra_path=[]):
        self._app = app
        self.dirname = dirname
        self.builddirname = None
        self.destdirname = None
        self._bind_readonly_mount = None

        self.use_chroot = use_chroot
        self.env = build_env.env
        self.env.update(extra_env)

        if use_chroot:
            path = extra_path + build_env.extra_path + self._base_path
        else:
            rel_path = extra_path + build_env.extra_path
            full_path = [os.path.normpath(dirname + p) for p in rel_path]
            path = full_path + os.environ['PATH'].split(':')
        self.env['PATH'] = ':'.join(path)

    # Wrapper to be overridden by unit tests.
    def _mkdir(self, dirname):  # pragma: no cover
        os.makedirs(dirname)

    def _dir_for_source(self, source, suffix):
        dirname = os.path.join(self.dirname,
                               '%s.%s' % (str(source.name), suffix))
        self._mkdir(dirname)
        return dirname

    def builddir(self, source):
        '''Create a build directory for a given source project.

        Return path to directory.

        '''

        return self._dir_for_source(source, 'build')

    def destdir(self, source):
        '''Create an installation target directory for a given source project.

        This is meant to be used as $DESTDIR when installing chunks.
        Return path to directory.

        '''

        return self._dir_for_source(source, 'inst')

    def relative(self, filename):
        '''Return a filename relative to the staging area.'''

        if not self.use_chroot:
            return filename

        dirname = self.dirname
        if not dirname.endswith('/'):
            dirname += '/'

        assert filename.startswith(dirname)
        return filename[len(dirname) - 1:]  # include leading slash

    def hardlink_all_files(self, srcpath, destpath): # pragma: no cover
        '''Hardlink every file in the path to the staging-area

        If an exception is raised, the staging-area is indeterminate.

        '''

        file_stat = os.lstat(srcpath)
        mode = file_stat.st_mode

        if stat.S_ISDIR(mode):
            # Ensure directory exists in destination, then recurse.
            if not os.path.lexists(destpath):
                os.makedirs(destpath)
            dest_stat = os.stat(os.path.realpath(destpath))
            if not stat.S_ISDIR(dest_stat.st_mode):
                raise IOError('Destination not a directory. source has %s'
                              ' destination has %s' % (srcpath, destpath))

            for entry in os.listdir(srcpath):
                self.hardlink_all_files(os.path.join(srcpath, entry),
                                        os.path.join(destpath, entry))
        elif stat.S_ISLNK(mode):
            # Copy the symlink.
            if os.path.lexists(destpath):
                os.remove(destpath)
            os.symlink(os.readlink(srcpath), destpath)

        elif stat.S_ISREG(mode):
            # Hardlink the file.
            if os.path.lexists(destpath):
                os.remove(destpath)
            os.link(srcpath, destpath)

        elif stat.S_ISCHR(mode) or stat.S_ISBLK(mode):
            # Block or character device. Put contents of st_dev in a mknod.
            if os.path.lexists(destpath):
                os.remove(destpath)
            os.mknod(destpath, file_stat.st_mode, file_stat.st_rdev)
            os.chmod(destpath, file_stat.st_mode)

        else:
            # Unsupported type.
            raise IOError('Cannot extract %s into staging-area. Unsupported'
                          ' type.' % srcpath)

    def install_artifact(self, handle):
        '''Install a build artifact into the staging area.

        We access the artifact via an open file handle. For now, we assume
        the artifact is a tarball.

        '''

        chunk_cache_dir = os.path.join(self._app.settings['tempdir'], 'chunks')
        unpacked_artifact = os.path.join(
            chunk_cache_dir, os.path.basename(handle.name) + '.d')
        if not os.path.exists(unpacked_artifact):
            self._app.status(
                msg='Unpacking chunk from cache %(filename)s',
                filename=os.path.basename(handle.name))
            savedir = tempfile.mkdtemp(dir=chunk_cache_dir)
            try:
                morphlib.bins.unpack_binary_from_file(
                    handle, savedir + '/')
            except BaseException, e: # pragma: no cover
                shutil.rmtree(savedir)
                raise
            # TODO: This rename is not concurrency safe if two builds are
            #       extracting the same chunk, one build will fail because
            #       the other renamed its tempdir here first.
            os.rename(savedir, unpacked_artifact)

        if not os.path.exists(self.dirname):
            self._mkdir(self.dirname)

        self.hardlink_all_files(unpacked_artifact, self.dirname)

    def remove(self):
        '''Remove the entire staging area.

        Do not expect anything with the staging area to work after this
        method is called. Be careful about calling this method if
        the filesystem root directory was given as the dirname.

        '''

        shutil.rmtree(self.dirname)

    to_mount = (
        ('dev/shm', 'tmpfs', 'none'),
    )

    def ccache_dir(self, source): #pragma: no cover
        ccache_dir = self._app.settings['compiler-cache-dir']
        if not os.path.isdir(ccache_dir):
            os.makedirs(ccache_dir)
        # Get a path for the repo's ccache
        ccache_url = source.repo.url
        ccache_path = urlparse(ccache_url).path
        ccache_repobase = os.path.basename(ccache_path)
        if ':' in ccache_repobase: # the basename is a repo-alias
            resolver = morphlib.repoaliasresolver.RepoAliasResolver(
                self._app.settings['repo-alias'])
            ccache_url = resolver.pull_url(ccache_repobase)
            ccache_path = urlparse(ccache_url).path
            ccache_repobase = os.path.basename(ccache_path)
        if ccache_repobase.endswith('.git'):
            ccache_repobase = ccache_repobase[:-len('.git')]

        ccache_repodir = os.path.join(ccache_dir, ccache_repobase)
        # Make sure that directory exists
        if not os.path.isdir(ccache_repodir):
            os.mkdir(ccache_repodir)
        # Get the destination path
        ccache_destdir= os.path.join(self.dirname, 'tmp', 'ccache')
        # Make sure that the destination exists. We'll create /tmp if necessary
        # to avoid breaking when faced with an empty staging area.
        if not os.path.isdir(ccache_destdir):
            os.makedirs(ccache_destdir)
        return ccache_repodir

    def chroot_open(self, source, setup_mounts): # pragma: no cover
        '''Setup staging area for use as a chroot.'''

        assert self.builddirname == None and self.destdirname == None

        builddir = self.builddir(source)
        destdir = self.destdir(source)
        self.builddirname = builddir
        self.destdirname = destdir

        return builddir, destdir

    def chroot_close(self): # pragma: no cover
        '''Undo changes by chroot_open.

        This should be called after the staging area is no longer needed.

        '''
        # No cleanup is currently required
        pass

    def runcmd(self, argv, **kwargs):  # pragma: no cover
        '''Run a command in a chroot in the staging area.'''
        assert 'env' not in kwargs
        kwargs['env'] = dict(self.env)
        if 'extra_env' in kwargs:
            kwargs['env'].update(kwargs['extra_env'])
            del kwargs['extra_env']

        ccache_dir = kwargs.pop('ccache_dir', None)

        chroot_dir = self.dirname if self.use_chroot else '/'
        temp_dir = kwargs["env"].get("TMPDIR", "/tmp")

        staging_dirs = [self.builddirname, self.destdirname]
        if self.use_chroot:
            staging_dirs += ["dev", "proc", temp_dir.lstrip('/')]
        do_not_mount_dirs = [os.path.join(self.dirname, d)
                             for d in staging_dirs]
        if not self.use_chroot:
            do_not_mount_dirs += [temp_dir]
        logging.debug("Not mounting dirs %r" % do_not_mount_dirs)

        mount_proc = self.use_chroot
        if ccache_dir and not self._app.settings['no-ccache']:
            ccache_target = os.path.join(
                    self.dirname, kwargs['env']['CCACHE_DIR'].lstrip('/'))
            binds = ((ccache_dir, ccache_target),)
        else:
            binds = ()

        cmdline = morphlib.util.containerised_cmdline(
                    argv, cwd=kwargs.pop('cwd', '/'),
                    root=chroot_dir, mounts=self.to_mount,
                    binds=binds, mount_proc=mount_proc,
                    writable_paths=do_not_mount_dirs)
        try:
            if kwargs.get('logfile') != None:
                logfile = kwargs.pop('logfile')
                teecmd = ['tee', '-a', logfile]
                return self._app.runcmd(cmdline, teecmd, **kwargs)
            else:
                return self._app.runcmd(cmdline, **kwargs)
        except cliapp.AppException as e:
            raise cliapp.AppException('In staging area %s: running '
                                      'command \'%s\' failed.' % 
                                      (self.dirname, ' '.join(argv)))

    def abort(self): # pragma: no cover
        '''Handle what to do with a staging area in the case of failure.
           This may either remove it or save it for later inspection.
        '''
        # TODO: when we add the option to throw away failed builds,
        #       hook it up here


        dest_dir = os.path.join(self._app.settings['tempdir'],
                                'failed', os.path.basename(self.dirname))
        os.rename(self.dirname, dest_dir)
        self.dirname = dest_dir