summaryrefslogtreecommitdiff
path: root/morphlib/extensions.py
blob: 7290f7c1c965a21136a4299dc5f0e50de50d9fb9 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
# Copyright (C) 2014-2015  Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program.  If not, see <http://www.gnu.org/licenses/>.

import asyncore
import asynchat
import glob
import logging
import os
import stat
import subprocess
import tempfile

import cliapp

import morphlib


class ExtensionError(morphlib.Error):
    pass

class ExtensionNotFoundError(ExtensionError):
    pass

class ExtensionNotExecutableError(ExtensionError):
    pass


def _get_morph_extension_directory():
    code_dir = os.path.dirname(morphlib.__file__)
    return os.path.join(code_dir, 'exts')

def _list_repo_extension_filenames(definitions_repo, kind): #pragma: no cover
    files = definitions_repo.list_files()
    return (f for f in files if os.path.splitext(f)[1] == kind)

def _list_morph_extension_filenames(kind):
    return glob.glob(os.path.join(_get_morph_extension_directory(),
                                  '*' + kind))

def _get_extension_name(filename):
    return os.path.basename(filename)

def _get_repo_extension_contents(definitions_repo, name, kind):
    return definitions_repo.read_file(name + kind)

def _get_morph_extension_filename(name, kind):
    return os.path.join(_get_morph_extension_directory(), name + kind)

def _is_executable(filename):
    st = os.stat(filename)
    mask = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
    return (stat.S_IMODE(st.st_mode) & mask) != 0

def _list_extensions(kind):
    repo_extension_filenames = []
    try:
        definitions_repo = morphlib.definitions_repo.open(
            '.', search_for_root=True)
        repo_extension_filenames = \
                _list_repo_extension_filenames(definitions_repo, kind)
    except morphlib.definitions_repo.DefinitionsRepoNotFound:
        # Squash this and just return no system branch extensions
        pass
    morph_extension_filenames = _list_morph_extension_filenames(kind)

    repo_extension_names = \
            (_get_extension_name(f) for f in repo_extension_filenames)
    morph_extension_names = \
            (_get_extension_name(f) for f in morph_extension_filenames)

    extension_names = set(repo_extension_names)
    extension_names.update(set(morph_extension_names))
    return list(extension_names)

def list_extensions(kind=None):
    """
    List all available extensions by 'kind'.

    'kind' should be one of '.write' or '.configure'.
    If 'kind' is not provided available extensions of both
    types will be returned.

    '.check' extensions are not listed here as they should
    be associated with a '.write' extension of the same name.
    """
    if kind:
        return _list_extensions(kind)
    else:
        configure_extensions = _list_extensions('.configure')
        write_extensions = _list_extensions('.write')

        return configure_extensions + write_extensions

class get_extension_filename():
    """
    Find the filename of an extension by its 'name' and 'kind'.

    'kind' should be one of '.configure', '.write' or '.check'.

    '.help' files for the extensions may also be retrieved by
    passing the kind as '.write.help' or '.configure.help'.

    If the extension is in the build repository then a temporary
    file will be created, which will be deleted on exting the with block.
    """
    def __init__(self, definitions_repo, name, kind, executable=True):
        self.definitions_repo = definitions_repo
        self.name = name
        self.kind = kind
        self.executable = executable
        self.delete = False

    def __enter__(self):
        ext_filename = None
        try:
            ext_contents = _get_repo_extension_contents(
                self.definitions_repo, self.name, self.kind)
        except (IOError, cliapp.AppException):
            # Not found: look for it in the Morph code.
            ext_filename = _get_morph_extension_filename(self.name, self.kind)
            if not os.path.exists(ext_filename):
                raise ExtensionNotFoundError(
                    'Could not find extension %s%s' % (self.name, self.kind))
            if self.executable and not _is_executable(ext_filename):
                raise ExtensionNotExecutableError(
                    'Extension not executable: %s' % ext_filename)
        else:
            # Found it in the system morphology's repository.
            fd, ext_filename = tempfile.mkstemp()
            os.write(fd, ext_contents)
            os.close(fd)
            os.chmod(ext_filename, 0o700)
            self.delete = True

        self.ext_filename = ext_filename
        return ext_filename

    def __exit__(self, type, value, trace):
        if self.delete:
            os.remove(self.ext_filename)


class _EOFWrapper(asyncore.file_wrapper):
    '''File object that reports when it hits EOF

    The async_chat class doesn't notice that its input file has hit EOF,
    so if we give it one of these instead, it will mark the chatter for
    closiure and ensure any in-progress buffers are flushed.
    '''
    def __init__(self, dispatcher, fd):
        self._dispatcher = dispatcher
        asyncore.file_wrapper.__init__(self, fd)

    def recv(self, *args):
        data = asyncore.file_wrapper.recv(self, *args)
        if not data:
            self._dispatcher.close_when_done()
            # ensure any unterminated data is flushed
            return self._dispatcher.get_terminator()
        return data


class _OutputDispatcher(asynchat.async_chat, asyncore.file_dispatcher):
    '''asyncore dispatcher that calls line_handler per line.'''
    def __init__(self, fd, line_handler, map=None):
        asynchat.async_chat.__init__(self, sock=None, map=map)
        asyncore.file_dispatcher.__init__(self, fd=fd, map=map)
        self.set_terminator('\n')
        self._line_handler = line_handler
    collect_incoming_data = asynchat.async_chat._collect_incoming_data
    def set_file(self, fd):
        self.socket = _EOFWrapper(self, fd)
        self._fileno = self.socket.fileno()
        self.add_channel()
    def found_terminator(self):
        self._line_handler(''.join(self.incoming))
        self.incoming = []

class ExtensionSubprocess(object):

    def __init__(self, report_stdout, report_stderr, report_logger):
        self._report_stdout = report_stdout
        self._report_stderr = report_stderr
        self._report_logger = report_logger

    def run(self, filename, args, cwd, env, separate_mount_namespace=True):
        '''Run an extension.

        Anything written by the extension to stdout is passed to status(), thus
        normally echoed to Morph's stdout. An extra FD is passed in the
        environment variable MORPH_LOG_FD, and anything written here will be
        included as debug messages in Morph's log file.

        '''

        log_read_fd, log_write_fd = os.pipe()

        try:
            new_env = env.copy()
            new_env['MORPH_LOG_FD'] = str(log_write_fd)

            # Because we don't have python 3.2's pass_fds, we have to
            # play games with preexec_fn to close the fds we don't
            # need to inherit
            def close_read_end():
                os.close(log_read_fd)

            cmdline = [filename] + list(args)

            if separate_mount_namespace:
                cmdline = morphlib.util.unshared_cmdline(cmdline)

            p = subprocess.Popen(
                cmdline,
                cwd=cwd, env=new_env,
                stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                preexec_fn=close_read_end)
            os.close(log_write_fd)
            log_write_fd = None

            return self._watch_extension_subprocess(p, log_read_fd)
        finally:
            os.close(log_read_fd)
            if log_write_fd is not None:
                os.close(log_write_fd)

    def _watch_extension_subprocess(self, p, log_read_fd):
        '''Follow stdout, stderr and log output of an extension subprocess.'''

        try:
            socket_map = {}
            for handler, fd in ((self._report_stdout, p.stdout),
                                (self._report_stderr, p.stderr),
                                (self._report_logger, log_read_fd)):
                _OutputDispatcher(line_handler=handler, fd=fd,
                                  map=socket_map)
            asyncore.loop(use_poll=True, map=socket_map)

            returncode = p.wait()
            assert returncode is not None
        except BaseException as e:
            logging.debug('Received exception %r watching extension' % e)
            p.terminate()
            p.wait()
            raise
        finally:
            p.stdout.close()
            p.stderr.close()

        return returncode