summaryrefslogtreecommitdiff
path: root/morphlib/definitions_repo.py
blob: 6d14fcbbe712a5841af0c967583137fa0d2a857c (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
# Copyright (C) 2015  Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program.  If not, see <http://www.gnu.org/licenses/>.


'''Handles the Git repository containing Baserock definitions.'''


import cliapp

import contextlib
import logging
import os
import urlparse
import uuid
import warnings

import morphlib
import gitdir


class DefinitionsRepoNotFound(cliapp.AppException):
    def __init__(self):
        cliapp.AppException.__init__(self,
            'This command must be run from inside a Git repository '
            'containing Baserock definitions.')


class FileOutsideRepo(cliapp.AppException):
    def __init__(self, path, repo):
        cliapp.AppException.__init__(self,
            'File %s is not in repo %s.' % (path, repo))


class DefinitionsRepo(gitdir.GitDirectory):
    '''Represents a definitions.git repo checked out locally.

    This can either be a normal Git clone, or a Git clone inside an old-style
    Morph workspace.

    If the repo is inside a Morph workspace, certain behaviours are enabled for
    consistency with old versions of Morph. See function documentation for
    details.

    '''
    def __init__(self, path, search_for_root=False, system_branch=None,
                 allow_missing=False):
        morphlib.gitdir.GitDirectory.__init__(
            self, path, search_for_root=search_for_root,
            allow_missing=allow_missing)
        self.system_branch = system_branch

    @property
    def HEAD(self):
        '''Return the ref considered to be HEAD of this definitions repo.

        In a normal Git checkout, this will return whatever ref is checked out
        as the working tree (HEAD, in Git terminology).

        If this definitions repo is in an old-style Morph system branch, it
        will return the ref that was checked out with `morph branch` or `morph
        checkout`, which will NOT necessarily correspond to what is checked out
        in the Git repo.

        '''
        if self.system_branch is None:
            return morphlib.gitdir.GitDirectory.HEAD.fget(self)
        else:
            return self.system_branch.get_config('branch.name')

    @property
    def remote_url(self):
        '''Return the 'upstream' URL of this repo.

        If this repo is inside a Morph system branch checkout, this will be
        whatever URL was passed to `morph checkout` or `morph branch`. That may
        be a keyed URL such as baserock:baserock/definitions.

        Otherwise, the fetch URL of the 'origin' remote is returned.

        '''
        if self.system_branch is None:
            return self.get_remote('origin').get_fetch_url()
        else:
            return self.system_branch.root_repository_url

    def branch_with_local_changes(self, uuid, push=True, build_ref_prefix=None,
                                  git_user_name=None, git_user_email=None,
                                  status_cb=None):
        '''Yield a branch that includes the user's local changes to this repo.

        When operating on local repos, this isn't really necessary. But when
        doing distributed building, any local changes the user has made need
        to be pushed. As a convenience for the user, Morph supports creating
        temporary branches with their local changes and pushing them to the
        'origin' remote of the repo they are working in.

        If there are no local changes, there is no temporary branch created,
        and the function yields whatever branch was checked out.

        The 'git_user_name' and 'git_user_email' parameters are used when
        creating commits in the temporary branch. The 'build_ref_prefix' is
        prepended to the ref name of the temporary branch. Pushing is limited
        to only certain refs in some Git servers.

        '''
        if status_cb:
            status_cb(msg='Looking for uncommitted changes (pass '
                          '--local-changes=ignore to skip)')

        if self.system_branch:
            bb = morphlib.buildbranch.BuildBranch(
                build_ref_prefix, uuid, system_branch=self.system_branch)
        else:
            bb = morphlib.buildbranch.BuildBranch(
                build_ref_prefix, uuid, definitions_repo=self)

        pbb = morphlib.buildbranch.pushed_build_branch(
            bb, changes_need_pushing=push, name=git_user_name,
            email=git_user_email, build_uuid=uuid,
            status=status_cb)
        return pbb   # (repo_url, commit, original_ref)

    @contextlib.contextmanager
    def source_pool(self, lrc, rrc, cachedir, ref, system_filename,
                    include_local_changes=False, push_local_changes=False,
                    update_repos=True, status_cb=None, build_ref_prefix=None,
                    git_user_name=None, git_user_email=None):
        '''Load the system defined in 'morph' and all the sources it contains.

        This is a context manager, because depending on the settings given it
        may create and push a temporary build branch. This is useful when there
        are local changes that you would like distributed build workers to
        build.

        If 'include_local_changes' is False, the on-disk definitions.git repo
        is used only to query the HEAD ref. Morph then looks for this ref in
        its local clone of that repo's 'origin' remote, which ensures that the
        changes it is building are pushed to the configured Git server (at
        least at time it is building them).

        When 'include_local_changes' is True, Morph will create a temporary
        branch in the repo including any local changes. If the definitions.git
        repo is inside an old-style Morph system branch, it will create
        temporary branches in all repos that have been marked with `morph
        edit`.  The branch is cleaned up when the context manager exits. The
        'user_name', 'user_email' and 'build_ref_prefix' settings must be
        passed if a temporary build branch is created.

        FIXME: if not inside an old-style Morph system branch, the temporary
        branch is redundant as Morph could just read the files from the disk
        as-is. This requires changes to SourceResolver before it is possible.

        The 'push_local_changes' option isn't much use. You probably want to
        use branch_with_local_changes() instead. It is present so that the
        `morph build` command continues to honour the 'push-build-branches'
        setting, but that was probably only useful for `morph distbuild` and
        that now uses branch_with_local_changes().

        The 'lrc' and 'rrc' parameters are local and remote Git repo caches.
        Use morphlib.util.new_repo_caches() to obtain these. The 'cachedir'
        parameter points to where Git repos are cached by Morph,
        app.settings['cachedir'] tells you that.

        The 'update_repos' flag allows you to disable updating Git repos, to
        honour app.settings['no-git-update']. If one of the refs in the build
        graph is not available locally and update_repos is False, you will see
        a morphlib.gitdir.InvalidRefError exception.

        The 'status_cb' function will be called if set to output progress and
        status messages to the user.

        The function yields a morphlib.srcpool.SourcePool instance, which is
        all you need to resolve cache keys, and construct a usable build graph.
        See morphlib.buildcommand.BuildCommand.resolve_artifacts() for a way
        of doing this.

        '''
        # FIXME: currently the way this function is implemented causes the
        # `deploy` command to re-create a temporary build branch for each
        # system that is deployed. This is a regression in terms of
        # performance. But it seems to me that the sourcepool object should
        # be able to contain multiple systems, and so the correct fix is to
        # extend this function to handle multiple systems, rather than split
        # up the 'process local changes' stage from the 'create source pool'
        # stage.
        if include_local_changes:
            build_uuid = uuid.uuid4().hex
            temporary_branch = DefinitionsRepo.branch_with_local_changes(
                self, build_uuid, push=push_local_changes,
                build_ref_prefix=build_ref_prefix, git_user_name=git_user_name,
                git_user_email=git_user_email, status_cb=status_cb)
            with temporary_branch as (repo_url, commit, original_ref):
                if status_cb:
                    status_cb(msg='Deciding on task order')

                yield morphlib.sourceresolver.create_source_pool(
                    lrc, rrc, repo_url, commit, [system_filename],
                    cachedir=cachedir, original_ref=original_ref,
                    update_repos=update_repos, status_cb=status_cb)
        else:
            repo_url = self.remote_url
            commit = self.resolve_ref_to_commit(ref)

            if status_cb:
                status_cb(msg='Deciding on task order')

            try:
                yield morphlib.sourceresolver.create_source_pool(
                    lrc, rrc, repo_url, commit, [system_filename],
                    cachedir=cachedir, original_ref=ref,
                    update_repos=update_repos, status_cb=status_cb)
            except morphlib.sourceresolver.InvalidDefinitionsRefError as e:
                raise cliapp.AppException(
                    'Commit %s wasn\'t found in the "origin" remote %s. '
                    'You either need to push your local commits on branch %s '
                    'to "origin", or use the --local-changes=include feature '
                    'of Morph.' % (e.ref, e.repo_url, ref))

    def get_morphology_loader(self):
        '''Return a MorphologyLoader instance.

        This may read the VERSION and DEFAULTS file and pass appropriate
        information to the MorphologyLoader constructor.

        '''
        mf = morphlib.morphologyfinder.MorphologyFinder(self)

        version_text = mf.read_file('VERSION')
        version = morphlib.definitions_version.check_version_file(version_text)

        defaults_text = mf.read_file('DEFAULTS', allow_missing=True)

        if version < 7:
            if defaults_text is not None:
                warnings.warn(
                    "Ignoring DEFAULTS file, because these definitions are "
                    "version %i" % version)
                defaults_text = None
        else:
            if defaults_text is None:
                warnings.warn("No DEFAULTS file found.")

        defaults = morphlib.defaults.Defaults(version,
                                              text=defaults_text)

        loader = morphlib.morphloader.MorphologyLoader(
            predefined_build_systems=defaults.build_systems())

        return loader

    def load_all_morphologies(self, loader=None):
        loader = loader or self.get_morphology_loader()

        mf = morphlib.morphologyfinder.MorphologyFinder(self)
        for filename in (f for f in mf.list_morphologies()
                         if not self.is_symlink(f)):
            text = mf.read_file(filename)
            m = loader.load_from_string(text, filename=filename)
            m.repo_url = self.remote_url
            m.ref = self.HEAD
            yield m

    def relative_path(self, path, cwd='.'):
        '''Make 'path' relative to the top directory of this repo.

        If 'path' is a relative path, it is taken to be relative to the
        current working directory. Thus, the result of this function will
        be different depending on the value of os.getcwd().

        If the given path is outside the repo, a PathOutsideRepo exception
        is raised.

        '''
        def path_is_outside_repo(path):
            return path.split(os.sep, 1)[0] == '..'

        absolute_path = os.path.join(cwd, os.path.abspath(path))
        repo_relative_path = os.path.relpath(absolute_path, self.dirname)

        if path_is_outside_repo(repo_relative_path):
            raise FileOutsideRepo(repo_relative_path, self)

        return repo_relative_path

    def relative_path_to_chunk(self, repo_url):
        '''Return a sensible directory to check out repo_url.

        This will be a path in the directory that contains this definitions
        repo, with its name based on 'repo_url'.

        '''
        # Parse the URL. If the path component is absolute, we assume
        # it's a real URL; otherwise, an aliased URL.
        parts = urlparse.urlparse(repo_url)

        # Remove .git suffix, if any.
        path = parts.path
        if path.endswith('.git'):
            path = path[:-len('.git')]

        relative = os.path.basename(parts.path)

        # Replace colons with slashes.
        relative = '/'.join(relative.split(':'))

        # Remove anyleading slashes, or os.path.join below will only
        # use the relative part (since it's absolute, not relative).
        relative = relative.lstrip('/')

        return os.path.join(os.path.dirname(self.dirname), relative)


class DefinitionsRepoWithApp(DefinitionsRepo):
    '''Wrapper class for DefinitionsRepo that understands Morph settings.

    The DefinitionsRepo class does not require a morphlib.app.Application
    instance to use it. However, this means you need to pass quite a lot
    of parameters in. Code inside Morph can use this class instead to save
    duplicating code.

    '''
    def __init__(self, app, *args, **kwargs):
        DefinitionsRepo.__init__(self, *args, **kwargs)
        self.app = app

        self._git_user_name = morphlib.git.get_user_name(app.runcmd)
        self._git_user_email = morphlib.git.get_user_email(app.runcmd)

        self._lrc, self._rrc = morphlib.util.new_repo_caches(app)

    def branch_with_local_changes(self, uuid, push=False):
        '''Equivalent to DefinitionsRepo.branch_with_local_changes().'''

        return DefinitionsRepo.branch_with_local_changes(
            self, uuid,
            push=(push or self.app.settings['push-build-branches']),
            build_ref_prefix=self.app.settings['build-ref-prefix'],
            git_user_name=self._git_user_name,
            git_user_email=self._git_user_email,
            status_cb=self.app.status,)

    def source_pool(self, ref, system_filename):
        '''Equivalent to DefinitionsRepo.source_pool().'''

        local_changes = self.app.settings['local-changes']
        return DefinitionsRepo.source_pool(
            self, self._lrc, self._rrc, self.app.settings['cachedir'],
            ref, system_filename,
            include_local_changes=(local_changes == 'include'),
            push_local_changes=self.app.settings['push-build-branches'],
            build_ref_prefix=self.app.settings['build-ref-prefix'],
            git_user_name=self._git_user_name,
            git_user_email=self._git_user_email,
            status_cb=self.app.status,
            update_repos=(not self.app.settings['no-git-update']))


def _system_branch(path):
    '''Open an old-style Morph system branch in an old-style Morph workspace.

    Raises morphlib.workspace.NotInWorkspace or
    morphlib.sysbranchdir.NotInSystemBranch if either workspace or
    system-branch are not found.

    '''
    morphlib.workspace.open(path)
    system_branch = morphlib.sysbranchdir.open_from_within(path)
    return system_branch


def _local_definitions_repo(path, search_for_root, system_branch=None,
                            app=None):
    '''Open a local Git repo containing Baserock definitions, at 'path'.

    Raises morphlib.gitdir.NoGitRepoError if there is no repo found at 'path'.

    '''
    if app:
        gitdir = morphlib.definitions_repo.DefinitionsRepoWithApp(
            app, path, search_for_root=search_for_root,
            system_branch=system_branch)
    else:
        gitdir = morphlib.definitions_repo.DefinitionsRepo(
            path, search_for_root=search_for_root, system_branch=system_branch)
    return gitdir


def open(path, search_for_root=False, search_workspace=False, app=None):
    '''Open the definitions.git repo at 'path'.

    Returns a DefinitionsRepo instance.

    If 'search_for_root' is True, this function will traverse up from 'path'
    to find a .git directory, and assume that is the top of the Git repository.
    If you are trying to find the repo based on the current working directory,
    you should set this to True. If you are trying to find the repo based on a
    path entered manually by the user, you may want to set this to False to
    avoid confusion.

    If 'search_workspace' is True, this function will check if 'path' is inside
    an old-style Morph workspace. If it is, there will be two changes to its
    behaviour. First, the definitions.git will be returned even if 'path' is
    inside a different repo, because the old-style Morph system branch will
    identify which is the correct definitions.git repo. Second, the value
    returned for HEAD will not be the ref checked out in the definitions.git
    repo, but rather the ref that was passed to `morph checkout` or `morph
    branch` when the system branch was originally checked out. This behaviour
    may seem confusing if you are new to Morph, but in fact Morph forced users
    to work this way for several years, so we need preserve this behaviour for
    a while to avoid disrupting existing users.

    '''
    sb = None

    if search_workspace:
        try:
            sb = _system_branch(path)
        except (morphlib.workspace.NotInWorkspace,
                morphlib.sysbranchdir.NotInSystemBranch):
            logging.debug('Did not find old-style Morph system branch')

    if sb:
        path = sb.get_git_directory_name(sb.root_repository_url)
        definitions_repo = _local_definitions_repo(
            path=path, search_for_root=False, system_branch=sb, app=app)
        logging.info('Opened definitions repo %s from Morph system branch %s',
                     definitions_repo, sb)
    else:
        try:
            definitions_repo = _local_definitions_repo(
                path, search_for_root=search_for_root, app=app)
        except morphlib.gitdir.NoGitRepoError:
            raise DefinitionsRepoNotFound()
        logging.info('Opened definitions repo %s', definitions_repo)

    return definitions_repo