summaryrefslogtreecommitdiff
path: root/morphlib/morphloader.py
blob: 45416a19ea8904bb51e4c1b7a4c8a69b12051012 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
# Copyright (C) 2013-2014  Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
# =*= License: GPL-2 =*=


import collections
import logging
import warnings
import yaml

import morphlib


class MorphologyObsoleteFieldWarning(UserWarning):

    def __init__(self, morphology, spec, field):
        self.kind = morphology['kind']
        self.morphology_name = morphology.get('name', '<unknown>')
        self.stratum_name = spec.get('alias', spec['morph'])
        self.field = field

    def __str__(self):
        format_string = ('%(kind)s morphology %(morphology_name)s refers to '
                         'stratum %(stratum_name)s with the %(field)s field. '
                         'Defaulting to null.')
        return format_string % self.__dict__


class MorphologySyntaxError(morphlib.Error):
    pass


class MorphologyNotYamlError(MorphologySyntaxError):

    def __init__(self, morphology, errmsg):
        self.msg = 'Syntax error in morphology %s:\n%s' % (morphology, errmsg)


class NotADictionaryError(MorphologySyntaxError):

    def __init__(self, morph_filename):
        self.msg = 'Not a dictionary: morphology %s' % morph_filename


class MorphologyValidationError(morphlib.Error):
    pass


class UnknownKindError(MorphologyValidationError):

    def __init__(self, kind, morph_filename):
        self.msg = (
            'Unknown kind %s in morphology %s' % (kind, morph_filename))


class MissingFieldError(MorphologyValidationError):

    def __init__(self, field, morphology_name):
        self.field = field
        self.morphology_name = morphology_name
        self.msg = (
            'Missing field %s from morphology %s' % (field, morphology_name))


class InvalidFieldError(MorphologyValidationError):

    def __init__(self, field, morphology_name):
        self.field = field
        self.morphology_name = morphology_name
        self.msg = (
            'Field %s not allowed in morphology %s' % (field, morphology_name))


class InvalidTypeError(MorphologyValidationError):

    def __init__(self, field, expected, actual, morphology_name):
        self.field = field
        self.expected = expected
        self.actual = actual
        self.morphology_name = morphology_name
        self.msg = (
            'Field %s expected type %s, got %s in morphology %s' %
            (field, expected, actual, morphology_name))


class ObsoleteFieldsError(MorphologyValidationError):

    def __init__(self, fields, morph_filename):
        self.msg = (
           'Morphology %s uses obsolete fields: %s' % 
           (morph_filename, ' '.join(fields)))


class UnknownArchitectureError(MorphologyValidationError):

    def __init__(self, arch, morph_filename):
        self.msg = ('Unknown architecture %s in morphology %s'
                    % (arch, morph_filename))


class NoBuildDependenciesError(MorphologyValidationError):

    def __init__(self, stratum_name, chunk_name, morph_filename):
        self.msg = (
            'Stratum %s has no build dependencies for chunk %s in %s' %
                (stratum_name, chunk_name, morph_filename))


class NoStratumBuildDependenciesError(MorphologyValidationError):

    def __init__(self, stratum_name, morph_filename):
        self.msg = (
            'Stratum %s has no build dependencies in %s' %
                (stratum_name, morph_filename))


class EmptyStratumError(MorphologyValidationError):

    def __init__(self, stratum_name, morph_filename):
        self.msg = (
            'Stratum %s has no chunks in %s' %
                (stratum_name, morph_filename))


class DuplicateChunkError(MorphologyValidationError):

    def __init__(self, stratum_name, chunk_name):
        self.stratum_name = stratum_name
        self.chunk_name = chunk_name
        MorphologyValidationError.__init__(
            self, 'Duplicate chunk %(chunk_name)s '\
                  'in stratum %(stratum_name)s' % locals())


class EmptyRefError(MorphologyValidationError):

    def __init__(self, ref_location, morph_filename):
        self.ref_location = ref_location
        self.morph_filename = morph_filename
        MorphologyValidationError.__init__(
            self, 'Empty ref found for %(ref_location)s '\
                  'in %(morph_filename)s' % locals())


class ChunkSpecRefNotStringError(MorphologyValidationError):

    def __init__(self, ref_value, chunk_name, stratum_name):
        self.ref_value = ref_value
        self.chunk_name = chunk_name
        self.stratum_name = stratum_name
        MorphologyValidationError.__init__(
            self, 'Ref %(ref_value)s for %(chunk_name)s '\
                  'in stratum %(stratum_name)s is not a string' % locals())


class SystemStrataNotListError(MorphologyValidationError):

    def __init__(self, system_name, strata_type):
        self.system_name = system_name
        self.strata_type = strata_type
        typename = strata_type.__name__
        MorphologyValidationError.__init__(
            self, 'System %(system_name)s has the wrong type for its strata: '\
                  '%(typename)s, expected list' % locals())


class DuplicateStratumError(MorphologyValidationError):

    def __init__(self, system_name, stratum_name):
        self.system_name = system_name
        self.stratum_name = stratum_name
        MorphologyValidationError.__init__(
            self, 'Duplicate stratum %(stratum_name)s '\
                  'in system %(system_name)s' % locals())


class SystemStratumSpecsNotMappingError(MorphologyValidationError):

    def __init__(self, system_name, strata):
        self.system_name = system_name
        self.strata = strata
        MorphologyValidationError.__init__(
            self, 'System %(system_name)s has stratum specs '\
                  'that are not mappings.' % locals())


class EmptySystemError(MorphologyValidationError):

    def __init__(self, system_name):
        MorphologyValidationError.__init__(
            self, 'System %(system_name)s has no strata.' % locals())


class MultipleValidationErrors(MorphologyValidationError):

    def __init__(self, name, errors):
        self.name = name
        self.errors = errors
        self.msg = 'Multiple errors when validating %(name)s:'
        for error in errors:
            self.msg += ('\n' + str(error))


class DuplicateDeploymentNameError(MorphologyValidationError):

    def __init__(self, cluster_filename, duplicates):
        self.duplicates = duplicates
        self.cluster_filename = cluster_filename
        morphlib.Error.__init__(self,
            'Cluster %s contains the following duplicate deployment names:%s'
            % (cluster_filename, '\n    ' + '\n    '.join(duplicates)))


class OrderedDumper(yaml.SafeDumper):
    keyorder = (
        'name',
        'kind',
        'description',
        'arch',
        'strata',
        'configuration-extensions',
        'morph',
        'repo',
        'ref',
        'unpetrify-ref',
        'build-depends',
        'build-mode',
        'artifacts',
        'max-jobs',
        'products',
        'chunks',
        'build-system',
        'pre-configure-commands',
        'configure-commands',
        'post-configure-commands',
        'pre-build-commands',
        'build-commands',
        'post-build-commands',
        'pre-install-commands',
        'install-commands',
        'post-install-commands',
        'artifact',
        'include',
        'systems',
        'deploy',
        'type',
        'location',
    )

    @classmethod
    def _iter_in_global_order(cls, mapping):
        for key in cls.keyorder:
            if key in mapping:
                yield key, mapping[key]
        for key in sorted(mapping.iterkeys()):
            if key not in cls.keyorder:
                yield key, mapping[key]

    @classmethod
    def _represent_dict(cls, dumper, mapping):
        return dumper.represent_mapping('tag:yaml.org,2002:map',
                                        cls._iter_in_global_order(mapping))

    def __init__(self, *args, **kwargs):
        yaml.SafeDumper.__init__(self, *args, **kwargs)
        self.add_representer(dict, self._represent_dict)


class MorphologyLoader(object):

    '''Load morphologies from disk, or save them back to disk.'''

    _required_fields = {
        'chunk': [
            'name',
        ],
        'stratum': [
            'name',
        ],
        'system': [
            'name',
            'arch',
            'strata',
        ],
        'cluster': [
          'name',
          'systems',
        ],
    }

    _obsolete_fields = {
        'system': [
            'system-kind',
            'disk-size',
        ],
    }

    _static_defaults = {
        'chunk': {
            'description': '',
            'pre-configure-commands': [],
            'configure-commands': [],
            'post-configure-commands': [],
            'pre-build-commands': [],
            'build-commands': [],
            'post-build-commands': [],
            'pre-test-commands': [],
            'test-commands': [],
            'post-test-commands': [],
            'pre-install-commands': [],
            'install-commands': [],
            'post-install-commands': [],
            'devices': [],
            'products': [],
            'max-jobs': None,
            'build-system': 'manual',
        },
        'stratum': {
            'chunks': [],
            'description': '',
            'build-depends': [],
            'products': [],
        },
        'system': {
            'description': '',
            'arch': None,
            'configuration-extensions': [],
        },
        'cluster': {
            'description': '',
        },
    }

    def parse_morphology_text(self, text, morph_filename):
        '''Parse a textual morphology.

        The text may be a string, or an open file handle.

        Return the new Morphology object, or raise an error indicating
        the problem. This method does minimal validation: a syntactically
        correct morphology is fine, even if none of the fields are
        valid. It also does not set any default values for any of the
        fields. See validate and set_defaults.

        '''

        try:
            obj = yaml.safe_load(text)
        except yaml.error.YAMLError as e:
            raise MorphologyNotYamlError(morph_filename, e)

        if not isinstance(obj, dict):
            raise NotADictionaryError(morph_filename)

        return morphlib.morph3.Morphology(obj)

    def load_from_string(self, string, filename='string'):
        '''Load a morphology from a string.

        Return the Morphology object.

        '''

        m = self.parse_morphology_text(string, filename)
        m.filename = filename
        self.validate(m)
        self.set_defaults(m)
        return m

    def load_from_file(self, filename):
        '''Load a morphology from a named file.

        Return the Morphology object.

        '''

        with open(filename) as f:
            text = f.read()
        return self.load_from_string(text, filename=filename)

    def save_to_string(self, morphology):
        '''Return normalised textual form of morphology.'''

        return yaml.dump(morphology.data, Dumper=OrderedDumper,
                         default_flow_style=False)

    def save_to_file(self, filename, morphology):
        '''Save a morphology object to a named file.'''

        text = self.save_to_string(morphology)
        with morphlib.savefile.SaveFile(filename, 'w') as f:
            f.write(text)

    def validate(self, morph):
        '''Validate a morphology.'''

        # Validate that the kind field is there.
        self._require_field('kind', morph)

        # The rest of the validation is dependent on the kind.

        # FIXME: move validation of clusters from morph2 to
        # here, and use morphload to load the morphology
        kind = morph['kind']
        if kind not in ('system', 'stratum', 'chunk', 'cluster'):
            raise UnknownKindError(morph['kind'], morph.filename)

        required = ['kind'] + self._required_fields[kind]
        obsolete = self._obsolete_fields.get(kind, [])
        allowed = self._static_defaults[kind].keys()
        self._require_fields(required, morph)
        self._deny_obsolete_fields(obsolete, morph)
        self._deny_unknown_fields(required + allowed, morph)

        getattr(self, '_validate_%s' % kind)(morph)

    def _validate_cluster(self, morph):
        # Deployment names must be unique within a cluster
        deployments = collections.Counter()
        for system in morph['systems']:
            deployments.update(system['deploy'].iterkeys())
            if 'subsystems' in system:
                deployments.update(self._get_subsystem_names(system))
        duplicates = set(deployment for deployment, count
                         in deployments.iteritems() if count > 1)
        if duplicates:
            raise DuplicateDeploymentNameError(morph.filename, duplicates)

    def _get_subsystem_names(self, system): # pragma: no cover
        for subsystem in system.get('subsystems', []):
            for name in subsystem['deploy'].iterkeys():
                yield name
            for name in self._get_subsystem_names(subsystem):
                yield name

    def _validate_system(self, morph):
        # A system must contain at least one stratum
        strata = morph['strata']
        if (not isinstance(strata, collections.Iterable)
            or isinstance(strata, collections.Mapping)):

            raise SystemStrataNotListError(morph['name'],
                                           type(strata))

        if not strata:
            raise EmptySystemError(morph['name'])

        if not all(isinstance(o, collections.Mapping) for o in strata):
            raise SystemStratumSpecsNotMappingError(morph['name'], strata)

        # All stratum names should be unique within a system.
        names = set()
        for spec in strata:
            name = spec.get('alias', spec['morph'])
            if name in names:
                raise DuplicateStratumError(morph['name'], name)
            names.add(name)

        # Validate stratum spec fields
        self._validate_stratum_specs_fields(morph, 'strata')

        # We allow the ARMv7 little-endian architecture to be specified
        # as armv7 and armv7l. Normalise.
        if morph['arch'] == 'armv7':
            morph['arch'] = 'armv7l'

        # Architecture name must be known.
        if morph['arch'] not in morphlib.valid_archs:
            raise UnknownArchitectureError(morph['arch'], morph.filename)

    def _validate_stratum(self, morph):
        # Require at least one chunk.
        if len(morph.get('chunks', [])) == 0:
            raise EmptyStratumError(morph['name'], morph.filename)

        # All chunk names must be unique within a stratum.
        names = set()
        for spec in morph['chunks']:
            name = spec.get('alias', spec['name'])
            if name in names:
                raise DuplicateChunkError(morph['name'], name)
            names.add(name)

        # All chunk refs must be strings.
        for spec in morph['chunks']:
            if 'ref' in spec:
                ref = spec['ref']
                if ref == None:
                    raise EmptyRefError(
                        spec.get('alias', spec['name']), morph.filename)
                elif not isinstance(ref, basestring):
                    raise ChunkSpecRefNotStringError(
                        ref, spec.get('alias', spec['name']), morph.filename)

        # Require build-dependencies for the stratum itself, unless
        # it has chunks built in bootstrap mode.
        if 'build-depends' not in morph:
            for spec in morph['chunks']:
                if spec.get('build-mode') in ['bootstrap', 'test']:
                    break
            else:
                raise NoStratumBuildDependenciesError(
                    morph['name'], morph.filename)

        # Validate build-dependencies if specified
        self._validate_stratum_specs_fields(morph, 'build-depends')

        # Require build-dependencies for each chunk.
        for spec in morph['chunks']:
            if 'build-depends' not in spec:
                raise NoBuildDependenciesError(
                    morph['name'],
                    spec.get('alias', spec['name']),
                    morph.filename)

    @classmethod
    def _validate_chunk(cls, morphology):
        errors = []

        if 'products' in morphology:
            cls._validate_products(morphology['name'],
                                   morphology['products'], errors)

        if len(errors) == 1:
            raise errors[0]
        elif errors:
            raise MultipleValidationErrors(morphology['name'], errors)

    @classmethod
    def _validate_products(cls, morphology_name, products, errors):
        '''Validate the products field is of the correct type.'''
        if (not isinstance(products, collections.Iterable)
            or isinstance(products, collections.Mapping)):
            raise InvalidTypeError('products', list,
                                   type(products), morphology_name)

        for spec_index, spec in enumerate(products):

            if not isinstance(spec, collections.Mapping):
                e = InvalidTypeError('products[%d]' % spec_index,
                                     dict, type(spec), morphology_name)
                errors.append(e)
                continue

            cls._validate_products_spec_fields_exist(morphology_name,
                                                     spec_index, spec, errors)

            if 'include' in spec:
                cls._validate_products_specs_include(
                    morphology_name, spec_index, spec['include'], errors)

    product_spec_required_fields = ('artifact', 'include')
    @classmethod
    def _validate_products_spec_fields_exist(
        cls, morphology_name, spec_index, spec, errors):

        given_fields = sorted(spec.iterkeys())
        missing = (field for field in cls.product_spec_required_fields
                   if field not in given_fields)
        for field in missing:
            e = MissingFieldError('products[%d].%s' % (spec_index, field),
                                  morphology_name)
            errors.append(e)
        unexpected = (field for field in given_fields
                      if field not in cls.product_spec_required_fields)
        for field in unexpected:
            e = InvalidFieldError('products[%d].%s' % (spec_index, field),
                                  morphology_name)
            errors.append(e)

    @classmethod
    def _validate_products_specs_include(cls, morphology_name, spec_index,
                                         include_patterns, errors):
        '''Validate that products' include field is a list of strings.'''
        # Allow include to be most iterables, but not a mapping
        # or a string, since iter of a mapping is just the keys,
        # and the iter of a string is a 1 character length string,
        # which would also validate as an iterable of strings.
        if (not isinstance(include_patterns, collections.Iterable)
            or isinstance(include_patterns, collections.Mapping)
            or isinstance(include_patterns, basestring)):

            e = InvalidTypeError('products[%d].include' % spec_index, list,
                                 type(include_patterns), morphology_name)
            errors.append(e)
        else:
            for pattern_index, pattern in enumerate(include_patterns):
                pattern_path = ('products[%d].include[%d]' %
                                (spec_index, pattern_index))
                if not isinstance(pattern, basestring):
                    e = InvalidTypeError(pattern_path, str,
                                         type(pattern), morphology_name)
                    errors.append(e)

    @classmethod
    def _warn_obsolete_field(cls, morphology, spec, field):
        warnings.warn(MorphologyObsoleteFieldWarning(morphology, spec, field),
                      stacklevel=2)

    @classmethod
    def _validate_stratum_specs_fields(cls, morphology, specs_field):
        for spec in morphology.get(specs_field, None) or []:
            for obsolete_field in ('repo', 'ref'):
                if obsolete_field in spec:
                    cls._warn_obsolete_field(morphology, spec, obsolete_field)

    def _require_field(self, field, morphology):
        if field not in morphology:
            raise MissingFieldError(field, morphology.filename)

    def _require_fields(self, fields, morphology):
        for field in fields:
            self._require_field(field, morphology)

    def _deny_obsolete_fields(self, fields, morphology):
        obsolete_ones = [x for x in morphology if x in fields]
        if obsolete_ones:
            raise ObsoleteFieldsError(obsolete_ones, morphology.filename)

    def _deny_unknown_fields(self, allowed, morphology):
        for field in morphology:
            if field not in allowed:
                raise InvalidFieldError(field, morphology.filename)

    def set_defaults(self, morphology):
        '''Set all missing fields in the morpholoy to their defaults.

        The morphology is assumed to be valid.

        '''

        kind = morphology['kind']
        defaults = self._static_defaults[kind]
        for key in defaults:
            if key not in morphology:
                morphology[key] = defaults[key]

        getattr(self, '_set_%s_defaults' % kind)(morphology)

    def unset_defaults(self, morphology):
        '''If a field is equal to its default, delete it.

        The morphology is assumed to be valid.

        '''

        kind = morphology['kind']
        defaults = self._static_defaults[kind]
        for key in defaults:
            if key in morphology and morphology[key] == defaults[key]:
                del morphology[key]

        if kind in ('system', 'stratum', 'cluster'):
            getattr(self, '_unset_%s_defaults' % kind)(morphology)

    @classmethod
    def _set_stratum_specs_defaults(cls, morphology, specs_field):
        for spec in morphology.get(specs_field, None) or []:
            for obsolete_field in ('repo', 'ref'):
                if obsolete_field in spec:
                    del spec[obsolete_field]

    @classmethod
    def _unset_stratum_specs_defaults(cls, morphology, specs_field):
        for spec in morphology.get(specs_field, []):
            for obsolete_field in ('repo', 'ref'):
                if obsolete_field in spec:
                    del spec[obsolete_field]

    def _set_cluster_defaults(self, morph):
        for system in morph.get('systems', []):
            if 'deploy-defaults' not in system:
                system['deploy-defaults'] = {}
            if 'deploy' not in system:
                system['deploy'] = {}

    def _unset_cluster_defaults(self, morph):
        for system in morph.get('systems', []):
            if 'deploy-defaults' in system and system['deploy-defaults'] == {}:
                del system['deploy-defaults']
            if 'deploy' in system and system['deploy'] == {}:
                del system['deploy']

    def _set_system_defaults(self, morph):
        self._set_stratum_specs_defaults(morph, 'strata')

    def _unset_system_defaults(self, morph):
        self._unset_stratum_specs_defaults(morph, 'strata')

    def _set_stratum_defaults(self, morph):
        for spec in morph['chunks']:
            if 'repo' not in spec:
                spec['repo'] = spec['name']
            if 'morph' not in spec:
                spec['morph'] = spec['name']
        self._set_stratum_specs_defaults(morph, 'build-depends')

    def _unset_stratum_defaults(self, morph):
        for spec in morph['chunks']:
            if 'repo' in spec and spec['repo'] == spec['name']:
                del spec['repo']
            if 'morph' in spec and spec['morph'] == spec['name']:
                del spec['morph']
        self._unset_stratum_specs_defaults(morph, 'strata')

    def _set_chunk_defaults(self, morph):
        if morph['max-jobs'] is not None:
            morph['max-jobs'] = int(morph['max-jobs'])