diff options
Diffstat (limited to 'morphlib')
| -rw-r--r-- | morphlib/__init__.py | 1 | ||||
| -rw-r--r-- | morphlib/artifactresolver_tests.py | 88 | ||||
| -rw-r--r-- | morphlib/artifactsplitrule.py | 18 | ||||
| -rw-r--r-- | morphlib/buildsystem.py | 15 | ||||
| -rw-r--r-- | morphlib/buildsystem_tests.py | 9 | ||||
| -rw-r--r-- | morphlib/cachekeycomputer_tests.py | 12 | ||||
| -rw-r--r-- | morphlib/defaults.py | 108 | ||||
| -rw-r--r-- | morphlib/definitions_repo.py | 21 | ||||
| -rw-r--r-- | morphlib/definitions_version.py | 2 | ||||
| -rw-r--r-- | morphlib/morphloader.py | 37 | ||||
| -rw-r--r-- | morphlib/morphloader_tests.py | 41 | ||||
| -rw-r--r-- | morphlib/morphologyfinder.py | 14 | ||||
| -rw-r--r-- | morphlib/morphologyfinder_tests.py | 4 | ||||
| -rw-r--r-- | morphlib/schemas/defaults.json-schema | 66 | ||||
| -rw-r--r-- | morphlib/source.py | 9 | ||||
| -rw-r--r-- | morphlib/sourceresolver.py | 69 | ||||
| -rw-r--r-- | morphlib/util.py | 7 |
17 files changed, 416 insertions, 105 deletions
diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 81540a31..36e5c3a8 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -60,6 +60,7 @@ import builder import cachedrepo import cachekeycomputer import cmdline_parse_utils +import defaults import definitions_repo import definitions_version import extensions diff --git a/morphlib/artifactresolver_tests.py b/morphlib/artifactresolver_tests.py index 20617c65..4bf42a93 100644 --- a/morphlib/artifactresolver_tests.py +++ b/morphlib/artifactresolver_tests.py @@ -20,6 +20,12 @@ import yaml import morphlib +default_split_rules = { + 'chunk': morphlib.artifactsplitrule.DEFAULT_CHUNK_RULES, + 'stratum': morphlib.artifactsplitrule.DEFAULT_STRATUM_RULES, +} + + def get_chunk_morphology(name, artifact_names=[]): assert(isinstance(artifact_names, list)) @@ -88,9 +94,9 @@ class ArtifactResolverTests(unittest.TestCase): pool = morphlib.sourcepool.SourcePool() morph = get_chunk_morphology('chunk') - sources = morphlib.source.make_sources('repo', 'ref', - 'chunk.morph', 'sha1', - 'tree', morph) + sources = morphlib.source.make_sources( + 'repo', 'ref', 'chunk.morph', 'sha1', 'tree', morph, + default_split_rules=default_split_rules) for source in sources: pool.add(source) @@ -109,9 +115,9 @@ class ArtifactResolverTests(unittest.TestCase): pool = morphlib.sourcepool.SourcePool() morph = get_chunk_morphology('chunk', ['chunk-foobar']) - sources = morphlib.source.make_sources('repo', 'ref', - 'chunk.morph', 'sha1', - 'tree', morph) + sources = morphlib.source.make_sources( + 'repo', 'ref', 'chunk.morph', 'sha1', 'tree', morph, + default_split_rules=default_split_rules) for source in sources: pool.add(source) @@ -129,9 +135,9 @@ class ArtifactResolverTests(unittest.TestCase): pool = morphlib.sourcepool.SourcePool() morph = get_chunk_morphology('chunk', ['chunk-baz', 'chunk-qux']) - sources = morphlib.source.make_sources('repo', 'ref', - 'chunk.morph', 'sha1', - 'tree', morph) + sources = morphlib.source.make_sources( + 'repo', 'ref', 'chunk.morph', 'sha1', 'tree', morph, + default_split_rules=default_split_rules) for source in sources: pool.add(source) @@ -151,18 +157,18 @@ class ArtifactResolverTests(unittest.TestCase): pool = morphlib.sourcepool.SourcePool() morph = get_chunk_morphology('chunk') - sources = morphlib.source.make_sources('repo', 'ref', - 'chunk.morph', 'sha1', - 'tree', morph) + sources = morphlib.source.make_sources( + 'repo', 'ref', 'chunk.morph', 'sha1', 'tree', morph, + default_split_rules=default_split_rules) for chunk in sources: pool.add(chunk) morph = get_stratum_morphology( 'stratum', chunks=[('chunk', 'chunk', 'repo', 'ref')]) - stratum_sources = set(morphlib.source.make_sources('repo', 'ref', - 'stratum.morph', - 'sha1', 'tree', - morph)) + stratum_sources = set( + morphlib.source.make_sources( + 'repo', 'ref', 'stratum.morph', 'sha1', 'tree', morph, + default_split_rules=default_split_rules)) for stratum in stratum_sources: pool.add(stratum) @@ -197,9 +203,9 @@ class ArtifactResolverTests(unittest.TestCase): pool = morphlib.sourcepool.SourcePool() morph = get_chunk_morphology('chunk', ['chunk-foo', 'chunk-bar']) - sources = morphlib.source.make_sources('repo', 'ref', - 'chunk.morph', 'sha1', - 'tree', morph) + sources = morphlib.source.make_sources( + 'repo', 'ref', 'chunk.morph', 'sha1', 'tree', morph, + default_split_rules=default_split_rules) for chunk in sources: pool.add(chunk) @@ -208,10 +214,10 @@ class ArtifactResolverTests(unittest.TestCase): chunks=[ ('chunk', 'chunk', 'repo', 'ref'), ]) - stratum_sources = set(morphlib.source.make_sources('repo', 'ref', - 'stratum.morph', - 'sha1', 'tree', - morph)) + stratum_sources = set( + morphlib.source.make_sources( + 'repo', 'ref', 'stratum.morph', 'sha1', 'tree', morph, + default_split_rules=default_split_rules)) for stratum in stratum_sources: pool.add(stratum) @@ -246,7 +252,8 @@ class ArtifactResolverTests(unittest.TestCase): chunk = get_chunk_morphology('chunk1') chunk1, = morphlib.source.make_sources( - 'repo', 'original/ref', 'chunk1.morph', 'sha1', 'tree', chunk) + 'repo', 'original/ref', 'chunk1.morph', 'sha1', 'tree', chunk, + default_split_rules=default_split_rules) pool.add(chunk1) morph = get_stratum_morphology( @@ -254,15 +261,16 @@ class ArtifactResolverTests(unittest.TestCase): chunks=[(loader.save_to_string(chunk), 'chunk1.morph', 'repo', 'original/ref')], build_depends=['stratum2']) - sources = morphlib.source.make_sources('repo', 'original/ref', - 'stratum1.morph', 'sha1', - 'tree', morph) + sources = morphlib.source.make_sources( + 'repo', 'original/ref', 'stratum1.morph', 'sha1', 'tree', morph, + default_split_rules=default_split_rules) for stratum1 in sources: pool.add(stratum1) chunk = get_chunk_morphology('chunk2') chunk2, = morphlib.source.make_sources( - 'repo', 'original/ref', 'chunk2.morph', 'sha1', 'tree', chunk) + 'repo', 'original/ref', 'chunk2.morph', 'sha1', 'tree', chunk, + default_split_rules=default_split_rules) pool.add(chunk2) morph = get_stratum_morphology( @@ -270,9 +278,9 @@ class ArtifactResolverTests(unittest.TestCase): chunks=[(loader.save_to_string(chunk), 'chunk2.morph', 'repo', 'original/ref')], build_depends=['stratum1']) - sources = morphlib.source.make_sources('repo', 'original/ref', - 'stratum2.morph', 'sha1', - 'tree', morph) + sources = morphlib.source.make_sources( + 'repo', 'original/ref', 'stratum2.morph', 'sha1', 'tree', morph, + default_split_rules=default_split_rules) for stratum2 in sources: pool.add(stratum2) @@ -301,23 +309,23 @@ class ArtifactResolverTests(unittest.TestCase): build-system: manual build-depends: [] ''') - sources = morphlib.source.make_sources('repo', 'original/ref', - 'stratum.morph', 'sha1', - 'tree', morph) + sources = morphlib.source.make_sources( + 'repo', 'original/ref', 'stratum.morph', 'sha1', 'tree', morph, + default_split_rules=default_split_rules) for stratum in sources: pool.add(stratum) morph = get_chunk_morphology('chunk1') - sources = morphlib.source.make_sources('repo', 'original/ref', - 'chunk1.morph', 'sha1', - 'tree', morph) + sources = morphlib.source.make_sources( + 'repo', 'original/ref', 'chunk1.morph', 'sha1', 'tree', morph, + default_split_rules=default_split_rules) for chunk1 in sources: pool.add(chunk1) morph = get_chunk_morphology('chunk2') - sources = morphlib.source.make_sources('repo', 'original/ref', - 'chunk2.morph', 'sha1', - 'tree', morph) + sources = morphlib.source.make_sources( + 'repo', 'original/ref', 'chunk2.morph', 'sha1', 'tree', morph, + default_split_rules=default_split_rules) for chunk2 in sources: pool.add(chunk2) diff --git a/morphlib/artifactsplitrule.py b/morphlib/artifactsplitrule.py index b5ebdf83..ba5abe02 100644 --- a/morphlib/artifactsplitrule.py +++ b/morphlib/artifactsplitrule.py @@ -189,10 +189,6 @@ class SplitRules(collections.Iterable): for artifact, rule in self._rules) -# TODO: Work out a good way to feed new defaults in. This is good for -# the usual Linux userspace, but we may find issues and need a -# migration path to a more useful set, or develop a system with -# a different layout, like Android. DEFAULT_CHUNK_RULES = [ ('-bins', [ r"(usr/)?s?bin/.*" ]), ('-libs', [ @@ -229,6 +225,14 @@ DEFAULT_STRATUM_RULES = [ ] +# A 'no-op' set of split rules. An empty list would cause everything to be +# ignored, which is unlikely to ever be what a user wants, and breaks some +# internal bits of Morph. +EMPTY_RULES = [ + ('', [r'.*']) +] + + def unify_chunk_matches(morphology, default_rules=DEFAULT_CHUNK_RULES): '''Create split rules including defaults and per-chunk rules. @@ -237,6 +241,8 @@ def unify_chunk_matches(morphology, default_rules=DEFAULT_CHUNK_RULES): by building the chunk to the chunk artifact they should be put in. ''' + if default_rules is None or len(default_rules) == 0: + default_rules = EMPTY_RULES split_rules = SplitRules() @@ -265,6 +271,8 @@ def unify_stratum_matches(morphology, default_rules=DEFAULT_STRATUM_RULES): strata to the stratum artifact they should be put in. ''' + if default_rules is None or len(default_rules) == 0: + default_rules = EMPTY_RULES assignment_split_rules = SplitRules() for spec in morphology['chunks']: @@ -296,7 +304,7 @@ def unify_stratum_matches(morphology, default_rules=DEFAULT_STRATUM_RULES): match_split_rules)) -def unify_system_matches(morphology): +def unify_system_matches(morphology, default_rules=[]): '''Create split rules including defaults and per-chunk rules. With rules specified in the morphology's 'products' field and the diff --git a/morphlib/buildsystem.py b/morphlib/buildsystem.py index d7fbaf73..29d35627 100644 --- a/morphlib/buildsystem.py +++ b/morphlib/buildsystem.py @@ -44,11 +44,10 @@ _STRIP_COMMAND = r'''find "$DESTDIR" -type f \ class BuildSystem(object): - '''Predefined commands for common build systems. + '''Predefined command sequences for a given build system. - Some build systems are well known: autotools, for example. We provide - pre-defined build commands for these so that they don't need to be copied - and pasted many times in the build instructions. + For example, you can have an 'autotools' build system, which runs + 'configure', 'make' and 'make install'. ''' @@ -69,6 +68,14 @@ class BuildSystem(object): self.strip_commands = [] self.post_strip_commands = [] + def from_dict(self, name, commands): + self.name = name + + self.configure_commands = commands.get('configure-commands', []) + self.build_commands = commands.get('build-commands', []) + self.install_commands = commands.get('install-commands', []) + self.strip_commands = commands.get('strip-commands', []) + def __getitem__(self, key): key = '_'.join(key.split('-')) return getattr(self, key) diff --git a/morphlib/buildsystem_tests.py b/morphlib/buildsystem_tests.py index b49d30ae..6b6bf30f 100644 --- a/morphlib/buildsystem_tests.py +++ b/morphlib/buildsystem_tests.py @@ -43,6 +43,15 @@ class BuildSystemTests(unittest.TestCase): morph = self.bs.get_morphology('foobar') self.assertTrue(morph.__class__.__name__ == 'Morphology') + def test_construct_from_dict(self): + '''Test parsing a dict of information from a DEFAULTS file.''' + + commands_dict = { + 'configure-commands': 'foo' + } + self.bs.from_dict('test', commands_dict) + self.assertEqual(self.bs.configure_commands, 'foo') + class LookupBuildSystemTests(unittest.TestCase): diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py index fbf680f0..6e736796 100644 --- a/morphlib/cachekeycomputer_tests.py +++ b/morphlib/cachekeycomputer_tests.py @@ -28,6 +28,12 @@ class DummyBuildEnvironment: self.env = env +default_split_rules = { + 'chunk': morphlib.artifactsplitrule.DEFAULT_CHUNK_RULES, + 'stratum': morphlib.artifactsplitrule.DEFAULT_STRATUM_RULES, +} + + class CacheKeyComputerTests(unittest.TestCase): def setUp(self): @@ -86,9 +92,9 @@ class CacheKeyComputerTests(unittest.TestCase): ''', }.iteritems(): morph = loader.load_from_string(text) - sources = morphlib.source.make_sources('repo', 'original/ref', - name, 'sha1', - 'tree', morph) + sources = morphlib.source.make_sources( + 'repo', 'original/ref', name, 'sha1', 'tree', morph, + default_split_rules=default_split_rules) for source in sources: self.source_pool.add(source) self.build_env = DummyBuildEnvironment({ diff --git a/morphlib/defaults.py b/morphlib/defaults.py new file mode 100644 index 00000000..9e695a90 --- /dev/null +++ b/morphlib/defaults.py @@ -0,0 +1,108 @@ +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. +# +# =*= License: GPL-2 =*= + + +import cliapp +import jsonschema +import yaml + +import os + +import morphlib + + +class Defaults(object): + '''Represents predefined default values specific to Baserock definitions. + + The DEFAULTS file was added in definitions format version 7, which lets + users set these defaults. The text of DEFAULTS file can be passed in as + 'text', and will be validated and parsed if definitions_version >= 7. + + Prior to version 7, the defaults were hardcoded in Morph. These defaults + will be returned if definitions_version < 7. + + ''' + def __init__(self, definitions_version, text=None): + self._build_systems = {} + self._split_rules = {} + + schema_path = os.path.join(morphlib.util.schemas_directory(), + 'defaults.json-schema') + with open(schema_path) as f: + self.schema = yaml.load(f) + + if definitions_version >= 7: + if text: + self._build_systems, self._split_rules = self._parse(text) + else: + self._build_systems, self._split_rules = self._builtins() + + def _parse(self, text): + build_systems = {} + split_rules = {} + + # This reports errors against <string> rather than the actual filename, + # which is sad. + data = yaml.safe_load(text) + + if data is None: + # It's OK to be empty, I guess. + return build_systems, split_rules + + try: + # It would be nice if this could give line numbers when it spotted + # errors. Seems tricky. + jsonschema.validate(data, self.schema) + except jsonschema.ValidationError as e: + raise cliapp.AppException('Invalid DEFAULTS file: %s' % e.message) + + build_system_data = data.get('build-systems', {}) + for name, commands in build_system_data.items(): + build_system = morphlib.buildsystem.BuildSystem() + build_system.from_dict(name, commands) + build_systems[name] = build_system + + # It would make sense to create artifactsplitrule.SplitRule instances + # here, instead of an unlabelled data structure. That would need some + # changes to source.make_sources() and the 'artifactsplitrule' module. + split_rules_data = data.get('split-rules', {}) + for kind, rules in split_rules_data.items(): + split_rules[kind] = [] + for rule in rules: + rule_unlabelled = (rule['artifact'], rule['include']) + split_rules[kind].append(rule_unlabelled) + + return build_systems, split_rules + + def _builtins(self): + build_systems = {} + split_rules = {} + + for build_system in morphlib.buildsystem.build_systems: + build_systems[build_system.name] = build_system + + split_rules['chunk'] = \ + morphlib.artifactsplitrule.DEFAULT_CHUNK_RULES + split_rules['stratum'] = \ + morphlib.artifactsplitrule.DEFAULT_STRATUM_RULES + + return build_systems, split_rules + + def build_systems(self): + return self._build_systems + + def split_rules(self): + return self._split_rules diff --git a/morphlib/definitions_repo.py b/morphlib/definitions_repo.py index 301e5b86..6d14fcbb 100644 --- a/morphlib/definitions_repo.py +++ b/morphlib/definitions_repo.py @@ -23,6 +23,7 @@ import logging import os import urlparse import uuid +import warnings import morphlib import gitdir @@ -236,9 +237,25 @@ class DefinitionsRepo(gitdir.GitDirectory): mf = morphlib.morphologyfinder.MorphologyFinder(self) version_text = mf.read_file('VERSION') - morphlib.definitions_version.check_version_file(version_text) + version = morphlib.definitions_version.check_version_file(version_text) - loader = morphlib.morphloader.MorphologyLoader() + defaults_text = mf.read_file('DEFAULTS', allow_missing=True) + + if version < 7: + if defaults_text is not None: + warnings.warn( + "Ignoring DEFAULTS file, because these definitions are " + "version %i" % version) + defaults_text = None + else: + if defaults_text is None: + warnings.warn("No DEFAULTS file found.") + + defaults = morphlib.defaults.Defaults(version, + text=defaults_text) + + loader = morphlib.morphloader.MorphologyLoader( + predefined_build_systems=defaults.build_systems()) return loader diff --git a/morphlib/definitions_version.py b/morphlib/definitions_version.py index bc72d2ad..2fb7785a 100644 --- a/morphlib/definitions_version.py +++ b/morphlib/definitions_version.py @@ -24,7 +24,7 @@ import yaml import morphlib -SUPPORTED_VERSIONS = [6] +SUPPORTED_VERSIONS = [6, 7] class DefinitionsVersionError(cliapp.AppException): diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index f85c5d4d..ba48b778 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -109,6 +109,13 @@ class UnknownArchitectureError(MorphologyValidationError): % (arch, morph_filename)) +class UnknownBuildSystemError(MorphologyValidationError): + + def __init__(self, build_system, morph_filename): + self.msg = ('Undefined build system %s in morphology %s' + % (build_system, morph_filename)) + + class NoStratumBuildDependenciesError(MorphologyValidationError): def __init__(self, stratum_name, morph_filename): @@ -404,8 +411,12 @@ class MorphologyLoader(object): } def __init__(self, - lookup_build_system=morphlib.buildsystem.lookup_build_system): - self._lookup_build_system = lookup_build_system + predefined_build_systems={}): + self._predefined_build_systems = predefined_build_systems.copy() + + if 'manual' not in self._predefined_build_systems: + self._predefined_build_systems['manual'] = \ + morphlib.buildsystem.ManualBuildSystem() def parse_morphology_text(self, text, morph_filename): '''Parse a textual morphology. @@ -811,10 +822,12 @@ class MorphologyLoader(object): if morph['max-jobs'] is not None: morph['max-jobs'] = int(morph['max-jobs']) - def _unset_chunk_defaults(self, morph): # pragma: no cover + def _unset_chunk_defaults(self, morph): # pragma: no cover + # This is only used by the deprecated branch-and-merge plugin, and + # probably doesn't work correctly for definitions V7 and newer. default_bs = self._static_defaults['chunk']['build-system'] - bs = self._lookup_build_system( - morph.get('build-system', default_bs)) + bs_name = morph.get('build-system', default_bs) + bs = self.lookup_build_system(bs_name) for key in self._static_defaults['chunk']: if key not in morph: continue if 'commands' not in key: continue @@ -823,11 +836,19 @@ class MorphologyLoader(object): if morph[key] == default_value: del morph[key] + def lookup_build_system(self, name): + return self._predefined_build_systems[name] + def set_commands(self, morph): - default = self._static_defaults['chunk']['build-system'] - bs = self._lookup_build_system( - morph.get('build-system', default)) if morph['kind'] == 'chunk': + default = self._static_defaults['chunk']['build-system'] + bs_name = morph.get('build-system', default) + + try: + bs = self.lookup_build_system(bs_name) + except KeyError: + raise UnknownBuildSystemError(bs_name, morph['name']) + for key in self._static_defaults['chunk']: if 'commands' not in key: continue if key not in morph: diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py index 6117573e..392b0cbc 100644 --- a/morphlib/morphloader_tests.py +++ b/morphlib/morphloader_tests.py @@ -60,12 +60,12 @@ class MorphologyLoaderTests(unittest.TestCase): string = '''\ name: foo kind: chunk -build-system: dummy +build-system: manual ''' morph = self.loader.parse_morphology_text(string, 'test') self.assertEqual(morph['kind'], 'chunk') self.assertEqual(morph['name'], 'foo') - self.assertEqual(morph['build-system'], 'dummy') + self.assertEqual(morph['build-system'], 'manual') def test_fails_to_parse_utter_garbage(self): self.assertRaises( @@ -494,43 +494,43 @@ chunks: string = '''\ name: foo kind: chunk -build-system: dummy +build-system: manual ''' morph = self.loader.load_from_string(string) self.assertEqual(morph['kind'], 'chunk') self.assertEqual(morph['name'], 'foo') - self.assertEqual(morph['build-system'], 'dummy') + self.assertEqual(morph['build-system'], 'manual') def test_loads_json_from_string(self): string = '''\ { "name": "foo", "kind": "chunk", - "build-system": "dummy" + "build-system": "manual" } ''' morph = self.loader.load_from_string(string) self.assertEqual(morph['kind'], 'chunk') self.assertEqual(morph['name'], 'foo') - self.assertEqual(morph['build-system'], 'dummy') + self.assertEqual(morph['build-system'], 'manual') def test_loads_from_file(self): with open(self.filename, 'w') as f: f.write('''\ name: foo kind: chunk -build-system: dummy +build-system: manual ''') morph = self.loader.load_from_file(self.filename) self.assertEqual(morph['kind'], 'chunk') self.assertEqual(morph['name'], 'foo') - self.assertEqual(morph['build-system'], 'dummy') + self.assertEqual(morph['build-system'], 'manual') def test_saves_to_string(self): morph = morphlib.morphology.Morphology({ 'name': 'foo', 'kind': 'chunk', - 'build-system': 'dummy', + 'build-system': 'manual', }) text = self.loader.save_to_string(morph) @@ -539,14 +539,14 @@ build-system: dummy self.assertEqual(text, '''\ name: foo kind: chunk -build-system: dummy +build-system: manual ''') def test_saves_to_file(self): morph = morphlib.morphology.Morphology({ 'name': 'foo', 'kind': 'chunk', - 'build-system': 'dummy', + 'build-system': 'manual', }) self.loader.save_to_file(self.filename, morph) @@ -558,7 +558,7 @@ build-system: dummy self.assertEqual(text, '''\ name: foo kind: chunk -build-system: dummy +build-system: manual ''') def test_validate_does_not_set_defaults(self): @@ -966,11 +966,12 @@ build-system: dummy ) s = self.loader.save_to_string(m) - def test_smoketest_strip_commands(self): - dummy_buildsystem = morphlib.buildsystem.DummyBuildSystem() - loader = morphlib.morphloader.MorphologyLoader( - lookup_build_system=lambda x: dummy_buildsystem) - m = morphlib.morphology.Morphology( - {'name': 'test', 'kind': 'chunk', 'build-system': 'dummy'}) - loader.set_commands(m) - self.assertEqual(m['strip-commands'], dummy_buildsystem.strip_commands) + + def test_unknown_build_system(self): + m = morphlib.morphology.Morphology({ + 'kind': 'chunk', + 'name': 'foo', + 'build-system': 'monkey scientist', + }) + with self.assertRaises(morphlib.morphloader.UnknownBuildSystemError): + s = self.loader.set_commands(m) diff --git a/morphlib/morphologyfinder.py b/morphlib/morphologyfinder.py index 9184c160..a334350e 100644 --- a/morphlib/morphologyfinder.py +++ b/morphlib/morphologyfinder.py @@ -15,9 +15,7 @@ # =*= License: GPL-2 =*= -import cliapp - -import morphlib +import errno class MorphologyFinder(object): @@ -33,9 +31,15 @@ class MorphologyFinder(object): self.gitdir = gitdir self.ref = ref - def read_file(self, filename): + def read_file(self, filename, allow_missing=False): '''Return the text of a file inside the Git repo.''' - return self.gitdir.read_file(filename, self.ref) + try: + return self.gitdir.read_file(filename, self.ref) + except IOError as e: + if allow_missing and e.errno == errno.ENOENT: + return None + else: + raise def list_morphologies(self): '''Return the filenames of all morphologies in the (repo, ref). diff --git a/morphlib/morphologyfinder_tests.py b/morphlib/morphologyfinder_tests.py index 59772e23..3312b627 100644 --- a/morphlib/morphologyfinder_tests.py +++ b/morphlib/morphologyfinder_tests.py @@ -103,6 +103,10 @@ class MorphologyFinderTests(unittest.TestCase): mf = morphlib.morphologyfinder.MorphologyFinder(gd) self.assertEqual(mf.read_file('foo.morph'), "altered morphology text") + with self.assertRaises(IOError): + mf.read_file('missing') + self.assertEqual(mf.read_file('missing', allow_missing=True), + None) def test_read_morph_raises_no_worktree_no_ref(self): gd = morphlib.gitdir.GitDirectory(self.mirror) diff --git a/morphlib/schemas/defaults.json-schema b/morphlib/schemas/defaults.json-schema new file mode 100644 index 00000000..2f713425 --- /dev/null +++ b/morphlib/schemas/defaults.json-schema @@ -0,0 +1,66 @@ +$schema: http://json-schema.org/draft-04/schema# +id: http://git.baserock.org/cgi-bin/cgit.cgi/baserock/baserock/definitions.git/tree/schemas/defaults.json-schema + +description: | + This is a JSON-Schema description of the DEFAULTS file specified in the + Baserock definitions format. DEFAULTS is a YAML file that contains global + defaults for a set of Baserock definitions. + + This JSON-Schema file is valid for VERSION 7 of the Baserock definitions + YAML serialisation format. + + The Baserock definitions YAML serialisation format is the recommended way of + representing Baserock definitions on disk. The actual data model is described + separately. See <https://wiki.baserock.org/definitions> for more information. + + This schema is represented as YAML, so that it can be edited more easily. + You may need to convert to JSON if using a JSON-Schema tool that expects + its input to be an actual string containing data serialised as JSON. + +definitions: + command-sequence: + type: array + items: {type: string} + + build-system: + type: object + additionalProperties: false + properties: + build-commands: {$ref: '#/definitions/command-sequence'} + configure-commands: {$ref: '#/definitions/command-sequence'} + install-commands: {$ref: '#/definitions/command-sequence'} + strip-commands: {$ref: '#/definitions/command-sequence'} + + split-rules: + type: array + items: + type: object + + required: [artifact, include] + additionalProperties: false + + properties: + artifact: {type: string} + include: + type: array + items: + type: string + format: regex + +type: object +additionalProperties: false + +properties: + # Predefined build systems. + build-systems: + type: object + patternProperties: + ^.*$: {$ref: '#/definitions/build-system'} + + # Predefined artifact splitting rules. + split-rules: + type: object + additionalProperties: false + properties: + chunk: {$ref: '#/definitions/split-rules'} + stratum: {$ref: '#/definitions/split-rules'} diff --git a/morphlib/source.py b/morphlib/source.py index 2c96bcbd..135c14cc 100644 --- a/morphlib/source.py +++ b/morphlib/source.py @@ -77,12 +77,14 @@ class Source(object): return artifact in self.dependencies -def make_sources(reponame, ref, filename, absref, tree, morphology): +def make_sources(reponame, ref, filename, absref, tree, morphology, + default_split_rules={}): kind = morphology['kind'] if kind in ('system', 'chunk'): unifier = getattr(morphlib.artifactsplitrule, 'unify_%s_matches' % kind) - split_rules = unifier(morphology) + split_rules = unifier(morphology, + default_rules=default_split_rules.get(kind, {})) # chunk and system sources are named after the morphology source_name = morphology['name'] source = morphlib.source.Source(source_name, reponame, ref, @@ -93,7 +95,8 @@ def make_sources(reponame, ref, filename, absref, tree, morphology): yield source elif kind == 'stratum': # pragma: no cover unifier = morphlib.artifactsplitrule.unify_stratum_matches - split_rules = unifier(morphology) + split_rules = unifier(morphology, + default_rules=default_split_rules.get(kind, {})) for name in split_rules.artifacts: source = morphlib.source.Source( name, # stratum source name is artifact name diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index 0b32598f..1056af16 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -19,7 +19,7 @@ import cPickle import logging import os import pylru -import yaml +import warnings import cliapp @@ -259,6 +259,36 @@ class SourceResolver(object): return morphlib.definitions_version.check_version_file(version_text) + def _get_defaults(self, definitions_checkout_dir, + definitions_version=7): # pragma: no cover + '''Return the default build system commands, and default split rules. + + This function returns a tuple with two dicts. + + The defaults are read from a file named DEFAULTS in the definitions + directory, if the definitions follow format version 7 or later. If the + definitions follow version 6 or earlier, hardcoded defaults are used. + + ''' + # Read default build systems and split rules from DEFAULTS file. + defaults_text = self._get_file_contents_from_definitions( + definitions_checkout_dir, 'DEFAULTS') + + if definitions_version < 7: + if defaults_text is not None: + warnings.warn( + "Ignoring DEFAULTS file, because these definitions are " + "version %i" % definitions_version) + defaults_text = None + else: + if defaults_text is None: + warnings.warn("No DEFAULTS file found.") + + defaults = morphlib.defaults.Defaults(definitions_version, + text=defaults_text) + + return defaults.build_systems(), defaults.split_rules() + def _get_morphology(self, resolved_morphologies, definitions_checkout_dir, morph_loader, filename): '''Read the morphology at the specified location. @@ -287,7 +317,8 @@ class SourceResolver(object): definitions_tree, morph_loader, system_filenames, - visit): + visit, + predefined_split_rules): definitions_queue = collections.deque(system_filenames) chunk_queue = set() @@ -304,7 +335,8 @@ class SourceResolver(object): raise MorphologyNotFoundError(filename) visit(definitions_repo, definitions_ref, filename, - definitions_absref, definitions_tree, morphology) + definitions_absref, definitions_tree, morphology, + predefined_split_rules) if morphology['kind'] == 'cluster': raise cliapp.AppException( @@ -349,8 +381,8 @@ class SourceResolver(object): def process_chunk(self, resolved_morphologies, resolved_trees, definitions_checkout_dir, morph_loader, chunk_repo, - chunk_ref, filename, chunk_buildsystem, - visit): + chunk_ref, filename, chunk_buildsystem, visit, + predefined_split_rules): absref, tree = self._resolve_ref(resolved_trees, chunk_repo, chunk_ref) if chunk_buildsystem is None: @@ -370,7 +402,8 @@ class SourceResolver(object): morphology = self._create_morphology_for_build_system( morph_loader, buildsystem, filename) - visit(chunk_repo, chunk_ref, filename, absref, tree, morphology) + visit(chunk_repo, chunk_ref, filename, absref, tree, morphology, + predefined_split_rules) def traverse_morphs(self, definitions_repo, definitions_ref, system_filenames, @@ -400,7 +433,13 @@ class SourceResolver(object): definitions_version = self._check_version_file( definitions_checkout_dir) - morph_loader = morphlib.morphloader.MorphologyLoader() + + predefined_build_systems, predefined_split_rules = \ + self._get_defaults( + definitions_checkout_dir, definitions_version) + + morph_loader = morphlib.morphloader.MorphologyLoader( + predefined_build_systems=predefined_build_systems) # First, process the system and its stratum morphologies. These # will all live in the same Git repository, and will point to @@ -408,14 +447,15 @@ class SourceResolver(object): chunk_queue = self._process_definitions_with_children( resolved_morphologies, definitions_checkout_dir, definitions_repo, definitions_ref, definitions_absref, - definitions_tree, morph_loader, - system_filenames, visit) + definitions_tree, morph_loader, system_filenames, visit, + predefined_split_rules) # Now process all the chunks involved in the build. for repo, ref, filename, buildsystem in chunk_queue: self.process_chunk(resolved_morphologies, resolved_trees, definitions_checkout_dir, morph_loader, - repo, ref, filename, buildsystem, visit) + repo, ref, filename, buildsystem, visit, + predefined_split_rules) def create_source_pool(lrc, rrc, repo, ref, filenames, cachedir, @@ -437,10 +477,11 @@ def create_source_pool(lrc, rrc, repo, ref, filenames, cachedir, ''' pool = morphlib.sourcepool.SourcePool() - def add_to_pool(reponame, ref, filename, absref, tree, morphology): - sources = morphlib.source.make_sources(reponame, ref, - filename, absref, - tree, morphology) + def add_to_pool(reponame, ref, filename, absref, tree, morphology, + predefined_split_rules): + sources = morphlib.source.make_sources( + reponame, ref, filename, absref, tree, morphology, + predefined_split_rules) for source in sources: pool.add(source) diff --git a/morphlib/util.py b/morphlib/util.py index 284fe305..e34799df 100644 --- a/morphlib/util.py +++ b/morphlib/util.py @@ -734,6 +734,7 @@ def temp_dir(*args, **kwargs): #pragma: no cover if cleanup_on_success: shutil.rmtree(td, ignore_errors=True) + def copyfileobj(fsrc, fdst, length=16*1024, callback=lambda x: None): #pragma: no cover ''' This is similar to shutil.copyfileobj @@ -775,3 +776,9 @@ class ProgressBar(object): progress, self._expected_size, self._unit) sys.stderr.write(s) sys.stderr.flush() + + +def schemas_directory(): # pragma: no cover + '''Returns a path to the schemas/ subdirectory of the 'morphlib' module.''' + code_dir = os.path.dirname(morphlib.__file__) + return os.path.join(code_dir, 'schemas') |