diff options
48 files changed, 570 insertions, 582 deletions
@@ -109,6 +109,7 @@ then echo 'Checking source code for silliness' if ! (git ls-files --cached | grep -v '\.gz$' | + grep -v '\.json-schema$' | grep -Ev 'tests[^/]*/.*\.std(out|err)' | grep -vF 'tests.build/build-system-autotools.script' | xargs -r scripts/check-silliness); then diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 81540a31..36e5c3a8 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -60,6 +60,7 @@ import builder import cachedrepo import cachekeycomputer import cmdline_parse_utils +import defaults import definitions_repo import definitions_version import extensions diff --git a/morphlib/artifactresolver_tests.py b/morphlib/artifactresolver_tests.py index 141ff948..52ed0811 100644 --- a/morphlib/artifactresolver_tests.py +++ b/morphlib/artifactresolver_tests.py @@ -20,6 +20,12 @@ import yaml import morphlib +default_split_rules = { + 'chunk': morphlib.artifactsplitrule.DEFAULT_CHUNK_RULES, + 'stratum': morphlib.artifactsplitrule.DEFAULT_STRATUM_RULES, +} + + def get_chunk_morphology(name, artifact_names=[]): assert(isinstance(artifact_names, list)) @@ -88,9 +94,9 @@ class ArtifactResolverTests(unittest.TestCase): pool = morphlib.sourcepool.SourcePool() morph = get_chunk_morphology('chunk') - sources = morphlib.source.make_sources('repo', 'ref', - 'chunk.morph', 'sha1', - 'tree', morph) + sources = morphlib.source.make_sources( + 'repo', 'ref', 'chunk.morph', 'sha1', 'tree', morph, + default_split_rules=default_split_rules) for source in sources: pool.add(source) @@ -109,9 +115,9 @@ class ArtifactResolverTests(unittest.TestCase): pool = morphlib.sourcepool.SourcePool() morph = get_chunk_morphology('chunk', ['chunk-foobar']) - sources = morphlib.source.make_sources('repo', 'ref', - 'chunk.morph', 'sha1', - 'tree', morph) + sources = morphlib.source.make_sources( + 'repo', 'ref', 'chunk.morph', 'sha1', 'tree', morph, + default_split_rules=default_split_rules) for source in sources: pool.add(source) @@ -129,9 +135,9 @@ class ArtifactResolverTests(unittest.TestCase): pool = morphlib.sourcepool.SourcePool() morph = get_chunk_morphology('chunk', ['chunk-baz', 'chunk-qux']) - sources = morphlib.source.make_sources('repo', 'ref', - 'chunk.morph', 'sha1', - 'tree', morph) + sources = morphlib.source.make_sources( + 'repo', 'ref', 'chunk.morph', 'sha1', 'tree', morph, + default_split_rules=default_split_rules) for source in sources: pool.add(source) @@ -151,18 +157,18 @@ class ArtifactResolverTests(unittest.TestCase): pool = morphlib.sourcepool.SourcePool() morph = get_chunk_morphology('chunk') - sources = morphlib.source.make_sources('repo', 'ref', - 'chunk.morph', 'sha1', - 'tree', morph) + sources = morphlib.source.make_sources( + 'repo', 'ref', 'chunk.morph', 'sha1', 'tree', morph, + default_split_rules=default_split_rules) for chunk in sources: pool.add(chunk) morph = get_stratum_morphology( 'stratum', chunks=[('chunk', 'chunk', 'repo', 'ref')]) - stratum_sources = set(morphlib.source.make_sources('repo', 'ref', - 'stratum.morph', - 'sha1', 'tree', - morph)) + stratum_sources = set( + morphlib.source.make_sources( + 'repo', 'ref', 'stratum.morph', 'sha1', 'tree', morph, + default_split_rules=default_split_rules)) for stratum in stratum_sources: pool.add(stratum) @@ -197,9 +203,9 @@ class ArtifactResolverTests(unittest.TestCase): pool = morphlib.sourcepool.SourcePool() morph = get_chunk_morphology('chunk', ['chunk-foo', 'chunk-bar']) - sources = morphlib.source.make_sources('repo', 'ref', - 'chunk.morph', 'sha1', - 'tree', morph) + sources = morphlib.source.make_sources( + 'repo', 'ref', 'chunk.morph', 'sha1', 'tree', morph, + default_split_rules=default_split_rules) for chunk in sources: pool.add(chunk) @@ -208,10 +214,10 @@ class ArtifactResolverTests(unittest.TestCase): chunks=[ ('chunk', 'chunk', 'repo', 'ref'), ]) - stratum_sources = set(morphlib.source.make_sources('repo', 'ref', - 'stratum.morph', - 'sha1', 'tree', - morph)) + stratum_sources = set( + morphlib.source.make_sources( + 'repo', 'ref', 'stratum.morph', 'sha1', 'tree', morph, + default_split_rules=default_split_rules)) for stratum in stratum_sources: pool.add(stratum) @@ -246,7 +252,8 @@ class ArtifactResolverTests(unittest.TestCase): chunk = get_chunk_morphology('chunk1') chunk1, = morphlib.source.make_sources( - 'repo', 'original/ref', 'chunk1.morph', 'sha1', 'tree', chunk) + 'repo', 'original/ref', 'chunk1.morph', 'sha1', 'tree', chunk, + default_split_rules=default_split_rules) pool.add(chunk1) morph = get_stratum_morphology( @@ -254,15 +261,16 @@ class ArtifactResolverTests(unittest.TestCase): chunks=[(loader.save_to_string(chunk), 'chunk1.morph', 'repo', 'original/ref')], build_depends=['stratum2']) - sources = morphlib.source.make_sources('repo', 'original/ref', - 'stratum1.morph', 'sha1', - 'tree', morph) + sources = morphlib.source.make_sources( + 'repo', 'original/ref', 'stratum1.morph', 'sha1', 'tree', morph, + default_split_rules=default_split_rules) for stratum1 in sources: pool.add(stratum1) chunk = get_chunk_morphology('chunk2') chunk2, = morphlib.source.make_sources( - 'repo', 'original/ref', 'chunk2.morph', 'sha1', 'tree', chunk) + 'repo', 'original/ref', 'chunk2.morph', 'sha1', 'tree', chunk, + default_split_rules=default_split_rules) pool.add(chunk2) morph = get_stratum_morphology( @@ -270,9 +278,9 @@ class ArtifactResolverTests(unittest.TestCase): chunks=[(loader.save_to_string(chunk), 'chunk2.morph', 'repo', 'original/ref')], build_depends=['stratum1']) - sources = morphlib.source.make_sources('repo', 'original/ref', - 'stratum2.morph', 'sha1', - 'tree', morph) + sources = morphlib.source.make_sources( + 'repo', 'original/ref', 'stratum2.morph', 'sha1', 'tree', morph, + default_split_rules=default_split_rules) for stratum2 in sources: pool.add(stratum2) @@ -299,23 +307,23 @@ class ArtifactResolverTests(unittest.TestCase): ref: original/ref build-depends: [] ''') - sources = morphlib.source.make_sources('repo', 'original/ref', - 'stratum.morph', 'sha1', - 'tree', morph) + sources = morphlib.source.make_sources( + 'repo', 'original/ref', 'stratum.morph', 'sha1', 'tree', morph, + default_split_rules=default_split_rules) for stratum in sources: pool.add(stratum) morph = get_chunk_morphology('chunk1') - sources = morphlib.source.make_sources('repo', 'original/ref', - 'chunk1.morph', 'sha1', - 'tree', morph) + sources = morphlib.source.make_sources( + 'repo', 'original/ref', 'chunk1.morph', 'sha1', 'tree', morph, + default_split_rules=default_split_rules) for chunk1 in sources: pool.add(chunk1) morph = get_chunk_morphology('chunk2') - sources = morphlib.source.make_sources('repo', 'original/ref', - 'chunk2.morph', 'sha1', - 'tree', morph) + sources = morphlib.source.make_sources( + 'repo', 'original/ref', 'chunk2.morph', 'sha1', 'tree', morph, + default_split_rules=default_split_rules) for chunk2 in sources: pool.add(chunk2) diff --git a/morphlib/artifactsplitrule.py b/morphlib/artifactsplitrule.py index b5ebdf83..ba5abe02 100644 --- a/morphlib/artifactsplitrule.py +++ b/morphlib/artifactsplitrule.py @@ -189,10 +189,6 @@ class SplitRules(collections.Iterable): for artifact, rule in self._rules) -# TODO: Work out a good way to feed new defaults in. This is good for -# the usual Linux userspace, but we may find issues and need a -# migration path to a more useful set, or develop a system with -# a different layout, like Android. DEFAULT_CHUNK_RULES = [ ('-bins', [ r"(usr/)?s?bin/.*" ]), ('-libs', [ @@ -229,6 +225,14 @@ DEFAULT_STRATUM_RULES = [ ] +# A 'no-op' set of split rules. An empty list would cause everything to be +# ignored, which is unlikely to ever be what a user wants, and breaks some +# internal bits of Morph. +EMPTY_RULES = [ + ('', [r'.*']) +] + + def unify_chunk_matches(morphology, default_rules=DEFAULT_CHUNK_RULES): '''Create split rules including defaults and per-chunk rules. @@ -237,6 +241,8 @@ def unify_chunk_matches(morphology, default_rules=DEFAULT_CHUNK_RULES): by building the chunk to the chunk artifact they should be put in. ''' + if default_rules is None or len(default_rules) == 0: + default_rules = EMPTY_RULES split_rules = SplitRules() @@ -265,6 +271,8 @@ def unify_stratum_matches(morphology, default_rules=DEFAULT_STRATUM_RULES): strata to the stratum artifact they should be put in. ''' + if default_rules is None or len(default_rules) == 0: + default_rules = EMPTY_RULES assignment_split_rules = SplitRules() for spec in morphology['chunks']: @@ -296,7 +304,7 @@ def unify_stratum_matches(morphology, default_rules=DEFAULT_STRATUM_RULES): match_split_rules)) -def unify_system_matches(morphology): +def unify_system_matches(morphology, default_rules=[]): '''Create split rules including defaults and per-chunk rules. With rules specified in the morphology's 'products' field and the diff --git a/morphlib/buildsystem.py b/morphlib/buildsystem.py index 4655f2ee..95bf4116 100644 --- a/morphlib/buildsystem.py +++ b/morphlib/buildsystem.py @@ -44,15 +44,10 @@ _STRIP_COMMAND = r'''find "$DESTDIR" -type f \ class BuildSystem(object): - '''An abstraction of an upstream build system. + '''Predefined command sequences for a given build system. - Some build systems are well known: autotools, for example. - Others are purely manual: there's a set of commands to run that - are specific for that project, and (almost) no other project uses them. - The Linux kernel would be an example of that. - - This class provides an abstraction for these, including a method - to autodetect well known build systems. + For example, you can have an 'autotools' build system, which runs + 'configure', 'make' and 'make install'. ''' @@ -73,6 +68,14 @@ class BuildSystem(object): self.strip_commands = [] self.post_strip_commands = [] + def from_dict(self, name, commands): + self.name = name + + self.configure_commands = commands.get('configure-commands', []) + self.build_commands = commands.get('build-commands', []) + self.install_commands = commands.get('install-commands', []) + self.strip_commands = commands.get('strip-commands', []) + def __getitem__(self, key): key = '_'.join(key.split('-')) return getattr(self, key) diff --git a/morphlib/buildsystem_tests.py b/morphlib/buildsystem_tests.py index 80898ebd..506698a5 100644 --- a/morphlib/buildsystem_tests.py +++ b/morphlib/buildsystem_tests.py @@ -53,6 +53,15 @@ class BuildSystemTests(unittest.TestCase): morph = self.bs.get_morphology('foobar') self.assertTrue(morph.__class__.__name__ == 'Morphology') + def test_construct_from_dict(self): + '''Test parsing a dict of information from a DEFAULTS file.''' + + commands_dict = { + 'configure-commands': 'foo' + } + self.bs.from_dict('test', commands_dict) + self.assertEqual(self.bs.configure_commands, 'foo') + class ManualBuildSystemTests(unittest.TestCase): diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py index aa217dfc..2a5852fe 100644 --- a/morphlib/cachekeycomputer_tests.py +++ b/morphlib/cachekeycomputer_tests.py @@ -28,6 +28,12 @@ class DummyBuildEnvironment: self.env = env +default_split_rules = { + 'chunk': morphlib.artifactsplitrule.DEFAULT_CHUNK_RULES, + 'stratum': morphlib.artifactsplitrule.DEFAULT_STRATUM_RULES, +} + + class CacheKeyComputerTests(unittest.TestCase): def setUp(self): @@ -83,9 +89,9 @@ class CacheKeyComputerTests(unittest.TestCase): ''', }.iteritems(): morph = loader.load_from_string(text) - sources = morphlib.source.make_sources('repo', 'original/ref', - name, 'sha1', - 'tree', morph) + sources = morphlib.source.make_sources( + 'repo', 'original/ref', name, 'sha1', 'tree', morph, + default_split_rules=default_split_rules) for source in sources: self.source_pool.add(source) self.build_env = DummyBuildEnvironment({ diff --git a/morphlib/defaults.py b/morphlib/defaults.py new file mode 100644 index 00000000..9e695a90 --- /dev/null +++ b/morphlib/defaults.py @@ -0,0 +1,108 @@ +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see <http://www.gnu.org/licenses/>. +# +# =*= License: GPL-2 =*= + + +import cliapp +import jsonschema +import yaml + +import os + +import morphlib + + +class Defaults(object): + '''Represents predefined default values specific to Baserock definitions. + + The DEFAULTS file was added in definitions format version 7, which lets + users set these defaults. The text of DEFAULTS file can be passed in as + 'text', and will be validated and parsed if definitions_version >= 7. + + Prior to version 7, the defaults were hardcoded in Morph. These defaults + will be returned if definitions_version < 7. + + ''' + def __init__(self, definitions_version, text=None): + self._build_systems = {} + self._split_rules = {} + + schema_path = os.path.join(morphlib.util.schemas_directory(), + 'defaults.json-schema') + with open(schema_path) as f: + self.schema = yaml.load(f) + + if definitions_version >= 7: + if text: + self._build_systems, self._split_rules = self._parse(text) + else: + self._build_systems, self._split_rules = self._builtins() + + def _parse(self, text): + build_systems = {} + split_rules = {} + + # This reports errors against <string> rather than the actual filename, + # which is sad. + data = yaml.safe_load(text) + + if data is None: + # It's OK to be empty, I guess. + return build_systems, split_rules + + try: + # It would be nice if this could give line numbers when it spotted + # errors. Seems tricky. + jsonschema.validate(data, self.schema) + except jsonschema.ValidationError as e: + raise cliapp.AppException('Invalid DEFAULTS file: %s' % e.message) + + build_system_data = data.get('build-systems', {}) + for name, commands in build_system_data.items(): + build_system = morphlib.buildsystem.BuildSystem() + build_system.from_dict(name, commands) + build_systems[name] = build_system + + # It would make sense to create artifactsplitrule.SplitRule instances + # here, instead of an unlabelled data structure. That would need some + # changes to source.make_sources() and the 'artifactsplitrule' module. + split_rules_data = data.get('split-rules', {}) + for kind, rules in split_rules_data.items(): + split_rules[kind] = [] + for rule in rules: + rule_unlabelled = (rule['artifact'], rule['include']) + split_rules[kind].append(rule_unlabelled) + + return build_systems, split_rules + + def _builtins(self): + build_systems = {} + split_rules = {} + + for build_system in morphlib.buildsystem.build_systems: + build_systems[build_system.name] = build_system + + split_rules['chunk'] = \ + morphlib.artifactsplitrule.DEFAULT_CHUNK_RULES + split_rules['stratum'] = \ + morphlib.artifactsplitrule.DEFAULT_STRATUM_RULES + + return build_systems, split_rules + + def build_systems(self): + return self._build_systems + + def split_rules(self): + return self._split_rules diff --git a/morphlib/definitions_repo.py b/morphlib/definitions_repo.py index 53328e85..143b5241 100644 --- a/morphlib/definitions_repo.py +++ b/morphlib/definitions_repo.py @@ -23,6 +23,7 @@ import logging import os import urlparse import uuid +import warnings import morphlib import gitdir @@ -238,8 +239,24 @@ class DefinitionsRepo(gitdir.GitDirectory): version_text = mf.read_file('VERSION') version = morphlib.definitions_version.check_version_file(version_text) + defaults_text = mf.read_file('DEFAULTS', allow_missing=True) + + if version < 7: + if defaults_text is not None: + warnings.warn( + "Ignoring DEFAULTS file, because these definitions are " + "version %i" % version) + defaults_text = None + else: + if defaults_text is None: + warnings.warn("No DEFAULTS file found.") + + defaults = morphlib.defaults.Defaults(version, + text=defaults_text) + loader = morphlib.morphloader.MorphologyLoader( - definitions_version=version) + definitions_version=version, + predefined_build_systems=defaults.build_systems()) return loader diff --git a/morphlib/definitions_version.py b/morphlib/definitions_version.py index 56cef164..ec2df928 100644 --- a/morphlib/definitions_version.py +++ b/morphlib/definitions_version.py @@ -24,7 +24,7 @@ import yaml import morphlib -SUPPORTED_VERSIONS = [3, 4, 5, 6] +SUPPORTED_VERSIONS = [3, 4, 5, 6, 7] class DefinitionsVersionError(cliapp.AppException): diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py index 479bc8fb..e58b3164 100644 --- a/morphlib/morphloader.py +++ b/morphlib/morphloader.py @@ -111,6 +111,13 @@ class UnknownArchitectureError(MorphologyValidationError): % (arch, morph_filename)) +class UnknownBuildSystemError(MorphologyValidationError): + + def __init__(self, build_system, morph_filename): + self.msg = ('Undefined build system %s in morphology %s' + % (build_system, morph_filename)) + + class NoStratumBuildDependenciesError(MorphologyValidationError): def __init__(self, stratum_name, morph_filename): @@ -403,7 +410,7 @@ class MorphologyLoader(object): } def __init__(self, definitions_version=0, - lookup_build_system=morphlib.buildsystem.lookup_build_system): + predefined_build_systems={}): if definitions_version >= 5: # pragma: no cover self._static_defaults = copy.deepcopy(self._static_defaults) self._static_defaults['chunk'].update({ @@ -412,7 +419,12 @@ class MorphologyLoader(object): 'post-strip-commands': None}) self._definitions_version = definitions_version - self._lookup_build_system = lookup_build_system + + self._predefined_build_systems = predefined_build_systems.copy() + + if 'manual' not in self._predefined_build_systems: + self._predefined_build_systems['manual'] = \ + morphlib.buildsystem.ManualBuildSystem() def parse_morphology_text(self, text, morph_filename): '''Parse a textual morphology. @@ -820,10 +832,12 @@ class MorphologyLoader(object): if morph['max-jobs'] is not None: morph['max-jobs'] = int(morph['max-jobs']) - def _unset_chunk_defaults(self, morph): # pragma: no cover + def _unset_chunk_defaults(self, morph): # pragma: no cover + # This is only used by the deprecated branch-and-merge plugin, and + # probably doesn't work correctly for definitions V7 and newer. default_bs = self._static_defaults['chunk']['build-system'] - bs = self._lookup_build_system( - morph.get('build-system', default_bs)) + bs_name = morph.get('build-system', default_bs) + bs = self._predefined_build_systems[bs_name] for key in self._static_defaults['chunk']: if key not in morph: continue if 'commands' not in key: continue @@ -832,11 +846,19 @@ class MorphologyLoader(object): if morph[key] == default_value: del morph[key] + def lookup_build_system(self, name): + return self._predefined_build_systems[name] + def set_commands(self, morph): - default = self._static_defaults['chunk']['build-system'] - bs = self._lookup_build_system( - morph.get('build-system', default)) if morph['kind'] == 'chunk': + default = self._static_defaults['chunk']['build-system'] + bs_name = morph.get('build-system', default) + + try: + bs = self.lookup_build_system(bs_name) + except KeyError: + raise UnknownBuildSystemError(bs_name, morph['name']) + for key in self._static_defaults['chunk']: if 'commands' not in key: continue if key not in morph: diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py index 6cb93094..1a2ee107 100644 --- a/morphlib/morphloader_tests.py +++ b/morphlib/morphloader_tests.py @@ -50,7 +50,7 @@ class MorphologyLoaderTests(unittest.TestCase): def setUp(self): self.loader = morphlib.morphloader.MorphologyLoader( - definitions_version=6) + definitions_version=7) self.tempdir = tempfile.mkdtemp() self.filename = os.path.join(self.tempdir, 'foo.morph') @@ -61,12 +61,12 @@ class MorphologyLoaderTests(unittest.TestCase): string = '''\ name: foo kind: chunk -build-system: dummy +build-system: manual ''' morph = self.loader.parse_morphology_text(string, 'test') self.assertEqual(morph['kind'], 'chunk') self.assertEqual(morph['name'], 'foo') - self.assertEqual(morph['build-system'], 'dummy') + self.assertEqual(morph['build-system'], 'manual') def test_fails_to_parse_utter_garbage(self): self.assertRaises( @@ -495,43 +495,43 @@ chunks: string = '''\ name: foo kind: chunk -build-system: dummy +build-system: manual ''' morph = self.loader.load_from_string(string) self.assertEqual(morph['kind'], 'chunk') self.assertEqual(morph['name'], 'foo') - self.assertEqual(morph['build-system'], 'dummy') + self.assertEqual(morph['build-system'], 'manual') def test_loads_json_from_string(self): string = '''\ { "name": "foo", "kind": "chunk", - "build-system": "dummy" + "build-system": "manual" } ''' morph = self.loader.load_from_string(string) self.assertEqual(morph['kind'], 'chunk') self.assertEqual(morph['name'], 'foo') - self.assertEqual(morph['build-system'], 'dummy') + self.assertEqual(morph['build-system'], 'manual') def test_loads_from_file(self): with open(self.filename, 'w') as f: f.write('''\ name: foo kind: chunk -build-system: dummy +build-system: manual ''') morph = self.loader.load_from_file(self.filename) self.assertEqual(morph['kind'], 'chunk') self.assertEqual(morph['name'], 'foo') - self.assertEqual(morph['build-system'], 'dummy') + self.assertEqual(morph['build-system'], 'manual') def test_saves_to_string(self): morph = morphlib.morphology.Morphology({ 'name': 'foo', 'kind': 'chunk', - 'build-system': 'dummy', + 'build-system': 'manual', }) text = self.loader.save_to_string(morph) @@ -540,14 +540,14 @@ build-system: dummy self.assertEqual(text, '''\ name: foo kind: chunk -build-system: dummy +build-system: manual ''') def test_saves_to_file(self): morph = morphlib.morphology.Morphology({ 'name': 'foo', 'kind': 'chunk', - 'build-system': 'dummy', + 'build-system': 'manual', }) self.loader.save_to_file(self.filename, morph) @@ -559,7 +559,7 @@ build-system: dummy self.assertEqual(text, '''\ name: foo kind: chunk -build-system: dummy +build-system: manual ''') def test_validate_does_not_set_defaults(self): @@ -967,12 +967,11 @@ build-system: dummy ) s = self.loader.save_to_string(m) - def test_smoketest_strip_commands(self): - dummy_buildsystem = morphlib.buildsystem.DummyBuildSystem() - loader = morphlib.morphloader.MorphologyLoader( - definitions_version=5, - lookup_build_system=lambda x: dummy_buildsystem) - m = morphlib.morphology.Morphology( - {'name': 'test', 'kind': 'chunk', 'build-system': 'dummy'}) - loader.set_commands(m) - self.assertEqual(m['strip-commands'], dummy_buildsystem.strip_commands) + def test_unknown_build_system(self): + m = morphlib.morphology.Morphology({ + 'kind': 'chunk', + 'name': 'foo', + 'build-system': 'monkey scientist', + }) + with self.assertRaises(morphlib.morphloader.UnknownBuildSystemError): + s = self.loader.set_commands(m) diff --git a/morphlib/morphologyfinder.py b/morphlib/morphologyfinder.py index 335c01b6..2c864bea 100644 --- a/morphlib/morphologyfinder.py +++ b/morphlib/morphologyfinder.py @@ -15,9 +15,7 @@ # =*= License: GPL-2 =*= -import cliapp - -import morphlib +import errno class MorphologyFinder(object): @@ -33,7 +31,7 @@ class MorphologyFinder(object): self.gitdir = gitdir self.ref = ref - def read_file(self, filename): + def read_file(self, filename, allow_missing=False): '''Return the un-parsed text of a morphology. For the given morphology name, locate and return the contents @@ -43,7 +41,13 @@ class MorphologyFinder(object): is handled by the MorphologyLoader class. ''' - return self.gitdir.read_file(filename, self.ref) + try: + return self.gitdir.read_file(filename, self.ref) + except IOError as e: + if allow_missing and e.errno == errno.ENOENT: + return None + else: + raise def list_morphologies(self): '''Return the filenames of all morphologies in the (repo, ref). diff --git a/morphlib/morphologyfinder_tests.py b/morphlib/morphologyfinder_tests.py index 59772e23..3312b627 100644 --- a/morphlib/morphologyfinder_tests.py +++ b/morphlib/morphologyfinder_tests.py @@ -103,6 +103,10 @@ class MorphologyFinderTests(unittest.TestCase): mf = morphlib.morphologyfinder.MorphologyFinder(gd) self.assertEqual(mf.read_file('foo.morph'), "altered morphology text") + with self.assertRaises(IOError): + mf.read_file('missing') + self.assertEqual(mf.read_file('missing', allow_missing=True), + None) def test_read_morph_raises_no_worktree_no_ref(self): gd = morphlib.gitdir.GitDirectory(self.mirror) diff --git a/morphlib/schemas/defaults.json-schema b/morphlib/schemas/defaults.json-schema new file mode 100644 index 00000000..2f713425 --- /dev/null +++ b/morphlib/schemas/defaults.json-schema @@ -0,0 +1,66 @@ +$schema: http://json-schema.org/draft-04/schema# +id: http://git.baserock.org/cgi-bin/cgit.cgi/baserock/baserock/definitions.git/tree/schemas/defaults.json-schema + +description: | + This is a JSON-Schema description of the DEFAULTS file specified in the + Baserock definitions format. DEFAULTS is a YAML file that contains global + defaults for a set of Baserock definitions. + + This JSON-Schema file is valid for VERSION 7 of the Baserock definitions + YAML serialisation format. + + The Baserock definitions YAML serialisation format is the recommended way of + representing Baserock definitions on disk. The actual data model is described + separately. See <https://wiki.baserock.org/definitions> for more information. + + This schema is represented as YAML, so that it can be edited more easily. + You may need to convert to JSON if using a JSON-Schema tool that expects + its input to be an actual string containing data serialised as JSON. + +definitions: + command-sequence: + type: array + items: {type: string} + + build-system: + type: object + additionalProperties: false + properties: + build-commands: {$ref: '#/definitions/command-sequence'} + configure-commands: {$ref: '#/definitions/command-sequence'} + install-commands: {$ref: '#/definitions/command-sequence'} + strip-commands: {$ref: '#/definitions/command-sequence'} + + split-rules: + type: array + items: + type: object + + required: [artifact, include] + additionalProperties: false + + properties: + artifact: {type: string} + include: + type: array + items: + type: string + format: regex + +type: object +additionalProperties: false + +properties: + # Predefined build systems. + build-systems: + type: object + patternProperties: + ^.*$: {$ref: '#/definitions/build-system'} + + # Predefined artifact splitting rules. + split-rules: + type: object + additionalProperties: false + properties: + chunk: {$ref: '#/definitions/split-rules'} + stratum: {$ref: '#/definitions/split-rules'} diff --git a/morphlib/source.py b/morphlib/source.py index 2c96bcbd..135c14cc 100644 --- a/morphlib/source.py +++ b/morphlib/source.py @@ -77,12 +77,14 @@ class Source(object): return artifact in self.dependencies -def make_sources(reponame, ref, filename, absref, tree, morphology): +def make_sources(reponame, ref, filename, absref, tree, morphology, + default_split_rules={}): kind = morphology['kind'] if kind in ('system', 'chunk'): unifier = getattr(morphlib.artifactsplitrule, 'unify_%s_matches' % kind) - split_rules = unifier(morphology) + split_rules = unifier(morphology, + default_rules=default_split_rules.get(kind, {})) # chunk and system sources are named after the morphology source_name = morphology['name'] source = morphlib.source.Source(source_name, reponame, ref, @@ -93,7 +95,8 @@ def make_sources(reponame, ref, filename, absref, tree, morphology): yield source elif kind == 'stratum': # pragma: no cover unifier = morphlib.artifactsplitrule.unify_stratum_matches - split_rules = unifier(morphology) + split_rules = unifier(morphology, + default_rules=default_split_rules.get(kind, {})) for name in split_rules.artifacts: source = morphlib.source.Source( name, # stratum source name is artifact name diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py index cbab0f7f..42398ce7 100644 --- a/morphlib/sourceresolver.py +++ b/morphlib/sourceresolver.py @@ -21,6 +21,7 @@ import os import pylru import shutil import tempfile +import warnings import yaml import cliapp @@ -289,6 +290,36 @@ class SourceResolver(object): return morphlib.definitions_version.check_version_file(version_text) + def _get_defaults(self, definitions_checkout_dir, + definitions_version=7): # pragma: no cover + '''Return the default build system commands, and default split rules. + + This function returns a tuple with two dicts. + + The defaults are read from a file named DEFAULTS in the definitions + directory, if the definitions follow format version 7 or later. If the + definitions follow version 6 or earlier, hardcoded defaults are used. + + ''' + # Read default build systems and split rules from DEFAULTS file. + defaults_text = self._get_file_contents_from_definitions( + definitions_checkout_dir, 'DEFAULTS') + + if definitions_version < 7: + if defaults_text is not None: + warnings.warn( + "Ignoring DEFAULTS file, because these definitions are " + "version %i" % definitions_version) + defaults_text = None + else: + if defaults_text is None: + warnings.warn("No DEFAULTS file found.") + + defaults = morphlib.defaults.Defaults(definitions_version, + text=defaults_text) + + return defaults.build_systems(), defaults.split_rules() + def _get_morphology(self, resolved_morphologies, definitions_checkout_dir, definitions_repo, definitions_absref, morph_loader, reponame, sha1, filename): # pragma: no cover @@ -311,17 +342,12 @@ class SourceResolver(object): return morph - def _process_definitions_with_children(self, - resolved_morphologies, - definitions_checkout_dir, - definitions_repo, - definitions_ref, - definitions_absref, - definitions_tree, - definitions_version, - morph_loader, - system_filenames, - visit): # pragma: no cover + def _process_definitions_with_children( + self, resolved_morphologies, definitions_checkout_dir, + definitions_repo, definitions_ref, definitions_absref, + definitions_tree, definitions_version, morph_loader, + system_filenames, visit, + predefined_split_rules): # pragma: no cover definitions_queue = collections.deque(system_filenames) chunk_queue = set() @@ -341,7 +367,8 @@ class SourceResolver(object): raise MorphologyNotFoundError(filename) visit(definitions_repo, definitions_ref, filename, - definitions_absref, definitions_tree, morphology) + definitions_absref, definitions_tree, morphology, + predefined_split_rules) if morphology['kind'] == 'cluster': raise cliapp.AppException( @@ -462,7 +489,8 @@ class SourceResolver(object): resolved_buildsystems, definitions_checkout_dir, definitions_repo, definitions_absref, definitions_version, morph_loader, chunk_repo, chunk_ref, - filename, chunk_buildsystem, visit): # pragma: no cover + filename, chunk_buildsystem, visit, + predefined_split_rules): # pragma: no cover absref = None tree = None chunk_key = None @@ -483,7 +511,8 @@ class SourceResolver(object): if morphology: absref, tree = self._resolve_ref(resolved_trees, chunk_repo, chunk_ref) - visit(chunk_repo, chunk_ref, filename, absref, tree, morphology) + visit(chunk_repo, chunk_ref, filename, absref, tree, morphology, + predefined_split_rules) return absref, tree = self._resolve_ref(resolved_trees, chunk_repo, chunk_ref) @@ -552,7 +581,8 @@ class SourceResolver(object): morphology = generate_morph_and_cache_buildsystem( buildsystem) - visit(chunk_repo, chunk_ref, filename, absref, tree, morphology) + visit(chunk_repo, chunk_ref, filename, absref, tree, morphology, + predefined_split_rules) def traverse_morphs(self, definitions_repo, definitions_ref, system_filenames, @@ -582,9 +612,15 @@ class SourceResolver(object): definitions_absref, definitions_checkout_dir) definitions_version = self._check_version_file( - definitions_checkout_dir) + definitions_checkout_dir) + + predefined_build_systems, predefined_split_rules = \ + self._get_defaults( + definitions_checkout_dir, definitions_version) + morph_loader = morphlib.morphloader.MorphologyLoader( - definitions_version=definitions_version) + definitions_version=definitions_version, + predefined_build_systems=predefined_build_systems) # First, process the system and its stratum morphologies. These # will all live in the same Git repository, and will point to @@ -593,7 +629,7 @@ class SourceResolver(object): resolved_morphologies, definitions_checkout_dir, definitions_repo, definitions_ref, definitions_absref, definitions_tree, definitions_version, morph_loader, - system_filenames, visit) + system_filenames, visit, predefined_split_rules) # Now process all the chunks involved in the build. for repo, ref, filename, buildsystem in chunk_queue: @@ -602,7 +638,8 @@ class SourceResolver(object): definitions_checkout_dir, definitions_repo, definitions_absref, definitions_version, morph_loader, repo, - ref, filename, buildsystem, visit) + ref, filename, buildsystem, visit, + predefined_split_rules) def create_source_pool(lrc, rrc, repo, ref, filenames, cachedir, @@ -624,10 +661,11 @@ def create_source_pool(lrc, rrc, repo, ref, filenames, cachedir, ''' pool = morphlib.sourcepool.SourcePool() - def add_to_pool(reponame, ref, filename, absref, tree, morphology): - sources = morphlib.source.make_sources(reponame, ref, - filename, absref, - tree, morphology) + def add_to_pool(reponame, ref, filename, absref, tree, morphology, + predefined_split_rules): + sources = morphlib.source.make_sources( + reponame, ref, filename, absref, tree, morphology, + predefined_split_rules) for source in sources: pool.add(source) diff --git a/morphlib/sourceresolver_tests.py b/morphlib/sourceresolver_tests.py index 5985579c..58512172 100644 --- a/morphlib/sourceresolver_tests.py +++ b/morphlib/sourceresolver_tests.py @@ -32,7 +32,7 @@ class FakeRemoteRepoCache(object): return '''{ "name": "%s", "kind": "chunk", - "build-system": "dummy" + "build-system": "manual" }''' % filename[:-len('.morph')] return 'text' @@ -46,12 +46,12 @@ class FakeLocalRepo(object): 'chunk.morph': ''' name: chunk kind: chunk - build-system: dummy + build-system: manual ''', 'chunk-split.morph': ''' name: chunk-split kind: chunk - build-system: dummy + build-system: manual products: - artifact: chunk-split-runtime include: [] @@ -119,7 +119,7 @@ class FakeLocalRepo(object): return ''' name: %s kind: chunk - build-system: dummy''' % filename[:-len('.morph')] + build-system: manual''' % filename[:-len('.morph')] return 'text' def list_files(self, ref, recurse): diff --git a/morphlib/util.py b/morphlib/util.py index a92b7f37..27139c66 100644 --- a/morphlib/util.py +++ b/morphlib/util.py @@ -731,3 +731,9 @@ def temp_dir(*args, **kwargs): #pragma: no cover else: if cleanup_on_success: shutil.rmtree(td, ignore_errors=True) + + +def schemas_directory(): # pragma: no cover + '''Returns a path to the schemas/ subdirectory of the 'morphlib' module.''' + code_dir = os.path.dirname(morphlib.__file__) + return os.path.join(code_dir, 'schemas') diff --git a/tests.branching/branch-creates-new-system-branch-not-from-master.stdout b/tests.branching/branch-creates-new-system-branch-not-from-master.stdout index 78b3ca33..c5682475 100644 --- a/tests.branching/branch-creates-new-system-branch-not-from-master.stdout +++ b/tests.branching/branch-creates-new-system-branch-not-from-master.stdout @@ -7,6 +7,7 @@ d ./newbranch/test d ./newbranch/test/morphs d ./newbranch/test/morphs/.git f ./newbranch/.morph-system-branch/config +f ./newbranch/test/morphs/DEFAULTS f ./newbranch/test/morphs/VERSION f ./newbranch/test/morphs/hello-stratum.morph f ./newbranch/test/morphs/hello-system.morph diff --git a/tests.branching/branch-creates-new-system-branch.stdout b/tests.branching/branch-creates-new-system-branch.stdout index 882d5401..238afe5e 100644 --- a/tests.branching/branch-creates-new-system-branch.stdout +++ b/tests.branching/branch-creates-new-system-branch.stdout @@ -7,6 +7,7 @@ d ./newbranch/test d ./newbranch/test/morphs d ./newbranch/test/morphs/.git f ./newbranch/.morph-system-branch/config +f ./newbranch/test/morphs/DEFAULTS f ./newbranch/test/morphs/VERSION f ./newbranch/test/morphs/hello-stratum.morph f ./newbranch/test/morphs/hello-system.morph diff --git a/tests.branching/branch-works-anywhere.stdout b/tests.branching/branch-works-anywhere.stdout index 4e6c3e27..e3c7bfb2 100644 --- a/tests.branching/branch-works-anywhere.stdout +++ b/tests.branching/branch-works-anywhere.stdout @@ -7,6 +7,7 @@ d ./branch1/test d ./branch1/test/morphs d ./branch1/test/morphs/.git f ./branch1/.morph-system-branch/config +f ./branch1/test/morphs/DEFAULTS f ./branch1/test/morphs/VERSION f ./branch1/test/morphs/hello-stratum.morph f ./branch1/test/morphs/hello-system.morph @@ -24,10 +25,12 @@ d ./branch2/test d ./branch2/test/morphs d ./branch2/test/morphs/.git f ./branch1/.morph-system-branch/config +f ./branch1/test/morphs/DEFAULTS f ./branch1/test/morphs/VERSION f ./branch1/test/morphs/hello-stratum.morph f ./branch1/test/morphs/hello-system.morph f ./branch2/.morph-system-branch/config +f ./branch2/test/morphs/DEFAULTS f ./branch2/test/morphs/VERSION f ./branch2/test/morphs/hello-stratum.morph f ./branch2/test/morphs/hello-system.morph @@ -50,14 +53,17 @@ d ./branch3/test d ./branch3/test/morphs d ./branch3/test/morphs/.git f ./branch1/.morph-system-branch/config +f ./branch1/test/morphs/DEFAULTS f ./branch1/test/morphs/VERSION f ./branch1/test/morphs/hello-stratum.morph f ./branch1/test/morphs/hello-system.morph f ./branch2/.morph-system-branch/config +f ./branch2/test/morphs/DEFAULTS f ./branch2/test/morphs/VERSION f ./branch2/test/morphs/hello-stratum.morph f ./branch2/test/morphs/hello-system.morph f ./branch3/.morph-system-branch/config +f ./branch3/test/morphs/DEFAULTS f ./branch3/test/morphs/VERSION f ./branch3/test/morphs/hello-stratum.morph f ./branch3/test/morphs/hello-system.morph @@ -85,18 +91,22 @@ d ./branch4/test d ./branch4/test/morphs d ./branch4/test/morphs/.git f ./branch1/.morph-system-branch/config +f ./branch1/test/morphs/DEFAULTS f ./branch1/test/morphs/VERSION f ./branch1/test/morphs/hello-stratum.morph f ./branch1/test/morphs/hello-system.morph f ./branch2/.morph-system-branch/config +f ./branch2/test/morphs/DEFAULTS f ./branch2/test/morphs/VERSION f ./branch2/test/morphs/hello-stratum.morph f ./branch2/test/morphs/hello-system.morph f ./branch3/.morph-system-branch/config +f ./branch3/test/morphs/DEFAULTS f ./branch3/test/morphs/VERSION f ./branch3/test/morphs/hello-stratum.morph f ./branch3/test/morphs/hello-system.morph f ./branch4/.morph-system-branch/config +f ./branch4/test/morphs/DEFAULTS f ./branch4/test/morphs/VERSION f ./branch4/test/morphs/hello-stratum.morph f ./branch4/test/morphs/hello-system.morph diff --git a/tests.branching/checkout-existing-branch.stdout b/tests.branching/checkout-existing-branch.stdout index f2b704e9..c3a806e2 100644 --- a/tests.branching/checkout-existing-branch.stdout +++ b/tests.branching/checkout-existing-branch.stdout @@ -7,6 +7,7 @@ d ./master/test d ./master/test/morphs d ./master/test/morphs/.git f ./master/.morph-system-branch/config +f ./master/test/morphs/DEFAULTS f ./master/test/morphs/VERSION f ./master/test/morphs/hello-stratum.morph f ./master/test/morphs/hello-system.morph diff --git a/tests.branching/checkout-works-anywhere.stdout b/tests.branching/checkout-works-anywhere.stdout index d7f6903a..737a605e 100644 --- a/tests.branching/checkout-works-anywhere.stdout +++ b/tests.branching/checkout-works-anywhere.stdout @@ -7,6 +7,7 @@ d ./master/test d ./master/test/morphs d ./master/test/morphs/.git f ./master/.morph-system-branch/config +f ./master/test/morphs/DEFAULTS f ./master/test/morphs/VERSION f ./master/test/morphs/hello-stratum.morph f ./master/test/morphs/hello-system.morph @@ -24,10 +25,12 @@ d ./newbranch/test d ./newbranch/test/morphs d ./newbranch/test/morphs/.git f ./master/.morph-system-branch/config +f ./master/test/morphs/DEFAULTS f ./master/test/morphs/VERSION f ./master/test/morphs/hello-stratum.morph f ./master/test/morphs/hello-system.morph f ./newbranch/.morph-system-branch/config +f ./newbranch/test/morphs/DEFAULTS f ./newbranch/test/morphs/VERSION f ./newbranch/test/morphs/hello-stratum.morph f ./newbranch/test/morphs/hello-system.morph diff --git a/tests.branching/setup b/tests.branching/setup index 8e82f32c..0d645446 100755 --- a/tests.branching/setup +++ b/tests.branching/setup @@ -46,7 +46,10 @@ mkdir "$DATADIR/morphs" ## Create a link to this repo that has a .git suffix ln -s "$DATADIR/morphs" "$DATADIR/morphs.git" -echo 'version: 6' > "$DATADIR/morphs/VERSION" +echo 'version: 7' > "$DATADIR/morphs/VERSION" + +# We don't build anything, so empty DEFAULTS file is fine. +echo '' > "$DATADIR/morphs/DEFAULTS" cat <<EOF > "$DATADIR/morphs/hello-system.morph" name: hello-system diff --git a/tests.build/build-chunk-failures-dump-log.script b/tests.build/build-chunk-failures-dump-log.script index 81361ad0..75c3caf2 100755 --- a/tests.build/build-chunk-failures-dump-log.script +++ b/tests.build/build-chunk-failures-dump-log.script @@ -20,13 +20,13 @@ set -eu # Make 'hello' chunk fail to build -chunkrepo="$DATADIR/chunk-repo" -cd "$chunkrepo" -git checkout --quiet farrokh +morphsrepo="$DATADIR/morphs-repo" +cd "$morphsrepo" cat <<EOF >hello.morph name: hello kind: chunk -build-system: dummy +configure-commands: + - echo dummy configure build-commands: - echo The next command will fail - "false" diff --git a/tests.build/build-stratum-with-submodules.script b/tests.build/build-stratum-with-submodules.script index bd6b97ce..6753d899 100755 --- a/tests.build/build-stratum-with-submodules.script +++ b/tests.build/build-stratum-with-submodules.script @@ -24,13 +24,7 @@ set -eu parent="$DATADIR/parent-repo" mkdir "$parent" -cat <<EOF > "$parent/parent.morph" -name: parent -kind: chunk -build-system: manual -build-commands: - - test -f le-sub/README -EOF +echo "This is nothing interesting" > "$parent/README" "$SRCDIR/scripts/run-git-in" "$parent" init --quiet "$SRCDIR/scripts/run-git-in" "$parent" add . @@ -42,17 +36,26 @@ EOF # Modify the stratum to refer to the parent, not the submodule. morphs="$DATADIR/morphs-repo" +cat <<EOF > "$morphs/parent.morph" +name: parent +kind: chunk +build-system: manual +build-commands: + - test -f le-sub/README +EOF + cat <<EOF > "$morphs/hello-stratum.morph" name: hello-stratum kind: stratum chunks: - name: parent + morph: parent.morph repo: test:parent-repo ref: master build-depends: [] build-mode: test EOF -"$SRCDIR/scripts/run-git-in" "$morphs" add hello-stratum.morph +"$SRCDIR/scripts/run-git-in" "$morphs" add hello-stratum.morph parent.morph "$SRCDIR/scripts/run-git-in" "$morphs" commit --quiet -m 'foo' diff --git a/tests.build/build-system-autotools-fails-if-autogen-fails.exit b/tests.build/build-system-autotools-fails-if-autogen-fails.exit deleted file mode 100644 index d00491fd..00000000 --- a/tests.build/build-system-autotools-fails-if-autogen-fails.exit +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/tests.build/build-system-autotools-fails-if-autogen-fails.script b/tests.build/build-system-autotools-fails-if-autogen-fails.script deleted file mode 100755 index 0b009b54..00000000 --- a/tests.build/build-system-autotools-fails-if-autogen-fails.script +++ /dev/null @@ -1,40 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2012-2013,2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -## Test that the autotools build system fails if it runs autogen.sh and that -## fails. - -set -eu - -cd "$DATADIR/chunk-repo" -git checkout -q farrokh - -cat <<EOF > autogen.sh -#!/bin/sh -echo "in failing autogen.sh" -exit 1 -EOF -chmod a+x autogen.sh - -git add autogen.sh -git rm -q hello.morph -git commit -q -m "Convert hello to a broken autotools project" - -"$SRCDIR/scripts/test-morph" build-morphology \ - test:morphs-repo master hello-system \ - >/dev/null 2> /dev/null - diff --git a/tests.build/build-system-autotools.script b/tests.build/build-system-autotools.script index 710a8f98..a421eaeb 100755 --- a/tests.build/build-system-autotools.script +++ b/tests.build/build-system-autotools.script @@ -18,8 +18,20 @@ ## Convert the hello-chunk project to something autotools-like, then ## build it. +## The 'autotools' build system is defined in the DEFAULTS file created +## by the 'setup' script. + set -eu +morphsrepo="$DATADIR/morphs-repo" +cd "$morphsrepo" + +cat <<EOF >> hello.morph +build-system: autotools +configure-commands: [] +EOF +git commit --quiet --all -m "Make hello into an autotools project" + chunkrepo="$DATADIR/chunk-repo" cd "$chunkrepo" @@ -35,14 +47,7 @@ install: all EOF git add Makefile -cat <<EOF > hello.morph -name: hello -kind: chunk -build-system: autotools -configure-commands: [] -EOF -git add hello.morph -git commit --quiet -m "Convert hello to an autotools project" +git commit --quiet -m "Make hello a real autotools project" "$SRCDIR/scripts/test-morph" build-morphology \ test:morphs-repo master hello-system diff --git a/tests.build/build-system-cmake.script b/tests.build/build-system-cmake.script deleted file mode 100755 index fe02f9dc..00000000 --- a/tests.build/build-system-cmake.script +++ /dev/null @@ -1,55 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2011-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -## Convert the hello-chunk project to something cmake-like, then -## build it. - -set -eu - -chunkrepo="$DATADIR/chunk-repo" -cd "$chunkrepo" - -git checkout --quiet farrokh - -cat <<'EOF' >CMakeLists.txt -cmake_minimum_required(VERSION 2.8) -project(hello) - -set(hello_SOURCES hello.c) -add_executable(hello ${hello_SOURCES}) -install(TARGETS hello RUNTIME DESTINATION ${CMAKE_INSTALL_PREFIX}/bin) -EOF - -git add CMakeLists.txt - -cat <<EOF > hello.morph -name: hello -kind: chunk -build-system: cmake -install-commands: - - make DESTDIR="\$DESTDIR" install -EOF -git add hello.morph -git commit --quiet -m "Convert hello to a cmake project" - -"$SRCDIR/scripts/test-morph" build-morphology \ - test:morphs-repo master hello-system - -for chunk in "$DATADIR/cache/artifacts/"*.chunk.* -do - tar -tf "$chunk" -done | LC_ALL=C sort -u | sed '/^\.\/./s:^\./::' | grep -Ee '^(usr/)?(bin|etc)' diff --git a/tests.build/build-system-cmake.stdout b/tests.build/build-system-cmake.stdout deleted file mode 100644 index 3410b113..00000000 --- a/tests.build/build-system-cmake.stdout +++ /dev/null @@ -1,2 +0,0 @@ -usr/bin/ -usr/bin/hello diff --git a/tests.build/build-system-cpan.script b/tests.build/build-system-cpan.script deleted file mode 100755 index 103d5466..00000000 --- a/tests.build/build-system-cpan.script +++ /dev/null @@ -1,77 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2011-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -## Convert the hello-chunk project to perl with CPAN and build. - -set -eu - -chunkrepo="$DATADIR/chunk-repo" -cd "$chunkrepo" - -git checkout --quiet farrokh - -git rm --quiet hello.c - -cat <<EOF >hello -#!/usr/bin/perl -print "hello, world\n" -EOF -git add hello - -cat <<EOF >Makefile.PL -use strict; -use warnings; -use ExtUtils::MakeMaker; -WriteMakefile( - EXE_FILES => ['hello'], -) -EOF -git add Makefile.PL - -cat <<EOF >hello.morph -name: hello -kind: chunk -build-system: cpan -EOF -git add hello.morph - -git commit --quiet -m 'convert hello into a perl cpan project' - -# Set 'prefix' of hello to something custom -cd "$DATADIR/morphs-repo" -cat <<EOF > hello-stratum.morph -name: hello-stratum -kind: stratum -chunks: - - name: hello - repo: test:chunk-repo - ref: farrokh - build-depends: [] - build-mode: test - prefix: / -EOF -git add hello-stratum.morph -git commit -q -m "Set custom install prefix for hello" - - -"$SRCDIR/scripts/test-morph" build-morphology \ - test:morphs-repo master hello-system - -for chunk in "$DATADIR/cache/artifacts/"*.chunk.* -do - tar -tf "$chunk" -done | LC_ALL=C sort | sed '/^\.\/./s:^\./::' | grep -F 'bin/hello' diff --git a/tests.build/build-system-cpan.stdout b/tests.build/build-system-cpan.stdout deleted file mode 100644 index 180e949b..00000000 --- a/tests.build/build-system-cpan.stdout +++ /dev/null @@ -1 +0,0 @@ -bin/hello diff --git a/tests.build/build-system-python-distutils.script b/tests.build/build-system-python-distutils.script deleted file mode 100755 index e5c0ea74..00000000 --- a/tests.build/build-system-python-distutils.script +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2011-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -## Convert the hello-chunk project to python with distutils and build. - -set -eu - -chunkrepo="$DATADIR/chunk-repo" -cd "$chunkrepo" - -git checkout --quiet farrokh - -git rm --quiet hello.c -cat <<EOF >hello -#!/usr/bin/python -print "hello, world" -EOF -git add hello - -cat <<EOF >setup.py -#!/usr/bin/python -from distutils.core import setup -setup(name='hello', - scripts=['hello']) -EOF -git add setup.py - -cat <<EOF >hello.morph -name: hello -kind: chunk -build-system: python-distutils -EOF -git add hello.morph - -git commit --quiet -m 'convert hello into a python project' - - -# Set 'prefix' of hello to something custom -cd "$DATADIR/morphs-repo" -cat <<EOF > hello-stratum.morph -name: hello-stratum -kind: stratum -chunks: - - name: hello - repo: test:chunk-repo - ref: farrokh - build-depends: [] - build-mode: test - prefix: "" -EOF -git add hello-stratum.morph -git commit -q -m "Set custom install prefix for hello" - - -"$SRCDIR/scripts/test-morph" build-morphology \ - test:morphs-repo master hello-system - -for chunk in "$DATADIR/cache/artifacts/"*.chunk.* -do - tar -tf "$chunk" -done | LC_ALL=C sort -u | sed '/^\.\/./s:^\./::' | grep -Ee '^(bin|lib)' | -sed -e 's:^local/::' \ - -e 's:lib/python2.[6-9]/:lib/python2.x/:' \ - -e 's:/hello-0\.0\.0[^/]*\.egg-info$:/hello.egg-info/:' \ - -e 's:[^/]*-packages:packages:' \ - -e '/^$/d' diff --git a/tests.build/build-system-python-distutils.stdout b/tests.build/build-system-python-distutils.stdout deleted file mode 100644 index 4d4c3a1e..00000000 --- a/tests.build/build-system-python-distutils.stdout +++ /dev/null @@ -1,6 +0,0 @@ -bin/ -bin/hello -lib/ -lib/python2.x/ -lib/python2.x/packages/ -lib/python2.x/packages/hello.egg-info/ diff --git a/tests.build/build-system-qmake.script b/tests.build/build-system-qmake.script deleted file mode 100755 index d430fba7..00000000 --- a/tests.build/build-system-qmake.script +++ /dev/null @@ -1,65 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2011-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -## Convert the hello-chunk project to something qmake-like, then -## build it. - -set -eu - -if ! command -v qmake > /dev/null ; then - # There is no qmake, so skip this test. - cat "$SRCDIR/tests.build/build-system-qmake.stdout" - exit 0 -fi - -chunkrepo="$DATADIR/chunk-repo" -cd "$chunkrepo" - -git checkout --quiet farrokh - -cat <<'EOF' >hello.pro -TEMPLATE = app -TARGET = hello -DEPENDPATH += . -INCLUDEPATH += . - -SOURCES += hello.c -hello.path = /usr/bin -hello.files = hello -INSTALLS += hello -EOF -git add hello.pro - -cat <<EOF > hello.morph -name: hello -kind: chunk -build-system: qmake -install-commands: - - make INSTALL_ROOT="\$DESTDIR" install -EOF -git add hello.morph -git commit --quiet -m "Convert hello to an qmake project" - -"$SRCDIR/scripts/test-morph" build-morphology \ - test:morphs-repo master hello-system - -for chunk in "$DATADIR/cache/artifacts/"*.chunk.* -do - echo "$chunk:" | sed 's/[^.]*//' - tar -tf "$chunk" | LC_ALL=C sort | sed '/^\.\/./s:^\./::' - echo -done diff --git a/tests.build/build-system-qmake.stdout b/tests.build/build-system-qmake.stdout deleted file mode 100644 index ccf80a86..00000000 --- a/tests.build/build-system-qmake.stdout +++ /dev/null @@ -1,8 +0,0 @@ -.chunk.hello: -./ -baserock/ -baserock/hello.meta -usr/ -usr/bin/ -usr/bin/hello - diff --git a/tests.build/morphless-chunks.script b/tests.build/morphless-chunks.script deleted file mode 100755 index 5b19bc4a..00000000 --- a/tests.build/morphless-chunks.script +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2012-2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -## Try to build a morphless chunk. - -set -eu - -# Make 'hello' chunk into an auto-detectable chunk. - -cd "$DATADIR/chunk-repo" -git checkout -q farrokh - -touch configure -chmod +x configure -# FIXME: If we leave the file empty, busybox sh on ARMv7 fails to execute it. -echo '#!/bin/sh' > configure - -cat << EOF > Makefile -all install: -EOF - -git rm -q hello.morph -git add Makefile configure -git commit -q -m "Convert hello into an autodetectable chunk" - - -"$SRCDIR/scripts/test-morph" build-morphology \ - test:morphs-repo master hello-system - -for chunk in "$DATADIR/cache/artifacts/"*.chunk.* -do - tar -tf "$chunk" -done | cat >/dev/null # No files get installed apart from metadata diff --git a/tests.build/morphless-chunks.stdout b/tests.build/morphless-chunks.stdout deleted file mode 100644 index e69de29b..00000000 --- a/tests.build/morphless-chunks.stdout +++ /dev/null diff --git a/tests.build/prefix.script b/tests.build/prefix.script index 662be704..5713e125 100755 --- a/tests.build/prefix.script +++ b/tests.build/prefix.script @@ -19,9 +19,10 @@ set -eu -# Create two chunks which print out PATH and PREFIX from their environment. -cd "$DATADIR/chunk-repo" +cd "$DATADIR/morphs-repo" git checkout -q master + +# Create two chunks which print out PATH and PREFIX from their environment. cat <<\EOF > xyzzy.morph name: xyzzy kind: chunk @@ -44,18 +45,19 @@ git add plugh.morph git commit -q -m "Add chunks" # Change stratum to include those two chunks, and use a custom install prefix -cd "$DATADIR/morphs-repo" cat <<EOF > hello-stratum.morph name: hello-stratum kind: stratum chunks: - name: xyzzy + morph: xyzzy.morph repo: test:chunk-repo ref: master build-depends: [] build-mode: test prefix: /plover - name: plugh + morph: plugh.morph repo: test:chunk-repo ref: master build-mode: test @@ -69,6 +71,6 @@ git commit -q -m "Update stratum" test:morphs-repo master hello-system cd "$DATADIR/cache/artifacts" -first_chunk=$(ls -1 *.chunk.xyzzy-* | head -n1 | cut -c -64) -second_chunk=$(ls -1 *.chunk.plugh-* | head -n1 | cut -c -64) +first_chunk=$(ls -1 *.chunk.xyzzy* | head -n1 | cut -c -64) +second_chunk=$(ls -1 *.chunk.plugh* | head -n1 | cut -c -64) cat $first_chunk.build-log $second_chunk.build-log diff --git a/tests.build/rebuild-cached-stratum.script b/tests.build/rebuild-cached-stratum.script index e2e0face..bc1b83fd 100755 --- a/tests.build/rebuild-cached-stratum.script +++ b/tests.build/rebuild-cached-stratum.script @@ -41,7 +41,7 @@ cache="$DATADIR/cache/artifacts" "$SRCDIR/scripts/test-morph" build-morphology \ test:morphs-repo rebuild-cached-stratum hello-system echo "first build:" -(cd "$cache" && ls *.chunk.* *hello-stratum-* | sed 's/^[^.]*\./ /' | +(cd "$cache" && ls *.chunk.* *hello-stratum* | sed 's/^[^.]*\./ /' | LC_ALL=C sort -u) # Change the chunk. @@ -53,6 +53,6 @@ echo "first build:" "$SRCDIR/scripts/test-morph" build-morphology \ test:morphs-repo rebuild-cached-stratum hello-system echo "second build:" -(cd "$cache" && ls *.chunk.* *hello-stratum-* | sed 's/^[^.]*\./ /' | +(cd "$cache" && ls *.chunk.* *hello-stratum* | sed 's/^[^.]*\./ /' | LC_ALL=C sort -u) diff --git a/tests.build/rebuild-cached-stratum.stdout b/tests.build/rebuild-cached-stratum.stdout index 9c53ee60..4672e7f2 100644 --- a/tests.build/rebuild-cached-stratum.stdout +++ b/tests.build/rebuild-cached-stratum.stdout @@ -1,22 +1,8 @@ first build: - chunk.hello-bins - chunk.hello-devel - chunk.hello-doc - chunk.hello-libs - chunk.hello-locale - chunk.hello-misc - stratum.hello-stratum-devel - stratum.hello-stratum-devel.meta - stratum.hello-stratum-runtime - stratum.hello-stratum-runtime.meta + chunk.hello-chunk + stratum.hello-stratum + stratum.hello-stratum.meta second build: - chunk.hello-bins - chunk.hello-devel - chunk.hello-doc - chunk.hello-libs - chunk.hello-locale - chunk.hello-misc - stratum.hello-stratum-devel - stratum.hello-stratum-devel.meta - stratum.hello-stratum-runtime - stratum.hello-stratum-runtime.meta + chunk.hello-chunk + stratum.hello-stratum + stratum.hello-stratum.meta diff --git a/tests.build/setup b/tests.build/setup index b7dc5074..7d7b51b5 100755 --- a/tests.build/setup +++ b/tests.build/setup @@ -54,22 +54,7 @@ int main(void) } EOF git add hello.c - -cat <<EOF > hello.morph -name: hello -kind: chunk -build-system: dummy -build-commands: - - gcc -o hello hello.c -install-commands: - - install -d "\$DESTDIR"/etc - - install -d "\$DESTDIR"/bin - - install hello "\$DESTDIR"/bin/hello -EOF -git add hello.morph - -git commit --quiet -m "add a hello world program and morph" - +git commit --quiet -m "add a hello world program" git checkout --quiet master @@ -81,14 +66,44 @@ mkdir "$morphsrepo" cd "$morphsrepo" git init --quiet -echo 'version: 5' > VERSION +echo 'version: 7' > VERSION git add VERSION +cat <<'EOF' > DEFAULTS +# This is a deliberately minimal DEFAULTS file. + +# There are no splitting rules defined, because it's important that Morph +# still works correctly when the user didn't define any. + +build-systems: + autotools: + configure-commands: + - ./configure + build-commands: + - make + install-commands: + - make DESTDIR="$DESTDIR" install +EOF +git add DEFAULTS + +cat <<EOF > hello.morph +name: hello-chunk +kind: chunk +build-commands: + - gcc -o hello hello.c +install-commands: + - install -d "\$DESTDIR"/etc + - install -d "\$DESTDIR"/bin + - install hello "\$DESTDIR"/bin/hello +EOF +git add hello.morph + cat <<EOF > hello-stratum.morph name: hello-stratum kind: stratum chunks: - name: hello + morph: hello.morph repo: test:chunk-repo ref: farrokh build-mode: test diff --git a/tests.build/setup-build-essential b/tests.build/setup-build-essential index 281ff7ec..474c831c 100755 --- a/tests.build/setup-build-essential +++ b/tests.build/setup-build-essential @@ -17,6 +17,7 @@ # Set up a stratum which resembles Baserock's 'build-essential' slightly. Used # for testing 'morph cross-bootstrap' and the 'bootstrap' build mode. +# Add a mock compiler chunk. mkdir -p "$DATADIR/cc-repo" cd "$DATADIR/cc-repo" @@ -26,6 +27,12 @@ echo "I'm a compiler!" EOF chmod +x morph-test-cc +git init -q +git add morph-test-cc +git commit -q -m "Create compiler chunk" + +cd "$DATADIR/morphs-repo" + cat <<EOF > "stage1-cc.morph" name: stage1-cc kind: chunk @@ -43,15 +50,11 @@ install-commands: - install -d "\$DESTDIR\$PREFIX/bin" - install -m 755 morph-test-cc "\$DESTDIR\$PREFIX/bin/morph-test-cc" EOF - -git init -q -git add morph-test-cc cc.morph stage1-cc.morph -git commit -q -m "Create compiler chunk" +git add cc.morph stage1-cc.morph +git commit -q -m "Add build instructions for mock compiler." # Require 'cc' in hello-chunk. We should have the second version available # but *not* the first one. -cd "$DATADIR/chunk-repo" -git checkout -q farrokh cat <<EOF > "hello.morph" name: hello kind: chunk @@ -70,18 +73,20 @@ git commit -q -m "Make 'hello' require our mock compiler" # Add 'build-essential' stratum and make hello-stratum depend upon it. Only # the *second* 'cc' chunk should make it into the build-essential stratum # artifact, and neither should make it into the system. -cd "$DATADIR/morphs-repo" + cat <<EOF > "build-essential.morph" name: build-essential kind: stratum chunks: - name: stage1-cc + morph: stage1-cc.morph repo: test:cc-repo ref: master build-depends: [] build-mode: bootstrap prefix: /tools - name: cc + morph: cc.morph repo: test:cc-repo ref: master build-depends: @@ -96,6 +101,7 @@ build-depends: - morph: build-essential chunks: - name: hello + morph: hello.morph repo: test:chunk-repo ref: farrokh build-depends: [] diff --git a/without-test-modules b/without-test-modules index 95f5c13e..a008e03f 100644 --- a/without-test-modules +++ b/without-test-modules @@ -62,3 +62,4 @@ distbuild/worker_build_scheduler.py # Not unit tested, since it needs a full system branch morphlib/buildbranch.py morphlib/definitions_repo.py +morphlib/defaults.py diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index 3bef0374..a7f3d070 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -238,7 +238,37 @@ another to hold a chunk. mkdir "$DATADIR/gits/morphs" cd "$DATADIR/gits/morphs" git init . - echo 'version: 6' > VERSION + echo 'version: 7' > VERSION + + install -m644 -D /dev/stdin << EOF "DEFAULTS" + # This is a simplified version of the DEFAULTS file supplied with the + # Baserock reference system definitions. + build-systems: + autotools: + configure-commands: + - ./configure + build-commands: + - make + install-commands: + - make install + split-rules: + chunk: + - artifact: -devel + include: + - (usr/)?include/.* + - (usr/)?lib/.*\.a + - (usr/)?share/man/.* + - artifact: -runtime + include: + - .* + stratum: + - artifact: -devel + include: + - .*-devel + - artifact: -runtime + include: + - .*-runtime + EOF arch=$(run_morph print-architecture) install -m644 -D /dev/stdin << EOF "systems/test-system.morph" @@ -321,7 +351,7 @@ another to hold a chunk. install-commands: - copy files system-integration: - test-chunk-bins: + test-chunk-runtime: 00-passwd: - | create file /etc/passwd diff --git a/yarns/regression.yarn b/yarns/regression.yarn index c424f437..aa98eec5 100644 --- a/yarns/regression.yarn +++ b/yarns/regression.yarn @@ -59,17 +59,17 @@ source it depended on. GIVEN a workspace AND a git server AND system systems/test-system.morph uses core-runtime from core - AND stratum strata/core.morph has match rules: [{artifact: core-runtime, include: [.*-(bins|libs|locale)]}, {artifact: core-devel, include: [.*-(devel|doc|misc)]}] + AND stratum strata/core.morph has match rules: [{artifact: core-runtime, include: [.*-devel]}, {artifact: core-devel, include: [.*-runtime]}] WHEN the user checks out the system branch called master GIVEN a cluster called test-cluster.morph in system branch master AND a system in cluster test-cluster.morph in branch master called test-system AND system test-system in cluster test-cluster.morph in branch master builds systems/test-system.morph AND system test-system in cluster test-cluster.morph in branch master has deployment type: tar WHEN the user builds the system systems/test-system.morph in branch master - GIVEN stratum strata/core.morph in system branch master has match rules: [{artifact: core-runtime, include: [.*-(bins|libs|misc)]}, {artifact: core-devel, include: [.*-(devel|doc|locale)]}] + GIVEN stratum strata/core.morph in system branch master has match rules: [{artifact: core-runtime, include: [.*-runtime]}, {artifact: core-devel, include: [.*-devel]}] WHEN the user builds the system systems/test-system.morph in branch master AND the user deploys the cluster test-cluster.morph in branch master with options test-system.location="$DATADIR/test.tar" - THEN tarball test.tar contains baserock/test-chunk-misc.meta + THEN tarball test.tar contains baserock/test-chunk-runtime.meta FINALLY the git server is shut down |