summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorSam Thursfield <sam.thursfield@codethink.co.uk>2015-07-28 15:23:09 +0000
committerSam Thursfield <sam.thursfield@codethink.co.uk>2015-10-14 14:34:29 +0000
commitb8b8de2b123ff96e6469196934359485bcd06b04 (patch)
tree2bb774dcaf3db88d6dafb8ab34322b5df4550ce0
parent3c59628c80ffe47992bf7347268ca587fc6d368d (diff)
downloadmorph-b8b8de2b123ff96e6469196934359485bcd06b04.tar.gz
Add support for Baserock definitions version 7
This adds a new 'Defaults' class to represent definitions defaults The Python 'jsonschema' module is used to validate the contents of the Defaults file. This module is already included in Baserock 'build' and 'devel' reference systems by way of the 'openstack-common' stratum. This commit embeds a copy of the JSON-Schema schema for the DEFAULTS file. I think the canonical location of this schema should be in the reference definitions.git, for now. In future, the schemas should maybe have their own repos. Either way, Morph should embed a copy for the time being so that we are sure the schema matches how Morph expects to parse the file. Morph's automated tests are all updated to use definitions version 7. I removed most of the tests for built-in build systems, because the built-ins themselves are no longer part of Morph. Only the mechanism for defining them needs to be tested now. Change-Id: I65f8f1c967683ef605852bfae5c68518e53f9981
-rwxr-xr-xcheck1
-rw-r--r--morphlib/__init__.py1
-rw-r--r--morphlib/artifactresolver_tests.py88
-rw-r--r--morphlib/artifactsplitrule.py18
-rw-r--r--morphlib/buildsystem.py15
-rw-r--r--morphlib/buildsystem_tests.py9
-rw-r--r--morphlib/cachekeycomputer_tests.py12
-rw-r--r--morphlib/defaults.py108
-rw-r--r--morphlib/definitions_repo.py21
-rw-r--r--morphlib/definitions_version.py2
-rw-r--r--morphlib/morphloader.py37
-rw-r--r--morphlib/morphloader_tests.py41
-rw-r--r--morphlib/morphologyfinder.py14
-rw-r--r--morphlib/morphologyfinder_tests.py4
-rw-r--r--morphlib/schemas/defaults.json-schema66
-rw-r--r--morphlib/source.py9
-rw-r--r--morphlib/sourceresolver.py69
-rw-r--r--morphlib/util.py7
-rw-r--r--tests.branching/branch-creates-new-system-branch-not-from-master.stdout1
-rw-r--r--tests.branching/branch-works-anywhere.stdout10
-rw-r--r--tests.branching/checkout-works-anywhere.stdout3
-rwxr-xr-xtests.branching/setup5
-rwxr-xr-xtests.build/ambiguous-refs.script3
-rwxr-xr-xtests.build/build-stratum-with-submodules.script2
-rwxr-xr-xtests.build/prefix.script6
-rwxr-xr-xtests.build/rebuild-cached-stratum.script4
-rw-r--r--tests.build/rebuild-cached-stratum.stdout26
-rwxr-xr-xtests.build/setup25
-rwxr-xr-xtests.build/setup-build-essential12
-rw-r--r--without-test-modules1
-rw-r--r--yarns/implementations.yarn34
-rw-r--r--yarns/regression.yarn6
32 files changed, 518 insertions, 142 deletions
diff --git a/check b/check
index 5aea6f55..1d56c514 100755
--- a/check
+++ b/check
@@ -134,6 +134,7 @@ then
echo 'Checking source code for silliness'
if ! (git ls-files --cached |
grep -v '\.gz$' |
+ grep -v '\.json-schema$' |
grep -Ev 'tests[^/]*/.*\.std(out|err)' |
grep -vF 'tests.build/build-system-autotools.script' |
xargs -r scripts/check-silliness); then
diff --git a/morphlib/__init__.py b/morphlib/__init__.py
index 81540a31..36e5c3a8 100644
--- a/morphlib/__init__.py
+++ b/morphlib/__init__.py
@@ -60,6 +60,7 @@ import builder
import cachedrepo
import cachekeycomputer
import cmdline_parse_utils
+import defaults
import definitions_repo
import definitions_version
import extensions
diff --git a/morphlib/artifactresolver_tests.py b/morphlib/artifactresolver_tests.py
index 20617c65..4bf42a93 100644
--- a/morphlib/artifactresolver_tests.py
+++ b/morphlib/artifactresolver_tests.py
@@ -20,6 +20,12 @@ import yaml
import morphlib
+default_split_rules = {
+ 'chunk': morphlib.artifactsplitrule.DEFAULT_CHUNK_RULES,
+ 'stratum': morphlib.artifactsplitrule.DEFAULT_STRATUM_RULES,
+}
+
+
def get_chunk_morphology(name, artifact_names=[]):
assert(isinstance(artifact_names, list))
@@ -88,9 +94,9 @@ class ArtifactResolverTests(unittest.TestCase):
pool = morphlib.sourcepool.SourcePool()
morph = get_chunk_morphology('chunk')
- sources = morphlib.source.make_sources('repo', 'ref',
- 'chunk.morph', 'sha1',
- 'tree', morph)
+ sources = morphlib.source.make_sources(
+ 'repo', 'ref', 'chunk.morph', 'sha1', 'tree', morph,
+ default_split_rules=default_split_rules)
for source in sources:
pool.add(source)
@@ -109,9 +115,9 @@ class ArtifactResolverTests(unittest.TestCase):
pool = morphlib.sourcepool.SourcePool()
morph = get_chunk_morphology('chunk', ['chunk-foobar'])
- sources = morphlib.source.make_sources('repo', 'ref',
- 'chunk.morph', 'sha1',
- 'tree', morph)
+ sources = morphlib.source.make_sources(
+ 'repo', 'ref', 'chunk.morph', 'sha1', 'tree', morph,
+ default_split_rules=default_split_rules)
for source in sources:
pool.add(source)
@@ -129,9 +135,9 @@ class ArtifactResolverTests(unittest.TestCase):
pool = morphlib.sourcepool.SourcePool()
morph = get_chunk_morphology('chunk', ['chunk-baz', 'chunk-qux'])
- sources = morphlib.source.make_sources('repo', 'ref',
- 'chunk.morph', 'sha1',
- 'tree', morph)
+ sources = morphlib.source.make_sources(
+ 'repo', 'ref', 'chunk.morph', 'sha1', 'tree', morph,
+ default_split_rules=default_split_rules)
for source in sources:
pool.add(source)
@@ -151,18 +157,18 @@ class ArtifactResolverTests(unittest.TestCase):
pool = morphlib.sourcepool.SourcePool()
morph = get_chunk_morphology('chunk')
- sources = morphlib.source.make_sources('repo', 'ref',
- 'chunk.morph', 'sha1',
- 'tree', morph)
+ sources = morphlib.source.make_sources(
+ 'repo', 'ref', 'chunk.morph', 'sha1', 'tree', morph,
+ default_split_rules=default_split_rules)
for chunk in sources:
pool.add(chunk)
morph = get_stratum_morphology(
'stratum', chunks=[('chunk', 'chunk', 'repo', 'ref')])
- stratum_sources = set(morphlib.source.make_sources('repo', 'ref',
- 'stratum.morph',
- 'sha1', 'tree',
- morph))
+ stratum_sources = set(
+ morphlib.source.make_sources(
+ 'repo', 'ref', 'stratum.morph', 'sha1', 'tree', morph,
+ default_split_rules=default_split_rules))
for stratum in stratum_sources:
pool.add(stratum)
@@ -197,9 +203,9 @@ class ArtifactResolverTests(unittest.TestCase):
pool = morphlib.sourcepool.SourcePool()
morph = get_chunk_morphology('chunk', ['chunk-foo', 'chunk-bar'])
- sources = morphlib.source.make_sources('repo', 'ref',
- 'chunk.morph', 'sha1',
- 'tree', morph)
+ sources = morphlib.source.make_sources(
+ 'repo', 'ref', 'chunk.morph', 'sha1', 'tree', morph,
+ default_split_rules=default_split_rules)
for chunk in sources:
pool.add(chunk)
@@ -208,10 +214,10 @@ class ArtifactResolverTests(unittest.TestCase):
chunks=[
('chunk', 'chunk', 'repo', 'ref'),
])
- stratum_sources = set(morphlib.source.make_sources('repo', 'ref',
- 'stratum.morph',
- 'sha1', 'tree',
- morph))
+ stratum_sources = set(
+ morphlib.source.make_sources(
+ 'repo', 'ref', 'stratum.morph', 'sha1', 'tree', morph,
+ default_split_rules=default_split_rules))
for stratum in stratum_sources:
pool.add(stratum)
@@ -246,7 +252,8 @@ class ArtifactResolverTests(unittest.TestCase):
chunk = get_chunk_morphology('chunk1')
chunk1, = morphlib.source.make_sources(
- 'repo', 'original/ref', 'chunk1.morph', 'sha1', 'tree', chunk)
+ 'repo', 'original/ref', 'chunk1.morph', 'sha1', 'tree', chunk,
+ default_split_rules=default_split_rules)
pool.add(chunk1)
morph = get_stratum_morphology(
@@ -254,15 +261,16 @@ class ArtifactResolverTests(unittest.TestCase):
chunks=[(loader.save_to_string(chunk), 'chunk1.morph',
'repo', 'original/ref')],
build_depends=['stratum2'])
- sources = morphlib.source.make_sources('repo', 'original/ref',
- 'stratum1.morph', 'sha1',
- 'tree', morph)
+ sources = morphlib.source.make_sources(
+ 'repo', 'original/ref', 'stratum1.morph', 'sha1', 'tree', morph,
+ default_split_rules=default_split_rules)
for stratum1 in sources:
pool.add(stratum1)
chunk = get_chunk_morphology('chunk2')
chunk2, = morphlib.source.make_sources(
- 'repo', 'original/ref', 'chunk2.morph', 'sha1', 'tree', chunk)
+ 'repo', 'original/ref', 'chunk2.morph', 'sha1', 'tree', chunk,
+ default_split_rules=default_split_rules)
pool.add(chunk2)
morph = get_stratum_morphology(
@@ -270,9 +278,9 @@ class ArtifactResolverTests(unittest.TestCase):
chunks=[(loader.save_to_string(chunk), 'chunk2.morph',
'repo', 'original/ref')],
build_depends=['stratum1'])
- sources = morphlib.source.make_sources('repo', 'original/ref',
- 'stratum2.morph', 'sha1',
- 'tree', morph)
+ sources = morphlib.source.make_sources(
+ 'repo', 'original/ref', 'stratum2.morph', 'sha1', 'tree', morph,
+ default_split_rules=default_split_rules)
for stratum2 in sources:
pool.add(stratum2)
@@ -301,23 +309,23 @@ class ArtifactResolverTests(unittest.TestCase):
build-system: manual
build-depends: []
''')
- sources = morphlib.source.make_sources('repo', 'original/ref',
- 'stratum.morph', 'sha1',
- 'tree', morph)
+ sources = morphlib.source.make_sources(
+ 'repo', 'original/ref', 'stratum.morph', 'sha1', 'tree', morph,
+ default_split_rules=default_split_rules)
for stratum in sources:
pool.add(stratum)
morph = get_chunk_morphology('chunk1')
- sources = morphlib.source.make_sources('repo', 'original/ref',
- 'chunk1.morph', 'sha1',
- 'tree', morph)
+ sources = morphlib.source.make_sources(
+ 'repo', 'original/ref', 'chunk1.morph', 'sha1', 'tree', morph,
+ default_split_rules=default_split_rules)
for chunk1 in sources:
pool.add(chunk1)
morph = get_chunk_morphology('chunk2')
- sources = morphlib.source.make_sources('repo', 'original/ref',
- 'chunk2.morph', 'sha1',
- 'tree', morph)
+ sources = morphlib.source.make_sources(
+ 'repo', 'original/ref', 'chunk2.morph', 'sha1', 'tree', morph,
+ default_split_rules=default_split_rules)
for chunk2 in sources:
pool.add(chunk2)
diff --git a/morphlib/artifactsplitrule.py b/morphlib/artifactsplitrule.py
index b5ebdf83..ba5abe02 100644
--- a/morphlib/artifactsplitrule.py
+++ b/morphlib/artifactsplitrule.py
@@ -189,10 +189,6 @@ class SplitRules(collections.Iterable):
for artifact, rule in self._rules)
-# TODO: Work out a good way to feed new defaults in. This is good for
-# the usual Linux userspace, but we may find issues and need a
-# migration path to a more useful set, or develop a system with
-# a different layout, like Android.
DEFAULT_CHUNK_RULES = [
('-bins', [ r"(usr/)?s?bin/.*" ]),
('-libs', [
@@ -229,6 +225,14 @@ DEFAULT_STRATUM_RULES = [
]
+# A 'no-op' set of split rules. An empty list would cause everything to be
+# ignored, which is unlikely to ever be what a user wants, and breaks some
+# internal bits of Morph.
+EMPTY_RULES = [
+ ('', [r'.*'])
+]
+
+
def unify_chunk_matches(morphology, default_rules=DEFAULT_CHUNK_RULES):
'''Create split rules including defaults and per-chunk rules.
@@ -237,6 +241,8 @@ def unify_chunk_matches(morphology, default_rules=DEFAULT_CHUNK_RULES):
by building the chunk to the chunk artifact they should be put in.
'''
+ if default_rules is None or len(default_rules) == 0:
+ default_rules = EMPTY_RULES
split_rules = SplitRules()
@@ -265,6 +271,8 @@ def unify_stratum_matches(morphology, default_rules=DEFAULT_STRATUM_RULES):
strata to the stratum artifact they should be put in.
'''
+ if default_rules is None or len(default_rules) == 0:
+ default_rules = EMPTY_RULES
assignment_split_rules = SplitRules()
for spec in morphology['chunks']:
@@ -296,7 +304,7 @@ def unify_stratum_matches(morphology, default_rules=DEFAULT_STRATUM_RULES):
match_split_rules))
-def unify_system_matches(morphology):
+def unify_system_matches(morphology, default_rules=[]):
'''Create split rules including defaults and per-chunk rules.
With rules specified in the morphology's 'products' field and the
diff --git a/morphlib/buildsystem.py b/morphlib/buildsystem.py
index d7fbaf73..29d35627 100644
--- a/morphlib/buildsystem.py
+++ b/morphlib/buildsystem.py
@@ -44,11 +44,10 @@ _STRIP_COMMAND = r'''find "$DESTDIR" -type f \
class BuildSystem(object):
- '''Predefined commands for common build systems.
+ '''Predefined command sequences for a given build system.
- Some build systems are well known: autotools, for example. We provide
- pre-defined build commands for these so that they don't need to be copied
- and pasted many times in the build instructions.
+ For example, you can have an 'autotools' build system, which runs
+ 'configure', 'make' and 'make install'.
'''
@@ -69,6 +68,14 @@ class BuildSystem(object):
self.strip_commands = []
self.post_strip_commands = []
+ def from_dict(self, name, commands):
+ self.name = name
+
+ self.configure_commands = commands.get('configure-commands', [])
+ self.build_commands = commands.get('build-commands', [])
+ self.install_commands = commands.get('install-commands', [])
+ self.strip_commands = commands.get('strip-commands', [])
+
def __getitem__(self, key):
key = '_'.join(key.split('-'))
return getattr(self, key)
diff --git a/morphlib/buildsystem_tests.py b/morphlib/buildsystem_tests.py
index b49d30ae..6b6bf30f 100644
--- a/morphlib/buildsystem_tests.py
+++ b/morphlib/buildsystem_tests.py
@@ -43,6 +43,15 @@ class BuildSystemTests(unittest.TestCase):
morph = self.bs.get_morphology('foobar')
self.assertTrue(morph.__class__.__name__ == 'Morphology')
+ def test_construct_from_dict(self):
+ '''Test parsing a dict of information from a DEFAULTS file.'''
+
+ commands_dict = {
+ 'configure-commands': 'foo'
+ }
+ self.bs.from_dict('test', commands_dict)
+ self.assertEqual(self.bs.configure_commands, 'foo')
+
class LookupBuildSystemTests(unittest.TestCase):
diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py
index fbf680f0..6e736796 100644
--- a/morphlib/cachekeycomputer_tests.py
+++ b/morphlib/cachekeycomputer_tests.py
@@ -28,6 +28,12 @@ class DummyBuildEnvironment:
self.env = env
+default_split_rules = {
+ 'chunk': morphlib.artifactsplitrule.DEFAULT_CHUNK_RULES,
+ 'stratum': morphlib.artifactsplitrule.DEFAULT_STRATUM_RULES,
+}
+
+
class CacheKeyComputerTests(unittest.TestCase):
def setUp(self):
@@ -86,9 +92,9 @@ class CacheKeyComputerTests(unittest.TestCase):
''',
}.iteritems():
morph = loader.load_from_string(text)
- sources = morphlib.source.make_sources('repo', 'original/ref',
- name, 'sha1',
- 'tree', morph)
+ sources = morphlib.source.make_sources(
+ 'repo', 'original/ref', name, 'sha1', 'tree', morph,
+ default_split_rules=default_split_rules)
for source in sources:
self.source_pool.add(source)
self.build_env = DummyBuildEnvironment({
diff --git a/morphlib/defaults.py b/morphlib/defaults.py
new file mode 100644
index 00000000..9e695a90
--- /dev/null
+++ b/morphlib/defaults.py
@@ -0,0 +1,108 @@
+# Copyright (C) 2015 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+# =*= License: GPL-2 =*=
+
+
+import cliapp
+import jsonschema
+import yaml
+
+import os
+
+import morphlib
+
+
+class Defaults(object):
+ '''Represents predefined default values specific to Baserock definitions.
+
+ The DEFAULTS file was added in definitions format version 7, which lets
+ users set these defaults. The text of DEFAULTS file can be passed in as
+ 'text', and will be validated and parsed if definitions_version >= 7.
+
+ Prior to version 7, the defaults were hardcoded in Morph. These defaults
+ will be returned if definitions_version < 7.
+
+ '''
+ def __init__(self, definitions_version, text=None):
+ self._build_systems = {}
+ self._split_rules = {}
+
+ schema_path = os.path.join(morphlib.util.schemas_directory(),
+ 'defaults.json-schema')
+ with open(schema_path) as f:
+ self.schema = yaml.load(f)
+
+ if definitions_version >= 7:
+ if text:
+ self._build_systems, self._split_rules = self._parse(text)
+ else:
+ self._build_systems, self._split_rules = self._builtins()
+
+ def _parse(self, text):
+ build_systems = {}
+ split_rules = {}
+
+ # This reports errors against <string> rather than the actual filename,
+ # which is sad.
+ data = yaml.safe_load(text)
+
+ if data is None:
+ # It's OK to be empty, I guess.
+ return build_systems, split_rules
+
+ try:
+ # It would be nice if this could give line numbers when it spotted
+ # errors. Seems tricky.
+ jsonschema.validate(data, self.schema)
+ except jsonschema.ValidationError as e:
+ raise cliapp.AppException('Invalid DEFAULTS file: %s' % e.message)
+
+ build_system_data = data.get('build-systems', {})
+ for name, commands in build_system_data.items():
+ build_system = morphlib.buildsystem.BuildSystem()
+ build_system.from_dict(name, commands)
+ build_systems[name] = build_system
+
+ # It would make sense to create artifactsplitrule.SplitRule instances
+ # here, instead of an unlabelled data structure. That would need some
+ # changes to source.make_sources() and the 'artifactsplitrule' module.
+ split_rules_data = data.get('split-rules', {})
+ for kind, rules in split_rules_data.items():
+ split_rules[kind] = []
+ for rule in rules:
+ rule_unlabelled = (rule['artifact'], rule['include'])
+ split_rules[kind].append(rule_unlabelled)
+
+ return build_systems, split_rules
+
+ def _builtins(self):
+ build_systems = {}
+ split_rules = {}
+
+ for build_system in morphlib.buildsystem.build_systems:
+ build_systems[build_system.name] = build_system
+
+ split_rules['chunk'] = \
+ morphlib.artifactsplitrule.DEFAULT_CHUNK_RULES
+ split_rules['stratum'] = \
+ morphlib.artifactsplitrule.DEFAULT_STRATUM_RULES
+
+ return build_systems, split_rules
+
+ def build_systems(self):
+ return self._build_systems
+
+ def split_rules(self):
+ return self._split_rules
diff --git a/morphlib/definitions_repo.py b/morphlib/definitions_repo.py
index 301e5b86..6d14fcbb 100644
--- a/morphlib/definitions_repo.py
+++ b/morphlib/definitions_repo.py
@@ -23,6 +23,7 @@ import logging
import os
import urlparse
import uuid
+import warnings
import morphlib
import gitdir
@@ -236,9 +237,25 @@ class DefinitionsRepo(gitdir.GitDirectory):
mf = morphlib.morphologyfinder.MorphologyFinder(self)
version_text = mf.read_file('VERSION')
- morphlib.definitions_version.check_version_file(version_text)
+ version = morphlib.definitions_version.check_version_file(version_text)
- loader = morphlib.morphloader.MorphologyLoader()
+ defaults_text = mf.read_file('DEFAULTS', allow_missing=True)
+
+ if version < 7:
+ if defaults_text is not None:
+ warnings.warn(
+ "Ignoring DEFAULTS file, because these definitions are "
+ "version %i" % version)
+ defaults_text = None
+ else:
+ if defaults_text is None:
+ warnings.warn("No DEFAULTS file found.")
+
+ defaults = morphlib.defaults.Defaults(version,
+ text=defaults_text)
+
+ loader = morphlib.morphloader.MorphologyLoader(
+ predefined_build_systems=defaults.build_systems())
return loader
diff --git a/morphlib/definitions_version.py b/morphlib/definitions_version.py
index bc72d2ad..2fb7785a 100644
--- a/morphlib/definitions_version.py
+++ b/morphlib/definitions_version.py
@@ -24,7 +24,7 @@ import yaml
import morphlib
-SUPPORTED_VERSIONS = [6]
+SUPPORTED_VERSIONS = [6, 7]
class DefinitionsVersionError(cliapp.AppException):
diff --git a/morphlib/morphloader.py b/morphlib/morphloader.py
index f85c5d4d..ba48b778 100644
--- a/morphlib/morphloader.py
+++ b/morphlib/morphloader.py
@@ -109,6 +109,13 @@ class UnknownArchitectureError(MorphologyValidationError):
% (arch, morph_filename))
+class UnknownBuildSystemError(MorphologyValidationError):
+
+ def __init__(self, build_system, morph_filename):
+ self.msg = ('Undefined build system %s in morphology %s'
+ % (build_system, morph_filename))
+
+
class NoStratumBuildDependenciesError(MorphologyValidationError):
def __init__(self, stratum_name, morph_filename):
@@ -404,8 +411,12 @@ class MorphologyLoader(object):
}
def __init__(self,
- lookup_build_system=morphlib.buildsystem.lookup_build_system):
- self._lookup_build_system = lookup_build_system
+ predefined_build_systems={}):
+ self._predefined_build_systems = predefined_build_systems.copy()
+
+ if 'manual' not in self._predefined_build_systems:
+ self._predefined_build_systems['manual'] = \
+ morphlib.buildsystem.ManualBuildSystem()
def parse_morphology_text(self, text, morph_filename):
'''Parse a textual morphology.
@@ -811,10 +822,12 @@ class MorphologyLoader(object):
if morph['max-jobs'] is not None:
morph['max-jobs'] = int(morph['max-jobs'])
- def _unset_chunk_defaults(self, morph): # pragma: no cover
+ def _unset_chunk_defaults(self, morph): # pragma: no cover
+ # This is only used by the deprecated branch-and-merge plugin, and
+ # probably doesn't work correctly for definitions V7 and newer.
default_bs = self._static_defaults['chunk']['build-system']
- bs = self._lookup_build_system(
- morph.get('build-system', default_bs))
+ bs_name = morph.get('build-system', default_bs)
+ bs = self.lookup_build_system(bs_name)
for key in self._static_defaults['chunk']:
if key not in morph: continue
if 'commands' not in key: continue
@@ -823,11 +836,19 @@ class MorphologyLoader(object):
if morph[key] == default_value:
del morph[key]
+ def lookup_build_system(self, name):
+ return self._predefined_build_systems[name]
+
def set_commands(self, morph):
- default = self._static_defaults['chunk']['build-system']
- bs = self._lookup_build_system(
- morph.get('build-system', default))
if morph['kind'] == 'chunk':
+ default = self._static_defaults['chunk']['build-system']
+ bs_name = morph.get('build-system', default)
+
+ try:
+ bs = self.lookup_build_system(bs_name)
+ except KeyError:
+ raise UnknownBuildSystemError(bs_name, morph['name'])
+
for key in self._static_defaults['chunk']:
if 'commands' not in key: continue
if key not in morph:
diff --git a/morphlib/morphloader_tests.py b/morphlib/morphloader_tests.py
index 6117573e..392b0cbc 100644
--- a/morphlib/morphloader_tests.py
+++ b/morphlib/morphloader_tests.py
@@ -60,12 +60,12 @@ class MorphologyLoaderTests(unittest.TestCase):
string = '''\
name: foo
kind: chunk
-build-system: dummy
+build-system: manual
'''
morph = self.loader.parse_morphology_text(string, 'test')
self.assertEqual(morph['kind'], 'chunk')
self.assertEqual(morph['name'], 'foo')
- self.assertEqual(morph['build-system'], 'dummy')
+ self.assertEqual(morph['build-system'], 'manual')
def test_fails_to_parse_utter_garbage(self):
self.assertRaises(
@@ -494,43 +494,43 @@ chunks:
string = '''\
name: foo
kind: chunk
-build-system: dummy
+build-system: manual
'''
morph = self.loader.load_from_string(string)
self.assertEqual(morph['kind'], 'chunk')
self.assertEqual(morph['name'], 'foo')
- self.assertEqual(morph['build-system'], 'dummy')
+ self.assertEqual(morph['build-system'], 'manual')
def test_loads_json_from_string(self):
string = '''\
{
"name": "foo",
"kind": "chunk",
- "build-system": "dummy"
+ "build-system": "manual"
}
'''
morph = self.loader.load_from_string(string)
self.assertEqual(morph['kind'], 'chunk')
self.assertEqual(morph['name'], 'foo')
- self.assertEqual(morph['build-system'], 'dummy')
+ self.assertEqual(morph['build-system'], 'manual')
def test_loads_from_file(self):
with open(self.filename, 'w') as f:
f.write('''\
name: foo
kind: chunk
-build-system: dummy
+build-system: manual
''')
morph = self.loader.load_from_file(self.filename)
self.assertEqual(morph['kind'], 'chunk')
self.assertEqual(morph['name'], 'foo')
- self.assertEqual(morph['build-system'], 'dummy')
+ self.assertEqual(morph['build-system'], 'manual')
def test_saves_to_string(self):
morph = morphlib.morphology.Morphology({
'name': 'foo',
'kind': 'chunk',
- 'build-system': 'dummy',
+ 'build-system': 'manual',
})
text = self.loader.save_to_string(morph)
@@ -539,14 +539,14 @@ build-system: dummy
self.assertEqual(text, '''\
name: foo
kind: chunk
-build-system: dummy
+build-system: manual
''')
def test_saves_to_file(self):
morph = morphlib.morphology.Morphology({
'name': 'foo',
'kind': 'chunk',
- 'build-system': 'dummy',
+ 'build-system': 'manual',
})
self.loader.save_to_file(self.filename, morph)
@@ -558,7 +558,7 @@ build-system: dummy
self.assertEqual(text, '''\
name: foo
kind: chunk
-build-system: dummy
+build-system: manual
''')
def test_validate_does_not_set_defaults(self):
@@ -966,11 +966,12 @@ build-system: dummy
)
s = self.loader.save_to_string(m)
- def test_smoketest_strip_commands(self):
- dummy_buildsystem = morphlib.buildsystem.DummyBuildSystem()
- loader = morphlib.morphloader.MorphologyLoader(
- lookup_build_system=lambda x: dummy_buildsystem)
- m = morphlib.morphology.Morphology(
- {'name': 'test', 'kind': 'chunk', 'build-system': 'dummy'})
- loader.set_commands(m)
- self.assertEqual(m['strip-commands'], dummy_buildsystem.strip_commands)
+
+ def test_unknown_build_system(self):
+ m = morphlib.morphology.Morphology({
+ 'kind': 'chunk',
+ 'name': 'foo',
+ 'build-system': 'monkey scientist',
+ })
+ with self.assertRaises(morphlib.morphloader.UnknownBuildSystemError):
+ s = self.loader.set_commands(m)
diff --git a/morphlib/morphologyfinder.py b/morphlib/morphologyfinder.py
index 9184c160..a334350e 100644
--- a/morphlib/morphologyfinder.py
+++ b/morphlib/morphologyfinder.py
@@ -15,9 +15,7 @@
# =*= License: GPL-2 =*=
-import cliapp
-
-import morphlib
+import errno
class MorphologyFinder(object):
@@ -33,9 +31,15 @@ class MorphologyFinder(object):
self.gitdir = gitdir
self.ref = ref
- def read_file(self, filename):
+ def read_file(self, filename, allow_missing=False):
'''Return the text of a file inside the Git repo.'''
- return self.gitdir.read_file(filename, self.ref)
+ try:
+ return self.gitdir.read_file(filename, self.ref)
+ except IOError as e:
+ if allow_missing and e.errno == errno.ENOENT:
+ return None
+ else:
+ raise
def list_morphologies(self):
'''Return the filenames of all morphologies in the (repo, ref).
diff --git a/morphlib/morphologyfinder_tests.py b/morphlib/morphologyfinder_tests.py
index 59772e23..3312b627 100644
--- a/morphlib/morphologyfinder_tests.py
+++ b/morphlib/morphologyfinder_tests.py
@@ -103,6 +103,10 @@ class MorphologyFinderTests(unittest.TestCase):
mf = morphlib.morphologyfinder.MorphologyFinder(gd)
self.assertEqual(mf.read_file('foo.morph'),
"altered morphology text")
+ with self.assertRaises(IOError):
+ mf.read_file('missing')
+ self.assertEqual(mf.read_file('missing', allow_missing=True),
+ None)
def test_read_morph_raises_no_worktree_no_ref(self):
gd = morphlib.gitdir.GitDirectory(self.mirror)
diff --git a/morphlib/schemas/defaults.json-schema b/morphlib/schemas/defaults.json-schema
new file mode 100644
index 00000000..2f713425
--- /dev/null
+++ b/morphlib/schemas/defaults.json-schema
@@ -0,0 +1,66 @@
+$schema: http://json-schema.org/draft-04/schema#
+id: http://git.baserock.org/cgi-bin/cgit.cgi/baserock/baserock/definitions.git/tree/schemas/defaults.json-schema
+
+description: |
+ This is a JSON-Schema description of the DEFAULTS file specified in the
+ Baserock definitions format. DEFAULTS is a YAML file that contains global
+ defaults for a set of Baserock definitions.
+
+ This JSON-Schema file is valid for VERSION 7 of the Baserock definitions
+ YAML serialisation format.
+
+ The Baserock definitions YAML serialisation format is the recommended way of
+ representing Baserock definitions on disk. The actual data model is described
+ separately. See <https://wiki.baserock.org/definitions> for more information.
+
+ This schema is represented as YAML, so that it can be edited more easily.
+ You may need to convert to JSON if using a JSON-Schema tool that expects
+ its input to be an actual string containing data serialised as JSON.
+
+definitions:
+ command-sequence:
+ type: array
+ items: {type: string}
+
+ build-system:
+ type: object
+ additionalProperties: false
+ properties:
+ build-commands: {$ref: '#/definitions/command-sequence'}
+ configure-commands: {$ref: '#/definitions/command-sequence'}
+ install-commands: {$ref: '#/definitions/command-sequence'}
+ strip-commands: {$ref: '#/definitions/command-sequence'}
+
+ split-rules:
+ type: array
+ items:
+ type: object
+
+ required: [artifact, include]
+ additionalProperties: false
+
+ properties:
+ artifact: {type: string}
+ include:
+ type: array
+ items:
+ type: string
+ format: regex
+
+type: object
+additionalProperties: false
+
+properties:
+ # Predefined build systems.
+ build-systems:
+ type: object
+ patternProperties:
+ ^.*$: {$ref: '#/definitions/build-system'}
+
+ # Predefined artifact splitting rules.
+ split-rules:
+ type: object
+ additionalProperties: false
+ properties:
+ chunk: {$ref: '#/definitions/split-rules'}
+ stratum: {$ref: '#/definitions/split-rules'}
diff --git a/morphlib/source.py b/morphlib/source.py
index 2c96bcbd..135c14cc 100644
--- a/morphlib/source.py
+++ b/morphlib/source.py
@@ -77,12 +77,14 @@ class Source(object):
return artifact in self.dependencies
-def make_sources(reponame, ref, filename, absref, tree, morphology):
+def make_sources(reponame, ref, filename, absref, tree, morphology,
+ default_split_rules={}):
kind = morphology['kind']
if kind in ('system', 'chunk'):
unifier = getattr(morphlib.artifactsplitrule,
'unify_%s_matches' % kind)
- split_rules = unifier(morphology)
+ split_rules = unifier(morphology,
+ default_rules=default_split_rules.get(kind, {}))
# chunk and system sources are named after the morphology
source_name = morphology['name']
source = morphlib.source.Source(source_name, reponame, ref,
@@ -93,7 +95,8 @@ def make_sources(reponame, ref, filename, absref, tree, morphology):
yield source
elif kind == 'stratum': # pragma: no cover
unifier = morphlib.artifactsplitrule.unify_stratum_matches
- split_rules = unifier(morphology)
+ split_rules = unifier(morphology,
+ default_rules=default_split_rules.get(kind, {}))
for name in split_rules.artifacts:
source = morphlib.source.Source(
name, # stratum source name is artifact name
diff --git a/morphlib/sourceresolver.py b/morphlib/sourceresolver.py
index 0b32598f..1056af16 100644
--- a/morphlib/sourceresolver.py
+++ b/morphlib/sourceresolver.py
@@ -19,7 +19,7 @@ import cPickle
import logging
import os
import pylru
-import yaml
+import warnings
import cliapp
@@ -259,6 +259,36 @@ class SourceResolver(object):
return morphlib.definitions_version.check_version_file(version_text)
+ def _get_defaults(self, definitions_checkout_dir,
+ definitions_version=7): # pragma: no cover
+ '''Return the default build system commands, and default split rules.
+
+ This function returns a tuple with two dicts.
+
+ The defaults are read from a file named DEFAULTS in the definitions
+ directory, if the definitions follow format version 7 or later. If the
+ definitions follow version 6 or earlier, hardcoded defaults are used.
+
+ '''
+ # Read default build systems and split rules from DEFAULTS file.
+ defaults_text = self._get_file_contents_from_definitions(
+ definitions_checkout_dir, 'DEFAULTS')
+
+ if definitions_version < 7:
+ if defaults_text is not None:
+ warnings.warn(
+ "Ignoring DEFAULTS file, because these definitions are "
+ "version %i" % definitions_version)
+ defaults_text = None
+ else:
+ if defaults_text is None:
+ warnings.warn("No DEFAULTS file found.")
+
+ defaults = morphlib.defaults.Defaults(definitions_version,
+ text=defaults_text)
+
+ return defaults.build_systems(), defaults.split_rules()
+
def _get_morphology(self, resolved_morphologies, definitions_checkout_dir,
morph_loader, filename):
'''Read the morphology at the specified location.
@@ -287,7 +317,8 @@ class SourceResolver(object):
definitions_tree,
morph_loader,
system_filenames,
- visit):
+ visit,
+ predefined_split_rules):
definitions_queue = collections.deque(system_filenames)
chunk_queue = set()
@@ -304,7 +335,8 @@ class SourceResolver(object):
raise MorphologyNotFoundError(filename)
visit(definitions_repo, definitions_ref, filename,
- definitions_absref, definitions_tree, morphology)
+ definitions_absref, definitions_tree, morphology,
+ predefined_split_rules)
if morphology['kind'] == 'cluster':
raise cliapp.AppException(
@@ -349,8 +381,8 @@ class SourceResolver(object):
def process_chunk(self, resolved_morphologies, resolved_trees,
definitions_checkout_dir, morph_loader, chunk_repo,
- chunk_ref, filename, chunk_buildsystem,
- visit):
+ chunk_ref, filename, chunk_buildsystem, visit,
+ predefined_split_rules):
absref, tree = self._resolve_ref(resolved_trees, chunk_repo, chunk_ref)
if chunk_buildsystem is None:
@@ -370,7 +402,8 @@ class SourceResolver(object):
morphology = self._create_morphology_for_build_system(
morph_loader, buildsystem, filename)
- visit(chunk_repo, chunk_ref, filename, absref, tree, morphology)
+ visit(chunk_repo, chunk_ref, filename, absref, tree, morphology,
+ predefined_split_rules)
def traverse_morphs(self, definitions_repo, definitions_ref,
system_filenames,
@@ -400,7 +433,13 @@ class SourceResolver(object):
definitions_version = self._check_version_file(
definitions_checkout_dir)
- morph_loader = morphlib.morphloader.MorphologyLoader()
+
+ predefined_build_systems, predefined_split_rules = \
+ self._get_defaults(
+ definitions_checkout_dir, definitions_version)
+
+ morph_loader = morphlib.morphloader.MorphologyLoader(
+ predefined_build_systems=predefined_build_systems)
# First, process the system and its stratum morphologies. These
# will all live in the same Git repository, and will point to
@@ -408,14 +447,15 @@ class SourceResolver(object):
chunk_queue = self._process_definitions_with_children(
resolved_morphologies, definitions_checkout_dir,
definitions_repo, definitions_ref, definitions_absref,
- definitions_tree, morph_loader,
- system_filenames, visit)
+ definitions_tree, morph_loader, system_filenames, visit,
+ predefined_split_rules)
# Now process all the chunks involved in the build.
for repo, ref, filename, buildsystem in chunk_queue:
self.process_chunk(resolved_morphologies, resolved_trees,
definitions_checkout_dir, morph_loader,
- repo, ref, filename, buildsystem, visit)
+ repo, ref, filename, buildsystem, visit,
+ predefined_split_rules)
def create_source_pool(lrc, rrc, repo, ref, filenames, cachedir,
@@ -437,10 +477,11 @@ def create_source_pool(lrc, rrc, repo, ref, filenames, cachedir,
'''
pool = morphlib.sourcepool.SourcePool()
- def add_to_pool(reponame, ref, filename, absref, tree, morphology):
- sources = morphlib.source.make_sources(reponame, ref,
- filename, absref,
- tree, morphology)
+ def add_to_pool(reponame, ref, filename, absref, tree, morphology,
+ predefined_split_rules):
+ sources = morphlib.source.make_sources(
+ reponame, ref, filename, absref, tree, morphology,
+ predefined_split_rules)
for source in sources:
pool.add(source)
diff --git a/morphlib/util.py b/morphlib/util.py
index 284fe305..e34799df 100644
--- a/morphlib/util.py
+++ b/morphlib/util.py
@@ -734,6 +734,7 @@ def temp_dir(*args, **kwargs): #pragma: no cover
if cleanup_on_success:
shutil.rmtree(td, ignore_errors=True)
+
def copyfileobj(fsrc, fdst, length=16*1024,
callback=lambda x: None): #pragma: no cover
''' This is similar to shutil.copyfileobj
@@ -775,3 +776,9 @@ class ProgressBar(object):
progress, self._expected_size, self._unit)
sys.stderr.write(s)
sys.stderr.flush()
+
+
+def schemas_directory(): # pragma: no cover
+ '''Returns a path to the schemas/ subdirectory of the 'morphlib' module.'''
+ code_dir = os.path.dirname(morphlib.__file__)
+ return os.path.join(code_dir, 'schemas')
diff --git a/tests.branching/branch-creates-new-system-branch-not-from-master.stdout b/tests.branching/branch-creates-new-system-branch-not-from-master.stdout
index 78b3ca33..c5682475 100644
--- a/tests.branching/branch-creates-new-system-branch-not-from-master.stdout
+++ b/tests.branching/branch-creates-new-system-branch-not-from-master.stdout
@@ -7,6 +7,7 @@ d ./newbranch/test
d ./newbranch/test/morphs
d ./newbranch/test/morphs/.git
f ./newbranch/.morph-system-branch/config
+f ./newbranch/test/morphs/DEFAULTS
f ./newbranch/test/morphs/VERSION
f ./newbranch/test/morphs/hello-stratum.morph
f ./newbranch/test/morphs/hello-system.morph
diff --git a/tests.branching/branch-works-anywhere.stdout b/tests.branching/branch-works-anywhere.stdout
index 4e6c3e27..e3c7bfb2 100644
--- a/tests.branching/branch-works-anywhere.stdout
+++ b/tests.branching/branch-works-anywhere.stdout
@@ -7,6 +7,7 @@ d ./branch1/test
d ./branch1/test/morphs
d ./branch1/test/morphs/.git
f ./branch1/.morph-system-branch/config
+f ./branch1/test/morphs/DEFAULTS
f ./branch1/test/morphs/VERSION
f ./branch1/test/morphs/hello-stratum.morph
f ./branch1/test/morphs/hello-system.morph
@@ -24,10 +25,12 @@ d ./branch2/test
d ./branch2/test/morphs
d ./branch2/test/morphs/.git
f ./branch1/.morph-system-branch/config
+f ./branch1/test/morphs/DEFAULTS
f ./branch1/test/morphs/VERSION
f ./branch1/test/morphs/hello-stratum.morph
f ./branch1/test/morphs/hello-system.morph
f ./branch2/.morph-system-branch/config
+f ./branch2/test/morphs/DEFAULTS
f ./branch2/test/morphs/VERSION
f ./branch2/test/morphs/hello-stratum.morph
f ./branch2/test/morphs/hello-system.morph
@@ -50,14 +53,17 @@ d ./branch3/test
d ./branch3/test/morphs
d ./branch3/test/morphs/.git
f ./branch1/.morph-system-branch/config
+f ./branch1/test/morphs/DEFAULTS
f ./branch1/test/morphs/VERSION
f ./branch1/test/morphs/hello-stratum.morph
f ./branch1/test/morphs/hello-system.morph
f ./branch2/.morph-system-branch/config
+f ./branch2/test/morphs/DEFAULTS
f ./branch2/test/morphs/VERSION
f ./branch2/test/morphs/hello-stratum.morph
f ./branch2/test/morphs/hello-system.morph
f ./branch3/.morph-system-branch/config
+f ./branch3/test/morphs/DEFAULTS
f ./branch3/test/morphs/VERSION
f ./branch3/test/morphs/hello-stratum.morph
f ./branch3/test/morphs/hello-system.morph
@@ -85,18 +91,22 @@ d ./branch4/test
d ./branch4/test/morphs
d ./branch4/test/morphs/.git
f ./branch1/.morph-system-branch/config
+f ./branch1/test/morphs/DEFAULTS
f ./branch1/test/morphs/VERSION
f ./branch1/test/morphs/hello-stratum.morph
f ./branch1/test/morphs/hello-system.morph
f ./branch2/.morph-system-branch/config
+f ./branch2/test/morphs/DEFAULTS
f ./branch2/test/morphs/VERSION
f ./branch2/test/morphs/hello-stratum.morph
f ./branch2/test/morphs/hello-system.morph
f ./branch3/.morph-system-branch/config
+f ./branch3/test/morphs/DEFAULTS
f ./branch3/test/morphs/VERSION
f ./branch3/test/morphs/hello-stratum.morph
f ./branch3/test/morphs/hello-system.morph
f ./branch4/.morph-system-branch/config
+f ./branch4/test/morphs/DEFAULTS
f ./branch4/test/morphs/VERSION
f ./branch4/test/morphs/hello-stratum.morph
f ./branch4/test/morphs/hello-system.morph
diff --git a/tests.branching/checkout-works-anywhere.stdout b/tests.branching/checkout-works-anywhere.stdout
index d7f6903a..737a605e 100644
--- a/tests.branching/checkout-works-anywhere.stdout
+++ b/tests.branching/checkout-works-anywhere.stdout
@@ -7,6 +7,7 @@ d ./master/test
d ./master/test/morphs
d ./master/test/morphs/.git
f ./master/.morph-system-branch/config
+f ./master/test/morphs/DEFAULTS
f ./master/test/morphs/VERSION
f ./master/test/morphs/hello-stratum.morph
f ./master/test/morphs/hello-system.morph
@@ -24,10 +25,12 @@ d ./newbranch/test
d ./newbranch/test/morphs
d ./newbranch/test/morphs/.git
f ./master/.morph-system-branch/config
+f ./master/test/morphs/DEFAULTS
f ./master/test/morphs/VERSION
f ./master/test/morphs/hello-stratum.morph
f ./master/test/morphs/hello-system.morph
f ./newbranch/.morph-system-branch/config
+f ./newbranch/test/morphs/DEFAULTS
f ./newbranch/test/morphs/VERSION
f ./newbranch/test/morphs/hello-stratum.morph
f ./newbranch/test/morphs/hello-system.morph
diff --git a/tests.branching/setup b/tests.branching/setup
index 816d5830..c10b4fee 100755
--- a/tests.branching/setup
+++ b/tests.branching/setup
@@ -47,7 +47,10 @@ mkdir "$DATADIR/morphs"
## Create a link to this repo that has a .git suffix
ln -s "$DATADIR/morphs" "$DATADIR/morphs.git"
-echo 'version: 6' > "$DATADIR/morphs/VERSION"
+echo 'version: 7' > "$DATADIR/morphs/VERSION"
+
+# We don't build anything, so empty DEFAULTS file is fine.
+echo '' > "$DATADIR/morphs/DEFAULTS"
cat <<EOF > "$DATADIR/morphs/hello-system.morph"
name: hello-system
diff --git a/tests.build/ambiguous-refs.script b/tests.build/ambiguous-refs.script
index 181af3cc..84fca86e 100755
--- a/tests.build/ambiguous-refs.script
+++ b/tests.build/ambiguous-refs.script
@@ -19,6 +19,9 @@
## instead of 'git rev-parse --verify': show-ref returns a list of partial
## matches sorted alphabetically, so any code using it may resolve refs
+## The 'autotools' build system is defined in the DEFAULTS file created
+## by the 'setup' script.
+
set -eu
# Create a ref that will show up in 'git show-ref' before the real master ref
diff --git a/tests.build/build-stratum-with-submodules.script b/tests.build/build-stratum-with-submodules.script
index 2dd2b924..b3073af1 100755
--- a/tests.build/build-stratum-with-submodules.script
+++ b/tests.build/build-stratum-with-submodules.script
@@ -24,6 +24,7 @@ set -eu
parent="$DATADIR/parent-repo"
mkdir "$parent"
+
echo "No real content here" > "$parent/dummy"
"$SRCDIR/scripts/run-git-in" "$parent" init --quiet
@@ -49,6 +50,7 @@ name: hello-stratum
kind: stratum
chunks:
- name: parent
+ morph: parent.morph
repo: test:parent-repo
ref: master
morph: parent.morph
diff --git a/tests.build/prefix.script b/tests.build/prefix.script
index 291b25cf..0ff077b8 100755
--- a/tests.build/prefix.script
+++ b/tests.build/prefix.script
@@ -49,6 +49,7 @@ name: hello-stratum
kind: stratum
chunks:
- name: xyzzy
+ morph: xyzzy.morph
repo: test:chunk-repo
ref: master
morph: xyzzy.morph
@@ -56,6 +57,7 @@ chunks:
build-mode: test
prefix: /plover
- name: plugh
+ morph: plugh.morph
repo: test:chunk-repo
ref: master
morph: plugh.morph
@@ -70,6 +72,6 @@ git commit -q -m "Update build definitions"
test:morphs-repo master hello-system
cd "$DATADIR/cache/artifacts"
-first_chunk=$(ls -1 *.chunk.xyzzy-* | head -n1 | cut -c -64)
-second_chunk=$(ls -1 *.chunk.plugh-* | head -n1 | cut -c -64)
+first_chunk=$(ls -1 *.chunk.xyzzy* | head -n1 | cut -c -64)
+second_chunk=$(ls -1 *.chunk.plugh* | head -n1 | cut -c -64)
cat $first_chunk.build-log $second_chunk.build-log
diff --git a/tests.build/rebuild-cached-stratum.script b/tests.build/rebuild-cached-stratum.script
index e2e0face..bc1b83fd 100755
--- a/tests.build/rebuild-cached-stratum.script
+++ b/tests.build/rebuild-cached-stratum.script
@@ -41,7 +41,7 @@ cache="$DATADIR/cache/artifacts"
"$SRCDIR/scripts/test-morph" build-morphology \
test:morphs-repo rebuild-cached-stratum hello-system
echo "first build:"
-(cd "$cache" && ls *.chunk.* *hello-stratum-* | sed 's/^[^.]*\./ /' |
+(cd "$cache" && ls *.chunk.* *hello-stratum* | sed 's/^[^.]*\./ /' |
LC_ALL=C sort -u)
# Change the chunk.
@@ -53,6 +53,6 @@ echo "first build:"
"$SRCDIR/scripts/test-morph" build-morphology \
test:morphs-repo rebuild-cached-stratum hello-system
echo "second build:"
-(cd "$cache" && ls *.chunk.* *hello-stratum-* | sed 's/^[^.]*\./ /' |
+(cd "$cache" && ls *.chunk.* *hello-stratum* | sed 's/^[^.]*\./ /' |
LC_ALL=C sort -u)
diff --git a/tests.build/rebuild-cached-stratum.stdout b/tests.build/rebuild-cached-stratum.stdout
index 9c53ee60..4672e7f2 100644
--- a/tests.build/rebuild-cached-stratum.stdout
+++ b/tests.build/rebuild-cached-stratum.stdout
@@ -1,22 +1,8 @@
first build:
- chunk.hello-bins
- chunk.hello-devel
- chunk.hello-doc
- chunk.hello-libs
- chunk.hello-locale
- chunk.hello-misc
- stratum.hello-stratum-devel
- stratum.hello-stratum-devel.meta
- stratum.hello-stratum-runtime
- stratum.hello-stratum-runtime.meta
+ chunk.hello-chunk
+ stratum.hello-stratum
+ stratum.hello-stratum.meta
second build:
- chunk.hello-bins
- chunk.hello-devel
- chunk.hello-doc
- chunk.hello-libs
- chunk.hello-locale
- chunk.hello-misc
- stratum.hello-stratum-devel
- stratum.hello-stratum-devel.meta
- stratum.hello-stratum-runtime
- stratum.hello-stratum-runtime.meta
+ chunk.hello-chunk
+ stratum.hello-stratum
+ stratum.hello-stratum.meta
diff --git a/tests.build/setup b/tests.build/setup
index 7936584b..ca60d426 100755
--- a/tests.build/setup
+++ b/tests.build/setup
@@ -55,8 +55,7 @@ int main(void)
EOF
git add hello.c
-git commit --quiet -m "add a hello world program and morph"
-
+git commit --quiet -m "add a hello world program"
git checkout --quiet master
@@ -68,11 +67,28 @@ mkdir "$morphsrepo"
cd "$morphsrepo"
git init --quiet
-echo 'version: 6' > VERSION
+echo 'version: 7' > VERSION
git add VERSION
+cat <<'EOF' > DEFAULTS
+# This is a deliberately minimal DEFAULTS file.
+
+# There are no splitting rules defined, because it's important that Morph
+# still works correctly when the user didn't define any.
+
+build-systems:
+ autotools:
+ configure-commands:
+ - ./configure
+ build-commands:
+ - make
+ install-commands:
+ - make DESTDIR="$DESTDIR" install
+EOF
+git add DEFAULTS
+
cat <<EOF > hello.morph
-name: hello
+name: hello-chunk
kind: chunk
build-commands:
- gcc -o hello hello.c
@@ -88,6 +104,7 @@ name: hello-stratum
kind: stratum
chunks:
- name: hello
+ morph: hello.morph
repo: test:chunk-repo
ref: farrokh
morph: hello.morph
diff --git a/tests.build/setup-build-essential b/tests.build/setup-build-essential
index 8868db68..28847d76 100755
--- a/tests.build/setup-build-essential
+++ b/tests.build/setup-build-essential
@@ -17,6 +17,7 @@
# Set up a stratum which resembles Baserock's 'build-essential' slightly. Used
# for testing 'morph cross-bootstrap' and the 'bootstrap' build mode.
+# Add a mock compiler chunk.
mkdir -p "$DATADIR/cc-repo"
cd "$DATADIR/cc-repo"
@@ -32,9 +33,6 @@ git commit -q -m "Create compiler chunk"
cd "$DATADIR/morphs-repo"
-# Require 'cc' in hello-chunk. We should have the second version available
-# but *not* the first one.
-
cat <<EOF > "stage1-cc.morph"
name: stage1-cc
kind: chunk
@@ -53,6 +51,11 @@ install-commands:
- install -m 755 morph-test-cc "\$DESTDIR\$PREFIX/bin/morph-test-cc"
EOF
+git add cc.morph stage1-cc.morph
+git commit -q -m "Add build instructions for mock compiler."
+
+# Require 'cc' in hello-chunk. We should have the second version available
+# but *not* the first one.
cat <<EOF > "hello.morph"
name: hello
kind: chunk
@@ -76,6 +79,7 @@ name: build-essential
kind: stratum
chunks:
- name: stage1-cc
+ morph: stage1-cc.morph
repo: test:cc-repo
ref: master
morph: stage1-cc.morph
@@ -83,6 +87,7 @@ chunks:
build-mode: bootstrap
prefix: /tools
- name: cc
+ morph: cc.morph
repo: test:cc-repo
ref: master
morph: cc.morph
@@ -98,6 +103,7 @@ build-depends:
- morph: build-essential
chunks:
- name: hello
+ morph: hello.morph
repo: test:chunk-repo
ref: farrokh
morph: hello.morph
diff --git a/without-test-modules b/without-test-modules
index f42f0f94..77803473 100644
--- a/without-test-modules
+++ b/without-test-modules
@@ -62,3 +62,4 @@ distbuild/worker_build_scheduler.py
morphlib/buildbranch.py
morphlib/definitions_repo.py
morphlib/sourceresolver.py
+morphlib/defaults.py
diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn
index ff1971f8..9f23107b 100644
--- a/yarns/implementations.yarn
+++ b/yarns/implementations.yarn
@@ -250,7 +250,37 @@ another to hold a chunk.
mkdir "$DATADIR/gits/morphs"
cd "$DATADIR/gits/morphs"
git init .
- echo 'version: 6' > VERSION
+ echo 'version: 7' > VERSION
+
+ install -m644 -D /dev/stdin << EOF "DEFAULTS"
+ # This is a simplified version of the DEFAULTS file supplied with the
+ # Baserock reference system definitions.
+ build-systems:
+ autotools:
+ configure-commands:
+ - ./configure
+ build-commands:
+ - make
+ install-commands:
+ - make install
+ split-rules:
+ chunk:
+ - artifact: -devel
+ include:
+ - (usr/)?include/.*
+ - (usr/)?lib/.*\.a
+ - (usr/)?share/man/.*
+ - artifact: -runtime
+ include:
+ - .*
+ stratum:
+ - artifact: -devel
+ include:
+ - .*-devel
+ - artifact: -runtime
+ include:
+ - .*-runtime
+ EOF
arch=$(run_morph print-architecture)
install -m644 -D /dev/stdin << EOF "systems/test-system.morph"
@@ -333,7 +363,7 @@ another to hold a chunk.
install-commands:
- copy files
system-integration:
- test-chunk-bins:
+ test-chunk-runtime:
00-passwd:
- |
create file /etc/passwd
diff --git a/yarns/regression.yarn b/yarns/regression.yarn
index c424f437..aa98eec5 100644
--- a/yarns/regression.yarn
+++ b/yarns/regression.yarn
@@ -59,17 +59,17 @@ source it depended on.
GIVEN a workspace
AND a git server
AND system systems/test-system.morph uses core-runtime from core
- AND stratum strata/core.morph has match rules: [{artifact: core-runtime, include: [.*-(bins|libs|locale)]}, {artifact: core-devel, include: [.*-(devel|doc|misc)]}]
+ AND stratum strata/core.morph has match rules: [{artifact: core-runtime, include: [.*-devel]}, {artifact: core-devel, include: [.*-runtime]}]
WHEN the user checks out the system branch called master
GIVEN a cluster called test-cluster.morph in system branch master
AND a system in cluster test-cluster.morph in branch master called test-system
AND system test-system in cluster test-cluster.morph in branch master builds systems/test-system.morph
AND system test-system in cluster test-cluster.morph in branch master has deployment type: tar
WHEN the user builds the system systems/test-system.morph in branch master
- GIVEN stratum strata/core.morph in system branch master has match rules: [{artifact: core-runtime, include: [.*-(bins|libs|misc)]}, {artifact: core-devel, include: [.*-(devel|doc|locale)]}]
+ GIVEN stratum strata/core.morph in system branch master has match rules: [{artifact: core-runtime, include: [.*-runtime]}, {artifact: core-devel, include: [.*-devel]}]
WHEN the user builds the system systems/test-system.morph in branch master
AND the user deploys the cluster test-cluster.morph in branch master with options test-system.location="$DATADIR/test.tar"
- THEN tarball test.tar contains baserock/test-chunk-misc.meta
+ THEN tarball test.tar contains baserock/test-chunk-runtime.meta
FINALLY the git server is shut down