10 files changed, 801 insertions, 0 deletions
diff --git a/migrations/000-version-info.py b/migrations/000-version-info.py
new file mode 100755
index 00000000..2bff51f4
--- /dev/null
+++ b/migrations/000-version-info.py
@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+# Copyright (C) 2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+'''Migration to Baserock Definitions format version 0.
+
+The format of version 0 is not formally specified, except by the Morph
+codebase. It marks the starting point of the work to formalise the Baserock
+Definitions format.
+
+'''
+
+
+import os
+import sys
+
+import migrations
+
+
+TO_VERSION = 0
+
+
+try:
+    if os.path.exists('./VERSION'):
+        # This will raise an exception if the VERSION file is invalid, which
+        # might be useful.
+        migrations.check_definitions_version(TO_VERSION)
+
+        sys.stdout.write("Nothing to do.\n")
+        sys.exit(0)
+    else:
+        sys.stdout.write("No VERSION file found, creating one.\n")
+        migrations.set_definitions_version(TO_VERSION)
+        sys.exit(0)
+except RuntimeError as e:
+    sys.stderr.write("Error: %s\n" % e.message)
+    sys.exit(1)
diff --git a/migrations/001-empty-build-depends.py b/migrations/001-empty-build-depends.py
new file mode 100755
index 00000000..5d4296d6
--- /dev/null
+++ b/migrations/001-empty-build-depends.py
@@ -0,0 +1,82 @@
+#!/usr/bin/env python
+# Copyright (C) 2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+'''Migration to Baserock Definitions format version 1.
+
+In version 1, the 'build-depends' parameter was made optional. It was
+previously mandatory to specify 'build-depends' for a chunk, even if it was an
+empty list.
+
+'''
+
+
+import sys
+import warnings
+
+import migrations
+
+
+TO_VERSION = 1
+
+
+def check_empty_build_depends(contents, filename):
+    assert contents['kind'] == 'stratum'
+
+    valid = True
+    for chunk_ref in contents.get('chunks', []):
+        if 'build-depends' not in chunk_ref:
+            chunk_ref_name = chunk_ref.get('name', chunk_ref.get('morph'))
+            warnings.warn(
+                "%s:%s has no build-depends field, which "
+                "is invalid in definitions version 0." %
+                (contents['name'], chunk_ref_name))
+            valid = False
+
+    return valid
+
+
+def remove_empty_build_depends(contents, filename):
+    assert contents['kind'] == 'stratum'
+
+    changed = False
+    for chunk_ref in contents.get('chunks', []):
+        if 'build-depends' in chunk_ref:
+            if len(chunk_ref['build-depends']) == 0:
+                del chunk_ref['build-depends']
+                changed = True
+
+    return changed
+
+
+try:
+    if migrations.check_definitions_version(TO_VERSION - 1):
+        success = migrations.process_definitions(
+            path='.', kinds=['stratum'],
+            validate_cb=check_empty_build_depends,
+            modify_cb=remove_empty_build_depends)
+        if success:
+            migrations.set_definitions_version(TO_VERSION)
+            sys.stdout.write("Migration completed successfully.\n")
+            sys.exit(0)
+        else:
+            sys.stderr.write("Migration failed due to warnings.\n")
+            sys.exit(1)
+    else:
+        sys.stdout.write("Nothing to do.\n")
+        sys.exit(0)
+except RuntimeError as e:
+    sys.stderr.write("Error: %s\n" % e.message)
+    sys.exit(1)
diff --git a/migrations/002-missing-chunk-morphs.py b/migrations/002-missing-chunk-morphs.py
new file mode 100755
index 00000000..2c93804e
--- /dev/null
+++ b/migrations/002-missing-chunk-morphs.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+# Copyright (C) 2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+'''Migration to Baserock Definitions format version 2.
+
+In version 2, the processing of the 'morph:' field within stratum .morph files
+became more strict.  This migration checks whether definitions are valid
+according to version 2 of the format.
+
+'''
+
+
+import os
+import sys
+import warnings
+
+import migrations
+
+
+TO_VERSION = 2
+
+
+def check_missing_chunk_morphs(contents, filename):
+    assert contents['kind'] == 'stratum'
+
+    valid = True
+
+    for chunk_ref in contents.get('chunks', []):
+        if 'morph' in chunk_ref:
+            chunk_path = os.path.join('.', chunk_ref['morph'])
+            if not os.path.exists(chunk_path):
+                # There's no way we can really fix this, so
+                # just warn and say the migration failed.
+                warnings.warn(
+                    "%s points to non-existant file %s" %
+                    (contents['name'], chunk_ref['morph']))
+                valid = False
+
+    return valid
+
+
+try:
+    if migrations.check_definitions_version(TO_VERSION - 1):
+        safe_to_migrate = migrations.process_definitions(
+            kinds=['stratum'], validate_cb=check_missing_chunk_morphs)
+
+        if not safe_to_migrate:
+            sys.stderr.write(
+                "Migration failed due to one or more warnings.\n")
+            sys.exit(1)
+        else:
+            migrations.set_definitions_version(TO_VERSION)
+            sys.stdout.write("Migration completed successfully.\n")
+            sys.exit(0)
+    else:
+        sys.stdout.write("Nothing to do.\n")
+        sys.exit(0)
+except RuntimeError as e:
+    sys.stderr.write("Error: %s\n" % e.message)
+    sys.exit(1)
diff --git a/migrations/003-arch-armv5.py b/migrations/003-arch-armv5.py
new file mode 100755
index 00000000..58eb79de
--- /dev/null
+++ b/migrations/003-arch-armv5.py
@@ -0,0 +1,89 @@
+#!/usr/bin/env python
+# Copyright (C) 2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+'''Migration to Baserock Definitions format version 3.
+
+In version 3, there were two additions:
+
+    - the 'armv5' architecture
+    - the install-essential-files.configure configuration extension
+
+This migration checks that neither of these are in use in the input (version 2)
+definitions. Which isn't particularly useful.
+
+'''
+
+
+import sys
+import warnings
+
+import migrations
+
+
+TO_VERSION = 3
+
+
+def check_arch(contents, filename):
+    assert contents['kind'] == 'system'
+
+    valid = True
+
+    if contents['arch'] == 'armv5':
+        warnings.warn(
+            "%s uses armv5 architecture that is not understood until version "
+            "3." % filename)
+        valid = False
+
+    return valid
+
+
+def check_configuration_extensions(contents, filename):
+    assert contents['kind'] == 'system'
+
+    valid = True
+
+    for extension in contents.get('configuration-extensions', []):
+        if extension == 'install-essential-files':
+            warnings.warn(
+                "%s uses install-essential-files.configure extension, which "
+                "was not present in morph.git until commit 423dc974a61f1c0 "
+                "(tag baserock-definitions-v3)." % filename)
+            valid = False
+
+    return valid
+
+
+try:
+    if migrations.check_definitions_version(TO_VERSION - 1):
+        safe_to_migrate = migrations.process_definitions(
+            kinds=['system'], validate_cb=check_arch)
+        safe_to_migrate = migrations.process_definitions(
+            kinds=['system'], validate_cb=check_configuration_extensions)
+
+        if not safe_to_migrate:
+            sys.stderr.write(
+                "Migration failed due to one or more warnings.\n")
+            sys.exit(1)
+        else:
+            migrations.set_definitions_version(TO_VERSION)
+            sys.stdout.write("Migration completed successfully.\n")
+            sys.exit(0)
+    else:
+        sys.stdout.write("Nothing to do.\n")
+        sys.exit(0)
+except RuntimeError as e:
+    sys.stderr.write("Error: %s\n" % e.message)
+    sys.exit(1)
diff --git a/migrations/004-install-files-overwrite-symlink.py b/migrations/004-install-files-overwrite-symlink.py
new file mode 100755
index 00000000..6853dcb5
--- /dev/null
+++ b/migrations/004-install-files-overwrite-symlink.py
@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+# Copyright (C) 2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+'''Migration to Baserock Definitions format version 4.
+
+This change to the format was made to work around a bug in a deployment
+extension present in morph.git.
+
+Automated migration is not really possible for this change, and unless you
+are experiencing the install-files.configure extension crashing, you can ignore
+it completely.
+
+We have now moved all .configure and .write extensions into the definitions.git
+repository. Changes like this no longer require a marking a new version of the
+Baserock definitions format in order to prevent build tools crashing.
+
+Morph commit c373f5a403b0ec introduces version 4 of the definitions format. In
+older versions of Morph the install-files.configure extension would crash if it
+tried to overwrite a symlink. This bug is fixed in the version of Morph that
+can build definitions version 4.
+
+If you need to overwrite a symlink at deploytime using install-files.configure,
+please use VERSION to 4 or above in your definitions.git repo so older versions
+of Morph gracefully refuse to deploy, instead of crashing.
+
+'''
+
+
+import sys
+
+import migrations
+
+
+TO_VERSION = 4
+
+
+try:
+    if migrations.check_definitions_version(TO_VERSION - 1):
+        migrations.set_definitions_version(TO_VERSION)
+        sys.stdout.write("Migration completed successfully.\n")
+        sys.exit(0)
+    else:
+        sys.stdout.write("Nothing to do.\n")
+        sys.exit(0)
+except RuntimeError as e:
+    sys.stderr.write("Error: %s\n" % e.message)
+    sys.exit(1)
diff --git a/migrations/005-strip-commands.py b/migrations/005-strip-commands.py
new file mode 100755
index 00000000..da3de940
--- /dev/null
+++ b/migrations/005-strip-commands.py
@@ -0,0 +1,78 @@
+#!/usr/bin/env python
+# Copyright (C) 2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+'''Migration to Baserock Definitions format version 5.
+
+Version 5 of the definitions format adds a 'strip-commands' field that can
+be set in chunk definitions.
+
+Version 5 also allows deployment extensions to live in definitions.git instead
+of morph.git. This greatly reduces the interface surface of the Baserock
+definitions format specification, because we no longer have to mark a new
+version of the definitions format each time an extension in morph.git is added,
+removed, or changes its API in any way.
+
+In commit 6f4929946 of git://git.baserock.org/baserock/baserock/definitions.git
+the deployment extensions were moved into an extensions/ subdirectory, and the
+system and cluster .morph files that referred to them were all updated to
+prepend 'extension/' to the filenames. This migration doesn't (re)do that
+change.
+
+'''
+
+
+import sys
+import warnings
+
+import migrations
+
+
+TO_VERSION = 5
+
+
+def check_strip_commands(contents, filename):
+    assert contents['kind'] == 'chunk'
+
+    valid = True
+
+    if 'strip-commands' in contents:
+        warnings.warn(
+            "%s has strip-commands, which are not valid until version 5" %
+            filename)
+        valid = False
+
+    return valid
+
+
+try:
+    if migrations.check_definitions_version(TO_VERSION - 1):
+        safe_to_migrate = migrations.process_definitions(
+            kinds=['chunk'], validate_cb=check_strip_commands)
+
+        if not safe_to_migrate:
+            sys.stderr.write(
+                "Migration failed due to one or more warnings.\n")
+            sys.exit(1)
+        else:
+            migrations.set_definitions_version(TO_VERSION)
+            sys.stdout.write("Migration completed successfully.\n")
+            sys.exit(0)
+    else:
+        sys.stdout.write("Nothing to do.\n")
+        sys.exit(0)
+except RuntimeError as e:
+    sys.stderr.write("Error: %s\n" % e.message)
+    sys.exit(1)
diff --git a/migrations/GUIDELINES b/migrations/GUIDELINES
new file mode 100644
index 00000000..3694e2c9
--- /dev/null
+++ b/migrations/GUIDELINES
@@ -0,0 +1,35 @@
+Guidelines for writing migrations
+---------------------------------
+
+All changes to the definitions format must have a migration, but it is valid
+for the migration to do nothing except update the version number (see
+004-install-files-overwrite-symlink.py for an example of that).
+
+This small set of rules exists to ensure that the migrations are consistent and
+easy to understand. If you are writing a migration and these rules don't make
+any sense, we should probably change them. Please sign up to the
+baserock-dev@baserock.org mailing list to suggest the change.
+
+- Write migrations in Python. They must be valid Python 3. For now, since
+  only Python 2 is available in Baserock 'build' and 'devel' reference systems
+  up to the 15.25 release of Baserock, they must also be valid Python 2.
+
+- Don't use any external libraries.
+
+- Follow the existing file naming pattern, and the existing code convention.
+
+- Keep the migration code as simple as possible.
+
+- Avoid crashing on malformed input data, where practical. For example, use
+  contents.get('field') instead of contents['field'] to avoid crashing when
+  'field' is not present. The idea of this is to avoid a "cascade of errors"
+  problem when running the migrations on bad inputs. It is confusing when
+  migrations break on problems that are unrelated to the actual area where
+  they operate, even if they are theoretically "within their rights" to do so.
+
+- Migrate the definitions in line with current best practices. For example,
+  migrations/001-empty-build-depends.py doesn't need to remove empty
+  build-depends fields: they are still valid in version 1 of the format. But
+  best practice is now to remove them. Users who don't agree with this practice
+  can choose to not run that migration, which can be done with `chmod -x
+  migrations/xxx.py`.
diff --git a/migrations/indent b/migrations/indent
new file mode 100755
index 00000000..8d6f034f
--- /dev/null
+++ b/migrations/indent
@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+# Copyright (C) 2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+'''Automatically reformat a set of Baserock definition files.
+
+This tool expects to be able to use ruamel.yaml to load and write YAML.
+It will totally ruin things if used with PyYAML.
+
+It makes sense to run this script on your definitions, and check through
+and commit the result, before running any of the automated migrations. This
+way, you can be sure that the migrations will only change things that they need 
+to in the .morph files.
+
+'''
+
+
+import migrations
+
+
+def force_rewrite(contents, filename):
+    return True
+
+migrations.process_definitions(path='.', modify_cb=force_rewrite)
diff --git a/migrations/migrations.py b/migrations/migrations.py
new file mode 100644
index 00000000..22ed1328
--- /dev/null
+++ b/migrations/migrations.py
@@ -0,0 +1,228 @@
+#!/usr/bin/env python
+# Copyright (C) 2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+'''Tools for migrating Baserock definitions from one format version to another.
+
+'''
+
+
+# ruamel.yaml is a fork of PyYAML which allows rewriting YAML files without
+# destroying all of the comments, ordering and formatting. The more
+# widely-used PyYAML library will produce output totally different to the
+# input file in most cases.
+#
+# See: <https://bitbucket.org/ruamel/yaml>
+import ruamel.yaml as yaml
+
+import logging
+import os
+import warnings
+
+
+# Uncomment this to cause all log messages to be written to stdout. By
+# default they are hidden, but if you are debugging something this might help!
+#
+# logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+
+
+def pretty_warnings(message, category, filename, lineno,
+                    file=None, line=None):
+    '''Format warning messages from warnings.warn().'''
+    return 'WARNING: %s\n' % (message)
+
+# Override the default warning formatter (which is ugly), and add a filter to
+# ensure duplicate warnings only get displayed once.
+warnings.simplefilter("once", append=True)
+warnings.formatwarning = pretty_warnings
+
+
+
+def parse_yaml_with_roundtrip_info(text):
+    return yaml.load(text, yaml.RoundTripLoader)
+
+def write_yaml_with_roundtrip_info(contents, stream, **kwargs):
+    yaml.dump(contents, stream, Dumper=yaml.RoundTripDumper, **kwargs)
+
+
+
+class VersionFileError(RuntimeError):
+    '''Represents errors in the version marker file (./VERSION).'''
+    pass
+
+
+class MigrationOutOfOrderError(RuntimeError):
+    '''Raised if a migration is run on too old a version of definitions.
+
+    It's not an error to run a migration on a version that is already migrated.
+
+    '''
+    pass
+
+
+def check_definitions_version(from_version, version_file='./VERSION',
+                              to_version=None):
+    '''Check if migration between 'from_version' and 'to_version' is needed.
+
+    Both 'from_version' and 'to_version' should be whole numbers. The
+    'to_version' defaults to from_version + 1.
+
+    This function reads the version marker file specified by 'version_file'.
+    Returns True if the version is between 'from_version' and 'to_version',
+    indicating that migration needs to be done. Returns False if the version is
+    already at or beyond 'to_version'. Raises MigrationOutOfOrderError if the
+    version is below 'from_version'.
+
+    If 'version_file' is missing or invalid, it raises VersionFileError. The
+    version file is expected to follow the following format:
+
+        version: 1
+
+    '''
+    to_version = to_version or (from_version + 1)
+    need_to_migrate = False
+
+    if os.path.exists(version_file):
+        logging.info("Found version information file: %s" % version_file)
+
+        with open(version_file) as f:
+            version_text = f.read()
+
+        if len(version_text) == 0:
+            raise VersionFileError(
+                "File %s exists but is empty." % version_file)
+
+        try:
+            version_info = yaml.safe_load(version_text)
+            current_version = version_info['version']
+
+            if current_version >= to_version:
+                logging.info(
+                    "Already at version %i." % current_version)
+            elif current_version < from_version:
+                raise MigrationOutOfOrderError(
+                    "This tool expects to migrate from version %i to version "
+                    "%i of the Baserock Definitions syntax. These definitions "
+                    "claim to be version %i." % (
+                        from_version, to_version, current_version))
+            else:
+                logging.info("Need to migrate from %i to %i.",
+                             current_version, to_version)
+                need_to_migrate = True
+        except (KeyError, TypeError, ValueError) as e:
+            logging.exception(e)
+            raise VersionFileError(
+                "Invalid version info: '%s'" % version_text)
+    else:
+        raise VersionFileError(
+            "No file %s was found. Please run the migration scripts in order,"
+            "starting from 000-version-info.py." % version_file)
+
+    return need_to_migrate
+
+
+def set_definitions_version(new_version, version_file='./VERSION'):
+    '''Update the version information stored in 'version_file'.
+
+    The new version must be a whole number. If 'version_file' doesn't exist,
+    it will be created.
+
+    '''
+    version_info = {'version': new_version}
+    with open(version_file, 'w') as f:
+        # If 'default_flow_style' is True (the default) then the output here
+        # will look like "{version: 0}" instead of "version: 0".
+        yaml.safe_dump(version_info, f, default_flow_style=False)
+
+
+def walk_definition_files(path='.', extensions=['.morph']):
+    '''Recursively yield all files under 'path' with the given extension(s).
+
+    This is safe to run in the top level of a Git repository, as anything under
+    '.git' will be ignored.
+
+    '''
+    for dirname, dirnames, filenames in os.walk('.'):
+        filenames.sort()
+        dirnames.sort()
+        if '.git' in dirnames:
+            dirnames.remove('.git')
+        for filename in filenames:
+            for extension in extensions:
+                if filename.endswith(extension):
+                    yield os.path.join(dirname, filename)
+
+
+ALL_KINDS = ['cluster', 'system', 'stratum', 'chunk']
+
+
+def process_definitions(path='.', kinds=ALL_KINDS, validate_cb=None,
+                        modify_cb=None):
+    '''Run callbacks for all Baserock definitions found in 'path'.
+
+    If 'validate_cb' is set, it will be called for each definition and can
+    return True or False to indicate whether that definition is valid according
+    a new version of the format. The process_definitions() function will return
+    True if all definitions were valid according to validate_cb(), and False
+    otherwise.
+
+    If 'modify_cb' is set, it will be called for each definition and can
+    modify the 'content' dict. It should return True if the dict was modified,
+    and in this case the definition file will be overwritten with the new
+    contents. The 'ruamel.yaml' library is used if it is available, which will
+    try to preserve comments, ordering and some formatting in the YAML
+    definition files.
+
+    If 'validate_cb' is set and returns False for a definition, 'modify_cb'
+    will not be called.
+
+    Both callbacks are passed two parameters: a dict containing the contents of
+    the definition file, and its filename. The filename is passed so you can
+    use it when reporting errors.
+
+    The 'kinds' setting can be used to ignore some definitions according to the
+    'kind' field.
+
+    '''
+    all_valid = True
+
+    for filename in walk_definition_files(path=path):
+        with open(filename) as f:
+            text = f.read()
+
+        if modify_cb is None:
+            contents = yaml.load(text)
+        else:
+            contents = parse_yaml_with_roundtrip_info(text)
+
+        if 'kind' in contents:
+            if contents['kind'] in kinds:
+                valid = True
+                changed = False
+
+                if validate_cb is not None:
+                    valid = validate_cb(contents, filename)
+                    all_valid &= valid
+
+                if valid and modify_cb is not None:
+                    changed = modify_cb(contents, filename)
+
+                if changed:
+                    with open(filename, 'w') as f:
+                        write_yaml_with_roundtrip_info(contents, f, width=80)
+        else:
+            warnings.warn("%s is invalid: no 'kind' field set." % filename)
+
+    return all_valid
diff --git a/migrations/run-all b/migrations/run-all
new file mode 100755
index 00000000..8df00074
--- /dev/null
+++ b/migrations/run-all
@@ -0,0 +1,72 @@
+#!/usr/bin/env python
+# Copyright (C) 2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+'''Run a set of migration scripts.
+
+This script does exactly what `PYTHONPATH=. run-parts --exit-on-error` would
+do. I avoided using 'run-parts' purely because the implementation in Fedora 22
+doesn't have an '--exit-on-error' option. The Busybox and Debian
+implementations do have that option.
+
+Please fix run-parts in https://git.fedorahosted.org/cgit/crontabs.git/tree/ 
+so we can simplify this script :-)
+
+'''
+
+
+import os
+import subprocess
+import sys
+
+
+if len(sys.argv) == 2:
+    migration_dir = sys.argv[1]
+elif len(sys.argv) == 1:
+    migration_dir = os.path.dirname(__file__)
+else:
+    sys.stderr.write("Usage: %s [MIGRATION_DIR]\n" % sys.argv[0])
+    sys.exit(1)
+
+
+def is_executable(fpath):
+    return os.path.isfile(fpath) and os.access(fpath, os.X_OK)
+
+env = os.environ
+if 'PYTHONPATH' in env:
+    env['PYTHONPATH'] = env['PYTHONPATH'] + ':' + migration_dir
+else:
+    env['PYTHONPATH'] = migration_dir
+
+try:
+    migrations_found = 0
+    for fname in sorted(os.listdir(migration_dir)):
+        migration_fpath = os.path.join(migration_dir, fname)
+        if is_executable(migration_fpath):
+            if not os.path.samefile(migration_fpath, __file__):
+                migrations_found += 1
+                sys.stdout.write(migration_fpath + ":\n")
+                subprocess.check_call(
+                    migration_fpath, env=env)
+
+    if migrations_found == 0:
+        sys.stderr.write("No migration files found in '%s'\n" % migration_dir)
+        sys.exit(1)
+    else:
+        sys.exit(0)
+
+except (subprocess.CalledProcessError, RuntimeError) as e:
+    sys.stderr.write(str(e) + '\n')
+    sys.exit(1)