From 08b60b1721f0e7c0d287980f35872edcc9d2c95c Mon Sep 17 00:00:00 2001 From: Paul Sherwood Date: Sun, 28 Feb 2016 18:40:33 +0000 Subject: Remove migrations - this content is now in baserock/spec.git Change-Id: Icf333b87e3ae5d6e9984d6e534d8b8c0ae15c7a4 --- migrations | 3 + migrations/000-version-info.py | 49 --- migrations/001-empty-build-depends.py | 82 ----- migrations/002-missing-chunk-morphs.py | 73 ----- migrations/003-arch-armv5.py | 89 ------ migrations/004-install-files-overwrite-symlink.py | 59 ---- migrations/005-strip-commands.py | 78 ----- migrations/006-specify-build-system.py | 354 ---------------------- migrations/007-defaults-in-definitions.py | 67 ---- migrations/007-initial-defaults | 199 ------------ migrations/008-submodules-in-strata.py | 220 -------------- migrations/GUIDELINES | 35 --- migrations/indent | 36 --- migrations/migrations.py | 228 -------------- migrations/run-all | 73 ----- 15 files changed, 3 insertions(+), 1642 deletions(-) create mode 100644 migrations delete mode 100755 migrations/000-version-info.py delete mode 100755 migrations/001-empty-build-depends.py delete mode 100755 migrations/002-missing-chunk-morphs.py delete mode 100755 migrations/003-arch-armv5.py delete mode 100755 migrations/004-install-files-overwrite-symlink.py delete mode 100755 migrations/005-strip-commands.py delete mode 100755 migrations/006-specify-build-system.py delete mode 100755 migrations/007-defaults-in-definitions.py delete mode 100644 migrations/007-initial-defaults delete mode 100755 migrations/008-submodules-in-strata.py delete mode 100644 migrations/GUIDELINES delete mode 100755 migrations/indent delete mode 100644 migrations/migrations.py delete mode 100755 migrations/run-all diff --git a/migrations b/migrations new file mode 100644 index 00000000..2b0104d2 --- /dev/null +++ b/migrations @@ -0,0 +1,3 @@ +The content of the definitions/migrations directory is now at + +http://git.baserock.org/cgit/baserock/baserock/spec.git/tree/migrations diff --git a/migrations/000-version-info.py b/migrations/000-version-info.py deleted file mode 100755 index 2bff51f4..00000000 --- a/migrations/000-version-info.py +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . - - -'''Migration to Baserock Definitions format version 0. - -The format of version 0 is not formally specified, except by the Morph -codebase. It marks the starting point of the work to formalise the Baserock -Definitions format. - -''' - - -import os -import sys - -import migrations - - -TO_VERSION = 0 - - -try: - if os.path.exists('./VERSION'): - # This will raise an exception if the VERSION file is invalid, which - # might be useful. - migrations.check_definitions_version(TO_VERSION) - - sys.stdout.write("Nothing to do.\n") - sys.exit(0) - else: - sys.stdout.write("No VERSION file found, creating one.\n") - migrations.set_definitions_version(TO_VERSION) - sys.exit(0) -except RuntimeError as e: - sys.stderr.write("Error: %s\n" % e.message) - sys.exit(1) diff --git a/migrations/001-empty-build-depends.py b/migrations/001-empty-build-depends.py deleted file mode 100755 index 5d4296d6..00000000 --- a/migrations/001-empty-build-depends.py +++ /dev/null @@ -1,82 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . - - -'''Migration to Baserock Definitions format version 1. - -In version 1, the 'build-depends' parameter was made optional. It was -previously mandatory to specify 'build-depends' for a chunk, even if it was an -empty list. - -''' - - -import sys -import warnings - -import migrations - - -TO_VERSION = 1 - - -def check_empty_build_depends(contents, filename): - assert contents['kind'] == 'stratum' - - valid = True - for chunk_ref in contents.get('chunks', []): - if 'build-depends' not in chunk_ref: - chunk_ref_name = chunk_ref.get('name', chunk_ref.get('morph')) - warnings.warn( - "%s:%s has no build-depends field, which " - "is invalid in definitions version 0." % - (contents['name'], chunk_ref_name)) - valid = False - - return valid - - -def remove_empty_build_depends(contents, filename): - assert contents['kind'] == 'stratum' - - changed = False - for chunk_ref in contents.get('chunks', []): - if 'build-depends' in chunk_ref: - if len(chunk_ref['build-depends']) == 0: - del chunk_ref['build-depends'] - changed = True - - return changed - - -try: - if migrations.check_definitions_version(TO_VERSION - 1): - success = migrations.process_definitions( - path='.', kinds=['stratum'], - validate_cb=check_empty_build_depends, - modify_cb=remove_empty_build_depends) - if success: - migrations.set_definitions_version(TO_VERSION) - sys.stdout.write("Migration completed successfully.\n") - sys.exit(0) - else: - sys.stderr.write("Migration failed due to warnings.\n") - sys.exit(1) - else: - sys.stdout.write("Nothing to do.\n") - sys.exit(0) -except RuntimeError as e: - sys.stderr.write("Error: %s\n" % e.message) - sys.exit(1) diff --git a/migrations/002-missing-chunk-morphs.py b/migrations/002-missing-chunk-morphs.py deleted file mode 100755 index 2c93804e..00000000 --- a/migrations/002-missing-chunk-morphs.py +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . - - -'''Migration to Baserock Definitions format version 2. - -In version 2, the processing of the 'morph:' field within stratum .morph files -became more strict. This migration checks whether definitions are valid -according to version 2 of the format. - -''' - - -import os -import sys -import warnings - -import migrations - - -TO_VERSION = 2 - - -def check_missing_chunk_morphs(contents, filename): - assert contents['kind'] == 'stratum' - - valid = True - - for chunk_ref in contents.get('chunks', []): - if 'morph' in chunk_ref: - chunk_path = os.path.join('.', chunk_ref['morph']) - if not os.path.exists(chunk_path): - # There's no way we can really fix this, so - # just warn and say the migration failed. - warnings.warn( - "%s points to non-existant file %s" % - (contents['name'], chunk_ref['morph'])) - valid = False - - return valid - - -try: - if migrations.check_definitions_version(TO_VERSION - 1): - safe_to_migrate = migrations.process_definitions( - kinds=['stratum'], validate_cb=check_missing_chunk_morphs) - - if not safe_to_migrate: - sys.stderr.write( - "Migration failed due to one or more warnings.\n") - sys.exit(1) - else: - migrations.set_definitions_version(TO_VERSION) - sys.stdout.write("Migration completed successfully.\n") - sys.exit(0) - else: - sys.stdout.write("Nothing to do.\n") - sys.exit(0) -except RuntimeError as e: - sys.stderr.write("Error: %s\n" % e.message) - sys.exit(1) diff --git a/migrations/003-arch-armv5.py b/migrations/003-arch-armv5.py deleted file mode 100755 index 58eb79de..00000000 --- a/migrations/003-arch-armv5.py +++ /dev/null @@ -1,89 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . - - -'''Migration to Baserock Definitions format version 3. - -In version 3, there were two additions: - - - the 'armv5' architecture - - the install-essential-files.configure configuration extension - -This migration checks that neither of these are in use in the input (version 2) -definitions. Which isn't particularly useful. - -''' - - -import sys -import warnings - -import migrations - - -TO_VERSION = 3 - - -def check_arch(contents, filename): - assert contents['kind'] == 'system' - - valid = True - - if contents['arch'] == 'armv5': - warnings.warn( - "%s uses armv5 architecture that is not understood until version " - "3." % filename) - valid = False - - return valid - - -def check_configuration_extensions(contents, filename): - assert contents['kind'] == 'system' - - valid = True - - for extension in contents.get('configuration-extensions', []): - if extension == 'install-essential-files': - warnings.warn( - "%s uses install-essential-files.configure extension, which " - "was not present in morph.git until commit 423dc974a61f1c0 " - "(tag baserock-definitions-v3)." % filename) - valid = False - - return valid - - -try: - if migrations.check_definitions_version(TO_VERSION - 1): - safe_to_migrate = migrations.process_definitions( - kinds=['system'], validate_cb=check_arch) - safe_to_migrate = migrations.process_definitions( - kinds=['system'], validate_cb=check_configuration_extensions) - - if not safe_to_migrate: - sys.stderr.write( - "Migration failed due to one or more warnings.\n") - sys.exit(1) - else: - migrations.set_definitions_version(TO_VERSION) - sys.stdout.write("Migration completed successfully.\n") - sys.exit(0) - else: - sys.stdout.write("Nothing to do.\n") - sys.exit(0) -except RuntimeError as e: - sys.stderr.write("Error: %s\n" % e.message) - sys.exit(1) diff --git a/migrations/004-install-files-overwrite-symlink.py b/migrations/004-install-files-overwrite-symlink.py deleted file mode 100755 index 6853dcb5..00000000 --- a/migrations/004-install-files-overwrite-symlink.py +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . - -'''Migration to Baserock Definitions format version 4. - -This change to the format was made to work around a bug in a deployment -extension present in morph.git. - -Automated migration is not really possible for this change, and unless you -are experiencing the install-files.configure extension crashing, you can ignore -it completely. - -We have now moved all .configure and .write extensions into the definitions.git -repository. Changes like this no longer require a marking a new version of the -Baserock definitions format in order to prevent build tools crashing. - -Morph commit c373f5a403b0ec introduces version 4 of the definitions format. In -older versions of Morph the install-files.configure extension would crash if it -tried to overwrite a symlink. This bug is fixed in the version of Morph that -can build definitions version 4. - -If you need to overwrite a symlink at deploytime using install-files.configure, -please use VERSION to 4 or above in your definitions.git repo so older versions -of Morph gracefully refuse to deploy, instead of crashing. - -''' - - -import sys - -import migrations - - -TO_VERSION = 4 - - -try: - if migrations.check_definitions_version(TO_VERSION - 1): - migrations.set_definitions_version(TO_VERSION) - sys.stdout.write("Migration completed successfully.\n") - sys.exit(0) - else: - sys.stdout.write("Nothing to do.\n") - sys.exit(0) -except RuntimeError as e: - sys.stderr.write("Error: %s\n" % e.message) - sys.exit(1) diff --git a/migrations/005-strip-commands.py b/migrations/005-strip-commands.py deleted file mode 100755 index da3de940..00000000 --- a/migrations/005-strip-commands.py +++ /dev/null @@ -1,78 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . - - -'''Migration to Baserock Definitions format version 5. - -Version 5 of the definitions format adds a 'strip-commands' field that can -be set in chunk definitions. - -Version 5 also allows deployment extensions to live in definitions.git instead -of morph.git. This greatly reduces the interface surface of the Baserock -definitions format specification, because we no longer have to mark a new -version of the definitions format each time an extension in morph.git is added, -removed, or changes its API in any way. - -In commit 6f4929946 of git://git.baserock.org/baserock/baserock/definitions.git -the deployment extensions were moved into an extensions/ subdirectory, and the -system and cluster .morph files that referred to them were all updated to -prepend 'extension/' to the filenames. This migration doesn't (re)do that -change. - -''' - - -import sys -import warnings - -import migrations - - -TO_VERSION = 5 - - -def check_strip_commands(contents, filename): - assert contents['kind'] == 'chunk' - - valid = True - - if 'strip-commands' in contents: - warnings.warn( - "%s has strip-commands, which are not valid until version 5" % - filename) - valid = False - - return valid - - -try: - if migrations.check_definitions_version(TO_VERSION - 1): - safe_to_migrate = migrations.process_definitions( - kinds=['chunk'], validate_cb=check_strip_commands) - - if not safe_to_migrate: - sys.stderr.write( - "Migration failed due to one or more warnings.\n") - sys.exit(1) - else: - migrations.set_definitions_version(TO_VERSION) - sys.stdout.write("Migration completed successfully.\n") - sys.exit(0) - else: - sys.stdout.write("Nothing to do.\n") - sys.exit(0) -except RuntimeError as e: - sys.stderr.write("Error: %s\n" % e.message) - sys.exit(1) diff --git a/migrations/006-specify-build-system.py b/migrations/006-specify-build-system.py deleted file mode 100755 index b66736c6..00000000 --- a/migrations/006-specify-build-system.py +++ /dev/null @@ -1,354 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . - - -# THIS MIGRATION REQUIRES NETWORK ACCESS TO A BASEROCK GIT CACHE SERVER! If -# you do not have your own Trove, or don't know what a Trove is, it should -# work as-is, provided you have internet access that allows access to -# http://git.baserock.org:8080/. -# -# If you do have your own Trove, change the value of TROVE_HOST below to -# point to it. -# -# This migration uses the same autodetection mechanism that Morph and YBD use -# at build time, in order to fill in the 'build-system' field in strata where -# it is now needed. - - -'''Migration to Baserock Definitions format version 6. - -In definitions version 6, build system autodetection no longer happens. This -means that any chunk that wants to use one of the predefined build systems -(those built into Morph) must say so explicitly, using the 'build-system' -field. - -The build-system field for a chunk can now be specified within a stratum that -contains it. Previously you needed to add a .morph file for a chunk in order to -specify its build-system, but we want to avoid needing a .morph file for -components that follow standard patterns. - -Previously, if build-system wasn't given, Morph would scan the contents of the -chunk's Git repo and try to autodetect which build system was used. This could -be slow, could fail in confusing ways, and meant that to fully parse -definitions you needed access to some or all of the repos they referenced. - -The chosen build-system affects which predefined command sequences are set for -a chunk. It is valid to omit the field if a chunk has its own build commands -defined in a .morph file. When listing the chunks included in a stratum, either -'morph' or 'build-system' must be specified, but not both (to avoid the -possibility of conflicting values). - -''' - - -import requests -import yaml - -import logging -import os -import sys -import warnings - -import migrations - - -TROVE_HOST = 'git.baserock.org' - -REPO_ALIASES = { - 'baserock:': 'git://%s/baserock/' % TROVE_HOST, - 'freedesktop:': 'git://anongit.freedesktop.org/', - 'github:': 'git://github.com/', - 'gnome:': 'git://git.gnome.org/', - 'upstream:': 'git://%s/delta/' % TROVE_HOST, -} - -GIT_CACHE_SERVER_URL = 'http://%s:8080/' % TROVE_HOST - -FAIL_ON_REMOTE_CACHE_ERRORS = False - - -TO_VERSION = 6 - - -# From ybd.git file repos.py at commit eb3bf397ba729387f0d4145a8df8d3c1f9eb707f - -def get_repo_url(repo): - for alias, url in REPO_ALIASES.items(): - repo = repo.replace(alias, url) - if repo.endswith('.git'): - repo = repo[:-4] - return repo - - -# Based on morph.git file buildsystem.py at commit a7748f9cdaaf4112c30d7c1. -# -# I have copied and pasted this code here, as it should not be needed anywhere -# once everyone has migrated to definitions version 6. - -class BuildSystem(object): - def used_by_project(self, file_list): - '''Does a project use this build system? - - ``exists`` is a function that returns a boolean telling if a - filename, relative to the project source directory, exists or not. - - ''' - raise NotImplementedError() # pragma: no cover - - -class ManualBuildSystem(BuildSystem): - - '''A manual build system where the morphology must specify all commands.''' - - name = 'manual' - - def used_by_project(self, file_list): - return False - - -class DummyBuildSystem(BuildSystem): - - '''A dummy build system, useful for debugging morphologies.''' - - name = 'dummy' - - def used_by_project(self, file_list): - return False - - -class AutotoolsBuildSystem(BuildSystem): - - '''The automake/autoconf/libtool holy trinity.''' - - name = 'autotools' - - def used_by_project(self, file_list): - indicators = [ - 'autogen', - 'autogen.sh', - 'configure', - 'configure.ac', - 'configure.in', - 'configure.in.in', - ] - - return any(x in file_list for x in indicators) - - -class PythonDistutilsBuildSystem(BuildSystem): - - '''The Python distutils build systems.''' - - name = 'python-distutils' - - def used_by_project(self, file_list): - indicators = [ - 'setup.py', - ] - - return any(x in file_list for x in indicators) - - -class CPANBuildSystem(BuildSystem): - - '''The Perl cpan build system.''' - - name = 'cpan' - - def used_by_project(self, file_list): - indicators = [ - 'Makefile.PL', - ] - - return any(x in file_list for x in indicators) - - -class CMakeBuildSystem(BuildSystem): - - '''The cmake build system.''' - - name = 'cmake' - - def used_by_project(self, file_list): - indicators = [ - 'CMakeLists.txt', - ] - - return any(x in file_list for x in indicators) - - -class QMakeBuildSystem(BuildSystem): - - '''The Qt build system.''' - - name = 'qmake' - - def used_by_project(self, file_list): - indicator = '.pro' - - for x in file_list: - if x.endswith(indicator): - return True - - return False - - -build_systems = [ - ManualBuildSystem(), - AutotoolsBuildSystem(), - PythonDistutilsBuildSystem(), - CPANBuildSystem(), - CMakeBuildSystem(), - QMakeBuildSystem(), - DummyBuildSystem(), -] - - -def detect_build_system(file_list): - '''Automatically detect the build system, if possible. - - If the build system cannot be detected automatically, return None. - For ``exists`` see the ``BuildSystem.exists`` method. - - ''' - for bs in build_systems: - if bs.used_by_project(file_list): - return bs - return None - - -## End of code based on morph.git file buildsystem.py. - -def get_toplevel_file_list_from_repo(url, ref): - '''Try to list the set of files in the root directory of the repo at 'url'. - - ''' - try: - response = requests.get( - GIT_CACHE_SERVER_URL + '1.0/trees', - params={'repo': url, 'ref': ref}, - headers={'Accept': 'application/json'}, - timeout=9) - logging.debug("Got response: %s" % response) - try: - response.raise_for_status() - toplevel_tree = response.json()['tree'] - except Exception as e: - raise RuntimeError( - "Unexpected response from server %s for repo %s: %s" % - (GIT_CACHE_SERVER_URL, url, e.message)) - toplevel_filenames = toplevel_tree.keys() - except requests.exceptions.ConnectionError as e: - raise RuntimeError("Unable to connect to cache server %s while trying " - "to query file list of repo %s. Error was: %s" % - (GIT_CACHE_SERVER_URL, url, e.message)) - return toplevel_filenames - - -def validate_chunk_refs(contents, filename): - assert contents['kind'] == 'stratum' - - valid = True - for chunk_ref in contents.get('chunks', []): - if chunk_ref.get('morph') is None: - # No chunk .morph file -- this stratum was relying on build-system - # autodetection here. - - if 'repo' not in chunk_ref: - warnings.warn("%s: Chunk %s doesn't specify a source repo." % - (filename, chunk_ref.get('name'))) - valid = False - - if 'ref' not in chunk_ref: - warnings.warn("%s: Chunk %s doesn't specify a source ref." % - (filename, chunk_ref.get('name'))) - valid = False - return valid - - -def move_dict_entry_last(dict_object, key, error_if_missing=False): - '''Move an entry in a ordered dict to the end.''' - - # This is a hack, I couldn't find a method on the 'CommentedMap' type dict - # that we receive from ruamel.yaml that would allow doing this neatly. - if key in dict_object: - value = dict_object[key] - del dict_object[key] - dict_object[key] = value - else: - if error_if_missing: - raise KeyError(key) - - -def ensure_buildsystem_defined_where_needed(contents, filename): - assert contents['kind'] == 'stratum' - - changed = False - for chunk_ref in contents.get('chunks', []): - if chunk_ref.get('morph') is None: - # No chunk .morph file -- this stratum was relying on build-system - # autodetection here. - - chunk_git_url = get_repo_url(chunk_ref['repo']) - chunk_git_ref = chunk_ref['ref'] - - try: - toplevel_file_list = get_toplevel_file_list_from_repo( - chunk_git_url, chunk_git_ref) - except Exception as e: - warnings.warn(str(e)) - message = ( - "Unable to look up one or more repos on remote Git " - "server %s. If you are using a Trove that is not %s, " - "please edit the TROVE_HOST constant in this script " - "and run it again." % (TROVE_HOST, TROVE_HOST)) - if FAIL_ON_REMOTE_CACHE_ERRORS: - raise RuntimeError(message) - else: - warnings.warn(message) - continue - - logging.debug( - '%s: got file list %s', chunk_git_url, toplevel_file_list) - build_system = detect_build_system(toplevel_file_list) - - chunk_ref['build-system'] = build_system.name - move_dict_entry_last(chunk_ref, 'build-depends') - - changed = True - - return changed - - -try: - if migrations.check_definitions_version(TO_VERSION - 1): - success = migrations.process_definitions( - kinds=['stratum'], - validate_cb=validate_chunk_refs, - modify_cb=ensure_buildsystem_defined_where_needed) - if not success: - sys.stderr.write( - "Migration failed due to one or more warnings.\n") - sys.exit(1) - else: - migrations.set_definitions_version(TO_VERSION) - sys.stderr.write("Migration completed successfully.\n") - sys.exit(0) - else: - sys.stderr.write("Nothing to do.\n") - sys.exit(0) -except RuntimeError as e: - sys.stderr.write("Error: %s\n" % e.message) - sys.exit(1) diff --git a/migrations/007-defaults-in-definitions.py b/migrations/007-defaults-in-definitions.py deleted file mode 100755 index 489baf9b..00000000 --- a/migrations/007-defaults-in-definitions.py +++ /dev/null @@ -1,67 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . - - -'''Migration to Baserock Definitions format version 7. - -Definitions version 7 adds a file named DEFAULTS which sets the default -build commands and default split rules for the set of definitions in that -repo. - -''' - - -import os -import shutil -import sys -import warnings - -import migrations - - -TO_VERSION = 7 - - - -try: - if migrations.check_definitions_version(TO_VERSION - 1): - if os.path.exists('DEFAULTS'): - warnings.warn( - "DEFAULTS file already exists in these definitions.") - valid = False - else: - shutil.copy( - 'migrations/007-initial-defaults', - 'DEFAULTS') - valid = True - - if valid: - migrations.set_definitions_version(TO_VERSION) - sys.stdout.write("Migration completed successfully.\n") - sys.exit(0) - else: - sys.stderr.write( - "Migration failed due to one or more warnings.\n") - sys.exit(1) - else: - if not os.path.exists('DEFAULTS'): - warnings.warn( - "These definitions are marked as version 7 but there is no " - "DEFAULTS file.") - sys.stdout.write("Nothing to do.\n") - sys.exit(0) -except RuntimeError as e: - sys.stderr.write("Error: %s\n" % e.message) - sys.exit(1) diff --git a/migrations/007-initial-defaults b/migrations/007-initial-defaults deleted file mode 100644 index ab034a0b..00000000 --- a/migrations/007-initial-defaults +++ /dev/null @@ -1,199 +0,0 @@ -# Baserock definitions defaults -# ============================= -# -# The DEFAULTS file is treated specially by Baserock build tools. -# -# For more information, see: . - - -# Predefined build commands -# ------------------------- -# -# Common patterns in build instructions can be defined here, which can save -# users from having to write lots of similar-looking chunk .morph files. -# -# There are pre- and post- variants for each set of commands. These exist so -# you can add more commands without having to copy the defaults. For example, -# to create an extra symlink after running `make install`, you can use -# post-install-commands. Since these exist as a way of extending the defaults, -# you cannot set default values for the pre- and post- commands. -# -# The set of environment variables available when these commands are executed -# is not formally specified right now, but you can assume PREFIX, DESTDIR and -# MORPH_ARCH are all set. -# -build-systems: - manual: - # The special, default 'no-op' build system. - configure-commands: [] - build-commands: [] - install-commands: [] - strip-commands: [] - - autotools: - # GNU Autoconf and GNU Automake, or anything which follow the same pattern. - # - # See also: https://github.com/cgwalters/build-api/blob/master/build-api.md - configure-commands: - - >- - export NOCONFIGURE=1; - if [ -e autogen ]; then ./autogen; - elif [ -e autogen.sh ]; then ./autogen.sh; - elif [ -e bootstrap ]; then ./bootstrap; - elif [ -e bootstrap.sh ]; then ./bootstrap.sh; - elif [ ! -e ./configure ]; then autoreconf -ivf; - fi - - ./configure --prefix="$PREFIX" - build-commands: - - make - install-commands: - - make DESTDIR="$DESTDIR" install - strip-commands: - # TODO: Make idempotent when files are hardlinks - # Strip all ELF binary files that are executable or named like a library. - # .so files for C, .cmxs for OCaml and .node for Node. - # - # The file name and permissions checks are done with the `find` command before - # the ELF header is checked with the shell command, because it is a lot cheaper - # to check the mode and file name first, because it is a metadata check, rather - # than a subprocess and a file read. - # - # `file` is not used, to keep the dependency requirements down. - - &generic-strip-command | - find "$DESTDIR" -type f \ - '(' -perm -111 -o -name '*.so*' -o -name '*.cmxs' -o -name '*.node' ')' \ - -exec sh -ec \ - 'read -n4 hdr <"$1" # check for elf header - if [ "$hdr" != "$(printf \\x7fELF)" ]; then - exit 0 - fi - debugfile="$DESTDIR$PREFIX/lib/debug/$(basename "$1")" - mkdir -p "$(dirname "$debugfile")" - objcopy --only-keep-debug "$1" "$debugfile" - chmod 644 "$debugfile" - strip --remove-section=.comment --remove-section=.note --strip-unneeded "$1" - objcopy --add-gnu-debuglink "$debugfile" "$1"' - {} ';' - - python-distutils: - # The Python distutils build systems. - configure-commands: [] - build-commands: - - python setup.py build - install-commands: - - python setup.py install --prefix "$PREFIX" --root "$DESTDIR" - strip-commands: - - *generic-strip-command - - cpan: - # The Perl ExtUtil::MakeMaker build system. This is called the 'cpan' build - # system for historical reasons. - # - # To install perl distributions into the correct location in our chroot - # we need to set PREFIX to / in the configure-commands. - # - # The mapping between PREFIX and the final installation - # directories is complex and depends upon the configuration of perl - # see, - # https://metacpan.org/pod/distribution/perl/INSTALL#Installation-Directories - # and ExtUtil::MakeMaker's documentation for more details. - configure-commands: - - perl Makefile.PL PREFIX=$DESTDIR$PREFIX - build-commands: - - make - install-commands: - - make install - strip-commands: - - *generic-strip-command - - module-build: - # The Module::Build build system - # - # See the comment in ExtUtilsMakeMakerBuildSystem to see why --prefix is - # set to $DESTDIR$PREFIX here (--prefix in Module::Build has the same - # meaning as PREFIX in ExtUtils::MakeMaker). - configure-commands: - - perl Build.PL --prefix "$DESTDIR$PREFIX" - build-commands: - - ./Build - install-commands: - - ./Build install - strip-commands: - - *generic-strip-command - - cmake: - # The CMake build system. - configure-commands: - - cmake -DCMAKE_INSTALL_PREFIX="$PREFIX" - build-commands: - - make - install-commands: - - make DESTDIR="$DESTDIR" install - strip-commands: - - *generic-strip-command - - qmake: - # The Qt build system. - configure-commands: - - qmake -makefile - build-commands: - - make - install-commands: - - make INSTALL_ROOT="$DESTDIR" install - strip-commands: - - *generic-strip-command - - -# Predefined artifact splitting rules -# ----------------------------------- -# -# Once a build has completed, you have some files that have been installed into -# $DESTDIR. The splitting rules control how many 'artifact' tarballs are -# generated as a result of the build, and which files from $DESTDIR end up in -# which 'artifact'. -# -# The default split rules are defined here. These can be overriden in -# individual chunk .morph files and stratum .morph files using the 'products' -# field. -# -split-rules: - chunk: - - artifact: -bins - include: - - (usr/)?s?bin/.* - - artifact: -libs - include: - - (usr/)?lib(32|64)?/lib[^/]*\.so(\.\d+)* - - (usr/)libexec/.* - - artifact: -devel - include: - - (usr/)?include/.* - - (usr/)?lib(32|64)?/lib.*\.a - - (usr/)?lib(32|64)?/lib.*\.la - - (usr/)?(lib(32|64)?|share)/pkgconfig/.*\.pc - - artifact: -doc - include: - - (usr/)?share/doc/.* - - (usr/)?share/man/.* - - (usr/)?share/info/.* - - artifact: -locale - include: - - (usr/)?share/locale/.* - - (usr/)?share/i18n/.* - - (usr/)?share/zoneinfo/.* - - artifact: -misc - include: - - .* - - stratum: - - artifact: -devel - include: - - .*-devel - - .*-debug - - .*-doc - - artifact: -runtime - include: - - .*-bins - - .*-libs - - .*-locale - - .*-misc - - .* diff --git a/migrations/008-submodules-in-strata.py b/migrations/008-submodules-in-strata.py deleted file mode 100755 index 92ec865d..00000000 --- a/migrations/008-submodules-in-strata.py +++ /dev/null @@ -1,220 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2016 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . - - -# THIS MIGRATION REQUIRES NETWORK ACCESS TO A BASEROCK GIT CACHE SERVER! If -# you do not have your own Trove, or don't know what a Trove is, it should -# work as-is, provided you have internet access that allows access to -# http://git.baserock.org:8080/. -# -# If you do have your own Trove, change the value of TROVE_HOST below to -# point to it. -# - -'''Migration to Baserock Definitions format version 8. - -In definitions version 8, submodules must be declared explicitly for all chunks -that contains a .gitmodules file in their root. This is so that mirrored source -repositories don't need to maintain branches that point to the mirrored -submodules, and can instead translate these at build time. - -''' - -import requests -import string -import logging -import re -import os -import sys -import warnings -import migrations -from subprocess import call, Popen -from ConfigParser import RawConfigParser -from StringIO import StringIO - - -TROVE_HOST = 'git.baserock.org' - -REPO_ALIASES = { - 'baserock:': 'git://%s/baserock/' % TROVE_HOST, - 'freedesktop:': 'git://anongit.freedesktop.org/', - 'github:': 'git://github.com/', - 'gnome:': 'git://git.gnome.org/', - 'upstream:': 'git://%s/delta/' % TROVE_HOST, -} - -GIT_CACHE_SERVER_URL = 'http://%s:8080/' % TROVE_HOST - -FAIL_ON_REMOTE_CACHE_ERRORS = False - - -TO_VERSION = 8 - - -# From ybd.git file repos.py at commit eb3bf397ba729387f0d4145a8df8d3c1f9eb707f - -def get_repo_url(repo): - for alias, url in REPO_ALIASES.items(): - repo = repo.replace(alias, url) - if repo.endswith('.git'): - repo = repo[:-4] - return repo - -def get_repo_name(repo): - ''' Convert URIs to strings that only contain digits, letters, _ and %. - NOTE: this naming scheme is based on what lorry uses - ''' - valid_chars = string.digits + string.ascii_letters + '%_' - transl = lambda x: x if x in valid_chars else '_' - return ''.join([transl(x) for x in get_repo_url(repo)]) - - -## End of code based on ybd repos.py - -def get_toplevel_file_list_from_repo(url, ref): - '''Try to list the set of files in the root directory of the repo at 'url'. - - ''' - try: - response = requests.get( - GIT_CACHE_SERVER_URL + '1.0/trees', - params={'repo': url, 'ref': ref}, - headers={'Accept': 'application/json'}, - timeout=9) - logging.debug("Got response: %s" % response) - try: - response.raise_for_status() - toplevel_tree = response.json()['tree'] - except Exception as e: - raise RuntimeError( - "Unexpected response from server %s for repo %s: %s" % - (GIT_CACHE_SERVER_URL, url, e.message)) - toplevel_filenames = toplevel_tree.keys() - except requests.exceptions.ConnectionError as e: - raise RuntimeError("Unable to connect to cache server %s while trying " - "to query file list of repo %s. Error was: %s" % - (GIT_CACHE_SERVER_URL, url, e.message)) - return toplevel_filenames - - -def validate_chunk_refs(contents, filename): - assert contents['kind'] == 'stratum' - - valid = True - for chunk_ref in contents.get('chunks', []): - if chunk_ref.get('morph') is None: - # No chunk .morph file -- this stratum was relying on build-system - # autodetection here. - - if 'repo' not in chunk_ref: - warnings.warn("%s: Chunk %s doesn't specify a source repo." % - (filename, chunk_ref.get('name'))) - valid = False - - if 'ref' not in chunk_ref: - warnings.warn("%s: Chunk %s doesn't specify a source ref." % - (filename, chunk_ref.get('name'))) - valid = False - return valid - - -def move_dict_entry_last(dict_object, key, error_if_missing=False): - '''Move an entry in a ordered dict to the end.''' - - # This is a hack, I couldn't find a method on the 'CommentedMap' type dict - # that we receive from ruamel.yaml that would allow doing this neatly. - if key in dict_object: - value = dict_object[key] - del dict_object[key] - dict_object[key] = value - else: - if error_if_missing: - raise KeyError(key) - -def submodules_to_dict(path): - with open(os.path.join(path, '.gitmodules'), "r") as gitfile: - content = '\n'.join([l.strip() for l in gitfile.read().splitlines()]) - io = StringIO(content) - parser = RawConfigParser() - parser.readfp(io) - stuff = {} - for section in parser.sections(): - submodule = re.sub(r'submodule "(.*)"', r'\1', section) - url = parser.get(section, 'url') - path = parser.get(section, 'path') - stuff[submodule] = {'url': url} - return stuff - -def add_submodules_to_strata(contents, filename): - assert contents['kind'] == 'stratum' - - changed = False - for chunk_ref in contents.get('chunks', []): - chunk_git_url = get_repo_url(chunk_ref['repo']) - chunk_git_ref = chunk_ref['ref'] - - if 'submodules' in chunk_ref: - continue - try: - toplevel_file_list = get_toplevel_file_list_from_repo( - chunk_git_url, chunk_git_ref) - except Exception as e: - message = ( - "Unable to look up repo %s on remote Git server %s. Check that " - "the repo URL is correct." % (chunk_git_url, TROVE_HOST)) - warning = ( - "If you are using a Trove that is not %s, please edit the " - "TROVE_HOST constant in this script and run it again." % - TROVE_HOST) - if FAIL_ON_REMOTE_CACHE_ERRORS: - raise RuntimeError(message + " " + warning) - else: - warnings.warn(message) - warnings.warn(warning) - continue - - logging.debug( - "%s: got file list %s", chunk_git_url, toplevel_file_list) - - path = get_repo_name(chunk_git_url) - if u'.gitmodules' in toplevel_file_list: - call(['git', 'clone', chunk_git_url, path]) - p_co = Popen(['git', 'checkout', chunk_git_ref], cwd=path) - p_co.wait() - chunk_ref['submodules'] = submodules_to_dict(path) - call(['rm', '-rf', path]) - changed = True - - return changed -try: - if migrations.check_definitions_version(TO_VERSION - 1): - success = migrations.process_definitions( - kinds=['stratum'], - validate_cb=validate_chunk_refs, - modify_cb=add_submodules_to_strata) - if not success: - sys.stderr.write( - "Migration failed due to one or more warnings.\n") - sys.exit(1) - else: - migrations.set_definitions_version(TO_VERSION) - sys.stderr.write("Migration completed successfully.\n") - sys.exit(0) - else: - sys.stderr.write("Nothing to do.\n") - sys.exit(0) -except RuntimeError as e: - sys.stderr.write("Error: %s\n" % e.message) - sys.exit(1) diff --git a/migrations/GUIDELINES b/migrations/GUIDELINES deleted file mode 100644 index 3694e2c9..00000000 --- a/migrations/GUIDELINES +++ /dev/null @@ -1,35 +0,0 @@ -Guidelines for writing migrations ---------------------------------- - -All changes to the definitions format must have a migration, but it is valid -for the migration to do nothing except update the version number (see -004-install-files-overwrite-symlink.py for an example of that). - -This small set of rules exists to ensure that the migrations are consistent and -easy to understand. If you are writing a migration and these rules don't make -any sense, we should probably change them. Please sign up to the -baserock-dev@baserock.org mailing list to suggest the change. - -- Write migrations in Python. They must be valid Python 3. For now, since - only Python 2 is available in Baserock 'build' and 'devel' reference systems - up to the 15.25 release of Baserock, they must also be valid Python 2. - -- Don't use any external libraries. - -- Follow the existing file naming pattern, and the existing code convention. - -- Keep the migration code as simple as possible. - -- Avoid crashing on malformed input data, where practical. For example, use - contents.get('field') instead of contents['field'] to avoid crashing when - 'field' is not present. The idea of this is to avoid a "cascade of errors" - problem when running the migrations on bad inputs. It is confusing when - migrations break on problems that are unrelated to the actual area where - they operate, even if they are theoretically "within their rights" to do so. - -- Migrate the definitions in line with current best practices. For example, - migrations/001-empty-build-depends.py doesn't need to remove empty - build-depends fields: they are still valid in version 1 of the format. But - best practice is now to remove them. Users who don't agree with this practice - can choose to not run that migration, which can be done with `chmod -x - migrations/xxx.py`. diff --git a/migrations/indent b/migrations/indent deleted file mode 100755 index 8d6f034f..00000000 --- a/migrations/indent +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . - - -'''Automatically reformat a set of Baserock definition files. - -This tool expects to be able to use ruamel.yaml to load and write YAML. -It will totally ruin things if used with PyYAML. - -It makes sense to run this script on your definitions, and check through -and commit the result, before running any of the automated migrations. This -way, you can be sure that the migrations will only change things that they need -to in the .morph files. - -''' - - -import migrations - - -def force_rewrite(contents, filename): - return True - -migrations.process_definitions(path='.', modify_cb=force_rewrite) diff --git a/migrations/migrations.py b/migrations/migrations.py deleted file mode 100644 index 22ed1328..00000000 --- a/migrations/migrations.py +++ /dev/null @@ -1,228 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . - - -'''Tools for migrating Baserock definitions from one format version to another. - -''' - - -# ruamel.yaml is a fork of PyYAML which allows rewriting YAML files without -# destroying all of the comments, ordering and formatting. The more -# widely-used PyYAML library will produce output totally different to the -# input file in most cases. -# -# See: -import ruamel.yaml as yaml - -import logging -import os -import warnings - - -# Uncomment this to cause all log messages to be written to stdout. By -# default they are hidden, but if you are debugging something this might help! -# -# logging.basicConfig(stream=sys.stdout, level=logging.DEBUG) - - -def pretty_warnings(message, category, filename, lineno, - file=None, line=None): - '''Format warning messages from warnings.warn().''' - return 'WARNING: %s\n' % (message) - -# Override the default warning formatter (which is ugly), and add a filter to -# ensure duplicate warnings only get displayed once. -warnings.simplefilter("once", append=True) -warnings.formatwarning = pretty_warnings - - - -def parse_yaml_with_roundtrip_info(text): - return yaml.load(text, yaml.RoundTripLoader) - -def write_yaml_with_roundtrip_info(contents, stream, **kwargs): - yaml.dump(contents, stream, Dumper=yaml.RoundTripDumper, **kwargs) - - - -class VersionFileError(RuntimeError): - '''Represents errors in the version marker file (./VERSION).''' - pass - - -class MigrationOutOfOrderError(RuntimeError): - '''Raised if a migration is run on too old a version of definitions. - - It's not an error to run a migration on a version that is already migrated. - - ''' - pass - - -def check_definitions_version(from_version, version_file='./VERSION', - to_version=None): - '''Check if migration between 'from_version' and 'to_version' is needed. - - Both 'from_version' and 'to_version' should be whole numbers. The - 'to_version' defaults to from_version + 1. - - This function reads the version marker file specified by 'version_file'. - Returns True if the version is between 'from_version' and 'to_version', - indicating that migration needs to be done. Returns False if the version is - already at or beyond 'to_version'. Raises MigrationOutOfOrderError if the - version is below 'from_version'. - - If 'version_file' is missing or invalid, it raises VersionFileError. The - version file is expected to follow the following format: - - version: 1 - - ''' - to_version = to_version or (from_version + 1) - need_to_migrate = False - - if os.path.exists(version_file): - logging.info("Found version information file: %s" % version_file) - - with open(version_file) as f: - version_text = f.read() - - if len(version_text) == 0: - raise VersionFileError( - "File %s exists but is empty." % version_file) - - try: - version_info = yaml.safe_load(version_text) - current_version = version_info['version'] - - if current_version >= to_version: - logging.info( - "Already at version %i." % current_version) - elif current_version < from_version: - raise MigrationOutOfOrderError( - "This tool expects to migrate from version %i to version " - "%i of the Baserock Definitions syntax. These definitions " - "claim to be version %i." % ( - from_version, to_version, current_version)) - else: - logging.info("Need to migrate from %i to %i.", - current_version, to_version) - need_to_migrate = True - except (KeyError, TypeError, ValueError) as e: - logging.exception(e) - raise VersionFileError( - "Invalid version info: '%s'" % version_text) - else: - raise VersionFileError( - "No file %s was found. Please run the migration scripts in order," - "starting from 000-version-info.py." % version_file) - - return need_to_migrate - - -def set_definitions_version(new_version, version_file='./VERSION'): - '''Update the version information stored in 'version_file'. - - The new version must be a whole number. If 'version_file' doesn't exist, - it will be created. - - ''' - version_info = {'version': new_version} - with open(version_file, 'w') as f: - # If 'default_flow_style' is True (the default) then the output here - # will look like "{version: 0}" instead of "version: 0". - yaml.safe_dump(version_info, f, default_flow_style=False) - - -def walk_definition_files(path='.', extensions=['.morph']): - '''Recursively yield all files under 'path' with the given extension(s). - - This is safe to run in the top level of a Git repository, as anything under - '.git' will be ignored. - - ''' - for dirname, dirnames, filenames in os.walk('.'): - filenames.sort() - dirnames.sort() - if '.git' in dirnames: - dirnames.remove('.git') - for filename in filenames: - for extension in extensions: - if filename.endswith(extension): - yield os.path.join(dirname, filename) - - -ALL_KINDS = ['cluster', 'system', 'stratum', 'chunk'] - - -def process_definitions(path='.', kinds=ALL_KINDS, validate_cb=None, - modify_cb=None): - '''Run callbacks for all Baserock definitions found in 'path'. - - If 'validate_cb' is set, it will be called for each definition and can - return True or False to indicate whether that definition is valid according - a new version of the format. The process_definitions() function will return - True if all definitions were valid according to validate_cb(), and False - otherwise. - - If 'modify_cb' is set, it will be called for each definition and can - modify the 'content' dict. It should return True if the dict was modified, - and in this case the definition file will be overwritten with the new - contents. The 'ruamel.yaml' library is used if it is available, which will - try to preserve comments, ordering and some formatting in the YAML - definition files. - - If 'validate_cb' is set and returns False for a definition, 'modify_cb' - will not be called. - - Both callbacks are passed two parameters: a dict containing the contents of - the definition file, and its filename. The filename is passed so you can - use it when reporting errors. - - The 'kinds' setting can be used to ignore some definitions according to the - 'kind' field. - - ''' - all_valid = True - - for filename in walk_definition_files(path=path): - with open(filename) as f: - text = f.read() - - if modify_cb is None: - contents = yaml.load(text) - else: - contents = parse_yaml_with_roundtrip_info(text) - - if 'kind' in contents: - if contents['kind'] in kinds: - valid = True - changed = False - - if validate_cb is not None: - valid = validate_cb(contents, filename) - all_valid &= valid - - if valid and modify_cb is not None: - changed = modify_cb(contents, filename) - - if changed: - with open(filename, 'w') as f: - write_yaml_with_roundtrip_info(contents, f, width=80) - else: - warnings.warn("%s is invalid: no 'kind' field set." % filename) - - return all_valid diff --git a/migrations/run-all b/migrations/run-all deleted file mode 100755 index 5a817ee5..00000000 --- a/migrations/run-all +++ /dev/null @@ -1,73 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see . - - -'''Run a set of migration scripts. - -This script does exactly what `PYTHONPATH=. run-parts --exit-on-error` would -do. I avoided using 'run-parts' purely because the implementation in Fedora 22 -doesn't have an '--exit-on-error' option. The Busybox and Debian -implementations do have that option. - -Please fix run-parts in https://git.fedorahosted.org/cgit/crontabs.git/tree/ -so we can simplify this script :-) - -''' - - -import os -import subprocess -import sys - - -if len(sys.argv) == 2: - migration_dir = sys.argv[1] -elif len(sys.argv) == 1: - migration_dir = os.path.dirname(__file__) -else: - sys.stderr.write("Usage: %s [MIGRATION_DIR]\n" % sys.argv[0]) - sys.exit(1) - - -def is_executable(fpath): - return os.path.isfile(fpath) and os.access(fpath, os.X_OK) - -env = os.environ -if 'PYTHONPATH' in env: - env['PYTHONPATH'] = env['PYTHONPATH'] + ':' + migration_dir -else: - env['PYTHONPATH'] = migration_dir - -try: - migrations_found = 0 - for fname in sorted(os.listdir(migration_dir)): - migration_fpath = os.path.join(migration_dir, fname) - if is_executable(migration_fpath): - if not os.path.samefile(migration_fpath, __file__) and \ - fname != 'indent': - migrations_found += 1 - sys.stdout.write(migration_fpath + ":\n") - subprocess.check_call( - migration_fpath, env=env) - - if migrations_found == 0: - sys.stderr.write("No migration files found in '%s'\n" % migration_dir) - sys.exit(1) - else: - sys.exit(0) - -except (subprocess.CalledProcessError, RuntimeError) as e: - sys.stderr.write(str(e) + '\n') - sys.exit(1) -- cgit v1.2.1