diff options
Diffstat (limited to 'migrations/006-specify-build-system.py')
| -rwxr-xr-x | migrations/006-specify-build-system.py | 354 |
1 files changed, 0 insertions, 354 deletions
diff --git a/migrations/006-specify-build-system.py b/migrations/006-specify-build-system.py deleted file mode 100755 index b66736c6..00000000 --- a/migrations/006-specify-build-system.py +++ /dev/null @@ -1,354 +0,0 @@ -#!/usr/bin/env python -# Copyright (C) 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program. If not, see <http://www.gnu.org/licenses/>. - - -# THIS MIGRATION REQUIRES NETWORK ACCESS TO A BASEROCK GIT CACHE SERVER! If -# you do not have your own Trove, or don't know what a Trove is, it should -# work as-is, provided you have internet access that allows access to -# http://git.baserock.org:8080/. -# -# If you do have your own Trove, change the value of TROVE_HOST below to -# point to it. -# -# This migration uses the same autodetection mechanism that Morph and YBD use -# at build time, in order to fill in the 'build-system' field in strata where -# it is now needed. - - -'''Migration to Baserock Definitions format version 6. - -In definitions version 6, build system autodetection no longer happens. This -means that any chunk that wants to use one of the predefined build systems -(those built into Morph) must say so explicitly, using the 'build-system' -field. - -The build-system field for a chunk can now be specified within a stratum that -contains it. Previously you needed to add a .morph file for a chunk in order to -specify its build-system, but we want to avoid needing a .morph file for -components that follow standard patterns. - -Previously, if build-system wasn't given, Morph would scan the contents of the -chunk's Git repo and try to autodetect which build system was used. This could -be slow, could fail in confusing ways, and meant that to fully parse -definitions you needed access to some or all of the repos they referenced. - -The chosen build-system affects which predefined command sequences are set for -a chunk. It is valid to omit the field if a chunk has its own build commands -defined in a .morph file. When listing the chunks included in a stratum, either -'morph' or 'build-system' must be specified, but not both (to avoid the -possibility of conflicting values). - -''' - - -import requests -import yaml - -import logging -import os -import sys -import warnings - -import migrations - - -TROVE_HOST = 'git.baserock.org' - -REPO_ALIASES = { - 'baserock:': 'git://%s/baserock/' % TROVE_HOST, - 'freedesktop:': 'git://anongit.freedesktop.org/', - 'github:': 'git://github.com/', - 'gnome:': 'git://git.gnome.org/', - 'upstream:': 'git://%s/delta/' % TROVE_HOST, -} - -GIT_CACHE_SERVER_URL = 'http://%s:8080/' % TROVE_HOST - -FAIL_ON_REMOTE_CACHE_ERRORS = False - - -TO_VERSION = 6 - - -# From ybd.git file repos.py at commit eb3bf397ba729387f0d4145a8df8d3c1f9eb707f - -def get_repo_url(repo): - for alias, url in REPO_ALIASES.items(): - repo = repo.replace(alias, url) - if repo.endswith('.git'): - repo = repo[:-4] - return repo - - -# Based on morph.git file buildsystem.py at commit a7748f9cdaaf4112c30d7c1. -# -# I have copied and pasted this code here, as it should not be needed anywhere -# once everyone has migrated to definitions version 6. - -class BuildSystem(object): - def used_by_project(self, file_list): - '''Does a project use this build system? - - ``exists`` is a function that returns a boolean telling if a - filename, relative to the project source directory, exists or not. - - ''' - raise NotImplementedError() # pragma: no cover - - -class ManualBuildSystem(BuildSystem): - - '''A manual build system where the morphology must specify all commands.''' - - name = 'manual' - - def used_by_project(self, file_list): - return False - - -class DummyBuildSystem(BuildSystem): - - '''A dummy build system, useful for debugging morphologies.''' - - name = 'dummy' - - def used_by_project(self, file_list): - return False - - -class AutotoolsBuildSystem(BuildSystem): - - '''The automake/autoconf/libtool holy trinity.''' - - name = 'autotools' - - def used_by_project(self, file_list): - indicators = [ - 'autogen', - 'autogen.sh', - 'configure', - 'configure.ac', - 'configure.in', - 'configure.in.in', - ] - - return any(x in file_list for x in indicators) - - -class PythonDistutilsBuildSystem(BuildSystem): - - '''The Python distutils build systems.''' - - name = 'python-distutils' - - def used_by_project(self, file_list): - indicators = [ - 'setup.py', - ] - - return any(x in file_list for x in indicators) - - -class CPANBuildSystem(BuildSystem): - - '''The Perl cpan build system.''' - - name = 'cpan' - - def used_by_project(self, file_list): - indicators = [ - 'Makefile.PL', - ] - - return any(x in file_list for x in indicators) - - -class CMakeBuildSystem(BuildSystem): - - '''The cmake build system.''' - - name = 'cmake' - - def used_by_project(self, file_list): - indicators = [ - 'CMakeLists.txt', - ] - - return any(x in file_list for x in indicators) - - -class QMakeBuildSystem(BuildSystem): - - '''The Qt build system.''' - - name = 'qmake' - - def used_by_project(self, file_list): - indicator = '.pro' - - for x in file_list: - if x.endswith(indicator): - return True - - return False - - -build_systems = [ - ManualBuildSystem(), - AutotoolsBuildSystem(), - PythonDistutilsBuildSystem(), - CPANBuildSystem(), - CMakeBuildSystem(), - QMakeBuildSystem(), - DummyBuildSystem(), -] - - -def detect_build_system(file_list): - '''Automatically detect the build system, if possible. - - If the build system cannot be detected automatically, return None. - For ``exists`` see the ``BuildSystem.exists`` method. - - ''' - for bs in build_systems: - if bs.used_by_project(file_list): - return bs - return None - - -## End of code based on morph.git file buildsystem.py. - -def get_toplevel_file_list_from_repo(url, ref): - '''Try to list the set of files in the root directory of the repo at 'url'. - - ''' - try: - response = requests.get( - GIT_CACHE_SERVER_URL + '1.0/trees', - params={'repo': url, 'ref': ref}, - headers={'Accept': 'application/json'}, - timeout=9) - logging.debug("Got response: %s" % response) - try: - response.raise_for_status() - toplevel_tree = response.json()['tree'] - except Exception as e: - raise RuntimeError( - "Unexpected response from server %s for repo %s: %s" % - (GIT_CACHE_SERVER_URL, url, e.message)) - toplevel_filenames = toplevel_tree.keys() - except requests.exceptions.ConnectionError as e: - raise RuntimeError("Unable to connect to cache server %s while trying " - "to query file list of repo %s. Error was: %s" % - (GIT_CACHE_SERVER_URL, url, e.message)) - return toplevel_filenames - - -def validate_chunk_refs(contents, filename): - assert contents['kind'] == 'stratum' - - valid = True - for chunk_ref in contents.get('chunks', []): - if chunk_ref.get('morph') is None: - # No chunk .morph file -- this stratum was relying on build-system - # autodetection here. - - if 'repo' not in chunk_ref: - warnings.warn("%s: Chunk %s doesn't specify a source repo." % - (filename, chunk_ref.get('name'))) - valid = False - - if 'ref' not in chunk_ref: - warnings.warn("%s: Chunk %s doesn't specify a source ref." % - (filename, chunk_ref.get('name'))) - valid = False - return valid - - -def move_dict_entry_last(dict_object, key, error_if_missing=False): - '''Move an entry in a ordered dict to the end.''' - - # This is a hack, I couldn't find a method on the 'CommentedMap' type dict - # that we receive from ruamel.yaml that would allow doing this neatly. - if key in dict_object: - value = dict_object[key] - del dict_object[key] - dict_object[key] = value - else: - if error_if_missing: - raise KeyError(key) - - -def ensure_buildsystem_defined_where_needed(contents, filename): - assert contents['kind'] == 'stratum' - - changed = False - for chunk_ref in contents.get('chunks', []): - if chunk_ref.get('morph') is None: - # No chunk .morph file -- this stratum was relying on build-system - # autodetection here. - - chunk_git_url = get_repo_url(chunk_ref['repo']) - chunk_git_ref = chunk_ref['ref'] - - try: - toplevel_file_list = get_toplevel_file_list_from_repo( - chunk_git_url, chunk_git_ref) - except Exception as e: - warnings.warn(str(e)) - message = ( - "Unable to look up one or more repos on remote Git " - "server %s. If you are using a Trove that is not %s, " - "please edit the TROVE_HOST constant in this script " - "and run it again." % (TROVE_HOST, TROVE_HOST)) - if FAIL_ON_REMOTE_CACHE_ERRORS: - raise RuntimeError(message) - else: - warnings.warn(message) - continue - - logging.debug( - '%s: got file list %s', chunk_git_url, toplevel_file_list) - build_system = detect_build_system(toplevel_file_list) - - chunk_ref['build-system'] = build_system.name - move_dict_entry_last(chunk_ref, 'build-depends') - - changed = True - - return changed - - -try: - if migrations.check_definitions_version(TO_VERSION - 1): - success = migrations.process_definitions( - kinds=['stratum'], - validate_cb=validate_chunk_refs, - modify_cb=ensure_buildsystem_defined_where_needed) - if not success: - sys.stderr.write( - "Migration failed due to one or more warnings.\n") - sys.exit(1) - else: - migrations.set_definitions_version(TO_VERSION) - sys.stderr.write("Migration completed successfully.\n") - sys.exit(0) - else: - sys.stderr.write("Nothing to do.\n") - sys.exit(0) -except RuntimeError as e: - sys.stderr.write("Error: %s\n" % e.message) - sys.exit(1) |