#!/usr/bin/env python # Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import cliapp import contextlib import os import re import yaml import morphlib class EditMorph(cliapp.Application): '''Tools for performing set operations on large morphologies''' def add_settings(self): self.settings.boolean(['no-git-update'], 'do not update the cached git repositories ' 'automatically') def load_morphology(self, file_name, expected_kind = None): loader = morphlib.morphloader.MorphologyLoader() morphology = loader.load_from_file(file_name) if expected_kind is not None and morphology['kind'] != expected_kind: raise morphlib.Error("Expected: a %s morphology" % expected_kind) return morphology, text def cmd_remove_chunk(self, args): '''Removes from STRATUM all reference of CHUNK''' if len(args) != 2: raise cliapp.AppException("remove-chunk expects a morphology file " "name and a chunk name") file_name = args[0] chunk_name = args[1] morphology, text = self.load_morphology(file_name, expected_kind='stratum') component_count = 0 build_depends_count = 0 new_chunks = morphology['chunks'] for info in morphology['chunks']: if info['name'] == chunk_name: new_chunks.remove(info) component_count += 1 elif chunk_name in info['build-depends']: info['build-depends'].remove(chunk_name) build_depends_count += 1 morphology['chunks'] = new_chunks loader = morphlib.morphloader.MorphologyLoader() loader.save_to_file(file_name, morphology) self.output.write("Removed: %i chunk(s) and %i build depend(s).\n" % (component_count, build_depends_count)) def cmd_sort(self, args): """Sort STRATUM""" if len(args) != 1: raise cliapp.AppException("sort expects a morphology file name") file_name = args[0] morphology, text = self.load_morphology(file_name, expected_kind='stratum') for chunk in morphology['chunks']: chunk['build-depends'].sort() morphology['chunks'] = self.sort_chunks(morphology['chunks']) loader = morphlib.morphloader.MorphologyLoader() loader.save_to_file(file_name, morphology) def sort_chunks(self, chunks_list): """Sort stratum chunks The order is something like alphabetical reverse dependency order. Chunks on which nothing depends are sorted at the bottom. The algorithm used is a simple-minded recursive sort. """ chunk_dict = {} for chunk in chunks_list: chunk_dict[chunk['name']] = chunk reverse_deps_dict = {} for chunk_name in chunk_dict.keys(): chunk = chunk_dict[chunk_name] for dep in chunk['build-depends']: if dep not in reverse_deps_dict: reverse_deps_dict[dep] = [chunk_name] else: reverse_deps_dict[dep].append(chunk_name) sort_order = list(chunk_dict.keys()) sort_order.sort(key=unicode.lower) result = [] satisfied_list = [] repeat_count = 0 while len(sort_order) > 0: postponed_list = [] # Move any chunk into the result order that has all its # dependencies satisfied in the result already. for chunk_name in sort_order: deps_satisfied = True chunk = chunk_dict[chunk_name] for dep in chunk['build-depends']: if dep not in satisfied_list: deps_satisfied = False if dep not in sort_order: raise cliapp.AppException( 'Invalid build-dependency for %s: %s' % (chunk['name'], dep)) break if deps_satisfied: result.append(chunk) satisfied_list.append(chunk_name) else: postponed_list.append(chunk_name) if len(postponed_list) == len(sort_order): # This is not the smartest algorithm possible (but it works!) repeat_count += 1 if repeat_count > 10: raise cliapp.AppException('Stuck in loop while sorting') assert(len(postponed_list) + len(result) == len(chunk_dict.keys())) sort_order = postponed_list # Move chunks which are not build-depends of other chunks to the end. targets = [c for c in chunk_dict.keys() if c not in reverse_deps_dict] targets.sort(key=unicode.lower) for chunk_name in targets: result.remove(chunk_dict[chunk_name]) result.append(chunk_dict[chunk_name]) return result @staticmethod @contextlib.contextmanager def _open_yaml(path): with open(path, 'r') as f: d = yaml.load(f) yield d with open(path, 'w') as f: yaml.dump(d, f, default_flow_style=False) def cmd_set_system_artifact_depends(self, args): '''Change the artifacts used by a System. Usage: MORPHOLOGY_FILE STRATUM_NAME ARTIFACTS ARTIFACTS is an English language string describing which artifacts to include, since the primary use of this command is to assist yarn tests. Example: edit-morph set-system-artifact-depends system.morph \ build-essential "build-essential-minimal, build-essential-runtime and build-essential-devel" ''' file_path = args[0] stratum_name = args[1] artifacts = re.split(r"\s+and\s+|,?\s*", args[2]) with self._open_yaml(file_path) as d: for spec in d["strata"]: if spec.get("alias", spec["name"]) == stratum_name: spec["artifacts"] = artifacts def cmd_set_stratum_match_rules(self, (file_path, match_rules)): '''Set a stratum's match rules. Usage: FILE_PATH MATCH_RULES_YAML This sets the stratum's "products" field, which is used to determine which chunk artifacts go into which stratum artifacts the stratum produces. The match rules must be a string that yaml can parse. ''' with self._open_yaml(file_path) as d: d['products'] = yaml.load(match_rules) @classmethod def _splice_cluster_system(cls, syslist, syspath): sysname = syspath[0] syspath = syspath[1:] for system in syslist: if sysname in system['deploy']: break else: system = { 'morph': None, 'deploy': { sysname: { 'type': None, 'location': None, }, }, } syslist.append(system) if syspath: cls._splice_cluster_system( system.setdefault('subsystems', []), syspath) @classmethod def _find_cluster_system(cls, syslist, syspath): sysname = syspath[0] syspath = syspath[1:] for system in syslist: if sysname in system['deploy']: break if syspath: return cls._find_cluster_system(system['subsystems'], syspath) return system def cmd_cluster_init(self, (cluster_file,)): with open(cluster_file, 'w') as f: d = { 'name': os.path.splitext(os.path.basename(cluster_file))[0], 'kind': 'cluster', } yaml.dump(d, f) def cmd_cluster_system_init(self, (cluster_file, system_path)): syspath = system_path.split('.') with self._open_yaml(cluster_file) as d: self._splice_cluster_system(d.setdefault('systems', []), syspath) def cmd_cluster_system_set_morphology(self, (cluster_file, system_path, morphology)): syspath = system_path.split('.') with self._open_yaml(cluster_file) as d: system = self._find_cluster_system(d['systems'], syspath) system['morph'] = morphology def cmd_cluster_system_set_deploy_type(self, (cluster_file, system_path, deploy_type)): syspath = system_path.split('.') with self._open_yaml(cluster_file) as d: system = self._find_cluster_system(d['systems'], syspath) system['deploy'][syspath[-1]]['type'] = deploy_type def cmd_cluster_system_set_deploy_location(self, (cluster_file, system_path, deploy_location)): syspath = system_path.split('.') with self._open_yaml(cluster_file) as d: system = self._find_cluster_system(d['systems'], syspath) system['deploy'][syspath[-1]]['location'] = deploy_location def cmd_cluster_system_set_deploy_variable(self, (cluster_file, system_path, key, val)): syspath = system_path.split('.') with self._open_yaml(cluster_file) as d: system = self._find_cluster_system(d['systems'], syspath) system['deploy'][syspath[-1]][key] = val EditMorph().run()