From 8fdc122f65e128782eeac6f699b58a64e378bd12 Mon Sep 17 00:00:00 2001 From: Edward Cragg Date: Fri, 8 Apr 2016 14:32:30 +0100 Subject: extensions/strip-gplv3.configure: Fix to work with YBD Add an object and functions allowing metadata to be read more easily from multiple sources, either morph metadata, or with a conversion function for YBD formatted metadata. Change-Id: I94ea2425edecb4a659968134b0900081e22913d3 --- extensions/strip-gplv3.configure | 31 ++++------- scripts/scriptslib.py | 110 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 119 insertions(+), 22 deletions(-) diff --git a/extensions/strip-gplv3.configure b/extensions/strip-gplv3.configure index 2dac0907..70a08d77 100755 --- a/extensions/strip-gplv3.configure +++ b/extensions/strip-gplv3.configure @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2016 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -21,7 +21,6 @@ to find the files created by that chunk, then remove them. ''' -import json import os import re import subprocess @@ -29,6 +28,9 @@ import sys import writeexts +import imp +scriptslib = imp.load_source('scriptslib', 'scripts/scriptslib.py') + class StripGPLv3ConfigureExtension(writeexts.Extension): gplv3_chunks = [ ['autoconf', ''], @@ -59,29 +61,16 @@ class StripGPLv3ConfigureExtension(writeexts.Extension): def process_args(self, args): target_root = args[0] meta_dir = os.path.join(target_root, 'baserock') + metadata = scriptslib.meta_load_from_dir(meta_dir) for chunk in self.gplv3_chunks: - regex = os.path.join(meta_dir, "%s-[^-]\+\.meta" % chunk[0]) - artifacts = subprocess.check_output(['busybox', 'find', meta_dir, - '-regex', regex]) - - for artifact in artifacts.split(): - self.remove_chunk(target_root, artifact, chunk[1]) - - os.symlink(os.path.join(os.sep, 'bin', 'busybox'), - os.path.join(target_root, 'usr', 'bin', 'awk')) + for meta in metadata.get_name(chunk[0]): + self.remove_chunk( + target_root, reversed(meta['contents']), chunk[1]) - def remove_chunk(self, target_root, chunk, pattern): - chunk_meta_path = os.path.join(target_root, 'baserock', chunk) - - with open(chunk_meta_path, 'r') as f: - chunk_meta_data = json.load(f) - - if not 'contents' in chunk_meta_data: - raise writeexts.ExtensionError( - 'Chunk %s does not have a "contents" list' % chunk) + def remove_chunk(self, target_root, chunk_contents, pattern): updated_contents = [] - for content_entry in reversed(chunk_meta_data['contents']): + for content_entry in chunk_contents: pat = re.compile(pattern) if len(pattern) == 0 or not pat.match(content_entry): self.remove_content_entry(target_root, content_entry) diff --git a/scripts/scriptslib.py b/scripts/scriptslib.py index 68470c28..1f79fb95 100644 --- a/scripts/scriptslib.py +++ b/scripts/scriptslib.py @@ -17,7 +17,8 @@ import yaml import subprocess - +import os +import sys aliases = { 'baserock:': 'git://%(trove)s/baserock/', @@ -43,3 +44,110 @@ def definitions_root(): def load_yaml_file(yaml_file): with open(yaml_file, 'r') as f: return yaml.safe_load(f) + + +class BaserockMeta(object): + '''An object representing Baserock metadata contained in a Baserock + system image, for available metadata formats''' + + def __init__(self): + self.metas = {} + + def get_each(self): + '''Yield an iterable for the whole list of metas''' + for key in self.metas: + yield self.metas[key] + + def get_name(self, name): + '''Yield an iterable of metadata matched by name, e.g. `bash`''' + for key in self.metas: + if self.metas[key]['source-name'] == name: + yield self.metas[key] + + def import_meta(self, meta_text): + importers = (self.import_meta_ybd, + self.import_meta_morph) + + for i in importers: + try: + i(meta_text) + return + except (KeyError, Exception) as err: + pass + + # Shouldn't get here + sys.stderr.write('Metadata format not recognised.\n' + 'Error:\n') + raise err + + def import_meta_morph(self, meta_text): + self._add_meta(yaml.load(meta_text)) + + def import_meta_ybd(self, meta_text): + source = yaml.load(meta_text) + + null = '0' * 32 + + if 'repo' not in source: + kind = 'stratum' + contents = 'components' + source['repo'] = 'upstream:definitions' + source['ref'] = null # No ref info + else: + kind = 'chunk' + contents = 'files' + + repo = parse_repo_alias(source['repo']) + source_name = '-'.join( + source['products'][0]['artifact'].split('-')[:-1]) + + # Needed until YBD provides cache-key in metadata + if not 'cache-key' in source: + source['cache-key'] = null + + for product in source['products']: + + self._add_meta({ + 'kind': kind, + 'source-name': source_name, + 'artifact-name': product['artifact'], + 'contents': product[contents], + 'repo': repo, + 'repo-alias': source['repo'], + 'sha1': source['ref'], + 'original_ref': source['ref'], + 'cache-key': source['cache-key'] + }) + + def _add_meta(self, meta_dict): + '''Validate and add a meta''' + + ignore = ('configuration', + 'system-artifact-name') + + for i in ignore: + if i in meta_dict: + return + + required_fields = ('repo', 'sha1', 'contents') + for f in required_fields: + if not f in meta_dict: + raise Exception('Metadata format not recognised, no ' + 'value for \'%s\'. Data: \'%s\''% (f, str(meta_dict))) + + self.metas[meta_dict['artifact-name']] = meta_dict + + +def meta_load_from_dir(meta_dir_path): + '''Read Baserock metadata from a directory''' + + files = [f for f in os.listdir(meta_dir_path) + if os.path.isfile(os.path.join(meta_dir_path, f))] + + meta = BaserockMeta() + for f in files: + if f.endswith('.meta'): + meta.import_meta( + open(os.path.join(meta_dir_path, f), 'r').read()) + + return meta -- cgit v1.2.1