From b36f03f8a12c2ea4131bd79baa045643c3f9112f Mon Sep 17 00:00:00 2001 From: Michael Drake Date: Mon, 13 Apr 2015 14:19:19 +0000 Subject: Add WIP CSV manifest generator. It produces CSV files next to the system morph files in definitions. You pass it a list of systems to generate manifests for. $ morph system-manifest . HEAD \ systems/devel-system-x86_64-generic.morph \ systems/xfce-system.morph Currently the CSV contains only chunk name and version guess. Change-Id: I712de11eaa6e1564c5bae5f5f396e5e77484972f --- morphlib/plugins/system_manifests_plugin.py | 166 ++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 morphlib/plugins/system_manifests_plugin.py diff --git a/morphlib/plugins/system_manifests_plugin.py b/morphlib/plugins/system_manifests_plugin.py new file mode 100644 index 00000000..7a461820 --- /dev/null +++ b/morphlib/plugins/system_manifests_plugin.py @@ -0,0 +1,166 @@ +# Copyright (C) 2014-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# This plugin is used as part of the Baserock automated release process. +# +# See: for more information. + +import csv +import os +import warnings + +import cliapp +import morphlib + +class SystemManifestsPlugin(cliapp.Plugin): + + def enable(self): + self.app.add_subcommand( + 'system-manifest', self.manifests, + arg_synopsis='REPO REF MORPH [MORPH]...') + + def disable(self): + pass + + def manifests(self, args): + '''Generate manifest(s) for the given system(s). + + Command line arguments: + + * `REPO` is a git repository URL. + * `REF` is a branch or other commit reference in that repository. + * `MORPH` is a system morphology name at that ref. + + ''' + + if len(args) < 3: + raise cliapp.AppException( + 'Wrong number of arguments to system-manifests command ' + '(see help)') + + repo, ref = args[0], args[1] + system_filenames = map(morphlib.util.sanitise_morphology_path, + args[2:]) + + self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app) + self.resolver = morphlib.artifactresolver.ArtifactResolver() + + for system_filename in system_filenames: + self.system_manifest(repo, ref, system_filename) + + def system_manifest(self, repo, ref, system_filename): + '''Generate manifest for given system.''' + + self.app.status( + msg='Creating source pool for %s' % system_filename, chatty=True) + source_pool = morphlib.sourceresolver.create_source_pool( + self.lrc, self.rrc, repo, ref, system_filename, + cachedir=self.app.settings['cachedir'], + update_repos = not self.app.settings['no-git-update'], + status_cb=self.app.status) + + self.app.status( + msg='Resolving artifacts for %s' % system_filename, chatty=True) + root_artifacts = self.resolver.resolve_root_artifacts(source_pool) + + def find_artifact_by_name(artifacts_list, filename): + for a in artifacts_list: + if a.source.filename == filename: + return a + raise ValueError + + system_artifact = find_artifact_by_name(root_artifacts, + system_filename) + + self.app.status( + msg='Computing cache keys for %s' % system_filename, chatty=True) + build_env = morphlib.buildenvironment.BuildEnvironment( + self.app.settings, system_artifact.source.morphology['arch']) + ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) + + aliases = self.app.settings['repo-alias'] + resolver = morphlib.repoaliasresolver.RepoAliasResolver(aliases) + + manifest = Manifest(system_filename) + + for source in set(a.source for a in system_artifact.walk()): + source.cache_key = ckc.compute_key(source) + source.cache_id = ckc.get_cache_id(source) + + if source.morphology['kind'] != 'chunk': + continue + + name = source.morphology['name'] + ref = source.original_ref + + # Ensure we have a cache of the repo + if not self.lrc.has_repo(source.repo_name): + self.lrc.cache_repo(source.repo_name) + + cached = self.lrc.get_repo(source.repo_name) + + manifest.add_chunk(name, ref, cached) + +def get_main_license(dir): # pragma: no cover + license = 'UNKNOWN' + if os.path.exists(dir + '/COPYING'): + license_file = dir + '/COPYING' + elif os.path.exists(dir + '/LICENSE'): + license_file = dir + '/LICENSE' + else: + return license + + # TODO: Some sensible place to get this functionality from + licensecheck_path = os.path.join(os.getcwd(), + 'scripts/licensecheck.pl') + output = cliapp.runcmd(['perl', licensecheck_path, '-l', + '500' , license_file]) + if output is not None: + license = output[(len(license_file) + 2):].strip() + + return license + +def get_all_licenses(dir): # pragma: no cover + license_list = 'UNKNOWN' + # TODO: Return list of all unique licenses used in all the files + # in the directroy tree provided + + return license_list + +class Manifest: + """ + Writes out a manifest of what's included in a system + """ + + def __init__(self, system_name): + path = os.path.join(os.getcwd(), system_name + '-manifest.csv') + print('Creating {}'.format(path)) + self.file = open(path, 'wb') + self.writer = csv.writer(self.file, quoting=csv.QUOTE_ALL) + + def _write_chunk(self, chunk_name, version_guess, + license, license_list, upstream): + self.writer.writerow([chunk_name, version_guess, + license, license_list, upstream]) + + def add_chunk(self, chunk_name, ref, cached_repo): + print(' Inspecting chunk: {}'.format(chunk_name)) + version_guess = cached_repo.version_guess(ref) + # TODO: Clone cached repo into temp dir + license = None#get_main_license(dir) + license_list = None#get_all_licenses(dir) + upstream = None#get_upstream_address(dir) + self._write_chunk(chunk_name, version_guess, license, + license_list, upstream) -- cgit v1.2.1