Add WIP CSV manifest generator.tlsa/manifests

It produces CSV files next to the system morph files in definitions.

You pass it a list of systems to generate manifests for.

  $ morph system-manifest . HEAD \
        systems/devel-system-x86_64-generic.morph \
        systems/xfce-system.morph

Currently the CSV contains only chunk name and version guess.

Change-Id: I712de11eaa6e1564c5bae5f5f396e5e77484972f

1 files changed, 166 insertions, 0 deletions

diff --git a/morphlib/plugins/system_manifests_plugin.py b/morphlib/plugins/system_manifests_plugin.py
new file mode 100644
index 00000000..7a461820
--- /dev/null
+++ b/morphlib/plugins/system_manifests_plugin.py
@@ -0,0 +1,166 @@
+# Copyright (C) 2014-2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# This plugin is used as part of the Baserock automated release process.
+#
+# See: <http://wiki.baserock.org/guides/release-process> for more information.
+
+import csv
+import os
+import warnings
+
+import cliapp
+import morphlib
+
+class SystemManifestsPlugin(cliapp.Plugin):
+
+    def enable(self):
+        self.app.add_subcommand(
+            'system-manifest', self.manifests,
+            arg_synopsis='REPO REF MORPH [MORPH]...')
+
+    def disable(self):
+        pass
+
+    def manifests(self, args):
+        '''Generate manifest(s) for the given system(s).
+
+        Command line arguments:
+
+        * `REPO` is a git repository URL.
+        * `REF` is a branch or other commit reference in that repository.
+        * `MORPH` is a system morphology name at that ref.
+
+        '''
+
+        if len(args) < 3:
+            raise cliapp.AppException(
+                'Wrong number of arguments to system-manifests command '
+                '(see help)')
+
+        repo, ref = args[0], args[1]
+        system_filenames = map(morphlib.util.sanitise_morphology_path,
+                               args[2:])
+
+        self.lrc, self.rrc = morphlib.util.new_repo_caches(self.app)
+        self.resolver = morphlib.artifactresolver.ArtifactResolver()
+
+        for system_filename in system_filenames:
+            self.system_manifest(repo, ref, system_filename)
+
+    def system_manifest(self, repo, ref, system_filename):
+        '''Generate manifest for given system.'''
+
+        self.app.status(
+            msg='Creating source pool for %s' % system_filename, chatty=True)
+        source_pool = morphlib.sourceresolver.create_source_pool(
+            self.lrc, self.rrc, repo, ref, system_filename,
+            cachedir=self.app.settings['cachedir'],
+            update_repos = not self.app.settings['no-git-update'],
+            status_cb=self.app.status)
+
+        self.app.status(
+            msg='Resolving artifacts for %s' % system_filename, chatty=True)
+        root_artifacts = self.resolver.resolve_root_artifacts(source_pool)
+
+        def find_artifact_by_name(artifacts_list, filename):
+            for a in artifacts_list:
+                if a.source.filename == filename:
+                    return a
+            raise ValueError
+
+        system_artifact = find_artifact_by_name(root_artifacts,
+                                                system_filename)
+
+        self.app.status(
+            msg='Computing cache keys for %s' % system_filename, chatty=True)
+        build_env = morphlib.buildenvironment.BuildEnvironment(
+            self.app.settings, system_artifact.source.morphology['arch'])
+        ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env)
+
+        aliases = self.app.settings['repo-alias']
+        resolver = morphlib.repoaliasresolver.RepoAliasResolver(aliases)
+
+        manifest = Manifest(system_filename)
+
+        for source in set(a.source for a in system_artifact.walk()):
+            source.cache_key = ckc.compute_key(source)
+            source.cache_id = ckc.get_cache_id(source)
+
+            if source.morphology['kind'] != 'chunk':
+                continue
+
+            name = source.morphology['name']
+            ref = source.original_ref
+
+            # Ensure we have a cache of the repo
+            if not self.lrc.has_repo(source.repo_name):
+                self.lrc.cache_repo(source.repo_name)
+
+            cached = self.lrc.get_repo(source.repo_name)
+
+            manifest.add_chunk(name, ref, cached)
+
+def get_main_license(dir): # pragma: no cover
+    license = 'UNKNOWN'
+    if os.path.exists(dir + '/COPYING'):
+        license_file = dir + '/COPYING'
+    elif os.path.exists(dir + '/LICENSE'):
+        license_file = dir + '/LICENSE'
+    else:
+        return license
+
+    # TODO: Some sensible place to get this functionality from
+    licensecheck_path = os.path.join(os.getcwd(),
+                                     'scripts/licensecheck.pl')
+    output = cliapp.runcmd(['perl', licensecheck_path, '-l',
+                            '500' , license_file])
+    if output is not None:
+        license = output[(len(license_file) + 2):].strip()
+
+    return license
+
+def get_all_licenses(dir): # pragma: no cover
+    license_list = 'UNKNOWN'
+    # TODO: Return list of all unique licenses used in all the files
+    #       in the directroy tree provided
+
+    return license_list
+
+class Manifest:
+    """
+    Writes out a manifest of what's included in a system
+    """
+
+    def __init__(self, system_name):
+        path = os.path.join(os.getcwd(), system_name + '-manifest.csv')
+        print('Creating {}'.format(path))
+        self.file = open(path, 'wb')
+        self.writer = csv.writer(self.file, quoting=csv.QUOTE_ALL)
+
+    def _write_chunk(self, chunk_name, version_guess,
+                     license, license_list, upstream):
+        self.writer.writerow([chunk_name, version_guess,
+                              license, license_list, upstream])
+
+    def add_chunk(self, chunk_name, ref, cached_repo):
+        print('  Inspecting chunk: {}'.format(chunk_name))
+        version_guess = cached_repo.version_guess(ref)
+        # TODO: Clone cached repo into temp dir
+        license = None#get_main_license(dir)
+        license_list = None#get_all_licenses(dir)
+        upstream = None#get_upstream_address(dir)
+        self._write_chunk(chunk_name, version_guess, license,
+                          license_list, upstream)