Merge remote-tracking branch 'definitions/master'pedro/openssl101t

Change-Id: I3c7b31d2006dafd8b69386cbee41d0d568b348eb

3 files changed, 203 insertions, 53 deletions

diff --git a/scripts/check-unpetrify-refs.py b/scripts/check-unpetrify-refs.py
index 27792c4a..c70b680d 100755
--- a/scripts/check-unpetrify-refs.py
+++ b/scripts/check-unpetrify-refs.py
@@ -15,11 +15,12 @@
 # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
 
 import os
-import sys
 import glob
-import yaml
+import argparse
 import subprocess
 
+import scriptslib
+
 
 '''
 Script for checking unpetrify-refs in strata.
@@ -31,14 +32,6 @@ a missing or non-existent unpetrify-ref and if it fails to check the remote
 '''
 
 strata_dir = "strata"
-trove_host = "git.baserock.org"
-aliases = {
-  'baserock:': 'git://%(trove)s/baserock/',
-  'freedesktop:': 'git://anongit.freedesktop.org/',
-  'github:': 'git://github.com/',
-  'gnome:': 'git://git.gnome.org/',
-  'upstream:': 'git://%(trove)s/delta/'
-}
 
 def ref_exists(remote, ref):
     output = subprocess.check_output(
@@ -46,43 +39,31 @@ def ref_exists(remote, ref):
         stderr=subprocess.STDOUT).strip()
     return True if output else False
 
-def get_repo_url(repo):
-    remote = repo[:repo.find(':') + 1]
-    return repo.replace(remote, aliases[remote])
-
-def definitions_root():
-    return subprocess.check_output(
-        ["git", "rev-parse", "--show-toplevel"]).strip()
-
-def load_yaml_file(yaml_file):
-    with open(yaml_file, 'r') as f:
-        return yaml.safe_load(f)
-
-def main(args):
-    global trove_host, aliases
-    opt = next(((i, j.split('=')[1]) for i, j in enumerate(args)
-              if j.startswith("--trove-host=")), None)
-    if opt:
-        trove_host = opt[1]
-        del args[opt[0]]
-    aliases = {k: v % {'trove': trove_host} for k, v in aliases.iteritems()}
+def main():
+    parser = argparse.ArgumentParser(
+        description="Sanity checks unpetrify-refs in Baserock strata")
+    parser.add_argument("--trove-host", default="git.baserock.org",
+                        help="Trove host to map repo aliases to")
+    parser.add_argument("strata", nargs="*", metavar="STRATA",
+                        help="The strata to check (checks all by default)")
+    args = parser.parse_args()
 
-    if args:
-        strata = args
+    if args.strata:
+        strata = args.strata
     else:
-        strata_path = os.path.join(definitions_root(), strata_dir)
+        strata_path = os.path.join(scriptslib.definitions_root(), strata_dir)
         strata = glob.glob("%s/*.morph" % strata_path)
 
     for stratum in strata:
         path = os.path.relpath(stratum)
-        morphology = load_yaml_file(stratum)
+        morphology = scriptslib.load_yaml_file(stratum)
         for chunk in morphology['chunks']:
             unpetrify_ref = chunk.get("unpetrify-ref")
             if not unpetrify_ref:
                 print ("%s: '%s' has no unpetrify-ref!" %
                        (path, chunk['name']))
                 continue
-            remote = get_repo_url(chunk['repo'])
+            remote = scriptslib.parse_repo_alias(chunk['repo'], args.trove_host)
             try:
                 if not ref_exists(remote, unpetrify_ref):
                     print ("%s: unpetrify-ref for '%s' is not present on the "
@@ -92,4 +73,4 @@ def main(args):
                        (path, remote, chunk['name'], e.output.strip()))
 
 if __name__ == "__main__":
-    main(sys.argv[1:])
+    main()
diff --git a/scripts/licensecheck.py b/scripts/licensecheck.py
index c5f72f07..08d0e1b4 100755
--- a/scripts/licensecheck.py
+++ b/scripts/licensecheck.py
@@ -23,10 +23,12 @@ import string
 import subprocess
 import sys
 import tempfile
-import yaml
+
+import scriptslib
 
 
 gpl3_chunks = ("autoconf",
+               "autoconf-tarball",
                "automake",
                "bash",
                "binutils",
@@ -37,10 +39,11 @@ gpl3_chunks = ("autoconf",
                "gawk",
                "gcc",
                "gdbm",
-               "gettext",
+               "gettext-tarball",
                "gperf",
                "groff",
                "libtool",
+               "libtool-tarball",
                "m4-tarball",
                "make",
                "nano",
@@ -49,16 +52,6 @@ gpl3_chunks = ("autoconf",
                "texinfo-tarball")
 
 
-def definitions_root():
-    return subprocess.check_output(
-        ["git", "rev-parse", "--show-toplevel"]).strip()
-
-
-def load_yaml_file(yaml_file):
-    with open(yaml_file, 'r') as f:
-        return yaml.safe_load(f)
-
-
 def license_file_name(repo_name, sha, licenses_dir):
     license_file = os.path.join(licenses_dir, repo_name + '-' + sha)
     return license_file
@@ -91,6 +84,8 @@ def check_repo_if_needed(name, repo, ref, repos_dir, licenses_dir):
     if repo_name.endswith(".git"):
         repo_name = repo_name[:-4]
 
+    repo_url = scriptslib.parse_repo_alias(repo)
+
     # Check if ref is sha1 to speedup
     if len(ref) == 40 and all(c in string.hexdigits for c in ref):
         license_file = license_file_name(repo_name, ref, licenses_dir)
@@ -105,13 +100,31 @@ def check_repo_if_needed(name, repo, ref, repos_dir, licenses_dir):
             subprocess.check_call([
                 "git", "remote", "update", "origin", "--prune"],
                 stderr=devnull, stdout=devnull, cwd=clone_path)
+            # Update submodules
+            subprocess.check_call(
+                ["git", "submodule", "update", "--recursive"],
+                stderr=devnull, stdout=devnull, cwd=clone_path)
             subprocess.check_call(["git", "checkout", ref], stderr=devnull,
                 stdout=devnull, cwd=clone_path)
     else:
         sys.stderr.write("Getting repo '%s' ...\n" % repo_name)
         with open(os.devnull, 'w') as devnull:
-            subprocess.check_call(["morph", "get-repo", name, clone_path],
-                stdout=devnull, stderr=devnull)
+            try:
+                # Attempt to use morph to obtain a repository, from morph's
+                # existing local git cache if possible
+                subprocess.check_call(
+                    ["morph", "get-repo", name, clone_path],
+                    stdout=devnull, stderr=devnull)
+
+            except (OSError, subprocess.CalledProcessError):
+                # Fall back to git clone, when morph hasn't been found on the
+                # system, or otherwise fails to get a repo. This is required
+                # where morph isn't available, e.g. when using YBD to build.
+                # YBD currently doesn't offer a similar 'get-repo' feature.
+                sys.stderr.write("Falling back to git clone.\n")
+                subprocess.check_call(
+                    ["git", "clone", "--recursive", repo_url, clone_path],
+                    stdout=devnull, stderr=devnull) # also clone submodules
 
     sha = subprocess.check_output(
         ["git", "rev-parse", "HEAD"], cwd=clone_path).strip()
@@ -124,7 +137,7 @@ def check_repo_if_needed(name, repo, ref, repos_dir, licenses_dir):
 
 
 def check_stratum(stratum_file, repos_dir, licenses_dir):
-    stratum = load_yaml_file(stratum_file)
+    stratum = scriptslib.load_yaml_file(stratum_file)
     license_files = []
     for chunk in stratum['chunks']:
 
@@ -152,11 +165,14 @@ def main():
 
     args = parser.parse_args()
 
-    system = load_yaml_file(args.system)
+    if not os.path.exists(args.repos_dir):
+        os.makedirs(args.repos_dir)
+
+    system = scriptslib.load_yaml_file(args.system)
     license_files = []
     for stratum in system['strata']:
         stratum_file = stratum['morph']
-        stratum_path = os.path.join(definitions_root(), stratum_file)
+        stratum_path = os.path.join(scriptslib.definitions_root(), stratum_file)
         license_files.extend(check_stratum(stratum_path, args.repos_dir, args.licenses_dir))
 
     for chunk_repo, chunk_license in license_files:
diff --git a/scripts/scriptslib.py b/scripts/scriptslib.py
new file mode 100644
index 00000000..1f79fb95
--- /dev/null
+++ b/scripts/scriptslib.py
@@ -0,0 +1,153 @@
+# Copyright (C) 2016 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# Small library of useful things for the scripts that live here.
+
+import yaml
+import subprocess
+import os
+import sys
+
+aliases = {
+  'baserock:': 'git://%(trove)s/baserock/',
+  'freedesktop:': 'git://anongit.freedesktop.org/',
+  'github:': 'git://github.com/',
+  'gnome:': 'git://git.gnome.org/',
+  'upstream:': 'git://%(trove)s/delta/'
+}
+
+def parse_repo_alias(repo, trove_host='git.baserock.org'):
+    global aliases
+    remote = repo[:repo.find(':') + 1]
+    aliases = {k: v % {'trove': trove_host} for k, v in aliases.iteritems()}
+    try:
+        return repo.replace(remote, aliases[remote])
+    except KeyError as e:
+        raise Exception("Unknown repo-alias \"%s\"" % repo)
+
+def definitions_root():
+    return subprocess.check_output(
+        ["git", "rev-parse", "--show-toplevel"]).strip()
+
+def load_yaml_file(yaml_file):
+    with open(yaml_file, 'r') as f:
+        return yaml.safe_load(f)
+
+
+class BaserockMeta(object):
+    '''An object representing Baserock metadata contained in a Baserock
+       system image, for available metadata formats'''
+
+    def __init__(self):
+        self.metas = {}
+
+    def get_each(self):
+        '''Yield an iterable for the whole list of metas'''
+        for key in self.metas:
+            yield self.metas[key]
+
+    def get_name(self, name):
+        '''Yield an iterable of metadata matched by name, e.g. `bash`'''
+        for key in self.metas:
+            if self.metas[key]['source-name'] == name:
+                yield self.metas[key]
+
+    def import_meta(self, meta_text):
+        importers = (self.import_meta_ybd,
+                     self.import_meta_morph)
+
+        for i in importers:
+            try:
+                i(meta_text)
+                return
+            except (KeyError, Exception) as err:
+                pass
+
+        # Shouldn't get here
+        sys.stderr.write('Metadata format not recognised.\n'
+                         'Error:\n')
+        raise err
+
+    def import_meta_morph(self, meta_text):
+        self._add_meta(yaml.load(meta_text))
+
+    def import_meta_ybd(self, meta_text):
+        source = yaml.load(meta_text)
+
+        null = '0' * 32
+
+        if 'repo' not in source:
+            kind = 'stratum'
+            contents = 'components'
+            source['repo'] = 'upstream:definitions'
+            source['ref'] = null # No ref info
+        else:
+            kind = 'chunk'
+            contents = 'files'
+
+        repo = parse_repo_alias(source['repo'])
+        source_name = '-'.join(
+                      source['products'][0]['artifact'].split('-')[:-1])
+
+        # Needed until YBD provides cache-key in metadata
+        if not 'cache-key' in source:
+            source['cache-key'] = null
+
+        for product in source['products']:
+
+            self._add_meta({
+                'kind':           kind,
+                'source-name':    source_name,
+                'artifact-name':  product['artifact'],
+                'contents':       product[contents],
+                'repo':           repo,
+                'repo-alias':     source['repo'],
+                'sha1':           source['ref'],
+                'original_ref':   source['ref'],
+                'cache-key':      source['cache-key']
+                })
+
+    def _add_meta(self, meta_dict):
+        '''Validate and add a meta'''
+
+        ignore = ('configuration',
+                  'system-artifact-name')
+
+        for i in ignore:
+            if i in meta_dict:
+                return
+
+        required_fields = ('repo', 'sha1', 'contents')
+        for f in required_fields:
+            if not f in meta_dict:
+                raise Exception('Metadata format not recognised, no '
+                    'value for \'%s\'. Data: \'%s\''% (f, str(meta_dict)))
+
+        self.metas[meta_dict['artifact-name']] = meta_dict
+
+
+def meta_load_from_dir(meta_dir_path):
+    '''Read Baserock metadata from a directory'''
+
+    files = [f for f in os.listdir(meta_dir_path)
+             if os.path.isfile(os.path.join(meta_dir_path, f))]
+
+    meta = BaserockMeta()
+    for f in files:
+        if f.endswith('.meta'):
+            meta.import_meta(
+                open(os.path.join(meta_dir_path, f), 'r').read())
+
+    return meta