Merge branch 'master' of git://git.baserock.org/baserock/baserock/definitions

Conflicts:
	README
	strata/python2-core/pyyaml.morph~
	strata/trove.morph

2 files changed, 280 insertions, 0 deletions

diff --git a/scripts/check-unpetrify-refs.py b/scripts/check-unpetrify-refs.py
new file mode 100755
index 00000000..27792c4a
--- /dev/null
+++ b/scripts/check-unpetrify-refs.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python
+# Copyright (C) 2016 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+import sys
+import glob
+import yaml
+import subprocess
+
+
+'''
+Script for checking unpetrify-refs in strata.
+
+Without args this script will check everything in strata/, or each stratum
+given on the command-line and will print on stdout whether a chunk has
+a missing or non-existent unpetrify-ref and if it fails to check the remote
+(missing repo).
+'''
+
+strata_dir = "strata"
+trove_host = "git.baserock.org"
+aliases = {
+  'baserock:': 'git://%(trove)s/baserock/',
+  'freedesktop:': 'git://anongit.freedesktop.org/',
+  'github:': 'git://github.com/',
+  'gnome:': 'git://git.gnome.org/',
+  'upstream:': 'git://%(trove)s/delta/'
+}
+
+def ref_exists(remote, ref):
+    output = subprocess.check_output(
+        ["git", "ls-remote", remote, str(ref)],
+        stderr=subprocess.STDOUT).strip()
+    return True if output else False
+
+def get_repo_url(repo):
+    remote = repo[:repo.find(':') + 1]
+    return repo.replace(remote, aliases[remote])
+
+def definitions_root():
+    return subprocess.check_output(
+        ["git", "rev-parse", "--show-toplevel"]).strip()
+
+def load_yaml_file(yaml_file):
+    with open(yaml_file, 'r') as f:
+        return yaml.safe_load(f)
+
+def main(args):
+    global trove_host, aliases
+    opt = next(((i, j.split('=')[1]) for i, j in enumerate(args)
+              if j.startswith("--trove-host=")), None)
+    if opt:
+        trove_host = opt[1]
+        del args[opt[0]]
+    aliases = {k: v % {'trove': trove_host} for k, v in aliases.iteritems()}
+
+    if args:
+        strata = args
+    else:
+        strata_path = os.path.join(definitions_root(), strata_dir)
+        strata = glob.glob("%s/*.morph" % strata_path)
+
+    for stratum in strata:
+        path = os.path.relpath(stratum)
+        morphology = load_yaml_file(stratum)
+        for chunk in morphology['chunks']:
+            unpetrify_ref = chunk.get("unpetrify-ref")
+            if not unpetrify_ref:
+                print ("%s: '%s' has no unpetrify-ref!" %
+                       (path, chunk['name']))
+                continue
+            remote = get_repo_url(chunk['repo'])
+            try:
+                if not ref_exists(remote, unpetrify_ref):
+                    print ("%s: unpetrify-ref for '%s' is not present on the "
+                           "remote (%s)!" % (path, chunk['name'], remote))
+            except subprocess.CalledProcessError as e:
+                print ("%s: failed to ls-remote (%s) for chunk '%s':\n%s" %
+                       (path, remote, chunk['name'], e.output.strip()))
+
+if __name__ == "__main__":
+    main(sys.argv[1:])
diff --git a/scripts/licensecheck.py b/scripts/licensecheck.py
new file mode 100755
index 00000000..c5f72f07
--- /dev/null
+++ b/scripts/licensecheck.py
@@ -0,0 +1,185 @@
+#!/usr/bin/env python
+# Copyright (C) 2016 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import argparse
+import errno
+import os
+import pipes
+import re
+import string
+import subprocess
+import sys
+import tempfile
+import yaml
+
+
+gpl3_chunks = ("autoconf",
+               "automake",
+               "bash",
+               "binutils",
+               "bison",
+               "ccache",
+               "cmake",
+               "flex",
+               "gawk",
+               "gcc",
+               "gdbm",
+               "gettext",
+               "gperf",
+               "groff",
+               "libtool",
+               "m4-tarball",
+               "make",
+               "nano",
+               "patch",
+               "rsync",
+               "texinfo-tarball")
+
+
+def definitions_root():
+    return subprocess.check_output(
+        ["git", "rev-parse", "--show-toplevel"]).strip()
+
+
+def load_yaml_file(yaml_file):
+    with open(yaml_file, 'r') as f:
+        return yaml.safe_load(f)
+
+
+def license_file_name(repo_name, sha, licenses_dir):
+    license_file = os.path.join(licenses_dir, repo_name + '-' + sha)
+    return license_file
+
+
+def check_license(repo_name, sha, clone_path, license_file):
+
+    licenses_dir = os.path.dirname(license_file)
+
+    try:
+        os.makedirs(licenses_dir)
+    except OSError as e:
+        if e.errno != errno.EEXIST:
+            raise
+
+    _, license_file_temp = tempfile.mkstemp(dir=licenses_dir)
+    with open(license_file_temp,"wb") as out:
+        sys.stderr.write("Checking license of '%s' ...\n" % repo_name)
+        with open(os.devnull, 'w') as devnull:
+            subprocess.check_call("perl scripts/licensecheck.pl -r "
+                + pipes.quote(clone_path) + "|cut -d: -f2- | sort -u",
+                stdout=out, stderr=devnull, shell=True)
+
+    os.rename(license_file_temp, license_file)
+    return license_file
+
+
+def check_repo_if_needed(name, repo, ref, repos_dir, licenses_dir):
+    repo_name = re.split('/|:',repo)[-1]
+    if repo_name.endswith(".git"):
+        repo_name = repo_name[:-4]
+
+    # Check if ref is sha1 to speedup
+    if len(ref) == 40 and all(c in string.hexdigits for c in ref):
+        license_file = license_file_name(repo_name, ref, licenses_dir)
+        if os.path.isfile(license_file):
+            return (repo, license_file)
+
+    clone_path = os.path.join(repos_dir, repo_name)
+
+    if os.path.isdir(clone_path):
+        sys.stderr.write("Updating repo '%s' ...\n" % repo_name)
+        with open(os.devnull, 'w') as devnull:
+            subprocess.check_call([
+                "git", "remote", "update", "origin", "--prune"],
+                stderr=devnull, stdout=devnull, cwd=clone_path)
+            subprocess.check_call(["git", "checkout", ref], stderr=devnull,
+                stdout=devnull, cwd=clone_path)
+    else:
+        sys.stderr.write("Getting repo '%s' ...\n" % repo_name)
+        with open(os.devnull, 'w') as devnull:
+            subprocess.check_call(["morph", "get-repo", name, clone_path],
+                stdout=devnull, stderr=devnull)
+
+    sha = subprocess.check_output(
+        ["git", "rev-parse", "HEAD"], cwd=clone_path).strip()
+
+    license_file = license_file_name(repo_name, sha, licenses_dir)
+    if os.path.isfile(license_file):
+        return (repo, license_file)
+
+    return (repo, check_license(repo_name, sha, clone_path, license_file))
+
+
+def check_stratum(stratum_file, repos_dir, licenses_dir):
+    stratum = load_yaml_file(stratum_file)
+    license_files = []
+    for chunk in stratum['chunks']:
+
+        name = chunk["name"]
+        build_mode = chunk.get("build-mode") # Allowed to be None
+
+        # Don't include bootstrap chunks and stripped gplv3 chunks
+        if name in gpl3_chunks or build_mode == "bootstrap":
+            continue
+        repo = chunk["repo"]
+        ref = chunk["ref"]
+        yield check_repo_if_needed(name, repo, ref, repos_dir, licenses_dir)
+
+
+def main():
+
+    parser = argparse.ArgumentParser(
+        description='Checks licenses of the components of a given System.')
+    parser.add_argument('system', metavar='SYSTEM', type=str,
+        help='System to check for licenses')
+    parser.add_argument('--repos-dir', default="./repos",
+            help='DIR to clone all the repos (default ./repos)')
+    parser.add_argument('--licenses-dir', default="./licenses",
+            help='DIR to store chunk license files (default ./licenses)')
+
+    args = parser.parse_args()
+
+    system = load_yaml_file(args.system)
+    license_files = []
+    for stratum in system['strata']:
+        stratum_file = stratum['morph']
+        stratum_path = os.path.join(definitions_root(), stratum_file)
+        license_files.extend(check_stratum(stratum_path, args.repos_dir, args.licenses_dir))
+
+    for chunk_repo, chunk_license in license_files:
+        try:
+            # Print repo name
+            sys.stdout.write("%s\n%s\n" % (chunk_repo, '-' * len(chunk_repo)))
+
+            # Print license file of the repo
+            with open(chunk_license, 'r') as f:
+                for line in f:
+                    sys.stdout.write(line)
+            sys.stdout.write("\n")
+        except IOError:
+        # stdout is closed, no point in continuing
+        # Attempt to close them explicitly to prevent cleanup problems:
+            try:
+                sys.stdout.flush()
+                sys.stdout.close()
+            except IOError:
+                pass
+            finally:
+                exit()
+
+
+if __name__ == "__main__":
+    main()