Add script to submit builds of artifacts to the server

1 files changed, 220 insertions, 0 deletions

diff --git a/scripts/submit-build b/scripts/submit-build
new file mode 100755
index 0000000..7da0750
--- /dev/null
+++ b/scripts/submit-build
@@ -0,0 +1,220 @@
+#!/usr/bin/env python
+#
+# Copyright (C) 2015 Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+
+'''Submit information on builds of Baserock artifacts.
+
+Usage: submit-build --host example.com --builder-name "Bob T. Builder" FILE [...]
+
+'''
+
+
+import requests
+
+import argparse
+import hashlib
+import logging
+import os
+import re
+import sys
+import tarfile
+
+
+class MetadataError(RuntimeError):
+    pass
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description='submit builds to morph-cache-server')
+    parser.add_argument('--builder-name', '-b', type=str, required=True)
+    parser.add_argument('--host', type=str)
+    parser.add_argument('--url', type=str)
+
+    parser.add_argument('files', metavar='files', nargs='+', type=str)
+
+    args = parser.parse_args()
+
+    if args.url and args.host:
+        raise RuntimeError("Please only specify one of --host and --url.")
+
+    if not args.url and not args.host:
+        raise RuntimeError("Please specify one of --host and --url.")
+
+    if args.url is None and args.host:
+        url = 'http://%s:8080/' % args.host
+    else:
+        url = args.url
+
+    return args.files, url, args.builder_name
+
+
+# Morph artifacts are: SHA256.type.name
+# YBD artifacts are: name.SHA256
+# This just checks that there's a SHA256 pattern in there somewhere.
+artifact_name_pattern = re.compile('.*[a-f0-9]{64}.*')
+
+
+def filename_looks_like_a_baserock_artifact(filename):
+    '''Rough heuristic to determine if a file is a Morph or YBD artifact.
+
+    This is just to avoid having lots of accidental submissions of files that
+    obviously aren't build artifacts.
+
+    FIXME: the name logic should be server-side, not client-side!
+
+    '''
+    match = artifact_name_pattern.match(os.path.basename(filename))
+    return (match is not None)
+
+
+def parse_ybd_artifact_metadata(text):
+    lines = text.splitlines()
+
+    if len(lines) < 2:
+        raise MetadataError("Not enough lines in metadata.")
+
+    repo_line, ref_line = lines[0:2]
+
+    if repo_line.startswith('repo:'):
+        repo = repo_line[6:]
+    else:
+        raise MetadataError("Expected 'repo:' on first line.")
+
+    if ref_line.startswith('ref:'):
+        ref = ref_line[5:]
+    else:
+        raise MetadataError("Expected 'ref:' on second line.")
+
+    # FIXME: this feels like a big hack. I think YBD should put the actual URL
+    # in the metadata.
+    if repo.startswith('baserock:'):
+        repo = 'git://git.baserock.org/baserock/' + repo[9:]
+    if repo.startswith('upstream:'):
+        repo = 'git://git.baserock.org/delta/' + repo[9:]
+
+    return repo, ref
+
+
+def read_metadata_from_artifact(artifact_tarfile):
+    members = artifact_tarfile.getmembers()
+    metadata_files = sorted(t for t in members if
+                            t.name.startswith('./baserock'))
+
+    if len(metadata_files) == 0:
+        raise MetadataError(
+            "No ./baserock entry in tarfile; this seems not to be a Baserock "
+            "artifact.")
+    elif len(metadata_files) == 1:
+        raise MetadataError(
+            "Found %s in tarfile, but expected a ./baserock directory and "
+            "a .meta file.", metadata_files[0].name)
+    elif len(metadata_files) == 2:
+        # Because we used sorted() above, the ./baserock directory will be
+        # first, and the .meta file second.
+        metadata_file = metadata_files[1]
+    else:
+        # It mostly ignores system artifacts because I'm not sure how to know
+        # which metadata file is the right one...
+        raise MetadataError(
+            "Looks like a system artifact, ignoring it.")
+
+    # FIXME: support Morph metadata too
+    f = artifact_tarfile.extractfile(metadata_file)
+    if f is None:
+        raise MetadataError(
+            "Unable to read %s from artifact." % metadata_file.name)
+    metadata_text = f.read()
+
+    repo, ref = parse_ybd_artifact_metadata(metadata_text)
+
+    return repo, ref
+
+
+def checksum(filename, hasher):
+    with open(filename,'rb') as f:
+        for chunk in iter(lambda: f.read(128 * hasher.block_size), b''):
+            hasher.update(chunk)
+    return hasher.hexdigest()
+
+
+def submit_build_info_for_file(url, builder_name, filename, source_repo=None,
+                               source_ref=None):
+    log = logging.getLogger()
+    log.debug("Calculating SHA1 hash of %s", filename)
+
+    # Flush the logger here because calculating the hash is pretty slow.
+    # If you are tailing the log output it's nice to know what the program
+    # is doing.
+    for handler in log.handlers:
+        handler.flush()
+
+    hash_sha1 = checksum(filename, hashlib.sha1())
+    logging.debug("Checksum for %s: %s", filename, hash_sha1)
+
+    ctime = os.stat(filename).st_ctime
+    logging.debug("Creation time of %s: %s", filename, ctime)
+
+    logging.debug("Submitting to remote cache.")
+    params = {
+        'cache_name': os.path.basename(filename),
+        'builder_name': builder_name,
+        'build_datetime': ctime,
+        'hash_sha1': hash_sha1,
+        'source_repo': source_repo,
+        'source_ref': source_ref,
+    }
+    requests.put(url + '2.0/builds', params=params)
+
+
+def run():
+    files, cache_server_url, builder_name = parse_args()
+
+    bad_files = []
+    submitted_files = []
+    for filename in files:
+
+        if not os.path.isfile(filename):
+            sys.stderr.write("%s: not a file.\n" % filename)
+            bad_files.append(filename)
+        elif not filename_looks_like_a_baserock_artifact(filename):
+            sys.stderr.write(
+                "%s: No SHA256 hash in the filename, probably not a Baserock "
+                "build artifact.\n" % filename)
+            bad_files.append(filename)
+        elif not tarfile.is_tarfile(filename):
+            sys.stderr.write(
+                "%s: Not a valid tarfile, according to Python 'tarfile' "
+                "module.\n" % filename)
+            bad_files.append(filename)
+        else:
+            try:
+                with tarfile.open(filename) as f:
+                    repo, ref = read_metadata_from_artifact(f)
+                submit_build_info_for_file(
+                    cache_server_url, builder_name, filename, repo, ref)
+                submitted_files.append(filename)
+            except MetadataError as e:
+                sys.stderr.write("%s: %s\n" % (filename, e))
+
+
+if __name__ == '__main__':
+    try:
+        logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+        run()
+    except RuntimeError as e:
+        sys.stderr.write('ERROR: %s\n' % e)
+        sys.exit(1)