ci: Add an HTML report generator

The JUnit cover report is useful, but only up to a point; for instance, it's not used unless it's part of a merge request. This means you don't get a report if you're pushing to a branch that does not have an MR open. With a simple Python script and some minimal templating, we can generate an HTML report from the "I Can't Believe it's not JSON™" log that Meson produces, and keep it as a CI artifact.
author: Emmanuele Bassi <ebassi@gnome.org> 2019-04-13 14:11:30 +0100
committer: Emmanuele Bassi <ebassi@gnome.org> 2019-04-13 14:11:30 +0100
commit: 3bc8ab91a2694380fc7c46d2b53351be8e6f12f3 (patch)
tree: b7955e110ffd27d72921a8d90229366e347e9505 /.gitlab-ci
parent: f9d586977428dc775796db5d256cb610b7fcb49d (diff)
download: gtk+-3bc8ab91a2694380fc7c46d2b53351be8e6f12f3.tar.gz
3 files changed, 173 insertions, 1 deletions
diff --git a/.gitlab-ci/Dockerfile b/.gitlab-ci/Dockerfile
index 8c32e8fa5b..57b1983fda 100644
--- a/.gitlab-ci/Dockerfile
+++ b/.gitlab-ci/Dockerfile
@@ -73,6 +73,8 @@ RUN dnf -y install \
 
 RUN pip3 install meson==0.50.0
 
+RUN pip3 install jinja2
+
 ARG HOST_USER_ID=5555
 ENV HOST_USER_ID ${HOST_USER_ID}
 RUN useradd -u $HOST_USER_ID -ms /bin/bash user
diff --git a/.gitlab-ci/meson-html-report.py b/.gitlab-ci/meson-html-report.py
new file mode 100755
index 0000000000..a5d1d82409
--- /dev/null
+++ b/.gitlab-ci/meson-html-report.py
@@ -0,0 +1,164 @@
+#!/usr/bin/env python3
+
+# Copyright 2019  GNOME Foundation
+
+# Turns a test log generated by Meson into an HTML report
+
+import argparse
+import datetime
+import json
+import os
+import sys
+from jinja2 import Template
+
+REPORT_TEMPLATE = '''
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <title>{{ report.project_name }} Test Report</title>
+  <meta charset="utf-8" />
+</head>
+<body>
+  <header>
+    <h1>{{ report.project_name }} :: Test Reports</h1>
+    <div class="report-meta">
+      <p><strong>Branch:</strong> {{ report.branch_name }}</p>
+      <p><strong>Date:</strong> <time datetime="{{ report.date.isoformat() }}">{{ report.locale_date }}</time></p>
+      {% if report.job_id %}<p><strong>Job ID:</strong> {{ report.job_id }}</p>{% endif %}
+    </div>
+  </header>
+
+  <article>
+    <section>
+      <div class="summary">
+        <h3>Summary</h3>
+        <ul>
+          <li><strong>Total units:</strong> {{ report.total_units }}</li>
+          <li><strong>Passed:</strong> {{ report.total_successes }}</li>
+          <li><strong>Failed:</strong> {{ report.total_failures }}</li>
+        </u>
+      </div>
+    </section>
+
+    {% for suite_result in report.results_list %}
+    <section>
+      <div class="result">
+        <h3>Suite: {{ suite_result.suite_name }}</h3>
+        <ul>
+          <li><strong>Units:</strong> {{ suite_result.n_units }}</li>
+          <li><strong>Passed:</strong> {{ suite_result.n_successes }}</li>
+          <li><strong>Failed:</strong> {{ suite_result.n_failures }}</li>
+        </ul>
+        {% for failure in suite_result.failures %}
+            {% if loop.first %}
+        <div>
+          <h4>Failures</h4>
+          <ul>
+            {% endif %}
+            <li>{{ failure.name }} - result: <span class="failure">{{ failure.result }}</span><br/>
+            <pre>{{ failure.stdout }}</pre>
+            </li>
+            {% if loop.last %}
+          </ul>
+        </div>
+            {% endif %}
+        {% endfor %}
+    </section>
+    {% endfor %}
+
+  </article>
+</body>
+</html>
+'''
+
+aparser = argparse.ArgumentParser(description='Turns a Meson test log into an HTML report')
+aparser.add_argument('--project-name', metavar='NAME',
+                     help='The project name',
+                     default='Unknown')
+aparser.add_argument('--job-id', metavar='ID',
+                     help='The job ID for the report',
+                     default=None)
+aparser.add_argument('--branch', metavar='NAME',
+                     help='Branch of the project being tested',
+                     default='master')
+aparser.add_argument('--output', metavar='FILE',
+                     help='The output HTML file, stdout by default',
+                     type=argparse.FileType('w', encoding='UTF-8'),
+                     default=sys.stdout)
+aparser.add_argument('infile', metavar='FILE',
+                     help='The input testlog.json, stdin by default',
+                     type=argparse.FileType('r', encoding='UTF-8'),
+                     default=sys.stdin)
+
+args = aparser.parse_args()
+
+outfile = args.output
+
+suites = {}
+for line in args.infile:
+    data = json.loads(line)
+    (full_suite, unit_name) = data['name'].split(' / ')
+    (project_name, suite_name) = full_suite.split(':')
+
+    unit = {
+        'project-name': project_name,
+        'suite': suite_name,
+        'name': unit_name,
+        'duration': data['duration'],
+        'returncode': data['returncode'],
+        'result': data['result'],
+        'stdout': data['stdout'],
+    }
+
+    units = suites.setdefault(full_suite, [])
+    units.append(unit)
+
+report = {}
+report['date'] = datetime.datetime.utcnow()
+report['locale_date'] = report['date'].strftime("%c")
+report['project_name'] = args.project_name
+report['job_id'] = args.job_id
+report['branch_name'] = args.branch
+report['total_successes'] = 0
+report['total_failures'] = 0
+report['total_units'] = 0
+report['results_list'] = []
+
+for name, units in suites.items():
+    (project_name, suite_name) = name.split(':')
+    print('Processing {} suite {}:'.format(project_name, suite_name))
+
+    def if_failed(unit):
+        if unit['result'] in ['FAIL', 'TIMEOUT']:
+            return True
+        return False
+
+    def if_succeded(unit):
+        if unit['result'] in ['OK', 'EXPECTEDFAIL', 'SKIP']:
+            return True
+        return False
+
+    successes = list(filter(if_succeded, units))
+    failures = list(filter(if_failed, units))
+
+    n_units = len(units)
+    n_successes = len(successes)
+    n_failures = len(failures)
+
+    report['total_units'] += n_units
+    report['total_successes'] += n_successes
+    report['total_failures'] += n_failures
+    print(' - {}: {} total, {} pass, {} fail'.format(suite_name, n_units, n_successes, n_failures))
+
+    suite_report = {
+        'suite_name': suite_name,
+        'n_units': n_units,
+        'successes': successes,
+        'n_successes': n_successes,
+        'failures': failures,
+        'n_failures': n_failures,
+    }
+    report['results_list'].append(suite_report)
+
+template = Template(REPORT_TEMPLATE)
+outfile.write(template.render({'report': report}))
diff --git a/.gitlab-ci/test-docker.sh b/.gitlab-ci/test-docker.sh
index 95ed71edbd..45de67000a 100755
--- a/.gitlab-ci/test-docker.sh
+++ b/.gitlab-ci/test-docker.sh
@@ -37,11 +37,17 @@ xvfb-run -a -s "-screen 0 1024x768x24" \
 # Save the exit code
 exit_code=$?
 
-# We always want to run the report generator
+# We always want to run the report generators
 $srcdir/.gitlab-ci/meson-junit-report.py \
         --project-name=gtk \
         --job-id="${CI_JOB_NAME}" \
         --output=report.xml \
         meson-logs/testlog.json
 
+$srcdir/.gitlab-ci/meson-html-report.py \
+        --project-name=GTK \
+        --job-id="${CI_JOB_NAME}" \
+        --output=report.html \
+        meson-logs/testlog.json
+
 exit $exit_code
author	Emmanuele Bassi <ebassi@gnome.org>	2019-04-13 14:11:30 +0100
committer	Emmanuele Bassi <ebassi@gnome.org>	2019-04-13 14:11:30 +0100
commit	3bc8ab91a2694380fc7c46d2b53351be8e6f12f3 (patch)
tree	b7955e110ffd27d72921a8d90229366e347e9505 /.gitlab-ci
parent	f9d586977428dc775796db5d256cb610b7fcb49d (diff)
download	gtk+-3bc8ab91a2694380fc7c46d2b53351be8e6f12f3.tar.gz