gitlab-ci: add python script to submit lava jobs

Covert the job submission process to a python script for more
robustness and control. allowing easier manipulation of job data.

As a result, it adds retry logic to deal with Infrastructure Errors in LAVA.

_call_proxy() is equipped with a robust retry logic, which I have been
using already in the past few weeks in stress testing to run hundreds
of jobs.

Signed-off-by: Gustavo Padovan <gustavo.padovan@collabora.com>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/11079>

4 files changed, 196 insertions, 59 deletions

diff --git a/.gitlab-ci/generate_lava.py b/.gitlab-ci/generate_lava.py
deleted file mode 100755
index 66b898841f4..00000000000
--- a/.gitlab-ci/generate_lava.py
+++ /dev/null
@@ -1,49 +0,0 @@
-#!/usr/bin/env python3
-
-from jinja2 import Environment, FileSystemLoader
-import argparse
-import os
-import datetime
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--template")
-parser.add_argument("--pipeline-info")
-parser.add_argument("--base-artifacts-url")
-parser.add_argument("--mesa-url")
-parser.add_argument("--device-type")
-parser.add_argument("--dtb", nargs='?', default="")
-parser.add_argument("--kernel-image-name")
-parser.add_argument("--kernel-image-type", nargs='?', default="")
-parser.add_argument("--gpu-version")
-parser.add_argument("--boot-method")
-parser.add_argument("--lava-tags", nargs='?', default="")
-parser.add_argument("--env-vars", nargs='?', default="")
-parser.add_argument("--deqp-version")
-parser.add_argument("--ci-node-index")
-parser.add_argument("--ci-node-total")
-parser.add_argument("--job-type")
-args = parser.parse_args()
-
-env = Environment(loader = FileSystemLoader(os.path.dirname(args.template)), trim_blocks=True, lstrip_blocks=True)
-template = env.get_template(os.path.basename(args.template))
-
-env_vars = "%s CI_NODE_INDEX=%s CI_NODE_TOTAL=%s" % (args.env_vars, args.ci_node_index, args.ci_node_total)
-
-values = {}
-values['pipeline_info'] = args.pipeline_info
-values['base_artifacts_url'] = args.base_artifacts_url
-values['mesa_url'] = args.mesa_url
-values['device_type'] = args.device_type
-values['dtb'] = args.dtb
-values['kernel_image_name'] = args.kernel_image_name
-values['kernel_image_type'] = args.kernel_image_type
-values['gpu_version'] = args.gpu_version
-values['boot_method'] = args.boot_method
-values['tags'] = args.lava_tags
-values['env_vars'] = env_vars
-values['deqp_version'] = args.deqp_version
-
-f = open(os.path.splitext(os.path.basename(args.template))[0], "w")
-f.write(template.render(values))
-f.close()
-
diff --git a/.gitlab-ci/lava-gitlab-ci.yml b/.gitlab-ci/lava-gitlab-ci.yml
index e919a50bdbd..c36581b642f 100644
--- a/.gitlab-ci/lava-gitlab-ci.yml
+++ b/.gitlab-ci/lava-gitlab-ci.yml
@@ -21,7 +21,7 @@
     - rm -rf results
     - mkdir -p results
     - >
-      artifacts/generate_lava.py \
+      artifacts/lava_job_submitter.py \
         --template artifacts/lava.yml.jinja2 \
         --pipeline-info "$CI_PIPELINE_URL on $CI_COMMIT_REF_NAME ${CI_NODE_INDEX}/${CI_NODE_TOTAL}" \
         --base-artifacts-url ${ARTIFACTS_URL} \
@@ -36,15 +36,8 @@
         --boot-method ${BOOT_METHOD} \
         --lava-tags "${LAVA_TAGS}" \
         --ci-node-index "${CI_NODE_INDEX}" \
-        --ci-node-total "${CI_NODE_TOTAL}"
-    - lava_job_id=`lavacli jobs submit lava.yml` || lavacli jobs submit lava.yml
-    - echo $lava_job_id
+        --ci-node-total "${CI_NODE_TOTAL}" | tee results/lava.log
     - cp lava.yml results/.
-    - lavacli jobs logs $lava_job_id | tee results/lava-$lava_job_id.log
-    - lavacli jobs show $lava_job_id
-    - result=`lavacli results $lava_job_id 0_mesa mesa | head -1`
-    - echo $result
-    - '[[ "$result" == "pass" ]]'
   artifacts:
     name: "mesa_${CI_JOB_NAME}"
     when: always
diff --git a/.gitlab-ci/lava_job_submitter.py b/.gitlab-ci/lava_job_submitter.py
new file mode 100755
index 00000000000..2bd0b7322c1
--- /dev/null
+++ b/.gitlab-ci/lava_job_submitter.py
@@ -0,0 +1,193 @@
+#!/usr/bin/env python3
+#
+# Copyright (C) 2020, 2021 Collabora Limited
+# Author: Gustavo Padovan <gustavo.padovan@collabora.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a
+# copy of this software and associated documentation files (the "Software"),
+# to deal in the Software without restriction, including without limitation
+# the rights to use, copy, modify, merge, publish, distribute, sublicense,
+# and/or sell copies of the Software, and to permit persons to whom the
+# Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice (including the next
+# paragraph) shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+# THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+"""Send a job to LAVA, track it and collect log back"""
+
+import argparse
+import jinja2
+import lavacli
+import os
+import sys
+import time
+import urllib.parse
+import xmlrpc
+import yaml
+
+from datetime import datetime
+from lavacli.utils import loader
+
+
+def log_msg(msg):
+    return "{}: {}".format(datetime.now(), msg)
+
+
+def print_log(msg):
+    print(log_msg(msg))
+
+
+def generate_lava_yaml(args):
+    env = jinja2.Environment(loader = jinja2.FileSystemLoader(os.path.dirname(args.template)), trim_blocks=True, lstrip_blocks=True)
+    template = env.get_template(os.path.basename(args.template))
+
+    env_vars = "%s CI_NODE_INDEX=%s CI_NODE_TOTAL=%s" % (args.env_vars, args.ci_node_index, args.ci_node_total)
+
+    values = {}
+    values['pipeline_info'] = args.pipeline_info
+    values['base_artifacts_url'] = args.base_artifacts_url
+    values['mesa_url'] = args.mesa_url
+    values['device_type'] = args.device_type
+    values['dtb'] = args.dtb
+    values['kernel_image_name'] = args.kernel_image_name
+    values['kernel_image_type'] = args.kernel_image_type
+    values['gpu_version'] = args.gpu_version
+    values['boot_method'] = args.boot_method
+    values['tags'] = args.lava_tags
+    values['env_vars'] = env_vars
+    values['deqp_version'] = args.deqp_version
+
+    f = open(os.path.splitext(os.path.basename(args.template))[0], "w")
+    yaml = template.render(values)
+    f.write(yaml)
+    f.close()
+
+    return yaml
+
+
+def setup_lava_proxy():
+    config = lavacli.load_config("default")
+    uri, usr, tok = (config.get(key) for key in ("uri", "username", "token"))
+    uri_obj = urllib.parse.urlparse(uri)
+    uri_str = "{}://{}:{}@{}{}".format(uri_obj.scheme, usr, tok, uri_obj.netloc, uri_obj.path)
+    transport = lavacli.RequestsTransport(
+        uri_obj.scheme,
+        config.get("proxy"),
+        config.get("timeout", 120.0),
+        config.get("verify_ssl_cert", True),
+    )
+    proxy = xmlrpc.client.ServerProxy(
+        uri_str, allow_none=True, transport=transport)
+
+    print_log("Proxy for {} created.".format(config['uri']))
+
+    return proxy
+
+
+def _call_proxy(fn, *args):
+    retries = 60
+    for n in range(1, retries + 1):
+        try:
+            return fn(*args)
+        except xmlrpc.client.ProtocolError as err:
+            if n == retries:
+                sys.exit(log_msg("A protocol error occurred (Err {} {})".format(err.errcode, err.errmsg)))
+            else:
+                time.sleep(15)
+                pass
+        except xmlrpc.client.Fault as err:
+            sys.exit(log_msg("FATAL: Fault: {} (code: {})".format(err.faultString, err.faultCode)))
+
+
+def get_job_results(proxy, job_id, test_suite, test_case):
+    results_yaml = _call_proxy(proxy.results.get_testjob_results_yaml, job_id)
+    results = yaml.load(results_yaml, Loader=loader(False))
+
+    metadata = results[0]['metadata']
+    if metadata['result'] == 'fail' and 'error_type' in metadata and metadata['error_type'] == "Infrastructure":
+        print_log("LAVA job {} failed with Infrastructure Error. Retry.".format(job_id))
+        return False
+
+    results_yaml = _call_proxy(proxy.results.get_testcase_results_yaml, job_id, test_suite, test_case)
+    results = yaml.load(results_yaml, Loader=loader(False))
+    if results:
+        print_log("LAVA: result for test_suite '{}', test_case '{}': {}".format(test_suite, test_case, results[0]['result']))
+    else:
+        print_log("LAVA: no result for test_suite '{}', test_case '{}'".format(test_suite, test_case))
+
+    return True
+
+
+def follow_job_execution(proxy, job_id):
+    line_count = 0
+    finished = False
+    while not finished:
+        (finished, data) = _call_proxy(proxy.scheduler.jobs.logs, job_id, line_count)
+        logs = yaml.load(str(data), Loader=loader(False))
+        if logs:
+            for line in logs:
+                print("{} {}".format(line["dt"], line["msg"]))
+
+            line_count += len(logs)
+
+
+def show_job_data(proxy, job_id):
+    show = _call_proxy(proxy.scheduler.jobs.show, job_id)
+    for field, value in show.items():
+        print("{}\t: {}".format(field, value))
+
+
+def submit_job(proxy, job_file):
+    return _call_proxy(proxy.scheduler.jobs.submit, job_file)
+
+
+def main(args):
+    proxy = setup_lava_proxy()
+
+    yaml_file = generate_lava_yaml(args)
+
+    while True:
+        job_id = submit_job(proxy, yaml_file)
+
+        print_log("LAVA job id: {}".format(job_id))
+
+        follow_job_execution(proxy, job_id)
+
+        show_job_data(proxy, job_id)
+
+        if get_job_results(proxy,  job_id, "0_mesa", "mesa") == True:
+             break
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser("LAVA job submitter")
+
+    parser.add_argument("--template")
+    parser.add_argument("--pipeline-info")
+    parser.add_argument("--base-artifacts-url")
+    parser.add_argument("--mesa-url")
+    parser.add_argument("--device-type")
+    parser.add_argument("--dtb", nargs='?', default="")
+    parser.add_argument("--kernel-image-name")
+    parser.add_argument("--kernel-image-type", nargs='?', default="")
+    parser.add_argument("--gpu-version")
+    parser.add_argument("--boot-method")
+    parser.add_argument("--lava-tags", nargs='?', default="")
+    parser.add_argument("--env-vars", nargs='?', default="")
+    parser.add_argument("--deqp-version")
+    parser.add_argument("--ci-node-index")
+    parser.add_argument("--ci-node-total")
+    parser.add_argument("--job-type")
+
+    parser.set_defaults(func=main)
+    args = parser.parse_args()
+    args.func(args)
diff --git a/.gitlab-ci/prepare-artifacts.sh b/.gitlab-ci/prepare-artifacts.sh
index a16f6eae6e9..9acbd023c58 100755
--- a/.gitlab-ci/prepare-artifacts.sh
+++ b/.gitlab-ci/prepare-artifacts.sh
@@ -45,7 +45,7 @@ tar -cf artifacts/install.tar install
 
 if [ -n "$MINIO_ARTIFACT_NAME" ]; then
     # Pass needed files to the test stage
-    cp $CI_PROJECT_DIR/.gitlab-ci/generate_lava.py artifacts/.
+    cp $CI_PROJECT_DIR/.gitlab-ci/lava_job_submitter.py artifacts/.
     cp $CI_PROJECT_DIR/.gitlab-ci/lava.yml.jinja2 artifacts/.
 
     MINIO_ARTIFACT_NAME="$MINIO_ARTIFACT_NAME.tar.gz"