From a04720dc1c16bee72a18609069387df0862f00bd Mon Sep 17 00:00:00 2001 From: David Heidelberg Date: Sat, 17 Dec 2022 17:34:36 +0100 Subject: ci: move the CI handling scripts into bin/ci Any change in .gitlab-ci/ directory triggers an entire CI run, which is unnecessary for just changing the developers' scripts. Added symlink to `.gitlab-ci/bin` to keep original invocation compatibility. Reviewed-by: Martin Roukala Reviewed-by: Guilherme Gallo Signed-off-by: David Heidelberg Part-of: --- bin/ci/.gitignore | 2 + bin/ci/ci_run_n_monitor.py | 308 +++++++++++++++++++++++++++++++++++++++ bin/ci/download_gl_schema.sh | 11 ++ bin/ci/gitlab_common.py | 42 ++++++ bin/ci/gitlab_gql.py | 303 ++++++++++++++++++++++++++++++++++++++ bin/ci/job_details.gql | 7 + bin/ci/pipeline_details.gql | 86 +++++++++++ bin/ci/requirements.txt | 8 + bin/ci/update_traces_checksum.py | 142 ++++++++++++++++++ 9 files changed, 909 insertions(+) create mode 100644 bin/ci/.gitignore create mode 100755 bin/ci/ci_run_n_monitor.py create mode 100755 bin/ci/download_gl_schema.sh create mode 100644 bin/ci/gitlab_common.py create mode 100755 bin/ci/gitlab_gql.py create mode 100644 bin/ci/job_details.gql create mode 100644 bin/ci/pipeline_details.gql create mode 100644 bin/ci/requirements.txt create mode 100755 bin/ci/update_traces_checksum.py (limited to 'bin') diff --git a/bin/ci/.gitignore b/bin/ci/.gitignore new file mode 100644 index 00000000000..9739d32aaae --- /dev/null +++ b/bin/ci/.gitignore @@ -0,0 +1,2 @@ +schema.graphql +gitlab_gql.py.cache.db diff --git a/bin/ci/ci_run_n_monitor.py b/bin/ci/ci_run_n_monitor.py new file mode 100755 index 00000000000..3e497e6dec0 --- /dev/null +++ b/bin/ci/ci_run_n_monitor.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +# Copyright © 2020 - 2022 Collabora Ltd. +# Authors: +# Tomeu Vizoso +# David Heidelberg +# +# SPDX-License-Identifier: MIT + +""" +Helper script to restrict running only required CI jobs +and show the job(s) logs. +""" + +import argparse +import re +from subprocess import check_output +import sys +import time +from concurrent.futures import ThreadPoolExecutor +from functools import partial +from itertools import chain +from typing import Optional + +import gitlab +from colorama import Fore, Style +from gitlab_common import get_gitlab_project, read_token, wait_for_pipeline +from gitlab_gql import GitlabGQL, create_job_needs_dag, filter_dag, print_dag + +REFRESH_WAIT_LOG = 10 +REFRESH_WAIT_JOBS = 6 + +URL_START = "\033]8;;" +URL_END = "\033]8;;\a" + +STATUS_COLORS = { + "created": "", + "running": Fore.BLUE, + "success": Fore.GREEN, + "failed": Fore.RED, + "canceled": Fore.MAGENTA, + "manual": "", + "pending": "", + "skipped": "", +} + +COMPLETED_STATUSES = ["success", "failed"] + + +def print_job_status(job) -> None: + """It prints a nice, colored job status with a link to the job.""" + if job.status == "canceled": + return + + print( + STATUS_COLORS[job.status] + + "🞋 job " + + URL_START + + f"{job.web_url}\a{job.name}" + + URL_END + + f" :: {job.status}" + + Style.RESET_ALL + ) + + +def print_job_status_change(job) -> None: + """It reports job status changes.""" + if job.status == "canceled": + return + + print( + STATUS_COLORS[job.status] + + "🗘 job " + + URL_START + + f"{job.web_url}\a{job.name}" + + URL_END + + f" has new status: {job.status}" + + Style.RESET_ALL + ) + + +def pretty_wait(sec: int) -> None: + """shows progressbar in dots""" + for val in range(sec, 0, -1): + print(f"⏲ {val} seconds", end="\r") + time.sleep(1) + + +def monitor_pipeline( + project, + pipeline, + target_job: Optional[str], + dependencies, + force_manual: bool, + stress: bool, +) -> tuple[Optional[int], Optional[int]]: + """Monitors pipeline and delegate canceling jobs""" + statuses = {} + target_statuses = {} + stress_succ = 0 + stress_fail = 0 + + if target_job: + target_jobs_regex = re.compile(target_job.strip()) + + while True: + to_cancel = [] + for job in pipeline.jobs.list(all=True, sort="desc"): + # target jobs + if target_job and target_jobs_regex.match(job.name): + if force_manual and job.status == "manual": + enable_job(project, job, True) + + if stress and job.status in ["success", "failed"]: + if job.status == "success": + stress_succ += 1 + if job.status == "failed": + stress_fail += 1 + retry_job(project, job) + + if (job.id not in target_statuses) or ( + job.status not in target_statuses[job.id] + ): + print_job_status_change(job) + target_statuses[job.id] = job.status + else: + print_job_status(job) + + continue + + # all jobs + if (job.id not in statuses) or (job.status not in statuses[job.id]): + print_job_status_change(job) + statuses[job.id] = job.status + + # dependencies and cancelling the rest + if job.name in dependencies: + if job.status == "manual": + enable_job(project, job, False) + + elif target_job and job.status not in [ + "canceled", + "success", + "failed", + "skipped", + ]: + to_cancel.append(job) + + if target_job: + cancel_jobs(project, to_cancel) + + if stress: + print( + "∑ succ: " + str(stress_succ) + "; fail: " + str(stress_fail), + flush=False, + ) + pretty_wait(REFRESH_WAIT_JOBS) + continue + + print("---------------------------------", flush=False) + + if len(target_statuses) == 1 and {"running"}.intersection( + target_statuses.values() + ): + return next(iter(target_statuses)), None + + if {"failed", "canceled"}.intersection(target_statuses.values()): + return None, 1 + + if {"success", "manual"}.issuperset(target_statuses.values()): + return None, 0 + + pretty_wait(REFRESH_WAIT_JOBS) + + +def enable_job(project, job, target: bool) -> None: + """enable manual job""" + pjob = project.jobs.get(job.id, lazy=True) + pjob.play() + if target: + jtype = "🞋 " + else: + jtype = "(dependency)" + print(Fore.MAGENTA + f"{jtype} job {job.name} manually enabled" + Style.RESET_ALL) + + +def retry_job(project, job) -> None: + """retry job""" + pjob = project.jobs.get(job.id, lazy=True) + pjob.retry() + jtype = "↻" + print(Fore.MAGENTA + f"{jtype} job {job.name} manually enabled" + Style.RESET_ALL) + + +def cancel_job(project, job) -> None: + """Cancel GitLab job""" + pjob = project.jobs.get(job.id, lazy=True) + pjob.cancel() + print(f"♲ {job.name}") + + +def cancel_jobs(project, to_cancel) -> None: + """Cancel unwanted GitLab jobs""" + if not to_cancel: + return + + with ThreadPoolExecutor(max_workers=6) as exe: + part = partial(cancel_job, project) + exe.map(part, to_cancel) + + +def print_log(project, job_id) -> None: + """Print job log into output""" + printed_lines = 0 + while True: + job = project.jobs.get(job_id) + + # GitLab's REST API doesn't offer pagination for logs, so we have to refetch it all + lines = job.trace().decode("raw_unicode_escape").splitlines() + for line in lines[printed_lines:]: + print(line) + printed_lines = len(lines) + + if job.status in COMPLETED_STATUSES: + print(Fore.GREEN + f"Job finished: {job.web_url}" + Style.RESET_ALL) + return + pretty_wait(REFRESH_WAIT_LOG) + + +def parse_args() -> None: + """Parse args""" + parser = argparse.ArgumentParser( + description="Tool to trigger a subset of container jobs " + + "and monitor the progress of a test job", + epilog="Example: mesa-monitor.py --rev $(git rev-parse HEAD) " + + '--target ".*traces" ', + ) + parser.add_argument("--target", metavar="target-job", help="Target job") + parser.add_argument( + "--rev", metavar="revision", help="repository git revision (default: HEAD)" + ) + parser.add_argument( + "--token", + metavar="token", + help="force GitLab token, otherwise it's read from ~/.config/gitlab-token", + ) + parser.add_argument( + "--force-manual", action="store_true", help="Force jobs marked as manual" + ) + parser.add_argument("--stress", action="store_true", help="Stresstest job(s)") + return parser.parse_args() + + +def find_dependencies(target_job: str, project_path: str, sha: str) -> set[str]: + gql_instance = GitlabGQL() + dag, _ = create_job_needs_dag( + gql_instance, {"projectPath": project_path.path_with_namespace, "sha": sha} + ) + + target_dep_dag = filter_dag(dag, target_job) + if not target_dep_dag: + print(Fore.RED + "The job(s) were not found in the pipeline." + Fore.RESET) + sys.exit(1) + print(Fore.YELLOW) + print("Detected job dependencies:") + print() + print_dag(target_dep_dag) + print(Fore.RESET) + return set(chain.from_iterable(target_dep_dag.values())) + + +if __name__ == "__main__": + try: + t_start = time.perf_counter() + + args = parse_args() + + token = read_token(args.token) + + gl = gitlab.Gitlab(url="https://gitlab.freedesktop.org", private_token=token) + + cur_project = get_gitlab_project(gl, "mesa") + + REV: str = args.rev + if not REV: + REV = check_output(['git', 'rev-parse', 'HEAD']).decode('ascii').strip() + print(f"Revision: {REV}") + pipe = wait_for_pipeline(cur_project, REV) + print(f"Pipeline: {pipe.web_url}") + deps = set() + if args.target: + print("🞋 job: " + Fore.BLUE + args.target + Style.RESET_ALL) + deps = find_dependencies( + target_job=args.target, sha=REV, project_path=cur_project + ) + target_job_id, ret = monitor_pipeline( + cur_project, pipe, args.target, deps, args.force_manual, args.stress + ) + + if target_job_id: + print_log(cur_project, target_job_id) + + t_end = time.perf_counter() + spend_minutes = (t_end - t_start) / 60 + print(f"⏲ Duration of script execution: {spend_minutes:0.1f} minutes") + + sys.exit(ret) + except KeyboardInterrupt: + sys.exit(1) diff --git a/bin/ci/download_gl_schema.sh b/bin/ci/download_gl_schema.sh new file mode 100755 index 00000000000..41858df4c12 --- /dev/null +++ b/bin/ci/download_gl_schema.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +# Helper script to download the schema GraphQL from Gitlab to enable IDEs to +# assist the developer to edit gql files + +SOURCE_DIR=$(dirname "$(realpath "$0")") + +( + cd $SOURCE_DIR || exit 1 + gql-cli https://gitlab.freedesktop.org/api/graphql --print-schema > schema.graphql +) diff --git a/bin/ci/gitlab_common.py b/bin/ci/gitlab_common.py new file mode 100644 index 00000000000..85313cc1deb --- /dev/null +++ b/bin/ci/gitlab_common.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 +# Copyright © 2020 - 2022 Collabora Ltd. +# Authors: +# Tomeu Vizoso +# David Heidelberg +# +# SPDX-License-Identifier: MIT +'''Shared functions between the scripts.''' + +import os +import time +from typing import Optional + + +def get_gitlab_project(glab, name: str): + """Finds a specified gitlab project for given user""" + glab.auth() + username = glab.user.username + return glab.projects.get(f"{username}/mesa") + + +def read_token(token_arg: Optional[str]) -> str: + """pick token from args or file""" + if token_arg: + return token_arg + return ( + open(os.path.expanduser("~/.config/gitlab-token"), encoding="utf-8") + .readline() + .rstrip() + ) + + +def wait_for_pipeline(project, sha: str): + """await until pipeline appears in Gitlab""" + print("⏲ for the pipeline to appear..", end="") + while True: + pipelines = project.pipelines.list(sha=sha) + if pipelines: + print("", flush=True) + return pipelines[0] + print("", end=".", flush=True) + time.sleep(1) diff --git a/bin/ci/gitlab_gql.py b/bin/ci/gitlab_gql.py new file mode 100755 index 00000000000..bd58f320b42 --- /dev/null +++ b/bin/ci/gitlab_gql.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 + +import re +from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser, Namespace +from dataclasses import dataclass, field +from os import getenv +from pathlib import Path +from typing import Any, Iterable, Optional, Pattern, Union + +import yaml +from filecache import DAY, filecache +from gql import Client, gql +from gql.transport.aiohttp import AIOHTTPTransport +from graphql import DocumentNode + +Dag = dict[str, list[str]] +TOKEN_DIR = Path(getenv("XDG_CONFIG_HOME") or Path.home() / ".config") + + +def get_token_from_default_dir() -> str: + try: + token_file = TOKEN_DIR / "gitlab-token" + return token_file.resolve() + except FileNotFoundError as ex: + print( + f"Could not find {token_file}, please provide a token file as an argument" + ) + raise ex + + +def get_project_root_dir(): + root_path = Path(__file__).parent.parent.parent.resolve() + gitlab_file = root_path / ".gitlab-ci.yml" + assert gitlab_file.exists() + + return root_path + + +@dataclass +class GitlabGQL: + _transport: Any = field(init=False) + client: Client = field(init=False) + url: str = "https://gitlab.freedesktop.org/api/graphql" + token: Optional[str] = None + + def __post_init__(self): + self._setup_gitlab_gql_client() + + def _setup_gitlab_gql_client(self) -> Client: + # Select your transport with a defined url endpoint + headers = {} + if self.token: + headers["Authorization"] = f"Bearer {self.token}" + self._transport = AIOHTTPTransport(url=self.url, headers=headers) + + # Create a GraphQL client using the defined transport + self.client = Client( + transport=self._transport, fetch_schema_from_transport=True + ) + + @filecache(DAY) + def query( + self, gql_file: Union[Path, str], params: dict[str, Any] + ) -> dict[str, Any]: + # Provide a GraphQL query + source_path = Path(__file__).parent + pipeline_query_file = source_path / gql_file + + query: DocumentNode + with open(pipeline_query_file, "r") as f: + pipeline_query = f.read() + query = gql(pipeline_query) + + # Execute the query on the transport + return self.client.execute(query, variable_values=params) + + def invalidate_query_cache(self): + self.query._db.clear() + + +def create_job_needs_dag( + gl_gql: GitlabGQL, params +) -> tuple[Dag, dict[str, dict[str, Any]]]: + + result = gl_gql.query("pipeline_details.gql", params) + dag = {} + jobs = {} + pipeline = result["project"]["pipeline"] + if not pipeline: + raise RuntimeError(f"Could not find any pipelines for {params}") + + for stage in pipeline["stages"]["nodes"]: + for stage_job in stage["groups"]["nodes"]: + for job in stage_job["jobs"]["nodes"]: + needs = job.pop("needs")["nodes"] + jobs[job["name"]] = job + dag[job["name"]] = {node["name"] for node in needs} + + for job, needs in dag.items(): + needs: set + partial = True + + while partial: + next_depth = {n for dn in needs for n in dag[dn]} + partial = not needs.issuperset(next_depth) + needs = needs.union(next_depth) + + dag[job] = needs + + return dag, jobs + + +def filter_dag(dag: Dag, regex: Pattern) -> Dag: + return {job: needs for job, needs in dag.items() if re.match(regex, job)} + + +def print_dag(dag: Dag) -> None: + for job, needs in dag.items(): + print(f"{job}:") + print(f"\t{' '.join(needs)}") + print() + + +def fetch_merged_yaml(gl_gql: GitlabGQL, params) -> dict[Any]: + gitlab_yml_file = get_project_root_dir() / ".gitlab-ci.yml" + content = Path(gitlab_yml_file).read_text().strip() + params["content"] = content + raw_response = gl_gql.query("job_details.gql", params) + if merged_yaml := raw_response["ciConfig"]["mergedYaml"]: + return yaml.safe_load(merged_yaml) + + gl_gql.invalidate_query_cache() + raise ValueError( + """ + Could not fetch any content for merged YAML, + please verify if the git SHA exists in remote. + Maybe you forgot to `git push`? """ + ) + + +def recursive_fill(job, relationship_field, target_data, acc_data: dict, merged_yaml): + if relatives := job.get(relationship_field): + if isinstance(relatives, str): + relatives = [relatives] + + for relative in relatives: + parent_job = merged_yaml[relative] + acc_data = recursive_fill(parent_job, acc_data, merged_yaml) + + acc_data |= job.get(target_data, {}) + + return acc_data + + +def get_variables(job, merged_yaml, project_path, sha) -> dict[str, str]: + p = get_project_root_dir() / ".gitlab-ci" / "image-tags.yml" + image_tags = yaml.safe_load(p.read_text()) + + variables = image_tags["variables"] + variables |= merged_yaml["variables"] + variables |= job["variables"] + variables["CI_PROJECT_PATH"] = project_path + variables["CI_PROJECT_NAME"] = project_path.split("/")[1] + variables["CI_REGISTRY_IMAGE"] = "registry.freedesktop.org/${CI_PROJECT_PATH}" + variables["CI_COMMIT_SHA"] = sha + + while recurse_among_variables_space(variables): + pass + + return variables + + +# Based on: https://stackoverflow.com/a/2158532/1079223 +def flatten(xs): + for x in xs: + if isinstance(x, Iterable) and not isinstance(x, (str, bytes)): + yield from flatten(x) + else: + yield x + + +def get_full_script(job) -> list[str]: + script = [] + for script_part in ("before_script", "script", "after_script"): + script.append(f"# {script_part}") + lines = flatten(job.get(script_part, [])) + script.extend(lines) + script.append("") + + return script + + +def recurse_among_variables_space(var_graph) -> bool: + updated = False + for var, value in var_graph.items(): + value = str(value) + dep_vars = [] + if match := re.findall(r"(\$[{]?[\w\d_]*[}]?)", value): + all_dep_vars = [v.lstrip("${").rstrip("}") for v in match] + # print(value, match, all_dep_vars) + dep_vars = [v for v in all_dep_vars if v in var_graph] + + for dep_var in dep_vars: + dep_value = str(var_graph[dep_var]) + new_value = var_graph[var] + new_value = new_value.replace(f"${{{dep_var}}}", dep_value) + new_value = new_value.replace(f"${dep_var}", dep_value) + var_graph[var] = new_value + updated |= dep_value != new_value + + return updated + + +def get_job_final_definiton(job_name, merged_yaml, project_path, sha): + job = merged_yaml[job_name] + variables = get_variables(job, merged_yaml, project_path, sha) + + print("# --------- variables ---------------") + for var, value in sorted(variables.items()): + print(f"export {var}={value!r}") + + # TODO: Recurse into needs to get full script + # TODO: maybe create a extra yaml file to avoid too much rework + script = get_full_script(job) + print() + print() + print("# --------- full script ---------------") + print("\n".join(script)) + + if image := variables.get("MESA_IMAGE"): + print() + print() + print("# --------- container image ---------------") + print(image) + + +def parse_args() -> Namespace: + parser = ArgumentParser( + formatter_class=ArgumentDefaultsHelpFormatter, + description="CLI and library with utility functions to debug jobs via Gitlab GraphQL", + epilog=f"""Example: + {Path(__file__).name} --rev $(git rev-parse HEAD) --print-job-dag""", + ) + parser.add_argument("-pp", "--project-path", type=str, default="mesa/mesa") + parser.add_argument("--sha", "--rev", type=str, required=True) + parser.add_argument( + "--regex", + type=str, + required=False, + help="Regex pattern for the job name to be considered", + ) + parser.add_argument("--print-dag", action="store_true", help="Print job needs DAG") + parser.add_argument( + "--print-merged-yaml", + action="store_true", + help="Print the resulting YAML for the specific SHA", + ) + parser.add_argument( + "--print-job-manifest", type=str, help="Print the resulting job data" + ) + parser.add_argument( + "--gitlab-token-file", + type=str, + default=get_token_from_default_dir(), + help="force GitLab token, otherwise it's read from $XDG_CONFIG_HOME/gitlab-token", + ) + + args = parser.parse_args() + args.gitlab_token = Path(args.gitlab_token_file).read_text() + return args + + +def main(): + args = parse_args() + gl_gql = GitlabGQL(token=args.gitlab_token) + + if args.print_dag: + dag, jobs = create_job_needs_dag( + gl_gql, {"projectPath": args.project_path, "sha": args.sha} + ) + + if args.regex: + dag = filter_dag(dag, re.compile(args.regex)) + print_dag(dag) + + if args.print_merged_yaml: + print( + fetch_merged_yaml( + gl_gql, {"projectPath": args.project_path, "sha": args.sha} + ) + ) + + if args.print_job_manifest: + merged_yaml = fetch_merged_yaml( + gl_gql, {"projectPath": args.project_path, "sha": args.sha} + ) + get_job_final_definiton( + args.print_job_manifest, merged_yaml, args.project_path, args.sha + ) + + +if __name__ == "__main__": + main() diff --git a/bin/ci/job_details.gql b/bin/ci/job_details.gql new file mode 100644 index 00000000000..8b8e4b2b885 --- /dev/null +++ b/bin/ci/job_details.gql @@ -0,0 +1,7 @@ +query getCiConfigData($projectPath: ID!, $sha: String, $content: String!) { + ciConfig(projectPath: $projectPath, sha: $sha, content: $content) { + errors + mergedYaml + __typename + } +} diff --git a/bin/ci/pipeline_details.gql b/bin/ci/pipeline_details.gql new file mode 100644 index 00000000000..e735ed8d396 --- /dev/null +++ b/bin/ci/pipeline_details.gql @@ -0,0 +1,86 @@ +fragment LinkedPipelineData on Pipeline { + id + iid + path + cancelable + retryable + userPermissions { + updatePipeline + } + status: detailedStatus { + id + group + label + icon + } + sourceJob { + id + name + } + project { + id + name + fullPath + } +} + +query getPipelineDetails($projectPath: ID!, $sha: String!) { + project(fullPath: $projectPath) { + id + pipeline(sha: $sha) { + id + iid + complete + downstream { + nodes { + ...LinkedPipelineData + } + } + upstream { + ...LinkedPipelineData + } + stages { + nodes { + id + name + status: detailedStatus { + id + action { + id + icon + path + title + } + } + groups { + nodes { + id + status: detailedStatus { + id + label + group + icon + } + name + size + jobs { + nodes { + id + name + kind + scheduledAt + needs { + nodes { + id + name + } + } + } + } + } + } + } + } + } + } +} diff --git a/bin/ci/requirements.txt b/bin/ci/requirements.txt new file mode 100644 index 00000000000..5b67ec7a1ef --- /dev/null +++ b/bin/ci/requirements.txt @@ -0,0 +1,8 @@ +aiohttp==3.8.1 +colorama==0.4.5 +filecache==0.81 +gql==3.4.0 +python-gitlab==3.5.0 +PyYAML==6.0 +ruamel.yaml.clib==0.2.6 +ruamel.yaml==0.17.21 diff --git a/bin/ci/update_traces_checksum.py b/bin/ci/update_traces_checksum.py new file mode 100755 index 00000000000..0e9c8437b07 --- /dev/null +++ b/bin/ci/update_traces_checksum.py @@ -0,0 +1,142 @@ +#!/usr/bin/env python3 +# Copyright © 2022 Collabora Ltd. +# Authors: +# David Heidelberg +# +# SPDX-License-Identifier: MIT + +""" +Helper script to update traces checksums +""" + +import argparse +import bz2 +import glob +import re +import json +import sys +from ruamel.yaml import YAML + +import gitlab +from colorama import Fore, Style +from gitlab_common import get_gitlab_project, read_token, wait_for_pipeline + + +DESCRIPTION_FILE = "export PIGLIT_REPLAY_DESCRIPTION_FILE='.*/install/(.*)'$" +DEVICE_NAME = "export PIGLIT_REPLAY_DEVICE_NAME='(.*)'$" + + +def gather_results( + project, + pipeline, +) -> None: + """Gather results""" + + target_jobs_regex = re.compile(".*-traces([:].*)?$") + + for job in pipeline.jobs.list(all=True, sort="desc"): + if target_jobs_regex.match(job.name) and job.status == "failed": + cur_job = project.jobs.get(job.id) + # get variables + print(f"👁 {job.name}...") + log: list[str] = cur_job.trace().decode("unicode_escape").splitlines() + filename: str = '' + dev_name: str = '' + for logline in log: + desc_file = re.search(DESCRIPTION_FILE, logline) + device_name = re.search(DEVICE_NAME, logline) + if desc_file: + filename = desc_file.group(1) + if device_name: + dev_name = device_name.group(1) + + if not filename or not dev_name: + print(Fore.RED + "Couldn't find device name or YML file in the logs!" + Style.RESET_ALL) + return + + print(f"👁 Found {dev_name} and file {filename}") + + # find filename in Mesa source + traces_file = glob.glob('./**/' + filename, recursive=True) + # write into it + with open(traces_file[0], 'r', encoding='utf-8') as target_file: + yaml = YAML() + yaml.compact(seq_seq=False, seq_map=False) + yaml.version = 1,2 + yaml.width = 2048 # do not break the text fields + yaml.default_flow_style = None + target = yaml.load(target_file) + + # parse artifact + results_json_bz2 = cur_job.artifact(path="results/results.json.bz2", streamed=False) + results_json = bz2.decompress(results_json_bz2).decode("utf-8") + results = json.loads(results_json) + + for _, value in results["tests"].items(): + if ( + not value['images'] or + not value['images'][0] or + "image_desc" not in value['images'][0] + ): + continue + + trace: str = value['images'][0]['image_desc'] + checksum: str = value['images'][0]['checksum_render'] + + if not checksum: + print(Fore.RED + f"{dev_name}: {trace}: checksum is missing! Crash?" + Style.RESET_ALL) + continue + + if checksum == "error": + print(Fore.RED + f"{dev_name}: {trace}: crashed" + Style.RESET_ALL) + continue + + if target['traces'][trace][dev_name].get('checksum') == checksum: + continue + + if "label" in target['traces'][trace][dev_name]: + print(f'{dev_name}: {trace}: please verify that label {Fore.BLUE}{target["traces"][trace][dev_name]["label"]}{Style.RESET_ALL} is still valid') + + print(Fore.GREEN + f'{dev_name}: {trace}: checksum updated' + Style.RESET_ALL) + target['traces'][trace][dev_name]['checksum'] = checksum + + with open(traces_file[0], 'w', encoding='utf-8') as target_file: + yaml.dump(target, target_file) + + + +def parse_args() -> None: + """Parse args""" + parser = argparse.ArgumentParser( + description="Tool to generate patch from checksums ", + epilog="Example: update_traces_checksum.py --rev $(git rev-parse HEAD) " + ) + parser.add_argument( + "--rev", metavar="revision", help="repository git revision", required=True + ) + parser.add_argument( + "--token", + metavar="token", + help="force GitLab token, otherwise it's read from ~/.config/gitlab-token", + ) + return parser.parse_args() + + +if __name__ == "__main__": + try: + args = parse_args() + + token = read_token(args.token) + + gl = gitlab.Gitlab(url="https://gitlab.freedesktop.org", private_token=token) + + cur_project = get_gitlab_project(gl, "mesa") + + print(f"Revision: {args.rev}") + pipe = wait_for_pipeline(cur_project, args.rev) + print(f"Pipeline: {pipe.web_url}") + gather_results(cur_project, pipe) + + sys.exit() + except KeyboardInterrupt: + sys.exit(1) -- cgit v1.2.1