path: root/xen/scripts/xen_analysis/cppcheck_analysis.py

#!/usr/bin/env python3

import os, re, shutil
from . import settings, utils, cppcheck_report_utils, exclusion_file_list
from .exclusion_file_list import ExclusionFileListError

class GetMakeVarsPhaseError(Exception):
    pass

class CppcheckDepsPhaseError(Exception):
    pass

class CppcheckReportPhaseError(Exception):
    pass

CPPCHECK_BUILD_DIR = "build-dir-cppcheck"
CPPCHECK_HTMLREPORT_OUTDIR = "cppcheck-htmlreport"
CPPCHECK_REPORT_OUTDIR = "cppcheck-report"
cppcheck_extra_make_args = ""
xen_cc = ""

def get_make_vars():
    global xen_cc
    invoke_make = utils.invoke_command(
            "make -C {} {} export-variable-CC"
                .format(settings.xen_dir, settings.make_forward_args),
            True, GetMakeVarsPhaseError,
            "Error occured retrieving make vars:\n{}"
        )

    cc_var_regex = re.search('^CC=(.*)$', invoke_make, flags=re.M)
    if cc_var_regex:
        xen_cc = cc_var_regex.group(1)

    if xen_cc == "":
        raise GetMakeVarsPhaseError("CC variable not found in Xen make output")


def __generate_suppression_list(out_file):
    # The following lambda function will return a file if it contains lines with
    # a comment containing "cppcheck-suppress[*]" on a single line.
    grep_action = lambda x: utils.grep(x,
                    r'^[ \t]*/\* cppcheck-suppress\[(.*)\] \*/$')
    # Look for a list of .h files that matches the condition above
    headers = utils.recursive_find_file(settings.xen_dir, r'.*\.h$',
                                        grep_action)

    try:
        with open(out_file, "wt") as supplist_file:
            # Add this rule to skip every finding in the autogenerated
            # header for cppcheck
            supplist_file.write("*:*generated/compiler-def.h\n")

            try:
                exclusion_file = \
                    "{}/docs/misra/exclude-list.json".format(settings.repo_dir)
                exclusion_list = \
                    exclusion_file_list.load_exclusion_file_list(exclusion_file)
            except ExclusionFileListError as e:
                raise CppcheckDepsPhaseError(
                    "Issue with reading file {}: {}".format(exclusion_file, e)
                )

            # Append excluded files to the suppression list, using * as error id
            # to suppress every findings on that file
            for path in exclusion_list:
                supplist_file.write("*:{}\n".format(path))

            for entry in headers:
                filename = entry["file"]
                try:
                    with open(filename, "rt") as infile:
                        header_content = infile.readlines()
                except OSError as e:
                    raise CppcheckDepsPhaseError(
                            "Issue with reading file {}: {}"
                                .format(filename, e)
                          )
                header_lines_len = len(header_content)
                # line_num in entry will be header_content[line_num-1], here we
                # are going to search the first line after line_num that have
                # anything different from comments or empty line, because the
                # in-code comment suppression is related to that line then.
                for line_num in entry["matches"]:
                    cppcheck_violation_id = ""
                    tmp_line = line_num
                    # look up to which line is referring the comment at
                    # line_num (which would be header_content[tmp_line-1])
                    comment_section = False
                    while tmp_line < header_lines_len:
                        line = header_content[tmp_line]
                        # Matches a line with just optional spaces/tabs and the
                        # start of a comment '/*'
                        comment_line_starts = re.match('^[ \t]*/\*.*$', line)
                        # Matches a line with text and the end of a comment '*/'
                        comment_line_stops = re.match('^.*\*/$', line)
                        if (not comment_section) and comment_line_starts:
                            comment_section = True
                        if (len(line.strip()) != 0) and (not comment_section):
                            cppcheck_violation_id = entry["matches"][line_num][0]
                            break
                        if comment_section and comment_line_stops:
                            comment_section = False
                        tmp_line = tmp_line + 1

                    if cppcheck_violation_id == "":
                        raise CppcheckDepsPhaseError(
                            "Error matching cppcheck comment in {} at line {}."
                                .format(filename, line_num)
                          )
                    # Write [error id]:[filename]:[line]
                    # tmp_line refers to the array index, so translated to the
                    # file line (that begins with 1) it is tmp_line+1
                    supplist_file.write(
                            "{}:{}:{}\n".format(cppcheck_violation_id, filename,
                                                (tmp_line + 1))
                        )
    except OSError as e:
        raise CppcheckDepsPhaseError("Issue with writing file {}: {}"
                                     .format(out_file, e))


def generate_cppcheck_deps():
    global cppcheck_extra_make_args

    # Compile flags to pass to cppcheck:
    # - include config.h as this is passed directly to the compiler.
    # - define CPPCHECK as we use it to disable or enable some specific part of
    #   the code to solve some cppcheck issues.
    # - explicitely enable some cppcheck checks as we do not want to use "all"
    #   which includes unusedFunction which gives wrong positives as we check
    #   file per file.
    # - Explicitly suppress warnings on compiler-def.h because cppcheck throws
    #   an unmatchedSuppression due to the rule we put in suppression-list.txt
    #   to skip every finding in the file.
    # - Explicitly suppress findings for unusedStructMember that is not very
    #   reliable and causes lots of false positives.
    #
    # Compiler defines are in compiler-def.h which is included in config.h
    #
    cppcheck_flags="""
--cppcheck-build-dir={}/{}
 --max-ctu-depth=10
 --enable=style,information,missingInclude
 --template=\'{{file}}({{line}},{{column}}):{{id}}:{{severity}}:{{message}}\'
 --relative-paths={}
 --inline-suppr
 --suppressions-list={}/suppression-list.txt
 --suppress='unmatchedSuppression:*'
 --suppress='unusedStructMember:*'
 --include={}/include/xen/config.h
 -DCPPCHECK
""".format(settings.outdir, CPPCHECK_BUILD_DIR, settings.xen_dir,
           settings.outdir, settings.xen_dir)

    invoke_cppcheck = utils.invoke_command(
            "{} --version".format(settings.cppcheck_binpath),
            True, CppcheckDepsPhaseError,
            "Error occured retrieving cppcheck version:\n{}\n\n{}"
        )

    version_regex = re.search('^Cppcheck (.*)$', invoke_cppcheck, flags=re.M)
    # Currently, only cppcheck version >= 2.7 is supported, but version 2.8 is
    # known to be broken, please refer to docs/misra/cppcheck.txt
    if (not version_regex) or (not version_regex.group(1).startswith("2.7")):
        raise CppcheckDepsPhaseError(
                "Can't find cppcheck version or version is not 2.7"
              )

    # If misra option is selected, append misra addon and generate cppcheck
    # files for misra analysis
    if settings.cppcheck_misra:
        cppcheck_flags = cppcheck_flags + " --addon=cppcheck-misra.json"

        skip_rules_arg = ""
        if settings.cppcheck_skip_rules != "":
            skip_rules_arg = "-s {}".format(settings.cppcheck_skip_rules)

        utils.invoke_command(
            "{}/convert_misra_doc.py -i {}/docs/misra/rules.rst"
            " -o {}/cppcheck-misra.txt -j {}/cppcheck-misra.json {}"
                .format(settings.tools_dir, settings.repo_dir,
                        settings.outdir, settings.outdir, skip_rules_arg),
            False, CppcheckDepsPhaseError,
            "An error occured when running:\n{}"
        )

    # Generate compiler macros
    os.makedirs("{}/include/generated".format(settings.outdir), exist_ok=True)
    utils.invoke_command(
            "{} -dM -E -o \"{}/include/generated/compiler-def.h\" - < /dev/null"
                .format(xen_cc, settings.outdir),
            False, CppcheckDepsPhaseError,
            "An error occured when running:\n{}"
        )

    # Generate cppcheck suppression list
    __generate_suppression_list(
        "{}/suppression-list.txt".format(settings.outdir))

    # Generate cppcheck build folder
    os.makedirs("{}/{}".format(settings.outdir, CPPCHECK_BUILD_DIR),
                exist_ok=True)

    cppcheck_cc_flags = """--compiler={} --cppcheck-cmd={} {}
 --cppcheck-plat={}/cppcheck-plat --ignore-path=tools/
 --ignore-path=arch/x86/efi/check.c
""".format(xen_cc, settings.cppcheck_binpath, cppcheck_flags,
           settings.tools_dir)

    if settings.cppcheck_html:
        cppcheck_cc_flags = cppcheck_cc_flags + " --cppcheck-html"

    # Generate the extra make argument to pass the cppcheck-cc.sh wrapper as CC
    cppcheck_extra_make_args = "CC=\"{}/cppcheck-cc.sh {} --\"".format(
                                        settings.tools_dir,
                                        cppcheck_cc_flags
                                    ).replace("\n", "")


def generate_cppcheck_report():
    # Prepare text report
    # Look for a list of .cppcheck.txt files, those are the txt report
    # fragments
    fragments = utils.recursive_find_file(settings.outdir, r'.*\.cppcheck.txt$')
    text_report_dir = "{}/{}".format(settings.outdir,
                                        CPPCHECK_REPORT_OUTDIR)
    report_filename = "{}/xen-cppcheck.txt".format(text_report_dir)
    os.makedirs(text_report_dir, exist_ok=True)
    try:
        cppcheck_report_utils.cppcheck_merge_txt_fragments(fragments,
                                                           report_filename,
                                                           [settings.xen_dir])
    except cppcheck_report_utils.CppcheckTXTReportError as e:
        raise CppcheckReportPhaseError(e)

    # If HTML output is requested
    if settings.cppcheck_html:
        # Look for a list of .cppcheck.xml files, those are the XML report
        # fragments
        fragments = utils.recursive_find_file(settings.outdir,
                                              r'.*\.cppcheck.xml$')
        html_report_dir = "{}/{}".format(settings.outdir,
                                         CPPCHECK_HTMLREPORT_OUTDIR)
        xml_filename = "{}/xen-cppcheck.xml".format(html_report_dir)
        os.makedirs(html_report_dir, exist_ok=True)
        try:
            cppcheck_report_utils.cppcheck_merge_xml_fragments(fragments,
                                                               xml_filename,
                                                               settings.xen_dir,
                                                               settings.outdir)
        except cppcheck_report_utils.CppcheckHTMLReportError as e:
            raise CppcheckReportPhaseError(e)
        # Call cppcheck-htmlreport utility to generate the HTML output
        utils.invoke_command(
            "{} --file={} --source-dir={} --report-dir={}/html --title=Xen"
                .format(settings.cppcheck_htmlreport_binpath, xml_filename,
                        settings.xen_dir, html_report_dir),
            False, CppcheckReportPhaseError,
            "Error occured generating Cppcheck HTML report:\n{}"
        )
        # Strip src and obj path from *.html files
        html_files = utils.recursive_find_file(html_report_dir, r'.*\.html$')
        try:
            cppcheck_report_utils.cppcheck_strip_path_html(html_files,
                                                           (settings.xen_dir,
                                                            settings.outdir))
        except cppcheck_report_utils.CppcheckHTMLReportError as e:
            raise CppcheckReportPhaseError(e)


def clean_analysis_artifacts():
    clean_files = ("suppression-list.txt", "cppcheck-misra.txt",
                   "cppcheck-misra.json")
    cppcheck_build_dir = "{}/{}".format(settings.outdir, CPPCHECK_BUILD_DIR)
    if os.path.isdir(cppcheck_build_dir):
        shutil.rmtree(cppcheck_build_dir)
    artifact_files = utils.recursive_find_file(settings.outdir,
                                r'.*\.(?:c\.json|cppcheck\.txt|cppcheck\.xml)$')
    for file in clean_files:
        file = "{}/{}".format(settings.outdir, file)
        if os.path.isfile(file):
            artifact_files.append(file)
    for delfile in artifact_files:
        os.remove(delfile)


def clean_reports():
    text_report_dir = "{}/{}".format(settings.outdir,
                                     CPPCHECK_REPORT_OUTDIR)
    html_report_dir = "{}/{}".format(settings.outdir,
                                     CPPCHECK_HTMLREPORT_OUTDIR)
    if os.path.isdir(text_report_dir):
        shutil.rmtree(text_report_dir)
    if os.path.isdir(html_report_dir):
        shutil.rmtree(html_report_dir)