zephyr: Add zmake command coverage.

Added a new zmake sub-command `coverage`, which builds all projects
with coverage, runs unit tests, and creates a html coverage report.

BUG=b:183007888
TEST=sudo emerge zephyr-build-tools && \
zmake coverage build/ztest-coverage
BRANCH=none

Change-Id: Idb6af59c223ece00d3eb09982778cb1b500d8db4
Signed-off-by: Jeremy Bettis <jbettis@google.com>
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/ec/+/2794925
Tested-by: Jeremy Bettis <jbettis@chromium.org>
Reviewed-by: Jack Rosenthal <jrosenth@chromium.org>
Reviewed-by: Yuval Peress <peress@chromium.org>
Reviewed-by: Keith Short <keithshort@chromium.org>
Commit-Queue: Keith Short <keithshort@chromium.org>

4 files changed, 181 insertions, 47 deletions

diff --git a/docs/code_coverage.md b/docs/code_coverage.md
index 790de68f84..ec8053ca93 100644
--- a/docs/code_coverage.md
+++ b/docs/code_coverage.md
@@ -36,32 +36,15 @@ appear to be caused in part by using relative paths instead of absolute paths.)
 
 ## Zephyr ztest code coverage
 
-This needs some work, but you can generate coverage reports with these commands:
-
-```
-# Get initial (0 lines executed) coverage for as many boards as possible
-for project in $(cd zephyr/projects; find -name zmake.yaml -print)
-do
-        project="$(dirname ${project#./})"
-        echo "Building initial coverage for ${project}"
-        builddir="build/ztest-coverage/projects/$project"
-        infopath="build/ztest-coverage/projects/${project/\//_}.info"
-        zmake configure --coverage -B ${builddir} zephyr/projects/$project
-        for buildsubdir in ${builddir}/build-* ; do ninja -C ${buildsubdir} all.libraries ; done
-        lcov --gcov-tool $HOME/trunk/src/platform/ec/util/llvm-gcov.sh -q -o - -c -d ${builddir} -t "${project/\//_}" \
-          --exclude "*/build-*/zephyr/*/generated/*" --exclude '*/test/*' --exclude '*/testsuite/*' \
-          -i | util/normalize_symlinks.py >${infopath}
-done
-
-# Get unit test coverage
-for i in zephyr/test/* ; do
-        builddir="build/ztest-coverage/$(basename $i)"
-        zmake configure --coverage --test -B ${builddir} $i
-        lcov --gcov-tool $HOME/trunk/src/platform/ec/util/llvm-gcov.sh -q -o - -c -d ${builddir} -t "$(basename $i)" \
-          --exclude '*/build-singleimage/zephyr/*/generated/*' --exclude '*/test/*' --exclude '*/testsuite/*' \
-          | util/normalize_symlinks.py >${builddir}.info
-done
-
-# Merge into a nice html report
-genhtml -q -o build/ztest-coverage/coverage_rpt -t "Zephyr EC Unittest" -p /mnt/host/source/src -s build/ztest-coverage/*.info build/ztest-coverage/projects/*.info
-```
+To build the Zephyr unit tests for code coverage run:
+
+`zmake coverage build/ztest-coverage`
+
+This target will compile, without linking, all zephyr projects with
+`CONFIG_COVERAGE` Kconfig option enabled, run the tests, and then process the
+profiling data into a code coverage report using the `lcov` and `genhtml`
+tools. This requires the `HAS_COVERAGE_SUPPORT` option, which can only be
+selected in `Kconfig.board`.
+
+The coverage report top-level page is
+`build/ztest-coverage/coverage_rpt/index.html`.
diff --git a/util/normalize_symlinks.py b/util/normalize_symlinks.py
deleted file mode 100755
index 112202b39e..0000000000
--- a/util/normalize_symlinks.py
+++ /dev/null
@@ -1,17 +0,0 @@
-#!/usr/bin/env python3
-# Copyright 2021 The Chromium OS Authors. All rights reserved.
-# Use of this source code is governed by a BSD-style license that can be
-# found in the LICENSE file.
-
-"""Takes a lcov info file as input and normalizes symlinks from SF: lines."""
-
-import fileinput
-import os
-import sys
-
-for line in fileinput.input():
-    if line.startswith('SF:'):
-        path = line[3:].rstrip()
-        sys.stdout.write('SF:%s\n' % os.path.realpath(path))
-    else:
-        sys.stdout.write(line)
diff --git a/zephyr/zmake/zmake/__main__.py b/zephyr/zmake/zmake/__main__.py
index e3c527cb01..654c49d085 100644
--- a/zephyr/zmake/zmake/__main__.py
+++ b/zephyr/zmake/zmake/__main__.py
@@ -115,6 +115,12 @@ def main(argv=None):
     testall.add_argument('--fail-fast', action='store_true',
                          help='stop testing after the first error')
 
+    coverage = sub.add_parser('coverage')
+    coverage.add_argument('--fail-fast', action='store_true',
+                         help='stop testing after the first error')
+    coverage.add_argument('build_dir', type=pathlib.Path,
+                      help='The build directory used during configuration')
+
     opts = parser.parse_args(argv)
 
     if opts.no_log_label:
diff --git a/zephyr/zmake/zmake/zmake.py b/zephyr/zmake/zmake/zmake.py
index 865e477698..0d9f9ee33c 100644
--- a/zephyr/zmake/zmake/zmake.py
+++ b/zephyr/zmake/zmake/zmake.py
@@ -6,13 +6,14 @@
 import logging
 import os
 import pathlib
+import re
 import shutil
 import subprocess
 import tempfile
 
 import zmake.build_config
-import zmake.modules
 import zmake.jobserver
+import zmake.modules
 import zmake.multiproc
 import zmake.project
 import zmake.toolchains as toolchains
@@ -409,3 +410,164 @@ class Zmake:
         for tmpdir in tmp_dirs:
             shutil.rmtree(tmpdir)
         return rv
+
+    def _run_lcov(self, build_dir, lcov_file, initial=False):
+        with self.jobserver.get_job():
+            if initial:
+                self.logger.info('Running (initial) lcov on %s.', build_dir)
+            else:
+                self.logger.info('Running lcov on %s.', build_dir)
+            cmd = ['/usr/bin/lcov', '--gcov-tool',
+                   self.module_paths['ec'] /
+                   'util/llvm-gcov.sh', '-q', '-o', '-',
+                   '-c', '-d', build_dir, '-t', lcov_file.stem, '--exclude',
+                   '*/build-*/zephyr/*/generated/*', '--exclude', '*/test/*',
+                   '--exclude', '*/testsuite/*']
+            if initial:
+                cmd += ['-i']
+            proc = self.jobserver.popen(cmd,
+                                        claim_job=False,
+                                        stdout=subprocess.PIPE,
+                                        stderr=subprocess.PIPE,
+                                        encoding='utf-8',
+                                        errors='replace')
+            zmake.multiproc.log_output(
+                self.logger, logging.WARNING, proc.stderr)
+
+            with open(lcov_file, 'w') as outfile:
+                for line in proc.stdout:
+                    if line.startswith('SF:'):
+                        path = line[3:].rstrip()
+                        outfile.write('SF:%s\n' % os.path.realpath(path))
+                    else:
+                        outfile.write(line)
+            if proc.wait():
+                raise OSError(get_process_failure_msg(proc))
+
+            return 0
+
+    def _coverage_compile_only(self, project, build_dir, lcov_file):
+        self.logger.info("Building %s in %s",
+                         project.project_dir, build_dir)
+        rv = self.configure(
+            project_dir=project.project_dir,
+            build_dir=build_dir,
+            build_after_configure=False,
+            test_after_configure=False,
+            coverage=True)
+        if rv:
+            return rv
+
+        # Use ninja to compile the all.libraries target.
+        build_project = zmake.project.Project(build_dir / 'project')
+
+        procs = []
+        dirs = {}
+        for build_name, build_config in build_project.iter_builds():
+            self.logger.info('Building %s:%s all.libraries.',
+                             build_dir, build_name)
+            dirs[build_name] = build_dir / 'build-{}'.format(build_name)
+            proc = self.jobserver.popen(
+                ['/usr/bin/ninja', '-C', dirs[build_name], 'all.libraries'],
+                # Ninja will connect as a job client instead and claim
+                # many jobs.
+                claim_job=False,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                encoding='utf-8',
+                errors='replace')
+            zmake.multiproc.log_output(
+                logger=self.logger,
+                log_level=logging.DEBUG,
+                file_descriptor=proc.stdout,
+                log_level_override_func=ninja_log_level_override)
+            zmake.multiproc.log_output(self.logger, logging.ERROR, proc.stderr)
+            procs.append(proc)
+
+        for proc in procs:
+            if proc.wait():
+                raise OSError(get_process_failure_msg(proc))
+
+        return self._run_lcov(build_dir, lcov_file, initial=True)
+
+    def _coverage_run_test(self, project, build_dir, lcov_file):
+        self.logger.info("Running test %s in %s",
+                         project.project_dir, build_dir)
+        rv = self.configure(
+            project_dir=project.project_dir,
+            build_dir=build_dir,
+            build_after_configure=True,
+            test_after_configure=True,
+            coverage=True)
+        if rv:
+            return rv
+        return self._run_lcov(build_dir, lcov_file, initial=False)
+
+    def coverage(self, build_dir, fail_fast=False):
+        """Builds all targets with coverage enabled, and then runs the tests."""
+        executor = zmake.multiproc.Executor(fail_fast=fail_fast)
+        all_lcov_files = []
+        root_dir = self.module_paths['ec'] / 'zephyr'
+        for project in zmake.project.find_projects(root_dir):
+            is_test = project.config.is_test
+            rel_path = project.project_dir.relative_to(root_dir)
+            project_build_dir = pathlib.Path(build_dir).joinpath(rel_path)
+            lcov_file = pathlib.Path(build_dir).joinpath(
+                str(rel_path).replace('/', '_') + '.info')
+            all_lcov_files.append(lcov_file)
+            if is_test:
+                # Configure and run the test.
+                executor.append(
+                    func=lambda: self._coverage_run_test(
+                        project,
+                        project_build_dir,
+                        lcov_file))
+            else:
+                # Configure and compile the non-test project.
+                executor.append(
+                    func=lambda: self._coverage_compile_only(
+                        project,
+                        project_build_dir,
+                        lcov_file))
+
+        rv = executor.wait()
+        if rv:
+            return rv
+
+        with self.jobserver.get_job():
+            # Get the build version
+            proc = self.jobserver.popen(
+                [self.module_paths['ec'] /
+                 'util/getversion.sh'],
+                claim_job=False,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                encoding='utf-8',
+                errors='replace')
+            zmake.multiproc.log_output(self.logger, logging.ERROR, proc.stderr)
+            version = ''
+            for line in proc.stdout:
+                match = re.search(r'#define VERSION "(.*)"', line)
+                if match:
+                    version = match.group(1)
+            if proc.wait():
+                raise OSError(get_process_failure_msg(proc))
+
+            # Merge into a nice html report
+            self.logger.info("Creating coverage report %s.",
+                             build_dir / 'coverage_rpt')
+            proc = self.jobserver.popen(
+                ['/usr/bin/genhtml', '-q', '-o',
+                 build_dir / 'coverage_rpt', '-t',
+                 "Zephyr EC Unittest {}".format(version), '-p',
+                 self.checkout / 'src', '-s'] + all_lcov_files,
+                claim_job=False,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                encoding='utf-8',
+                errors='replace')
+            zmake.multiproc.log_output(self.logger, logging.ERROR, proc.stderr)
+            zmake.multiproc.log_output(self.logger, logging.DEBUG, proc.stdout)
+            if proc.wait():
+                raise OSError(get_process_failure_msg(proc))
+            return 0