zmake: add compare-builds option

Add compare-builds as a sub command to zmake. This sub command performs
two separate checkouts of the EC repo and verifies the resulting
binaries are unchanged.  The zephyr and module repos are checked out at
HEAD from the respective repos.

Typical usage compares the EC repo at HEAD and HEAD~
    zmake compare-builds -a

BUG=none
BRANCH=none
TEST=zmake compare-builds -a
TEST=modify skyrim build, verify only skyrim board doesn't compare

Signed-off-by: Keith Short <keithshort@chromium.org>
Change-Id: I6ffa0e41bbcb649f4376853a86778baa3b9e62f1
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/ec/+/3957415
Reviewed-by: Wai-Hong Tam <waihong@google.com>

4 files changed, 360 insertions, 0 deletions

diff --git a/zephyr/zmake/README.md b/zephyr/zmake/README.md
index 6efc495ec5..224c67653c 100644
--- a/zephyr/zmake/README.md
+++ b/zephyr/zmake/README.md
@@ -87,6 +87,28 @@ Chromium OS's meta-build tool for Zephyr
 | `--extra-cflags EXTRA_CFLAGS` | Additional CFLAGS to use for target builds |
 | `-a`, `--all` | Select all projects |
 
+### zmake compare-builds
+
+**Usage:** `zmake compare-builds [-h] [--ref1 REF1] [--ref2 REF2] [-k] [-t TOOLCHAIN] [--extra-cflags EXTRA_CFLAGS] (-a | project_name [project_name ...])`
+
+#### Positional Arguments
+
+|   |   |
+|---|---|
+| `project_name` | Name(s) of the project(s) to build |
+
+#### Optional Arguments
+
+|   |   |
+|---|---|
+| `-h`, `--help` | show this help message and exit |
+| `--ref1 REF1` | 1st git reference (commit, branch, etc), default=HEAD |
+| `--ref2 REF2` | 2nd git reference (commit, branch, etc), default=HEAD~ |
+| `-k`, `--keep-temps` | Keep temporary build directories on exit |
+| `-t TOOLCHAIN`, `--toolchain TOOLCHAIN` | Name of toolchain to use |
+| `--extra-cflags EXTRA_CFLAGS` | Additional CFLAGS to use for target builds |
+| `-a`, `--all` | Select all projects |
+
 ### zmake list-projects
 
 **Usage:** `zmake list-projects [-h] [--format FMT] [search_dir]`
diff --git a/zephyr/zmake/zmake/__main__.py b/zephyr/zmake/zmake/__main__.py
index f0f2098cf5..23fb58eca6 100644
--- a/zephyr/zmake/zmake/__main__.py
+++ b/zephyr/zmake/zmake/__main__.py
@@ -186,6 +186,27 @@ def get_argparser():
     add_common_configure_args(build)
     add_common_build_args(build)
 
+    compare_builds = sub.add_parser(
+        "compare-builds", help="Compare output binaries from two commits"
+    )
+    compare_builds.add_argument(
+        "--ref1",
+        default="HEAD",
+        help="1st git reference (commit, branch, etc), default=HEAD",
+    )
+    compare_builds.add_argument(
+        "--ref2",
+        default="HEAD~",
+        help="2nd git reference (commit, branch, etc), default=HEAD~",
+    )
+    compare_builds.add_argument(
+        "-k",
+        "--keep-temps",
+        action="store_true",
+        help="Keep temporary build directories on exit",
+    )
+    add_common_build_args(compare_builds)
+
     list_projects = sub.add_parser(
         "list-projects",
         help="List projects known to zmake.",
diff --git a/zephyr/zmake/zmake/compare_builds.py b/zephyr/zmake/zmake/compare_builds.py
new file mode 100644
index 0000000000..92e197de75
--- /dev/null
+++ b/zephyr/zmake/zmake/compare_builds.py
@@ -0,0 +1,237 @@
+# Copyright 2022 The ChromiumOS Authors
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Module to compare Zephyr EC builds"""
+
+import dataclasses
+import logging
+import os
+import pathlib
+import subprocess
+import sys
+
+from zmake.output_packers import packer_registry
+
+
+def get_git_hash(ref):
+    """Get the full git commit hash for a git reference
+
+    Args:
+        ref: Git reference (e.g. HEAD, m/main, sha256)
+
+    Returns:
+        A string, with the full hash of the git reference
+    """
+
+    try:
+        result = subprocess.run(
+            ["git", "rev-parse", ref],
+            check=True,
+            stdout=subprocess.PIPE,
+            stderr=subprocess.DEVNULL,
+            encoding="utf-8",
+        )
+    except subprocess.CalledProcessError:
+        logging.error("Failed to determine hash for git reference %s", ref)
+        sys.exit(1)
+    else:
+        full_reference = result.stdout.strip()
+
+    return full_reference
+
+
+def git_do_checkout(module_name, work_dir, git_source, dst_dir, git_ref):
+    """Clone a repository and perform a checkout.
+
+    Args:
+        module_name: The module name to checkout.
+        work_dir: Root directory for the checktout.
+        git_source: Path to the repository for the module.
+        dst_dir: Destination directory for the checkout, relative to the work_dir.
+        git_ref: Git reference to checkout.
+    """
+    cmd = ["git", "clone", "--quiet", "--no-checkout", git_source, dst_dir]
+
+    try:
+        subprocess.run(
+            cmd,
+            cwd=work_dir,
+            check=True,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+    except subprocess.CalledProcessError:
+        logging.error("Clone failed for %s", module_name)
+        sys.exit(1)
+
+    cmd = ["git", "-C", dst_dir, "checkout", "--quiet", git_ref]
+    try:
+        subprocess.run(
+            cmd,
+            cwd=work_dir,
+            check=True,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+    except subprocess.CalledProcessError:
+        logging.error("Checkout of %s failed for %s", git_ref, module_name)
+        sys.exit(1)
+
+
+def create_bin_from_elf(elf_input, bin_output):
+    """Create a plain binary from an ELF executable
+
+    Args:
+        elf_input - ELF output file, created by zmake
+        bin_output - Output binary filename. Created by this function.
+    """
+
+    cmd = ["objcopy", "-O", "binary"]
+    # Some native-posix builds include a GNU build ID, which is guaranteed
+    # unique from build to build. Remove this section during conversion
+    # binary format.
+    cmd.extend(["-R", ".note.gnu.build-id"])
+    cmd.extend([elf_input, bin_output])
+    try:
+        subprocess.run(
+            cmd,
+            check=True,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL,
+        )
+    except subprocess.CalledProcessError:
+        logging.error("Failed to create binary: %s", bin_output)
+        sys.exit(1)
+
+
+@dataclasses.dataclass
+class CheckoutConfig:
+    """All the information needed to build the EC at a specific checkout."""
+
+    temp_dir: str
+    ref: str
+    full_ref: str = dataclasses.field(default_factory=str)
+    work_dir: pathlib.Path = dataclasses.field(default_factory=pathlib.Path)
+    zephyr_dir: pathlib.Path = dataclasses.field(default_factory=pathlib.Path)
+    modules_dir: pathlib.Path = dataclasses.field(default_factory=pathlib.Path)
+
+    def __post_init__(self):
+        self.full_ref = get_git_hash(self.ref)
+        self.work_dir = pathlib.Path(self.temp_dir) / self.full_ref
+        self.zephyr_dir = self.work_dir / "zephyr-base"
+        self.modules_dir = self.work_dir / "modules"
+
+        os.mkdir(self.work_dir)
+
+
+class CompareBuilds:
+    """Information required to build Zephyr EC projects at a specific EC git
+        commit reference.
+
+    Args:
+        temp_dir: Temporary directory where all sources will be checked out
+            and built.
+        ref1: 1st git reference for the EC repository.  May be a partial hash,
+            local branch name, or remote branch name.
+        ref2: 2nd git reference for the EC repository.
+
+    Attributes:
+        checkouts: list of CheckoutConfig objects containing information
+            about the code checkout at each EC git reference.
+    """
+
+    def __init__(self, temp_dir, ref1, ref2):
+        self.checkouts = []
+        self.checkouts.append(CheckoutConfig(temp_dir, ref1))
+        self.checkouts.append(CheckoutConfig(temp_dir, ref2))
+
+    def do_checkouts(self, zephyr_base, module_paths):
+        """Checkout all EC sources at a specific commit.
+
+        Args:
+            zephyr_base: The location of the zephyr sources.
+            module_paths: The location of the module sources.
+        """
+
+        for checkout in self.checkouts:
+            for module_name, git_source in module_paths.items():
+                dst_dir = checkout.modules_dir / module_name
+                git_ref = checkout.full_ref if module_name == "ec" else "HEAD"
+                git_do_checkout(
+                    module_name=module_name,
+                    work_dir=checkout.work_dir,
+                    git_source=git_source,
+                    dst_dir=dst_dir,
+                    git_ref=git_ref,
+                )
+
+            git_do_checkout(
+                module_name="zephyr",
+                work_dir=checkout.work_dir,
+                git_source=zephyr_base,
+                dst_dir="zephyr-base",
+                git_ref="HEAD",
+            )
+
+    def check_binaries(self, projects):
+        """Compare Zephyr EC binaries for two different source trees
+
+        Args:
+            projects: List of projects to compare the output binaries.
+
+        Returns:
+            A list of projects that failed to compare.  An empty list indicates that
+            all projects compared successfully.
+        """
+
+        failed_projects = []
+        for project in projects:
+            if project.config.is_test:
+                continue
+
+            output_path = (
+                pathlib.Path("ec")
+                / "build"
+                / "zephyr"
+                / pathlib.Path(project.config.project_name)
+                / "output"
+            )
+
+            output_dir1 = self.checkouts[0].modules_dir / output_path
+            output_dir2 = self.checkouts[1].modules_dir / output_path
+
+            bin_output1 = output_dir1 / "ec.bin"
+            bin_output2 = output_dir2 / "ec.bin"
+
+            # ELF executables don't compare due to meta data.  Convert to a binary
+            # for the comparison
+            if project.config.output_packer == packer_registry["elf"]:
+                create_bin_from_elf(
+                    elf_input=output_dir1 / "zephyr.elf", bin_output=bin_output1
+                )
+                create_bin_from_elf(
+                    elf_input=output_dir2 / "zephyr.elf", bin_output=bin_output2
+                )
+
+            bin1_path = pathlib.Path(bin_output1)
+            bin2_path = pathlib.Path(bin_output2)
+            if not os.path.isfile(bin1_path) or not os.path.isfile(bin2_path):
+                failed_projects.append(project.config.project_name)
+                logging.error(
+                    "Zephyr EC binary not found for project %s",
+                    project.config.project_name,
+                )
+                continue
+
+            try:
+                subprocess.run(
+                    ["cmp", bin_output1, bin_output2],
+                    check=True,
+                    stdout=subprocess.DEVNULL,
+                    stderr=subprocess.DEVNULL,
+                )
+            except subprocess.CalledProcessError:
+                failed_projects.append(project.config.project_name)
+
+        return failed_projects
diff --git a/zephyr/zmake/zmake/zmake.py b/zephyr/zmake/zmake/zmake.py
index b7a81a3b9b..f81f157054 100644
--- a/zephyr/zmake/zmake/zmake.py
+++ b/zephyr/zmake/zmake/zmake.py
@@ -3,6 +3,7 @@
 # found in the LICENSE file.
 
 """Module encapsulating Zmake wrapper object."""
+import atexit
 import difflib
 import functools
 import logging
@@ -11,9 +12,11 @@ import pathlib
 import re
 import shutil
 import subprocess
+import tempfile
 from typing import Dict, Optional, Set, Union
 
 import zmake.build_config
+import zmake.compare_builds
 import zmake.generate_readme
 import zmake.jobserver
 import zmake.modules
@@ -323,6 +326,83 @@ class Zmake:
             save_temps=save_temps,
         )
 
+    def compare_builds(
+        self,
+        ref1,
+        ref2,
+        project_names,
+        toolchain=None,
+        all_projects=False,
+        extra_cflags=None,
+        keep_temps=False,
+    ):
+        """Compare EC builds at two commits."""
+        temp_dir = tempfile.mkdtemp(prefix="zcompare-")
+        if not keep_temps:
+            atexit.register(shutil.rmtree, temp_dir)
+        else:
+            self.logger.info("Temporary dir %s will be retained", temp_dir)
+
+        projects = self._resolve_projects(
+            project_names,
+            all_projects=all_projects,
+        )
+
+        self.logger.info("Compare zephyr builds")
+
+        cmp_builds = zmake.compare_builds.CompareBuilds(temp_dir, ref1, ref2)
+
+        for checkout in cmp_builds.checkouts:
+            self.logger.info(
+                "Checkout %s: full hash %s", checkout.ref, checkout.full_ref
+            )
+
+        cmp_builds.do_checkouts(self.zephyr_base, self.module_paths)
+
+        for checkout in cmp_builds.checkouts:
+            # Now that the sources have been checked out, transform the
+            # zephyr-base and module-paths to use the temporary directory
+            # created by BuildInfo.
+            for module_name in self.module_paths.keys():
+                new_path = checkout.modules_dir / module_name
+                transformed_module = {module_name: new_path}
+                self.module_paths.update(transformed_module)
+
+            self.zephyr_base = checkout.zephyr_dir
+
+            self.logger.info("Building projects at %s", checkout.ref)
+            result = self.configure(
+                project_names,
+                build_dir=None,
+                toolchain=toolchain,
+                clobber=False,
+                bringup=False,
+                coverage=False,
+                allow_warnings=False,
+                all_projects=all_projects,
+                extra_cflags=extra_cflags,
+                build_after_configure=True,
+                delete_intermediates=False,
+                static_version=True,
+                save_temps=False,
+            )
+
+            if result:
+                self.logger.error(
+                    "compare-builds failed to build all projects at %s",
+                    checkout.ref,
+                )
+                return result
+
+        self.failed_projects = cmp_builds.check_binaries(projects)
+
+        if len(self.failed_projects) == 0:
+            self.logger.info("Zephyr compare builds successful:")
+            for checkout in cmp_builds.checkouts:
+                self.logger.info("   %s: %s", checkout.ref, checkout.full_ref)
+
+        return len(self.failed_projects)
+
     def test(  # pylint: disable=unused-argument
         self,
         project_names,