zephyr: zmake: Add config and device tree check capability to compare-builds

The invocation of command with -con arg now will compare config
files, -dt will compare devicetree files along with binary files
and outputs the failed projects lists separately along with combined
list. With -bin arg now binaries comparisons can be avoided.

BUG=b:262374758
BRANCH=none
TEST=zmake compare-builds it8xxx2_evb -bin -con -dt
TEST=Modified dt and config in Rex and saw comparison failing

Change-Id: I29751c640b833792f6bfbacbdb80b9119bdb8efc
Signed-off-by: Madhurima Paruchuri <mparuchuri@google.com>
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/ec/+/4169805
Reviewed-by: Fabio Baltieri <fabiobaltieri@google.com>

4 files changed, 178 insertions, 48 deletions

diff --git a/zephyr/zmake/README.md b/zephyr/zmake/README.md
index f8adbdc81e..87af4242f3 100644
--- a/zephyr/zmake/README.md
+++ b/zephyr/zmake/README.md
@@ -91,7 +91,7 @@ Chromium OS's meta-build tool for Zephyr
 
 ### zmake compare-builds
 
-**Usage:** `zmake compare-builds [-h] [--ref1 REF1] [--ref2 REF2] [-k] [-t TOOLCHAIN] [--extra-cflags EXTRA_CFLAGS] (-a | project_name [project_name ...])`
+**Usage:** `zmake compare-builds [-h] [--ref1 REF1] [--ref2 REF2] [-k] [-n] [-b] [-d] [-t TOOLCHAIN] [--extra-cflags EXTRA_CFLAGS] (-a | project_name [project_name ...])`
 
 #### Positional Arguments
 
@@ -107,6 +107,9 @@ Chromium OS's meta-build tool for Zephyr
 | `--ref1 REF1` | 1st git reference (commit, branch, etc), default=HEAD |
 | `--ref2 REF2` | 2nd git reference (commit, branch, etc), default=HEAD~ |
 | `-k`, `--keep-temps` | Keep temporary build directories on exit |
+| `-n`, `--compare-configs` | Compare configs of build outputs |
+| `-b`, `--compare-binaries-disable` | Don't compare binaries of build outputs |
+| `-d`, `--compare-devicetrees` | Compare devicetrees of build outputs |
 | `-t TOOLCHAIN`, `--toolchain TOOLCHAIN` | Name of toolchain to use |
 | `--extra-cflags EXTRA_CFLAGS` | Additional CFLAGS to use for target builds |
 | `-a`, `--all` | Select all projects |
diff --git a/zephyr/zmake/zmake/__main__.py b/zephyr/zmake/zmake/__main__.py
index 99b69747ea..7b79fb23e6 100644
--- a/zephyr/zmake/zmake/__main__.py
+++ b/zephyr/zmake/zmake/__main__.py
@@ -205,6 +205,24 @@ def get_argparser():
         action="store_true",
         help="Keep temporary build directories on exit",
     )
+    compare_builds.add_argument(
+        "-n",
+        "--compare-configs",
+        action="store_true",
+        help="Compare configs of build outputs",
+    )
+    compare_builds.add_argument(
+        "-b",
+        "--compare-binaries-disable",
+        action="store_true",
+        help="Don't compare binaries of build outputs",
+    )
+    compare_builds.add_argument(
+        "-d",
+        "--compare-devicetrees",
+        action="store_true",
+        help="Compare devicetrees of build outputs",
+    )
     add_common_build_args(compare_builds)
 
     list_projects = sub.add_parser(
@@ -410,8 +428,15 @@ def main(argv=None):
         return result or wait_rv
     finally:
         multiproc.LogWriter.wait_for_log_end()
+        for file, failed_projects in zmake.cmp_failed_projects.items():
+            if failed_projects:
+                logging.error(
+                    "Failed projects by diff in %s: %s",
+                    file,
+                    ", ".join(failed_projects),
+                )
         if zmake.failed_projects:
-            logging.error("Failed projects: %s", zmake.failed_projects)
+            logging.error("All failed projects: %s", zmake.failed_projects)
 
 
 if __name__ == "__main__":
diff --git a/zephyr/zmake/zmake/compare_builds.py b/zephyr/zmake/zmake/compare_builds.py
index 92e197de75..a1cc1ee953 100644
--- a/zephyr/zmake/zmake/compare_builds.py
+++ b/zephyr/zmake/zmake/compare_builds.py
@@ -105,6 +105,18 @@ def create_bin_from_elf(elf_input, bin_output):
         sys.exit(1)
 
 
+def _compare_non_test_projects(projects, cmp_method, *args):
+    failed_projects = []
+    for project in projects:
+        if project.config.is_test:
+            continue
+
+        if not cmp_method(project, *args):
+            failed_projects.append(project.config.project_name)
+
+    return failed_projects
+
+
 @dataclasses.dataclass
 class CheckoutConfig:
     """All the information needed to build the EC at a specific checkout."""
@@ -174,6 +186,51 @@ class CompareBuilds:
                 git_ref="HEAD",
             )
 
+    def _compare_binaries(self, project):
+        output_path = (
+            pathlib.Path("ec")
+            / "build"
+            / "zephyr"
+            / pathlib.Path(project.config.project_name)
+            / "output"
+        )
+
+        output_dir1 = self.checkouts[0].modules_dir / output_path
+        output_dir2 = self.checkouts[1].modules_dir / output_path
+
+        bin_output1 = output_dir1 / "ec.bin"
+        bin_output2 = output_dir2 / "ec.bin"
+
+        # ELF executables don't compare due to meta data.  Convert to a binary
+        # for the comparison
+        if project.config.output_packer == packer_registry["elf"]:
+            create_bin_from_elf(
+                elf_input=output_dir1 / "zephyr.elf", bin_output=bin_output1
+            )
+            create_bin_from_elf(
+                elf_input=output_dir2 / "zephyr.elf", bin_output=bin_output2
+            )
+
+        bin1_path = pathlib.Path(bin_output1)
+        bin2_path = pathlib.Path(bin_output2)
+        if not os.path.isfile(bin1_path) or not os.path.isfile(bin2_path):
+            logging.error(
+                "Zephyr EC binary not found for project %s",
+                project.config.project_name,
+            )
+            return False
+
+        try:
+            subprocess.run(
+                ["cmp", bin_output1, bin_output2],
+                check=True,
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.DEVNULL,
+            )
+        except subprocess.CalledProcessError:
+            return False
+        return True
+
     def check_binaries(self, projects):
         """Compare Zephyr EC binaries for two different source trees
 
@@ -185,53 +242,83 @@ class CompareBuilds:
             all projects compared successfully.
         """
 
-        failed_projects = []
-        for project in projects:
-            if project.config.is_test:
-                continue
-
-            output_path = (
-                pathlib.Path("ec")
-                / "build"
-                / "zephyr"
-                / pathlib.Path(project.config.project_name)
-                / "output"
+        failed_projects = _compare_non_test_projects(
+            projects, self._compare_binaries
+        )
+        return failed_projects
+
+    def _compare_build_files(self, project, build_mode, file):
+        build_path = (
+            pathlib.Path("ec")
+            / "build"
+            / "zephyr"
+            / pathlib.Path(project.config.project_name)
+            / f"build-{build_mode}"
+            / "zephyr"
+        )
+
+        build_dir1 = self.checkouts[0].modules_dir / build_path
+        build_dir2 = self.checkouts[1].modules_dir / build_path
+
+        file1 = build_dir1 / file
+        file2 = build_dir2 / file
+
+        try:
+            data1 = ""
+            data2 = ""
+            with open(file1) as fp1, open(file2) as fp2:
+                data1 = fp1.read()
+                data2 = fp2.read()
+            data1 = data1.replace(self.checkouts[0].full_ref, "")
+            data2 = data2.replace(self.checkouts[1].full_ref, "")
+            return data1 == data2
+        except FileNotFoundError as err:
+            logging.error(
+                "Zephyr build-%s %s file not found for project %s: %s",
+                build_mode,
+                file,
+                project.config.project_name,
+                err,
             )
+        return False
 
-            output_dir1 = self.checkouts[0].modules_dir / output_path
-            output_dir2 = self.checkouts[1].modules_dir / output_path
+    def _check_build_files(self, project, file):
+        return self._compare_build_files(
+            project, "ro", file
+        ) and self._compare_build_files(project, "rw", file)
 
-            bin_output1 = output_dir1 / "ec.bin"
-            bin_output2 = output_dir2 / "ec.bin"
+    def check_configs(self, projects):
+        """Compare Zephyr EC Config files for two different source trees
 
-            # ELF executables don't compare due to meta data.  Convert to a binary
-            # for the comparison
-            if project.config.output_packer == packer_registry["elf"]:
-                create_bin_from_elf(
-                    elf_input=output_dir1 / "zephyr.elf", bin_output=bin_output1
-                )
-                create_bin_from_elf(
-                    elf_input=output_dir2 / "zephyr.elf", bin_output=bin_output2
-                )
+        Args:
+            projects: List of projects to compare the .config files.
 
-            bin1_path = pathlib.Path(bin_output1)
-            bin2_path = pathlib.Path(bin_output2)
-            if not os.path.isfile(bin1_path) or not os.path.isfile(bin2_path):
-                failed_projects.append(project.config.project_name)
-                logging.error(
-                    "Zephyr EC binary not found for project %s",
-                    project.config.project_name,
-                )
-                continue
-
-            try:
-                subprocess.run(
-                    ["cmp", bin_output1, bin_output2],
-                    check=True,
-                    stdout=subprocess.DEVNULL,
-                    stderr=subprocess.DEVNULL,
-                )
-            except subprocess.CalledProcessError:
-                failed_projects.append(project.config.project_name)
+        Returns:
+            A list of projects that failed to compare.  An empty list indicates that
+            all projects compared successfully.
+        """
+
+        failed_projects = _compare_non_test_projects(
+            projects,
+            self._check_build_files,
+            ".config",
+        )
+        return failed_projects
+
+    def check_devicetrees(self, projects):
+        """Compare Zephyr EC devicetree files for two different source trees
+
+        Args:
+            projects: List of projects to compare the zephyr.dts files.
+
+        Returns:
+            A list of projects that failed to compare.  An empty list indicates that
+            all projects compared successfully.
+        """
 
+        failed_projects = _compare_non_test_projects(
+            projects,
+            self._check_build_files,
+            "zephyr.dts",
+        )
         return failed_projects
diff --git a/zephyr/zmake/zmake/zmake.py b/zephyr/zmake/zmake/zmake.py
index 6454049d53..30d56cc08f 100644
--- a/zephyr/zmake/zmake/zmake.py
+++ b/zephyr/zmake/zmake/zmake.py
@@ -191,6 +191,7 @@ class Zmake:
 
         self.executor = zmake.multiproc.Executor()
         self._sequential = self.jobserver.is_sequential() and not goma
+        self.cmp_failed_projects = {}
         self.failed_projects = []
 
     @property
@@ -342,6 +343,9 @@ class Zmake:
         extra_cflags=None,
         keep_temps=False,
         cmake_defs=None,
+        compare_configs=False,
+        compare_binaries_disable=False,
+        compare_devicetrees=False,
     ):
         """Compare EC builds at two commits."""
         temp_dir = tempfile.mkdtemp(prefix="zcompare-")
@@ -403,9 +407,20 @@ class Zmake:
                     checkout.ref,
                 )
                 return result
-
-        self.failed_projects = cmp_builds.check_binaries(projects)
-
+        if not compare_binaries_disable:
+            failed_projects = cmp_builds.check_binaries(projects)
+            self.cmp_failed_projects["binary"] = failed_projects
+            self.failed_projects.extend(failed_projects)
+        if compare_configs:
+            failed_projects = cmp_builds.check_configs(projects)
+            self.cmp_failed_projects["config"] = failed_projects
+            self.failed_projects.extend(failed_projects)
+        if compare_devicetrees:
+            failed_projects = cmp_builds.check_devicetrees(projects)
+            self.cmp_failed_projects["devicetree"] = failed_projects
+            self.failed_projects.extend(failed_projects)
+
+        self.failed_projects = list(set(self.failed_projects))
         if len(self.failed_projects) == 0:
             self.logger.info("Zephyr compare builds successful:")
             for checkout in cmp_builds.checkouts: