HACK: vboot_api_kernel: Retry EC RW update on unexpected hash failurefirmware-gru-8785.B

Certain versions of kevin EC RO will occasionally flash corrupted data
to a sector rather than the intended flash payload. This manifests
itself after SW sync as a RW hash mismatch. When such a mismatch occurs,
retry the SW sync process up to five times before returning failure and
entering recovery mode.

BUG=b:35587287
BRANCH=gru
TEST=Stress test SW sync with faulty RO, verify that recovery mode is
never entered and logs show SW sync was retried after hash fail.

Signed-off-by: Shawn Nematbakhsh <shawnn@chromium.org>
Change-Id: I1f38300a95c31e889f94b42718e2996220e9f5a9
Reviewed-on: https://chromium-review.googlesource.com/447957
Reviewed-by: Julius Werner <jwerner@chromium.org>
Reviewed-by: Randall Spangler <rspangler@chromium.org>

2 files changed, 35 insertions, 5 deletions

diff --git a/firmware/include/vboot_api.h b/firmware/include/vboot_api.h
index 75b20f38..87a538b6 100644
--- a/firmware/include/vboot_api.h
+++ b/firmware/include/vboot_api.h
@@ -130,6 +130,10 @@ enum VbErrorPredefined_t {
 	/* VbExEcGetExpectedRWHash() may return the following codes */
 	/* Compute expected RW hash from the EC image; BIOS doesn't have it */
 	VBERROR_EC_GET_EXPECTED_HASH_FROM_IMAGE = 0x20000,
+
+	/* EcUpdateImage() may return the following codes */
+	/* EC flash seemed to complete OK but region hash value is unexpected */
+	VBERROR_EC_HASH_CMP_FAILED            = 0x30000,
 };
 
 
diff --git a/firmware/lib/vboot_api_kernel.c b/firmware/lib/vboot_api_kernel.c
index 4aa2fbdc..882d5c34 100644
--- a/firmware/lib/vboot_api_kernel.c
+++ b/firmware/lib/vboot_api_kernel.c
@@ -858,8 +858,12 @@ static VbError_t EcUpdateImage(int devidx, VbCommonParams *cparams,
 		VBDEBUG(("EcUpdateImage() - "
 			 "Failed to update EC-%s\n", rw_request ?
 			 "RW" : "RO"));
-		VbSetRecoveryRequest(VBNV_RECOVERY_EC_UPDATE);
-		return VBERROR_EC_REBOOT_TO_RO_REQUIRED;
+		if (rw_request) {
+			return VBERROR_EC_HASH_CMP_FAILED;
+		} else {
+			VbSetRecoveryRequest(VBNV_RECOVERY_EC_UPDATE);
+			return VBERROR_EC_REBOOT_TO_RO_REQUIRED;
+		}
 	}
 
 	return VBERROR_SUCCESS;
@@ -875,6 +879,7 @@ VbError_t VbEcSoftwareSync(int devidx, VbCommonParams *cparams)
 	enum VbSelectFirmware_t select_ro = VB_SELECT_FIRMWARE_READONLY;
 	int in_rw = 0;
 	int ro_try_count = 2;
+	int rw_try_count = 5;
 	int num_tries = 0;
 	uint32_t try_ro_sync, recovery_request;
 	int rv, updated_rw, updated_ro;
@@ -946,13 +951,33 @@ VbError_t VbEcSoftwareSync(int devidx, VbCommonParams *cparams)
 
 	VBDEBUG(("VbEcSoftwareSync() check for RW update.\n"));
 
-	/* Update the RW Image. */
-	rv = EcUpdateImage(devidx, cparams, select_rw, &updated_rw, in_rw);
+	/*
+	 * HACK(b:35587287): Retry EcUpdateImage on hash mismatch since
+	 * flashing is known to flake due to faulty EC RO.
+	 */
+	while (num_tries++ < rw_try_count) {
+		/* Update the RW Image. */
+		rv = EcUpdateImage(devidx,
+				   cparams,
+				   select_rw,
+				   &updated_rw,
+				   in_rw);
+
+		/* Stop on success, or error other than hash mismatch */
+		if (rv != VBERROR_EC_HASH_CMP_FAILED)
+			break;
+	}
 
 	if (rv != VBERROR_SUCCESS) {
 		VBDEBUG(("VbEcSoftwareSync() - "
 			 "EcUpdateImage() returned %d\n", rv));
-		return rv;
+
+		if (rv == VBERROR_EC_HASH_CMP_FAILED) {
+			/* All retries failed, go to recovery */
+			VbSetRecoveryRequest(VBNV_RECOVERY_EC_UPDATE);
+			return VBERROR_EC_REBOOT_TO_RO_REQUIRED;
+		} else
+			return rv;
 	}
 
 	/* Tell EC to jump to its RW image */
@@ -987,6 +1012,7 @@ VbError_t VbEcSoftwareSync(int devidx, VbCommonParams *cparams)
 		VbNvGet(&vnc, VBNV_RECOVERY_REQUEST, &recovery_request);
 
 		/* Update the RO Image. */
+		num_tries = 0;
 		while (num_tries < ro_try_count) {
 			VBDEBUG(("VbEcSoftwareSync() RO Software Sync\n"));