nvmem: reinitialize on catastrophic errors

If there is an NVMEM corruption which causes a reboot, persisting
corruption would cause rolling reboot of the device.

It is a harsh remedy, but at least the device remains functional.
Added a log entry to explicitly report NVMEM reinitialization.

BRANCH=cr50, cr50-mp
BUG=b:139326267
TEST=verified by erasing a flash page assigned to NVMEM and rebooting
     the device. Observed two new flash log entries.

Change-Id: Id292d7c66b81c03bbe3cd343ae75acb62d06582d
Signed-off-by: Vadim Bendebury <vbendeb@chromium.org>
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/ec/+/1758805
Reviewed-by: Andrey Pronin <apronin@chromium.org>

3 files changed, 88 insertions, 16 deletions

diff --git a/common/new_nvmem.c b/common/new_nvmem.c
index aeef5ccf85..90976e1ec0 100644
--- a/common/new_nvmem.c
+++ b/common/new_nvmem.c
@@ -316,7 +316,7 @@ static struct delete_candidates {
  */
 static uint8_t page_list[NEW_NVMEM_TOTAL_PAGES];
 static uint32_t next_evict_obj_base;
-
+static uint8_t init_in_progress;
 /*
  * Mutex to protect flash space containing NVMEM objects. All operations
  * modifying the flash contents or relying on its consistency (like searching
@@ -364,18 +364,41 @@ test_export_static struct access_tracker master_at;
 
 test_export_static enum ec_error_list browse_flash_contents(int print);
 static enum ec_error_list save_container(struct nn_container *nc);
+static void invalidate_nvmem_flash(void);
 
 /* Log NVMEM problem as per passed in payload and size, and reboot. */
 static void report_failure(struct nvmem_failure_payload *payload,
 			   size_t payload_union_size)
 {
-	flash_log_add_event(
-		FE_LOG_NVMEM,
-		payload_union_size +
-			offsetof(struct nvmem_failure_payload, size),
-		payload);
-	ccprintf("Logging failure %d\n", payload->failure_type);
+	if (init_in_progress) {
+		/*
+		 * This must be a rolling reboot, let's invalidate flash
+		 * storage to stop this.
+		 */
+		invalidate_nvmem_flash();
+	}
+
+	flash_log_add_event(FE_LOG_NVMEM,
+			    payload_union_size +
+				    offsetof(struct nvmem_failure_payload,
+					     size),
+			    payload);
+
+	ccprintf("Logging failure %d, will %sreinit\n", payload->failure_type,
+		 init_in_progress ? "" : "not ");
+
+	if (init_in_progress) {
+		struct nvmem_failure_payload fp;
+
+		fp.failure_type = NVMEMF_NVMEM_WIPE;
+
+		flash_log_add_event(
+			FE_LOG_NVMEM,
+			offsetof(struct nvmem_failure_payload, size), &fp);
+	}
+
 	cflush();
+
 	system_reset(SYSTEM_RESET_MANUALLY_TRIGGERED | SYSTEM_RESET_HARD);
 }
 
@@ -634,6 +657,26 @@ static enum ec_error_list write_to_flash(const void *flash_addr,
 }
 
 /*
+ * Corrupt headers of all active pages thus invalidating the entire NVMEM
+ * flash storage.
+ */
+static void invalidate_nvmem_flash(void)
+{
+	size_t i;
+	struct nn_page_header *ph;
+	struct nn_page_header bad_ph;
+
+	memset(&bad_ph, 0, sizeof(bad_ph));
+
+	for (i = 0; i < ARRAY_SIZE(page_list); i++) {
+		ph = list_element_to_ph(i);
+		if (!ph)
+			continue;
+		write_to_flash(ph, &bad_ph, sizeof(*ph));
+	}
+}
+
+/*
  * When initializing flash for the first time - set the proper first page
  * header.
  */
@@ -764,7 +807,21 @@ test_export_static enum ec_error_list get_next_object(struct access_tracker *at,
 
 		/* And calculate hash. */
 		if (!container_is_valid(ch)) {
-			ccprintf("%s: container hash mismatch!\n", __func__);
+			struct nvmem_failure_payload fp;
+
+			if (!init_in_progress)
+				report_no_payload_failure(
+					NVMEMF_CONTAINER_HASH_MISMATCH);
+			/*
+			 * During init there might be a way to deal with
+			 * this, let's just log this and continue.
+			 */
+			fp.failure_type = NVMEMF_CONTAINER_HASH_MISMATCH;
+			flash_log_add_event(
+				FE_LOG_NVMEM,
+				offsetof(struct nvmem_failure_payload, size),
+				&fp);
+
 			return EC_ERROR_INVAL;
 		}
 
@@ -2253,13 +2310,8 @@ static enum ec_error_list retrieve_nvmem_contents(void)
 			break;
 	}
 
-	if (rv != EC_SUCCESS) {
-		/*
-		 * Something is really messed up, need to wipe out and start
-		 * from scratch?
-		 */
-		ccprintf("%s:%d FAILURE!\n", __func__, __LINE__);
-	}
+	if (rv != EC_SUCCESS)
+		report_no_payload_failure(NVMEMF_UNRECOVERABLE_INIT);
 
 	rv = verify_reserved(res_bitmap, nc);
 
@@ -2276,6 +2328,8 @@ enum ec_error_list new_nvmem_init(void)
 	if (!crypto_enabled())
 		return EC_ERROR_INVAL;
 
+	init_in_progress = 1;
+
 	total_var_space = 0;
 
 	/* Initialize NVMEM indices. */
@@ -2293,6 +2347,8 @@ enum ec_error_list new_nvmem_init(void)
 
 	unlock_mutex(__LINE__);
 
+	init_in_progress = 0;
+
 	CPRINTS("init took %d", (uint32_t)(init.val - start.val));
 
 	return rv;
diff --git a/include/flash_log.h b/include/flash_log.h
index 64ee2703e3..e504df6ee7 100644
--- a/include/flash_log.h
+++ b/include/flash_log.h
@@ -65,7 +65,10 @@ enum nvmem_failure_type {
 	NVMEMF_PRE_ERASE_MISMATCH = 10,
 	NVMEMF_PAGE_LIST_OVERFLOW = 11,
 	NVMEMF_CIPHER_ERROR = 12,
-	NVMEMF_CORRUPTED_INIT = 13
+	NVMEMF_CORRUPTED_INIT = 13,
+	NVMEMF_CONTAINER_HASH_MISMATCH = 14,
+	NVMEMF_UNRECOVERABLE_INIT = 15,
+	NVMEMF_NVMEM_WIPE = 16,
 };
 
 /* Not all nvmem failures require payload. */
diff --git a/test/nvmem.c b/test/nvmem.c
index 4e7c3600be..df5457273b 100644
--- a/test/nvmem.c
+++ b/test/nvmem.c
@@ -11,6 +11,7 @@
 #include "console.h"
 #include "crc.h"
 #include "flash.h"
+#include "flash_log.h"
 #include "new_nvmem.h"
 #include "nvmem.h"
 #include "printf.h"
@@ -835,6 +836,7 @@ static int test_nvmem_incomplete_transaction(void)
 	uint8_t buf[nvmem_user_sizes[NVMEM_TPM]];
 	uint8_t *p;
 	size_t object_size;
+	union entry_u e;
 
 	TEST_ASSERT(prepare_post_migration_nvmem() == EC_SUCCESS);
 	num_objects = fill_obj_offsets(offsets, ARRAY_SIZE(offsets));
@@ -898,8 +900,19 @@ static int test_nvmem_incomplete_transaction(void)
 	failure_mode = TEST_SPANNING_PAGES;
 	new_nvmem_save();
 	failure_mode = TEST_NO_FAILURE;
+
+	/* Drain the event log. */
+	e.r.timestamp = 0;
+	while (flash_log_dequeue_event(e.r.timestamp, e.entry, sizeof(e)) > 0)
+		;
+
 	TEST_ASSERT(nvmem_init() == EC_SUCCESS);
 
+	/* Let's verify that a container mismatch event has been added. */
+	TEST_ASSERT(flash_log_dequeue_event(e.r.timestamp, e.entry, sizeof(e))
+		    > 0);
+	TEST_ASSERT(e.r.type == FE_LOG_NVMEM);
+	TEST_ASSERT(e.r.payload[0] == NVMEMF_CONTAINER_HASH_MISMATCH);
 	return EC_SUCCESS;
 }