nvmem: recover from failure saving object spanning flash pages

In case power was lost when saving an object spanning two pages, the
initialization process does not return the second flash page into the
pages pool, leaving NVMEM in an inconsistent state.

Proper recovery should reinitialize the second page and return it into
the pool of available flash pages.

If a failure like this happens, to recover the initialization sequence
will have to run one extra cycle, first one bringing the NVMEM state
to the previously covered state of the last object in NVMEM corrupted,
but fitting into a page.

A unit test added to verify proper behavior in this situation.

BRANCH=cr50, cr50-mp
BUG=b:139326267
TEST='make run-nvmem -j' passes. Also added temporary code cause reset
     when saving the second part of an object spanning two flash
     pages. Observed the device properly recover from this failure.

Change-Id: I76ebb6fc73ffc0b07bce34370302f3787914bfb2
Signed-off-by: Vadim Bendebury <vbendeb@chromium.org>
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/ec/+/1766092
Reviewed-by: Andrey Pronin <apronin@chromium.org>

4 files changed, 69 insertions, 7 deletions

diff --git a/common/new_nvmem.c b/common/new_nvmem.c
index a9e0435893..aeef5ccf85 100644
--- a/common/new_nvmem.c
+++ b/common/new_nvmem.c
@@ -1007,6 +1007,12 @@ static enum ec_error_list save_object(const struct nn_container *cont)
 		save_data = (const void *)((uintptr_t)save_data + top_room);
 		save_size -= top_room;
 		start_new_flash_page(save_size);
+#if defined(NVMEM_TEST_BUILD)
+		if (save_size && (failure_mode == TEST_SPANNING_PAGES)) {
+			ccprintf("%s:%d corrupting...\n", __func__, __LINE__);
+			return EC_SUCCESS;
+		}
+#endif
 	}
 
 	if (save_size) {
@@ -2134,8 +2140,10 @@ static enum ec_error_list verify_delimiter(struct nn_container *nc)
 		uint8_t i;
 
 		for (i = 0; i < master_at.list_index; i++)
-			if (list_element_to_ph(i) == dpt.mt.ph)
+			if (list_element_to_ph(i) == dpt.mt.ph) {
 				dpt.list_index = i;
+				break;
+			}
 	}
 
 	while ((rv = get_next_object(&dpt, nc, 0)) == EC_SUCCESS)
@@ -2150,8 +2158,35 @@ static enum ec_error_list verify_delimiter(struct nn_container *nc)
 		size_t remainder_size;
 		const void *p = page_cursor(&master_at.ct);
 
-		remainder_size =
-			CONFIG_FLASH_BANK_SIZE - master_at.ct.data_offset;
+		if (dpt.ct.ph != dpt.mt.ph) {
+			/*
+			 * The last retrieved object is spanning flash page
+			 * boundary.
+			 *
+			 * If this is not the last object in the flash, this
+			 * is an unrecoverable init failure.
+			 */
+			if ((dpt.mt.ph != master_at.mt.ph) ||
+			    (list_element_to_ph(dpt.list_index - 1) !=
+			     dpt.ct.ph))
+				report_no_payload_failure(
+					NVMEMF_CORRUPTED_INIT);
+			/*
+			 * Let's erase the page where the last object spilled
+			 * into.
+			 */
+			flash_physical_erase((uintptr_t)dpt.mt.ph -
+						     CONFIG_PROGRAM_MEMORY_BASE,
+					     CONFIG_FLASH_BANK_SIZE);
+			/*
+			 * And move it to the available pages part of the
+			 * pages list.
+			 */
+			master_at.list_index -= 1;
+			master_at.mt = dpt.ct;
+		}
+
+		remainder_size = CONFIG_FLASH_BANK_SIZE - dpt.ct.data_offset;
 		memset(nc, 0, remainder_size);
 		write_to_flash(p, nc, remainder_size);
 		/* Make sure compaction starts with the new page. */
@@ -2181,8 +2216,11 @@ static enum ec_error_list retrieve_nvmem_contents(void)
 	/* No saved object will exceed CONFIG_FLASH_BANK_SIZE in size. */
 	nc = get_scratch_buffer(CONFIG_FLASH_BANK_SIZE);
 
-	/* Depending on the state of flash, we might have to do this twice. */
-	for (tries = 0; tries < 2; tries++) {
+	/*
+	 * Depending on the state of flash, we might have to do this three
+	 * times.
+	 */
+	for (tries = 0; tries < 3; tries++) {
 		memset(&master_at, 0, sizeof(master_at));
 		memset(nvmem_cache_base(NVMEM_TPM), 0,
 		       nvmem_user_sizes[NVMEM_TPM]);
diff --git a/include/flash_log.h b/include/flash_log.h
index bb4baf651c..64ee2703e3 100644
--- a/include/flash_log.h
+++ b/include/flash_log.h
@@ -64,7 +64,8 @@ enum nvmem_failure_type {
 	NVMEMF_SECTION_VERIFY = 9,
 	NVMEMF_PRE_ERASE_MISMATCH = 10,
 	NVMEMF_PAGE_LIST_OVERFLOW = 11,
-	NVMEMF_CIPHER_ERROR = 12
+	NVMEMF_CIPHER_ERROR = 12,
+	NVMEMF_CORRUPTED_INIT = 13
 };
 
 /* Not all nvmem failures require payload. */
diff --git a/test/nvmem.c b/test/nvmem.c
index 7bac2f9dd7..4e7c3600be 100644
--- a/test/nvmem.c
+++ b/test/nvmem.c
@@ -834,6 +834,7 @@ static int test_nvmem_incomplete_transaction(void)
 	size_t num_objects;
 	uint8_t buf[nvmem_user_sizes[NVMEM_TPM]];
 	uint8_t *p;
+	size_t object_size;
 
 	TEST_ASSERT(prepare_post_migration_nvmem() == EC_SUCCESS);
 	num_objects = fill_obj_offsets(offsets, ARRAY_SIZE(offsets));
@@ -878,6 +879,27 @@ static int test_nvmem_incomplete_transaction(void)
 	/* And verify that nvmem can still successfully initialize. */
 	TEST_ASSERT(nvmem_init() == EC_SUCCESS);
 
+	/*
+	 * Now let's interrupt saving an object spanning two pages.
+	 *
+	 * First, fill up the current page to get close to the limit such that
+	 * the next save will have to span two flash pages.
+	 */
+	object_size = offsets[4] - offsets[3];
+	p = (uint8_t *)evictable_offs_to_addr(offsets[3]) + object_size - 10;
+	while ((master_at.mt.data_offset + object_size +
+		sizeof(struct nn_container)) <= CONFIG_FLASH_BANK_SIZE) {
+		(*p)++;
+		new_nvmem_save();
+	}
+
+	/* This will trigger spilling over the page boundary. */
+	(*p)++;
+	failure_mode = TEST_SPANNING_PAGES;
+	new_nvmem_save();
+	failure_mode = TEST_NO_FAILURE;
+	TEST_ASSERT(nvmem_init() == EC_SUCCESS);
+
 	return EC_SUCCESS;
 }
 
diff --git a/test/nvmem_test.h b/test/nvmem_test.h
index 6aba855bfe..58775d4bcb 100644
--- a/test/nvmem_test.h
+++ b/test/nvmem_test.h
@@ -20,7 +20,8 @@ enum test_failure_mode {
 	TEST_FAIL_WHEN_COMPACTING,
 	TEST_FAIL_SAVING_VAR,
 	TEST_FAIL_FINALIZING_VAR,
-	TEST_FAILED_HASH
+	TEST_FAILED_HASH,
+	TEST_SPANNING_PAGES
 };
 
 extern enum test_failure_mode failure_mode;