futility: updater: Add --manifest to scan and print archive info

The firmware updater packages used to rely on a pre-generated VERSION
file to report what files were included and their image versions. Its
format was hard to parse, and may be out-dated if people repack without
updating VERSION file.

The firmware updater today has the ability to read and parse version,
key hash, ... etc everything we need, so it seems more reasonable to
just let firmware updater scan updater package and print the information
in JSON format, so it will be very easy to fetch latest information.

To make sure the output is purely JSON, the start and end messages are
now sent to stderr instead of stdout.

BUG=chromium:875551
TEST=TEST=make futil; tests/futility/run_test_scripts.sh $(pwd)/build/futility
BRANCH=None

Change-Id: Ifa468fbb3adf798c7931f015258e6c6ce93de993
Signed-off-by: Hung-Te Lin <hungte@chromium.org>
Reviewed-on: https://chromium-review.googlesource.com/1260804

6 files changed, 346 insertions, 5 deletions

diff --git a/futility/cmd_update.c b/futility/cmd_update.c
index df09dc8c..f382b9ca 100644
--- a/futility/cmd_update.c
+++ b/futility/cmd_update.c
@@ -26,6 +26,7 @@ static struct option const long_opts[] = {
 	{"quirks", 1, NULL, 'f'},
 	{"list-quirks", 0, NULL, 'L'},
 	{"mode", 1, NULL, 'm'},
+	{"manifest", 0, NULL, 'A'},
 	{"factory", 0, NULL, 'Y'},
 	{"force", 0, NULL, 'F'},
 	{"programmer", 1, NULL, 'p'},
@@ -50,6 +51,7 @@ static void print_help(int argc, char *argv[])
 		"    --pd_image=FILE \tPD firmware image (i.e, pd.bin)\n"
 		"-t, --try           \tTry A/B update on reboot if possible\n"
 		"-a, --archive=PATH  \tRead resources from archive\n"
+		"    --manifest      \tPrint out a JSON manifest and exit\n"
 		"-p, --programmer=PRG\tChange AP (host) flashrom programmer\n"
 		"    --quirks=LIST   \tSpecify the quirks to apply\n"
 		"    --list-quirks   \tPrint all available quirks\n"
@@ -75,7 +77,7 @@ static int do_update(int argc, char *argv[])
 	struct updater_config_arguments args = {0};
 	int i, errorcnt = 0;
 
-	printf(">> Firmware updater started.\n");
+	fprintf(stderr, ">> Firmware updater started.\n");
 	cfg = updater_new_config();
 	assert(cfg);
 
@@ -106,6 +108,8 @@ static int do_update(int argc, char *argv[])
 		case 'm':
 			args.mode = optarg;
 			break;
+		case 'A':
+			args.do_manifest = 1;
 			break;
 		case 'Y':
 			args.is_factory = 1;
@@ -157,7 +161,7 @@ static int do_update(int argc, char *argv[])
 	}
 	if (!errorcnt)
 		errorcnt += updater_setup_config(cfg, &args);
-	if (!errorcnt) {
+	if (!errorcnt && !args.do_manifest) {
 		int r = update_firmware(cfg);
 		if (r != UPDATE_ERR_DONE) {
 			r = Min(r, UPDATE_ERR_UNKNOWN);
@@ -165,9 +169,9 @@ static int do_update(int argc, char *argv[])
 			errorcnt++;
 		}
 	}
-	printf(">> %s: Firmware updater %s.\n",
-	       errorcnt ? "FAILED": "DONE",
-	       errorcnt ? "stopped due to error" : "exited successfully");
+	fprintf(stderr, ">> %s: Firmware updater %s.\n",
+		errorcnt ? "FAILED": "DONE",
+		errorcnt ? "stopped due to error" : "exited successfully");
 
 	updater_delete_config(cfg);
 	return !!errorcnt;
diff --git a/futility/updater.c b/futility/updater.c
index 95ec6612..94e672dd 100644
--- a/futility/updater.c
+++ b/futility/updater.c
@@ -1673,12 +1673,19 @@ int updater_setup_config(struct updater_config *cfg,
 	const char *default_quirks = NULL;
 	int is_factory = arg->is_factory;
 	const char *archive_path = arg->archive;
+	struct manifest *manifest = NULL;
 
 	/* Setup values that may change output or decision of other argument. */
 	cfg->verbosity = arg->verbosity;
 	if (arg->force_update)
 		cfg->force_update = 1;
 
+	/* Check incompatible options and return early. */
+	if (arg->do_manifest && !arg->archive) {
+		ERROR("Manifest is only available for archive.");
+		return ++errorcnt;
+	}
+
 	/* Setup update mode. */
 	if (arg->try_update)
 		cfg->try_update = 1;
@@ -1735,9 +1742,20 @@ int updater_setup_config(struct updater_config *cfg,
 		ERROR("Failed to open archive: %s", archive_path);
 		return ++errorcnt;
 	}
+
 	errorcnt += updater_load_images(
 			cfg, arg->image, arg->ec_image, arg->pd_image);
 
+	if (arg->do_manifest) {
+		manifest = new_manifest_from_archive(cfg->archive);
+		if (!manifest) {
+			ERROR("Failure in archive: %s", archive_path);
+			return ++errorcnt;
+		}
+		print_json_manifest(manifest);
+		return errorcnt;
+	}
+
 	/*
 	 * Quirks must be loaded after images are loaded because we use image
 	 * contents to decide default quirks to load. Also, we have to load
@@ -1758,6 +1776,8 @@ int updater_setup_config(struct updater_config *cfg,
 		errorcnt++;
 		ERROR("Factory mode needs WP disabled.");
 	}
+	if (manifest)
+		delete_manifest(manifest);
 	return errorcnt;
 }
 
diff --git a/futility/updater.h b/futility/updater.h
index a6cb3a72..366ecf10 100644
--- a/futility/updater.h
+++ b/futility/updater.h
@@ -118,6 +118,20 @@ struct updater_config_arguments {
 	int verbosity;
 };
 
+struct model_config {
+	char *name;
+	char *image, *ec_image, *pd_image;
+	char *signature_id;
+};
+
+struct manifest {
+	int num;
+	int default_model;
+	int has_keyset;
+	struct model_config *models;
+	struct archive *archive;
+};
+
 enum updater_error_codes {
 	UPDATE_ERR_DONE,
 	UPDATE_ERR_NEED_RO_UPDATE,
@@ -265,4 +279,16 @@ int archive_has_entry(struct archive *ar, const char *name);
 int archive_read_file(struct archive *ar, const char *fname,
 		      uint8_t **data, uint32_t *size);
 
+/*
+ * Creates a new manifest object by scanning files in archive.
+ * Returns the manifest on success, otherwise NULL for failure.
+ */
+struct manifest *new_manifest_from_archive(struct archive *archive);
+
+/* Releases all resources allocated by given manifest object. */
+void delete_manifest(struct manifest *manifest);
+
+/* Prints the information of objects in manifest (models and images) in JSON. */
+void print_json_manifest(const struct manifest *manifest);
+
 #endif  /* VBOOT_REFERENCE_FUTILITY_UPDATER_H_ */
diff --git a/futility/updater_archive.c b/futility/updater_archive.c
index dd36f101..736d459e 100644
--- a/futility/updater_archive.c
+++ b/futility/updater_archive.c
@@ -22,6 +22,50 @@
 #include "updater.h"
 #include "vb2_common.h"
 
+/*
+ * A firmware update package (archive) is a file packed by either shar(1) or
+ * zip(1). See https://chromium.googlesource.com/chromiumos/platform/firmware/
+ * for more information.
+ *
+ * A package for single board (i.e., not Unified Build) will have all the image
+ * files in top folder:
+ *  - host: 'bios.bin'
+ *  - ec: 'ec.bin'
+ *  - pd: 'pd.bin'
+ * If white label is supported, a 'keyset/' folder will be available, with key
+ * files in it:
+ *  - rootkey.$WLTAG
+ *  - vblock_A.$WLTAG
+ *  - vblock_B.$WLTAG
+ * The $WLTAG should come from VPD value 'whitelabel_tag', or the
+ * 'customization_id'. Note 'customization_id' is in format LOEM[-VARIANT] and
+ * we can only take LOEM as $WLTAG, for example A-B => $WLTAG=A.
+ *
+ * A package for Unified Build is more complicated. There will be a models/
+ * folder, and each model (by $(mosys platform model) ) should appear as a sub
+ * folder, with a 'setvars.sh' file inside. The 'setvars.sh' is a shell script
+ * describing what files should be used and the signature ID ($SIGID) to use.
+ *
+ * Similar to write label in non-Unified-Build, the keys and vblock files will
+ * be in 'keyset/' folder:
+ *  - rootkey.$SIGID
+ *  - vblock_A.$SIGID
+ *  - vblock_B.$SIGID
+ * If $SIGID starts with 'sig-id-in-*' then we have to replace it by VPD value
+ * 'whitelabel_tag' as '$MODEL-$WLTAG'.
+ */
+
+static const char * const SETVARS_IMAGE_MAIN = "IMAGE_MAIN",
+		  * const SETVARS_IMAGE_EC = "IMAGE_EC",
+		  * const SETVARS_IMAGE_PD = "IMAGE_PD",
+		  * const SETVARS_SIGNATURE_ID = "SIGNATURE_ID",
+		  * const SIG_ID_IN_VPD_PREFIX = "sig-id-in",
+		  * const DIR_KEYSET = "keyset",
+		  * const DIR_MODELS = "models",
+		  * const DEFAULT_MODEL_NAME = "default",
+		  * const PATH_STARTSWITH_KEYSET = "keyset/",
+		  * const PATH_ENDSWITH_SERVARS = "/setvars.sh";
+
 struct archive {
 	void *handle;
 
@@ -35,6 +79,10 @@ struct archive {
 			 uint8_t **data, uint32_t *size);
 };
 
+/*
+ * -- Begin of archive implementations --
+ */
+
 /* Callback for archive_open on a general file system. */
 static void *archive_fallback_open(const char *name)
 {
@@ -309,3 +357,236 @@ int archive_read_file(struct archive *ar, const char *fname,
 		return archive_fallback_read_file(NULL, fname, data, size);
 	return ar->read_file(ar->handle, fname, data, size);
 }
+
+/*
+ * -- End of archive implementations --
+ */
+
+/* Returns 1 if name ends by given pattern, otherwise 0. */
+static int str_endswith(const char *name, const char *pattern)
+{
+	size_t name_len = strlen(name), pattern_len = strlen(pattern);
+	if (name_len < pattern_len)
+		return 0;
+	return strcmp(name + name_len - pattern_len, pattern) == 0;
+}
+
+/* Returns 1 if name starts by given pattern, otherwise 0. */
+static int str_startswith(const char *name, const char *pattern)
+{
+	return strncmp(name, pattern, strlen(pattern)) == 0;
+}
+
+/*
+ * Reads and parses a setvars type file from archive, then stores into config.
+ * Returns 0 on success (at least one entry found), otherwise failure.
+ */
+static int model_config_parse_setvars_file(
+		struct model_config *cfg, struct archive *archive,
+		const char *fpath)
+{
+	uint8_t *data;
+	uint32_t len;
+
+	char *ptr_line, *ptr_token;
+	char *line, *k, *v;
+	int valid = 0;
+
+	if (archive_read_file(archive, fpath, &data, &len) != 0) {
+		ERROR("Failed reading: %s", fpath);
+		return -1;
+	}
+
+	/* Valid content should end with \n, or \"; ensure ASCIIZ for parsing */
+	if (len)
+		data[len - 1] = '\0';
+
+	for (line = strtok_r((char *)data, "\n\r", &ptr_line); line;
+	     line = strtok_r(NULL, "\n\r", &ptr_line)) {
+		/* Format: KEY="value" */
+		k = strtok_r(line, "=", &ptr_token);
+		if (!k)
+			continue;
+		v = strtok_r(NULL, "\"", &ptr_token);
+		if (!v)
+			continue;
+
+		if (strcmp(k, SETVARS_IMAGE_MAIN) == 0)
+			cfg->image = strdup(v);
+		else if (strcmp(k, SETVARS_IMAGE_EC) == 0)
+			cfg->ec_image = strdup(v);
+		else if (strcmp(k, SETVARS_IMAGE_PD) == 0)
+			cfg->pd_image = strdup(v);
+		else if (strcmp(k, SETVARS_SIGNATURE_ID) == 0)
+			cfg->signature_id = strdup(v);
+		else
+			continue;
+		valid++;
+	}
+	free(data);
+	return valid == 0;
+}
+
+/*
+ * Adds and copies one new model config to the existing list of given manifest.
+ * Returns a pointer to the newly allocated config, or NULL on failure.
+ */
+static struct model_config *manifest_add_model(
+		struct manifest *manifest,
+		const struct model_config *cfg)
+{
+	struct model_config *model;
+	manifest->num++;
+	manifest->models = (struct model_config *)realloc(
+			manifest->models, manifest->num * sizeof(*model));
+	if (!manifest->models) {
+		ERROR("Internal error: failed to allocate buffer.");
+		return NULL;
+	}
+	model = &manifest->models[manifest->num - 1];
+	memcpy(model, cfg, sizeof(*model));
+	return model;
+}
+
+/*
+ * A callback function for manifest to scan files in archive.
+ * Returns 0 to keep scanning, or non-zero to stop.
+ */
+static int manifest_scan_entries(const char *name, void *arg)
+{
+	struct manifest *manifest = (struct manifest *)arg;
+	struct archive *archive = manifest->archive;
+	struct model_config model = {0};
+	char *slash;
+
+	if (str_startswith(name, PATH_STARTSWITH_KEYSET))
+		manifest->has_keyset = 1;
+	if (!str_endswith(name, PATH_ENDSWITH_SERVARS))
+		return 0;
+
+	/* name: models/$MODEL/setvars.sh */
+	model.name = strdup(strchr(name, '/') + 1);
+	slash = strchr(model.name, '/');
+	if (slash)
+		*slash = '\0';
+
+	DEBUG("Found model <%s> setvars: %s", model.name, name);
+	if (model_config_parse_setvars_file(&model, archive, name)) {
+		ERROR("Invalid setvars file: %s", name);
+		return 0;
+	}
+
+	/* In legacy setvars.sh, the ec_image and pd_image may not exist. */
+	if (model.ec_image && !archive_has_entry(archive, model.ec_image)) {
+		DEBUG("Ignore non-exist EC image: %s", model.ec_image);
+		free(model.ec_image);
+		model.ec_image = NULL;
+	}
+	if (model.pd_image && !archive_has_entry(archive, model.pd_image)) {
+		DEBUG("Ignore non-exist PD image: %s", model.pd_image);
+		free(model.pd_image);
+		model.pd_image = NULL;
+	}
+	return !manifest_add_model(manifest, &model);
+}
+
+/*
+ * Creates a new manifest object by scanning files in archive.
+ * Returns the manifest on success, otherwise NULL for failure.
+ */
+struct manifest *new_manifest_from_archive(struct archive *archive)
+{
+	struct manifest manifest = {0}, *new_manifest;
+	struct model_config model = {0};
+	const char * const image_name = "bios.bin",
+	           * const ec_name = "ec.bin",
+		   * const pd_name = "pd.bin";
+
+	manifest.archive = archive;
+	manifest.default_model = -1;
+	archive_walk(archive, &manifest, manifest_scan_entries);
+	if (manifest.num == 0) {
+		/* Try to load from current folder. */
+		if (!archive_has_entry(archive, image_name))
+			return 0;
+		model.image = strdup(image_name);
+		if (archive_has_entry(archive, ec_name))
+			model.ec_image = strdup(ec_name);
+		if (archive_has_entry(archive, pd_name))
+			model.pd_image = strdup(pd_name);
+		model.name = strdup(DEFAULT_MODEL_NAME);
+		manifest_add_model(&manifest, &model);
+		manifest.default_model = manifest.num - 1;
+	}
+	DEBUG("%d model(s) loaded.", manifest.num);
+	if (!manifest.num) {
+		ERROR("No valid configurations found from archive.");
+		return NULL;
+	}
+
+	new_manifest = (struct manifest *)malloc(sizeof(manifest));
+	if (!new_manifest) {
+		ERROR("Internal error: memory allocation error.");
+		return NULL;
+	}
+	memcpy(new_manifest, &manifest, sizeof(manifest));
+	return new_manifest;
+}
+
+/* Releases all resources allocated by given manifest object. */
+void delete_manifest(struct manifest *manifest)
+{
+	int i;
+	assert(manifest);
+	for (i = 0; i < manifest->num; i++) {
+		struct model_config *model = &manifest->models[i];
+		free(model->name);
+		free(model->signature_id);
+		free(model->image);
+		free(model->ec_image);
+		free(model->pd_image);
+	}
+	free(manifest->models);
+	free(manifest);
+}
+
+/* Prints the information of given image file in JSON format. */
+static void print_json_image(
+		const char *name, const char *fpath, struct archive *archive,
+		int indent, int is_host)
+{
+	struct firmware_image image = {0};
+	if (!fpath)
+		return;
+	load_firmware_image(&image, fpath, archive);
+	if (!is_host)
+		printf(",\n");
+	printf("%*s\"%s\": { \"versions\": { \"ro\": \"%s\", \"rw\": \"%s\" },",
+	       indent, "", name, image.ro_version, image.rw_version_a);
+	printf("\n%*s\"image\": \"%s\" }", indent + 2, "", fpath);
+	free_firmware_image(&image);
+}
+
+/* Prints the information of objects in manifest (models and images) in JSON. */
+void print_json_manifest(const struct manifest *manifest)
+{
+	int i, indent;
+	struct archive *ar = manifest->archive;
+
+	printf("{\n");
+	for (i = 0, indent = 2; i < manifest->num; i++) {
+		struct model_config *m = &manifest->models[i];
+		printf("%s%*s\"%s\": {\n", i ? ",\n" : "", indent, "", m->name);
+		indent += 2;
+		print_json_image("host", m->image, ar, indent, 1);
+		print_json_image("ec", m->ec_image, ar, indent, 0);
+		print_json_image("pd", m->pd_image, ar, indent, 0);
+		if (m->signature_id)
+			printf(",\n%*s\"signature_id\": \"%s\"", indent, "",
+			       m->signature_id);
+		printf("\n  }");
+		indent -= 2;
+		assert(indent == 2);
+	}
+	printf("\n}\n");
+}
diff --git a/tests/futility/link.manifest.json b/tests/futility/link.manifest.json
new file mode 100644
index 00000000..2cd5281b
--- /dev/null
+++ b/tests/futility/link.manifest.json
@@ -0,0 +1,6 @@
+{
+  "default": {
+    "host": { "versions": { "ro": "Google_Link.2695.1.133", "rw": "Google_Link.2695.1.133" },
+      "image": "bios.bin" }
+  }
+}
diff --git a/tests/futility/test_update.sh b/tests/futility/test_update.sh
index cd1bcb61..c7880486 100755
--- a/tests/futility/test_update.sh
+++ b/tests/futility/test_update.sh
@@ -283,11 +283,15 @@ test_update "Full update (--quirks min_platform_version)" \
 	-i "${TO_IMAGE}" --wp=0 --sys_props 0,0x10001,1,3
 
 mkdir -p "${TMP}.archive"
+cp -f "${LINK_BIOS}" "${TMP}.archive/bios.bin"
 cp -f "${TO_IMAGE}" "${TMP}.archive/image_in_archive"
 test_update "Full update (--archive)" \
 	"${FROM_IMAGE}" "${TMP}.expected.full" \
 	-a "${TMP}.archive" \
 	-i "image_in_archive" --wp=0 --sys_props 0,0x10001,1,3
+echo "TEST: Manifest (--manifest)"
+${FUTILITY} update -a "${TMP}.archive" --manifest >"${TMP}.json.out"
+cmp "${TMP}.json.out" "${SCRIPTDIR}/link.manifest.json"
 
 # Test special programmer
 if type flashrom >/dev/null 2>&1; then