futility: updater: support reading the archives using libarchive

The firmware archive files from buildbot are usually released in
tar+bzip2 format. Although the standard format of the archive
(chromeos-firmwareupdate) is ZIP, it is very helpful if the developers
can quickly run the update using the standard tar.bz2 archive files.

BUG=b:230679721
TEST=futility update --manifest -a \
       ~/Downloads/ChromeOS-firmware-R91-13885.3.0-asurada.tar.bz2
     QEMU_LD_PREFIX=/build/cherry qemu-arm /build/cherry/usr/bin/futility \
       update --manifest -a \
       ~/Downloads/ChromeOS-firmware-R91-13885.3.0-asurada.tar.bz2
BRANCH=None

Change-Id: Ibbab2e8226a00e8b5b292293af570eda37b31a8a
Signed-off-by: Hung-Te Lin <hungte@chromium.org>
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/vboot_reference/+/3644540
Reviewed-by: Yu-Ping Wu <yupingso@chromium.org>

2 files changed, 273 insertions, 14 deletions

diff --git a/Makefile b/Makefile
index 6340fb29..beaffafa 100644
--- a/Makefile
+++ b/Makefile
@@ -275,6 +275,13 @@ ifneq ($(filter-out 0,${HAVE_LIBZIP}),)
   LIBZIP_LIBS := $(shell ${PKG_CONFIG} --libs libzip)
 endif
 
+LIBARCHIVE_VERSION := $(shell ${PKG_CONFIG} --modversion libarchive 2>/dev/null)
+HAVE_LIBARCHIVE := $(if ${LIBARCHIVE_VERSION},1)
+ifneq ($(filter-out 0,${HAVE_LIBARCHIVE}),)
+  CFLAGS += -DHAVE_LIBARCHIVE $(shell ${PKG_CONFIG} --cflags libarchive)
+  LIBARCHIVE_LIBS := $(shell ${PKG_CONFIG} --libs libarchive)
+endif
+
 HAVE_CROSID := $(shell ${PKG_CONFIG} --exists crosid && echo 1)
 ifeq ($(HAVE_CROSID),1)
   CFLAGS += -DHAVE_CROSID $(shell ${PKG_CONFIG} --cflags crosid)
@@ -1040,7 +1047,8 @@ signing_install: $(if ${SDK_BUILD},\
 futil: ${FUTIL_BIN}
 
 # FUTIL_LIBS is shared by FUTIL_BIN and TEST_FUTIL_BINS.
-FUTIL_LIBS = ${CROSID_LIBS} ${CRYPTO_LIBS} ${LIBZIP_LIBS} ${FLASHROM_LIBS}
+FUTIL_LIBS = ${CROSID_LIBS} ${CRYPTO_LIBS} ${LIBZIP_LIBS} ${LIBARCHIVE_LIBS} \
+	${FLASHROM_LIBS}
 
 ${FUTIL_BIN}: LDLIBS += ${FUTIL_LIBS}
 ${FUTIL_BIN}: ${FUTIL_OBJS} ${UTILLIB} ${FWLIB}
diff --git a/futility/updater_archive.c b/futility/updater_archive.c
index aa3e2f93..dafce3c7 100644
--- a/futility/updater_archive.c
+++ b/futility/updater_archive.c
@@ -19,6 +19,11 @@
 #include <sys/time.h>
 #include <unistd.h>
 
+#ifdef HAVE_LIBARCHIVE
+#include <archive.h>
+#include <archive_entry.h>
+#endif
+
 #ifdef HAVE_LIBZIP
 #ifndef __clang__
 /* If libzip headers were built for Clang but later get included with GCC you
@@ -238,6 +243,226 @@ static int archive_fallback_write_file(void *handle, const char *fname,
 	return r;
 }
 
+#ifdef HAVE_LIBARCHIVE
+
+/*
+ * For stream-based archives (e.g., tar+gz) we want to create a cache for
+ * storing the names and contents for later processing.
+ */
+struct archive_cache {
+	char *name;
+	uint8_t *data;
+	int64_t mtime;
+	size_t size;
+	int has_data;
+	struct archive_cache *next;
+};
+
+/* Add a new cache node to an existing cache list and return the new head. */
+static struct archive_cache *archive_cache_new(struct archive_cache *cache,
+					       const char *name)
+{
+	struct archive_cache *c;
+
+	c = (struct archive_cache *)calloc(sizeof(*c), 1);
+	if (!c)
+		return NULL;
+
+	c->name = strdup(name);
+	if (!c->name) {
+		free(c);
+		return NULL;
+	}
+
+	c->next = cache;
+	return c;
+}
+
+/* Find and return an entry (by name) from the cache. */
+static struct archive_cache *archive_cache_find(struct archive_cache *c,
+						const char *name)
+{
+	for (; c; c = c->next) {
+		assert(c->name);
+		if (!strcmp(c->name, name))
+			return c;
+	}
+	return NULL;
+}
+
+/* Callback for archive_walk to process all entries in the cache. */
+static int archive_cache_walk(
+		struct archive_cache *c, void *arg,
+		int (*callback)(const char *name, void *arg))
+{
+	for (; c; c = c->next) {
+		assert(c->name);
+		if (callback(c->name, arg))
+			break;
+	}
+	return 0;
+}
+
+/* Delete all entries in the cache. */
+static void *archive_cache_free(struct archive_cache *c)
+{
+	struct archive_cache *next;
+
+	while (c) {
+		next = c->next;
+		free(c->name);
+		free(c->data);
+		free(c);
+		c = next;
+	}
+	return NULL;
+}
+
+enum {
+	FILTER_IGNORE,
+	FILTER_ABORT,
+	FILTER_NAME_ONLY,
+	FILTER_READ_ALL,
+};
+
+static struct archive_cache *libarchive_read_file_entries(
+		const char *fpath, int (*filter)(struct archive_entry *entry))
+{
+	struct archive *a = archive_read_new();
+	struct archive_entry *entry;
+	struct archive_cache *c, *cache = NULL;
+	int r;
+
+	assert(a);
+	archive_read_support_filter_all(a);
+	archive_read_support_format_all(a);
+	r = archive_read_open_filename(a, fpath, 10240);
+	if (r != ARCHIVE_OK) {
+		ERROR("Failed parsing archive using libarchive: %s\n", fpath);
+		archive_read_free(a);
+		return NULL;
+	}
+
+	WARN("Loading data from archive: %s ", fpath);
+	while (archive_read_next_header(a, &entry) == ARCHIVE_OK) {
+		fputc('.', stderr);
+		if (archive_entry_filetype(entry) != AE_IFREG)
+			continue;
+		if (filter)
+			r = filter(entry);
+		else
+			r = FILTER_READ_ALL;
+
+		if (r == FILTER_ABORT)
+			break;
+		if (r == FILTER_IGNORE)
+			continue;
+
+		c = archive_cache_new(cache, archive_entry_pathname(entry));
+		if (!c) {
+			ERROR("Internal error: out of memory.\n");
+			archive_cache_free(cache);
+			archive_read_free(a);
+			return NULL;
+		}
+		cache = c;
+
+		if (r == FILTER_NAME_ONLY)
+			continue;
+
+		assert(r == FILTER_READ_ALL);
+		c->size = archive_entry_size(entry);
+		c->mtime = archive_entry_mtime(entry);
+		c->data = (uint8_t *)calloc(1, c->size + 1);
+		if (!c->data) {
+			WARN("Out of memory when loading: %s\n", c->name);
+			continue;
+		}
+		if (archive_read_data(a, c->data, c->size) != c->size) {
+			WARN("Failed reading from archive: %s\n", c->name);
+			continue;
+		}
+		c->has_data = 1;
+	}
+	fputs("\r\n", stderr);  /* Flush the '.' */
+	VB2_DEBUG("Finished loading from archive: %s.\n", fpath);
+
+	archive_read_free(a);
+	return cache;
+}
+
+/* Callback for archive_open on an ARCHIVE file. */
+static void *archive_libarchive_open(const char *name)
+{
+	/*
+	 * The firmware archives today can usually all load into memory
+	 * so we are using a NULL filter. Change that to a specific list in
+	 * future if the /build/$BOARD/firmware archive becomes too large.
+	 */
+	return libarchive_read_file_entries(name, NULL);
+}
+
+/* Callback for archive_close on an ARCHIVE file. */
+static int archive_libarchive_close(void *handle)
+{
+	archive_cache_free(handle);
+	return 0;
+}
+
+/* Callback for archive_has_entry on an ARCHIVE file. */
+static int archive_libarchive_has_entry(void *handle, const char *fname)
+{
+	return archive_cache_find(handle, fname) != NULL;
+}
+
+/* Callback for archive_walk on an ARCHIVE file. */
+static int archive_libarchive_walk(
+		void *handle, void *arg,
+		int (*callback)(const char *name, void *arg))
+{
+	return archive_cache_walk(handle, arg, callback);
+}
+
+/* Callback for archive_read_file on an ARCHIVE file. */
+static int archive_libarchive_read_file(
+		void *handle, const char *fname, uint8_t **data,
+		uint32_t *size, int64_t *mtime)
+{
+	struct archive_cache *c = archive_cache_find(handle, fname);
+
+	if (!c)
+		return 1;
+
+	if (!c->has_data) {
+		/* TODO(hungte) Re-read. */
+		ERROR("Not in the cache: %s\n", fname);
+		return 1;
+	}
+
+	if (mtime)
+		*mtime = c->mtime;
+	if (size)
+		*size = c->size;
+	*data = (uint8_t *)malloc(c->size + 1);
+	if (!*data) {
+		ERROR("Out of memory when reading: %s\n", c->name);
+		return 1;
+	}
+	memcpy(*data, c->data, c->size);
+	(*data)[c->size] = '\0';
+	return 0;
+}
+
+/* Callback for archive_write_file on an ARCHIVE file. */
+static int archive_libarchive_write_file(
+		void *handle, const char *fname, uint8_t *data, uint32_t size,
+		int64_t mtime)
+{
+	ERROR("Not implemented!\n");
+	return 1;
+}
+#endif
+
 #ifdef HAVE_LIBZIP
 
 /* Callback for archive_open on a ZIP file. */
@@ -368,7 +593,7 @@ struct u_archive *archive_open(const char *path)
 		return NULL;
 	}
 
-	ar = (struct u_archive *)malloc(sizeof(*ar));
+	ar = (struct u_archive *)calloc(sizeof(*ar), 1);
 	if (!ar) {
 		ERROR("Internal error: allocation failure.\n");
 		return NULL;
@@ -384,22 +609,48 @@ struct u_archive *archive_open(const char *path)
 		ar->has_entry = archive_fallback_has_entry;
 		ar->read_file = archive_fallback_read_file;
 		ar->write_file = archive_fallback_write_file;
-	} else {
+	}
+
+	/* Format detection must try ZIP (the official format) first. */
 #ifdef HAVE_LIBZIP
-		VB2_DEBUG("Found file, use ZIP driver: %s\n", path);
-		ar->open = archive_zip_open;
-		ar->close = archive_zip_close;
-		ar->walk = archive_zip_walk;
-		ar->has_entry = archive_zip_has_entry;
-		ar->read_file = archive_zip_read_file;
-		ar->write_file = archive_zip_write_file;
-#else
-		ERROR("Found file, but no drivers were enabled: %s\n", path);
+	if (!ar->open) {
+		ar->handle = archive_zip_open(path);
+
+		if (ar->handle) {
+			VB2_DEBUG("Found a ZIP file: %s\n", path);
+			ar->open = archive_zip_open;
+			ar->close = archive_zip_close;
+			ar->walk = archive_zip_walk;
+			ar->has_entry = archive_zip_has_entry;
+			ar->read_file = archive_zip_read_file;
+			ar->write_file = archive_zip_write_file;
+		}
+	}
+#endif
+
+	/* LIBARCHIVE must be the last driver. */
+#ifdef HAVE_LIBARCHIVE
+	if (!ar->open) {
+		VB2_DEBUG("Found a file, use libarchive: %s\n", path);
+		ar->open = archive_libarchive_open;
+		ar->close = archive_libarchive_close;
+		ar->walk = archive_libarchive_walk;
+		ar->has_entry = archive_libarchive_has_entry;
+		ar->read_file = archive_libarchive_read_file;
+		ar->write_file = archive_libarchive_write_file;
+	}
+#endif
+
+	if (!ar->open) {
+		ERROR("Found a file, but no drivers were selected: %s\n", path);
 		free(ar);
 		return NULL;
-#endif
 	}
-	ar->handle = ar->open(path);
+
+	/* Some drivers may have already opened the archive. */
+	if (!ar->handle)
+		ar->handle = ar->open(path);
+
 	if (!ar->handle) {
 		ERROR("Failed to open archive: %s\n", path);
 		free(ar);