When ENOMEM happened in archive_string_append_from_wcs and archive_string_append_from_mbs,

those function should report the error to the caller instead of invoking __archive_errx().
We should report that ENOMEM error as possible as we can and we still need to further
improve reporting ENOEM.

SVN-Revision: 4193

9 files changed, 133 insertions, 44 deletions

diff --git a/libarchive/archive_acl.c b/libarchive/archive_acl.c
index 4747a4c5..6fccd25f 100644
--- a/libarchive/archive_acl.c
+++ b/libarchive/archive_acl.c
@@ -419,8 +419,11 @@ archive_acl_next(struct archive *a, struct archive_acl *acl, int want_type, int
 	*permset = acl->acl_p->permset;
 	*tag = acl->acl_p->tag;
 	*id = acl->acl_p->id;
-	if (archive_mstring_get_mbs(a, &acl->acl_p->name, name) != 0)
+	if (archive_mstring_get_mbs(a, &acl->acl_p->name, name) != 0) {
+		if (errno == ENOMEM)
+			__archive_errx(1, "No memory");
 		*name = NULL;
+	}
 	acl->acl_p = acl->acl_p->next;
 	return (ARCHIVE_OK);
 }
@@ -438,7 +441,7 @@ archive_acl_text_w(struct archive *a, struct archive_acl *acl, int flags)
 	const wchar_t *prefix;
 	wchar_t separator;
 	struct archive_acl_entry *ap;
-	int id;
+	int id, r;
 	wchar_t *wp;
 
 	if (acl->acl_text_w != NULL) {
@@ -458,9 +461,12 @@ archive_acl_text_w(struct archive *a, struct archive_acl *acl, int flags)
 				length += 8; /* "default:" */
 			length += 5; /* tag name */
 			length += 1; /* colon */
-			if (archive_mstring_get_wcs(a, &ap->name, &wname) == 0 &&
-			    wname != NULL)
+			r = archive_mstring_get_wcs(a, &ap->name, &wname);
+			if (r == 0 && wname != NULL)
 				length += wcslen(wname);
+			else if (r < 0 && errno == ENOMEM)
+				__archive_errx(1, "No memory to generate "
+				    "the text version of the ACL");
 			else
 				length += sizeof(uid_t) * 3 + 1;
 			length ++; /* colon */
@@ -499,8 +505,10 @@ archive_acl_text_w(struct archive *a, struct archive_acl *acl, int flags)
 
 		ap = acl->acl_head;
 		while (ap != NULL) {
+			r = 0;
 			if ((ap->type & ARCHIVE_ENTRY_ACL_TYPE_ACCESS) != 0 &&
-				archive_mstring_get_wcs(a, &ap->name, &wname) == 0) {
+			    (r = archive_mstring_get_wcs(a, &ap->name, &wname))
+			      == 0) {
 				*wp++ = separator;
 				if (flags & ARCHIVE_ENTRY_ACL_STYLE_EXTRA_ID)
 					id = ap->id;
@@ -509,7 +517,9 @@ archive_acl_text_w(struct archive *a, struct archive_acl *acl, int flags)
 				append_entry_w(&wp, NULL, ap->tag, wname,
 				    ap->permset, id);
 				count++;
-			}
+			} else if (r < 0 && errno == ENOMEM)
+				__archive_errx(1, "No memory to generate "
+				    "the text version of the ACL");
 			ap = ap->next;
 		}
 	}
@@ -524,7 +534,8 @@ archive_acl_text_w(struct archive *a, struct archive_acl *acl, int flags)
 		count = 0;
 		while (ap != NULL) {
 			if ((ap->type & ARCHIVE_ENTRY_ACL_TYPE_DEFAULT) != 0 &&
-				archive_mstring_get_wcs(a, &ap->name, &wname) == 0) {
+			    (r = archive_mstring_get_wcs(a, &ap->name,
+					&wname)) == 0) {
 				if (count > 0)
 					*wp++ = separator;
 				if (flags & ARCHIVE_ENTRY_ACL_STYLE_EXTRA_ID)
@@ -534,7 +545,9 @@ archive_acl_text_w(struct archive *a, struct archive_acl *acl, int flags)
 				append_entry_w(&wp, prefix, ap->tag,
 				    wname, ap->permset, id);
 				count ++;
-			}
+			} else if (r < 0 && errno == ENOMEM)
+				__archive_errx(1, "No memory to generate "
+				    "the text version of the ACL");
 			ap = ap->next;
 		}
 	}
diff --git a/libarchive/archive_entry.c b/libarchive/archive_entry.c
index cbdda8a4..4153dcd8 100644
--- a/libarchive/archive_entry.c
+++ b/libarchive/archive_entry.c
@@ -375,8 +375,11 @@ archive_entry_fflags_text(struct archive_entry *entry)
 	char *p;
 
 	if (archive_mstring_get_mbs(entry->archive,
-	    &entry->ae_fflags_text, &f) == 0 && f != NULL)
-		return (f);
+	    &entry->ae_fflags_text, &f) == 0) {
+		if (f != NULL)
+			return (f);
+	} else if (errno == ENOMEM)
+		__archive_errx(1, "No memory");
 
 	if (entry->ae_fflags_set == 0  &&  entry->ae_fflags_clear == 0)
 		return (NULL);
@@ -390,6 +393,8 @@ archive_entry_fflags_text(struct archive_entry *entry)
 	if (archive_mstring_get_mbs(entry->archive,
 	    &entry->ae_fflags_text, &f) == 0)
 		return (f);
+	if (errno == ENOMEM)
+		__archive_errx(1, "No memory");
 	return (NULL);
 }
 
@@ -405,6 +410,8 @@ archive_entry_gname(struct archive_entry *entry)
 	const char *p;
 	if (archive_mstring_get_mbs(entry->archive, &entry->ae_gname, &p) == 0)
 		return (p);
+	if (errno == ENOMEM)
+		__archive_errx(1, "No memory");
 	return (NULL);
 }
 
@@ -414,6 +421,8 @@ archive_entry_gname_w(struct archive_entry *entry)
 	const wchar_t *p;
 	if (archive_mstring_get_wcs(entry->archive, &entry->ae_gname, &p) == 0)
 		return (p);
+	if (errno == ENOMEM)
+		__archive_errx(1, "No memory");
 	return (NULL);
 }
 
@@ -428,9 +437,13 @@ const char *
 archive_entry_hardlink(struct archive_entry *entry)
 {
 	const char *p;
-	if ((entry->ae_set & AE_SET_HARDLINK) && archive_mstring_get_mbs(
+	if ((entry->ae_set & AE_SET_HARDLINK) == 0)
+		return (NULL);
+	if (archive_mstring_get_mbs(
 	    entry->archive, &entry->ae_hardlink, &p) == 0)
 		return (p);
+	if (errno == ENOMEM)
+		__archive_errx(1, "No memory");
 	return (NULL);
 }
 
@@ -438,9 +451,13 @@ const wchar_t *
 archive_entry_hardlink_w(struct archive_entry *entry)
 {
 	const wchar_t *p;
-	if ((entry->ae_set & AE_SET_HARDLINK) && archive_mstring_get_wcs(
+	if ((entry->ae_set & AE_SET_HARDLINK) == 0)
+		return (NULL);
+	if (archive_mstring_get_wcs(
 	    entry->archive, &entry->ae_hardlink, &p) == 0)
 		return (p);
+	if (errno == ENOMEM)
+		__archive_errx(1, "No memory");
 	return (NULL);
 }
 
@@ -511,6 +528,8 @@ archive_entry_pathname(struct archive_entry *entry)
 	if (archive_mstring_get_mbs(
 	    entry->archive, &entry->ae_pathname, &p) == 0)
 		return (p);
+	if (errno == ENOMEM)
+		__archive_errx(1, "No memory");
 	return (NULL);
 }
 
@@ -521,6 +540,8 @@ archive_entry_pathname_w(struct archive_entry *entry)
 	if (archive_mstring_get_wcs(
 	    entry->archive, &entry->ae_pathname, &p) == 0)
 		return (p);
+	if (errno == ENOMEM)
+		__archive_errx(1, "No memory");
 	return (NULL);
 }
 
@@ -584,6 +605,8 @@ archive_entry_sourcepath(struct archive_entry *entry)
 	if (archive_mstring_get_mbs(
 	    entry->archive, &entry->ae_sourcepath, &p) == 0)
 		return (p);
+	if (errno == ENOMEM)
+		__archive_errx(1, "No memory");
 	return (NULL);
 }
 
@@ -601,9 +624,13 @@ const char *
 archive_entry_symlink(struct archive_entry *entry)
 {
 	const char *p;
-	if ((entry->ae_set & AE_SET_SYMLINK) && archive_mstring_get_mbs(
+	if ((entry->ae_set & AE_SET_SYMLINK) == 0)
+		return (NULL);
+	if (archive_mstring_get_mbs(
 	    entry->archive, &entry->ae_symlink, &p) == 0)
 		return (p);
+	if (errno == ENOMEM)
+		__archive_errx(1, "No memory");
 	return (NULL);
 }
 
@@ -611,9 +638,13 @@ const wchar_t *
 archive_entry_symlink_w(struct archive_entry *entry)
 {
 	const wchar_t *p;
-	if ((entry->ae_set & AE_SET_SYMLINK) && archive_mstring_get_wcs(
+	if ((entry->ae_set & AE_SET_SYMLINK) == 0)
+		return (NULL);
+	if (archive_mstring_get_wcs(
 	    entry->archive, &entry->ae_symlink, &p) == 0)
 		return (p);
+	if (errno == ENOMEM)
+		__archive_errx(1, "No memory");
 	return (NULL);
 }
 
@@ -641,6 +672,8 @@ archive_entry_uname(struct archive_entry *entry)
 	const char *p;
 	if (archive_mstring_get_mbs(entry->archive, &entry->ae_uname, &p) == 0)
 		return (p);
+	if (errno == ENOMEM)
+		__archive_errx(1, "No memory");
 	return (NULL);
 }
 
@@ -650,6 +683,8 @@ archive_entry_uname_w(struct archive_entry *entry)
 	const wchar_t *p;
 	if (archive_mstring_get_wcs(entry->archive, &entry->ae_uname, &p) == 0)
 		return (p);
+	if (errno == ENOMEM)
+		__archive_errx(1, "No memory");
 	return (NULL);
 }
 
@@ -730,6 +765,8 @@ archive_entry_update_gname_utf8(struct archive_entry *entry, const char *name)
 	if (archive_mstring_update_utf8(entry->archive,
 	    &entry->ae_gname, name) == 0)
 		return (1);
+	if (errno == ENOMEM)
+		__archive_errx(1, "No memory");
 	return (0);
 }
 
@@ -796,6 +833,8 @@ archive_entry_update_hardlink_utf8(struct archive_entry *entry, const char *targ
 	if (archive_mstring_update_utf8(entry->archive,
 	    &entry->ae_hardlink, target) == 0)
 		return (1);
+	if (errno == ENOMEM)
+		__archive_errx(1, "No memory");
 	return (0);
 }
 
@@ -932,7 +971,11 @@ archive_entry_update_link_utf8(struct archive_entry *entry, const char *target)
 	else
 		r = archive_mstring_update_utf8(entry->archive,
 		    &entry->ae_hardlink, target);
-	return ((r == 0)? 1: 0);
+	if (r == 0)
+		return (1);
+	if (errno == ENOMEM)
+		__archive_errx(1, "No memory");
+	return (0);
 }
 
 int
@@ -1005,6 +1048,8 @@ archive_entry_update_pathname_utf8(struct archive_entry *entry, const char *name
 	if (archive_mstring_update_utf8(entry->archive,
 	    &entry->ae_pathname, name) == 0)
 		return (1);
+	if (errno == ENOMEM)
+		__archive_errx(1, "No memory");
 	return (0);
 }
 
@@ -1115,6 +1160,8 @@ archive_entry_update_symlink_utf8(struct archive_entry *entry, const char *linkn
 	if (archive_mstring_update_utf8(entry->archive,
 	    &entry->ae_symlink, linkname) == 0)
 		return (1);
+	if (errno == ENOMEM)
+		__archive_errx(1, "No memory");
 	return (0);
 }
 
@@ -1164,6 +1211,8 @@ archive_entry_update_uname_utf8(struct archive_entry *entry, const char *name)
 	if (archive_mstring_update_utf8(entry->archive,
 	    &entry->ae_uname, name) == 0)
 		return (1);
+	if (errno == ENOMEM)
+		__archive_errx(1, "No memory");
 	return (0);
 }
 
diff --git a/libarchive/archive_read_disk_posix.c b/libarchive/archive_read_disk_posix.c
index 5e39516c..8a42e28e 100644
--- a/libarchive/archive_read_disk_posix.c
+++ b/libarchive/archive_read_disk_posix.c
@@ -1303,8 +1303,12 @@ archive_read_disk_open_w(struct archive *_a, const wchar_t *pathname)
 	archive_string_init(&path);
 	if (archive_string_append_from_wcs(&path, pathname,
 	    wcslen(pathname)) != 0) {
-		archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
-		    "Can't convert a path to a char string");
+		if (errno == ENOMEM)
+			archive_set_error(&a->archive, ENOMEM,
+			    "Can't allocate memory");
+		else
+			archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
+			    "Can't convert a path to a char string");
 		a->archive.state = ARCHIVE_STATE_FATAL;
 		ret = ARCHIVE_FATAL;
 	} else
diff --git a/libarchive/archive_read_disk_windows.c b/libarchive/archive_read_disk_windows.c
index d8999988..0a373ba0 100644
--- a/libarchive/archive_read_disk_windows.c
+++ b/libarchive/archive_read_disk_windows.c
@@ -1166,8 +1166,12 @@ archive_read_disk_open(struct archive *_a, const char *pathname)
 	archive_string_init(&wpath);
 	if (archive_wstring_append_from_mbs(&wpath, pathname,
 	    strlen(pathname)) != 0) {
-		archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
-		    "Can't convert a path to a wchar_t string");
+		if (errno == ENOMEM)
+			archive_set_error(&a->archive, ENOMEM,
+			    "Can't allocate memory");
+		else
+			archive_set_error(&a->archive, ARCHIVE_ERRNO_MISC,
+			    "Can't convert a path to a wchar_t string");
 		a->archive.state = ARCHIVE_STATE_FATAL;
 		ret = ARCHIVE_FATAL;
 	} else
diff --git a/libarchive/archive_read_open_filename.c b/libarchive/archive_read_open_filename.c
index bf526972..cad2fd1e 100644
--- a/libarchive/archive_read_open_filename.c
+++ b/libarchive/archive_read_open_filename.c
@@ -130,9 +130,13 @@ archive_read_open_filename_w(struct archive *a, const wchar_t *wfilename,
 		archive_string_init(&fn);
 		if (archive_string_append_from_wcs(&fn, wfilename,
 		    wcslen(wfilename)) != 0) {
-			archive_set_error(a, EINVAL,
-			    "Failed to convert a wide-character filename to"
-			    " a multi-byte filename");
+			if (errno == ENOMEM)
+				archive_set_error(a, errno,
+				    "Can't allocate memory");
+			else
+				archive_set_error(a, EINVAL,
+				    "Failed to convert a wide-character"
+				    " filename to a multi-byte filename");
 			archive_string_free(&fn);
 			return (ARCHIVE_FATAL);
 		}
diff --git a/libarchive/archive_string.c b/libarchive/archive_string.c
index 906f12cd..b168e28d 100644
--- a/libarchive/archive_string.c
+++ b/libarchive/archive_string.c
@@ -443,10 +443,7 @@ int
 archive_wstring_append_from_mbs(struct archive_wstring *dest,
     const char *p, size_t len)
 {
-	int r = archive_wstring_append_from_mbs_in_codepage(dest, p, len, NULL);
-	if (r != 0 && errno == ENOMEM)
-		__archive_errx(1, "No memory");
-	return (r);
+	return archive_wstring_append_from_mbs_in_codepage(dest, p, len, NULL);
 }
 
 static int
@@ -612,8 +609,7 @@ archive_wstring_append_from_mbs(struct archive_wstring *dest,
 
 	memset(&shift_state, 0, sizeof(shift_state));
 	if (NULL == archive_wstring_ensure(dest, dest->length + wcs_length + 1))
-		__archive_errx(1,
-		    "No memory for archive_wstring_append_from_mbs()");
+		return (-1);
 	wcs = dest->s + dest->length;
 	r = mbsnrtowcs(wcs, &mbs, mbs_length, wcs_length, &shift_state);
 	if (r != (size_t)-1) {
@@ -650,8 +646,7 @@ archive_wstring_append_from_mbs(struct archive_wstring *dest,
 	memset(&shift_state, 0, sizeof(shift_state));
 #endif
 	if (NULL == archive_wstring_ensure(dest, dest->length + wcs_length + 1))
-		__archive_errx(1,
-		    "No memory for archive_wstring_append_from_mbs()");
+		return (-1);
 	wcs = dest->s + dest->length;
 	/*
 	 * We cannot use mbsrtowcs/mbstowcs here because those may convert
@@ -697,10 +692,7 @@ int
 archive_string_append_from_wcs(struct archive_string *as,
     const wchar_t *w, size_t len)
 {
-	int r = archive_string_append_from_wcs_in_codepage(as, w, len, NULL);
-	if (r != 0 && errno == ENOMEM)
-		__archive_errx(1, "No memory");
-	return (r);
+	return archive_string_append_from_wcs_in_codepage(as, w, len, NULL);
 }
 
 static int
@@ -817,7 +809,7 @@ archive_string_append_from_wcs(struct archive_string *as,
 	while (nwc > 0) {
 		/* Allocate buffer for MBS. */
 		if (archive_string_ensure(as, as->length + ndest + 1) == NULL)
-			__archive_errx(1, "Out of memory");
+			return (-1);
 
 		dest = as->s + as->length;
 		wpp = wp;
@@ -893,7 +885,7 @@ archive_string_append_from_wcs(struct archive_string *as,
 	 * as->s is still NULL.
 	 */
 	if (archive_string_ensure(as, as->length + len + 1) == NULL)
-		__archive_errx(1, "Out of memory");
+		return (-1);
 
 	p = as->s + as->length;
 	end = as->s + as->buffer_length - MB_CUR_MAX -1;
@@ -946,6 +938,7 @@ archive_string_append_from_wcs(struct archive_string *as,
 	(void)as;/* UNUSED */
 	(void)w;/* UNUSED */
 	(void)len;/* UNUSED */
+	errno = ENOSYS;
 	return (-1);
 }
 
diff --git a/libarchive/archive_string_sprintf.c b/libarchive/archive_string_sprintf.c
index 7d7d9713..fb752a21 100644
--- a/libarchive/archive_string_sprintf.c
+++ b/libarchive/archive_string_sprintf.c
@@ -146,7 +146,9 @@ archive_string_vsprintf(struct archive_string *as, const char *fmt,
 				pw = va_arg(ap, wchar_t *);
 				if (pw == NULL)
 					pw = L"(null)";
-				archive_string_append_from_wcs(as, pw, wcslen(pw));
+				if (archive_string_append_from_wcs(as, pw,
+				    wcslen(pw)) != 0 && errno == ENOMEM)
+					__archive_errx(1, "Out of memory");
 				break;
 			default:
 				p2 = va_arg(ap, char *);
@@ -160,7 +162,9 @@ archive_string_vsprintf(struct archive_string *as, const char *fmt,
 			pw = va_arg(ap, wchar_t *);
 			if (pw == NULL)
 				pw = L"(null)";
-			archive_string_append_from_wcs(as, pw, wcslen(pw));
+			if (archive_string_append_from_wcs(as, pw,
+			    wcslen(pw)) != 0 && errno == ENOMEM)
+				__archive_errx(1, "Out of memory");
 			break;
 		case 'o': case 'u': case 'x': case 'X':
 			/* Common handling for unsigned integer formats. */
diff --git a/libarchive/archive_util.c b/libarchive/archive_util.c
index e0852a3b..103c009d 100644
--- a/libarchive/archive_util.c
+++ b/libarchive/archive_util.c
@@ -243,8 +243,9 @@ __archive_mktemp(const char *tmpdir)
 		archive_wstrcpy(&temp_name, tmp);
 		free(tmp);
 	} else {
-		archive_wstring_append_from_mbs(&temp_name, tmpdir,
-		    strlen(tmpdir));
+		if (archive_wstring_append_from_mbs(&temp_name, tmpdir,
+		    strlen(tmpdir)) < 0)
+			goto exit_tmpfile;
 		if (temp_name.s[temp_name.length-1] != L'/')
 			archive_wstrappend_wchar(&temp_name, L'/');
 	}
diff --git a/libarchive/archive_write_set_format_iso9660.c b/libarchive/archive_write_set_format_iso9660.c
index 22c074b3..069e7324 100644
--- a/libarchive/archive_write_set_format_iso9660.c
+++ b/libarchive/archive_write_set_format_iso9660.c
@@ -4810,13 +4810,19 @@ isofile_gen_utility_names(struct archive_write *a, struct isofile *file)
 		struct archive_wstring ws;
 
 		if (wp != NULL) {
+			int r;
 			archive_string_init(&ws);
 			archive_wstrcpy(&ws, wp);
 			cleanup_backslash_2(ws.s);
 			archive_string_empty(&(file->parentdir));
-			archive_string_append_from_wcs(&(file->parentdir),
+			r = archive_string_append_from_wcs(&(file->parentdir),
 			    ws.s, ws.length);
 			archive_wstring_free(&ws);
+			if (r < 0 && errno == ENOMEM) {
+				archive_set_error(&a->archive, ENOMEM,
+				    "Can't allocate memory");
+				return (ARCHIVE_FATAL);
+			}
 		}
 	}
 #endif
@@ -4919,14 +4925,20 @@ isofile_gen_utility_names(struct archive_write *a, struct isofile *file)
 			struct archive_wstring ws;
 
 			if (wp != NULL) {
+				int r;
 				archive_string_init(&ws);
 				archive_wstrcpy(&ws, wp);
 				cleanup_backslash_2(ws.s);
 				archive_string_empty(&(file->symlink));
-				archive_string_append_from_wcs(
+				r = archive_string_append_from_wcs(
 				    &(file->symlink),
 				    ws.s, ws.length);
 				archive_wstring_free(&ws);
+				if (r < 0 && errno == ENOMEM) {
+					archive_set_error(&a->archive, ENOMEM,
+					    "Can't allocate memory");
+					return (ARCHIVE_FATAL);
+				}
 			}
 		}
 #endif
@@ -6258,9 +6270,14 @@ isoent_gen_joliet_identifier(struct archive_write *a, struct isoent *isoent,
 		 * Get a length of MBS of a full-pathname.
 		 */
 		if ((int)np->file->basename_utf16.length > ffmax) {
-			archive_strncpy_in_locale(&iso9660->mbs,
+			if (archive_strncpy_in_locale(&iso9660->mbs,
 			    (const char *)np->identifier, l,
-			    iso9660->sconv_from_utf16be);
+				iso9660->sconv_from_utf16be) != 0 &&
+			    errno == ENOMEM) {
+				archive_set_error(&a->archive, errno,
+				    "No memory");
+				return (ARCHIVE_FATAL);
+			}
 			np->mb_len = iso9660->mbs.length;
 			if (np->mb_len != (int)np->file->basename.length)
 				weight = np->mb_len;