Keeping track of rs6000-coff archive element pointers

rs6000-coff archives use a linked list of file offsets, where each
element points to the next element.  The idea is to allow updating of
large archives quickly without rewriting the whole archive.  (binutils
ar does not do this.)  Unfortunately this is an easy target for
fuzzers to create an archive that will cause ar or any other tool
processing archives to hang.  I'd implemented guards against pointing
back to the previous element, but of course that didn't last long.

So this patch implements a scheme to keep track of file offset ranges
used by elements as _bfd_read_ar_hdr is called for each element.  See
the add_range function comment.  I needed a place to stash the list,
so chose the obvious artdata.tdata backend extension to archive's
tdata, already used by xcoff.  That involved a little cleanup, because
while it would be possible to continue using different artdata.tdata
for the big and small archives, it's nicer to use a union.

If anyone is concerned this list of element ranges might grow large
and thus significantly slow down the tools, adjacent ranges are
merged.  In fact something like "ar t" will only ever have one range
on xcoff archives generated by binutils/ar.  I agree there might still
be a problem with ld random element access via the armap.

include/
	* coff/xcoff.h (SIZEOF_AR_FILE_HDR): Use sizeof.
	(SIZEOF_AR_FILE_HDR_BIG, SIZEOF_AR_HDR, SIZEOF_AR_HDR_BIG): Likewise.
	(struct ar_ranges, struct xcoff_artdata): New.
	(x_artdata): Define.
	(xcoff_big_format_p): Rewrite.
	(xcoff_ardata, xcoff_ardata_big): Delete.
bfd/
	* coff-rs6000.c: Replace uses of xcoff_ardata and
	xcoff_ardata_big throughout file.
	(_bfd_xcoff_archive_p): Adjust artdata.tdata allocation.
	(add_range): New function.
	(_bfd_xcoff_read_ar_hdr): Use it here.  Fix memory leak.
	(_bfd_xcoff_openr_next_archived_file): Remove old sanity
	checks.  Set up range for header.
	(xcoff_write_archive_contents_old): Make the temporary
	artdata.tdata used here to pass info down to
	_bfd_compute_and_write_armap a struct xcoff_artdata.
	(xcoff_write_archive_contents_big): Likewise.
	* coff64-rs6000.c: Replace uses of xcoff_ardata and
	xcoff_ardata_big throughout file.
	(xcoff64_archive_p): Adjust artdata.tdata allocation.

1 files changed, 34 insertions, 22 deletions

diff --git a/include/coff/xcoff.h b/include/coff/xcoff.h
index 049ccd863ac..08afc000bf0 100644
--- a/include/coff/xcoff.h
+++ b/include/coff/xcoff.h
@@ -526,7 +526,7 @@ struct xcoff_ar_file_hdr
   char freeoff[XCOFFARMAG_ELEMENT_SIZE];
 };
 
-#define SIZEOF_AR_FILE_HDR (SXCOFFARMAG + 5 * XCOFFARMAG_ELEMENT_SIZE)
+#define SIZEOF_AR_FILE_HDR (sizeof (struct xcoff_ar_file_hdr))
 
 /* This is the equivalent data structure for the big archive format.  */
 
@@ -557,7 +557,7 @@ struct xcoff_ar_file_hdr_big
   char freeoff[XCOFFARMAGBIG_ELEMENT_SIZE];
 };
 
-#define SIZEOF_AR_FILE_HDR_BIG (SXCOFFARMAG + 6 * XCOFFARMAGBIG_ELEMENT_SIZE)
+#define SIZEOF_AR_FILE_HDR_BIG (sizeof (struct xcoff_ar_file_hdr_big))
 
 /* Each XCOFF archive member starts with this (printable) structure.  */
 
@@ -595,7 +595,7 @@ struct xcoff_ar_hdr
      bytes is given in the size field.  */
 };
 
-#define SIZEOF_AR_HDR (3 * XCOFFARMAG_ELEMENT_SIZE + 4 * 12 + 4)
+#define SIZEOF_AR_HDR (sizeof (struct xcoff_ar_hdr))
 
 /* The equivalent for the big archive format.  */
 
@@ -633,35 +633,47 @@ struct xcoff_ar_hdr_big
      bytes is given in the size field.  */
 };
 
-#define SIZEOF_AR_HDR_BIG (3 * XCOFFARMAGBIG_ELEMENT_SIZE + 4 * 12 + 4)
+#define SIZEOF_AR_HDR_BIG (sizeof (struct xcoff_ar_hdr_big))
+
+/* Track archive file offsets used by elements and the header.  */
+struct ar_ranges
+{
+  ufile_ptr start, end;
+  struct ar_ranges *next;
+};
+
+/* An archive bfd has tdata pointing to a struct artdata.  The xcoff
+   backend has artdata.tdata pointing to the following.  */
+struct xcoff_artdata
+{
+  union
+  {
+    struct xcoff_ar_file_hdr hdr;
+    struct xcoff_ar_file_hdr_big bhdr;
+  } u;
+  struct ar_ranges ranges;
+  /* Anything less than this size can't hold an archive element.  */
+  unsigned int min_elt;
+};
+
+#define x_artdata(abfd) ((struct xcoff_artdata *) bfd_ardata (abfd)->tdata)
 
 /* We often have to distinguish between the old and big file format.
-   Make it a bit cleaner.  We can use `xcoff_ardata' here because the
-   `hdr' member has the same size and position in both formats.  
-   <bigaf> is the default format, return TRUE even when xcoff_ardata is 
-   NULL. */
+   u.hdr.magic and u.bhdr.magic have the same size and position.  */
 #ifndef SMALL_ARCHIVE
 /* Creates big archives by default */
 #define xcoff_big_format_p(abfd) \
-  ((NULL != bfd_ardata (abfd) && NULL == xcoff_ardata (abfd)) || \
-   ((NULL != bfd_ardata (abfd)) && \
-    (NULL != xcoff_ardata (abfd)) && \
-    (xcoff_ardata (abfd)->magic[1] == 'b')))
+  (bfd_ardata (abfd) != NULL				\
+   && (x_artdata (abfd) == NULL			\
+       || x_artdata (abfd)->u.hdr.magic[1] == 'b'))
 #else
 /* Creates small archives by default. */
 #define xcoff_big_format_p(abfd) \
-  (((NULL != bfd_ardata (abfd)) && \
-    (NULL != xcoff_ardata (abfd)) && \
-    (xcoff_ardata (abfd)->magic[1] == 'b')))
+  (bfd_ardata (abfd) != NULL				\
+   && x_artdata (abfd) != NULL			\
+   && x_artdata (abfd)->u.hdr.magic[1] == 'b')
 #endif
 
-/* We store a copy of the xcoff_ar_file_hdr in the tdata field of the
-   artdata structure.  Similar for the big archive.  */
-#define xcoff_ardata(abfd) \
-  ((struct xcoff_ar_file_hdr *) bfd_ardata (abfd)->tdata)
-#define xcoff_ardata_big(abfd) \
-  ((struct xcoff_ar_file_hdr_big *) bfd_ardata (abfd)->tdata)
-
 /* We store a copy of the xcoff_ar_hdr in the arelt_data field of an
    archive element.  Similar for the big archive.  */
 #define arch_eltdata(bfd) ((struct areltdata *) ((bfd)->arelt_data))