libctf: add LIBCTF_WRITE_FOREIGN_ENDIAN debugging option

libctf has always handled endianness differences by detecting
foreign-endian CTF dicts on the input and endian-flipping them: dicts
are always written in native endianness.  This makes endian-awareness
very low overhead, but it means that the foreign-endian code paths
almost never get routinely tested, since "make check" usually reads in
dicts ld has just written out: only a few corrupted-CTF tests are
actually in fixed endianness, and even they only test the foreign-
endian code paths when you run make check on a big-endian machine.
(And the fix is surely not to add more .s-based tests like that, because
they are a nightmare to maintain compared to the C-code-based ones.)

To improve on this, add a new environment variable,
LIBCTF_WRITE_FOREIGN_ENDIAN, which causes libctf to unconditionally
endian-flip at ctf_write time, so the output is always in the wrong
endianness.  This then tests the foreign-endian read paths properly
at open time.

Make this easier by restructuring the writeout code in ctf-serialize.c,
which duplicates the maybe-gzip-and-write-out code three times (once
for ctf_write_mem, with thresholding, and once each for
ctf_compress_write and ctf_write just so those can avoid thresholding
and/or compression).  Instead, have the latter two call the former
with thresholds of 0 or (size_t) -1, respectively.

The endian-flipping code itself gains a bit of complexity, because
one single endian-flipper (flip_types) was assuming the input to be
in foreign-endian form and assuming it could pull things out of the
input once they had been flipped and make sense of them. At the
cost of a few lines of duplicated initializations, teach it to
read before flipping if we're flipping to foreign-endianness instead
of away from it.

libctf/
	* ctf-impl.h (ctf_flip_header): No longer static.
	(ctf_flip): Likewise.
	* ctf-open.c (flip_header): Rename to...
	(ctf_flip_header): ... this, now it is not private to one file.
	(flip_ctf): Rename...
	(ctf_flip): ... this too.  Add FOREIGN_ENDIAN arg.
	(flip_types): Likewise.  Use it.
	(ctf_bufopen_internal): Adjust calls.
	* ctf-serialize.c (ctf_write_mem): Add flip_endian path via
	a newly-allocated bounce buffer.
	(ctf_compress_write): Move below ctf_write_mem and reimplement
	in terms of it.
	(ctf_write): Likewise.
	(ctf_gzwrite): Note that this obscure writeout function does not
	support endian-flipping.

3 files changed, 144 insertions, 111 deletions

diff --git a/libctf/ctf-impl.h b/libctf/ctf-impl.h
index f749b839ab3..6b6ec16291a 100644
--- a/libctf/ctf-impl.h
+++ b/libctf/ctf-impl.h
@@ -738,6 +738,8 @@ extern void ctf_arc_close_internal (struct ctf_archive *);
 extern const ctf_preamble_t *ctf_arc_bufpreamble (const ctf_sect_t *);
 extern void *ctf_set_open_errno (int *, int);
 extern unsigned long ctf_set_errno (ctf_dict_t *, int);
+extern void ctf_flip_header (ctf_header_t *);
+extern int ctf_flip (ctf_dict_t *, ctf_header_t *, unsigned char *, int);
 
 extern ctf_dict_t *ctf_simple_open_internal (const char *, size_t, const char *,
 					     size_t, size_t,
diff --git a/libctf/ctf-open.c b/libctf/ctf-open.c
index 3f8d336f895..f0e203e0a16 100644
--- a/libctf/ctf-open.c
+++ b/libctf/ctf-open.c
@@ -965,8 +965,8 @@ init_types (ctf_dict_t *fp, ctf_header_t *cth)
 
 /* Flip the endianness of the CTF header.  */
 
-static void
-flip_header (ctf_header_t *cth)
+void
+ctf_flip_header (ctf_header_t *cth)
 {
   swap_thing (cth->cth_preamble.ctp_magic);
   swap_thing (cth->cth_preamble.ctp_version);
@@ -1031,26 +1031,48 @@ flip_vars (void *start, size_t len)
    ctf_stype followed by variable data.  */
 
 static int
-flip_types (ctf_dict_t *fp, void *start, size_t len)
+flip_types (ctf_dict_t *fp, void *start, size_t len, int to_foreign)
 {
   ctf_type_t *t = start;
 
   while ((uintptr_t) t < ((uintptr_t) start) + len)
     {
+      uint32_t kind;
+      size_t size;
+      uint32_t vlen;
+      size_t vbytes;
+
+      if (to_foreign)
+	{
+	  kind = CTF_V2_INFO_KIND (t->ctt_info);
+	  size = t->ctt_size;
+	  vlen = CTF_V2_INFO_VLEN (t->ctt_info);
+	  vbytes = get_vbytes_v2 (fp, kind, size, vlen);
+	}
+
       swap_thing (t->ctt_name);
       swap_thing (t->ctt_info);
       swap_thing (t->ctt_size);
 
-      uint32_t kind = CTF_V2_INFO_KIND (t->ctt_info);
-      size_t size = t->ctt_size;
-      uint32_t vlen = CTF_V2_INFO_VLEN (t->ctt_info);
-      size_t vbytes = get_vbytes_v2 (fp, kind, size, vlen);
+      if (!to_foreign)
+	{
+	  kind = CTF_V2_INFO_KIND (t->ctt_info);
+	  size = t->ctt_size;
+	  vlen = CTF_V2_INFO_VLEN (t->ctt_info);
+	  vbytes = get_vbytes_v2 (fp, kind, size, vlen);
+	}
 
       if (_libctf_unlikely_ (size == CTF_LSIZE_SENT))
 	{
+	  if (to_foreign)
+	    size = CTF_TYPE_LSIZE (t);
+
 	  swap_thing (t->ctt_lsizehi);
 	  swap_thing (t->ctt_lsizelo);
-	  size = CTF_TYPE_LSIZE (t);
+
+	  if (!to_foreign)
+	    size = CTF_TYPE_LSIZE (t);
+
 	  t = (ctf_type_t *) ((uintptr_t) t + sizeof (ctf_type_t));
 	}
       else
@@ -1182,22 +1204,27 @@ flip_types (ctf_dict_t *fp, void *start, size_t len)
 }
 
 /* Flip the endianness of BUF, given the offsets in the (already endian-
-   converted) CTH.
+   converted) CTH.  If TO_FOREIGN is set, flip to foreign-endianness; if not,
+   flip away.
 
    All of this stuff happens before the header is fully initialized, so the
    LCTF_*() macros cannot be used yet.  Since we do not try to endian-convert v1
    data, this is no real loss.  */
 
-static int
-flip_ctf (ctf_dict_t *fp, ctf_header_t *cth, unsigned char *buf)
+int
+ctf_flip (ctf_dict_t *fp, ctf_header_t *cth, unsigned char *buf,
+	  int to_foreign)
 {
+  ctf_dprintf("flipping endianness\n");
+
   flip_lbls (buf + cth->cth_lbloff, cth->cth_objtoff - cth->cth_lbloff);
   flip_objts (buf + cth->cth_objtoff, cth->cth_funcoff - cth->cth_objtoff);
   flip_objts (buf + cth->cth_funcoff, cth->cth_objtidxoff - cth->cth_funcoff);
   flip_objts (buf + cth->cth_objtidxoff, cth->cth_funcidxoff - cth->cth_objtidxoff);
   flip_objts (buf + cth->cth_funcidxoff, cth->cth_varoff - cth->cth_funcidxoff);
   flip_vars (buf + cth->cth_varoff, cth->cth_typeoff - cth->cth_varoff);
-  return flip_types (fp, buf + cth->cth_typeoff, cth->cth_stroff - cth->cth_typeoff);
+  return flip_types (fp, buf + cth->cth_typeoff,
+		     cth->cth_stroff - cth->cth_typeoff, to_foreign);
 }
 
 /* Set up the ctl hashes in a ctf_dict_t.  Called by both writable and
@@ -1404,7 +1431,7 @@ ctf_bufopen_internal (const ctf_sect_t *ctfsect, const ctf_sect_t *symsect,
     upgrade_header (hp);
 
   if (foreign_endian)
-    flip_header (hp);
+    ctf_flip_header (hp);
   fp->ctf_openflags = hp->cth_flags;
   fp->ctf_size = hp->cth_stroff + hp->cth_strlen;
 
@@ -1610,9 +1637,9 @@ ctf_bufopen_internal (const ctf_sect_t *ctfsect, const ctf_sect_t *symsect,
   fp->ctf_syn_ext_strtab = syn_strtab;
 
   if (foreign_endian &&
-      (err = flip_ctf (fp, hp, fp->ctf_buf)) != 0)
+      (err = ctf_flip (fp, hp, fp->ctf_buf, 0)) != 0)
     {
-      /* We can be certain that flip_ctf() will have endian-flipped everything
+      /* We can be certain that ctf_flip() will have endian-flipped everything
 	 other than the types table when we return.  In particular the header
 	 is fine, so set it, to allow freeing to use the usual code path.  */
 
diff --git a/libctf/ctf-serialize.c b/libctf/ctf-serialize.c
index cc9e59d4836..c6b8b495568 100644
--- a/libctf/ctf-serialize.c
+++ b/libctf/ctf-serialize.c
@@ -1229,7 +1229,13 @@ err:
 
 /* File writing.  */
 
-/* Write the compressed CTF data stream to the specified gzFile descriptor.  */
+/* Write the compressed CTF data stream to the specified gzFile descriptor.  The
+   whole stream is compressed, and cannot be read by CTF opening functions in
+   this library until it is decompressed.  (The functions below this one leave
+   the header uncompressed, and the CTF opening functions work on them without
+   manual decompression.)
+
+   No support for (testing-only) endian-flipping.  */
 int
 ctf_gzwrite (ctf_dict_t *fp, gzFile fd)
 {
@@ -1260,85 +1266,25 @@ ctf_gzwrite (ctf_dict_t *fp, gzFile fd)
   return 0;
 }
 
-/* Compress the specified CTF data stream and write it to the specified file
-   descriptor.  */
-int
-ctf_compress_write (ctf_dict_t *fp, int fd)
-{
-  unsigned char *buf;
-  unsigned char *bp;
-  ctf_header_t h;
-  ctf_header_t *hp = &h;
-  ssize_t header_len = sizeof (ctf_header_t);
-  ssize_t compress_len;
-  ssize_t len;
-  int rc;
-  int err = 0;
-
-  if (ctf_serialize (fp) < 0)
-    return -1;					/* errno is set for us.  */
-
-  memcpy (hp, fp->ctf_header, header_len);
-  hp->cth_flags |= CTF_F_COMPRESS;
-  compress_len = compressBound (fp->ctf_size);
-
-  if ((buf = malloc (compress_len)) == NULL)
-    {
-      ctf_err_warn (fp, 0, 0, _("ctf_compress_write: cannot allocate %li bytes"),
-		    (unsigned long) compress_len);
-      return (ctf_set_errno (fp, ECTF_ZALLOC));
-    }
-
-  if ((rc = compress (buf, (uLongf *) &compress_len,
-		      fp->ctf_buf, fp->ctf_size)) != Z_OK)
-    {
-      err = ctf_set_errno (fp, ECTF_COMPRESS);
-      ctf_err_warn (fp, 0, 0, _("zlib deflate err: %s"), zError (rc));
-      goto ret;
-    }
-
-  while (header_len > 0)
-    {
-      if ((len = write (fd, hp, header_len)) < 0)
-	{
-	  err = ctf_set_errno (fp, errno);
-	  ctf_err_warn (fp, 0, 0, _("ctf_compress_write: error writing header"));
-	  goto ret;
-	}
-      header_len -= len;
-      hp += len;
-    }
-
-  bp = buf;
-  while (compress_len > 0)
-    {
-      if ((len = write (fd, bp, compress_len)) < 0)
-	{
-	  err = ctf_set_errno (fp, errno);
-	  ctf_err_warn (fp, 0, 0, _("ctf_compress_write: error writing"));
-	  goto ret;
-	}
-      compress_len -= len;
-      bp += len;
-    }
-
-ret:
-  free (buf);
-  return err;
-}
-
 /* Optionally compress the specified CTF data stream and return it as a new
-   dynamically-allocated string.  */
+   dynamically-allocated string.  Possibly write it with reversed
+   endianness.  */
 unsigned char *
 ctf_write_mem (ctf_dict_t *fp, size_t *size, size_t threshold)
 {
   unsigned char *buf;
   unsigned char *bp;
   ctf_header_t *hp;
+  unsigned char *flipped, *src;
   ssize_t header_len = sizeof (ctf_header_t);
   ssize_t compress_len;
+  int flip_endian;
+  int uncompressed;
   int rc;
 
+  flip_endian = getenv ("LIBCTF_WRITE_FOREIGN_ENDIAN") != NULL;
+  uncompressed = (fp->ctf_size < threshold);
+
   if (ctf_serialize (fp) < 0)
     return NULL;				/* errno is set for us.  */
 
@@ -1359,17 +1305,43 @@ ctf_write_mem (ctf_dict_t *fp, size_t *size, size_t threshold)
   bp = buf + sizeof (struct ctf_header);
   *size = sizeof (struct ctf_header);
 
-  if (fp->ctf_size < threshold)
+  if (uncompressed)
+    hp->cth_flags &= ~CTF_F_COMPRESS;
+  else
+    hp->cth_flags |= CTF_F_COMPRESS;
+
+  src = fp->ctf_buf;
+  flipped = NULL;
+
+  if (flip_endian)
     {
-      hp->cth_flags &= ~CTF_F_COMPRESS;
-      memcpy (bp, fp->ctf_buf, fp->ctf_size);
+      if ((flipped = malloc (fp->ctf_size)) == NULL)
+	{
+	  ctf_set_errno (fp, ENOMEM);
+	  ctf_err_warn (fp, 0, 0, _("ctf_write_mem: cannot allocate %li bytes"),
+			(unsigned long) fp->ctf_size + sizeof (struct ctf_header));
+	  return NULL;
+	}
+      ctf_flip_header (hp);
+      memcpy (flipped, fp->ctf_buf, fp->ctf_size);
+      if (ctf_flip (fp, fp->ctf_header, flipped, 1) < 0)
+	{
+	  free (buf);
+	  free (flipped);
+	  return NULL;				/* errno is set for us.  */
+	}
+      src = flipped;
+    }
+
+  if (uncompressed)
+    {
+      memcpy (bp, src, fp->ctf_size);
       *size += fp->ctf_size;
     }
   else
     {
-      hp->cth_flags |= CTF_F_COMPRESS;
       if ((rc = compress (bp, (uLongf *) &compress_len,
-			  fp->ctf_buf, fp->ctf_size)) != Z_OK)
+			  src, fp->ctf_size)) != Z_OK)
 	{
 	  ctf_set_errno (fp, ECTF_COMPRESS);
 	  ctf_err_warn (fp, 0, 0, _("zlib deflate err: %s"), zError (rc));
@@ -1378,45 +1350,77 @@ ctf_write_mem (ctf_dict_t *fp, size_t *size, size_t threshold)
 	}
       *size += compress_len;
     }
+
+  free (flipped);
+
   return buf;
 }
 
-/* Write the uncompressed CTF data stream to the specified file descriptor.  */
+/* Compress the specified CTF data stream and write it to the specified file
+   descriptor.  */
 int
-ctf_write (ctf_dict_t *fp, int fd)
+ctf_compress_write (ctf_dict_t *fp, int fd)
 {
-  const unsigned char *buf;
-  ssize_t resid;
+  unsigned char *buf;
+  unsigned char *bp;
+  size_t tmp;
+  ssize_t buf_len;
   ssize_t len;
+  int err = 0;
 
-  if (ctf_serialize (fp) < 0)
+  if ((buf = ctf_write_mem (fp, &tmp, 0)) == NULL)
     return -1;					/* errno is set for us.  */
 
-  resid = sizeof (ctf_header_t);
-  buf = (unsigned char *) fp->ctf_header;
-  while (resid != 0)
+  buf_len = tmp;
+  bp = buf;
+
+  while (buf_len > 0)
     {
-      if ((len = write (fd, buf, resid)) <= 0)
+      if ((len = write (fd, bp, buf_len)) < 0)
 	{
-	  ctf_err_warn (fp, 0, errno, _("ctf_write: error writing header"));
-	  return (ctf_set_errno (fp, errno));
+	  err = ctf_set_errno (fp, errno);
+	  ctf_err_warn (fp, 0, 0, _("ctf_compress_write: error writing"));
+	  goto ret;
 	}
-      resid -= len;
-      buf += len;
+      buf_len -= len;
+      bp += len;
     }
 
-  resid = fp->ctf_size;
-  buf = fp->ctf_buf;
-  while (resid != 0)
+ret:
+  free (buf);
+  return err;
+}
+
+/* Write the uncompressed CTF data stream to the specified file descriptor.  */
+int
+ctf_write (ctf_dict_t *fp, int fd)
+{
+  unsigned char *buf;
+  unsigned char *bp;
+  size_t tmp;
+  ssize_t buf_len;
+  ssize_t len;
+  int err = 0;
+
+  if ((buf = ctf_write_mem (fp, &tmp, (size_t) -1)) == NULL)
+    return -1;					/* errno is set for us.  */
+
+  buf_len = tmp;
+  bp = buf;
+
+  while (buf_len > 0)
     {
-      if ((len = write (fd, buf, resid)) <= 0)
+      if ((len = write (fd, bp, buf_len)) < 0)
 	{
-	  ctf_err_warn (fp, 0, errno, _("ctf_write: error writing"));
-	  return (ctf_set_errno (fp, errno));
+	  err = ctf_set_errno (fp, errno);
+	  ctf_err_warn (fp, 0, 0, _("ctf_compress_write: error writing"));
+	  goto ret;
 	}
-      resid -= len;
-      buf += len;
+      buf_len -= len;
+      bp += len;
     }
 
-  return 0;
+ret:
+  free (buf);
+  return err;
 }