base64: fix Curl_base64_encode and Curl_base64_decode interfaces

Previous interfaces for these libcurl internal functions did not allow to tell
apart a legitimate zero size result from an error condition. These functions
now return a CURLcode indicating function success or otherwise specific error.
Output size is returned using a pointer argument.

All usage of these two functions, and others closely related, has been adapted
to the new interfaces. Relative error and OOM handling adapted or added where
missing. Unit test 1302 also adapted.

1 files changed, 43 insertions, 21 deletions

diff --git a/lib/base64.c b/lib/base64.c
index 812fd67c2..23ebb4aa9 100644
--- a/lib/base64.c
+++ b/lib/base64.c
@@ -68,12 +68,19 @@ static void decodeQuantum(unsigned char *dest, const char *src)
 /*
  * Curl_base64_decode()
  *
- * Given a base64 string at src, decode it and return an allocated memory in
- * the *outptr. Returns the length of the decoded data.
+ * Given a base64 NUL-terminated string at src, decode it and return a
+ * pointer in *outptr to a newly allocated memory area holding decoded
+ * data. Size of decoded data is returned in variable pointed by outlen.
+ *
+ * Returns CURLE_OK on success, otherwise specific error code. Function
+ * output shall not be considered valid unless CURLE_OK is returned.
+ *
+ * When decoded data length is 0, returns NULL in *outptr.
  *
  * @unittest: 1302
  */
-size_t Curl_base64_decode(const char *src, unsigned char **outptr)
+CURLcode Curl_base64_decode(const char *src,
+                            unsigned char **outptr, size_t *outlen)
 {
   size_t length = 0;
   size_t equalsTerm = 0;
@@ -84,6 +91,7 @@ size_t Curl_base64_decode(const char *src, unsigned char **outptr)
   unsigned char *newstr;
 
   *outptr = NULL;
+  *outlen = 0;
 
   while((src[length] != '=') && src[length])
     length++;
@@ -97,7 +105,7 @@ size_t Curl_base64_decode(const char *src, unsigned char **outptr)
 
   /* Don't allocate a buffer if the decoded length is 0 */
   if(numQuantums == 0)
-    return 0;
+    return CURLE_OK;
 
   rawlen = (numQuantums * 3) - equalsTerm;
 
@@ -105,7 +113,7 @@ size_t Curl_base64_decode(const char *src, unsigned char **outptr)
   (which may be partially thrown out) and the zero terminator. */
   newstr = malloc(rawlen+4);
   if(!newstr)
-    return 0;
+    return CURLE_OUT_OF_MEMORY;
 
   *outptr = newstr;
 
@@ -124,23 +132,34 @@ size_t Curl_base64_decode(const char *src, unsigned char **outptr)
     newstr[i] = lastQuantum[i];
 
   newstr[i] = '\0'; /* zero terminate */
-  return rawlen;
+
+  *outlen = rawlen; /* return size of decoded data */
+
+  return CURLE_OK;
 }
 
 /*
  * Curl_base64_encode()
  *
- * Returns the length of the newly created base64 string. The third argument
- * is a pointer to an allocated area holding the base64 data. If something
- * went wrong, 0 is returned.
+ * Given a pointer to an input buffer and an input size, encode it and
+ * return a pointer in *outptr to a newly allocated memory area holding
+ * encoded data. Size of encoded data is returned in variable pointed by
+ * outlen.
+ *
+ * Input length of 0 indicates input buffer holds a NUL-terminated string.
+ *
+ * Returns CURLE_OK on success, otherwise specific error code. Function
+ * output shall not be considered valid unless CURLE_OK is returned.
+ *
+ * When encoded data length is 0, returns NULL in *outptr.
  *
  * @unittest: 1302
  */
-size_t Curl_base64_encode(struct SessionHandle *data,
-                          const char *inputbuff, size_t insize,
-                          char **outptr)
+CURLcode Curl_base64_encode(struct SessionHandle *data,
+                            const char *inputbuff, size_t insize,
+                            char **outptr, size_t *outlen)
 {
-  CURLcode res;
+  CURLcode error;
   unsigned char ibuf[3];
   unsigned char obuf[4];
   int i;
@@ -151,24 +170,25 @@ size_t Curl_base64_encode(struct SessionHandle *data,
 
   const char *indata = inputbuff;
 
-  *outptr = NULL; /* set to NULL in case of failure before we reach the end */
+  *outptr = NULL;
+  *outlen = 0;
 
   if(0 == insize)
     insize = strlen(indata);
 
   base64data = output = malloc(insize*4/3+4);
   if(NULL == output)
-    return 0;
+    return CURLE_OUT_OF_MEMORY;
 
   /*
    * The base64 data needs to be created using the network encoding
    * not the host encoding.  And we can't change the actual input
    * so we copy it to a buffer, translate it, and use that instead.
    */
-  res = Curl_convert_clone(data, indata, insize, &convbuf);
-  if(res) {
+  error = Curl_convert_clone(data, indata, insize, &convbuf);
+  if(error) {
     free(output);
-    return 0;
+    return error;
   }
 
   if(convbuf)
@@ -215,12 +235,14 @@ size_t Curl_base64_encode(struct SessionHandle *data,
     }
     output += 4;
   }
-  *output=0;
-  *outptr = base64data; /* make it return the actual data memory */
+  *output = '\0';
+  *outptr = base64data; /* return pointer to new data, allocated memory */
 
   if(convbuf)
     free(convbuf);
 
-  return strlen(base64data); /* return the length of the new data */
+  *outlen = strlen(base64data); /* return the length of the new data */
+
+  return CURLE_OK;
 }
 /* ---- End of Base64 Encoding ---- */