mime: do not perform more than one read in a row

Input buffer filling may delay the data sending if data reads are slow.
To overcome this problem, file and callback data reads do not accumulate
in buffer anymore. All other data (memory data and mime framing) are
considered as fast and still concatenated in buffer.
As this may highly impact performance in terms of data overhead, an early
end of part data check is added to spare a read call.
When encoding a part's data, an encoder may require more bytes than made
available by a single read. In this case, the above rule does not apply
and reads are performed until the encoder is able to deliver some data.

Tests 643, 644, 645, 650 and 654 have been adapted to the output data
changes, with test data size reduced to avoid the boredom of long lists of
1-byte chunks in verification data.
New test 667 checks mimepost using single-byte read callback with encoder.
New test 668 checks the end of part data early detection.

Fixes #4826
Reported-by: MrdUkk on github

15 files changed, 679 insertions, 77 deletions

diff --git a/lib/mime.c b/lib/mime.c
index 5f928a171..7d754e63b 100644
--- a/lib/mime.c
+++ b/lib/mime.c
@@ -26,6 +26,7 @@
 
 #include "mime.h"
 #include "non-ascii.h"
+#include "warnless.h"
 #include "urldata.h"
 #include "sendf.h"
 
@@ -52,6 +53,10 @@
 
 
 #define READ_ERROR                      ((size_t) -1)
+#define STOP_FILLING                    ((size_t) -2)
+
+static size_t mime_subparts_read(char *buffer, size_t size, size_t nitems,
+                                 void *instream, bool *hasread);
 
 /* Encoders. */
 static size_t encoder_nop_read(char *buffer, size_t size, bool ateof,
@@ -354,10 +359,15 @@ static size_t encoder_nop_read(char *buffer, size_t size, bool ateof,
 
   (void) ateof;
 
+  if(!size)
+    return STOP_FILLING;
+
   if(size > insize)
     size = insize;
+
   if(size)
     memcpy(buffer, st->buf, size);
+
   st->bufbeg += size;
   return size;
 }
@@ -377,6 +387,9 @@ static size_t encoder_7bit_read(char *buffer, size_t size, bool ateof,
 
   (void) ateof;
 
+  if(!size)
+    return STOP_FILLING;
+
   if(size > cursize)
     size = cursize;
 
@@ -404,8 +417,11 @@ static size_t encoder_base64_read(char *buffer, size_t size, bool ateof,
     /* Line full ? */
     if(st->pos > MAX_ENCODED_LINE_LENGTH - 4) {
       /* Yes, we need 2 characters for CRLF. */
-      if(size < 2)
+      if(size < 2) {
+        if(!cursize)
+          return STOP_FILLING;
         break;
+      }
       *ptr++ = '\r';
       *ptr++ = '\n';
       st->pos = 0;
@@ -414,7 +430,12 @@ static size_t encoder_base64_read(char *buffer, size_t size, bool ateof,
     }
 
     /* Be sure there is enough space and input data for a base64 group. */
-    if(size < 4 || st->bufend - st->bufbeg < 3)
+    if(size < 4) {
+      if(!cursize)
+        return STOP_FILLING;
+      break;
+    }
+    if(st->bufend - st->bufbeg < 3)
       break;
 
     /* Encode three bytes as four characters. */
@@ -431,25 +452,31 @@ static size_t encoder_base64_read(char *buffer, size_t size, bool ateof,
   }
 
   /* If at eof, we have to flush the buffered data. */
-  if(ateof && size >= 4) {
-    /* Buffered data size can only be 0, 1 or 2. */
-    ptr[2] = ptr[3] = '=';
-    i = 0;
-    switch(st->bufend - st->bufbeg) {
-    case 2:
-      i = (st->buf[st->bufbeg + 1] & 0xFF) << 8;
-      /* FALLTHROUGH */
-    case 1:
-      i |= (st->buf[st->bufbeg] & 0xFF) << 16;
-      ptr[0] = base64[(i >> 18) & 0x3F];
-      ptr[1] = base64[(i >> 12) & 0x3F];
-      if(++st->bufbeg != st->bufend) {
-        ptr[2] = base64[(i >> 6) & 0x3F];
-        st->bufbeg++;
+  if(ateof) {
+    if(size < 4) {
+      if(!cursize)
+        return STOP_FILLING;
+    }
+    else {
+      /* Buffered data size can only be 0, 1 or 2. */
+      ptr[2] = ptr[3] = '=';
+      i = 0;
+      switch(st->bufend - st->bufbeg) {
+      case 2:
+        i = (st->buf[st->bufbeg + 1] & 0xFF) << 8;
+        /* FALLTHROUGH */
+      case 1:
+        i |= (st->buf[st->bufbeg] & 0xFF) << 16;
+        ptr[0] = base64[(i >> 18) & 0x3F];
+        ptr[1] = base64[(i >> 12) & 0x3F];
+        if(++st->bufbeg != st->bufend) {
+          ptr[2] = base64[(i >> 6) & 0x3F];
+          st->bufbeg++;
+        }
+        cursize += 4;
+        st->pos += 4;
+        break;
       }
-      cursize += 4;
-      st->pos += 4;
-      break;
     }
   }
 
@@ -581,8 +608,11 @@ static size_t encoder_qp_read(char *buffer, size_t size, bool ateof,
     }
 
     /* If the output buffer would overflow, do not store. */
-    if(len > size)
+    if(len > size) {
+      if(!cursize)
+        return STOP_FILLING;
       break;
+    }
 
     /* Append to output buffer. */
     memcpy(ptr, buf, len);
@@ -612,16 +642,18 @@ static size_t mime_mem_read(char *buffer, size_t size, size_t nitems,
                             void *instream)
 {
   curl_mimepart *part = (curl_mimepart *) instream;
-  size_t sz = (size_t) part->datasize - part->state.offset;
+  size_t sz = curlx_sotouz(part->datasize - part->state.offset);
   (void) size;   /* Always 1.*/
 
+  if(!nitems)
+    return STOP_FILLING;
+
   if(sz > nitems)
     sz = nitems;
 
   if(sz)
-    memcpy(buffer, (char *) &part->data[part->state.offset], sz);
+    memcpy(buffer, part->data + curlx_sotouz(part->state.offset), sz);
 
-  part->state.offset += sz;
   return sz;
 }
 
@@ -641,7 +673,7 @@ static int mime_mem_seek(void *instream, curl_off_t offset, int whence)
   if(offset < 0 || offset > part->datasize)
     return CURL_SEEKFUNC_FAIL;
 
-  part->state.offset = (size_t) offset;
+  part->state.offset = offset;
   return CURL_SEEKFUNC_OK;
 }
 
@@ -668,6 +700,9 @@ static size_t mime_file_read(char *buffer, size_t size, size_t nitems,
 {
   curl_mimepart *part = (curl_mimepart *) instream;
 
+  if(!nitems)
+    return STOP_FILLING;
+
   if(mime_open_file(part))
     return READ_ERROR;
 
@@ -711,15 +746,16 @@ static size_t readback_bytes(mime_state *state,
                              const char *trail)
 {
   size_t sz;
+  size_t offset = curlx_sotouz(state->offset);
 
-  if(numbytes > state->offset) {
-    sz = numbytes - state->offset;
-    bytes += state->offset;
+  if(numbytes > offset) {
+    sz = numbytes - offset;
+    bytes += offset;
   }
   else {
     size_t tsz = strlen(trail);
 
-    sz = state->offset - numbytes;
+    sz = offset - numbytes;
     if(sz >= tsz)
       return 0;
     bytes = trail + sz;
@@ -736,7 +772,7 @@ static size_t readback_bytes(mime_state *state,
 
 /* Read a non-encoded part content. */
 static size_t read_part_content(curl_mimepart *part,
-                                char *buffer, size_t bufsize)
+                                char *buffer, size_t bufsize, bool *hasread)
 {
   size_t sz = 0;
 
@@ -745,26 +781,70 @@ static size_t read_part_content(curl_mimepart *part,
   case CURL_READFUNC_ABORT:
   case CURL_READFUNC_PAUSE:
   case READ_ERROR:
+    return part->lastreadstatus;
+  default:
+    break;
+  }
+
+  /* If we can determine we are at end of part data, spare a read. */
+  if(part->datasize != (curl_off_t) -1 &&
+     part->state.offset >= part->datasize) {
+    /* sz is already zero. */
+  }
+  else {
+    switch(part->kind) {
+    case MIMEKIND_MULTIPART:
+      /*
+       * Cannot be processed as other kinds since read function requires
+       * an additional parameter and is highly recursive.
+       */
+       sz = mime_subparts_read(buffer, 1, bufsize, part->arg, hasread);
+       break;
+    case MIMEKIND_FILE:
+      if(part->fp && feof(part->fp))
+        break;  /* At EOF. */
+      /* FALLTHROUGH */
+    default:
+      if(part->readfunc) {
+        if(!(part->flags & MIME_FAST_READ)) {
+          if(*hasread)
+            return STOP_FILLING;
+          *hasread = TRUE;
+        }
+        sz = part->readfunc(buffer, 1, bufsize, part->arg);
+      }
+      break;
+    }
+  }
+
+  switch(sz) {
+  case STOP_FILLING:
+    break;
+  case 0:
+  case CURL_READFUNC_ABORT:
+  case CURL_READFUNC_PAUSE:
+  case READ_ERROR:
+    part->lastreadstatus = sz;
     break;
   default:
-    if(part->readfunc)
-      sz = part->readfunc(buffer, 1, bufsize, part->arg);
+    part->state.offset += sz;
     part->lastreadstatus = sz;
     break;
   }
-  return part->lastreadstatus;
+
+  return sz;
 }
 
 /* Read and encode part content. */
-static size_t read_encoded_part_content(curl_mimepart *part,
-                                        char *buffer, size_t bufsize)
+static size_t read_encoded_part_content(curl_mimepart *part, char *buffer,
+                                        size_t bufsize, bool *hasread)
 {
   mime_encoder_state *st = &part->encstate;
   size_t cursize = 0;
   size_t sz;
   bool ateof = FALSE;
 
-  while(bufsize) {
+  for(;;) {
     if(st->bufbeg < st->bufend || ateof) {
       /* Encode buffered data. */
       sz = part->encoder->encodefunc(buffer, bufsize, ateof, part);
@@ -773,9 +853,8 @@ static size_t read_encoded_part_content(curl_mimepart *part,
         if(ateof)
           return cursize;
         break;
-      case CURL_READFUNC_ABORT:
-      case CURL_READFUNC_PAUSE:
       case READ_ERROR:
+      case STOP_FILLING:
         return cursize? cursize: sz;
       default:
         cursize += sz;
@@ -797,7 +876,7 @@ static size_t read_encoded_part_content(curl_mimepart *part,
     if(st->bufend >= sizeof(st->buf))
       return cursize? cursize: READ_ERROR;    /* Buffer full. */
     sz = read_part_content(part, st->buf + st->bufend,
-                           sizeof(st->buf) - st->bufend);
+                           sizeof(st->buf) - st->bufend, hasread);
     switch(sz) {
     case 0:
       ateof = TRUE;
@@ -805,6 +884,7 @@ static size_t read_encoded_part_content(curl_mimepart *part,
     case CURL_READFUNC_ABORT:
     case CURL_READFUNC_PAUSE:
     case READ_ERROR:
+    case STOP_FILLING:
       return cursize? cursize: sz;
     default:
       st->bufend += sz;
@@ -812,12 +892,12 @@ static size_t read_encoded_part_content(curl_mimepart *part,
     }
   }
 
-  return cursize;
+  /* NOTREACHED */
 }
 
 /* Readback a mime part. */
 static size_t readback_part(curl_mimepart *part,
-                            char *buffer, size_t bufsize)
+                            char *buffer, size_t bufsize, bool *hasread)
 {
   size_t cursize = 0;
 #ifdef CURL_DOES_CONVERSIONS
@@ -876,9 +956,9 @@ static size_t readback_part(curl_mimepart *part,
       break;
     case MIMESTATE_CONTENT:
       if(part->encoder)
-        sz = read_encoded_part_content(part, buffer, bufsize);
+        sz = read_encoded_part_content(part, buffer, bufsize, hasread);
       else
-        sz = read_part_content(part, buffer, bufsize);
+        sz = read_part_content(part, buffer, bufsize, hasread);
       switch(sz) {
       case 0:
         mimesetstate(&part->state, MIMESTATE_END, NULL);
@@ -891,6 +971,7 @@ static size_t readback_part(curl_mimepart *part,
       case CURL_READFUNC_ABORT:
       case CURL_READFUNC_PAUSE:
       case READ_ERROR:
+      case STOP_FILLING:
         return cursize? cursize: sz;
       }
       break;
@@ -919,9 +1000,9 @@ static size_t readback_part(curl_mimepart *part,
   return cursize;
 }
 
-/* Readback from mime. */
+/* Readback from mime. Warning: not a read callback function. */
 static size_t mime_subparts_read(char *buffer, size_t size, size_t nitems,
-                                 void *instream)
+                                 void *instream, bool *hasread)
 {
   curl_mime *mime = (curl_mime *) instream;
   size_t cursize = 0;
@@ -942,7 +1023,7 @@ static size_t mime_subparts_read(char *buffer, size_t size, size_t nitems,
 #endif
       mimesetstate(&mime->state, MIMESTATE_BOUNDARY1, mime->firstpart);
       /* The first boundary always follows the header termination empty line,
-         so is always preceded by a CRLK. We can then spare 2 characters
+         so is always preceded by a CRLF. We can then spare 2 characters
          by skipping the leading CRLF in boundary. */
       mime->state.offset += 2;
       break;
@@ -972,11 +1053,12 @@ static size_t mime_subparts_read(char *buffer, size_t size, size_t nitems,
         mimesetstate(&mime->state, MIMESTATE_END, NULL);
         break;
       }
-      sz = readback_part(part, buffer, nitems);
+      sz = readback_part(part, buffer, nitems, hasread);
       switch(sz) {
       case CURL_READFUNC_ABORT:
       case CURL_READFUNC_PAUSE:
       case READ_ERROR:
+      case STOP_FILLING:
         return cursize? cursize: sz;
       case 0:
 #ifdef CURL_DOES_CONVERSIONS
@@ -1084,6 +1166,7 @@ static void cleanup_part_content(curl_mimepart *part)
   part->datasize = (curl_off_t) 0;    /* No size yet. */
   cleanup_encoder_state(&part->encstate);
   part->kind = MIMEKIND_NONE;
+  part->flags &= ~MIME_FAST_READ;
   part->lastreadstatus = 1; /* Successful read status. */
 }
 
@@ -1341,6 +1424,7 @@ CURLcode curl_mime_data(curl_mimepart *part,
     part->readfunc = mime_mem_read;
     part->seekfunc = mime_mem_seek;
     part->freefunc = mime_mem_free;
+    part->flags |= MIME_FAST_READ;
     part->kind = MIMEKIND_DATA;
   }
 
@@ -1515,7 +1599,7 @@ CURLcode Curl_mime_set_subparts(curl_mimepart *part,
     }
 
     subparts->parent = part;
-    part->readfunc = mime_subparts_read;
+    /* Subparts are processed internally: no read callback. */
     part->seekfunc = mime_subparts_seek;
     part->freefunc = take_ownership? mime_subparts_free: mime_subparts_unbind;
     part->arg = subparts;
@@ -1537,9 +1621,23 @@ CURLcode curl_mime_subparts(curl_mimepart *part, curl_mime *subparts)
 size_t Curl_mime_read(char *buffer, size_t size, size_t nitems, void *instream)
 {
   curl_mimepart *part = (curl_mimepart *) instream;
+  size_t ret;
+  bool hasread;
 
   (void) size;   /* Always 1. */
-  return readback_part(part, buffer, nitems);
+
+  do {
+    hasread = FALSE;
+    ret = readback_part(part, buffer, nitems, &hasread);
+    /*
+     * If this is not possible to get some data without calling more than
+     * one read callback (probably because a content encoder is not able to
+     * deliver a new bunch for the few data accumulated so far), force another
+     * read until we get enough data or a special exit code.
+     */
+  } while(ret == STOP_FILLING);
+
+  return ret;
 }
 
 /* Rewind mime stream. */
diff --git a/lib/mime.h b/lib/mime.h
index c6d374ec1..d7f25132e 100644
--- a/lib/mime.h
+++ b/lib/mime.h
@@ -31,6 +31,7 @@
 /* Part flags. */
 #define MIME_USERHEADERS_OWNER  (1 << 0)
 #define MIME_BODY_ONLY          (1 << 1)
+#define MIME_FAST_READ          (1 << 2)
 
 #define FILE_CONTENTTYPE_DEFAULT        "application/octet-stream"
 #define MULTIPART_CONTENTTYPE_DEFAULT   "multipart/mixed"
@@ -87,7 +88,7 @@ typedef struct {
 typedef struct {
   enum mimestate state;       /* Current state token. */
   void *ptr;                  /* State-dependent pointer. */
-  size_t offset;              /* State-dependent offset. */
+  curl_off_t offset;          /* State-dependent offset. */
 }  mime_state;
 
 /* minimum buffer size for the boundary string */
diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc
index 99052007f..4cb7a8e74 100644
--- a/tests/data/Makefile.inc
+++ b/tests/data/Makefile.inc
@@ -85,7 +85,7 @@ test626 test627 test628 test629 test630 test631 test632 test633 test634 \
 test635 test636 test637 test638 test639 test640 test641 test642 \
 test643 test644 test645 test646 test647 test648 test649 test650 test651 \
 test652 test653 test654 test655 test656 test658 test659 test660 test661 \
-test662 test663 test664 test665 \
+test662 test663 test664 test665 test667 test668 \
 test670 test671 test672 test673 \
 \
 test700 test701 test702 test703 test704 test705 test706 test707 test708 \
diff --git a/tests/data/test643 b/tests/data/test643
index 0eaf22c5f..e8da2b1c4 100644
--- a/tests/data/test643
+++ b/tests/data/test643
@@ -69,18 +69,18 @@ s/boundary=------------------------[a-z0-9]*/boundary=--------------------------
 POST /643 HTTP/1.1

 Host: %HOSTIP:%HTTPPORT

 Accept: */*

-Content-Length: 718

+Content-Length: 640

 Content-Type: multipart/form-data; boundary=----------------------------

 

 ------------------------------

 Content-Disposition: form-data; name="sendfile"; filename="postit2.c"

 

-this is what we post to the silly web server
+dummy
 

 ------------------------------

 Content-Disposition: form-data; name="callbackdata"

 

-this is what we post to the silly web server
+dummy
 

 ------------------------------

 Content-Disposition: form-data; name="filename"

@@ -99,18 +99,18 @@ blah blah
 POST /643 HTTP/1.1

 Host: %HOSTIP:%HTTPPORT

 Accept: */*

-Content-Length: 732

+Content-Length: 654

 Content-Type: multipart/form-data; boundary=----------------------------

 

 ------------------------------

 Content-Disposition: form-data; name="sendfile alternative"; filename="file name 2"

 

-this is what we post to the silly web server
+dummy
 

 ------------------------------

 Content-Disposition: form-data; name="callbackdata"

 

-this is what we post to the silly web server
+dummy
 

 ------------------------------

 Content-Disposition: form-data; name="filename"

diff --git a/tests/data/test644 b/tests/data/test644
index 256d3379a..99337ae0f 100644
--- a/tests/data/test644
+++ b/tests/data/test644
@@ -44,7 +44,7 @@ s/boundary=------------------------[a-z0-9]*/boundary=--------------------------
 POST /644 HTTP/1.1

 Host: %HOSTIP:%HTTPPORT

 Accept: */*

-Content-Length: 718

+Content-Length: 640

 Content-Type: multipart/form-data; boundary=----------------------------

 

 ------------------------------

diff --git a/tests/data/test645 b/tests/data/test645
index 6533944b4..eeb15f994 100644
--- a/tests/data/test645
+++ b/tests/data/test645
@@ -73,16 +73,42 @@ Transfer-Encoding: chunked
 Content-Type: multipart/form-data; boundary=----------------------------

 Expect: 100-continue

 

-2ce

+76

 ------------------------------

 Content-Disposition: form-data; name="sendfile"; filename="postit2.c"

 

-this is what we post to the silly web server
+d

+1

+u

+1

+m

+1

+m

+1

+y

+1

+
+

+65

 

 ------------------------------

 Content-Disposition: form-data; name="callbackdata"

 

-this is what we post to the silly web server
+

+1

+d

+1

+u

+1

+m

+1

+m

+1

+y

+1

+
+

+19a

 

 ------------------------------

 Content-Disposition: form-data; name="filename"

@@ -108,16 +134,42 @@ Transfer-Encoding: chunked
 Content-Type: multipart/form-data; boundary=----------------------------

 Expect: 100-continue

 

-2dc

+84

 ------------------------------

 Content-Disposition: form-data; name="sendfile alternative"; filename="file name 2"

 

-this is what we post to the silly web server
+d

+1

+u

+1

+m

+1

+m

+1

+y

+1

+
+

+65

 

 ------------------------------

 Content-Disposition: form-data; name="callbackdata"

 

-this is what we post to the silly web server
+

+1

+d

+1

+u

+1

+m

+1

+m

+1

+y

+1

+
+

+19a

 

 ------------------------------

 Content-Disposition: form-data; name="filename"

diff --git a/tests/data/test650 b/tests/data/test650
index 1a06064c7..03fe43816 100644
--- a/tests/data/test650
+++ b/tests/data/test650
@@ -63,7 +63,7 @@ Transfer-Encoding: chunked
 Content-Type: multipart/form-data; boundary=----------------------------

 Expect: 100-continue

 

-60a

+361

 ------------------------------

 Content-Disposition: form-data; name="fieldname"

 Content-Type: text/plain

@@ -89,12 +89,16 @@ This is data from a file.
 Content-Disposition: attachment; filename="test650.filedata"

 Content-Type: text/whatever

 

+

+a5

 This is data from a file.
 

 ------------------------------

 Content-Disposition: attachment; filename="test650.filedata"

 Content-Type: text/whatever

 

+

+af

 This is data from a file.
 

 --------------------------------

@@ -102,6 +106,8 @@ This is data from a file.
 ------------------------------

 Content-Disposition: form-data; name="filecontents"

 

+

+10f

 This is data from a file.
 

 ------------------------------

@@ -112,8 +118,12 @@ Content-Disposition: form-data; name="formlength"
 Content-Disposition: form-data; name="standardinput"

 Content-Type: application/octet-stream

 

+

+16

  Some data from stdin
 

+30

+

 --------------------------------

 

 0

diff --git a/tests/data/test654 b/tests/data/test654
index 21697e763..9d4bf45a1 100644
--- a/tests/data/test654
+++ b/tests/data/test654
@@ -81,7 +81,7 @@ Transfer-Encoding: chunked
 Content-Type: multipart/form-data; boundary=----------------------------

 Expect: 100-continue

 

-20c

+1af

 ------------------------------

 Content-Disposition: form-data; name="greeting"

 Content-Type: application/X-Greeting

@@ -98,7 +98,21 @@ This is data from a file
 ------------------------------

 Content-Disposition: form-data

 

-this is what we post to the silly web server
+

+1

+d

+1

+u

+1

+m

+1

+m

+1

+y

+1

+
+

+30

 

 --------------------------------

 

diff --git a/tests/data/test667 b/tests/data/test667
new file mode 100644
index 000000000..2a183753b
--- /dev/null
+++ b/tests/data/test667
@@ -0,0 +1,85 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP POST
+HTTP MIME POST
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data>
+HTTP/1.1 200 OK

+Date: Thu, 09 Nov 2010 14:49:00 GMT

+Server: test-server/fake swsclose

+Connection: close

+Content-Type: text/html

+

+hello
+</data>
+<datacheck>
+HTTP/1.1 200 OK

+Date: Thu, 09 Nov 2010 14:49:00 GMT

+Server: test-server/fake swsclose

+Connection: close

+Content-Type: text/html

+

+hello
+</datacheck>
+</reply>
+
+# Client-side
+<client>
+<server>
+http
+</server>
+# tool is what to use instead of 'curl'
+<tool>
+lib667
+</tool>
+
+ <name>
+HTTP chunked mimepost using single-byte read callback with encoder
+ </name>
+ <command>
+http://%HOSTIP:%HTTPPORT/667
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<strippart>
+s/^--------------------------[a-z0-9]*/------------------------------/
+s/boundary=------------------------[a-z0-9]*/boundary=----------------------------/
+</strippart>
+# Note that the stripping above removes 12 bytes from every occurrence of the
+# boundary string and since 5 of them are in the body contents, we see
+# (5*12) == 60 bytes less
+<protocol>
+POST /667 HTTP/1.1

+Host: %HOSTIP:%HTTPPORT

+Accept: */*

+Transfer-Encoding: chunked

+Content-Type: multipart/form-data; boundary=----------------------------

+Expect: 100-continue

+

+7f

+------------------------------

+Content-Disposition: form-data; name="field"

+Content-Transfer-Encoding: base64

+

+

+4

+ZHVt

+34

+bXk=

+--------------------------------

+

+0

+

+</protocol>
+</verify>
+</testcase>
diff --git a/tests/data/test668 b/tests/data/test668
new file mode 100644
index 000000000..af0a72051
--- /dev/null
+++ b/tests/data/test668
@@ -0,0 +1,99 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP POST
+HTTP MIME POST
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+<data>
+HTTP/1.1 200 OK

+Date: Thu, 09 Nov 2010 14:49:00 GMT

+Server: test-server/fake swsclose

+Connection: close

+Content-Type: text/html

+

+hello
+</data>
+<datacheck>
+HTTP/1.1 200 OK

+Date: Thu, 09 Nov 2010 14:49:00 GMT

+Server: test-server/fake swsclose

+Connection: close

+Content-Type: text/html

+

+hello
+</datacheck>
+</reply>
+
+# Client-side
+<client>
+<server>
+http
+</server>
+# tool is what to use instead of 'curl'
+<tool>
+lib668
+</tool>
+
+ <name>
+HTTP mimepost early end of data detection
+ </name>
+ <command>
+http://%HOSTIP:%HTTPPORT/668
+</command>
+<file name="log/file668.txt">
+This is data from a file
+</file>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<strippart>
+s/^--------------------------[a-z0-9]*/------------------------------/
+s/boundary=------------------------[a-z0-9]*/boundary=----------------------------/
+</strippart>
+# Note that the stripping above removes 12 bytes from every occurrence of the
+# boundary string and since 5 of them are in the body contents, we see
+# (5*12) == 60 bytes less
+<protocol>
+POST /668 HTTP/1.1

+Host: %HOSTIP:%HTTPPORT

+Accept: */*

+Transfer-Encoding: chunked

+Content-Type: multipart/form-data; boundary=----------------------------

+Expect: 100-continue

+

+c1

+------------------------------

+Content-Disposition: form-data; name="field1"

+

+dummy

+------------------------------

+Content-Disposition: form-data; name="field2"

+

+

+5

+dummy

+91

+

+------------------------------

+Content-Disposition: form-data; name="field3"; filename="file668.txt"

+Content-Type: text/plain

+

+

+49

+This is data from a file
+

+--------------------------------

+

+0

+

+</protocol>
+</verify>
+</testcase>
diff --git a/tests/libtest/Makefile.inc b/tests/libtest/Makefile.inc
index 9652f03fe..a8a148e66 100644
--- a/tests/libtest/Makefile.inc
+++ b/tests/libtest/Makefile.inc
@@ -22,7 +22,7 @@ noinst_PROGRAMS = chkhostname libauthretry libntlmconnect                \
  lib571 lib572 lib573 lib574 lib575 lib576        lib578 lib579 lib582   \
  lib583 lib585 lib586 lib587 lib589 lib590 lib591 lib597 lib598 lib599   \
  lib643 lib644 lib645 lib650 lib651 lib652 lib653 lib654 lib655 lib658   \
- lib659 lib661 \
+ lib659 lib661 lib667 lib668 \
  lib670 lib671 lib672 lib673 \
  lib1156 \
  lib1500 lib1501 lib1502 lib1503 lib1504 lib1505 lib1506 lib1507 lib1508 \
@@ -349,6 +349,14 @@ lib659_CPPFLAGS = $(AM_CPPFLAGS)
 lib661_SOURCES = lib661.c $(SUPPORTFILES)
 lib661_CPPFLAGS = $(AM_CPPFLAGS)
 
+lib667_SOURCES = lib667.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS)
+lib667_LDADD = $(TESTUTIL_LIBS)
+lib667_CPPFLAGS = $(AM_CPPFLAGS)
+
+lib668_SOURCES = lib668.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS)
+lib668_LDADD = $(TESTUTIL_LIBS)
+lib668_CPPFLAGS = $(AM_CPPFLAGS)
+
 lib670_SOURCES = lib670.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS)
 lib670_LDADD = $(TESTUTIL_LIBS)
 lib670_CPPFLAGS = $(AM_CPPFLAGS) -DLIB670
diff --git a/tests/libtest/lib643.c b/tests/libtest/lib643.c
index 08c0f2e80..5af0f4a14 100644
--- a/tests/libtest/lib643.c
+++ b/tests/libtest/lib643.c
@@ -26,11 +26,9 @@
 static char data[]=
 #ifdef CURL_DOES_CONVERSIONS
   /* ASCII representation with escape sequences for non-ASCII platforms */
-  "\x74\x68\x69\x73\x20\x69\x73\x20\x77\x68\x61\x74\x20\x77\x65\x20\x70"
-  "\x6f\x73\x74\x20\x74\x6f\x20\x74\x68\x65\x20\x73\x69\x6c\x6c\x79\x20"
-  "\x77\x65\x62\x20\x73\x65\x72\x76\x65\x72\x0a";
+  "\x64\x75\x6d\x6d\x79\x0a";
 #else
-  "this is what we post to the silly web server\n";
+  "dummy\n";
 #endif
 
 struct WriteThis {
diff --git a/tests/libtest/lib654.c b/tests/libtest/lib654.c
index 45051a9c7..f9c8b9118 100644
--- a/tests/libtest/lib654.c
+++ b/tests/libtest/lib654.c
@@ -5,7 +5,7 @@
  *                            | (__| |_| |  _ <| |___
  *                             \___|\___/|_| \_\_____|
  *
- * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
+ * Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
  *
  * This software is licensed as described in the file COPYING, which
  * you should have received as part of this distribution. The terms
@@ -26,11 +26,9 @@
 static char data[]=
 #ifdef CURL_DOES_CONVERSIONS
   /* ASCII representation with escape sequences for non-ASCII platforms */
-  "\x74\x68\x69\x73\x20\x69\x73\x20\x77\x68\x61\x74\x20\x77\x65\x20\x70"
-  "\x6f\x73\x74\x20\x74\x6f\x20\x74\x68\x65\x20\x73\x69\x6c\x6c\x79\x20"
-  "\x77\x65\x62\x20\x73\x65\x72\x76\x65\x72\x0a";
+  "\x64\x75\x6d\x6d\x79\x0a";
 #else
-  "this is what we post to the silly web server\n";
+  "dummy\n";
 #endif
 
 struct WriteThis {
diff --git a/tests/libtest/lib667.c b/tests/libtest/lib667.c
new file mode 100644
index 000000000..8bf7be43f
--- /dev/null
+++ b/tests/libtest/lib667.c
@@ -0,0 +1,117 @@
+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at https://curl.haxx.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ ***************************************************************************/
+#include "test.h"
+
+#include "memdebug.h"
+
+static char data[]=
+#ifdef CURL_DOES_CONVERSIONS
+  /* ASCII representation with escape sequences for non-ASCII platforms */
+  "\x64\x75\x6d\x6d\x79";
+#else
+  "dummy";
+#endif
+
+struct WriteThis {
+  char *readptr;
+  curl_off_t sizeleft;
+};
+
+static size_t read_callback(char *ptr, size_t size, size_t nmemb, void *userp)
+{
+  struct WriteThis *pooh = (struct WriteThis *)userp;
+  int eof = !*pooh->readptr;
+
+  if(size*nmemb < 1)
+    return 0;
+
+  eof = pooh->sizeleft <= 0;
+  if(!eof)
+    pooh->sizeleft--;
+
+  if(!eof) {
+    *ptr = *pooh->readptr;           /* copy one single byte */
+    pooh->readptr++;                 /* advance pointer */
+    return 1;                        /* we return 1 byte at a time! */
+  }
+
+  return 0;                         /* no more data left to deliver */
+}
+
+int test(char *URL)
+{
+  CURL *easy = NULL;
+  curl_mime *mime = NULL;
+  curl_mimepart *part;
+  CURLcode result;
+  int res = TEST_ERR_FAILURE;
+  struct WriteThis pooh;
+
+  /*
+   * Check proper handling of mime encoder feature when the part read callback
+   * delivers data bytes one at a time. Use chunked encoding for accurate test.
+   */
+
+  if(curl_global_init(CURL_GLOBAL_ALL) != CURLE_OK) {
+    fprintf(stderr, "curl_global_init() failed\n");
+    return TEST_ERR_MAJOR_BAD;
+  }
+
+  easy = curl_easy_init();
+
+  /* First set the URL that is about to receive our POST. */
+  test_setopt(easy, CURLOPT_URL, URL);
+
+  /* get verbose debug output please */
+  test_setopt(easy, CURLOPT_VERBOSE, 1L);
+
+  /* include headers in the output */
+  test_setopt(easy, CURLOPT_HEADER, 1L);
+
+  /* Prepare the callback structure. */
+  pooh.readptr = data;
+  pooh.sizeleft = (curl_off_t) strlen(data);
+
+  /* Build the mime tree. */
+  mime = curl_mime_init(easy);
+  part = curl_mime_addpart(mime);
+  curl_mime_name(part, "field");
+  curl_mime_encoder(part, "base64");
+  /* Using an undefined length forces chunked transfer. */
+  curl_mime_data_cb(part, (curl_off_t) -1, read_callback, NULL, NULL, &pooh);
+
+  /* Bind mime data to its easy handle. */
+  test_setopt(easy, CURLOPT_MIMEPOST, mime);
+
+  /* Send data. */
+  result = curl_easy_perform(easy);
+  if(result) {
+    fprintf(stderr, "curl_easy_perform() failed\n");
+    res = (int) result;
+  }
+
+test_cleanup:
+  curl_easy_cleanup(easy);
+  curl_mime_free(mime);
+  curl_global_cleanup();
+  return res;
+}
diff --git a/tests/libtest/lib668.c b/tests/libtest/lib668.c
new file mode 100644
index 000000000..c0b608a0a
--- /dev/null
+++ b/tests/libtest/lib668.c
@@ -0,0 +1,122 @@
+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at https://curl.haxx.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ ***************************************************************************/
+#include "test.h"
+
+#include "memdebug.h"
+
+static char data[]=
+#ifdef CURL_DOES_CONVERSIONS
+  /* ASCII representation with escape sequences for non-ASCII platforms */
+  "\x64\x75\x6d\x6d\x79";
+#else
+  "dummy";
+#endif
+
+struct WriteThis {
+  char *readptr;
+  curl_off_t sizeleft;
+};
+
+static size_t read_callback(char *ptr, size_t size, size_t nmemb, void *userp)
+{
+  struct WriteThis *pooh = (struct WriteThis *)userp;
+  size_t len = strlen(pooh->readptr);
+
+  (void) size; /* Always 1.*/
+
+  if(len > nmemb)
+    len = nmemb;
+  if(len) {
+    memcpy(ptr, pooh->readptr, len);
+    pooh->readptr += len;
+  }
+  return len;
+}
+
+int test(char *URL)
+{
+  CURL *easy = NULL;
+  curl_mime *mime = NULL;
+  curl_mimepart *part;
+  CURLcode result;
+  int res = TEST_ERR_FAILURE;
+  struct WriteThis pooh1, pooh2;
+
+  /*
+   * Check early end of part data detection.
+   */
+
+  if(curl_global_init(CURL_GLOBAL_ALL) != CURLE_OK) {
+    fprintf(stderr, "curl_global_init() failed\n");
+    return TEST_ERR_MAJOR_BAD;
+  }
+
+  easy = curl_easy_init();
+
+  /* First set the URL that is about to receive our POST. */
+  test_setopt(easy, CURLOPT_URL, URL);
+
+  /* get verbose debug output please */
+  test_setopt(easy, CURLOPT_VERBOSE, 1L);
+
+  /* include headers in the output */
+  test_setopt(easy, CURLOPT_HEADER, 1L);
+
+  /* Prepare the callback structures. */
+  pooh1.readptr = data;
+  pooh1.sizeleft = (curl_off_t) strlen(data);
+  pooh2 = pooh1;
+
+  /* Build the mime tree. */
+  mime = curl_mime_init(easy);
+  part = curl_mime_addpart(mime);
+  curl_mime_name(part, "field1");
+  /* Early end of data detection can be done because the data size is known. */
+  curl_mime_data_cb(part, (curl_off_t) strlen(data),
+                    read_callback, NULL, NULL, &pooh1);
+  part = curl_mime_addpart(mime);
+  curl_mime_name(part, "field2");
+  /* Using an undefined length forces chunked transfer and disables early
+     end of data detection for this part. */
+  curl_mime_data_cb(part, (curl_off_t) -1, read_callback, NULL, NULL, &pooh2);
+  part = curl_mime_addpart(mime);
+  curl_mime_name(part, "field3");
+  /* Regular file part sources early end of data can be detected because
+     the file size is known. In addition, and EOF test is performed. */
+  curl_mime_filedata(part, "log/file668.txt");
+
+  /* Bind mime data to its easy handle. */
+  test_setopt(easy, CURLOPT_MIMEPOST, mime);
+
+  /* Send data. */
+  result = curl_easy_perform(easy);
+  if(result) {
+    fprintf(stderr, "curl_easy_perform() failed\n");
+    res = (int) result;
+  }
+
+test_cleanup:
+  curl_easy_cleanup(easy);
+  curl_mime_free(mime);
+  curl_global_cleanup();
+  return res;
+}