mime: use percent-escaping for multipart form field and file names

Until now, form field and file names where escaped using the
backslash-escaping algorithm defined for multipart mails. This commit
replaces this with the percent-escaping method for URLs.

As this may introduce incompatibilities with server-side applications, a
new libcurl option CURLOPT_MIME_OPTIONS with bitmask
CURLMIMEOPT_FORMESCAPE is introduced to revert to legacy use of
backslash-escaping. This is controlled by new cli tool option
--form-escape.

New tests and documentation are provided for this feature.

Reported by: Ryan Sleevi
Fixes #7789
Closes #7805

6 files changed, 278 insertions, 10 deletions

diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc
index b6a503e72..cbc709d63 100644
--- a/tests/data/Makefile.inc
+++ b/tests/data/Makefile.inc
@@ -144,8 +144,7 @@ test1152 test1153 test1154 test1155 test1156 test1157 test1158 test1159 \
 test1160 test1161 test1162 test1163 test1164 test1165 test1166 test1167 \
 test1168 test1169 test1170 test1171 test1172 test1173 test1174 test1175 \
 test1176 test1177 test1178 test1179 test1180 test1181 test1182 test1183 \
-test1184 test1185 \
-test1188 \
+test1184 test1185 test1186 test1187 test1188 test1189 \
 \
 test1190 test1191 test1192 test1193 test1194 test1195 test1196 test1197 \
 test1198 test1199 \
diff --git a/tests/data/test1158 b/tests/data/test1158
index fd21d74bb..3771f5d0b 100644
--- a/tests/data/test1158
+++ b/tests/data/test1158
@@ -50,11 +50,11 @@ POST /we/want/%TESTNUMBER HTTP/1.1
 Host: %HOSTIP:%HTTPPORT

 User-Agent: curl/%VERSION

 Accept: */*

-Content-Length: 954

+Content-Length: 958

 Content-Type: multipart/form-data; boundary=----------------------------24e78000bd32

 

 ------------------------------24e78000bd32

-Content-Disposition: form-data; name="file"; filename="test%TESTNUMBER\".txt"

+Content-Disposition: form-data; name="file"; filename="test%TESTNUMBER%22.txt"

 Content-Type: mo/foo

 

 foo bar
@@ -63,7 +63,7 @@ bar
 foo
 

 ------------------------------24e78000bd32

-Content-Disposition: form-data; name="file2"; filename="test%TESTNUMBER\".txt"

+Content-Disposition: form-data; name="file2"; filename="test%TESTNUMBER%22.txt"

 Content-Type: text/plain

 

 foo bar
@@ -75,7 +75,7 @@ foo
 Content-Disposition: form-data; name="file3"

 Content-Type: multipart/mixed; boundary=----------------------------7f0e85a48b0b

 

-Content-Disposition: attachment; filename="test%TESTNUMBER\".txt"

+Content-Disposition: attachment; filename="test%TESTNUMBER%22.txt"

 Content-Type: m/f

 

 foo bar
@@ -83,7 +83,7 @@ This is a bar foo
 bar
 foo
 

-Content-Disposition: attachment; filename="test%TESTNUMBER\".txt"

+Content-Disposition: attachment; filename="test%TESTNUMBER%22.txt"

 Content-Type: text/plain

 

 foo bar
diff --git a/tests/data/test1186 b/tests/data/test1186
new file mode 100644
index 000000000..e4662b697
--- /dev/null
+++ b/tests/data/test1186
@@ -0,0 +1,98 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP FORMPOST
+</keywords>
+</info>
+# Server-side
+<reply>
+<data>
+HTTP/1.1 200 OK

+Date: Tue, 09 Nov 2010 14:49:00 GMT

+Server: test-server/fake

+Content-Length: 10

+

+blablabla
+</data>
+</reply>
+
+# Client-side
+<client>
+<server>
+http
+</server>
+<name>
+Multipart formposting with backslash-escaping filename containing '"'
+</name>
+<command>
+http://%HOSTIP:%HTTPPORT/we/want/%TESTNUMBER --form-escape -F "file=@\"log/test%TESTNUMBER\\\".txt\";type=mo/foo;filename=\"test%TESTNUMBER\\\".txt\"" -F 'file2=@"log/test%TESTNUMBER\".txt"' -F 'file3=@"log/test%TESTNUMBER\".txt";type=m/f,"log/test%TESTNUMBER\".txt"'
+</command>
+<precheck>
+perl -e "print 'Test requires a system supporting double quotes in file names' if ($^O eq 'msys');"
+</precheck>
+# We create this file before the command is invoked!
+<file name=log/test%TESTNUMBER".txt>
+foo bar
+This is a bar foo
+bar
+foo
+</file>
+</client>
+
+# Verify data after the test has been "shot"
+<verify>
+<strip>
+^(Content-Type: multipart/form-data;|Content-Type: multipart/mixed; boundary=|-------).*
+</strip>
+<protocol>
+POST /we/want/%TESTNUMBER HTTP/1.1

+Host: %HOSTIP:%HTTPPORT

+User-Agent: curl/%VERSION

+Accept: */*

+Content-Length: 954

+Content-Type: multipart/form-data; boundary=----------------------------24e78000bd32

+

+------------------------------24e78000bd32

+Content-Disposition: form-data; name="file"; filename="test%TESTNUMBER\".txt"

+Content-Type: mo/foo

+

+foo bar
+This is a bar foo
+bar
+foo
+

+------------------------------24e78000bd32

+Content-Disposition: form-data; name="file2"; filename="test%TESTNUMBER\".txt"

+Content-Type: text/plain

+

+foo bar
+This is a bar foo
+bar
+foo
+

+------------------------------24e78000bd32

+Content-Disposition: form-data; name="file3"

+Content-Type: multipart/mixed; boundary=----------------------------7f0e85a48b0b

+

+Content-Disposition: attachment; filename="test%TESTNUMBER\".txt"

+Content-Type: m/f

+

+foo bar
+This is a bar foo
+bar
+foo
+

+Content-Disposition: attachment; filename="test%TESTNUMBER\".txt"

+Content-Type: text/plain

+

+foo bar
+This is a bar foo
+bar
+foo
+

+

+------------------------------24e78000bd32--

+</protocol>
+</verify>
+</testcase>
diff --git a/tests/data/test1187 b/tests/data/test1187
new file mode 100644
index 000000000..53abf908f
--- /dev/null
+++ b/tests/data/test1187
@@ -0,0 +1,63 @@
+<testcase>
+<info>
+<keywords>
+SMTP
+MULTIPART
+</keywords>
+</info>
+
+#
+# Server-side
+<reply>
+</reply>
+
+#
+# Client-side
+<client>
+<server>
+smtp
+</server>
+ <name>
+SMTP multipart with file name escaping
+ </name>
+<stdin>
+From: different

+To: another

+

+body

+</stdin>
+ <command>
+smtp://%HOSTIP:%SMTPPORT/%TESTNUMBER --mail-rcpt recipient@example.com --mail-from sender@example.com -F  "=This is the mail text" -F '=File content;filename="strange\file\"name"'
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<strippart>
+s/^--------------------------[a-z0-9]*/------------------------------/
+s/boundary=------------------------[a-z0-9]*/boundary=----------------------------/
+</strippart>
+<protocol>
+EHLO %TESTNUMBER

+MAIL FROM:<sender@example.com>

+RCPT TO:<recipient@example.com>

+DATA

+QUIT

+</protocol>
+<upload>
+Content-Type: multipart/mixed; boundary=----------------------------

+Mime-Version: 1.0

+

+------------------------------

+

+This is the mail text

+------------------------------

+Content-Disposition: attachment; filename="strange\\file\"name"

+

+File content

+--------------------------------

+.

+</upload>
+</verify>
+</testcase>
diff --git a/tests/data/test1189 b/tests/data/test1189
new file mode 100644
index 000000000..229e443d9
--- /dev/null
+++ b/tests/data/test1189
@@ -0,0 +1,108 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP FORMPOST
+</keywords>
+</info>
+# Server-side
+<reply>
+<data>
+HTTP/1.1 200 OK

+Date: Tue, 09 Nov 2010 14:49:00 GMT

+Server: test-server/fake

+Content-Length: 10

+

+blablabla
+</data>
+</reply>
+
+# Client-side
+<client>
+<server>
+http
+</server>
+ <name>
+Multipart formposting with backslash-escaping of name= and filename=
+ </name>
+ <command>
+http://%HOSTIP:%HTTPPORT/we/want/%TESTNUMBER --form-escape -F name=daniel -F tool=curl --form-string "str1=@literal" --form-string "str2=<verbatim;type=xxx/yyy" -F "file=@log/test%TESTNUMBER.txt;type=moo/foobar;filename=fakerfile" -F file2=@log/test%TESTNUMBER.txt -F "file3=@\"log/test%TESTNUMBER.txt\";type=mo/foo;filename=\"f\\\\\\\\ak\\\\\\er,\\\\an\\d;.t\\\"xt\"" -F 'file4=@"log/test%TESTNUMBER.txt"; filename="A\\AA\"\"\\\"ZZZ"'
+</command>
+# We create this file before the command is invoked!
+<file name="log/test%TESTNUMBER.txt">
+foo bar
+This is a bar foo
+bar
+foo
+</file>
+</client>
+
+# Verify data after the test has been "shot"
+<verify>
+<strip>
+^(Content-Type: multipart/form-data;|-------).*
+</strip>
+<protocol>
+POST /we/want/%TESTNUMBER HTTP/1.1

+Host: %HOSTIP:%HTTPPORT

+User-Agent: curl/%VERSION

+Accept: */*

+Content-Length: 1186

+Content-Type: multipart/form-data; boundary=----------------------------24e78000bd32

+

+------------------------------24e78000bd32

+Content-Disposition: form-data; name="name"

+

+daniel

+------------------------------24e78000bd32

+Content-Disposition: form-data; name="tool"

+

+curl

+------------------------------24e78000bd32

+Content-Disposition: form-data; name="str1"

+

+@literal

+------------------------------24e78000bd32

+Content-Disposition: form-data; name="str2"

+

+<verbatim;type=xxx/yyy

+------------------------------24e78000bd32

+Content-Disposition: form-data; name="file"; filename="fakerfile"

+Content-Type: moo/foobar

+

+foo bar
+This is a bar foo
+bar
+foo
+

+------------------------------24e78000bd32

+Content-Disposition: form-data; name="file2"; filename="test%TESTNUMBER.txt"

+Content-Type: text/plain

+

+foo bar
+This is a bar foo
+bar
+foo
+

+------------------------------24e78000bd32

+Content-Disposition: form-data; name="file3"; filename="f\\\\ak\\\\er,\\an\\d;.t\"xt"

+Content-Type: mo/foo

+

+foo bar
+This is a bar foo
+bar
+foo
+

+------------------------------24e78000bd32

+Content-Disposition: form-data; name="file4"; filename="A\\AA\"\"\\\"ZZZ"

+Content-Type: text/plain

+

+foo bar
+This is a bar foo
+bar
+foo
+

+------------------------------24e78000bd32--

+</protocol>
+</verify>
+</testcase>
diff --git a/tests/data/test39 b/tests/data/test39
index 8b00ada8c..81c7119bc 100644
--- a/tests/data/test39
+++ b/tests/data/test39
@@ -47,7 +47,7 @@ POST /we/want/%TESTNUMBER HTTP/1.1
 Host: %HOSTIP:%HTTPPORT

 User-Agent: curl/%VERSION

 Accept: */*

-Content-Length: 1184

+Content-Length: 1180

 Content-Type: multipart/form-data; boundary=----------------------------24e78000bd32

 

 ------------------------------24e78000bd32

@@ -85,7 +85,7 @@ bar
 foo
 

 ------------------------------24e78000bd32

-Content-Disposition: form-data; name="file3"; filename="f\\\\ak\\\\er,\\an\\d;.t\"xt"

+Content-Disposition: form-data; name="file3"; filename="f\\ak\\er,\an\d;.t%22xt"

 Content-Type: mo/foo

 

 foo bar
@@ -94,7 +94,7 @@ bar
 foo
 

 ------------------------------24e78000bd32

-Content-Disposition: form-data; name="file4"; filename="A\\AA\"\"\\\"ZZZ"

+Content-Disposition: form-data; name="file4"; filename="A\AA%22%22\%22ZZZ"

 Content-Type: text/plain

 

 foo bar