HTTP: implement Brotli content encoding

This uses the brotli external library (https://github.com/google/brotli).
Brotli becomes a feature: additional curl_version_info() bit and
structure fields are provided for it and CURLVERSION_NOW bumped.

Tests 314 and 315 check Brotli content unencoding with correct and
erroneous data.

Some tests are updated to accomodate with the now configuration dependent
parameters of the Accept-Encoding header.

10 files changed, 328 insertions, 7 deletions

diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc
index 305f6f318..6d253afa0 100644
--- a/tests/data/Makefile.inc
+++ b/tests/data/Makefile.inc
@@ -54,7 +54,7 @@ test271 test272 test273 test274 test275 test276 test277 test278 test279 \
 test280 test281 test282 test283 test284 test285 test286 test287 test288 \
 test289 test290 test291 test292 test293 test294 test295 test296 test297 \
 test298 test299 test300 test301 test302 test303 test304 test305 test306 \
-test307 test308 test309 test310 test311 test312 test313                 \
+test307 test308 test309 test310 test311 test312 test313 test314 test315 \
                                 test320 test321 test322 test323 test324 \
 test325 \
 test350 test351 test352 test353 test354 \
diff --git a/tests/data/test220 b/tests/data/test220
index 2fb0b8a6a..7fd264345 100644
--- a/tests/data/test220
+++ b/tests/data/test220
@@ -57,11 +57,14 @@ http://%HOSTIP:%HTTPPORT/220 --compressed
 <strip>
 ^User-Agent:.*
 </strip>
+<strippart>
+s/^Accept-Encoding: .*/Accept-Encoding: xxx/
+</strippart>
 <protocol>
 GET /220 HTTP/1.1

 Host: %HOSTIP:%HTTPPORT

 Accept: */*

-Accept-Encoding: deflate, gzip

+Accept-Encoding: xxx
 

 </protocol>
 </verify>
diff --git a/tests/data/test221 b/tests/data/test221
index 95edb4990..3a85439d0 100644
--- a/tests/data/test221
+++ b/tests/data/test221
@@ -57,11 +57,14 @@ http://%HOSTIP:%HTTPPORT/221 --compressed
 <strip>
 ^User-Agent:.*
 </strip>
+<strippart>
+s/^Accept-Encoding: .*/Accept-Encoding: xxx/
+</strippart>
 <protocol>
 GET /221 HTTP/1.1

 Host: %HOSTIP:%HTTPPORT

 Accept: */*

-Accept-Encoding: deflate, gzip

+Accept-Encoding: xxx
 

 </protocol>
 <errorcode>
diff --git a/tests/data/test222 b/tests/data/test222
index a4594869e..865266e07 100644
--- a/tests/data/test222
+++ b/tests/data/test222
@@ -188,11 +188,14 @@ http://%HOSTIP:%HTTPPORT/222 --compressed
 <strip>
 ^User-Agent:.*
 </strip>
+<strippart>
+s/^Accept-Encoding: .*/Accept-Encoding: xxx/
+</strippart>
 <protocol>
 GET /222 HTTP/1.1

 Host: %HOSTIP:%HTTPPORT

 Accept: */*

-Accept-Encoding: deflate, gzip

+Accept-Encoding: xxx
 

 </protocol>
 </verify>
diff --git a/tests/data/test223 b/tests/data/test223
index 196e78c80..884967e3f 100644
--- a/tests/data/test223
+++ b/tests/data/test223
@@ -78,11 +78,14 @@ http://%HOSTIP:%HTTPPORT/223 --compressed
 <strip>
 ^User-Agent:.*
 </strip>
+<strippart>
+s/^Accept-Encoding: .*/Accept-Encoding: xxx/
+</strippart>
 <protocol>
 GET /223 HTTP/1.1

 Host: %HOSTIP:%HTTPPORT

 Accept: */*

-Accept-Encoding: deflate, gzip

+Accept-Encoding: xxx
 

 </protocol>
 <errorcode>
diff --git a/tests/data/test224 b/tests/data/test224
index 1c8ad2380..a56046873 100644
--- a/tests/data/test224
+++ b/tests/data/test224
@@ -93,11 +93,14 @@ http://%HOSTIP:%HTTPPORT/224 --compressed
 <strip>
 ^User-Agent:.*
 </strip>
+<strippart>
+s/^Accept-Encoding: .*/Accept-Encoding: xxx/
+</strippart>
 <protocol>
 GET /224 HTTP/1.1

 Host: %HOSTIP:%HTTPPORT

 Accept: */*

-Accept-Encoding: deflate, gzip

+Accept-Encoding: xxx
 

 </protocol>
 </verify>
diff --git a/tests/data/test230 b/tests/data/test230
index cc166a3f1..2174434b3 100644
--- a/tests/data/test230
+++ b/tests/data/test230
@@ -189,11 +189,14 @@ http://%HOSTIP:%HTTPPORT/230 --compressed
 <strip>
 ^User-Agent:.*
 </strip>
+<strippart>
+s/^Accept-Encoding: .*/Accept-Encoding: xxx/
+</strippart>
 <protocol>
 GET /230 HTTP/1.1

 Host: %HOSTIP:%HTTPPORT

 Accept: */*

-Accept-Encoding: deflate, gzip

+Accept-Encoding: xxx
 

 </protocol>
 </verify>
diff --git a/tests/data/test314 b/tests/data/test314
new file mode 100644
index 000000000..f4703cdeb
--- /dev/null
+++ b/tests/data/test314
@@ -0,0 +1,198 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+compressed
+</keywords>
+</info>
+#
+# Server-side
+<reply>
+<data base64="yes">
+SFRUUC8xLjEgMjAwIE9LDQpEYXRlOiBNb24sIDI5IE5vdiAyMDA0IDIxOjU2OjUzIEdNVA0KU2Vy
+dmVyOiBBcGFjaGUvMS4zLjMxIChEZWJpYW4gR05VL0xpbnV4KSBtb2RfZ3ppcC8xLjMuMjYuMWEg
+UEhQLzQuMy45LTEgbW9kX3NzbC8yLjguMjAgT3BlblNTTC8wLjkuN2QgbW9kX3BlcmwvMS4yOQ0K
+VmFyeTogQWNjZXB0LUVuY29kaW5nDQpDb250ZW50LVR5cGU6IHRleHQvaHRtbDsgY2hhcnNldD1J
+U08tODg1OS0xDQpDb250ZW50LUVuY29kaW5nOiBicg0KQ29udGVudC1MZW5ndGg6IDEwNTYNCg0K
+G7ATAJwFdhtdgaQ8i+mZBoO/lwogPKuqHpeP38jV5TDITTB7/oJVCS69FFDKWDVtMk8y4SfMSu/a
+9vvLxWPweDCKePH/2y9VIkbF+EgCYSNs9v53J8QTIHT4ZucHCCRQiXRdT6XdE60KSlbIvobr5rJQ
+sRn7ipIjMVMq3Go+/UXtY2d0yP1qaaGSxCn8nZuUNGh74KOI7EEkgFl1tjYytkpc9mJJy9J+wTTI
++HroUQP2VR2DYkNoUECqgtOLlGcVEln4+eVzEWcrb8fNrcrVxLArJBpSd8FX8eZs8ebJUO7aBZ5e
+pHz6zel7lhLlfHoQIkGh34riaSVr7VTGDmmO6HjSCzKO27LybZ9I3CtMSD2Il4mB131Tlcbut1Bd
+zL4XU4DZYMLBN4jwVZEoHpjzHX+vQ3prnrNw4oB7OWOr/fBzjvfjDuO24WxwzPqPo+V6VNcthz1p
+fF1+sMK4yWY7He33m32EuQgQFSZ3a5Wu4FyQcAb45Z+wUxM5XCmX52YmdUR2YTs+W+bNw2EZSfMR
+cP3CinyJI/cTT+JubL3T4COkhz0Rffeoh/3E4c/6ugma1ubhokYecXp8HBwmeDL48d62H26u69DO
+yMhg1PFj+oVDWnK4K+L5AlRr0mpJLqoGHrzflMLQ6qL2oIo9hN6qCeZEEqXM+/KunVYpWVeTY+ht
+hA0y5p5RLLTTS4cehaJOpbFyAVxZOardIkJAVx0NshOZY4hDbts9BXsXzFEOgsFhrIQYgh04StZz
+llIRMVDptYlwGmpZCHHmVECdGiFIfEhkQ2INSwMCuuKpaycgSOO9hJA9UFKDBdzTiLJBP9oUVkKL
+bHjwicICCi3k0HcppcvQaW27AMI06kuQU4WUGizgnkaUDcZqCgsotMgG528UFlBo8SFpb05OAjJq
+2gEI0UgN93KS1OvAOYSLN5IaLOCeRnQpJXuLUwcm7urpg6lYxAk26uEoADdsRytHGkSWjOKP6T07
+wiceuNo7CXyu7ohtUZXoEWawRHGVkPDVJYqH+xa0DDRKSSgM4K3efLVPSTaUPvBGIZgnn2JBFFWa
+MsKZguUuUnz6qaSGqnmGAYiupdC1EFye58V4CLbWVjJU4NF2jrOUYR/Dv04zYwVQtQcFzgmK6H4N
+HAhmb0a6pQRKxZaZ+x2vCC7sCuIu4dNCATwqzk12ue6oEsxzYybLPNGJd084M43O9W8E+5/drd/F
+QVB2X4jlFlCuHuWeQxQo+w73Tb9swW692v3BlfQTP1ClWzuJ+RwuSb9m4V3QVa4MEL+0Xzc5FX9P
++YX1cgaL+6oMHw7L+IOjOt+n1BOloyqk35lLHX7RZmu8SckMnGP95XjWc4FRKP9x/iXrKaeCnut/
+zstyZdJS5FRmBT/wb5KK9YWBGnqPLO8isN2HS8gA
+</data>
+
+<datacheck>
+HTTP/1.1 200 OK

+Date: Mon, 29 Nov 2004 21:56:53 GMT

+Server: Apache/1.3.31 (Debian GNU/Linux) mod_gzip/1.3.26.1a PHP/4.3.9-1 mod_ssl/2.8.20 OpenSSL/0.9.7d mod_perl/1.29

+Vary: Accept-Encoding

+Content-Type: text/html; charset=ISO-8859-1

+Content-Encoding: br

+Content-Length: 1056

+

+<?xml version="1.0" encoding="ISO-8859-1"?>
+<!DOCTYPE project-listing SYSTEM "http://freshmeat.net/backend/fm-projects-0.4.dtd">
+<project-listing>
+  <project>
+    <project_id>1612</project_id>
+    <date_added>1998-08-21 04:01:29</date_added>
+    <date_updated>2004-10-18 02:22:23</date_updated>
+    <projectname_short>curl</projectname_short>
+    <projectname_full>curl and libcurl</projectname_full>
+    <desc_short>Command line tool and library for client-side URL transfers.</desc_short>
+    <desc_full>curl and libcurl is a tool for transferring files

+using URL syntax. It supports HTTP, HTTPS, FTP,

+FTPS, DICT, TELNET, LDAP, FILE, and GOPHER, as

+well as HTTP-post, HTTP-put, cookies, FTP upload,

+resumed transfers, passwords, portnumbers, SSL

+certificates, Kerberos, and proxies. It is powered

+by libcurl, the client-side URL transfer library.

+There are bindings to libcurl for over 20

+languages and environments.

+</desc_full>
+    <vitality_score>5784.57</vitality_score>
+    <vitality_percent>3.16</vitality_percent>
+    <vitality_rank>169</vitality_rank>
+    <popularity_score>6594.54</popularity_score>
+    <popularity_percent>13.81</popularity_percent>
+    <popularity_rank>105</popularity_rank>
+    <rating>8.50</rating>
+    <rating_count>21</rating_count>
+    <rating_rank>183</rating_rank>
+    <subscriptions>323</subscriptions>
+    <branch_name>Default</branch_name>
+    <url_project_page>http://freshmeat.net/projects/curl/</url_project_page>
+    <url_homepage>http://freshmeat.net/redir/curl/1612/url_homepage/</url_homepage>
+    <url_tgz>http://freshmeat.net/redir/curl/1612/url_tgz/</url_tgz>
+    <url_bz2>http://freshmeat.net/redir/curl/1612/url_bz2/</url_bz2>
+    <url_zip>http://freshmeat.net/redir/curl/1612/url_zip/</url_zip>
+    <url_changelog>http://freshmeat.net/redir/curl/1612/url_changelog/</url_changelog>
+    <url_rpm>http://freshmeat.net/redir/curl/1612/url_rpm/</url_rpm>
+    <url_deb>http://freshmeat.net/redir/curl/1612/url_deb/</url_deb>
+    <url_osx>http://freshmeat.net/redir/curl/1612/url_osx/</url_osx>
+    <url_bsdport>http://freshmeat.net/redir/curl/1612/url_bsdport/</url_bsdport>
+    <url_purchase></url_purchase>
+    <url_cvs>http://freshmeat.net/redir/curl/1612/url_cvs/</url_cvs>
+    <url_list>http://freshmeat.net/redir/curl/1612/url_list/</url_list>
+    <url_mirror>http://freshmeat.net/redir/curl/1612/url_mirror/</url_mirror>
+    <url_demo></url_demo>
+    <license>MIT/X Consortium License</license>
+    <latest_release>
+      <latest_release_version>7.12.2</latest_release_version>
+      <latest_release_id>176085</latest_release_id>
+      <latest_release_date>2004-10-18 02:22:23</latest_release_date>
+    </latest_release>
+    <screenshot_thumb></screenshot_thumb>
+    <authors>
+      <author>
+        <author_name>Daniel Stenberg</author_name>
+        <author_url>http://freshmeat.net/~bagder/</author_url>
+        <author_role>Owner</author_role>
+      </author>
+    </authors>
+    <descriminators>
+      <trove_id>12</trove_id>
+      <trove_id>226</trove_id>
+      <trove_id>3</trove_id>
+      <trove_id>2</trove_id>
+      <trove_id>188</trove_id>
+      <trove_id>216</trove_id>
+      <trove_id>200</trove_id>
+      <trove_id>220</trove_id>
+      <trove_id>164</trove_id>
+      <trove_id>90</trove_id>
+      <trove_id>89</trove_id>
+      <trove_id>809</trove_id>
+      <trove_id>150</trove_id>
+      <trove_id>224</trove_id>
+      <trove_id>900</trove_id>
+      <trove_id>839</trove_id>
+    </descriminators>
+    <dependencies>
+      <dependency type="recommended">
+        <dependency_release_id>0</dependency_release_id>
+        <dependency_branch_id>7464</dependency_branch_id>
+        <dependency_project_id>7464</dependency_project_id>
+        <dependency_project_title>OpenSSL (Default)</dependency_project_title>
+      </dependency>
+      <dependency type="optional">
+        <dependency_release_id>0</dependency_release_id>
+        <dependency_branch_id>0</dependency_branch_id>
+        <dependency_project_id>7443</dependency_project_id>
+        <dependency_project_title>OpenLDAP</dependency_project_title>
+      </dependency>
+      <dependency type="optional">
+        <dependency_release_id>0</dependency_release_id>
+        <dependency_branch_id>0</dependency_branch_id>
+        <dependency_project_id>12351</dependency_project_id>
+        <dependency_project_title>zlib</dependency_project_title>
+      </dependency>
+      <dependency type="optional">
+        <dependency_release_id>0</dependency_release_id>
+        <dependency_branch_id>0</dependency_branch_id>
+        <dependency_project_id>32047</dependency_project_id>
+        <dependency_project_title>Heimdal</dependency_project_title>
+      </dependency>
+      <dependency type="optional">
+        <dependency_release_id>0</dependency_release_id>
+        <dependency_branch_id>0</dependency_branch_id>
+        <dependency_project_id>44532</dependency_project_id>
+        <dependency_project_title>c-ares</dependency_project_title>
+      </dependency>
+    </dependencies>
+  </project>
+</project-listing>
+</datacheck>
+
+</reply>
+
+#
+# Client-side
+<client>
+<features>
+brotli
+</features>
+<server>
+http
+</server>
+ <name>
+HTTP GET brotli compressed content
+ </name>
+ <command>
+http://%HOSTIP:%HTTPPORT/314 --compressed
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<strip>
+^User-Agent:.*
+</strip>
+<strippart>
+s/^Accept-Encoding: .*/Accept-Encoding: xxx/
+</strippart>
+<protocol>
+GET /314 HTTP/1.1

+Host: %HOSTIP:%HTTPPORT

+Accept: */*

+Accept-Encoding: xxx
+

+</protocol>
+</verify>
+</testcase>
diff --git a/tests/data/test315 b/tests/data/test315
new file mode 100644
index 000000000..c75d9ae7b
--- /dev/null
+++ b/tests/data/test315
@@ -0,0 +1,91 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+compressed
+FAILURE
+</keywords>
+</info>
+#
+# Server-side
+<reply>
+# this brotli chunk has three bytes removed from the beginning
+<data base64="yes">
+SFRUUC8xLjEgMjAwIE9LDQpEYXRlOiBNb24sIDI5IE5vdiAyMDA0IDIxOjU2OjUzIEdNVA0KU2Vy
+dmVyOiBBcGFjaGUvMS4zLjMxIChEZWJpYW4gR05VL0xpbnV4KSBtb2RfZ3ppcC8xLjMuMjYuMWEg
+UEhQLzQuMy45LTEgbW9kX3NzbC8yLjguMjAgT3BlblNTTC8wLjkuN2QgbW9kX3BlcmwvMS4yOQ0K
+VmFyeTogQWNjZXB0LUVuY29kaW5nDQpDb250ZW50LVR5cGU6IHRleHQvaHRtbDsgY2hhcnNldD1J
+U08tODg1OS0xDQpDb250ZW50LUVuY29kaW5nOiBicg0KQ29udGVudC1MZW5ndGg6IDEwNTYNCg0K
+AJwFdhtdgaQ8i+mZBoO/lwogPKuqHpeP38jV5TDITTB7/oJVCS69FFDKWDVtMk8y4SfMSu/a9vvL
+xWPweDCKePH/2y9VIkbF+EgCYSNs9v53J8QTIHT4ZucHCCRQiXRdT6XdE60KSlbIvobr5rJQsRn7
+ipIjMVMq3Go+/UXtY2d0yP1qaaGSxCn8nZuUNGh74KOI7EEkgFl1tjYytkpc9mJJy9J+wTTI+Hro
+UQP2VR2DYkNoUECqgtOLlGcVEln4+eVzEWcrb8fNrcrVxLArJBpSd8FX8eZs8ebJUO7aBZ5epHz6
+zel7lhLlfHoQIkGh34riaSVr7VTGDmmO6HjSCzKO27LybZ9I3CtMSD2Il4mB131Tlcbut1BdzL4X
+U4DZYMLBN4jwVZEoHpjzHX+vQ3prnrNw4oB7OWOr/fBzjvfjDuO24WxwzPqPo+V6VNcthz1pfF1+
+sMK4yWY7He33m32EuQgQFSZ3a5Wu4FyQcAb45Z+wUxM5XCmX52YmdUR2YTs+W+bNw2EZSfMRcP3C
+inyJI/cTT+JubL3T4COkhz0Rffeoh/3E4c/6ugma1ubhokYecXp8HBwmeDL48d62H26u69DOyMhg
+1PFj+oVDWnK4K+L5AlRr0mpJLqoGHrzflMLQ6qL2oIo9hN6qCeZEEqXM+/KunVYpWVeTY+hthA0y
+5p5RLLTTS4cehaJOpbFyAVxZOardIkJAVx0NshOZY4hDbts9BXsXzFEOgsFhrIQYgh04StZzllIR
+MVDptYlwGmpZCHHmVECdGiFIfEhkQ2INSwMCuuKpaycgSOO9hJA9UFKDBdzTiLJBP9oUVkKLbHjw
+icICCi3k0HcppcvQaW27AMI06kuQU4WUGizgnkaUDcZqCgsotMgG528UFlBo8SFpb05OAjJq2gEI
+0UgN93KS1OvAOYSLN5IaLOCeRnQpJXuLUwcm7urpg6lYxAk26uEoADdsRytHGkSWjOKP6T07wice
+uNo7CXyu7ohtUZXoEWawRHGVkPDVJYqH+xa0DDRKSSgM4K3efLVPSTaUPvBGIZgnn2JBFFWaMsKZ
+guUuUnz6qaSGqnmGAYiupdC1EFye58V4CLbWVjJU4NF2jrOUYR/Dv04zYwVQtQcFzgmK6H4NHAhm
+b0a6pQRKxZaZ+x2vCC7sCuIu4dNCATwqzk12ue6oEsxzYybLPNGJd084M43O9W8E+5/drd/FQVB2
+X4jlFlCuHuWeQxQo+w73Tb9swW692v3BlfQTP1ClWzuJ+RwuSb9m4V3QVa4MEL+0Xzc5FX9P+YX1
+cgaL+6oMHw7L+IOjOt+n1BOloyqk35lLHX7RZmu8SckMnGP95XjWc4FRKP9x/iXrKaeCnut/zsty
+ZdJS5FRmBT/wb5KK9YWBGnqPLO8isN2HS8gA
+</data>
+
+<datacheck>
+HTTP/1.1 200 OK

+Date: Mon, 29 Nov 2004 21:56:53 GMT

+Server: Apache/1.3.31 (Debian GNU/Linux) mod_gzip/1.3.26.1a PHP/4.3.9-1 mod_ssl/2.8.20 OpenSSL/0.9.7d mod_perl/1.29

+Vary: Accept-Encoding

+Content-Type: text/html; charset=ISO-8859-1

+Content-Encoding: br

+Content-Length: 1056

+

+</datacheck>
+
+</reply>
+
+#
+# Client-side
+<client>
+<features>
+brotli
+</features>
+<server>
+http
+</server>
+ <name>
+HTTP GET brotli compressed content with broken header
+ </name>
+ <command>
+http://%HOSTIP:%HTTPPORT/315 --compressed
+</command>
+</client>
+
+#
+# Verify data after the test has been "shot"
+<verify>
+<strip>
+^User-Agent:.*
+</strip>
+<strippart>
+s/^Accept-Encoding: .*/Accept-Encoding: xxx/
+</strippart>
+<protocol>
+GET /315 HTTP/1.1

+Host: %HOSTIP:%HTTPPORT

+Accept: */*

+Accept-Encoding: xxx
+

+</protocol>
+<errorcode>
+61
+</errorcode>
+</verify>
+</testcase>
diff --git a/tests/runtests.pl b/tests/runtests.pl
index 7f6f81291..dcd9f8419 100755
--- a/tests/runtests.pl
+++ b/tests/runtests.pl
@@ -217,6 +217,7 @@ my $gopher_ipv6;    # set if Gopher server has IPv6 support
 my $has_ipv6;       # set if libcurl is built with IPv6 support
 my $has_unix;       # set if libcurl is built with Unix sockets support
 my $has_libz;       # set if libcurl is built with libz support
+my $has_brotli;     # set if libcurl is built with brotli support
 my $has_getrlimit;  # set if system has getrlimit()
 my $has_ntlm;       # set if libcurl is built with NTLM support
 my $has_ntlm_wb;    # set if libcurl is built with NTLM delegation to winbind
@@ -2880,6 +2881,9 @@ sub checksystem {
             if($feat =~ /libz/i) {
                 $has_libz = 1;
             }
+            if($feat =~ /brotli/i) {
+                $has_brotli = 1;
+            }
             if($feat =~ /NTLM/i) {
                 # NTLM enabled
                 $has_ntlm=1;
@@ -3396,6 +3400,11 @@ sub singletest {
                     next;
                 }
             }
+            elsif($1 eq "brotli") {
+                if($has_brotli) {
+                    next;
+                }
+            }
             elsif($1 eq "NTLM") {
                 if($has_ntlm) {
                     next;
@@ -3552,6 +3561,11 @@ sub singletest {
                         next;
                     }
                 }
+                elsif($1 eq "brotli") {
+                    if(!$has_brotli) {
+                        next;
+                    }
+                }
                 elsif($1 eq "NTLM") {
                     if(!$has_ntlm) {
                         next;