http: added options for allowing HTTP/0.9 responses

Added CURLOPT_HTTP09_ALLOWED and --http0.9 for this purpose.

For now, both the tool and library allow HTTP/0.9 by default.
docs/DEPRECATE.md lays out the plan for when to reverse that default: 6
months after the 7.64.0 release. The options are added already now so
that applications/scripts can start using them already now.

Fixes #2873
Closes #3383

8 files changed, 295 insertions, 51 deletions

diff --git a/docs/DEPRECATE.md b/docs/DEPRECATE.md
index bb3c05fe1..27bd22ff7 100644
--- a/docs/DEPRECATE.md
+++ b/docs/DEPRECATE.md
@@ -64,3 +64,18 @@ revert if need be.
 
 Remove all global-cache related code from curl around April 2019 (might be
 7.66.0).
+
+## HTTP/0.9
+
+Supporting this is non-obvious and might even come as a surprise to some
+users. Potentially even being a security risk in some cases.
+
+### State
+
+curl 7.64.0 introduces options to disable/enable support for this protocol
+version. The default remains supported for now.
+
+### Removal
+
+The support for HTTP/0.9 will be switched to disabled by default in 6 months,
+in the September 2019 release (possibly called curl 7.68.0).
diff --git a/docs/cmdline-opts/Makefile.inc b/docs/cmdline-opts/Makefile.inc
index 76fa5d45f..b99a142ee 100644
--- a/docs/cmdline-opts/Makefile.inc
+++ b/docs/cmdline-opts/Makefile.inc
@@ -1,53 +1,205 @@
 # Shared between Makefile.am and CMakeLists.txt
 
-DPAGES = abstract-unix-socket.d anyauth.d append.d basic.d cacert.d capath.d cert.d \
-  cert-status.d cert-type.d ciphers.d compressed.d compressed-ssh.d     \
-  config.d doh-url.d                                                    \
-  connect-timeout.d connect-to.d continue-at.d cookie.d cookie-jar.d    \
-  create-dirs.d crlf.d crlfile.d data-ascii.d data-binary.d data.d      \
-  data-raw.d data-urlencode.d delegation.d digest.d disable.d           \
-  disable-eprt.d disable-epsv.d dns-interface.d dns-ipv4-addr.d         \
-  dns-ipv6-addr.d dns-servers.d dump-header.d egd-file.d engine.d       \
-  expect100-timeout.d fail.d fail-early.d false-start.d                 \
-  form.d form-string.d ftp-account.d ftp-alternative-to-user.d          \
-  ftp-create-dirs.d ftp-method.d ftp-pasv.d ftp-port.d ftp-pret.d       \
-  ftp-skip-pasv-ip.d ftp-ssl-ccc.d ftp-ssl-ccc-mode.d ftp-ssl-control.d \
-  get.d globoff.d                                                       \
-  happy-eyeballs-timeout-ms.d                                           \
-  head.d header.d help.d hostpubmd5.d http1.0.d                         \
-  http1.1.d http2.d http2-prior-knowledge.d ignore-content-length.d     \
-  include.d insecure.d interface.d ipv4.d ipv6.d junk-session-cookies.d \
-  keepalive-time.d key.d key-type.d krb.d libcurl.d limit-rate.d        \
-  list-only.d local-port.d location.d location-trusted.d                \
-  login-options.d mail-auth.d mail-from.d mail-rcpt.d manual.d          \
-  max-filesize.d max-redirs.d max-time.d metalink.d negotiate.d netrc.d \
-  netrc-file.d netrc-optional.d next.d no-alpn.d no-buffer.d            \
-  no-keepalive.d no-npn.d noproxy.d no-sessionid.d ntlm.d ntlm-wb.d     \
-  oauth2-bearer.d output.d pass.d path-as-is.d pinnedpubkey.d post301.d \
-  post302.d post303.d preproxy.d progress-bar.d proto.d proto-default.d \
-  proto-redir.d proxy1.0.d proxy-anyauth.d proxy-basic.d proxy-cacert.d \
-  proxy-capath.d proxy-cert.d proxy-cert-type.d proxy-ciphers.d         \
-  proxy-crlfile.d proxy.d proxy-digest.d proxy-header.d                 \
-  proxy-insecure.d proxy-key.d proxy-key-type.d proxy-negotiate.d       \
-  proxy-ntlm.d proxy-pass.d proxy-service-name.d                        \
-  proxy-ssl-allow-beast.d proxy-tlsauthtype.d proxy-tlspassword.d       \
-  proxy-tlsuser.d proxy-tlsv1.d proxytunnel.d proxy-user.d pubkey.d     \
-  quote.d random-file.d range.d raw.d referer.d remote-header-name.d    \
-  remote-name-all.d remote-name.d remote-time.d request.d resolve.d     \
-  retry-connrefused.d retry.d retry-delay.d retry-max-time.d sasl-ir.d  \
-  service-name.d show-error.d silent.d socks4a.d socks4.d socks5.d      \
-  socks5-basic.d socks5-gssapi.d proxy-pinnedpubkey.d                   \
-  socks5-gssapi-nec.d socks5-gssapi-service.d socks5-hostname.d         \
-  speed-limit.d speed-time.d ssl-allow-beast.d ssl.d ssl-no-revoke.d    \
-  ssl-reqd.d sslv2.d sslv3.d stderr.d suppress-connect-headers.d        \
-  tcp-fastopen.d tcp-nodelay.d                                          \
-  telnet-option.d tftp-blksize.d tftp-no-options.d time-cond.d          \
-  tls-max.d                                                             \
-  tlsauthtype.d tlspassword.d tlsuser.d tlsv1.0.d tlsv1.1.d tlsv1.2.d   \
-  tlsv1.3.d tlsv1.d trace-ascii.d trace.d trace-time.d tr-encoding.d    \
-  unix-socket.d upload-file.d url.d use-ascii.d user-agent.d user.d     \
-  verbose.d version.d write-out.d xattr.d request-target.d              \
-  styled-output.d tls13-ciphers.d proxy-tls13-ciphers.d                 \
-  disallow-username-in-url.d haproxy-protocol.d
+DPAGES =					\
+  abstract-unix-socket.d			\
+  anyauth.d					\
+  append.d basic.d				\
+  cacert.d capath.d				\
+  cert-status.d					\
+  cert-type.d					\
+  cert.d					\
+  ciphers.d					\
+  compressed-ssh.d				\
+  compressed.d					\
+  config.d					\
+  connect-timeout.d				\
+  connect-to.d					\
+  continue-at.d					\
+  cookie-jar.d					\
+  cookie.d					\
+  create-dirs.d					\
+  crlf.d crlfile.d				\
+  data-ascii.d					\
+  data-binary.d					\
+  data-urlencode.d				\
+  data.d data-raw.d				\
+  delegation.d					\
+  digest.d					\
+  disable-eprt.d				\
+  disable-epsv.d				\
+  disable.d					\
+  disallow-username-in-url.d			\
+  dns-interface.d				\
+  dns-ipv4-addr.d				\
+  dns-ipv6-addr.d				\
+  dns-servers.d					\
+  doh-url.d					\
+  dump-header.d					\
+  egd-file.d					\
+  engine.d					\
+  expect100-timeout.d				\
+  fail-early.d					\
+  fail.d					\
+  false-start.d					\
+  form-string.d					\
+  form.d					\
+  ftp-account.d					\
+  ftp-alternative-to-user.d			\
+  ftp-create-dirs.d				\
+  ftp-method.d					\
+  ftp-pasv.d					\
+  ftp-port.d					\
+  ftp-pret.d					\
+  ftp-skip-pasv-ip.d				\
+  ftp-ssl-ccc-mode.d				\
+  ftp-ssl-ccc.d					\
+  ftp-ssl-control.d				\
+  get.d globoff.d				\
+  happy-eyeballs-timeout-ms.d			\
+  haproxy-protocol.d				\
+  head.d header.d				\
+  help.d					\
+  hostpubmd5.d					\
+  http0.9.d					\
+  http1.0.d					\
+  http1.1.d http2.d				\
+  http2-prior-knowledge.d			\
+  ignore-content-length.d			\
+  include.d					\
+  insecure.d					\
+  interface.d					\
+  ipv4.d ipv6.d					\
+  junk-session-cookies.d			\
+  keepalive-time.d				\
+  key.d key-type.d				\
+  krb.d libcurl.d				\
+  limit-rate.d					\
+  list-only.d					\
+  local-port.d					\
+  location-trusted.d				\
+  location.d					\
+  login-options.d				\
+  mail-auth.d					\
+  mail-from.d					\
+  mail-rcpt.d					\
+  manual.d					\
+  max-filesize.d				\
+  max-redirs.d					\
+  max-time.d					\
+  metalink.d					\
+  negotiate.d					\
+  netrc-file.d					\
+  netrc-optional.d				\
+  netrc.d					\
+  next.d no-alpn.d				\
+  no-buffer.d					\
+  no-keepalive.d				\
+  no-npn.d					\
+  no-sessionid.d				\
+  noproxy.d					\
+  ntlm.d ntlm-wb.d				\
+  oauth2-bearer.d				\
+  output.d pass.d				\
+  path-as-is.d					\
+  pinnedpubkey.d				\
+  post301.d					\
+  post302.d					\
+  post303.d					\
+  preproxy.d					\
+  progress-bar.d				\
+  proto-default.d				\
+  proto-redir.d					\
+  proto.d					\
+  proxy-anyauth.d				\
+  proxy-basic.d					\
+  proxy-cacert.d				\
+  proxy-capath.d				\
+  proxy-cert-type.d				\
+  proxy-cert.d					\
+  proxy-ciphers.d				\
+  proxy-crlfile.d				\
+  proxy-digest.d				\
+  proxy-header.d				\
+  proxy-insecure.d				\
+  proxy-key-type.d				\
+  proxy-key.d					\
+  proxy-negotiate.d				\
+  proxy-ntlm.d					\
+  proxy-pass.d					\
+  proxy-pinnedpubkey.d				\
+  proxy-service-name.d				\
+  proxy-ssl-allow-beast.d			\
+  proxy-tls13-ciphers.d				\
+  proxy-tlsauthtype.d				\
+  proxy-tlspassword.d				\
+  proxy-tlsuser.d				\
+  proxy-tlsv1.d					\
+  proxy-user.d					\
+  proxy.d					\
+  proxy1.0.d					\
+  proxytunnel.d					\
+  pubkey.d quote.d				\
+  random-file.d					\
+  range.d raw.d					\
+  referer.d					\
+  remote-header-name.d				\
+  remote-name-all.d				\
+  remote-name.d					\
+  remote-time.d					\
+  request-target.d				\
+  request.d					\
+  resolve.d					\
+  retry-connrefused.d				\
+  retry-delay.d					\
+  retry-max-time.d				\
+  retry.d					\
+  sasl-ir.d					\
+  service-name.d				\
+  show-error.d					\
+  silent.d					\
+  socks4.d socks5.d				\
+  socks4a.d					\
+  socks5-basic.d				\
+  socks5-gssapi-nec.d				\
+  socks5-gssapi-service.d			\
+  socks5-gssapi.d				\
+  socks5-hostname.d				\
+  speed-limit.d					\
+  speed-time.d					\
+  ssl-allow-beast.d				\
+  ssl-no-revoke.d				\
+  ssl-reqd.d					\
+  ssl.d						\
+  sslv2.d sslv3.d				\
+  stderr.d					\
+  styled-output.d				\
+  suppress-connect-headers.d			\
+  tcp-fastopen.d				\
+  tcp-nodelay.d					\
+  telnet-option.d				\
+  tftp-blksize.d				\
+  tftp-no-options.d				\
+  time-cond.d					\
+  tls-max.d					\
+  tls13-ciphers.d				\
+  tlsauthtype.d					\
+  tlspassword.d					\
+  tlsuser.d					\
+  tlsv1.0.d					\
+  tlsv1.1.d					\
+  tlsv1.2.d					\
+  tlsv1.3.d tlsv1.d				\
+  tr-encoding.d					\
+  trace-ascii.d					\
+  trace-time.d					\
+  trace.d					\
+  unix-socket.d					\
+  upload-file.d					\
+  url.d use-ascii.d				\
+  user-agent.d					\
+  user.d verbose.d				\
+  version.d					\
+  write-out.d					\
+  xattr.d
 
 OTHERPAGES = page-footer page-header
diff --git a/docs/cmdline-opts/http0.9.d b/docs/cmdline-opts/http0.9.d
new file mode 100644
index 000000000..33fe72d18
--- /dev/null
+++ b/docs/cmdline-opts/http0.9.d
@@ -0,0 +1,14 @@
+Long: http0.9
+Tags: Versions
+Protocols: HTTP
+Added:
+Help: Allow HTTP 0.9 responses
+---
+Tells curl to be fine with HTTP version 0.9 response.
+
+HTTP/0.9 is a completely headerless response and therefore you can also
+connect with this to non-HTTP servers and still get a response since curl will
+simply transparently downgrade - if allowed.
+
+A future curl version will deny continuing if the response isn't at least
+HTTP/1.0 unless this option is used.
diff --git a/docs/libcurl/curl_easy_setopt.3 b/docs/libcurl/curl_easy_setopt.3
index 1bec4c14d..6d63912d7 100644
--- a/docs/libcurl/curl_easy_setopt.3
+++ b/docs/libcurl/curl_easy_setopt.3
@@ -319,6 +319,8 @@ Do an HTTP GET request. See \fICURLOPT_HTTPGET(3)\fP
 Set the request target. \fICURLOPT_REQUEST_TARGET(3)\fP
 .IP CURLOPT_HTTP_VERSION
 HTTP version to use. \fICURLOPT_HTTP_VERSION(3)\fP
+.IP CURLOPT_HTTP09_ALLOWED
+Allow HTTP/0.9 responses. \fICURLOPT_HTTP09_ALLOWED(3)\fP
 .IP CURLOPT_IGNORE_CONTENT_LENGTH
 Ignore Content-Length. See \fICURLOPT_IGNORE_CONTENT_LENGTH(3)\fP
 .IP CURLOPT_HTTP_CONTENT_DECODING
diff --git a/docs/libcurl/opts/CURLOPT_HTTP09_ALLOWED.3 b/docs/libcurl/opts/CURLOPT_HTTP09_ALLOWED.3
new file mode 100644
index 000000000..3fa44993a
--- /dev/null
+++ b/docs/libcurl/opts/CURLOPT_HTTP09_ALLOWED.3
@@ -0,0 +1,58 @@
+.\" **************************************************************************
+.\" *                                  _   _ ____  _
+.\" *  Project                     ___| | | |  _ \| |
+.\" *                             / __| | | | |_) | |
+.\" *                            | (__| |_| |  _ <| |___
+.\" *                             \___|\___/|_| \_\_____|
+.\" *
+.\" * Copyright (C) 1998 - 2018, Daniel Stenberg, <daniel@haxx.se>, et al.
+.\" *
+.\" * This software is licensed as described in the file COPYING, which
+.\" * you should have received as part of this distribution. The terms
+.\" * are also available at https://curl.haxx.se/docs/copyright.html.
+.\" *
+.\" * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+.\" * copies of the Software, and permit persons to whom the Software is
+.\" * furnished to do so, under the terms of the COPYING file.
+.\" *
+.\" * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+.\" * KIND, either express or implied.
+.\" *
+.\" **************************************************************************
+.\"
+.TH CURLOPT_HTTP09_ALLOWED 3 "17 Dec 2018" "libcurl 7.64.0" "curl_easy_setopt options"
+.SH NAME
+CURLOPT_HTTP09 \- allow HTTP/0.9 response
+.SH SYNOPSIS
+#include <curl/curl.h>
+
+CURLcode curl_easy_setopt(CURL *handle, CURLOPT_HTTP09_ALLOWED, long allowed);
+.SH DESCRIPTION
+Pass the long argument \fIallowed\fP set to 1L to allow HTTP/0.9 responses.
+
+A HTTP/0.9 response is a server response entirely without headers and only a
+body, while you can connect to lots of random TCP services and still get a
+response that curl might consider to be HTTP/0.9.
+.SH DEFAULT
+curl allows HTTP/0.9 responses by default.
+
+A future curl version will require this option to be set to allow HTTP/0.9
+responses.
+.SH PROTOCOLS
+HTTP
+.SH EXAMPLE
+.nf
+CURL *curl = curl_easy_init();
+if(curl) {
+  CURLcode ret;
+  curl_easy_setopt(curl, CURLOPT_URL, "https://example.com/");
+  curl_easy_setopt(curl, CURLOPT_HTTP09_ALLOWED, 1L);
+  ret = curl_easy_perform(curl);
+}
+.fi
+.SH AVAILABILITY
+Option added in 7.64.0, present along with HTTP.
+.SH RETURN VALUE
+Returns CURLE_OK if HTTP is supported, and CURLE_UNKNOWN_OPTION if not.
+.SH "SEE ALSO"
+.BR CURLOPT_SSLVERSION "(3), " CURLOPT_HTTP_VERSION "(3), "
diff --git a/docs/libcurl/opts/CURLOPT_HTTP_VERSION.3 b/docs/libcurl/opts/CURLOPT_HTTP_VERSION.3
index 060db7578..7b7a08144 100644
--- a/docs/libcurl/opts/CURLOPT_HTTP_VERSION.3
+++ b/docs/libcurl/opts/CURLOPT_HTTP_VERSION.3
@@ -84,3 +84,4 @@ Along with HTTP
 Returns CURLE_OK if HTTP is supported, and CURLE_UNKNOWN_OPTION if not.
 .SH "SEE ALSO"
 .BR CURLOPT_SSLVERSION "(3), " CURLOPT_HTTP200ALIASES "(3), "
+.BR CURLOPT_HTTP09_ALLOWED "(3), "
diff --git a/docs/libcurl/opts/Makefile.inc b/docs/libcurl/opts/Makefile.inc
index 9bfd555f1..b21f32356 100644
--- a/docs/libcurl/opts/Makefile.inc
+++ b/docs/libcurl/opts/Makefile.inc
@@ -154,6 +154,7 @@ man_MANS =                                      \
   CURLOPT_HEADERDATA.3                          \
   CURLOPT_HEADERFUNCTION.3                      \
   CURLOPT_HEADEROPT.3                           \
+  CURLOPT_HTTP09_ALLOWED.3                      \
   CURLOPT_HTTP200ALIASES.3                      \
   CURLOPT_HTTPAUTH.3                            \
   CURLOPT_HTTPGET.3                             \
@@ -163,9 +164,9 @@ man_MANS =                                      \
   CURLOPT_HTTP_CONTENT_DECODING.3               \
   CURLOPT_HTTP_TRANSFER_DECODING.3              \
   CURLOPT_HTTP_VERSION.3                        \
-  CURLOPT_TRAILERFUNCTION.3                     \
-  CURLOPT_TRAILERDATA.3                         \
   CURLOPT_IGNORE_CONTENT_LENGTH.3               \
+  CURLOPT_TRAILERDATA.3                         \
+  CURLOPT_TRAILERFUNCTION.3                     \
   CURLOPT_INFILESIZE.3                          \
   CURLOPT_INFILESIZE_LARGE.3                    \
   CURLOPT_INTERFACE.3                           \
diff --git a/docs/libcurl/symbols-in-versions b/docs/libcurl/symbols-in-versions
index 8659346ce..f25009c2c 100644
--- a/docs/libcurl/symbols-in-versions
+++ b/docs/libcurl/symbols-in-versions
@@ -421,6 +421,7 @@ CURLOPT_HEADER                  7.1
 CURLOPT_HEADERDATA              7.10
 CURLOPT_HEADERFUNCTION          7.7.2
 CURLOPT_HEADEROPT               7.37.0
+CURLOPT_HTTP09_ALLOWED          7.64.0
 CURLOPT_HTTP200ALIASES          7.10.3
 CURLOPT_HTTPAUTH                7.10.6
 CURLOPT_HTTPGET                 7.8.1