CURLOPT_FOLLOWLOCATION: add a CURLFOLLOW_NO_CUSTOMREQUEST bitbagder/FOLLOW_IGNORE_CUSTOM

With this change, the argument passed to the CURLOPT_FOLLOWLOCATION
option is treated as a bitmask instead of just a long. If the new
CURLFOLLOW_NO_CUSTOMREQUEST bit is set in the bitmask, it means that
libcurl will NOT allow a custom method override the HTTP request method
after a redirect is followed. As is otherwise the default behavior (that
surprises many users).

This change is forward compatible because CURLOPT_FOLLOWLOCATION has
been documented to accept the exact value of '1' to enable redirect
following and therefore the other bits were left unused and
undefined. We now add value to another bit. Starting in 7.66.0, the
value 1 and the first bit still enables plain redirect following but the
second bit adds more meaning.

This change is backward compatible in the following way: setting the
CURLFOLLOW_NO_CUSTOMREQUEST bit in a program that still uses an older
libcurl installation at run-tim will have no effect. This is because
older libcurl code checked if the value was non-zero and then enabled
redirect following. Of course older libcurl will always let the set
CURLOPT_CUSTOMREQUEST string override the method, disregarding what the
HTTP response code suggests.

Test 1589 added to verify the functionality.

11 files changed, 199 insertions, 9 deletions

diff --git a/docs/libcurl/opts/CURLOPT_CUSTOMREQUEST.3 b/docs/libcurl/opts/CURLOPT_CUSTOMREQUEST.3
index 93ccaf8dd..8d290e191 100644
--- a/docs/libcurl/opts/CURLOPT_CUSTOMREQUEST.3
+++ b/docs/libcurl/opts/CURLOPT_CUSTOMREQUEST.3
@@ -5,7 +5,7 @@
 .\" *                            | (__| |_| |  _ <| |___
 .\" *                             \___|\___/|_| \_\_____|
 .\" *
-.\" * Copyright (C) 1998 - 2017, Daniel Stenberg, <daniel@haxx.se>, et al.
+.\" * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
 .\" *
 .\" * This software is licensed as described in the file COPYING, which
 .\" * you should have received as part of this distribution. The terms
@@ -58,6 +58,12 @@ possibly confuse the remote server badly. Use \fICURLOPT_POST(3)\fP and
 to replace or extend the set of headers sent by libcurl. Use
 \fICURLOPT_HTTP_VERSION(3)\fP to change HTTP version.
 
+If this option used together with \fICURLOPT_FOLLOWLOCATION(3)\fP, the custom
+set method will by default override the method libcurl could otherwise change
+to. Starting in libcurl 7.66.0, you can fine-tune that decision by using the
+\fICURLFOLLOW_IGNORE_CUSTOM\fP bit to \fICURLOPT_FOLLOWLOCATION(3)\fP to make
+redirects ignore the custom method.
+
 .IP FTP
 Instead of LIST and NLST when performing FTP directory listings.
 .IP IMAP
diff --git a/docs/libcurl/opts/CURLOPT_FOLLOWLOCATION.3 b/docs/libcurl/opts/CURLOPT_FOLLOWLOCATION.3
index f8d2b1889..54ef98db2 100644
--- a/docs/libcurl/opts/CURLOPT_FOLLOWLOCATION.3
+++ b/docs/libcurl/opts/CURLOPT_FOLLOWLOCATION.3
@@ -5,7 +5,7 @@
 .\" *                            | (__| |_| |  _ <| |___
 .\" *                             \___|\___/|_| \_\_____|
 .\" *
-.\" * Copyright (C) 1998 - 2015, Daniel Stenberg, <daniel@haxx.se>, et al.
+.\" * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
 .\" *
 .\" * This software is licensed as described in the file COPYING, which
 .\" * you should have received as part of this distribution. The terms
@@ -28,9 +28,15 @@ CURLOPT_FOLLOWLOCATION \- follow HTTP 3xx redirects
 
 CURLcode curl_easy_setopt(CURL *handle, CURLOPT_FOLLOWLOCATION, long enable);
 .SH DESCRIPTION
-A long parameter set to 1 tells the library to follow any Location: header
-that the server sends as part of an HTTP header in a 3xx response. The
-Location: header can specify a relative or an absolute URL to follow.
+The long parameter \fIenable\fP set to 1 tells the library to follow any
+Location: header that the server sends as part of an HTTP header in a 3xx
+response. The Location: header can specify a relative or an absolute URL to
+follow.
+
+\fIenable\fP is a bitmask. If you set the \fICURLFOLLOW_NO_CUSTOMREQUEST\fP
+bit, it will tell libcurl that the method set with
+\fICURLOPT_CUSTOMREQUEST(3)\fP will not be used after a redirect if the HTTP
+response says so.
 
 libcurl will issue another request for the new URL and follow new Location:
 headers all the way until no more such headers are returned.
@@ -72,6 +78,8 @@ if(curl) {
 .fi
 .SH AVAILABILITY
 Along with HTTP
+
+CURLFOLLOW_NO_CUSTOMREQEUST was added in 7.66.0
 .SH RETURN VALUE
 Returns CURLE_OK if HTTP is supported, and CURLE_UNKNOWN_OPTION if not.
 .SH "SEE ALSO"
diff --git a/docs/libcurl/symbols-in-versions b/docs/libcurl/symbols-in-versions
index 5244a7cdb..0c5864752 100644
--- a/docs/libcurl/symbols-in-versions
+++ b/docs/libcurl/symbols-in-versions
@@ -168,6 +168,8 @@ CURLFINFOFLAG_KNOWN_PERM        7.21.0
 CURLFINFOFLAG_KNOWN_SIZE        7.21.0
 CURLFINFOFLAG_KNOWN_TIME        7.21.0
 CURLFINFOFLAG_KNOWN_UID         7.21.0
+CURLFOLLOW_ENABLE               7.66.0
+CURLFOLLOW_NO_CUSTOMREQUEST     7.66.0
 CURLFORM_ARRAY                  7.9.1         7.56.0
 CURLFORM_ARRAY_END              7.9.1         7.9.5       7.9.6
 CURLFORM_ARRAY_START            7.9.1         7.9.5       7.9.6
diff --git a/include/curl/curl.h b/include/curl/curl.h
index 20f0d7155..db17f8683 100644
--- a/include/curl/curl.h
+++ b/include/curl/curl.h
@@ -165,6 +165,13 @@ typedef enum {
 #define CURLSSLBACKEND_CYASSL CURLSSLBACKEND_WOLFSSL
 #define CURLSSLBACKEND_DARWINSSL CURLSSLBACKEND_SECURETRANSPORT
 
+/* bits for the CURLOPT_FOLLOWLOCATION option */
+#define CURLFOLLOW_ENABLE (1<<0) /* generic follow redirects */
+
+/* Follow redirects, and don't use the custom method for anything but the
+   initial request if the response code says so. */
+#define CURLFOLLOW_NO_CUSTOMREQUEST (1<<1)
+
 struct curl_httppost {
   struct curl_httppost *next;       /* next entry in the list */
   char *name;                       /* pointer to allocated name */
diff --git a/lib/http.c b/lib/http.c
index 9fbd7201e..dfe57ad45 100644
--- a/lib/http.c
+++ b/lib/http.c
@@ -2050,8 +2050,10 @@ CURLcode Curl_http(struct connectdata *conn, bool *done)
     httpreq = HTTPREQ_PUT;
   }
 
-  /* Now set the 'request' pointer to the proper request string */
-  if(data->set.str[STRING_CUSTOMREQUEST])
+  /* Now set the 'request' pointer to the proper request string if
+     it isn't a redirect with redirect_clears_method set */
+  if(data->set.str[STRING_CUSTOMREQUEST] &&
+     (!data->state.this_is_a_follow || !data->set.redirect_clears_method))
     request = data->set.str[STRING_CUSTOMREQUEST];
   else {
     if(data->set.opt_no_body)
diff --git a/lib/setopt.c b/lib/setopt.c
index bdfe86ac7..2347941cf 100644
--- a/lib/setopt.c
+++ b/lib/setopt.c
@@ -435,8 +435,15 @@ CURLcode Curl_vsetopt(struct Curl_easy *data, CURLoption option, va_list param)
     /*
      * Follow Location: header hints on a HTTP-server.
      */
-    data->set.http_follow_location = (0 != va_arg(param, long)) ? TRUE : FALSE;
-    break;
+  {
+    bool clear;
+    uarg = va_arg(param, unsigned long);
+    data->set.http_follow_location =
+      (uarg & (CURLFOLLOW_ENABLE|CURLFOLLOW_NO_CUSTOMREQUEST)) ? TRUE : FALSE;
+    clear = (uarg & CURLFOLLOW_NO_CUSTOMREQUEST) ? TRUE : FALSE;
+    data->set.redirect_clears_method = clear;
+  }
+  break;
 
   case CURLOPT_UNRESTRICTED_AUTH:
     /*
diff --git a/lib/urldata.h b/lib/urldata.h
index fdc185b22..e1926bc72 100644
--- a/lib/urldata.h
+++ b/lib/urldata.h
@@ -1703,6 +1703,7 @@ struct UserDefined {
   bit http_fail_on_error:1;  /* fail on HTTP error codes >= 400 */
   bit http_keep_sending_on_error:1; /* for HTTP status codes >= 300 */
   bit http_follow_location:1; /* follow HTTP redirects */
+  bit redirect_clears_method:1; /* CUSTOMREQUEST is only for first request */
   bit http_transfer_encoding:1; /* request compressed HTTP
                                     transfer-encoding */
   bit allow_auth_to_other_hosts:1;
diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc
index 6ec5a3c18..710e6b78d 100644
--- a/tests/data/Makefile.inc
+++ b/tests/data/Makefile.inc
@@ -177,6 +177,7 @@ test1540 test1541 \
 test1550 test1551 test1552 test1553 test1554 test1555 test1556 test1557 \
 test1558 test1559 test1560 test1561 test1562 test1563 \
 \
+                                                               test1589 \
 test1590 test1591 test1592 test1593 \
 \
 test1600 test1601 test1602 test1603 test1604 test1605 test1606 test1607 \
diff --git a/tests/data/test1589 b/tests/data/test1589
new file mode 100644
index 000000000..6fd4b7b81
--- /dev/null
+++ b/tests/data/test1589
@@ -0,0 +1,95 @@
+<testcase>
+<info>
+<keywords>
+CURLFOLLOW_NO_CUSTOMREQUEST
+CURLOPT_FOLLOWLOCATION
+</keywords>
+</info>
+#
+# Server-side
+<reply>
+<data>
+HTTP/1.1 302 OK
+Date: Thu, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Location: 15890001
+Content-Length: 6
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+
+-foo-
+</data>
+<data1>
+HTTP/1.1 200 OK
+Content-Length: 6
+Connection: close
+Content-Type: text/html
+
+-bar-
+</data>
+<datacheck>
+HTTP/1.1 302 OK
+Date: Thu, 09 Nov 2010 14:49:00 GMT
+Server: test-server/fake
+Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT
+ETag: "21025-dc7-39462498"
+Accept-Ranges: bytes
+Location: 15890001
+Content-Length: 6
+Connection: close
+Content-Type: text/html
+Funny-head: yesyes
+
+HTTP/1.1 200 OK
+Content-Length: 6
+Connection: close
+Content-Type: text/html
+
+-bar-
+</datacheck>
+</reply>
+
+# Client-side
+<client>
+<server>
+http
+</server>
+<features>
+http
+</features>
+<tool>
+lib1589
+</tool>
+
+ <name>
+HTTP CURLFOLLOW_NO_CUSTOMREQUEST with custom method
+ </name>
+ <command>
+http://%HOSTIP:%HTTPPORT/1589
+</command>
+</client>
+
+# Verify data after the test has been "shot"
+<verify>
+<strip>
+^User-Agent:.*
+</strip>
+<protocol nonewline="yes">
+IGLOO /1589 HTTP/1.1

+Host: %HOSTIP:%HTTPPORT

+Accept: */*

+Content-Length: 3

+Content-Type: application/x-www-form-urlencoded

+

+mooGET /15890001 HTTP/1.1

+Host: %HOSTIP:%HTTPPORT

+Accept: */*

+

+
+</protocol>
+</verify>
+</testcase>
diff --git a/tests/libtest/Makefile.inc b/tests/libtest/Makefile.inc
index f5effd97d..b24f018f0 100644
--- a/tests/libtest/Makefile.inc
+++ b/tests/libtest/Makefile.inc
@@ -32,6 +32,7 @@ noinst_PROGRAMS = chkhostname libauthretry libntlmconnect                \
  lib1540 lib1541 \
  lib1550 lib1551 lib1552 lib1553 lib1554 lib1555 lib1556 lib1557 \
  lib1558 lib1559 lib1560 \
+ lib1589 \
  lib1591 lib1592 lib1593 \
  lib1900 lib1905 lib1906 \
  lib2033
@@ -533,6 +534,9 @@ lib1559_LDADD = $(TESTUTIL_LIBS)
 lib1560_SOURCES = lib1560.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS)
 lib1560_LDADD = $(TESTUTIL_LIBS)
 
+lib1589_SOURCES = lib1589.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS)
+lib1589_LDADD = $(TESTUTIL_LIBS)
+
 lib1591_SOURCES = lib1591.c $(SUPPORTFILES) $(TESTUTIL) $(WARNLESS)
 lib1591_LDADD = $(TESTUTIL_LIBS)
 lib1591_CPPFLAGS = $(AM_CPPFLAGS) -DLIB1591
diff --git a/tests/libtest/lib1589.c b/tests/libtest/lib1589.c
new file mode 100644
index 000000000..dca3a3055
--- /dev/null
+++ b/tests/libtest/lib1589.c
@@ -0,0 +1,57 @@
+/***************************************************************************
+ *                                  _   _ ____  _
+ *  Project                     ___| | | |  _ \| |
+ *                             / __| | | | |_) | |
+ *                            | (__| |_| |  _ <| |___
+ *                             \___|\___/|_| \_\_____|
+ *
+ * Copyright (C) 1998 - 2019, Daniel Stenberg, <daniel@haxx.se>, et al.
+ *
+ * This software is licensed as described in the file COPYING, which
+ * you should have received as part of this distribution. The terms
+ * are also available at https://curl.haxx.se/docs/copyright.html.
+ *
+ * You may opt to use, copy, modify, merge, publish, distribute and/or sell
+ * copies of the Software, and permit persons to whom the Software is
+ * furnished to do so, under the terms of the COPYING file.
+ *
+ * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY
+ * KIND, either express or implied.
+ *
+ ***************************************************************************/
+#include "test.h"
+
+#include "memdebug.h"
+
+int test(char *URL)
+{
+  CURLcode res;
+  CURL *curl;
+
+  if(curl_global_init(CURL_GLOBAL_ALL) != CURLE_OK) {
+    fprintf(stderr, "curl_global_init() failed\n");
+    return TEST_ERR_MAJOR_BAD;
+  }
+
+  curl = curl_easy_init();
+  if(!curl) {
+    fprintf(stderr, "curl_easy_init() failed\n");
+    curl_global_cleanup();
+    return TEST_ERR_MAJOR_BAD;
+  }
+
+  test_setopt(curl, CURLOPT_HEADER, 1L);
+  test_setopt(curl, CURLOPT_URL, URL);
+  test_setopt(curl, CURLOPT_POSTFIELDS, "moo");
+  test_setopt(curl, CURLOPT_CUSTOMREQUEST, "IGLOO");
+  test_setopt(curl, CURLOPT_FOLLOWLOCATION, CURLFOLLOW_NO_CUSTOMREQUEST);
+
+  res = curl_easy_perform(curl);
+
+test_cleanup:
+
+  curl_easy_cleanup(curl);
+  curl_global_cleanup();
+
+  return (int)res;
+}