curl: add --fail-with-bodybagder/fail-with-body

Prevent both --fail and --fail-with-body on the same command line.

Verify with test 349, 360 and 361.

Closes #6449

13 files changed, 179 insertions, 14 deletions

diff --git a/docs/cmdline-opts/Makefile.inc b/docs/cmdline-opts/Makefile.inc
index 642a4bb42..abfa38c97 100644
--- a/docs/cmdline-opts/Makefile.inc
+++ b/docs/cmdline-opts/Makefile.inc
@@ -5,7 +5,7 @@
 #                            | (__| |_| |  _ <| |___
 #                             \___|\___/|_| \_\_____|
 #
-# Copyright (C) 1998 - 2020, Daniel Stenberg, <daniel@haxx.se>, et al.
+# Copyright (C) 1998 - 2021, Daniel Stenberg, <daniel@haxx.se>, et al.
 #
 # This software is licensed as described in the file COPYING, which
 # you should have received as part of this distribution. The terms
@@ -61,10 +61,11 @@ DPAGES =					\
   dump-header.d					\
   egd-file.d					\
   engine.d					\
-  etag-compare.d                \
-  etag-save.d                   \
+  etag-compare.d                                \
+  etag-save.d                                   \
   expect100-timeout.d				\
   fail-early.d					\
+  fail-with-body.d				\
   fail.d					\
   false-start.d					\
   form-string.d					\
diff --git a/docs/cmdline-opts/fail-with-body.d b/docs/cmdline-opts/fail-with-body.d
new file mode 100644
index 000000000..91db8bde8
--- /dev/null
+++ b/docs/cmdline-opts/fail-with-body.d
@@ -0,0 +1,16 @@
+Long: fail-with-body
+Protocols: HTTP
+Help: Fail on HTTP errors but save the body
+Category: http output
+Added: 7.76.0
+See-also: fail
+---
+
+Return an error on server errors where the HTTP response code is 400 or
+greater). In normal cases when an HTTP server fails to deliver a document, it
+returns an HTML document stating so (which often also describes why and
+more). This flag will still allow curl to outputting and save that content but
+also to return error 22.
+
+This is an alternative option to --fail which makes curl fail for the same
+circumstances but without saving the content.
diff --git a/docs/cmdline-opts/fail.d b/docs/cmdline-opts/fail.d
index e5028a847..d4d65fba4 100644
--- a/docs/cmdline-opts/fail.d
+++ b/docs/cmdline-opts/fail.d
@@ -2,6 +2,7 @@ Long: fail
 Short: f
 Protocols: HTTP
 Help: Fail silently (no output at all) on HTTP errors
+See-also: fail-with-body
 Category: important http
 ---
 Fail silently (no output at all) on server errors. This is mostly done to
diff --git a/docs/options-in-versions b/docs/options-in-versions
index bdffeec7e..66a178565 100644
--- a/docs/options-in-versions
+++ b/docs/options-in-versions
@@ -59,6 +59,7 @@
 --expect100-timeout                  7.47.0
 --fail (-f)                          4.0
 --fail-early                         7.52.0
+--fail-with-body                     7.76.0
 --false-start                        7.42.0
 --form (-F)                          5.0
 --form-string                        7.13.2
diff --git a/src/tool_cfgable.h b/src/tool_cfgable.h
index 243cbd11b..b5987af07 100644
--- a/src/tool_cfgable.h
+++ b/src/tool_cfgable.h
@@ -118,6 +118,7 @@ struct OperationConfig {
   bool use_ascii;           /* select ascii or text transfer */
   bool autoreferer;         /* automatically set referer */
   bool failonerror;         /* fail on (HTTP) errors */
+  bool failwithbody;        /* fail on (HTTP) errors but still store body */
   bool show_headers;        /* show headers to data output */
   bool no_body;             /* don't get the body */
   bool dirlistonly;         /* only get the FTP dir list */
diff --git a/src/tool_getparam.c b/src/tool_getparam.c
index 812ce7fd9..d187643a7 100644
--- a/src/tool_getparam.c
+++ b/src/tool_getparam.c
@@ -280,6 +280,7 @@ static const struct LongShort aliases[]= {
   {"fa", "fail-early",               ARG_BOOL},
   {"fb", "styled-output",            ARG_BOOL},
   {"fc", "mail-rcpt-allowfails",     ARG_BOOL},
+  {"fd", "fail-with-body",           ARG_BOOL},
   {"F",  "form",                     ARG_STRING},
   {"Fs", "form-string",              ARG_STRING},
   {"g",  "globoff",                  ARG_BOOL},
@@ -1766,8 +1767,17 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */
       case 'c': /* --mail-rcpt-allowfails */
         config->mail_rcpt_allowfails = toggle;
         break;
+      case 'd': /* --fail-with-body */
+        config->failwithbody = toggle;
+        break;
       default: /* --fail (hard on errors)  */
         config->failonerror = toggle;
+        break;
+      }
+      if(config->failonerror && config->failwithbody) {
+        errorf(config->global, "You must select either --fail or "
+               "--fail-with-body, not both.\n");
+        return PARAM_BAD_USE;
       }
       break;
     case 'F':
diff --git a/src/tool_help.c b/src/tool_help.c
index a094450e5..1d8273002 100644
--- a/src/tool_help.c
+++ b/src/tool_help.c
@@ -268,6 +268,9 @@ static const struct helptxt helptext[] = {
   {"    --fail-early",
    "Fail on first transfer error, do not continue",
    CURLHELP_CURL},
+  {"    --fail-with-body",
+   "Fail on HTTP errors but save the body",
+   CURLHELP_HTTP},
   {"    --false-start",
    "Enable TLS False Start",
    CURLHELP_TLS},
diff --git a/src/tool_operate.c b/src/tool_operate.c
index 140142a32..de191717c 100644
--- a/src/tool_operate.c
+++ b/src/tool_operate.c
@@ -369,7 +369,18 @@ static CURLcode post_per_transfer(struct GlobalConfig *global,
       if(result == CURLE_PEER_FAILED_VERIFICATION)
         fputs(CURL_CA_CERT_ERRORMSG, global->errors);
     }
-
+    else if(config->failwithbody) {
+      /* if HTTP response >= 400, return error */
+      long code = 0;
+      curl_easy_getinfo(curl, CURLINFO_RESPONSE_CODE, &code);
+      if(code >= 400) {
+        if(global->showerror)
+          fprintf(global->errors,
+                  "curl: (%d) The requested URL returned error: %ld\n",
+                  CURLE_HTTP_RETURNED_ERROR, code);
+        result = CURLE_HTTP_RETURNED_ERROR;
+      }
+    }
   /* Set file extended attributes */
   if(!result && config->xattr && outs->fopened && outs->stream) {
     int rc = fwrite_xattr(curl, fileno(outs->stream));
@@ -669,7 +680,7 @@ static CURLcode post_per_transfer(struct GlobalConfig *global,
   free(per->outfile);
   free(per->uploadfile);
 
-  return CURLE_OK;
+  return result;
 }
 
 static void single_transfer_cleanup(struct OperationConfig *config)
@@ -2325,18 +2336,14 @@ static CURLcode serial_transfers(struct GlobalConfig *global,
 #endif
       result = curl_easy_perform(per->curl);
 
-    /* store the result of the actual transfer */
-    returncode = result;
-
-    result = post_per_transfer(global, per, result, &retry, &delay);
+    returncode = post_per_transfer(global, per, result, &retry, &delay);
     if(retry) {
       tool_go_sleep(delay);
       continue;
     }
 
     /* Bail out upon critical errors or --fail-early */
-    if(result || is_fatal_error(returncode) ||
-       (returncode && global->fail_early))
+    if(is_fatal_error(returncode) || (returncode && global->fail_early))
       bailout = TRUE;
     else {
       /* setup the next one just before we delete this */
diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc
index 6b6f2bc5f..9f99ed7ad 100644
--- a/tests/data/Makefile.inc
+++ b/tests/data/Makefile.inc
@@ -58,9 +58,10 @@ test307 test308 test309 test310 test311 test312 test313 test314 test315 \
 test316 test317 test318 test319 test320 test321 test322 test323 test324 \
 test325 test326 test327 test328 test329 test330 test331 test332 test333 \
 test334 test335 test336 test337 test338 test339 test340 test341 test342 \
-test343 test344 test345 test346 test347 test348 \
-test350 test351 test352 test353 test354 test355 test356 test357 test358 \
-test359 \
+test343 test344 test345 test346 test347 test348 test349 test350 test351 \
+test352 test353 test354 test355 test356 test357 test358 test359 test360 \
+test361 \
+\
 test393 test394 test395 test396 test397 \
 \
 test400 test401 test402 test403 test404 test405 test406 test407 test408 \
diff --git a/tests/data/test24 b/tests/data/test24
index 43e2da5b5..4c9d35e26 100644
--- a/tests/data/test24
+++ b/tests/data/test24
@@ -3,6 +3,7 @@
 <keywords>
 HTTP
 HTTP GET
+--fail
 </keywords>
 </info>
 # Server-side
diff --git a/tests/data/test349 b/tests/data/test349
new file mode 100644
index 000000000..472a4dc48
--- /dev/null
+++ b/tests/data/test349
@@ -0,0 +1,45 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+--fail-with-body
+</keywords>
+</info>
+# Server-side
+<reply>
+<data>
+HTTP/1.0 404 BAD BOY swsclose
+Content-Type: text/html
+
+This silly page doesn't reaaaaaly exist so you should not get it.
+</data>
+</reply>
+
+# Client-side
+<client>
+<server>
+http
+</server>
+ <name>
+HTTP GET --fail-with-body on HTTP error return
+ </name>
+ <command>
+http://%HOSTIP:%HTTPPORT/349 --fail-with-body
+</command>
+</client>
+
+# Verify data after the test has been "shot"
+<verify>
+<protocol>
+GET /349 HTTP/1.1

+Host: %HOSTIP:%HTTPPORT

+User-Agent: curl/%VERSION

+Accept: */*

+

+</protocol>
+<errorcode>
+22
+</errorcode>
+</verify>
+</testcase>
diff --git a/tests/data/test360 b/tests/data/test360
new file mode 100644
index 000000000..f466277b2
--- /dev/null
+++ b/tests/data/test360
@@ -0,0 +1,28 @@
+<testcase>
+<info>
+<keywords>
+--fail
+--fail-with-body
+</keywords>
+</info>
+
+# Client-side
+<client>
+<server>
+http
+</server>
+ <name>
+Error on both --fail-with-body and --fail
+ </name>
+ <command>
+http://%HOSTIP:%HTTPPORT/360 --fail-with-body --fail
+</command>
+</client>
+
+# Verify data after the test has been "shot"
+<verify>
+<errorcode>
+2
+</errorcode>
+</verify>
+</testcase>
diff --git a/tests/data/test361 b/tests/data/test361
new file mode 100644
index 000000000..7d41d9244
--- /dev/null
+++ b/tests/data/test361
@@ -0,0 +1,50 @@
+<testcase>
+<info>
+<keywords>
+HTTP
+HTTP GET
+--fail-with-body
+</keywords>
+</info>
+# Server-side
+<reply>
+<data>
+HTTP/1.0 404 BAD BOY swsclose
+Content-Type: text/html
+
+This silly page doesn't reaaaaaly exist so you should not get it.
+</data>
+</reply>
+
+# Client-side
+<client>
+<server>
+http
+</server>
+ <name>
+HTTP GET --fail-with-body on HTTP error return - twice
+ </name>
+ <command>
+http://%HOSTIP:%HTTPPORT/361 http://%HOSTIP:%HTTPPORT/361 --fail-with-body
+</command>
+</client>
+
+# Verify data after the test has been "shot"
+<verify>
+<protocol>
+GET /361 HTTP/1.1

+Host: %HOSTIP:%HTTPPORT

+User-Agent: curl/%VERSION

+Accept: */*

+

+GET /361 HTTP/1.1

+Host: %HOSTIP:%HTTPPORT

+User-Agent: curl/%VERSION

+Accept: */*

+

+</protocol>
+<errorcode>
+22
+</errorcode>
+</verify>
+</testcase>