From b778ae4c5e5bcb6da4de789e25971f40f0673d86 Mon Sep 17 00:00:00 2001 From: Daniel Stenberg Date: Mon, 19 Jun 2017 14:10:33 +0200 Subject: http: add --strip-path-slash and CURLOPT_STRIP_PATH_SLASH ... to enable sending "OPTIONS *" which wasn't possible previously. This option currently only works for HTTP. Added test cases 1298 + 1299 to verify Fixes #1280 Closes #1462 --- docs/TODO | 18 --------- docs/cmdline-opts/Makefile.inc | 2 +- docs/cmdline-opts/strip-path-slash.d | 7 ++++ docs/libcurl/curl_easy_setopt.3 | 4 +- docs/libcurl/opts/CURLOPT_CUSTOMREQUEST.3 | 1 + docs/libcurl/opts/CURLOPT_NOBODY.3 | 7 ++-- docs/libcurl/opts/CURLOPT_STRIP_PATH_SLASH.3 | 56 ++++++++++++++++++++++++++++ docs/libcurl/opts/Makefile.inc | 1 + docs/libcurl/symbols-in-versions | 1 + include/curl/curl.h | 3 ++ lib/http.c | 6 +++ lib/url.c | 4 ++ lib/urldata.h | 2 + src/tool_cfgable.h | 1 + src/tool_getparam.c | 7 +++- src/tool_help.c | 2 + src/tool_operate.c | 2 + tests/data/Makefile.inc | 2 +- tests/data/test1298 | 56 ++++++++++++++++++++++++++++ tests/data/test1299 | 55 +++++++++++++++++++++++++++ tests/server/sws.c | 8 ++++ 21 files changed, 220 insertions(+), 25 deletions(-) create mode 100644 docs/cmdline-opts/strip-path-slash.d create mode 100644 docs/libcurl/opts/CURLOPT_STRIP_PATH_SLASH.3 create mode 100644 tests/data/test1298 create mode 100644 tests/data/test1299 diff --git a/docs/TODO b/docs/TODO index 22c9beb6f..5db21e1bf 100644 --- a/docs/TODO +++ b/docs/TODO @@ -69,7 +69,6 @@ 5.9 Improve formpost API 5.10 Leave secure cookies alone 5.11 Chunked transfer multipart formpost - 5.12 OPTIONS * 6. TELNET 6.1 ditch stdin @@ -558,23 +557,6 @@ This is not detailed in any FTP specification. https://github.com/curl/curl/issues/1139 -5.12 OPTIONS * - - HTTP defines an OPTIONS method that can be sent with an asterisk option like - "OPTIONS *" to ask about options from the server and not a specific URL - resource. https://tools.ietf.org/html/rfc7230#section-5.3.4 - - libcurl as it currently works will always sent HTTP methods with a path that - starts with a slash so there's no way for an application to send a proper - "OPTIONS *" using libcurl. This should be fixed. - - I can't think of any other non-slash paths we should support so it will - probably make sense to add a new boolean option for issuign an "OPTIONS *" - request. CURLOPT_OPTIONSASTERISK perhaps (and a corresponding command line - option)? - - See https://github.com/curl/curl/issues/1280 - 6. TELNET diff --git a/docs/cmdline-opts/Makefile.inc b/docs/cmdline-opts/Makefile.inc index 4577fac7c..edc86fe1a 100644 --- a/docs/cmdline-opts/Makefile.inc +++ b/docs/cmdline-opts/Makefile.inc @@ -42,6 +42,6 @@ DPAGES = abstract-unix-socket.d anyauth.d append.d basic.d cacert.d capath.d cer tlsauthtype.d tlspassword.d tlsuser.d tlsv1.0.d tlsv1.1.d tlsv1.2.d \ tlsv1.3.d tlsv1.d trace-ascii.d trace.d trace-time.d tr-encoding.d \ unix-socket.d upload-file.d url.d use-ascii.d user-agent.d user.d \ - verbose.d version.d write-out.d xattr.d + verbose.d version.d write-out.d xattr.d strip-path-slash.d OTHERPAGES = page-footer page-header diff --git a/docs/cmdline-opts/strip-path-slash.d b/docs/cmdline-opts/strip-path-slash.d new file mode 100644 index 000000000..4aee947dd --- /dev/null +++ b/docs/cmdline-opts/strip-path-slash.d @@ -0,0 +1,7 @@ +Long: strip-path-slash +Help: Strip off the first slash of the path +Protocols: HTTP +--- +Tells curl to strip the leading slash from the path when it sends the path to +the server. Useful when wanting to issue HTTP requests without leading slash, +like "OPTIONS *". diff --git a/docs/libcurl/curl_easy_setopt.3 b/docs/libcurl/curl_easy_setopt.3 index cb6884766..70e1ee8fd 100644 --- a/docs/libcurl/curl_easy_setopt.3 +++ b/docs/libcurl/curl_easy_setopt.3 @@ -5,7 +5,7 @@ .\" * | (__| |_| | _ <| |___ .\" * \___|\___/|_| \_\_____| .\" * -.\" * Copyright (C) 1998 - 2016, Daniel Stenberg, , et al. +.\" * Copyright (C) 1998 - 2017, Daniel Stenberg, , et al. .\" * .\" * This software is licensed as described in the file COPYING, which .\" * you should have received as part of this distribution. The terms @@ -303,6 +303,8 @@ Start a new cookie session. See \fICURLOPT_COOKIESESSION(3)\fP Add or control cookies. See \fICURLOPT_COOKIELIST(3)\fP .IP CURLOPT_HTTPGET Do a HTTP GET request. See \fICURLOPT_HTTPGET(3)\fP +.IP CURLOPT_STRIP_PATH_SLASH +Cut off the leading slash from the path. \fICURLOPT_STRIP_PATH_SLASH(3)\fP .IP CURLOPT_HTTP_VERSION HTTP version to use. \fICURLOPT_HTTP_VERSION(3)\fP .IP CURLOPT_IGNORE_CONTENT_LENGTH diff --git a/docs/libcurl/opts/CURLOPT_CUSTOMREQUEST.3 b/docs/libcurl/opts/CURLOPT_CUSTOMREQUEST.3 index b20d6836c..e53504491 100644 --- a/docs/libcurl/opts/CURLOPT_CUSTOMREQUEST.3 +++ b/docs/libcurl/opts/CURLOPT_CUSTOMREQUEST.3 @@ -108,3 +108,4 @@ Returns CURLE_OK if the option is supported, CURLE_UNKNOWN_OPTION if not, or CURLE_OUT_OF_MEMORY if there was insufficient heap space. .SH "SEE ALSO" .BR CURLOPT_HTTPHEADER "(3), " CURLOPT_NOBODY "(3), " +.BR CURLOPT_STRIP_PATH_SLASH "(3), " diff --git a/docs/libcurl/opts/CURLOPT_NOBODY.3 b/docs/libcurl/opts/CURLOPT_NOBODY.3 index 340b9f454..d0116a9ed 100644 --- a/docs/libcurl/opts/CURLOPT_NOBODY.3 +++ b/docs/libcurl/opts/CURLOPT_NOBODY.3 @@ -5,7 +5,7 @@ .\" * | (__| |_| | _ <| |___ .\" * \___|\___/|_| \_\_____| .\" * -.\" * Copyright (C) 1998 - 2014, Daniel Stenberg, , et al. +.\" * Copyright (C) 1998 - 2017, Daniel Stenberg, , et al. .\" * .\" * This software is licensed as described in the file COPYING, which .\" * you should have received as part of this distribution. The terms @@ -44,10 +44,10 @@ curl = curl_easy_init(); if(curl) { curl_easy_setopt(curl, CURLOPT_URL, "http://example.com"); - /* get us the resource without a body! */ + /* get us the resource without a body! */ curl_easy_setopt(curl, CURLOPT_NOBODY, 1L); - /* Perform the request */ + /* Perform the request */ curl_easy_perform(curl); } .fi @@ -57,3 +57,4 @@ Always Returns CURLE_OK .SH "SEE ALSO" .BR CURLOPT_HTTPGET "(3), " CURLOPT_POST "(3), " +.BR CURLOPT_STRIP_PATH_SLASH "(3), " diff --git a/docs/libcurl/opts/CURLOPT_STRIP_PATH_SLASH.3 b/docs/libcurl/opts/CURLOPT_STRIP_PATH_SLASH.3 new file mode 100644 index 000000000..5d0886782 --- /dev/null +++ b/docs/libcurl/opts/CURLOPT_STRIP_PATH_SLASH.3 @@ -0,0 +1,56 @@ +.\" ************************************************************************** +.\" * _ _ ____ _ +.\" * Project ___| | | | _ \| | +.\" * / __| | | | |_) | | +.\" * | (__| |_| | _ <| |___ +.\" * \___|\___/|_| \_\_____| +.\" * +.\" * Copyright (C) 1998 - 2017, Daniel Stenberg, , et al. +.\" * +.\" * This software is licensed as described in the file COPYING, which +.\" * you should have received as part of this distribution. The terms +.\" * are also available at https://curl.haxx.se/docs/copyright.html. +.\" * +.\" * You may opt to use, copy, modify, merge, publish, distribute and/or sell +.\" * copies of the Software, and permit persons to whom the Software is +.\" * furnished to do so, under the terms of the COPYING file. +.\" * +.\" * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY +.\" * KIND, either express or implied. +.\" * +.\" ************************************************************************** +.\" +.TH CURLOPT_STRIP_PATH_SLASH 3 "17 Jun 2014" "libcurl 7.37.0" "curl_easy_setopt options" +.SH NAME +CURLOPT_STRIP_PATH_SLASH \- strip the leading slash from the path +.SH SYNOPSIS +#include + +CURLcode curl_easy_setopt(CURL *handle, CURLOPT_STRIP_PATH_SLASH, value); +.SH DESCRIPTION +Pass a long set to 1 to tell libcurl to strip out the leading slash from the +path when used on the server. +.SH DEFAULT +0 - use the leading slash. +.SH PROTOCOLS +HTTP +.SH EXAMPLE +.nf +curl = curl_easy_init(); +if(curl) { + curl_easy_setopt(curl, CURLOPT_URL, "http://example.com/*"); + curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "OPTIONS"); + + /* issue an OPTIONS * request (no leading slash) */ + curl_easy_setopt(curl, CURLOPT_STRIP_PATH_SLASH, 1L); + + /* Perform the request */ + curl_easy_perform(curl); +} +.fi +.SH AVAILABILITY +Added in 7.55.0 +.SH RETURN VALUE +Returns CURLE_OK if the option is supported, and CURLE_UNKNOWN_OPTION if not. +.SH "SEE ALSO" +.BR CURLOPT_CUSTOMREQUEST "(3), " CURLOPT_HTTPGET "(3), " diff --git a/docs/libcurl/opts/Makefile.inc b/docs/libcurl/opts/Makefile.inc index 5a201fe3f..40930dc5d 100644 --- a/docs/libcurl/opts/Makefile.inc +++ b/docs/libcurl/opts/Makefile.inc @@ -278,6 +278,7 @@ man_MANS = \ CURLOPT_STREAM_DEPENDS.3 \ CURLOPT_STREAM_DEPENDS_E.3 \ CURLOPT_STREAM_WEIGHT.3 \ + CURLOPT_STRIP_PATH_SLASH.3 \ CURLOPT_SUPPRESS_CONNECT_HEADERS.3 \ CURLOPT_TCP_FASTOPEN.3 \ CURLOPT_TCP_KEEPALIVE.3 \ diff --git a/docs/libcurl/symbols-in-versions b/docs/libcurl/symbols-in-versions index 169ec8fe3..c9a53cb1c 100644 --- a/docs/libcurl/symbols-in-versions +++ b/docs/libcurl/symbols-in-versions @@ -409,6 +409,7 @@ CURLOPT_HTTPPOST 7.1 CURLOPT_HTTPPROXYTUNNEL 7.3 CURLOPT_HTTPREQUEST 7.1 - 7.15.5 CURLOPT_HTTP_CONTENT_DECODING 7.16.2 +CURLOPT_STRIP_PATH_SLASH 7.55.0 CURLOPT_HTTP_TRANSFER_DECODING 7.16.2 CURLOPT_HTTP_VERSION 7.9.1 CURLOPT_IGNORE_CONTENT_LENGTH 7.14.1 diff --git a/include/curl/curl.h b/include/curl/curl.h index 75cc14abc..521aa5e78 100644 --- a/include/curl/curl.h +++ b/include/curl/curl.h @@ -1780,6 +1780,9 @@ typedef enum { /* Suppress proxy CONNECT response headers from user callbacks */ CINIT(SUPPRESS_CONNECT_HEADERS, LONG, 265), + /* Strip the initial slash from the path taken from the URL */ + CINIT(STRIP_PATH_SLASH, LONG, 266), + CURLOPT_LASTENTRY /* the last unused */ } CURLoption; diff --git a/lib/http.c b/lib/http.c index c65cb58d2..2d04705ba 100644 --- a/lib/http.c +++ b/lib/http.c @@ -1851,6 +1851,9 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) case HTTPREQ_PUT: request = "PUT"; break; + case HTTPREQ_OPTIONS: + request = "OPTIONS"; + break; default: /* this should never happen */ case HTTPREQ_GET: request = "GET"; @@ -2266,6 +2269,9 @@ CURLcode Curl_http(struct connectdata *conn, bool *done) if(result) return result; + if(data->set.strip_path_slash) + ppath++; + /* url */ if(paste_ftp_userpwd) result = Curl_add_bufferf(req_buffer, "ftp://%s:%s@%s", diff --git a/lib/url.c b/lib/url.c index 8cd83d3a7..23b243f83 100644 --- a/lib/url.c +++ b/lib/url.c @@ -829,6 +829,10 @@ CURLcode Curl_setopt(struct Curl_easy *data, CURLoption option, then this can be changed to HEAD later on) */ data->set.httpreq = HTTPREQ_GET; break; + case CURLOPT_STRIP_PATH_SLASH: + arg = va_arg(param, long); + data->set.strip_path_slash = (bool)arg; + break; case CURLOPT_FILETIME: /* * Try to get the file time of the remote document. The time will diff --git a/lib/urldata.h b/lib/urldata.h index 62037d0f2..9310abc17 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -1264,6 +1264,7 @@ typedef enum { HTTPREQ_POST_FORM, /* we make a difference internally */ HTTPREQ_PUT, HTTPREQ_HEAD, + HTTPREQ_OPTIONS, HTTPREQ_CUSTOM, HTTPREQ_LAST /* last in list */ } Curl_HttpReq; @@ -1698,6 +1699,7 @@ struct UserDefined { Curl_HttpReq httpreq; /* what kind of HTTP request (if any) is this */ long httpversion; /* when non-zero, a specific HTTP version requested to be used in the library's request(s) */ + bool strip_path_slash; /* strip off initial slash from path */ struct ssl_config_data ssl; /* user defined SSL stuff */ struct ssl_config_data proxy_ssl; /* user defined SSL stuff for proxy */ struct ssl_general_config general_ssl; /* general user defined SSL stuff */ diff --git a/src/tool_cfgable.h b/src/tool_cfgable.h index 8d74905d9..f6536e8e4 100644 --- a/src/tool_cfgable.h +++ b/src/tool_cfgable.h @@ -144,6 +144,7 @@ struct OperationConfig { bool readbusy; /* set when reading input returns EAGAIN */ bool globoff; bool use_httpget; + bool strip_path_slash; bool insecure_ok; /* set TRUE to allow insecure SSL connects */ bool proxy_insecure_ok; /* set TRUE to allow insecure SSL connects for proxy */ diff --git a/src/tool_getparam.c b/src/tool_getparam.c index bcb9e1ee2..642bba832 100644 --- a/src/tool_getparam.c +++ b/src/tool_getparam.c @@ -257,6 +257,7 @@ static const struct LongShort aliases[]= { {"Fs", "form-string", ARG_STRING}, {"g", "globoff", ARG_BOOL}, {"G", "get", ARG_NONE}, + {"Ga", "strip-path-slash", ARG_BOOL}, {"h", "help", ARG_BOOL}, {"H", "header", ARG_STRING}, {"Hp", "proxy-header", ARG_STRING}, @@ -1591,7 +1592,11 @@ ParameterError getparameter(const char *flag, /* f or -long-flag */ break; case 'G': /* HTTP GET */ - config->use_httpget = TRUE; + if(subletter == 'a') { /* --strip-path-slash */ + config->strip_path_slash = TRUE; + } + else + config->use_httpget = TRUE; break; case 'h': /* h for help */ diff --git a/src/tool_help.c b/src/tool_help.c index 08a81f590..6d36e550f 100644 --- a/src/tool_help.c +++ b/src/tool_help.c @@ -400,6 +400,8 @@ static const struct helptxt helptext[] = { "Use SSLv3"}, {" --stderr", "Where to redirect stderr"}, + {" --strip-path-slash", + "Strip off the first slash of the path"}, {" --suppress-connect-headers", "Suppress proxy CONNECT response headers"}, {" --tcp-fastopen", diff --git a/src/tool_operate.c b/src/tool_operate.c index b80a77118..51a9aa9f6 100644 --- a/src/tool_operate.c +++ b/src/tool_operate.c @@ -972,6 +972,8 @@ static CURLcode operate_do(struct GlobalConfig *global, #endif /* !CURL_DISABLE_PROXY */ my_setopt(curl, CURLOPT_FAILONERROR, config->failonerror?1L:0L); + my_setopt(curl, CURLOPT_STRIP_PATH_SLASH, + config->strip_path_slash?1L:0L); my_setopt(curl, CURLOPT_UPLOAD, uploadfile?1L:0L); my_setopt(curl, CURLOPT_DIRLISTONLY, config->dirlistonly?1L:0L); my_setopt(curl, CURLOPT_APPEND, config->ftp_append?1L:0L); diff --git a/tests/data/Makefile.inc b/tests/data/Makefile.inc index b1f947466..64eb0619b 100644 --- a/tests/data/Makefile.inc +++ b/tests/data/Makefile.inc @@ -133,7 +133,7 @@ test1260 test1261 test1262 \ \ test1280 test1281 test1282 test1283 test1284 test1285 test1286 test1287 \ test1288 \ -\ +test1298 test1299 \ test1300 test1301 test1302 test1303 test1304 test1305 test1306 test1307 \ test1308 test1309 test1310 test1311 test1312 test1313 test1314 test1315 \ test1316 test1317 test1318 test1319 test1320 test1321 test1322 \ diff --git a/tests/data/test1298 b/tests/data/test1298 new file mode 100644 index 000000000..c9c8b383a --- /dev/null +++ b/tests/data/test1298 @@ -0,0 +1,56 @@ + + + +HTTP +HTTP GET +--strip-path-slash + + + +# +# Server-side + + +HTTP/1.1 200 OK +Date: Thu, 09 Nov 2010 14:49:00 GMT +Server: test-server/fake +Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT +ETag: "21025-dc7-39462498" +Accept-Ranges: bytes +Content-Length: 0 +Connection: close +Content-Type: text/html +Funny-head: yesyes + + + + +# +# Client-side + + +http + + +HTTP GET special path with --strip-path-slash + + +--strip-path-slash "http://%HOSTIP:%HTTPPORT/XXX" -H "Testno: 1298" + + + +# +# Verify data after the test has been "shot" + + +^User-Agent:.* + + +GET XXX HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +Accept: */* +Testno: 1298 + + + + diff --git a/tests/data/test1299 b/tests/data/test1299 new file mode 100644 index 000000000..51bb71248 --- /dev/null +++ b/tests/data/test1299 @@ -0,0 +1,55 @@ + + + +HTTP +--strip-path-slash + + + +# +# Server-side + + +HTTP/1.1 200 OK +Date: Thu, 09 Nov 2010 14:49:00 GMT +Server: test-server/fake +Last-Modified: Tue, 13 Jun 2000 12:10:00 GMT +ETag: "21025-dc7-39462498" +Accept-Ranges: bytes +Content-Length: 0 +Connection: close +Content-Type: text/html +Funny-head: yesyes + + + + +# +# Client-side + + +http + + +Send "OPTIONS *" with --strip-path-slash + + +--strip-path-slash -X OPTIONS http://%HOSTIP:%HTTPPORT/* -H "Testno: 1299" + + + +# +# Verify data after the test has been "shot" + + +^User-Agent:.* + + +OPTIONS * HTTP/1.1 +Host: %HOSTIP:%HTTPPORT +Accept: */* +Testno: 1299 + + + + diff --git a/tests/server/sws.c b/tests/server/sws.c index b1199ccc6..bff30f212 100644 --- a/tests/server/sws.c +++ b/tests/server/sws.c @@ -601,6 +601,14 @@ static int ProcessRequest(struct httprequest *req) } } + if(req->testno == DOCNUMBER_NOTHING) { + /* check for a Testno: header with the test case number */ + char *testno = strstr(line, "\nTestno: "); + if(testno) { + req->testno = strtol(&testno[9], NULL, 10); + logmsg("Found test number %d in Testno: header!", req->testno); + } + } if(req->testno == DOCNUMBER_NOTHING) { /* Still no test case number. Try to get the the number off the last dot instead, IE we consider the TLD to be the test number. Test 123 can -- cgit v1.2.1