diff options
| -rw-r--r-- | CHANGES | 7 | ||||
| -rw-r--r-- | Makefile.am | 3 | ||||
| -rw-r--r-- | UPGRADE | 34 | ||||
| -rw-r--r-- | acconfig.h | 3 | ||||
| -rw-r--r-- | configure.in | 29 | ||||
| -rw-r--r-- | docs/THANKS | 1 | ||||
| -rw-r--r-- | include/curl/curl.h | 6 | ||||
| -rw-r--r-- | lib/Makefile.am | 6 | ||||
| -rw-r--r-- | lib/README.encoding | 53 | ||||
| -rw-r--r-- | lib/content_encoding.c | 122 | ||||
| -rw-r--r-- | lib/content_encoding.h | 34 | ||||
| -rw-r--r-- | lib/http.c | 3 | ||||
| -rw-r--r-- | lib/http_chunks.c | 29 | ||||
| -rw-r--r-- | lib/http_chunks.h | 1 | ||||
| -rw-r--r-- | lib/transfer.c | 70 | ||||
| -rw-r--r-- | lib/url.c | 17 | ||||
| -rw-r--r-- | lib/urldata.h | 22 | ||||
| -rw-r--r-- | src/Makefile.am | 2 | ||||
| -rw-r--r-- | src/main.c | 18 |
19 files changed, 433 insertions, 27 deletions
@@ -6,6 +6,13 @@ Changelog +Daniel (2 Sep 2002) +- James Gallagher added Content-Encoding support to libcurl so now curl and + libcurl-using apps can request compressed contents using the 'deflate' + method. See the special file lib/README.encoding for details. + + curl --compressed is now used to request compressed contents. + Daniel (30 Aug 2002) - Applied an anonymous SOCKS5-proxy patch. Not properly working in all situations though, as all getaddrinfo()-using libcurls will fail on this. diff --git a/Makefile.am b/Makefile.am index 298c31971..164f7cc51 100644 --- a/Makefile.am +++ b/Makefile.am @@ -4,8 +4,7 @@ AUTOMAKE_OPTIONS = foreign -EXTRA_DIST = \ - CHANGES LEGAL maketgz MITX.txt MPL-1.1.txt \ +EXTRA_DIST = CHANGES LEGAL maketgz MITX.txt MPL-1.1.txt UPGRADE \ reconf Makefile.dist curl-config.in build_vms.com curl-mode.el bin_SCRIPTS = curl-config diff --git a/UPGRADE b/UPGRADE new file mode 100644 index 000000000..38d9679c5 --- /dev/null +++ b/UPGRADE @@ -0,0 +1,34 @@ +Upgrading to curl/libcurl 7.10 from any previous version +======================================================== + +libcurl 7.10 performs peer SSL certificate verification by default. This is +done by installing a default CA cert bundle on 'make install' (or similar), +that is used by default on operations against SSL servers. + +Alas, if you use communicate with HTTPS servers using certifcates that are +signed by CAs present in the bundle, you will not notice and changed behavior +and you will seeminglessly get a higher security level on your SSL connections +since you will make sure that the remote server really is who it claims to be. + +If the remote server uses a self-signed certificate, or if you don't install +curl's CA cert bundle or if it uses a certificate signed by a CA that isn't +included in the bundle, then you need to do one of the following: + + 1. Tell libcurl to *not* verify the peer. With libcurl you disable with with + curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, FALSE); + + With the curl command tool, you disable this with -k/--insecure. + + 2. Get a CA certificate that can verify the remote server and use the proper + option to point out this CA cert for verification when connecting. For + libcurl hackers: curl_easy_setopt(curl, CURLOPT_CAPATH, capath); + + With the curl command tool: --cacert [file] + +This upgrade procedure has been deemed The Right Thing even though it adds +this extra trouble for some users, since it adds security to a majority of the +SSL connections that previously weren't really secure. + +It turned out many people were using previous versions of curl/libcurl without +realizing the need for the CA cert options to get truly secure SSL +connections. diff --git a/acconfig.h b/acconfig.h index d84b28a4d..cceaa577c 100644 --- a/acconfig.h +++ b/acconfig.h @@ -85,3 +85,6 @@ /* Define to disable TELNET */ #undef CURL_DISABLE_TELNET + +/* Define if you have zlib present */ +#undef HAVE_LIBZ diff --git a/configure.in b/configure.in index ba45c060c..f4702b768 100644 --- a/configure.in +++ b/configure.in @@ -522,20 +522,21 @@ dnl NOTE: We *always* look for ZLIB headers & libraries, all this option dnl does is change where we look (by adjusting LIBS and CPPFLAGS.) dnl -dnl AC_MSG_CHECKING(where to look for ZLIB) -dnl if test X"$OPT_ZLIB" = Xno -dnl then -dnl AC_MSG_RESULT([defaults (or given in environment)]) -dnl else -dnl test X"$OPT_ZLIB" = Xyes && OPT_ZLIB=/usr/local -dnl LIBS="$LIBS -L$OPT_ZLIB/lib" -dnl CPPFLAGS="$CPPFLAGS -I$OPT_ZLIB/include" -dnl AC_MSG_RESULT([$OPT_ZLIB]) -dnl fi - -dnl z lib? -dnl AC_CHECK_FUNC(gzread, , AC_CHECK_LIB(z, gzread)) +AC_MSG_CHECKING(where to look for ZLIB) +if test X"$OPT_ZLIB" = Xno +then + AC_MSG_RESULT([defaults (or given in environment)]) +else + test X"$OPT_ZLIB" = Xyes && OPT_ZLIB=/usr/local + LIBS="$LIBS -L$OPT_ZLIB/lib" + CPPFLAGS="$CPPFLAGS -I$OPT_ZLIB/include" + AC_MSG_RESULT([$OPT_ZLIB]) +fi +dnl AC_CHECK_FUNC(gzread, , AC_CHECK_LIB(z, gzread)) +AC_CHECK_LIB(z, gzread, [AM_CONDITIONAL(CONTENT_ENCODING, true) + AC_DEFINE(HAVE_LIBZ) + LIBS="$LIBS -lz"]) dnl Default is to try the thread-safe versions of a few functions OPT_THREAD=on @@ -606,7 +607,7 @@ AC_CHECK_HEADERS( \ ) dnl Check for libz header -dnl AC_CHECK_HEADERS(zlib.h) +AC_CHECK_HEADERS(zlib.h) dnl Checks for typedefs, structures, and compiler characteristics. AC_C_CONST diff --git a/docs/THANKS b/docs/THANKS index f58fdffc4..81fd21f9b 100644 --- a/docs/THANKS +++ b/docs/THANKS @@ -81,3 +81,4 @@ that have contributed with non-trivial parts: - Götz Babin-Ebell <babinebell@trustcenter.de> - Andreas Damm <andreas-sourceforge@radab.org> - Jacky Lam <sylam@emsoftltd.com> + - James Gallagher <jgallagher@gso.uri.edu> diff --git a/include/curl/curl.h b/include/curl/curl.h index 1b20a674e..e547136ac 100644 --- a/include/curl/curl.h +++ b/include/curl/curl.h @@ -200,6 +200,7 @@ typedef enum { CURLE_SSL_CERTPROBLEM, /* 58 - problem with the local certificate */ CURLE_SSL_CIPHER, /* 59 - couldn't use specified cipher */ CURLE_SSL_CACERT, /* 60 - problem with the CA cert (path?) */ + CURLE_BAD_CONTENT_ENCODING, /* 61 - Unrecognized transfer encoding */ CURL_LAST /* never use! */ } CURLcode; @@ -585,6 +586,11 @@ typedef enum { CURLPROXY_SOCKS4 and CURLPROXY_SOCKS5. */ CINIT(PROXYTYPE, LONG, 101), + /* Set the Accept-Encoding string. Use this to tell a server you would like + the response to be compressed. */ + CINIT(ENCODING, OBJECTPOINT, 102), + + CURLOPT_LASTENTRY /* the last unused */ } CURLoption; diff --git a/lib/Makefile.am b/lib/Makefile.am index 12a582d92..cf0258f2c 100644 --- a/lib/Makefile.am +++ b/lib/Makefile.am @@ -7,7 +7,7 @@ AUTOMAKE_OPTIONS = foreign nostdinc EXTRA_DIST = getdate.y Makefile.b32 Makefile.b32.resp Makefile.m32 \ Makefile.vc6 Makefile.riscos libcurl.def dllinit.c curllib.dsp \ curllib.dsw config-vms.h config-win32.h config-riscos.h config-mac.h \ - config.h.in ca-bundle.crt + config.h.in ca-bundle.crt README.encoding lib_LTLIBRARIES = libcurl.la @@ -45,6 +45,8 @@ libcurl_la_LDFLAGS = -version-info 2:2:0 # set age to 0. # +# I could not get conditional compilation using LIBADD or LDADD and +# EXTRA_..._SOURCES to work. 08/28/02 jhrg libcurl_la_SOURCES = \ arpa_telnet.h file.c getpass.h netrc.h timeval.c \ base64.c file.h hostip.c progress.c timeval.h \ @@ -60,7 +62,7 @@ escape.h getpass.c netrc.c telnet.h \ getinfo.c getinfo.h transfer.c strequal.c strequal.h easy.c \ security.h security.c krb4.c krb4.h memdebug.c memdebug.h inet_ntoa_r.h \ http_chunks.c http_chunks.h strtok.c strtok.h connect.c connect.h \ -llist.c llist.h hash.c hash.h multi.c share.c share.h +llist.c llist.h hash.c hash.h multi.c content_encoding.c noinst_HEADERS = setup.h transfer.h diff --git a/lib/README.encoding b/lib/README.encoding new file mode 100644 index 000000000..ef5c8036f --- /dev/null +++ b/lib/README.encoding @@ -0,0 +1,53 @@ + + Content Encoding Support for libcurl + +* About content encodings: + +HTTP/1.1 [RFC 2616] specifies that a client may request that a server encode +its response. This is usually used to compress a response using one of a set +of commonly available compression techniques. These schemes are `deflate' +(the zlib algorithm), `gzip' and `compress' [sec 3.5, RFC 2616]. A client +requests that the sever perform an encoding by including an Accept-Encoding +header in the request document. The value of the header should be one of the +recognized tokens `deflate', ... (there's a way to register new +schemes/tokens, see sec 3.5 of the spec). A server MAY honor the client's +encoding request. When a response is encoded, the server includes a +Content-Encoding header in the response. The value of the Content-Encoding +header indicates which scheme was used to encode the data. + +A client may tell a server that it can understand several different encoding +schemes. In this case the server may choose any one of those and use it to +encode the response (indicating which one using the Content-Encoding header). +It's also possible for a client to attach priorities to different schemes so +that the server knows which it prefers. See sec 14.3 of RFC 2616 for more +information on the Accept-Encoding header. + +* Current support for content encoding: + +I added support for the 'deflate' content encoding to both libcurl and curl. +Both regular and chunked transfers should work although I've tested only the +former. The library zlib is required for this feature. Places where I +modified the source code are commented and typically include my initials and +the date (e.g., 08/29/02 jhrg). + +* The libcurl interface: + +To cause libcurl to request a content encoding use: + + curl_easy_setopt(curl, CURLOPT_ENCODING, <string>) + +where <string> is the intended value of the Accept-Encoding header. + +Currently, libcurl only understands how to process responses that use the +`deflate' Content-Encoding, so the only value for CURLOPT_ENCODING that will +work (besides "identity," which does nothing) is "deflate." If a response is +encoded using either the `gzip' or `compress' methods, libcurl will return an +error indicating that the response could not be decoded. If <string> is null +or empty no Accept-Encoding header is generated. + +* The curl interface: + +Use the --compressed option with curl to cause it to ask servers to compress +responses using deflate. + +James Gallagher <jgallagher@gso.uri.edu> diff --git a/lib/content_encoding.c b/lib/content_encoding.c new file mode 100644 index 000000000..51b59c584 --- /dev/null +++ b/lib/content_encoding.c @@ -0,0 +1,122 @@ +/***************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2002, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * In order to be useful for every potential user, curl and libcurl are + * dual-licensed under the MPL and the MIT/X-derivate licenses. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the MPL or the MIT/X-derivate + * licenses. You may pick one of these licenses. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + * $Id$ + *****************************************************************************/ + +#include "setup.h" + +#ifdef HAVE_LIBZ + +#include "urldata.h" +#include <curl/curl.h> +#include <curl/types.h> +#include "sendf.h" + +#define DSIZ 4096 /* buffer size for decompressed data */ + + +static CURLcode +process_zlib_error(struct SessionHandle *data, z_stream *z) +{ + if (z->msg) + failf (data, "Error while processing content unencoding.\n%s", + z->msg); + else + failf (data, "Error while processing content unencoding.\n" + "Unknown failure within decompression software."); + + return CURLE_BAD_CONTENT_ENCODING; +} + +static CURLcode +exit_zlib(z_stream *z, bool *zlib_init, CURLcode result) +{ + inflateEnd(z); + *zlib_init = 0; + return result; +} + +CURLcode +Curl_unencode_deflate_write(struct SessionHandle *data, + struct Curl_transfer_keeper *k, + ssize_t nread) +{ + int status; /* zlib status */ + int result; /* Curl_client_write status */ + char decomp[DSIZ]; /* Put the decompressed data here. */ + z_stream *z = &k->z; /* zlib state structure */ + + /* Initialize zlib? */ + if (!k->zlib_init) { + z->zalloc = (alloc_func)Z_NULL; + z->zfree = (free_func)Z_NULL; + z->opaque = 0; /* of dubious use 08/27/02 jhrg */ + if (inflateInit(z) != Z_OK) + return process_zlib_error(data, z); + k->zlib_init = 1; + } + + /* Set the compressed input when this fucntion is called */ + z->next_in = (Bytef *)k->str; + z->avail_in = nread; + + /* because the buffer size is fixed, iteratively decompress + and transfer to the client via client_write. */ + for (;;) { + /* (re)set buffer for decompressed output for every iteration */ + z->next_out = (Bytef *)&decomp[0]; + z->avail_out = DSIZ; + + status = inflate(z, Z_SYNC_FLUSH); + if (status == Z_OK || status == Z_STREAM_END) { + result = Curl_client_write(data, CLIENTWRITE_BODY, decomp, + DSIZ - z->avail_out); + /* if !CURLE_OK, clean up, return */ + if (result) { + return exit_zlib(z, &k->zlib_init, result); + } + + /* Done?; clean up, return */ + if (status == Z_STREAM_END) { + if (inflateEnd(z) == Z_OK) + return exit_zlib(z, &k->zlib_init, result); + else + return exit_zlib(z, &k->zlib_init, process_zlib_error(data, z)); + } + + /* Done with these bytes, exit */ + if (status == Z_OK && z->avail_in == 0 && z->avail_out > 0) + return result; + } + else { /* Error; exit loop, handle below */ + return exit_zlib(z, &k->zlib_init, process_zlib_error(data, z)); + } + } +} +#endif /* HAVE_LIBZ */ + +/* + * local variables: + * eval: (load-file "../curl-mode.el") + * end: + * vim600: fdm=marker + * vim: et sw=2 ts=2 sts=2 tw=78 + */ diff --git a/lib/content_encoding.h b/lib/content_encoding.h new file mode 100644 index 000000000..91ba7d110 --- /dev/null +++ b/lib/content_encoding.h @@ -0,0 +1,34 @@ +/***************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2002, Daniel Stenberg, <daniel@haxx.se>, et al. + * + * In order to be useful for every potential user, curl and libcurl are + * dual-licensed under the MPL and the MIT/X-derivate licenses. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the MPL or the MIT/X-derivate + * licenses. You may pick one of these licenses. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + * $Id$ + *****************************************************************************/ + +CURLcode Curl_unencode_deflate_write(struct SessionHandle *data, + struct Curl_transfer_keeper *k, + ssize_t nread); + +/* + * local variables: + * eval: (load-file "../curl-mode.el") + * end: + * vim600: fdm=marker + * vim: et sw=2 ts=2 sts=2 tw=78 + */ diff --git a/lib/http.c b/lib/http.c index 08a5536dc..8957968d8 100644 --- a/lib/http.c +++ b/lib/http.c @@ -707,6 +707,7 @@ CURLcode Curl_http(struct connectdata *conn) "%s" /* host */ "%s" /* pragma */ "%s" /* accept */ + "%s" /* accept-encoding */ "%s", /* referer */ data->set.customrequest?data->set.customrequest: @@ -727,6 +728,8 @@ CURLcode Curl_http(struct connectdata *conn) (conn->allocptr.host?conn->allocptr.host:""), /* Host: host */ http->p_pragma?http->p_pragma:"", http->p_accept?http->p_accept:"", + (data->set.encoding && *data->set.encoding && conn->allocptr.accept_encoding)? + conn->allocptr.accept_encoding:"", /* 08/28/02 jhrg */ (data->change.referer && conn->allocptr.ref)?conn->allocptr.ref:"" /* Referer: <data> <CRLF> */ ); diff --git a/lib/http_chunks.c b/lib/http_chunks.c index 939e86a91..784d231fe 100644 --- a/lib/http_chunks.c +++ b/lib/http_chunks.c @@ -33,6 +33,8 @@ #include "urldata.h" /* it includes http_chunks.h */ #include "sendf.h" /* for the client write stuff */ +#include "content_encoding.h" /* 08/29/02 jhrg */ + #define _MPRINTF_REPLACE /* use our functions only */ #include <curl/mprintf.h> @@ -172,7 +174,32 @@ CHUNKcode Curl_httpchunk_read(struct connectdata *conn, piece = (ch->datasize >= length)?length:ch->datasize; /* Write the data portion available */ - result = Curl_client_write(conn->data, CLIENTWRITE_BODY, datap, piece); + /* Added content-encoding here; untested but almost identical to the + tested code in transfer.c. 08/29/02 jhrg */ +#ifdef HAVE_LIBZ + switch (conn->keep.content_encoding) { + case IDENTITY: +#endif + result = Curl_client_write(conn->data, CLIENTWRITE_BODY, datap, + piece); +#ifdef HAVE_LIBZ + break; + + case DEFLATE: + result = Curl_unencode_deflate_write(conn->data, &conn->keep, piece); + break; + + case GZIP: + case COMPRESS: + default: + failf (conn->data, + "Unrecognized content encoding type. " + "libcurl understands `identity' and `deflate' " + "content encodings."); + return CHUNKE_BAD_ENCODING; + } +#endif + if(result) return CHUNKE_WRITE_ERROR; *wrote += piece; diff --git a/lib/http_chunks.h b/lib/http_chunks.h index 48bdbd37b..482612d25 100644 --- a/lib/http_chunks.h +++ b/lib/http_chunks.h @@ -73,6 +73,7 @@ typedef enum { CHUNKE_BAD_CHUNK, CHUNKE_WRITE_ERROR, CHUNKE_STATE_ERROR, + CHUNKE_BAD_ENCODING, CHUNKE_LAST } CHUNKcode; diff --git a/lib/transfer.c b/lib/transfer.c index 20088b17d..2567ca91b 100644 --- a/lib/transfer.c +++ b/lib/transfer.c @@ -82,6 +82,8 @@ #include <curl/types.h> #include "netrc.h" +#include "content_encoding.h" /* content encoding support. 08/27/02 jhrg */ + #include "hostip.h" #include "transfer.h" #include "sendf.h" @@ -368,7 +370,7 @@ CURLcode Curl_readwrite(struct connectdata *conn, * we got: "417 Expectation Failed" this means: * we have made a HTTP call and our Expect Header * seems to cause a problem => abort the write operations - * (or prevent them from starting + * (or prevent them from starting). */ k->write_after_100_header = FALSE; k->keepon &= ~KEEP_WRITE; @@ -575,6 +577,34 @@ CURLcode Curl_readwrite(struct connectdata *conn, /* init our chunky engine */ Curl_httpchunk_init(conn); } + else if (strnequal("Content-Encoding:", k->p, 17) && + data->set.encoding) { + /* + * Process Content-Encoding. Look for the values: identity, gzip, + * defalte, compress, x-gzip and x-compress. x-gzip and + * x-compress are the same as gzip and compress. (Sec 3.5 RFC + * 2616). zlib cannot handle compress, and gzip is not currently + * implemented. However, errors are handled further down when the + * response body is processed 08/27/02 jhrg */ + char *start; + + /* Find the first non-space letter */ + for(start=k->p+18; + *start && isspace((int)*start); + start++); + + /* Record the content-encoding for later use. 08/27/02 jhrg */ + if (strnequal("identity", start, 8)) + k->content_encoding = IDENTITY; + else if (strnequal("deflate", start, 7)) + k->content_encoding = DEFLATE; + else if (strnequal("gzip", start, 4) + || strnequal("x-gzip", start, 6)) + k->content_encoding = GZIP; + else if (strnequal("compress", start, 8) + || strnequal("x-compress", start, 10)) + k->content_encoding = COMPRESS; + } else if (strnequal("Content-Range:", k->p, 14)) { if (sscanf (k->p+14, " bytes %d-", &k->offset) || sscanf (k->p+14, " bytes: %d-", &k->offset)) { @@ -737,6 +767,7 @@ CURLcode Curl_readwrite(struct connectdata *conn, * the name says read, this function both reads and writes away * the data. The returned 'nread' holds the number of actual * data it wrote to the client. */ + /* Handle chunking here? 08/27/02 jhrg */ CHUNKcode res = Curl_httpchunk_read(conn, k->str, nread, &nread); @@ -776,8 +807,39 @@ CURLcode Curl_readwrite(struct connectdata *conn, if(!conn->bits.chunk && nread) { /* If this is chunky transfer, it was already written */ - result = Curl_client_write(data, CLIENTWRITE_BODY, k->str, - nread); + + /* This switch handles various content encodings. If there's an + error here, be sure to check over the almost identical code in + http_chunk.c. 08/29/02 jhrg */ +#ifdef HAVE_LIBZ + switch (k->content_encoding) { + case IDENTITY: +#endif + /* This is the default when the server sends no + Content-Encoding header. See Curl_readwrite_init; the + memset() call initializes k->content_encoding to zero. + 08/28/02 jhrg */ + result = Curl_client_write(data, CLIENTWRITE_BODY, k->str, + nread); +#ifdef HAVE_LIBZ + break; + + case DEFLATE: + /* Assume CLIENTWRITE_BODY; headers are not encoded. */ + result = Curl_unencode_deflate_write(data, k, nread); + break; + + case GZIP: /* FIXME 08/27/02 jhrg */ + case COMPRESS: + default: + failf (data, "Unrecognized content encoding type. " + "libcurl understands `identity' and `deflate' " + "content encodings."); + result = CURLE_BAD_CONTENT_ENCODING; + break; + } +#endif + if(result) return result; } @@ -954,6 +1016,8 @@ CURLcode Curl_readwrite_init(struct connectdata *conn) struct SessionHandle *data = conn->data; struct Curl_transfer_keeper *k = &conn->keep; + /* NB: the content encoding software depends on this initialization of + Curl_transfer_keeper. 08/28/02 jhrg */ memset(k, 0, sizeof(struct Curl_transfer_keeper)); k->start = Curl_tvnow(); /* start time */ @@ -303,7 +303,7 @@ CURLcode Curl_open(struct SessionHandle **curl) data->set.ssl.verifyhost = 2; #ifdef CURL_CA_BUNDLE /* This is our prefered CA cert bundle since install time */ - data->set.ssl.CAfile = CURL_CA_BUNDLE; + data->set.ssl.CAfile = (char *)CURL_CA_BUNDLE; #endif @@ -781,6 +781,13 @@ CURLcode Curl_setopt(struct SessionHandle *data, CURLoption option, ...) */ data->set.useragent = va_arg(param, char *); break; + case CURLOPT_ENCODING: + /* + * String to use at the value of Accept-Encoding header. 08/28/02 jhrg + */ + data->set.encoding = va_arg(param, char *); + break; + case CURLOPT_USERPWD: /* * user:password to use in the operation @@ -1127,6 +1134,8 @@ CURLcode Curl_disconnect(struct connectdata *conn) free(conn->allocptr.uagent); if(conn->allocptr.userpwd) free(conn->allocptr.userpwd); + if(conn->allocptr.accept_encoding) + free(conn->allocptr.accept_encoding); if(conn->allocptr.rangeline) free(conn->allocptr.rangeline); if(conn->allocptr.ref) @@ -2715,6 +2724,12 @@ static CURLcode CreateConnection(struct SessionHandle *data, } } + if(data->set.encoding) { + if(conn->allocptr.accept_encoding) + free(conn->allocptr.accept_encoding); + conn->allocptr.accept_encoding = + aprintf("Accept-Encoding: %s\015\012", data->set.encoding); + } conn->bytecount = 0; conn->headerbytecount = 0; diff --git a/lib/urldata.h b/lib/urldata.h index 2f183711e..74a7c492e 100644 --- a/lib/urldata.h +++ b/lib/urldata.h @@ -82,6 +82,10 @@ #include "http_chunks.h" /* for the structs and enum stuff */ +#ifdef HAVE_ZLIB_H +#include <zlib.h> /* for content-encoding 08/28/02 jhrg */ +#endif + /* Download buffer size, keep it fairly big for speed reasons */ #define BUFSIZE CURL_MAX_WRITE_SIZE @@ -241,6 +245,20 @@ struct Curl_transfer_keeper { we received a 100-continue/timeout or directly */ + /* for content-encoding 08/28/02 jhrg */ + int content_encoding; /* What content encoding. sec 3.5, RFC2616. */ + +#define IDENTITY 0 /* No encoding */ +#define DEFLATE 1 /* zlib delfate [RFC 1950 & 1951] */ +#define GZIP 2 /* gzip algorithm [RFC 1952] */ +#define COMPRESS 3 /* Not handled, added for completeness */ + +#ifdef HAVE_LIBZ + bool zlib_init; /* True if zlib already initialized; + undefined if Content-Encdoing header. */ + z_stream z; /* State structure for zlib. */ +#endif + /* for the low speed checks: */ time_t timeofdoc; long bodywrites; @@ -365,6 +383,7 @@ struct connectdata { struct dynamically_allocated_data { char *proxyuserpwd; /* free later if not NULL! */ char *uagent; /* free later if not NULL! */ + char *accept_encoding; /* free later if not NULL! 08/28/02 jhrg */ char *userpwd; /* free later if not NULL! */ char *rangeline; /* free later if not NULL! */ char *ref; /* free later if not NULL! */ @@ -424,6 +443,8 @@ struct connectdata { supposed to be called, after ->curl_do() */ }; +/* The end of connectdata. 08/27/02 jhrg */ + /* * Struct to keep statistical and informational data. */ @@ -593,6 +614,7 @@ struct UserDefined { bool free_referer; /* set TRUE if 'referer' points to a string we allocated */ char *useragent; /* User-Agent string */ + char *encoding; /* Accept-Encoding string 08/28/02 jhrg */ char *postfields; /* if POST, set the fields' values here */ size_t postfieldsize; /* if POST, this might have a size to use instead of strlen(), and then the data *may* be binary (contain diff --git a/src/Makefile.am b/src/Makefile.am index 82433a659..27c4a0bcd 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -18,7 +18,7 @@ curl_SOURCES = main.c hugehelp.c urlglob.c writeout.c setup.h \ config-win32.h config-mac.h config-vms.h config-riscos.h \ urlglob.h version.h writeout.h writeenv.c writeenv.h -curl_LDADD = ../lib/libcurl.la +curl_LDADD = ../lib/libcurl.la -lz curl_DEPENDENCIES = ../lib/libcurl.la BUILT_SOURCES = hugehelp.c CLEANFILES = hugehelp.c diff --git a/src/main.c b/src/main.c index fa2b43f38..9c8172477 100644 --- a/src/main.c +++ b/src/main.c @@ -352,8 +352,10 @@ static void help(void) " --cacert <file> CA certifciate to verify peer against (SSL)\n" " --capath <directory> CA directory (made using c_rehash) to verify\n" " peer against (SSL, NOT Windows)\n" - " --ciphers <list> What SSL ciphers to use (SSL)"); + " --ciphers <list> What SSL ciphers to use (SSL)\n" + " --compressed Request a compressed response (using deflate)."); puts(" --connect-timeout <seconds> Maximum time allowed for connection\n" + " --crlf Convert LF to CRLF in upload. Useful for MVS (OS/390)\n" " -f/--fail Fail silently (no output at all) on errors (H)\n" " -F/--form <name=content> Specify HTTP POST data (H)\n" " -g/--globoff Disable URL sequences and ranges using {} and []\n" @@ -408,8 +410,7 @@ static void help(void) " -1/--tlsv1 Force usage of TLSv1 (H)\n" " -2/--sslv2 Force usage of SSLv2 (H)\n" " -3/--sslv3 Force usage of SSLv3 (H)"); - puts(" -#/--progress-bar Display transfer progress as a progress bar\n" - " --crlf Convert LF to CRLF in upload. Useful for MVS (OS/390)"); + puts(" -#/--progress-bar Display transfer progress as a progress bar"); } struct LongShort { @@ -427,6 +428,7 @@ struct Configurable { char *cookiejar; /* write to this file */ char *cookiefile; /* read from this file */ bool cookiesession; /* new session? */ + bool encoding; /* Accept-Encoding please */ bool use_resume; bool resume_from_current; bool disable_epsv; @@ -999,6 +1001,7 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */ {"5g", "trace", TRUE}, {"5h", "trace-ascii", TRUE}, {"5i", "limit-rate", TRUE}, + {"5j", "compressed", FALSE}, /* might take an arg someday */ {"0", "http1.0", FALSE}, {"1", "tlsv1", FALSE}, {"2", "sslv2", FALSE}, @@ -1213,6 +1216,11 @@ static ParameterError getparameter(char *flag, /* f or -long-flag */ config->sendpersecond = value; } break; + + case 'j': /* --compressed */ + config->encoding ^= TRUE; + break; + default: /* the URL! */ { struct getout *url; @@ -2806,6 +2814,10 @@ operate(struct Configurable *config, int argc, char *argv[]) } curl_easy_setopt(curl, CURLOPT_VERBOSE, config->conf&CONF_VERBOSE); + /* new in curl 7.10 */ + curl_easy_setopt(curl, CURLOPT_ENCODING, + (config->encoding) ? "deflate" : NULL); + res = curl_easy_perform(curl); if((config->progressmode == CURL_PROGRESS_BAR) && |