diff options
Diffstat (limited to 'src/http/modules')
-rw-r--r-- | src/http/modules/ngx_http_chunked_filter_module.c | 99 | ||||
-rw-r--r-- | src/http/modules/ngx_http_dav_module.c | 2 | ||||
-rw-r--r-- | src/http/modules/ngx_http_fastcgi_module.c | 82 | ||||
-rw-r--r-- | src/http/modules/ngx_http_gzip_filter_module.c | 4 | ||||
-rw-r--r-- | src/http/modules/ngx_http_headers_filter_module.c | 2 | ||||
-rw-r--r-- | src/http/modules/ngx_http_log_module.c | 2 | ||||
-rw-r--r-- | src/http/modules/ngx_http_memcached_module.c | 22 | ||||
-rw-r--r-- | src/http/modules/ngx_http_mp4_module.c | 184 | ||||
-rw-r--r-- | src/http/modules/ngx_http_proxy_module.c | 783 | ||||
-rw-r--r-- | src/http/modules/ngx_http_upstream_keepalive_module.c | 569 | ||||
-rw-r--r-- | src/http/modules/perl/nginx.pm | 2 |
11 files changed, 1683 insertions, 68 deletions
diff --git a/src/http/modules/ngx_http_chunked_filter_module.c b/src/http/modules/ngx_http_chunked_filter_module.c index 9c4c5de14..e6ae1c99d 100644 --- a/src/http/modules/ngx_http_chunked_filter_module.c +++ b/src/http/modules/ngx_http_chunked_filter_module.c @@ -9,6 +9,12 @@ #include <ngx_http.h> +typedef struct { + ngx_chain_t *free; + ngx_chain_t *busy; +} ngx_http_chunked_filter_ctx_t; + + static ngx_int_t ngx_http_chunked_filter_init(ngx_conf_t *cf); @@ -50,7 +56,8 @@ static ngx_http_output_body_filter_pt ngx_http_next_body_filter; static ngx_int_t ngx_http_chunked_header_filter(ngx_http_request_t *r) { - ngx_http_core_loc_conf_t *clcf; + ngx_http_core_loc_conf_t *clcf; + ngx_http_chunked_filter_ctx_t *ctx; if (r->headers_out.status == NGX_HTTP_NOT_MODIFIED || r->headers_out.status == NGX_HTTP_NO_CONTENT @@ -70,6 +77,14 @@ ngx_http_chunked_header_filter(ngx_http_request_t *r) if (clcf->chunked_transfer_encoding) { r->chunked = 1; + ctx = ngx_pcalloc(r->pool, + sizeof(ngx_http_chunked_filter_ctx_t)); + if (ctx == NULL) { + return NGX_ERROR; + } + + ngx_http_set_ctx(r, ctx, ngx_http_chunked_filter_module); + } else { r->keepalive = 0; } @@ -83,17 +98,21 @@ ngx_http_chunked_header_filter(ngx_http_request_t *r) static ngx_int_t ngx_http_chunked_body_filter(ngx_http_request_t *r, ngx_chain_t *in) { - u_char *chunk; - off_t size; - ngx_buf_t *b; - ngx_chain_t out, tail, *cl, *tl, **ll; + u_char *chunk; + off_t size; + ngx_int_t rc; + ngx_buf_t *b; + ngx_chain_t *out, *cl, *tl, **ll; + ngx_http_chunked_filter_ctx_t *ctx; if (in == NULL || !r->chunked || r->header_only) { return ngx_http_next_body_filter(r, in); } - out.buf = NULL; - ll = &out.next; + ctx = ngx_http_get_module_ctx(r, ngx_http_chunked_filter_module); + + out = NULL; + ll = &out; size = 0; cl = in; @@ -127,31 +146,46 @@ ngx_http_chunked_body_filter(ngx_http_request_t *r, ngx_chain_t *in) } if (size) { - b = ngx_calloc_buf(r->pool); - if (b == NULL) { + tl = ngx_chain_get_free_buf(r->pool, &ctx->free); + if (tl == NULL) { return NGX_ERROR; } - /* the "0000000000000000" is 64-bit hexadimal string */ + b = tl->buf; + chunk = b->start; - chunk = ngx_palloc(r->pool, sizeof("0000000000000000" CRLF) - 1); if (chunk == NULL) { - return NGX_ERROR; + /* the "0000000000000000" is 64-bit hexadecimal string */ + + chunk = ngx_palloc(r->pool, sizeof("0000000000000000" CRLF) - 1); + if (chunk == NULL) { + return NGX_ERROR; + } + + b->start = chunk; + b->end = chunk + sizeof("0000000000000000" CRLF) - 1; } + b->tag = (ngx_buf_tag_t) &ngx_http_chunked_filter_module; + b->memory = 0; b->temporary = 1; b->pos = chunk; b->last = ngx_sprintf(chunk, "%xO" CRLF, size); - out.buf = b; + tl->next = out; + out = tl; } if (cl->buf->last_buf) { - b = ngx_calloc_buf(r->pool); - if (b == NULL) { + tl = ngx_chain_get_free_buf(r->pool, &ctx->free); + if (tl == NULL) { return NGX_ERROR; } + b = tl->buf; + + b->tag = (ngx_buf_tag_t) &ngx_http_chunked_filter_module; + b->temporary = 0; b->memory = 1; b->last_buf = 1; b->pos = (u_char *) CRLF "0" CRLF CRLF; @@ -159,35 +193,38 @@ ngx_http_chunked_body_filter(ngx_http_request_t *r, ngx_chain_t *in) cl->buf->last_buf = 0; - if (size == 0) { - b->pos += 2; - out.buf = b; - out.next = NULL; - - return ngx_http_next_body_filter(r, &out); - } + *ll = tl; - } else { if (size == 0) { - *ll = NULL; - return ngx_http_next_body_filter(r, out.next); + b->pos += 2; } - b = ngx_calloc_buf(r->pool); - if (b == NULL) { + } else if (size > 0) { + tl = ngx_chain_get_free_buf(r->pool, &ctx->free); + if (tl == NULL) { return NGX_ERROR; } + b = tl->buf; + + b->tag = (ngx_buf_tag_t) &ngx_http_chunked_filter_module; + b->temporary = 0; b->memory = 1; b->pos = (u_char *) CRLF; b->last = b->pos + 2; + + *ll = tl; + + } else { + *ll = NULL; } - tail.buf = b; - tail.next = NULL; - *ll = &tail; + rc = ngx_http_next_body_filter(r, out); + + ngx_chain_update_chains(r->pool, &ctx->free, &ctx->busy, &out, + (ngx_buf_tag_t) &ngx_http_chunked_filter_module); - return ngx_http_next_body_filter(r, &out); + return rc; } diff --git a/src/http/modules/ngx_http_dav_module.c b/src/http/modules/ngx_http_dav_module.c index 0761c1654..3bd9d3e94 100644 --- a/src/http/modules/ngx_http_dav_module.c +++ b/src/http/modules/ngx_http_dav_module.c @@ -158,7 +158,7 @@ ngx_http_dav_handler(ngx_http_request_t *r) if (r->uri.data[r->uri.len - 1] == '/') { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, - "can not PUT to a collection"); + "cannot PUT to a collection"); return NGX_HTTP_CONFLICT; } diff --git a/src/http/modules/ngx_http_fastcgi_module.c b/src/http/modules/ngx_http_fastcgi_module.c index 0bc95f8da..6f639acc2 100644 --- a/src/http/modules/ngx_http_fastcgi_module.c +++ b/src/http/modules/ngx_http_fastcgi_module.c @@ -26,6 +26,8 @@ typedef struct { ngx_hash_t headers_hash; ngx_uint_t header_params; + ngx_flag_t keep_conn; + #if (NGX_HTTP_CACHE) ngx_http_complex_value_t cache_key; #endif @@ -77,6 +79,8 @@ typedef struct { #define NGX_HTTP_FASTCGI_RESPONDER 1 +#define NGX_HTTP_FASTCGI_KEEP_CONN 1 + #define NGX_HTTP_FASTCGI_BEGIN_REQUEST 1 #define NGX_HTTP_FASTCGI_ABORT_REQUEST 2 #define NGX_HTTP_FASTCGI_END_REQUEST 3 @@ -130,6 +134,7 @@ static ngx_int_t ngx_http_fastcgi_create_key(ngx_http_request_t *r); static ngx_int_t ngx_http_fastcgi_create_request(ngx_http_request_t *r); static ngx_int_t ngx_http_fastcgi_reinit_request(ngx_http_request_t *r); static ngx_int_t ngx_http_fastcgi_process_header(ngx_http_request_t *r); +static ngx_int_t ngx_http_fastcgi_input_filter_init(void *data); static ngx_int_t ngx_http_fastcgi_input_filter(ngx_event_pipe_t *p, ngx_buf_t *buf); static ngx_int_t ngx_http_fastcgi_process_record(ngx_http_request_t *r, @@ -437,6 +442,13 @@ static ngx_command_t ngx_http_fastcgi_commands[] = { offsetof(ngx_http_fastcgi_loc_conf_t, catch_stderr), NULL }, + { ngx_string("fastcgi_keep_conn"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_FLAG, + ngx_conf_set_flag_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_fastcgi_loc_conf_t, keep_conn), + NULL }, + ngx_null_command }; @@ -600,6 +612,8 @@ ngx_http_fastcgi_handler(ngx_http_request_t *r) u->pipe->input_filter = ngx_http_fastcgi_input_filter; u->pipe->input_ctx = r; + u->input_filter_init = ngx_http_fastcgi_input_filter_init; + rc = ngx_http_read_client_request_body(r, ngx_http_upstream_init); if (rc >= NGX_HTTP_SPECIAL_RESPONSE) { @@ -841,6 +855,9 @@ ngx_http_fastcgi_create_request(ngx_http_request_t *r) cl->buf = b; + ngx_http_fastcgi_request_start.br.flags = + flcf->keep_conn ? NGX_HTTP_FASTCGI_KEEP_CONN : 0; + ngx_memcpy(b->pos, &ngx_http_fastcgi_request_start, sizeof(ngx_http_fastcgi_request_start_t)); @@ -1574,14 +1591,30 @@ ngx_http_fastcgi_process_header(ngx_http_request_t *r) static ngx_int_t +ngx_http_fastcgi_input_filter_init(void *data) +{ + ngx_http_request_t *r = data; + ngx_http_fastcgi_loc_conf_t *flcf; + + flcf = ngx_http_get_module_loc_conf(r, ngx_http_fastcgi_module); + + r->upstream->pipe->length = flcf->keep_conn ? + (off_t) sizeof(ngx_http_fastcgi_header_t) : -1; + + return NGX_OK; +} + + +static ngx_int_t ngx_http_fastcgi_input_filter(ngx_event_pipe_t *p, ngx_buf_t *buf) { - u_char *m, *msg; - ngx_int_t rc; - ngx_buf_t *b, **prev; - ngx_chain_t *cl; - ngx_http_request_t *r; - ngx_http_fastcgi_ctx_t *f; + u_char *m, *msg; + ngx_int_t rc; + ngx_buf_t *b, **prev; + ngx_chain_t *cl; + ngx_http_request_t *r; + ngx_http_fastcgi_ctx_t *f; + ngx_http_fastcgi_loc_conf_t *flcf; if (buf->pos == buf->last) { return NGX_OK; @@ -1589,6 +1622,7 @@ ngx_http_fastcgi_input_filter(ngx_event_pipe_t *p, ngx_buf_t *buf) r = p->input_ctx; f = ngx_http_get_module_ctx(r, ngx_http_fastcgi_module); + flcf = ngx_http_get_module_loc_conf(r, ngx_http_fastcgi_module); b = NULL; prev = &buf->shadow; @@ -1611,7 +1645,10 @@ ngx_http_fastcgi_input_filter(ngx_event_pipe_t *p, ngx_buf_t *buf) if (f->type == NGX_HTTP_FASTCGI_STDOUT && f->length == 0) { f->state = ngx_http_fastcgi_st_version; - p->upstream_done = 1; + + if (!flcf->keep_conn) { + p->upstream_done = 1; + } ngx_log_debug0(NGX_LOG_DEBUG_HTTP, p->log, 0, "http fastcgi closed stdout"); @@ -1623,6 +1660,10 @@ ngx_http_fastcgi_input_filter(ngx_event_pipe_t *p, ngx_buf_t *buf) f->state = ngx_http_fastcgi_st_version; p->upstream_done = 1; + if (flcf->keep_conn) { + r->upstream->keepalive = 1; + } + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, p->log, 0, "http fastcgi sent end request"); @@ -1703,8 +1744,10 @@ ngx_http_fastcgi_input_filter(ngx_event_pipe_t *p, ngx_buf_t *buf) } if (p->free) { - b = p->free->buf; - p->free = p->free->next; + cl = p->free; + b = cl->buf; + p->free = cl->next; + ngx_free_chain(p->pool, cl); } else { b = ngx_alloc_buf(p->pool); @@ -1781,6 +1824,23 @@ ngx_http_fastcgi_input_filter(ngx_event_pipe_t *p, ngx_buf_t *buf) } + if (flcf->keep_conn) { + + /* set p->length, minimal amount of data we want to see */ + + if (f->state < ngx_http_fastcgi_st_data) { + p->length = 1; + + } else if (f->state == ngx_http_fastcgi_st_padding) { + p->length = f->padding; + + } else { + /* ngx_http_fastcgi_st_data */ + + p->length = f->length; + } + } + if (b) { b->shadow = buf; b->last_shadow = 1; @@ -2011,6 +2071,8 @@ ngx_http_fastcgi_create_loc_conf(ngx_conf_t *cf) conf->catch_stderr = NGX_CONF_UNSET_PTR; + conf->keep_conn = NGX_CONF_UNSET; + ngx_str_set(&conf->upstream.module, "fastcgi"); return conf; @@ -2254,6 +2316,8 @@ ngx_http_fastcgi_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) ngx_conf_merge_ptr_value(conf->catch_stderr, prev->catch_stderr, NULL); + ngx_conf_merge_value(conf->keep_conn, prev->keep_conn, 0); + ngx_conf_merge_str_value(conf->index, prev->index, ""); diff --git a/src/http/modules/ngx_http_gzip_filter_module.c b/src/http/modules/ngx_http_gzip_filter_module.c index d624e36ff..18824059a 100644 --- a/src/http/modules/ngx_http_gzip_filter_module.c +++ b/src/http/modules/ngx_http_gzip_filter_module.c @@ -378,7 +378,7 @@ ngx_http_gzip_body_filter(ngx_http_request_t *r, ngx_chain_t *in) cl = NULL; - ngx_chain_update_chains(&ctx->free, &ctx->busy, &cl, + ngx_chain_update_chains(r->pool, &ctx->free, &ctx->busy, &cl, (ngx_buf_tag_t) &ngx_http_gzip_filter_module); ctx->nomem = 0; } @@ -448,7 +448,7 @@ ngx_http_gzip_body_filter(ngx_http_request_t *r, ngx_chain_t *in) ngx_http_gzip_filter_free_copy_buf(r, ctx); - ngx_chain_update_chains(&ctx->free, &ctx->busy, &ctx->out, + ngx_chain_update_chains(r->pool, &ctx->free, &ctx->busy, &ctx->out, (ngx_buf_tag_t) &ngx_http_gzip_filter_module); ctx->last_out = &ctx->out; diff --git a/src/http/modules/ngx_http_headers_filter_module.c b/src/http/modules/ngx_http_headers_filter_module.c index 1d409582c..c3f74ae19 100644 --- a/src/http/modules/ngx_http_headers_filter_module.c +++ b/src/http/modules/ngx_http_headers_filter_module.c @@ -507,7 +507,7 @@ ngx_http_headers_expires(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) minus = 0; if (hcf->expires == NGX_HTTP_EXPIRES_MODIFIED) { - return "daily time can not be used with \"modified\" parameter"; + return "daily time cannot be used with \"modified\" parameter"; } hcf->expires = NGX_HTTP_EXPIRES_DAILY; diff --git a/src/http/modules/ngx_http_log_module.c b/src/http/modules/ngx_http_log_module.c index b2bfbf477..9728d5411 100644 --- a/src/http/modules/ngx_http_log_module.c +++ b/src/http/modules/ngx_http_log_module.c @@ -960,7 +960,7 @@ buffer: if (log->script) { ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, - "buffered logs can not have variables in name"); + "buffered logs cannot have variables in name"); return NGX_CONF_ERROR; } diff --git a/src/http/modules/ngx_http_memcached_module.c b/src/http/modules/ngx_http_memcached_module.c index 745f73aa5..dbe0fe838 100644 --- a/src/http/modules/ngx_http_memcached_module.c +++ b/src/http/modules/ngx_http_memcached_module.c @@ -344,8 +344,8 @@ found: while (*p && *p++ != CR) { /* void */ } - r->headers_out.content_length_n = ngx_atoof(len, p - len - 1); - if (r->headers_out.content_length_n == -1) { + u->headers_in.content_length_n = ngx_atoof(len, p - len - 1); + if (u->headers_in.content_length_n == -1) { ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, "memcached sent invalid length in response \"%V\" " "for key \"%V\"", @@ -366,6 +366,7 @@ found: u->headers_in.status_n = 404; u->state->status = 404; + u->keepalive = 1; return NGX_OK; } @@ -407,7 +408,7 @@ ngx_http_memcached_filter(void *data, ssize_t bytes) u = ctx->request->upstream; b = &u->buffer; - if (u->length == ctx->rest) { + if (u->length == (ssize_t) ctx->rest) { if (ngx_strncmp(b->last, ngx_http_memcached_end + NGX_HTTP_MEMCACHED_END - ctx->rest, @@ -426,6 +427,10 @@ ngx_http_memcached_filter(void *data, ssize_t bytes) u->length -= bytes; ctx->rest -= bytes; + if (u->length == 0) { + u->keepalive = 1; + } + return NGX_OK; } @@ -463,6 +468,13 @@ ngx_http_memcached_filter(void *data, ssize_t bytes) if (ngx_strncmp(last, ngx_http_memcached_end, b->last - last) != 0) { ngx_log_error(NGX_LOG_ERR, ctx->request->connection->log, 0, "memcached sent invalid trailer"); + + b->last = last; + cl->buf->last = last; + u->length = 0; + ctx->rest = 0; + + return NGX_OK; } ctx->rest -= b->last - last; @@ -470,6 +482,10 @@ ngx_http_memcached_filter(void *data, ssize_t bytes) cl->buf->last = last; u->length = ctx->rest; + if (u->length == 0) { + u->keepalive = 1; + } + return NGX_OK; } diff --git a/src/http/modules/ngx_http_mp4_module.c b/src/http/modules/ngx_http_mp4_module.c index fc21ec410..9a3362e8d 100644 --- a/src/http/modules/ngx_http_mp4_module.c +++ b/src/http/modules/ngx_http_mp4_module.c @@ -32,8 +32,10 @@ #define NGX_HTTP_MP4_STSZ_DATA 21 #define NGX_HTTP_MP4_STCO_ATOM 22 #define NGX_HTTP_MP4_STCO_DATA 23 +#define NGX_HTTP_MP4_CO64_ATOM 24 +#define NGX_HTTP_MP4_CO64_DATA 25 -#define NGX_HTTP_MP4_LAST_ATOM NGX_HTTP_MP4_STCO_DATA +#define NGX_HTTP_MP4_LAST_ATOM NGX_HTTP_MP4_CO64_DATA typedef struct { @@ -61,7 +63,7 @@ typedef struct { ngx_uint_t start_sample; ngx_uint_t start_chunk; ngx_uint_t chunk_samples; - ngx_uint_t chunk_samples_size; + uint64_t chunk_samples_size; off_t start_offset; size_t tkhd_size; @@ -96,8 +98,10 @@ typedef struct { ngx_buf_t stsc_data_buf; ngx_buf_t stsz_atom_buf; ngx_buf_t stsz_data_buf; - ngx_buf_t tsco_atom_buf; - ngx_buf_t tsco_data_buf; + ngx_buf_t stco_atom_buf; + ngx_buf_t stco_data_buf; + ngx_buf_t co64_atom_buf; + ngx_buf_t co64_data_buf; ngx_mp4_stsc_entry_t stsc_chunk_entry; } ngx_http_mp4_trak_t; @@ -268,6 +272,12 @@ static ngx_int_t ngx_http_mp4_update_stco_atom(ngx_http_mp4_file_t *mp4, ngx_http_mp4_trak_t *trak); static void ngx_http_mp4_adjust_stco_atom(ngx_http_mp4_file_t *mp4, ngx_http_mp4_trak_t *trak, int32_t adjustment); +static ngx_int_t ngx_http_mp4_read_co64_atom(ngx_http_mp4_file_t *mp4, + uint64_t atom_data_size); +static ngx_int_t ngx_http_mp4_update_co64_atom(ngx_http_mp4_file_t *mp4, + ngx_http_mp4_trak_t *trak); +static void ngx_http_mp4_adjust_co64_atom(ngx_http_mp4_file_t *mp4, + ngx_http_mp4_trak_t *trak, off_t adjustment); static char *ngx_http_mp4(ngx_conf_t *cf, ngx_command_t *cmd, void *conf); static void *ngx_http_mp4_create_conf(ngx_conf_t *cf); static char *ngx_http_mp4_merge_conf(ngx_conf_t *cf, void *parent, void *child); @@ -373,6 +383,7 @@ static ngx_http_mp4_atom_handler_t ngx_http_mp4_stbl_atoms[] = { { "stsc", ngx_http_mp4_read_stsc_atom }, { "stsz", ngx_http_mp4_read_stsz_atom }, { "stco", ngx_http_mp4_read_stco_atom }, + { "co64", ngx_http_mp4_read_co64_atom }, { NULL, NULL } }; @@ -676,8 +687,15 @@ ngx_http_mp4_process(ngx_http_mp4_file_t *mp4) ngx_http_mp4_update_stsz_atom(mp4, &trak[i]); - if (ngx_http_mp4_update_stco_atom(mp4, &trak[i]) != NGX_OK) { - return NGX_ERROR; + if (trak[i].out[NGX_HTTP_MP4_CO64_DATA].buf) { + if (ngx_http_mp4_update_co64_atom(mp4, &trak[i]) != NGX_OK) { + return NGX_ERROR; + } + + } else { + if (ngx_http_mp4_update_stco_atom(mp4, &trak[i]) != NGX_OK) { + return NGX_ERROR; + } } ngx_http_mp4_update_stbl_atom(mp4, &trak[i]); @@ -721,7 +739,11 @@ ngx_http_mp4_process(ngx_http_mp4_file_t *mp4) "mp4 adjustment:%D", adjustment); for (i = 0; i < mp4->trak.nelts; i++) { - ngx_http_mp4_adjust_stco_atom(mp4, &trak[i], (int32_t) adjustment); + if (trak[i].out[NGX_HTTP_MP4_CO64_DATA].buf) { + ngx_http_mp4_adjust_co64_atom(mp4, &trak[i], adjustment); + } else { + ngx_http_mp4_adjust_stco_atom(mp4, &trak[i], (int32_t) adjustment); + } } return NGX_OK; @@ -1702,7 +1724,6 @@ ngx_http_mp4_read_stsd_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) { u_char *atom_header, *atom_table; size_t atom_size; - uint32_t entries; ngx_buf_t *atom; ngx_mp4_stsd_atom_t *stsd_atom; ngx_http_mp4_trak_t *trak; @@ -1718,10 +1739,9 @@ ngx_http_mp4_read_stsd_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) ngx_mp4_set_32value(stsd_atom->size, atom_size); ngx_mp4_set_atom_name(stsd_atom, 's', 't', 's', 'd'); - entries = ngx_mp4_get_32value(stsd_atom->entries); - if ((uint64_t) (sizeof(ngx_mp4_stsd_atom_t) - sizeof(ngx_mp4_atom_header_t)) - > atom_data_size) { + > atom_data_size) + { ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, "\"%s\" mp4 stsd atom too large", mp4->file.name.data); @@ -2497,7 +2517,7 @@ ngx_http_mp4_update_stsz_atom(ngx_http_mp4_file_t *mp4, } ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, - "chunk samples sizes:%uD", trak->chunk_samples_size); + "chunk samples sizes:%uL", trak->chunk_samples_size); atom_size = sizeof(ngx_mp4_stsz_atom_t) + (data->last - data->pos); trak->size += atom_size; @@ -2554,12 +2574,12 @@ ngx_http_mp4_read_stco_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) trak = ngx_mp4_last_trak(mp4); trak->chunks = entries; - atom = &trak->tsco_atom_buf; + atom = &trak->stco_atom_buf; atom->temporary = 1; atom->pos = atom_header; atom->last = atom_table; - data = &trak->tsco_data_buf; + data = &trak->stco_data_buf; data->temporary = 1; data->pos = atom_table; data->last = atom_end; @@ -2648,6 +2668,142 @@ ngx_http_mp4_adjust_stco_atom(ngx_http_mp4_file_t *mp4, } +typedef struct { + u_char size[4]; + u_char name[4]; + u_char version[1]; + u_char flags[3]; + u_char entries[4]; +} ngx_mp4_co64_atom_t; + + +static ngx_int_t +ngx_http_mp4_read_co64_atom(ngx_http_mp4_file_t *mp4, uint64_t atom_data_size) +{ + u_char *atom_header, *atom_table, *atom_end; + uint32_t entries; + ngx_buf_t *atom, *data; + ngx_mp4_co64_atom_t *co64_atom; + ngx_http_mp4_trak_t *trak; + + /* chunk offsets atom */ + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, "mp4 co64 atom"); + + atom_header = ngx_mp4_atom_header(mp4); + co64_atom = (ngx_mp4_co64_atom_t *) atom_header; + ngx_mp4_set_atom_name(co64_atom, 'c', 'o', '6', '4'); + + entries = ngx_mp4_get_32value(co64_atom->entries); + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, "chunks:%uD", entries); + + atom_table = atom_header + sizeof(ngx_mp4_co64_atom_t); + atom_end = atom_table + entries * sizeof(uint64_t); + + if ((uint64_t) (atom_end - co64_atom->version) > atom_data_size) { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "\"%s\" mp4 co64 atom too large", mp4->file.name.data); + return NGX_ERROR; + } + + trak = ngx_mp4_last_trak(mp4); + trak->chunks = entries; + + atom = &trak->co64_atom_buf; + atom->temporary = 1; + atom->pos = atom_header; + atom->last = atom_table; + + data = &trak->co64_data_buf; + data->temporary = 1; + data->pos = atom_table; + data->last = atom_end; + + trak->out[NGX_HTTP_MP4_CO64_ATOM].buf = atom; + trak->out[NGX_HTTP_MP4_CO64_DATA].buf = data; + + ngx_mp4_atom_next(mp4, atom_data_size); + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_mp4_update_co64_atom(ngx_http_mp4_file_t *mp4, + ngx_http_mp4_trak_t *trak) +{ + size_t atom_size; + ngx_buf_t *atom, *data; + ngx_mp4_co64_atom_t *co64_atom; + + /* + * mdia.minf.stbl.co64 updating requires trak->start_chunk + * from mdia.minf.stbl.stsc which depends on value from mdia.mdhd + * atom which may reside after mdia.minf + */ + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, + "mp4 co64 atom update"); + + data = trak->out[NGX_HTTP_MP4_CO64_DATA].buf; + + if (data == NULL) { + ngx_log_error(NGX_LOG_ERR, mp4->file.log, 0, + "no mp4 co64 atoms were found in \"%s\"", + mp4->file.name.data); + return NGX_ERROR; + } + + data->pos += trak->start_chunk * sizeof(uint64_t); + atom_size = sizeof(ngx_mp4_co64_atom_t) + (data->last - data->pos); + trak->size += atom_size; + + trak->start_offset = ngx_mp4_get_64value(data->pos); + trak->start_offset += trak->chunk_samples_size; + ngx_mp4_set_64value(data->pos, trak->start_offset); + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, + "start chunk offset:%uL", trak->start_offset); + + atom = trak->out[NGX_HTTP_MP4_CO64_ATOM].buf; + co64_atom = (ngx_mp4_co64_atom_t *) atom->pos; + + ngx_mp4_set_32value(co64_atom->size, atom_size); + ngx_mp4_set_32value(co64_atom->entries, trak->chunks - trak->start_chunk); + + return NGX_OK; +} + + +static void +ngx_http_mp4_adjust_co64_atom(ngx_http_mp4_file_t *mp4, + ngx_http_mp4_trak_t *trak, off_t adjustment) +{ + uint64_t offset, *entry, *end; + ngx_buf_t *data; + + /* + * moov.trak.mdia.minf.stbl.co64 adjustment requires + * minimal start offset of all traks and new moov atom size + */ + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, mp4->file.log, 0, + "mp4 co64 atom adjustment"); + + data = trak->out[NGX_HTTP_MP4_CO64_DATA].buf; + entry = (uint64_t *) data->pos; + end = (uint64_t *) data->last; + + while (entry < end) { + offset = ngx_mp4_get_64value(entry); + offset += adjustment; + ngx_mp4_set_64value(entry, offset); + entry++; + } +} + + static char * ngx_http_mp4(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) { diff --git a/src/http/modules/ngx_http_proxy_module.c b/src/http/modules/ngx_http_proxy_module.c index 902cfb8f2..88ee42472 100644 --- a/src/http/modules/ngx_http_proxy_module.c +++ b/src/http/modules/ngx_http_proxy_module.c @@ -71,6 +71,8 @@ typedef struct { ngx_flag_t redirect; + ngx_uint_t http_version; + ngx_uint_t headers_hash_max_size; ngx_uint_t headers_hash_bucket_size; } ngx_http_proxy_loc_conf_t; @@ -80,6 +82,12 @@ typedef struct { ngx_http_status_t status; ngx_http_proxy_vars_t vars; size_t internal_body_length; + + ngx_uint_t state; + off_t size; + off_t length; + + ngx_uint_t head; /* unsigned head:1 */ } ngx_http_proxy_ctx_t; @@ -92,6 +100,15 @@ static ngx_int_t ngx_http_proxy_create_request(ngx_http_request_t *r); static ngx_int_t ngx_http_proxy_reinit_request(ngx_http_request_t *r); static ngx_int_t ngx_http_proxy_process_status_line(ngx_http_request_t *r); static ngx_int_t ngx_http_proxy_process_header(ngx_http_request_t *r); +static ngx_int_t ngx_http_proxy_input_filter_init(void *data); +static ngx_int_t ngx_http_proxy_copy_filter(ngx_event_pipe_t *p, + ngx_buf_t *buf); +static ngx_int_t ngx_http_proxy_chunked_filter(ngx_event_pipe_t *p, + ngx_buf_t *buf); +static ngx_int_t ngx_http_proxy_non_buffered_copy_filter(void *data, + ssize_t bytes); +static ngx_int_t ngx_http_proxy_non_buffered_chunked_filter(void *data, + ssize_t bytes); static void ngx_http_proxy_abort_request(ngx_http_request_t *r); static void ngx_http_proxy_finalize_request(ngx_http_request_t *r, ngx_int_t rc); @@ -157,6 +174,13 @@ static ngx_conf_bitmask_t ngx_http_proxy_next_upstream_masks[] = { }; +static ngx_conf_enum_t ngx_http_proxy_http_version[] = { + { ngx_string("1.0"), NGX_HTTP_VERSION_10 }, + { ngx_string("1.1"), NGX_HTTP_VERSION_11 }, + { ngx_null_string, 0 } +}; + + ngx_module_t ngx_http_proxy_module; @@ -432,6 +456,13 @@ static ngx_command_t ngx_http_proxy_commands[] = { offsetof(ngx_http_proxy_loc_conf_t, upstream.ignore_headers), &ngx_http_upstream_ignore_headers_masks }, + { ngx_string("proxy_http_version"), + NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_CONF_TAKE1, + ngx_conf_set_enum_slot, + NGX_HTTP_LOC_CONF_OFFSET, + offsetof(ngx_http_proxy_loc_conf_t, http_version), + &ngx_http_proxy_http_version }, + #if (NGX_HTTP_SSL) { ngx_string("proxy_ssl_session_reuse"), @@ -479,6 +510,7 @@ ngx_module_t ngx_http_proxy_module = { static char ngx_http_proxy_version[] = " HTTP/1.0" CRLF; +static char ngx_http_proxy_version_11[] = " HTTP/1.1" CRLF; static ngx_keyval_t ngx_http_proxy_headers[] = { @@ -486,6 +518,7 @@ static ngx_keyval_t ngx_http_proxy_headers[] = { { ngx_string("Connection"), ngx_string("close") }, { ngx_string("Keep-Alive"), ngx_string("") }, { ngx_string("Expect"), ngx_string("") }, + { ngx_string("Upgrade"), ngx_string("") }, { ngx_null_string, ngx_null_string } }; @@ -610,7 +643,12 @@ ngx_http_proxy_handler(ngx_http_request_t *r) return NGX_HTTP_INTERNAL_SERVER_ERROR; } - u->pipe->input_filter = ngx_event_pipe_copy_input_filter; + u->pipe->input_filter = ngx_http_proxy_copy_filter; + u->pipe->input_ctx = r; + + u->input_filter_init = ngx_http_proxy_input_filter_init; + u->input_filter = ngx_http_proxy_non_buffered_copy_filter; + u->input_filter_ctx = r; u->accel = 1; @@ -866,14 +904,20 @@ ngx_http_proxy_create_request(ngx_http_request_t *r) method.len++; } + ctx = ngx_http_get_module_ctx(r, ngx_http_proxy_module); + + if (method.len == 5 + && ngx_strncasecmp(method.data, (u_char *) "HEAD ", 5) == 0) + { + ctx->head = 1; + } + len = method.len + sizeof(ngx_http_proxy_version) - 1 + sizeof(CRLF) - 1; escape = 0; loc_len = 0; unparsed_uri = 0; - ctx = ngx_http_get_module_ctx(r, ngx_http_proxy_module); - if (plcf->proxy_lengths) { uri_len = ctx->vars.uri.len; @@ -1009,8 +1053,14 @@ ngx_http_proxy_create_request(ngx_http_request_t *r) u->uri.len = b->last - u->uri.data; - b->last = ngx_cpymem(b->last, ngx_http_proxy_version, - sizeof(ngx_http_proxy_version) - 1); + if (plcf->http_version == NGX_HTTP_VERSION_11) { + b->last = ngx_cpymem(b->last, ngx_http_proxy_version_11, + sizeof(ngx_http_proxy_version_11) - 1); + + } else { + b->last = ngx_cpymem(b->last, ngx_http_proxy_version, + sizeof(ngx_http_proxy_version) - 1); + } ngx_memzero(&e, sizeof(ngx_http_script_engine_t)); @@ -1158,8 +1208,11 @@ ngx_http_proxy_reinit_request(ngx_http_request_t *r) ctx->status.count = 0; ctx->status.start = NULL; ctx->status.end = NULL; + ctx->state = 0; r->upstream->process_header = ngx_http_proxy_process_status_line; + r->upstream->pipe->input_filter = ngx_http_proxy_copy_filter; + r->upstream->input_filter = ngx_http_proxy_non_buffered_copy_filter; r->state = 0; return NGX_OK; @@ -1210,6 +1263,7 @@ ngx_http_proxy_process_status_line(ngx_http_request_t *r) r->http_version = NGX_HTTP_VERSION_9; u->state->status = NGX_HTTP_OK; + u->headers_in.connection_close = 1; return NGX_OK; } @@ -1234,6 +1288,10 @@ ngx_http_proxy_process_status_line(ngx_http_request_t *r) "http proxy status %ui \"%V\"", u->headers_in.status_n, &u->headers_in.status_line); + if (ctx->status.http_version < NGX_HTTP_VERSION_11) { + u->headers_in.connection_close = 1; + } + u->process_header = ngx_http_proxy_process_header; return ngx_http_proxy_process_header(r); @@ -1245,6 +1303,8 @@ ngx_http_proxy_process_header(ngx_http_request_t *r) { ngx_int_t rc; ngx_table_elt_t *h; + ngx_http_upstream_t *u; + ngx_http_proxy_ctx_t *ctx; ngx_http_upstream_header_t *hh; ngx_http_upstream_main_conf_t *umcf; @@ -1340,6 +1400,30 @@ ngx_http_proxy_process_header(ngx_http_request_t *r) h->lowcase_key = (u_char *) "date"; } + /* clear content length if response is chunked */ + + u = r->upstream; + + if (u->headers_in.chunked) { + u->headers_in.content_length_n = -1; + } + + /* + * set u->keepalive if response has no body; this allows to keep + * connections alive in case of r->header_only or X-Accel-Redirect + */ + + ctx = ngx_http_get_module_ctx(r, ngx_http_proxy_module); + + if (u->headers_in.status_n == NGX_HTTP_NO_CONTENT + || u->headers_in.status_n == NGX_HTTP_NOT_MODIFIED + || ctx->head + || (!u->headers_in.chunked + && u->headers_in.content_length_n == 0)) + { + u->keepalive = !u->headers_in.connection_close; + } + return NGX_OK; } @@ -1357,6 +1441,690 @@ ngx_http_proxy_process_header(ngx_http_request_t *r) } +static ngx_int_t +ngx_http_proxy_input_filter_init(void *data) +{ + ngx_http_request_t *r = data; + ngx_http_upstream_t *u; + ngx_http_proxy_ctx_t *ctx; + + u = r->upstream; + ctx = ngx_http_get_module_ctx(r, ngx_http_proxy_module); + + ngx_log_debug4(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http proxy filter init s:%d h:%d c:%d l:%O", + u->headers_in.status_n, ctx->head, u->headers_in.chunked, + u->headers_in.content_length_n); + + /* as per RFC2616, 4.4 Message Length */ + + if (u->headers_in.status_n == NGX_HTTP_NO_CONTENT + || u->headers_in.status_n == NGX_HTTP_NOT_MODIFIED + || ctx->head) + { + /* 1xx, 204, and 304 and replies to HEAD requests */ + /* no 1xx since we don't send Expect and Upgrade */ + + u->pipe->length = 0; + u->length = 0; + u->keepalive = !u->headers_in.connection_close; + + } else if (u->headers_in.chunked) { + /* chunked */ + + u->pipe->input_filter = ngx_http_proxy_chunked_filter; + u->pipe->length = 3; /* "0" LF LF */ + + u->input_filter = ngx_http_proxy_non_buffered_chunked_filter; + u->length = -1; + + } else if (u->headers_in.content_length_n == 0) { + /* empty body: special case as filter won't be called */ + + u->pipe->length = 0; + u->length = 0; + u->keepalive = !u->headers_in.connection_close; + + } else { + /* content length or connection close */ + + u->pipe->length = u->headers_in.content_length_n; + u->length = u->headers_in.content_length_n; + } + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_proxy_copy_filter(ngx_event_pipe_t *p, ngx_buf_t *buf) +{ + ngx_buf_t *b; + ngx_chain_t *cl; + ngx_http_request_t *r; + + if (buf->pos == buf->last) { + return NGX_OK; + } + + if (p->free) { + cl = p->free; + b = cl->buf; + p->free = cl->next; + ngx_free_chain(p->pool, cl); + + } else { + b = ngx_alloc_buf(p->pool); + if (b == NULL) { + return NGX_ERROR; + } + } + + ngx_memcpy(b, buf, sizeof(ngx_buf_t)); + b->shadow = buf; + b->tag = p->tag; + b->last_shadow = 1; + b->recycled = 1; + buf->shadow = b; + + cl = ngx_alloc_chain_link(p->pool); + if (cl == NULL) { + return NGX_ERROR; + } + + cl->buf = b; + cl->next = NULL; + + ngx_log_debug1(NGX_LOG_DEBUG_EVENT, p->log, 0, "input buf #%d", b->num); + + if (p->in) { + *p->last_in = cl; + } else { + p->in = cl; + } + p->last_in = &cl->next; + + if (p->length == -1) { + return NGX_OK; + } + + p->length -= b->last - b->pos; + + if (p->length == 0) { + r = p->input_ctx; + p->upstream_done = 1; + r->upstream->keepalive = !r->upstream->headers_in.connection_close; + + } else if (p->length < 0) { + r = p->input_ctx; + p->upstream_done = 1; + + ngx_log_error(NGX_LOG_ERR, r->connection->log, 0, + "upstream sent too many data"); + } + + return NGX_OK; +} + + +static ngx_inline ngx_int_t +ngx_http_proxy_parse_chunked(ngx_http_request_t *r, ngx_buf_t *buf) +{ + u_char *pos, ch, c; + ngx_int_t rc; + ngx_http_proxy_ctx_t *ctx; + enum { + sw_chunk_start = 0, + sw_chunk_size, + sw_chunk_extension, + sw_chunk_extension_almost_done, + sw_chunk_data, + sw_after_data, + sw_after_data_almost_done, + sw_last_chunk_extension, + sw_last_chunk_extension_almost_done, + sw_trailer, + sw_trailer_almost_done, + sw_trailer_header, + sw_trailer_header_almost_done + } state; + + ctx = ngx_http_get_module_ctx(r, ngx_http_proxy_module); + state = ctx->state; + + if (state == sw_chunk_data && ctx->size == 0) { + state = sw_after_data; + } + + rc = NGX_AGAIN; + + for (pos = buf->pos; pos < buf->last; pos++) { + + ch = *pos; + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http proxy chunked byte: %02Xd s:%d", ch, state); + + switch (state) { + + case sw_chunk_start: + if (ch >= '0' && ch <= '9') { + state = sw_chunk_size; + ctx->size = ch - '0'; + break; + } + + c = (u_char) (ch | 0x20); + + if (c >= 'a' && c <= 'f') { + state = sw_chunk_size; + ctx->size = c - 'a' + 10; + break; + } + + goto invalid; + + case sw_chunk_size: + if (ch >= '0' && ch <= '9') { + ctx->size = ctx->size * 16 + (ch - '0'); + break; + } + + c = (u_char) (ch | 0x20); + + if (c >= 'a' && c <= 'f') { + ctx->size = ctx->size * 16 + (c - 'a' + 10); + break; + } + + if (ctx->size == 0) { + + switch (ch) { + case CR: + state = sw_last_chunk_extension_almost_done; + break; + case LF: + state = sw_trailer; + break; + case ';': + state = sw_last_chunk_extension; + break; + default: + goto invalid; + } + + break; + } + + switch (ch) { + case CR: + state = sw_chunk_extension_almost_done; + break; + case LF: + state = sw_chunk_data; + break; + case ';': + state = sw_chunk_extension; + break; + default: + goto invalid; + } + + break; + + case sw_chunk_extension: + switch (ch) { + case CR: + state = sw_chunk_extension_almost_done; + break; + case LF: + state = sw_chunk_data; + } + break; + + case sw_chunk_extension_almost_done: + if (ch == LF) { + state = sw_chunk_data; + break; + } + goto invalid; + + case sw_chunk_data: + rc = NGX_OK; + goto data; + + case sw_after_data: + switch (ch) { + case CR: + state = sw_after_data_almost_done; + break; + case LF: + state = sw_chunk_start; + } + break; + + case sw_after_data_almost_done: + if (ch == LF) { + state = sw_chunk_start; + break; + } + goto invalid; + + case sw_last_chunk_extension: + switch (ch) { + case CR: + state = sw_last_chunk_extension_almost_done; + break; + case LF: + state = sw_trailer; + } + break; + + case sw_last_chunk_extension_almost_done: + if (ch == LF) { + state = sw_trailer; + break; + } + goto invalid; + + case sw_trailer: + switch (ch) { + case CR: + state = sw_trailer_almost_done; + break; + case LF: + goto done; + default: + state = sw_trailer_header; + } + break; + + case sw_trailer_almost_done: + if (ch == LF) { + goto done; + } + goto invalid; + + case sw_trailer_header: + switch (ch) { + case CR: + state = sw_trailer_header_almost_done; + break; + case LF: + state = sw_trailer; + } + break; + + case sw_trailer_header_almost_done: + if (ch == LF) { + state = sw_trailer; + break; + } + goto invalid; + + } + } + +data: + + ctx->state = state; + buf->pos = pos; + + switch (state) { + + case sw_chunk_start: + ctx->length = 3 /* "0" LF LF */; + break; + case sw_chunk_size: + ctx->length = 2 /* LF LF */ + + (ctx->size ? ctx->size + 4 /* LF "0" LF LF */ : 0); + break; + case sw_chunk_extension: + case sw_chunk_extension_almost_done: + ctx->length = 1 /* LF */ + ctx->size + 4 /* LF "0" LF LF */; + break; + case sw_chunk_data: + ctx->length = ctx->size + 4 /* LF "0" LF LF */; + break; + case sw_after_data: + case sw_after_data_almost_done: + ctx->length = 4 /* LF "0" LF LF */; + break; + case sw_last_chunk_extension: + case sw_last_chunk_extension_almost_done: + ctx->length = 2 /* LF LF */; + break; + case sw_trailer: + case sw_trailer_almost_done: + ctx->length = 1 /* LF */; + break; + case sw_trailer_header: + case sw_trailer_header_almost_done: + ctx->length = 2 /* LF LF */; + break; + + } + + return rc; + +done: + + return NGX_DONE; + +invalid: + + ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, + "upstream sent invalid chunked response"); + + return NGX_ERROR; +} + + +static ngx_int_t +ngx_http_proxy_chunked_filter(ngx_event_pipe_t *p, ngx_buf_t *buf) +{ + ngx_int_t rc; + ngx_buf_t *b, **prev; + ngx_chain_t *cl; + ngx_http_request_t *r; + ngx_http_proxy_ctx_t *ctx; + + if (buf->pos == buf->last) { + return NGX_OK; + } + + r = p->input_ctx; + ctx = ngx_http_get_module_ctx(r, ngx_http_proxy_module); + + b = NULL; + prev = &buf->shadow; + + for ( ;; ) { + + rc = ngx_http_proxy_parse_chunked(r, buf); + + if (rc == NGX_OK) { + + /* a chunk has been parsed successfully */ + + if (p->free) { + cl = p->free; + b = cl->buf; + p->free = cl->next; + ngx_free_chain(p->pool, cl); + + } else { + b = ngx_alloc_buf(p->pool); + if (b == NULL) { + return NGX_ERROR; + } + } + + ngx_memzero(b, sizeof(ngx_buf_t)); + + b->pos = buf->pos; + b->start = buf->start; + b->end = buf->end; + b->tag = p->tag; + b->temporary = 1; + b->recycled = 1; + + *prev = b; + prev = &b->shadow; + + cl = ngx_alloc_chain_link(p->pool); + if (cl == NULL) { + return NGX_ERROR; + } + + cl->buf = b; + cl->next = NULL; + + if (p->in) { + *p->last_in = cl; + } else { + p->in = cl; + } + p->last_in = &cl->next; + + /* STUB */ b->num = buf->num; + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, p->log, 0, + "input buf #%d %p", b->num, b->pos); + + if (buf->last - buf->pos >= ctx->size) { + + buf->pos += ctx->size; + b->last = buf->pos; + ctx->size = 0; + + continue; + } + + ctx->size -= buf->last - buf->pos; + buf->pos = buf->last; + b->last = buf->last; + + continue; + } + + if (rc == NGX_DONE) { + + /* a whole response has been parsed successfully */ + + p->upstream_done = 1; + r->upstream->keepalive = !r->upstream->headers_in.connection_close; + + break; + } + + if (rc == NGX_AGAIN) { + + /* set p->length, minimal amount of data we want to see */ + + p->length = ctx->length; + + break; + } + + /* invalid response */ + + ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, + "upstream sent invalid chunked response"); + + return NGX_ERROR; + } + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http proxy chunked state %d, length %d", + ctx->state, p->length); + + if (b) { + b->shadow = buf; + b->last_shadow = 1; + + ngx_log_debug2(NGX_LOG_DEBUG_EVENT, p->log, 0, + "input buf %p %z", b->pos, b->last - b->pos); + + return NGX_OK; + } + + /* there is no data record in the buf, add it to free chain */ + + if (ngx_event_pipe_add_free_buf(p, buf) != NGX_OK) { + return NGX_ERROR; + } + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_proxy_non_buffered_copy_filter(void *data, ssize_t bytes) +{ + ngx_http_request_t *r = data; + + ngx_buf_t *b; + ngx_chain_t *cl, **ll; + ngx_http_upstream_t *u; + + u = r->upstream; + + for (cl = u->out_bufs, ll = &u->out_bufs; cl; cl = cl->next) { + ll = &cl->next; + } + + cl = ngx_chain_get_free_buf(r->pool, &u->free_bufs); + if (cl == NULL) { + return NGX_ERROR; + } + + *ll = cl; + + cl->buf->flush = 1; + cl->buf->memory = 1; + + b = &u->buffer; + + cl->buf->pos = b->last; + b->last += bytes; + cl->buf->last = b->last; + cl->buf->tag = u->output.tag; + + if (u->length == -1) { + return NGX_OK; + } + + u->length -= bytes; + + if (u->length == 0) { + u->keepalive = !u->headers_in.connection_close; + } + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_proxy_non_buffered_chunked_filter(void *data, ssize_t bytes) +{ + ngx_http_request_t *r = data; + + ngx_int_t rc; + ngx_buf_t *b, *buf; + ngx_chain_t *cl, **ll; + ngx_http_upstream_t *u; + ngx_http_proxy_ctx_t *ctx; + + ctx = ngx_http_get_module_ctx(r, ngx_http_proxy_module); + u = r->upstream; + buf = &u->buffer; + + buf->pos = buf->last; + buf->last += bytes; + + for (cl = u->out_bufs, ll = &u->out_bufs; cl; cl = cl->next) { + ll = &cl->next; + } + + for ( ;; ) { + + rc = ngx_http_proxy_parse_chunked(r, buf); + + if (rc == NGX_OK) { + + /* a chunk has been parsed successfully */ + + cl = ngx_chain_get_free_buf(r->pool, &u->free_bufs); + if (cl == NULL) { + return NGX_ERROR; + } + + *ll = cl; + ll = &cl->next; + + b = cl->buf; + + b->flush = 1; + b->memory = 1; + + b->pos = buf->pos; + b->tag = u->output.tag; + + if (buf->last - buf->pos >= ctx->size) { + buf->pos += ctx->size; + b->last = buf->pos; + ctx->size = 0; + + } else { + ctx->size -= buf->last - buf->pos; + buf->pos = buf->last; + b->last = buf->last; + } + + ngx_log_debug2(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http proxy out buf %p %z", + b->pos, b->last - b->pos); + + continue; + } + + if (rc == NGX_DONE) { + + /* a whole response has been parsed successfully */ + + u->keepalive = !u->headers_in.connection_close; + u->length = 0; + + break; + } + + if (rc == NGX_AGAIN) { + break; + } + + /* invalid response */ + + ngx_log_error(NGX_LOG_ALERT, r->connection->log, 0, + "upstream sent invalid chunked response"); + + return NGX_ERROR; + } + + /* provide continuous buffer for subrequests in memory */ + + if (r->subrequest_in_memory) { + + cl = u->out_bufs; + + if (cl) { + buf->pos = cl->buf->pos; + } + + buf->last = buf->pos; + + for (cl = u->out_bufs; cl; cl = cl->next) { + ngx_log_debug3(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "http proxy in memory %p-%p %uz", + cl->buf->pos, cl->buf->last, ngx_buf_size(cl->buf)); + + if (buf->last == cl->buf->pos) { + buf->last = cl->buf->last; + continue; + } + + buf->last = ngx_movemem(buf->last, cl->buf->pos, + cl->buf->last - cl->buf->pos); + + cl->buf->pos = buf->last - (cl->buf->last - cl->buf->pos); + cl->buf->last = buf->last; + } + } + + return NGX_OK; +} + + static void ngx_http_proxy_abort_request(ngx_http_request_t *r) { @@ -1705,6 +2473,8 @@ ngx_http_proxy_create_loc_conf(ngx_conf_t *cf) conf->redirect = NGX_CONF_UNSET; conf->upstream.change_buffering = 1; + conf->http_version = NGX_CONF_UNSET_UINT; + conf->headers_hash_max_size = NGX_CONF_UNSET_UINT; conf->headers_hash_bucket_size = NGX_CONF_UNSET_UINT; @@ -2008,6 +2778,9 @@ ngx_http_proxy_merge_loc_conf(ngx_conf_t *cf, void *parent, void *child) } #endif + ngx_conf_merge_uint_value(conf->http_version, prev->http_version, + NGX_HTTP_VERSION_10); + ngx_conf_merge_uint_value(conf->headers_hash_max_size, prev->headers_hash_max_size, 512); diff --git a/src/http/modules/ngx_http_upstream_keepalive_module.c b/src/http/modules/ngx_http_upstream_keepalive_module.c new file mode 100644 index 000000000..945f78b15 --- /dev/null +++ b/src/http/modules/ngx_http_upstream_keepalive_module.c @@ -0,0 +1,569 @@ + +/* + * Copyright (C) Maxim Dounin + */ + + +#include <ngx_config.h> +#include <ngx_core.h> +#include <ngx_http.h> + + +typedef struct { + ngx_uint_t max_cached; + ngx_uint_t single; /* unsigned:1 */ + + ngx_queue_t cache; + ngx_queue_t free; + + ngx_http_upstream_init_pt original_init_upstream; + ngx_http_upstream_init_peer_pt original_init_peer; + +} ngx_http_upstream_keepalive_srv_conf_t; + + +typedef struct { + ngx_http_upstream_keepalive_srv_conf_t *conf; + + ngx_http_upstream_t *upstream; + + void *data; + + ngx_event_get_peer_pt original_get_peer; + ngx_event_free_peer_pt original_free_peer; + +#if (NGX_HTTP_SSL) + ngx_event_set_peer_session_pt original_set_session; + ngx_event_save_peer_session_pt original_save_session; +#endif + + ngx_uint_t failed; /* unsigned:1 */ + +} ngx_http_upstream_keepalive_peer_data_t; + + +typedef struct { + ngx_http_upstream_keepalive_srv_conf_t *conf; + + ngx_queue_t queue; + ngx_connection_t *connection; + + socklen_t socklen; + u_char sockaddr[NGX_SOCKADDRLEN]; + +} ngx_http_upstream_keepalive_cache_t; + + +static ngx_int_t ngx_http_upstream_init_keepalive_peer(ngx_http_request_t *r, + ngx_http_upstream_srv_conf_t *us); +static ngx_int_t ngx_http_upstream_get_keepalive_peer(ngx_peer_connection_t *pc, + void *data); +static void ngx_http_upstream_free_keepalive_peer(ngx_peer_connection_t *pc, + void *data, ngx_uint_t state); + +static void ngx_http_upstream_keepalive_dummy_handler(ngx_event_t *ev); +static void ngx_http_upstream_keepalive_close_handler(ngx_event_t *ev); +static void ngx_http_upstream_keepalive_close(ngx_connection_t *c); + + +#if (NGX_HTTP_SSL) +static ngx_int_t ngx_http_upstream_keepalive_set_session( + ngx_peer_connection_t *pc, void *data); +static void ngx_http_upstream_keepalive_save_session(ngx_peer_connection_t *pc, + void *data); +#endif + +static void *ngx_http_upstream_keepalive_create_conf(ngx_conf_t *cf); +static char *ngx_http_upstream_keepalive(ngx_conf_t *cf, ngx_command_t *cmd, + void *conf); + + +static ngx_command_t ngx_http_upstream_keepalive_commands[] = { + + { ngx_string("keepalive"), + NGX_HTTP_UPS_CONF|NGX_CONF_TAKE12, + ngx_http_upstream_keepalive, + 0, + 0, + NULL }, + + ngx_null_command +}; + + +static ngx_http_module_t ngx_http_upstream_keepalive_module_ctx = { + NULL, /* preconfiguration */ + NULL, /* postconfiguration */ + + NULL, /* create main configuration */ + NULL, /* init main configuration */ + + ngx_http_upstream_keepalive_create_conf, /* create server configuration */ + NULL, /* merge server configuration */ + + NULL, /* create location configuration */ + NULL /* merge location configuration */ +}; + + +ngx_module_t ngx_http_upstream_keepalive_module = { + NGX_MODULE_V1, + &ngx_http_upstream_keepalive_module_ctx, /* module context */ + ngx_http_upstream_keepalive_commands, /* module directives */ + NGX_HTTP_MODULE, /* module type */ + NULL, /* init master */ + NULL, /* init module */ + NULL, /* init process */ + NULL, /* init thread */ + NULL, /* exit thread */ + NULL, /* exit process */ + NULL, /* exit master */ + NGX_MODULE_V1_PADDING +}; + + +static ngx_int_t +ngx_http_upstream_init_keepalive(ngx_conf_t *cf, + ngx_http_upstream_srv_conf_t *us) +{ + ngx_uint_t i; + ngx_http_upstream_keepalive_srv_conf_t *kcf; + ngx_http_upstream_keepalive_cache_t *cached; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, cf->log, 0, + "init keepalive"); + + kcf = ngx_http_conf_upstream_srv_conf(us, + ngx_http_upstream_keepalive_module); + + if (kcf->original_init_upstream(cf, us) != NGX_OK) { + return NGX_ERROR; + } + + kcf->original_init_peer = us->peer.init; + + us->peer.init = ngx_http_upstream_init_keepalive_peer; + + /* allocate cache items and add to free queue */ + + cached = ngx_pcalloc(cf->pool, + sizeof(ngx_http_upstream_keepalive_cache_t) * kcf->max_cached); + if (cached == NULL) { + return NGX_ERROR; + } + + ngx_queue_init(&kcf->cache); + ngx_queue_init(&kcf->free); + + for (i = 0; i < kcf->max_cached; i++) { + ngx_queue_insert_head(&kcf->free, &cached[i].queue); + cached[i].conf = kcf; + } + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_upstream_init_keepalive_peer(ngx_http_request_t *r, + ngx_http_upstream_srv_conf_t *us) +{ + ngx_http_upstream_keepalive_peer_data_t *kp; + ngx_http_upstream_keepalive_srv_conf_t *kcf; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0, + "init keepalive peer"); + + kcf = ngx_http_conf_upstream_srv_conf(us, + ngx_http_upstream_keepalive_module); + + kp = ngx_palloc(r->pool, sizeof(ngx_http_upstream_keepalive_peer_data_t)); + if (kp == NULL) { + return NGX_ERROR; + } + + if (kcf->original_init_peer(r, us) != NGX_OK) { + return NGX_ERROR; + } + + kp->conf = kcf; + kp->upstream = r->upstream; + kp->data = r->upstream->peer.data; + kp->original_get_peer = r->upstream->peer.get; + kp->original_free_peer = r->upstream->peer.free; + + r->upstream->peer.data = kp; + r->upstream->peer.get = ngx_http_upstream_get_keepalive_peer; + r->upstream->peer.free = ngx_http_upstream_free_keepalive_peer; + +#if (NGX_HTTP_SSL) + kp->original_set_session = r->upstream->peer.set_session; + kp->original_save_session = r->upstream->peer.save_session; + r->upstream->peer.set_session = ngx_http_upstream_keepalive_set_session; + r->upstream->peer.save_session = ngx_http_upstream_keepalive_save_session; +#endif + + return NGX_OK; +} + + +static ngx_int_t +ngx_http_upstream_get_keepalive_peer(ngx_peer_connection_t *pc, void *data) +{ + ngx_http_upstream_keepalive_peer_data_t *kp = data; + ngx_http_upstream_keepalive_cache_t *item; + + ngx_int_t rc; + ngx_queue_t *q, *cache; + ngx_connection_t *c; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "get keepalive peer"); + + kp->failed = 0; + + /* single pool of cached connections */ + + if (kp->conf->single && !ngx_queue_empty(&kp->conf->cache)) { + + q = ngx_queue_head(&kp->conf->cache); + + item = ngx_queue_data(q, ngx_http_upstream_keepalive_cache_t, queue); + c = item->connection; + + ngx_queue_remove(q); + ngx_queue_insert_head(&kp->conf->free, q); + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "get keepalive peer: using connection %p", c); + + c->idle = 0; + c->log = pc->log; + c->read->log = pc->log; + c->write->log = pc->log; + c->pool->log = pc->log; + + pc->connection = c; + pc->cached = 1; + + return NGX_DONE; + } + + rc = kp->original_get_peer(pc, kp->data); + + if (kp->conf->single || rc != NGX_OK) { + return rc; + } + + /* search cache for suitable connection */ + + cache = &kp->conf->cache; + + for (q = ngx_queue_head(cache); + q != ngx_queue_sentinel(cache); + q = ngx_queue_next(q)) + { + item = ngx_queue_data(q, ngx_http_upstream_keepalive_cache_t, queue); + c = item->connection; + + if (ngx_memn2cmp((u_char *) &item->sockaddr, (u_char *) pc->sockaddr, + item->socklen, pc->socklen) + == 0) + { + ngx_queue_remove(q); + ngx_queue_insert_head(&kp->conf->free, q); + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "get keepalive peer: using connection %p", c); + + c->idle = 0; + c->log = pc->log; + c->read->log = pc->log; + c->write->log = pc->log; + c->pool->log = pc->log; + + pc->connection = c; + pc->cached = 1; + + return NGX_DONE; + } + } + + return NGX_OK; +} + + +static void +ngx_http_upstream_free_keepalive_peer(ngx_peer_connection_t *pc, void *data, + ngx_uint_t state) +{ + ngx_http_upstream_keepalive_peer_data_t *kp = data; + ngx_http_upstream_keepalive_cache_t *item; + + ngx_queue_t *q; + ngx_connection_t *c; + ngx_http_upstream_t *u; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "free keepalive peer"); + + /* remember failed state - peer.free() may be called more than once */ + + if (state & NGX_PEER_FAILED) { + kp->failed = 1; + } + + /* cache valid connections */ + + u = kp->upstream; + c = pc->connection; + + if (kp->failed + || c == NULL + || c->read->eof + || c->read->error + || c->read->timedout + || c->write->error + || c->write->timedout) + { + goto invalid; + } + + if (!u->keepalive) { + goto invalid; + } + + if (ngx_handle_read_event(c->read, 0) != NGX_OK) { + goto invalid; + } + + ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pc->log, 0, + "free keepalive peer: saving connection %p", c); + + if (ngx_queue_empty(&kp->conf->free)) { + + q = ngx_queue_last(&kp->conf->cache); + ngx_queue_remove(q); + + item = ngx_queue_data(q, ngx_http_upstream_keepalive_cache_t, queue); + + ngx_http_upstream_keepalive_close(item->connection); + + } else { + q = ngx_queue_head(&kp->conf->free); + ngx_queue_remove(q); + + item = ngx_queue_data(q, ngx_http_upstream_keepalive_cache_t, queue); + } + + item->connection = c; + ngx_queue_insert_head(&kp->conf->cache, q); + + pc->connection = NULL; + + if (c->read->timer_set) { + ngx_del_timer(c->read); + } + if (c->write->timer_set) { + ngx_del_timer(c->write); + } + + c->write->handler = ngx_http_upstream_keepalive_dummy_handler; + c->read->handler = ngx_http_upstream_keepalive_close_handler; + + c->data = item; + c->idle = 1; + c->log = ngx_cycle->log; + c->read->log = ngx_cycle->log; + c->write->log = ngx_cycle->log; + c->pool->log = ngx_cycle->log; + + item->socklen = pc->socklen; + ngx_memcpy(&item->sockaddr, pc->sockaddr, pc->socklen); + + if (c->read->ready) { + ngx_http_upstream_keepalive_close_handler(c->read); + } + +invalid: + + kp->original_free_peer(pc, kp->data, state); +} + + +static void +ngx_http_upstream_keepalive_dummy_handler(ngx_event_t *ev) +{ + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ev->log, 0, + "keepalive dummy handler"); +} + + +static void +ngx_http_upstream_keepalive_close_handler(ngx_event_t *ev) +{ + ngx_http_upstream_keepalive_srv_conf_t *conf; + ngx_http_upstream_keepalive_cache_t *item; + + int n; + char buf[1]; + ngx_connection_t *c; + + ngx_log_debug0(NGX_LOG_DEBUG_HTTP, ev->log, 0, + "keepalive close handler"); + + c = ev->data; + + if (c->close) { + goto close; + } + + n = recv(c->fd, buf, 1, MSG_PEEK); + + if (n == -1 && ngx_socket_errno == NGX_EAGAIN) { + /* stale event */ + + if (ngx_handle_read_event(c->read, 0) != NGX_OK) { + goto close; + } + + return; + } + +close: + + item = c->data; + conf = item->conf; + + ngx_http_upstream_keepalive_close(c); + + ngx_queue_remove(&item->queue); + ngx_queue_insert_head(&conf->free, &item->queue); +} + + +static void +ngx_http_upstream_keepalive_close(ngx_connection_t *c) +{ + +#if (NGX_HTTP_SSL) + + if (c->ssl) { + c->ssl->no_wait_shutdown = 1; + c->ssl->no_send_shutdown = 1; + + if (ngx_ssl_shutdown(c) == NGX_AGAIN) { + c->ssl->handler = ngx_http_upstream_keepalive_close; + return; + } + } + +#endif + + ngx_destroy_pool(c->pool); + ngx_close_connection(c); +} + + +#if (NGX_HTTP_SSL) + +static ngx_int_t +ngx_http_upstream_keepalive_set_session(ngx_peer_connection_t *pc, void *data) +{ + ngx_http_upstream_keepalive_peer_data_t *kp = data; + + return kp->original_set_session(pc, kp->data); +} + + +static void +ngx_http_upstream_keepalive_save_session(ngx_peer_connection_t *pc, void *data) +{ + ngx_http_upstream_keepalive_peer_data_t *kp = data; + + kp->original_save_session(pc, kp->data); + return; +} + +#endif + + +static void * +ngx_http_upstream_keepalive_create_conf(ngx_conf_t *cf) +{ + ngx_http_upstream_keepalive_srv_conf_t *conf; + + conf = ngx_pcalloc(cf->pool, + sizeof(ngx_http_upstream_keepalive_srv_conf_t)); + if (conf == NULL) { + return NULL; + } + + /* + * set by ngx_pcalloc(): + * + * conf->original_init_upstream = NULL; + * conf->original_init_peer = NULL; + */ + + conf->max_cached = 1; + + return conf; +} + + +static char * +ngx_http_upstream_keepalive(ngx_conf_t *cf, ngx_command_t *cmd, void *conf) +{ + ngx_http_upstream_srv_conf_t *uscf; + ngx_http_upstream_keepalive_srv_conf_t *kcf; + + ngx_int_t n; + ngx_str_t *value; + ngx_uint_t i; + + uscf = ngx_http_conf_get_module_srv_conf(cf, ngx_http_upstream_module); + + kcf = ngx_http_conf_upstream_srv_conf(uscf, + ngx_http_upstream_keepalive_module); + + kcf->original_init_upstream = uscf->peer.init_upstream + ? uscf->peer.init_upstream + : ngx_http_upstream_init_round_robin; + + uscf->peer.init_upstream = ngx_http_upstream_init_keepalive; + + /* read options */ + + value = cf->args->elts; + + n = ngx_atoi(value[1].data, value[1].len); + + if (n == NGX_ERROR || n == 0) { + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid value \"%V\" in \"%V\" directive", + &value[1], &cmd->name); + return NGX_CONF_ERROR; + } + + kcf->max_cached = n; + + for (i = 2; i < cf->args->nelts; i++) { + + if (ngx_strcmp(value[i].data, "single") == 0) { + kcf->single = 1; + continue; + } + + goto invalid; + } + + return NGX_CONF_OK; + +invalid: + + ngx_conf_log_error(NGX_LOG_EMERG, cf, 0, + "invalid parameter \"%V\"", &value[i]); + + return NGX_CONF_ERROR; +} diff --git a/src/http/modules/perl/nginx.pm b/src/http/modules/perl/nginx.pm index 6dedc6d7d..429af0d1c 100644 --- a/src/http/modules/perl/nginx.pm +++ b/src/http/modules/perl/nginx.pm @@ -48,7 +48,7 @@ our @EXPORT = qw( HTTP_INSUFFICIENT_STORAGE ); -our $VERSION = '1.1.3'; +our $VERSION = '1.1.4'; require XSLoader; XSLoader::load('nginx', $VERSION); |