summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMaxim Dounin <mdounin@mdounin.ru>2013-02-11 14:34:00 +0000
committerMaxim Dounin <mdounin@mdounin.ru>2013-02-11 14:34:00 +0000
commiteef05677eb4a6b59ce0e9705aa66f0b796c08493 (patch)
tree463e7f245de28881e60df704e20ceefbb4cc2f4f
parent8fea19793236c9a21ac13599aafb3cec746640fd (diff)
downloadnginx-eef05677eb4a6b59ce0e9705aa66f0b796c08493.tar.gz
Merge of r4985, r4986, r4987, r4988, r4989, r5002: access_log gzip.
*) Access log: fixed redundant buffer reallocation. Previously a new buffer was allocated for every "access_log" directive with the same file path and "buffer=" parameters, while only one buffer per file is used. *) Reopening log files code moved to a separate function. The code refactored in a way to call custom handler that can do appropriate cleanup work (if any), like flushing buffers, finishing compress streams, finalizing connections to log daemon, etc.. *) Access log: the "flush" parameter of the "access_log" directive. *) Configure: added the NGX_ZLIB define. This was introduced for conditional compilation of the code that requires the zlib library. *) Access log: the "gzip" parameter of the "access_log" directive. Note: this requires zlib version 1.2.0.4 or above to work. *) The data pointer in ngx_open_file_t objects must be initialized. Uninitialized pointer may result in arbitrary segfaults if access_log is used without buffer and without variables in file path. Patch by Tatsuhiko Kubo (ticket #268).
-rw-r--r--auto/lib/zlib/conf5
-rw-r--r--src/core/ngx_conf_file.c23
-rw-r--r--src/core/ngx_conf_file.h11
-rw-r--r--src/core/ngx_cycle.c21
-rw-r--r--src/http/modules/ngx_http_log_module.c418
5 files changed, 389 insertions, 89 deletions
diff --git a/auto/lib/zlib/conf b/auto/lib/zlib/conf
index cefc874b2..26db642ac 100644
--- a/auto/lib/zlib/conf
+++ b/auto/lib/zlib/conf
@@ -9,11 +9,13 @@ if [ $ZLIB != NONE ]; then
case "$NGX_CC_NAME" in
msvc* | owc* | bcc)
+ have=NGX_ZLIB . auto/have
LINK_DEPS="$LINK_DEPS $ZLIB/zlib.lib"
CORE_LIBS="$CORE_LIBS $ZLIB/zlib.lib"
;;
icc*)
+ have=NGX_ZLIB . auto/have
LINK_DEPS="$LINK_DEPS $ZLIB/libz.a"
# to allow -ipo optimization we link with the *.o but not library
@@ -30,6 +32,7 @@ if [ $ZLIB != NONE ]; then
;;
*)
+ have=NGX_ZLIB . auto/have
LINK_DEPS="$LINK_DEPS $ZLIB/libz.a"
CORE_LIBS="$CORE_LIBS $ZLIB/libz.a"
#CORE_LIBS="$CORE_LIBS -L $ZLIB -lz"
@@ -45,7 +48,7 @@ else
# FreeBSD, Solaris, Linux
ngx_feature="zlib library"
- ngx_feature_name=
+ ngx_feature_name="NGX_ZLIB"
ngx_feature_run=no
ngx_feature_incs="#include <zlib.h>"
ngx_feature_path=
diff --git a/src/core/ngx_conf_file.c b/src/core/ngx_conf_file.c
index 12bb5cf8a..ed7a2ce92 100644
--- a/src/core/ngx_conf_file.c
+++ b/src/core/ngx_conf_file.c
@@ -945,7 +945,8 @@ ngx_conf_open_file(ngx_cycle_t *cycle, ngx_str_t *name)
file->name = *name;
}
- file->buffer = NULL;
+ file->flush = NULL;
+ file->data = NULL;
return file;
}
@@ -954,7 +955,6 @@ ngx_conf_open_file(ngx_cycle_t *cycle, ngx_str_t *name)
static void
ngx_conf_flush_files(ngx_cycle_t *cycle)
{
- ssize_t n, len;
ngx_uint_t i;
ngx_list_part_t *part;
ngx_open_file_t *file;
@@ -975,23 +975,8 @@ ngx_conf_flush_files(ngx_cycle_t *cycle)
i = 0;
}
- len = file[i].pos - file[i].buffer;
-
- if (file[i].buffer == NULL || len == 0) {
- continue;
- }
-
- n = ngx_write_fd(file[i].fd, file[i].buffer, len);
-
- if (n == -1) {
- ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
- ngx_write_fd_n " to \"%s\" failed",
- file[i].name.data);
-
- } else if (n != len) {
- ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
- ngx_write_fd_n " to \"%s\" was incomplete: %z of %uz",
- file[i].name.data, n, len);
+ if (file[i].flush) {
+ file[i].flush(&file[i], cycle->log);
}
}
}
diff --git a/src/core/ngx_conf_file.h b/src/core/ngx_conf_file.h
index 64a9e8fc8..237e6ecea 100644
--- a/src/core/ngx_conf_file.h
+++ b/src/core/ngx_conf_file.h
@@ -91,17 +91,8 @@ struct ngx_open_file_s {
ngx_fd_t fd;
ngx_str_t name;
- u_char *buffer;
- u_char *pos;
- u_char *last;
-
-#if 0
- /* e.g. append mode, error_log */
- ngx_uint_t flags;
- /* e.g. reopen db file */
- ngx_uint_t (*handler)(void *data, ngx_open_file_t *file);
+ void (*flush)(ngx_open_file_t *file, ngx_log_t *log);
void *data;
-#endif
};
diff --git a/src/core/ngx_cycle.c b/src/core/ngx_cycle.c
index e5dad4607..dc4dc8993 100644
--- a/src/core/ngx_cycle.c
+++ b/src/core/ngx_cycle.c
@@ -1115,7 +1115,6 @@ ngx_test_lockfile(u_char *file, ngx_log_t *log)
void
ngx_reopen_files(ngx_cycle_t *cycle, ngx_uid_t user)
{
- ssize_t n, len;
ngx_fd_t fd;
ngx_uint_t i;
ngx_list_part_t *part;
@@ -1139,24 +1138,8 @@ ngx_reopen_files(ngx_cycle_t *cycle, ngx_uid_t user)
continue;
}
- len = file[i].pos - file[i].buffer;
-
- if (file[i].buffer && len != 0) {
-
- n = ngx_write_fd(file[i].fd, file[i].buffer, len);
-
- if (n == -1) {
- ngx_log_error(NGX_LOG_ALERT, cycle->log, ngx_errno,
- ngx_write_fd_n " to \"%s\" failed",
- file[i].name.data);
-
- } else if (n != len) {
- ngx_log_error(NGX_LOG_ALERT, cycle->log, 0,
- ngx_write_fd_n " to \"%s\" was incomplete: %z of %uz",
- file[i].name.data, n, len);
- }
-
- file[i].pos = file[i].buffer;
+ if (file[i].flush) {
+ file[i].flush(&file[i], cycle->log);
}
fd = ngx_open_file(file[i].name.data, NGX_FILE_APPEND,
diff --git a/src/http/modules/ngx_http_log_module.c b/src/http/modules/ngx_http_log_module.c
index 0962c8475..aa6a3fcee 100644
--- a/src/http/modules/ngx_http_log_module.c
+++ b/src/http/modules/ngx_http_log_module.c
@@ -9,6 +9,10 @@
#include <ngx_core.h>
#include <ngx_http.h>
+#if (NGX_ZLIB)
+#include <zlib.h>
+#endif
+
typedef struct ngx_http_log_op_s ngx_http_log_op_t;
@@ -41,6 +45,17 @@ typedef struct {
typedef struct {
+ u_char *start;
+ u_char *pos;
+ u_char *last;
+
+ ngx_event_t *event;
+ ngx_msec_t flush;
+ ngx_int_t gzip;
+} ngx_http_log_buf_t;
+
+
+typedef struct {
ngx_array_t *lengths;
ngx_array_t *values;
} ngx_http_log_script_t;
@@ -78,6 +93,17 @@ static void ngx_http_log_write(ngx_http_request_t *r, ngx_http_log_t *log,
static ssize_t ngx_http_log_script_write(ngx_http_request_t *r,
ngx_http_log_script_t *script, u_char **name, u_char *buf, size_t len);
+#if (NGX_ZLIB)
+static ssize_t ngx_http_log_gzip(ngx_fd_t fd, u_char *buf, size_t len,
+ ngx_int_t level, ngx_log_t *log);
+
+static void *ngx_http_log_gzip_alloc(void *opaque, u_int items, u_int size);
+static void ngx_http_log_gzip_free(void *opaque, void *address);
+#endif
+
+static void ngx_http_log_flush(ngx_open_file_t *file, ngx_log_t *log);
+static void ngx_http_log_flush_handler(ngx_event_t *ev);
+
static u_char *ngx_http_log_pipe(ngx_http_request_t *r, u_char *buf,
ngx_http_log_op_t *op);
static u_char *ngx_http_log_time(ngx_http_request_t *r, u_char *buf,
@@ -132,7 +158,7 @@ static ngx_command_t ngx_http_log_commands[] = {
{ ngx_string("access_log"),
NGX_HTTP_MAIN_CONF|NGX_HTTP_SRV_CONF|NGX_HTTP_LOC_CONF|NGX_HTTP_LIF_CONF
- |NGX_HTTP_LMT_CONF|NGX_CONF_TAKE123,
+ |NGX_HTTP_LMT_CONF|NGX_CONF_1MORE,
ngx_http_log_set_log,
NGX_HTTP_LOC_CONF_OFFSET,
0,
@@ -216,8 +242,8 @@ ngx_http_log_handler(ngx_http_request_t *r)
size_t len;
ngx_uint_t i, l;
ngx_http_log_t *log;
- ngx_open_file_t *file;
ngx_http_log_op_t *op;
+ ngx_http_log_buf_t *buffer;
ngx_http_log_loc_conf_t *lcf;
ngx_log_debug0(NGX_LOG_DEBUG_HTTP, r->connection->log, 0,
@@ -258,21 +284,25 @@ ngx_http_log_handler(ngx_http_request_t *r)
len += NGX_LINEFEED_SIZE;
- file = log[l].file;
+ buffer = log[l].file ? log[l].file->data : NULL;
- if (file && file->buffer) {
+ if (buffer) {
- if (len > (size_t) (file->last - file->pos)) {
+ if (len > (size_t) (buffer->last - buffer->pos)) {
- ngx_http_log_write(r, &log[l], file->buffer,
- file->pos - file->buffer);
+ ngx_http_log_write(r, &log[l], buffer->start,
+ buffer->pos - buffer->start);
- file->pos = file->buffer;
+ buffer->pos = buffer->start;
}
- if (len <= (size_t) (file->last - file->pos)) {
+ if (len <= (size_t) (buffer->last - buffer->pos)) {
- p = file->pos;
+ p = buffer->pos;
+
+ if (buffer->event && p == buffer->start) {
+ ngx_add_timer(buffer->event, buffer->flush);
+ }
for (i = 0; i < log[l].format->ops->nelts; i++) {
p = op[i].run(r, p, &op[i]);
@@ -280,10 +310,14 @@ ngx_http_log_handler(ngx_http_request_t *r)
ngx_linefeed(p);
- file->pos = p;
+ buffer->pos = p;
continue;
}
+
+ if (buffer->event && buffer->event->timer_set) {
+ ngx_del_timer(buffer->event);
+ }
}
line = ngx_pnalloc(r->pool, len);
@@ -310,14 +344,29 @@ static void
ngx_http_log_write(ngx_http_request_t *r, ngx_http_log_t *log, u_char *buf,
size_t len)
{
- u_char *name;
- time_t now;
- ssize_t n;
- ngx_err_t err;
+ u_char *name;
+ time_t now;
+ ssize_t n;
+ ngx_err_t err;
+#if (NGX_ZLIB)
+ ngx_http_log_buf_t *buffer;
+#endif
if (log->script == NULL) {
name = log->file->name.data;
+
+#if (NGX_ZLIB)
+ buffer = log->file->data;
+
+ if (buffer && buffer->gzip) {
+ n = ngx_http_log_gzip(log->file->fd, buf, len, buffer->gzip,
+ r->connection->log);
+ } else {
+ n = ngx_write_fd(log->file->fd, buf, len);
+ }
+#else
n = ngx_write_fd(log->file->fd, buf, len);
+#endif
} else {
name = NULL;
@@ -465,6 +514,194 @@ ngx_http_log_script_write(ngx_http_request_t *r, ngx_http_log_script_t *script,
}
+#if (NGX_ZLIB)
+
+static ssize_t
+ngx_http_log_gzip(ngx_fd_t fd, u_char *buf, size_t len, ngx_int_t level,
+ ngx_log_t *log)
+{
+ int rc, wbits, memlevel;
+ u_char *out;
+ size_t size;
+ ssize_t n;
+ z_stream zstream;
+ ngx_err_t err;
+ ngx_pool_t *pool;
+
+ wbits = MAX_WBITS;
+ memlevel = MAX_MEM_LEVEL - 1;
+
+ while ((ssize_t) len < ((1 << (wbits - 1)) - 262)) {
+ wbits--;
+ memlevel--;
+ }
+
+ /*
+ * This is a formula from deflateBound() for conservative upper bound of
+ * compressed data plus 18 bytes of gzip wrapper.
+ */
+
+ size = len + ((len + 7) >> 3) + ((len + 63) >> 6) + 5 + 18;
+
+ ngx_memzero(&zstream, sizeof(z_stream));
+
+ pool = ngx_create_pool(256, log);
+ if (pool == NULL) {
+ /* simulate successful logging */
+ return len;
+ }
+
+ pool->log = log;
+
+ zstream.zalloc = ngx_http_log_gzip_alloc;
+ zstream.zfree = ngx_http_log_gzip_free;
+ zstream.opaque = pool;
+
+ out = ngx_pnalloc(pool, size);
+ if (out == NULL) {
+ goto done;
+ }
+
+ zstream.next_in = buf;
+ zstream.avail_in = len;
+ zstream.next_out = out;
+ zstream.avail_out = size;
+
+ rc = deflateInit2(&zstream, (int) level, Z_DEFLATED, wbits + 16, memlevel,
+ Z_DEFAULT_STRATEGY);
+
+ if (rc != Z_OK) {
+ ngx_log_error(NGX_LOG_ALERT, log, 0, "deflateInit2() failed: %d", rc);
+ goto done;
+ }
+
+ ngx_log_debug4(NGX_LOG_DEBUG_HTTP, log, 0,
+ "deflate in: ni:%p no:%p ai:%ud ao:%ud",
+ zstream.next_in, zstream.next_out,
+ zstream.avail_in, zstream.avail_out);
+
+ rc = deflate(&zstream, Z_FINISH);
+
+ if (rc != Z_STREAM_END) {
+ ngx_log_error(NGX_LOG_ALERT, log, 0,
+ "deflate(Z_FINISH) failed: %d", rc);
+ goto done;
+ }
+
+ ngx_log_debug5(NGX_LOG_DEBUG_HTTP, log, 0,
+ "deflate out: ni:%p no:%p ai:%ud ao:%ud rc:%d",
+ zstream.next_in, zstream.next_out,
+ zstream.avail_in, zstream.avail_out,
+ rc);
+
+ size -= zstream.avail_out;
+
+ rc = deflateEnd(&zstream);
+
+ if (rc != Z_OK) {
+ ngx_log_error(NGX_LOG_ALERT, log, 0, "deflateEnd() failed: %d", rc);
+ goto done;
+ }
+
+ n = ngx_write_fd(fd, out, size);
+
+ if (n != (ssize_t) size) {
+ err = (n == -1) ? ngx_errno : 0;
+
+ ngx_destroy_pool(pool);
+
+ ngx_set_errno(err);
+ return -1;
+ }
+
+done:
+
+ ngx_destroy_pool(pool);
+
+ /* simulate successful logging */
+ return len;
+}
+
+
+static void *
+ngx_http_log_gzip_alloc(void *opaque, u_int items, u_int size)
+{
+ ngx_pool_t *pool = opaque;
+
+ ngx_log_debug2(NGX_LOG_DEBUG_HTTP, pool->log, 0,
+ "gzip alloc: n:%ud s:%ud", items, size);
+
+ return ngx_palloc(pool, items * size);
+}
+
+
+static void
+ngx_http_log_gzip_free(void *opaque, void *address)
+{
+#if 0
+ ngx_pool_t *pool = opaque;
+
+ ngx_log_debug1(NGX_LOG_DEBUG_HTTP, pool->log, 0, "gzip free: %p", address);
+#endif
+}
+
+#endif
+
+
+static void
+ngx_http_log_flush(ngx_open_file_t *file, ngx_log_t *log)
+{
+ size_t len;
+ ssize_t n;
+ ngx_http_log_buf_t *buffer;
+
+ buffer = file->data;
+
+ len = buffer->pos - buffer->start;
+
+ if (len == 0) {
+ return;
+ }
+
+#if (NGX_ZLIB)
+ if (buffer->gzip) {
+ n = ngx_http_log_gzip(file->fd, buffer->start, len, buffer->gzip, log);
+ } else {
+ n = ngx_write_fd(file->fd, buffer->start, len);
+ }
+#else
+ n = ngx_write_fd(file->fd, buffer->start, len);
+#endif
+
+ if (n == -1) {
+ ngx_log_error(NGX_LOG_ALERT, log, ngx_errno,
+ ngx_write_fd_n " to \"%s\" failed",
+ file->name.data);
+
+ } else if ((size_t) n != len) {
+ ngx_log_error(NGX_LOG_ALERT, log, 0,
+ ngx_write_fd_n " to \"%s\" was incomplete: %z of %uz",
+ file->name.data, n, len);
+ }
+
+ buffer->pos = buffer->start;
+
+ if (buffer->event && buffer->event->timer_set) {
+ ngx_del_timer(buffer->event);
+ }
+}
+
+
+static void
+ngx_http_log_flush_handler(ngx_event_t *ev)
+{
+ ngx_log_debug0(NGX_LOG_DEBUG_EVENT, ev->log, 0,
+ "http log buffer flush handler");
+
+ ngx_http_log_flush(ev->data, ev->log);
+}
+
+
static u_char *
ngx_http_log_copy_short(ngx_http_request_t *r, u_char *buf,
ngx_http_log_op_t *op)
@@ -848,10 +1085,13 @@ ngx_http_log_set_log(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
{
ngx_http_log_loc_conf_t *llcf = conf;
- ssize_t buf;
+ ssize_t size;
+ ngx_int_t gzip;
ngx_uint_t i, n;
- ngx_str_t *value, name;
+ ngx_msec_t flush;
+ ngx_str_t *value, name, s;
ngx_http_log_t *log;
+ ngx_http_log_buf_t *buffer;
ngx_http_log_fmt_t *fmt;
ngx_http_log_main_conf_t *lmcf;
ngx_http_script_compile_t sc;
@@ -936,54 +1176,152 @@ ngx_http_log_set_log(ngx_conf_t *cf, ngx_command_t *cmd, void *conf)
&& ngx_strcasecmp(fmt[i].name.data, name.data) == 0)
{
log->format = &fmt[i];
- goto buffer;
+ break;
}
}
- ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
- "unknown log format \"%V\"", &name);
- return NGX_CONF_ERROR;
+ if (log->format == NULL) {
+ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+ "unknown log format \"%V\"", &name);
+ return NGX_CONF_ERROR;
+ }
+
+ size = 0;
+ flush = 0;
+ gzip = 0;
+
+ for (i = 3; i < cf->args->nelts; i++) {
+
+ if (ngx_strncmp(value[i].data, "buffer=", 7) == 0) {
+ s.len = value[i].len - 7;
+ s.data = value[i].data + 7;
+
+ size = ngx_parse_size(&s);
+
+ if (size == NGX_ERROR || size == 0) {
+ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+ "invalid buffer size \"%V\"", &s);
+ return NGX_CONF_ERROR;
+ }
-buffer:
+ continue;
+ }
+
+ if (ngx_strncmp(value[i].data, "flush=", 6) == 0) {
+ s.len = value[i].len - 6;
+ s.data = value[i].data + 6;
+
+ flush = ngx_parse_time(&s, 0);
+
+ if (flush == (ngx_msec_t) NGX_ERROR || flush == 0) {
+ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+ "invalid flush time \"%V\"", &s);
+ return NGX_CONF_ERROR;
+ }
- if (cf->args->nelts == 4) {
- if (ngx_strncmp(value[3].data, "buffer=", 7) != 0) {
+ continue;
+ }
+
+ if (ngx_strncmp(value[i].data, "gzip", 4) == 0
+ && (value[i].len == 4 || value[i].data[4] == '='))
+ {
+#if (NGX_ZLIB)
+ if (size == 0) {
+ size = 64 * 1024;
+ }
+
+ if (value[i].len == 4) {
+ gzip = Z_BEST_SPEED;
+ continue;
+ }
+
+ s.len = value[i].len - 5;
+ s.data = value[i].data + 5;
+
+ gzip = ngx_atoi(s.data, s.len);
+
+ if (gzip < 1 || gzip > 9) {
+ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+ "invalid compression level \"%V\"", &s);
+ return NGX_CONF_ERROR;
+ }
+
+ continue;
+
+#else
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
- "invalid parameter \"%V\"", &value[3]);
+ "nginx was built without zlib support");
return NGX_CONF_ERROR;
+#endif
}
+ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+ "invalid parameter \"%V\"", &value[i]);
+ return NGX_CONF_ERROR;
+ }
+
+ if (flush && size == 0) {
+ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+ "no buffer is defined for access_log \"%V\"",
+ &value[1]);
+ return NGX_CONF_ERROR;
+ }
+
+ if (size) {
+
if (log->script) {
ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
"buffered logs cannot have variables in name");
return NGX_CONF_ERROR;
}
- name.len = value[3].len - 7;
- name.data = value[3].data + 7;
+ if (log->file->data) {
+ buffer = log->file->data;
+
+ if (buffer->last - buffer->start != size
+ || buffer->flush != flush
+ || buffer->gzip != gzip)
+ {
+ ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
+ "access_log \"%V\" already defined "
+ "with conflicting parameters",
+ &value[1]);
+ return NGX_CONF_ERROR;
+ }
- buf = ngx_parse_size(&name);
+ return NGX_CONF_OK;
+ }
- if (buf == NGX_ERROR) {
- ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
- "invalid buffer value \"%V\"", &name);
+ buffer = ngx_pcalloc(cf->pool, sizeof(ngx_http_log_buf_t));
+ if (buffer == NULL) {
return NGX_CONF_ERROR;
}
- if (log->file->buffer && log->file->last - log->file->pos != buf) {
- ngx_conf_log_error(NGX_LOG_EMERG, cf, 0,
- "access_log \"%V\" already defined "
- "with different buffer size", &value[1]);
+ buffer->start = ngx_pnalloc(cf->pool, size);
+ if (buffer->start == NULL) {
return NGX_CONF_ERROR;
}
- log->file->buffer = ngx_palloc(cf->pool, buf);
- if (log->file->buffer == NULL) {
- return NGX_CONF_ERROR;
+ buffer->pos = buffer->start;
+ buffer->last = buffer->start + size;
+
+ if (flush) {
+ buffer->event = ngx_pcalloc(cf->pool, sizeof(ngx_event_t));
+ if (buffer->event == NULL) {
+ return NGX_CONF_ERROR;
+ }
+
+ buffer->event->data = log->file;
+ buffer->event->handler = ngx_http_log_flush_handler;
+ buffer->event->log = &cf->cycle->new_log;
+
+ buffer->flush = flush;
}
- log->file->pos = log->file->buffer;
- log->file->last = log->file->buffer + buf;
+ buffer->gzip = gzip;
+
+ log->file->flush = ngx_http_log_flush;
+ log->file->data = buffer;
}
return NGX_CONF_OK;