From 60bd8015f21fdb63d5409b1191f8ea9d8f1a1b87 Mon Sep 17 00:00:00 2001 From: "gabor@google.com" Date: Thu, 21 Jul 2011 02:40:18 +0000 Subject: Speed up Snappy uncompression, new Logger interface. - Removed one copy of an uncompressed block contents changing the signature of Snappy_Uncompress() so it uncompresses into a flat array instead of a std::string. Speeds up readrandom ~10%. - Instead of a combination of Env/WritableFile, we now have a Logger interface that can be easily overridden applications that want to supply their own logging. - Separated out the gcc and Sun Studio parts of atomic_pointer.h so we can use 'asm', 'volatile' keywords for Sun Studio. git-svn-id: https://leveldb.googlecode.com/svn/trunk@39 62dab493-f737-651d-591e-8d6aee1b9529 --- util/env.cc | 15 +++++--- util/env_chromium.cc | 74 +++++++-------------------------------- util/env_posix.cc | 74 +++++++-------------------------------- util/posix_logger.h | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 132 insertions(+), 128 deletions(-) create mode 100644 util/posix_logger.h (limited to 'util') diff --git a/util/env.cc b/util/env.cc index e5297e7..79e493e 100644 --- a/util/env.cc +++ b/util/env.cc @@ -18,14 +18,19 @@ RandomAccessFile::~RandomAccessFile() { WritableFile::~WritableFile() { } +Logger::~Logger() { +} + FileLock::~FileLock() { } -void Log(Env* env, WritableFile* info_log, const char* format, ...) { - va_list ap; - va_start(ap, format); - env->Logv(info_log, format, ap); - va_end(ap); +void Log(Logger* info_log, const char* format, ...) { + if (info_log != NULL) { + va_list ap; + va_start(ap, format); + info_log->Logv(format, ap); + va_end(ap); + } } Status WriteStringToFile(Env* env, const Slice& data, diff --git a/util/env_chromium.cc b/util/env_chromium.cc index 1af525a..975386b 100644 --- a/util/env_chromium.cc +++ b/util/env_chromium.cc @@ -23,6 +23,7 @@ #include "leveldb/slice.h" #include "port/port.h" #include "util/logging.h" +#include "util/posix_logger.h" #if defined(OS_WIN) #include @@ -406,9 +407,8 @@ class ChromiumEnv : public Env { return Status::OK(); } - virtual void Logv(WritableFile* info_log, const char* format, va_list ap) { - // TODO(jorlow): We may want to just use Chromium's built in logging. - + // TODO(user,user): Use Chromium's built-in logging? + static uint64_t gettid() { uint64_t thread_id = 0; // Coppied from base/logging.cc. #if defined(OS_WIN) @@ -422,65 +422,17 @@ class ChromiumEnv : public Env { pthread_t tid = pthread_self(); memcpy(&thread_id, &tid, min(sizeof(r), sizeof(tid))); #endif + return thread_id; + } - // We try twice: the first time with a fixed-size stack allocated buffer, - // and the second time with a much larger dynamically allocated buffer. - char buffer[500]; - for (int iter = 0; iter < 2; iter++) { - char* base; - int bufsize; - if (iter == 0) { - bufsize = sizeof(buffer); - base = buffer; - } else { - bufsize = 30000; - base = new char[bufsize]; - } - char* p = base; - char* limit = base + bufsize; - - ::base::Time::Exploded t; - ::base::Time::Now().LocalExplode(&t); - p += snprintf(p, limit - p, - "%04d/%02d/%02d-%02d:%02d:%02d.%06d %llx ", - t.year, - t.month, - t.day_of_month, - t.hour, - t.minute, - t.second, - static_cast(t.millisecond) * 1000, - static_cast(thread_id)); - - // Print the message - if (p < limit) { - va_list backup_ap; - va_copy(backup_ap, ap); - p += vsnprintf(p, limit - p, format, backup_ap); - va_end(backup_ap); - } - - // Truncate to available space if necessary - if (p >= limit) { - if (iter == 0) { - continue; // Try again with larger buffer - } else { - p = limit - 1; - } - } - - // Add newline if necessary - if (p == base || p[-1] != '\n') { - *p++ = '\n'; - } - - assert(p <= limit); - info_log->Append(Slice(base, p - base)); - info_log->Flush(); - if (base != buffer) { - delete[] base; - } - break; + virtual Status NewLogger(const std::string& fname, Logger** result) { + FILE* f = fopen(fname.c_str(), "w"); + if (f == NULL) { + *result = NULL; + return Status::IOError(fname, strerror(errno)); + } else { + *result = new PosixLogger(f, &ChromiumEnv::gettid); + return Status::OK(); } } diff --git a/util/env_posix.cc b/util/env_posix.cc index 46723e2..5127c89 100644 --- a/util/env_posix.cc +++ b/util/env_posix.cc @@ -23,6 +23,7 @@ #include "leveldb/slice.h" #include "port/port.h" #include "util/logging.h" +#include "util/posix_logger.h" namespace leveldb { @@ -427,72 +428,21 @@ class PosixEnv : public Env { return Status::OK(); } - virtual void Logv(WritableFile* info_log, const char* format, va_list ap) { + static uint64_t gettid() { pthread_t tid = pthread_self(); uint64_t thread_id = 0; memcpy(&thread_id, &tid, std::min(sizeof(thread_id), sizeof(tid))); + return thread_id; + } - // We try twice: the first time with a fixed-size stack allocated buffer, - // and the second time with a much larger dynamically allocated buffer. - char buffer[500]; - for (int iter = 0; iter < 2; iter++) { - char* base; - int bufsize; - if (iter == 0) { - bufsize = sizeof(buffer); - base = buffer; - } else { - bufsize = 30000; - base = new char[bufsize]; - } - char* p = base; - char* limit = base + bufsize; - - struct timeval now_tv; - gettimeofday(&now_tv, NULL); - const time_t seconds = now_tv.tv_sec; - struct tm t; - localtime_r(&seconds, &t); - p += snprintf(p, limit - p, - "%04d/%02d/%02d-%02d:%02d:%02d.%06d %llx ", - t.tm_year + 1900, - t.tm_mon + 1, - t.tm_mday, - t.tm_hour, - t.tm_min, - t.tm_sec, - static_cast(now_tv.tv_usec), - static_cast(thread_id)); - - // Print the message - if (p < limit) { - va_list backup_ap; - va_copy(backup_ap, ap); - p += vsnprintf(p, limit - p, format, backup_ap); - va_end(backup_ap); - } - - // Truncate to available space if necessary - if (p >= limit) { - if (iter == 0) { - continue; // Try again with larger buffer - } else { - p = limit - 1; - } - } - - // Add newline if necessary - if (p == base || p[-1] != '\n') { - *p++ = '\n'; - } - - assert(p <= limit); - info_log->Append(Slice(base, p - base)); - info_log->Flush(); - if (base != buffer) { - delete[] base; - } - break; + virtual Status NewLogger(const std::string& fname, Logger** result) { + FILE* f = fopen(fname.c_str(), "w"); + if (f == NULL) { + *result = NULL; + return IOError(fname, errno); + } else { + *result = new PosixLogger(f, &PosixEnv::gettid); + return Status::OK(); } } diff --git a/util/posix_logger.h b/util/posix_logger.h new file mode 100644 index 0000000..0dbdeaa --- /dev/null +++ b/util/posix_logger.h @@ -0,0 +1,97 @@ +// Copyright 2011 Google Inc. All Rights Reserved. +// Author: sanjay@google.com (Sanjay Ghemawat) +// +// Logger implementation that can be shared by all environments +// where enough posix functionality is available. + +#ifndef STORAGE_LEVELDB_UTIL_POSIX_LOGGER_H_ +#define STORAGE_LEVELDB_UTIL_POSIX_LOGGER_H_ + +#include +#include +#include +#include +#include "leveldb/env.h" + +namespace leveldb { + +class PosixLogger : public Logger { + private: + FILE* file_; + uint64_t (*gettid_)(); // Return the thread id for the current thread + public: + PosixLogger(FILE* f, uint64_t (*gettid)()) : file_(f), gettid_(gettid) { } + virtual ~PosixLogger() { + fclose(file_); + } + virtual void Logv(const char* format, va_list ap) { + const uint64_t thread_id = (*gettid_)(); + + // We try twice: the first time with a fixed-size stack allocated buffer, + // and the second time with a much larger dynamically allocated buffer. + char buffer[500]; + for (int iter = 0; iter < 2; iter++) { + char* base; + int bufsize; + if (iter == 0) { + bufsize = sizeof(buffer); + base = buffer; + } else { + bufsize = 30000; + base = new char[bufsize]; + } + char* p = base; + char* limit = base + bufsize; + + struct timeval now_tv; + gettimeofday(&now_tv, NULL); + const time_t seconds = now_tv.tv_sec; + struct tm t; + localtime_r(&seconds, &t); + p += snprintf(p, limit - p, + "%04d/%02d/%02d-%02d:%02d:%02d.%06d %llx ", + t.tm_year + 1900, + t.tm_mon + 1, + t.tm_mday, + t.tm_hour, + t.tm_min, + t.tm_sec, + static_cast(now_tv.tv_usec), + static_cast(thread_id)); + + // Print the message + if (p < limit) { + va_list backup_ap; + va_copy(backup_ap, ap); + p += vsnprintf(p, limit - p, format, backup_ap); + va_end(backup_ap); + } + + // Truncate to available space if necessary + if (p >= limit) { + if (iter == 0) { + continue; // Try again with larger buffer + } else { + p = limit - 1; + } + } + + // Add newline if necessary + if (p == base || p[-1] != '\n') { + *p++ = '\n'; + } + + assert(p <= limit); + fwrite(base, 1, p - base, file_); + fflush(file_); + if (base != buffer) { + delete[] base; + } + break; + } + } +}; + +} + +#endif // STORAGE_LEVELDB_UTIL_POSIX_LOGGER_H_ -- cgit v1.2.1