14 files changed, 1156 insertions, 310 deletions

diff --git a/src/common/ClassHandler.cc b/src/common/ClassHandler.cc
index a2a13075bf3..405ab7388e4 100644
--- a/src/common/ClassHandler.cc
+++ b/src/common/ClassHandler.cc
@@ -14,7 +14,7 @@
 
 #define DOUT_SUBSYS osd
 #undef dout_prefix
-#define dout_prefix *_dout << dbeginl
+#define dout_prefix *_dout
 
 static ClassHandler::ClassData null_cls_data;
 
diff --git a/src/common/DoutStreambuf.cc b/src/common/DoutStreambuf.cc
new file mode 100644
index 00000000000..4279e4fbebd
--- /dev/null
+++ b/src/common/DoutStreambuf.cc
@@ -0,0 +1,628 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2010 Dreamhost
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#include "config.h"
+#include "common/DoutStreambuf.h"
+#include "common/errno.h"
+#include "common/Mutex.h"
+
+#include <values.h>
+#include <assert.h>
+#include <errno.h>
+#include <fstream>
+#include <iostream>
+#include <memory>
+#include <sstream>
+#include <streambuf>
+#include <string.h>
+#include <syslog.h>
+
+///////////////////////////// Globals /////////////////////////////
+// TODO: get rid of this lock using thread-local storage
+extern Mutex _dout_lock;
+
+//////////////////////// Helper functions //////////////////////////
+// Try a 0-byte write to a file descriptor to see if it open.
+static bool fd_is_open(int fd)
+{
+  char buf;
+  ssize_t res = TEMP_FAILURE_RETRY(write(fd, &buf, 0));
+  return (res == 0);
+}
+
+static bool empty(const char *str)
+{
+  if (!str)
+    return true;
+  if (!str[0])
+    return true;
+  return false;
+}
+
+static string cpp_str(const char *str)
+{
+  if (!str)
+    return "(NULL)";
+  if (str[0] == '\0')
+    return "(empty)";
+  return str;
+}
+
+static std::string normalize_relative(const char *from)
+{
+  if (from[0] == '/')
+    return string(from);
+
+  char c[512];
+  char *cwd = getcwd(c, sizeof(c));
+  ostringstream oss;
+  oss << cwd << "/" << from;
+  return oss.str();
+}
+
+/* Complain about errors even without a logfile */
+static void primitive_log(const std::string &str)
+{
+  std::cerr << str;
+  std::cerr.flush();
+  syslog(LOG_USER | LOG_NOTICE, "%s", str.c_str());
+}
+
+static inline bool prio_is_visible_on_stderr(int prio)
+{
+  return prio <= 5;
+}
+
+static inline int dout_prio_to_syslog_prio(int prio)
+{
+  if (prio <= 3)
+    return LOG_CRIT;
+  if (prio <= 5)
+    return LOG_ERR;
+  if (prio <= 15)
+    return LOG_WARNING;
+  if (prio <= 30)
+    return LOG_NOTICE;
+  if (prio <= 40)
+    return LOG_INFO;
+  return LOG_DEBUG;
+}
+
+static int safe_write(int fd, const char *buf, signed int len)
+{
+  int res;
+
+  assert(len != 0);
+  while (1) {
+    res = write(fd, buf, len);
+    if (res < 0) {
+      int err = errno;
+      if (err != EINTR) {
+	ostringstream oss;
+	oss << __func__ << ": failed to write to fd " << fd << ": "
+	    << cpp_strerror(err) << "\n";
+	primitive_log(oss.str());
+	return err;
+      }
+    }
+    len -= res;
+    buf += res;
+    if (len <= 0)
+      return 0;
+  }
+}
+
+static std::string get_basename(const std::string &filename)
+{
+  size_t last_slash = filename.find_last_of("/");
+  if (last_slash == std::string::npos)
+    return filename;
+  return filename.substr(last_slash + 1);
+}
+
+static std::string get_dirname(const std::string &filename)
+{
+  size_t last_slash = filename.find_last_of("/");
+  if (last_slash == std::string::npos)
+    return ".";
+  if (last_slash == 0)
+    return filename;
+  return filename.substr(0, last_slash);
+}
+
+static int create_symlink(string oldpath, const string &newpath)
+{
+  // Create relative symlink if the files are in the same directory
+  if (get_dirname(oldpath) == get_dirname(newpath)) {
+    oldpath = string("./") + get_basename(oldpath);
+  }
+
+  while (1) {
+    if (::symlink(oldpath.c_str(), newpath.c_str()) == 0)
+      return 0;
+    int err = errno;
+    if (err == EEXIST) {
+      // Handle EEXIST
+      if (::unlink(newpath.c_str())) {
+	err = errno;
+	ostringstream oss;
+	oss << __func__ << ": failed to remove '" << newpath << "': "
+	    << cpp_strerror(err) << "\n";
+	primitive_log(oss.str());
+	return err;
+      }
+    }
+    else {
+      // Other errors
+      ostringstream oss;
+      oss << __func__ << ": failed to symlink(oldpath='" << oldpath
+	  << "', newpath='" << newpath << "'): " << cpp_strerror(err) << "\n";
+      primitive_log(oss.str());
+      return err;
+    }
+  }
+}
+
+///////////////////////////// DoutStreambuf /////////////////////////////
+template <typename charT, typename traits>
+DoutStreambuf<charT, traits>::DoutStreambuf()
+  : flags(0), ofd(-1)
+{
+  // Initialize get pointer to zero so that underflow is called on the first read.
+  this->setg(0, 0, 0);
+
+  // Initialize output_buffer
+  _clear_output_buffer();
+}
+
+// This function is called when the output buffer is filled.
+// In this function, the buffer should be written to wherever it should
+// be written to (in this case, the streambuf object that this is controlling).
+template <typename charT, typename traits>
+typename DoutStreambuf<charT, traits>::int_type
+DoutStreambuf<charT, traits>::overflow(DoutStreambuf<charT, traits>::int_type c)
+{
+  {
+    // zero-terminate the buffer
+    charT* end_ptr = this->pptr();
+    *end_ptr++ = '\0';
+    *end_ptr++ = '\0';
+//    char buf[1000];
+//    hex2str(obuf, end_ptr - obuf, buf, sizeof(buf));
+//    printf("overflow buffer: '%s'\n", buf);
+  }
+
+  // Loop over all lines in the buffer.
+  int prio = 100;
+  charT* start = obuf;
+  while (true) {
+    char* end = strchrnul(start, '\n');
+    if (start == end) {
+      if (*start == '\0')
+	break;
+      // skip zero-length lines
+      ++start;
+      continue;
+    }
+    if (*start == '\1') {
+      // Decode some control characters
+      ++start;
+      unsigned char tmp = *((unsigned char*)start);
+      prio = tmp - 11;
+      ++start;
+    }
+    *end = '\n';
+    char next = *(end+1);
+    *(end+1) = '\0';
+
+    // Now 'start' points to a NULL-terminated string, which we want to
+    // output with priority 'prio'
+    int len = strlen(start);
+    if (flags & DOUTSB_FLAG_SYSLOG) {
+      //printf("syslogging: '%s' len=%d\n", start, len);
+      syslog(LOG_USER | dout_prio_to_syslog_prio(prio), "%s", start);
+    }
+    if (flags & DOUTSB_FLAG_STDOUT) {
+      // Just write directly out to the stdout fileno. There's no point in
+      // using something like fputs to write to a temporary buffer,
+      // because we would just have to flush that temporary buffer
+      // immediately.
+      if (safe_write(STDOUT_FILENO, start, len))
+	flags &= ~DOUTSB_FLAG_STDOUT;
+    }
+    if (flags & DOUTSB_FLAG_STDERR) {
+      // Only write to stderr if the message is important enough.
+      if (prio_is_visible_on_stderr(prio)) {
+	if (safe_write(STDERR_FILENO, start, len))
+	  flags &= ~DOUTSB_FLAG_STDERR;
+      }
+    }
+    if (flags & DOUTSB_FLAG_OFILE) {
+      if (safe_write(ofd, start, len))
+	flags &= ~DOUTSB_FLAG_OFILE;
+    }
+
+    *(end+1) = next;
+    start = end + 1;
+  }
+
+  _clear_output_buffer();
+
+  // A value different than EOF (or traits::eof() for other traits) signals success.
+  // If the function fails, either EOF (or traits::eof() for other traits) is returned or an
+  // exception is thrown.
+  return traits_ty::not_eof(c);
+}
+
+template <typename charT, typename traits>
+void DoutStreambuf<charT, traits>::handle_stderr_closed()
+{
+  assert(_dout_lock.is_locked());
+  flags &= ~DOUTSB_FLAG_STDERR;
+}
+
+template <typename charT, typename traits>
+void DoutStreambuf<charT, traits>::handle_stdout_closed()
+{
+  assert(_dout_lock.is_locked());
+  flags &= ~DOUTSB_FLAG_STDOUT;
+}
+
+template <typename charT, typename traits>
+void DoutStreambuf<charT, traits>::read_global_config()
+{
+  assert(_dout_lock.is_locked());
+  flags = 0;
+
+  if (g_conf.log_to_syslog) {
+    flags |= DOUTSB_FLAG_SYSLOG;
+  }
+  if (g_conf.log_to_stdout) {
+    if (fd_is_open(STDOUT_FILENO)) {
+      flags |= DOUTSB_FLAG_STDOUT;
+    }
+  }
+  if (fd_is_open(STDERR_FILENO)) {
+    flags |= DOUTSB_FLAG_STDERR;
+  }
+  if (g_conf.log_to_file) {
+    if (_read_ofile_config()) {
+      flags |= DOUTSB_FLAG_OFILE;
+    }
+  }
+}
+
+template <typename charT, typename traits>
+void DoutStreambuf<charT, traits>::
+set_flags(int flags_)
+{
+  assert(_dout_lock.is_locked());
+  flags = flags_;
+}
+
+template <typename charT, typename traits>
+void DoutStreambuf<charT, traits>::
+set_prio(int prio)
+{
+  charT* p = this->pptr();
+  *p++ = '\1';
+  unsigned char val = (prio + 11);
+  *p++ = val;
+  this->pbump(2);
+}
+
+template <typename charT, typename traits>
+int DoutStreambuf<charT, traits>::handle_pid_change()
+{
+  assert(_dout_lock.is_locked());
+  if (!(flags & DOUTSB_FLAG_OFILE))
+    return 0;
+
+  string new_opath(_calculate_opath());
+  if (opath == new_opath)
+    return 0;
+
+  if (!isym_path.empty()) {
+    // Re-create the instance symlink
+    int ret = create_symlink(new_opath, isym_path);
+    if (ret) {
+      ostringstream oss;
+      oss << __func__ << ": failed to (re)create instance symlink";
+      primitive_log(oss.str());
+      return ret;
+    }
+  }
+
+  if (!rsym_path.empty()) {
+    // Re-create the rank symlink
+    int ret = create_symlink(new_opath, rsym_path);
+    if (ret) {
+      ostringstream oss;
+      oss << __func__ << ": failed to (re)create rank symlink";
+      primitive_log(oss.str());
+      return ret;
+    }
+  }
+
+  int ret = ::rename(opath.c_str(), new_opath.c_str());
+  if (ret) {
+    int err = errno;
+    ostringstream oss;
+    oss << __func__ << ": failed to rename '" << opath << "' to "
+        << "'" << new_opath << "': " << cpp_strerror(err) << "\n";
+    primitive_log(oss.str());
+    return err;
+  }
+
+  opath = new_opath;
+
+  return 0;
+}
+
+template <typename charT, typename traits>
+int DoutStreambuf<charT, traits>::create_rank_symlink(int n)
+{
+  Mutex::Locker l(_dout_lock);
+
+  if (!(flags & DOUTSB_FLAG_OFILE))
+    return 0;
+
+  ostringstream rss;
+  std::string symlink_dir(_get_symlink_dir());
+  rss << symlink_dir << "/" << g_conf.type << "." << n;
+  string rsym_path_(rss.str());
+  int ret = create_symlink(opath, rsym_path_);
+  if (ret) {
+    ostringstream oss;
+    oss << __func__ << ": failed to create rank symlink with n = "
+	<< n << "\n";
+    primitive_log(oss.str());
+    return ret;
+  }
+
+  rsym_path = rsym_path_;
+  return 0;
+}
+
+template <typename charT, typename traits>
+std::string DoutStreambuf<charT, traits>::config_to_str() const
+{
+  assert(_dout_lock.is_locked());
+  ostringstream oss;
+  oss << "g_conf.log_to_syslog = " << g_conf.log_to_syslog << "\n";
+  oss << "g_conf.log_to_stdout = " << g_conf.log_to_stdout << "\n";
+  oss << "g_conf.log_to_file = " << g_conf.log_to_file << "\n";
+  oss << "g_conf.log_file = '" << cpp_str(g_conf.log_file) << "'\n";
+  oss << "g_conf.log_dir = '" << cpp_str(g_conf.log_dir) << "'\n";
+  oss << "g_conf.g_conf.log_per_instance = '"
+      << g_conf.log_per_instance << "'\n";
+  oss << "flags = 0x" << std::hex << flags << std::dec << "\n";
+  oss << "ofd = " << ofd << "\n";
+  oss << "opath = '" << opath << "'\n";
+  oss << "isym_path = '" << isym_path << "'\n";
+  oss << "rsym_path = '" << rsym_path << "'\n";
+  oss << "log_sym_history = " << g_conf.log_sym_history  << "\n";
+  return oss.str();
+}
+
+// This is called to flush the buffer.
+// This is called when we're done with the file stream (or when .flush() is called).
+template <typename charT, typename traits>
+typename DoutStreambuf<charT, traits>::int_type
+DoutStreambuf<charT, traits>::sync()
+{
+  //std::cout << "flush!" << std::endl;
+  typename DoutStreambuf<charT, traits>::int_type
+    ret(this->overflow(traits_ty::eof()));
+  if (ret == traits_ty::eof())
+    return -1;
+
+  return 0;
+}
+
+template <typename charT, typename traits>
+typename DoutStreambuf<charT, traits>::int_type
+DoutStreambuf<charT, traits>::underflow()
+{
+  // We can't read from this
+  // TODO: some more elegant way of preventing callers from trying to get input from this stream
+  assert(0);
+}
+
+template <typename charT, typename traits>
+void DoutStreambuf<charT, traits>::_clear_output_buffer()
+{
+  // Set up the put pointer.
+  // Overflow is called when this buffer is filled
+  this->setp(obuf, obuf + OBUF_SZ - 5);
+}
+
+template <typename charT, typename traits>
+std::string DoutStreambuf<charT, traits>::_calculate_opath() const
+{
+  assert(_dout_lock.is_locked());
+
+  // If g_conf.log_file was specified, that takes the highest priority
+  if (!empty(g_conf.log_file)) {
+    return normalize_relative(g_conf.log_file);
+  }
+
+  string log_dir;
+  if (empty(g_conf.log_dir))
+    log_dir = normalize_relative(".");
+  else
+    log_dir = normalize_relative(g_conf.log_dir);
+
+  if (g_conf.log_per_instance) {
+    char hostname[255];
+    memset(hostname, 0, sizeof(hostname));
+    int ret = gethostname(hostname, sizeof(hostname));
+    if (ret) {
+      int err = errno;
+      ostringstream oss;
+      oss << __func__ << ": error calling gethostname: " << cpp_strerror(err) << "\n";
+      primitive_log(oss.str());
+      return "";
+    }
+    ostringstream oss;
+    oss << log_dir << "/" << hostname << "." << getpid();
+    return oss.str();
+  }
+  else {
+    ostringstream oss;
+    oss << log_dir << "/" << g_conf.type << "." << g_conf.id << ".log";
+    return oss.str();
+  }
+}
+
+template <typename charT, typename traits>
+std::string DoutStreambuf<charT, traits>::_get_symlink_dir() const
+{
+  if (!empty(g_conf.log_sym_dir))
+    return normalize_relative(g_conf.log_sym_dir);
+  else
+    return get_dirname(opath);
+}
+
+template <typename charT, typename traits>
+bool DoutStreambuf<charT, traits>::_read_ofile_config()
+{
+  int ret;
+
+  isym_path = "";
+  rsym_path = "";
+  opath = _calculate_opath();
+  if (opath.empty()) {
+    ostringstream oss;
+    oss << __func__ << ": _calculate_opath failed.\n";
+    primitive_log(oss.str());
+    return false;
+  }
+
+  if (empty(g_conf.log_file) && g_conf.log_per_instance) {
+    // Calculate instance symlink path (isym_path)
+    ostringstream iss;
+    std::string symlink_dir(_get_symlink_dir());
+    iss << symlink_dir << "/" << g_conf.type << "." << g_conf.id;
+    isym_path = iss.str();
+
+    // Rotate isym_path
+    ret = _rotate_files(isym_path);
+    if (ret) {
+      ostringstream oss;
+      oss << __func__ << ": failed to rotate instance symlinks";
+      primitive_log(oss.str());
+      return ret;
+    }
+
+    // Create isym_path
+    ret = create_symlink(opath, isym_path);
+    if (ret) {
+      ostringstream oss;
+      oss << __func__ << ": failed to create instance symlink";
+      primitive_log(oss.str());
+      return ret;
+    }
+  }
+
+  assert(ofd == -1);
+  ofd = open(opath.c_str(),
+	    O_CREAT | O_WRONLY | O_APPEND, S_IWUSR | S_IRUSR);
+  if (ofd < 0) {
+    int err = errno;
+    ostringstream oss;
+    oss << "failed to open log file '" << opath << "': "
+	<< cpp_strerror(err) << "\n";
+    primitive_log(oss.str());
+    return false;
+  }
+
+  return true;
+}
+
+template <typename charT, typename traits>
+int DoutStreambuf<charT, traits>::_rotate_files(const std::string &base)
+{
+  // Given a file name base, and a directory like this:
+  // base
+  // base.1
+  // base.2
+  // base.3
+  // base.4
+  // unrelated_blah
+  // unrelated_blah.1
+  //
+  // We'll take the following actions:
+  // base		  rename to base.1
+  // base.1		  rename to base.2
+  // base.2		  rename to base.3
+  // base.3		  (unlink)
+  // base.4		  (unlink)
+  // unrelated_blah	  (do nothing)
+  // unrelated_blah.1	  (do nothing)
+
+  signed int i;
+  for (i = -1; i < INT_MAX; ++i) {
+    ostringstream oss;
+    oss << base;
+    if (i != -1)
+      oss << "." << i;
+    string path(oss.str());
+
+    if (::access(path.c_str(), R_OK | W_OK))
+      break;
+  }
+  signed int max_symlink = i - 1;
+
+  for (signed int j = max_symlink; j >= -1; --j) {
+    ostringstream oss;
+    oss << base;
+    if (j != -1)
+      oss << "." << j;
+    string path(oss.str());
+
+    signed int k = j + 1;
+    if (k >= g_conf.log_sym_history) {
+      if (::unlink(path.c_str())) {
+	int err = errno;
+	ostringstream ess;
+	ess << __func__ << ": failed to unlink '" << path << "': "
+	    << cpp_strerror(err) << "\n";
+	primitive_log(ess.str());
+	return err;
+      }
+      //*_dout << "---- " << getpid() << " removed " << path << " ----"
+      //       << std::endl;
+    }
+    else {
+      ostringstream pss;
+      pss << base << "." << k;
+      string new_path(pss.str());
+      if (::rename(path.c_str(), new_path.c_str())) {
+	int err = errno;
+	ostringstream ess;
+	ess << __func__ << ": failed to rename '" << path << "' to "
+	    << "'" << new_path << "': " << cpp_strerror(err) << "\n";
+	primitive_log(ess.str());
+	return err;
+      }
+//      *_dout << "---- " << getpid() << " renamed " << path << " -> "
+//      << newpath << " ----" << std::endl;
+    }
+  }
+  return 0;
+}
+
+// Explicit template instantiation
+template class DoutStreambuf <char>;
diff --git a/src/common/DoutStreambuf.h b/src/common/DoutStreambuf.h
new file mode 100644
index 00000000000..76968198b24
--- /dev/null
+++ b/src/common/DoutStreambuf.h
@@ -0,0 +1,106 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2010 Dreamhost
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+/*
+ * DoutStreambuf
+ *
+ * The stream buffer used by dout
+ */
+#ifndef CEPH_DOUT_STREAMBUF_H
+#define CEPH_DOUT_STREAMBUF_H
+
+#include <iosfwd>
+#include <string>
+
+template <typename charT, typename traits = std::char_traits<charT> >
+class DoutStreambuf : public std::basic_streambuf<charT, traits>
+{
+public:
+  enum dout_streambuf_flags_t {
+    DOUTSB_FLAG_SYSLOG =          0x01,
+    DOUTSB_FLAG_STDOUT =          0x02,
+    DOUTSB_FLAG_STDERR =          0x04,
+    DOUTSB_FLAG_OFILE =           0x08,
+  };
+
+  typedef traits traits_ty;
+  typedef typename traits_ty::int_type int_type;
+  typedef typename traits_ty::pos_type pos_type;
+  typedef typename traits_ty::off_type off_type;
+
+  // The size of the output buffer.
+  static const size_t OBUF_SZ = 32000;
+
+  DoutStreambuf();
+
+  // Call when you close stderr.  Not strictly necessary, since we would get an
+  // error the next time we tried to write to stdedrr. But nicer than waiting
+  // for the error to happen.
+  void handle_stderr_closed();
+
+  // Call when you close stdout.
+  void handle_stdout_closed();
+
+  // Set the flags based on the global configuration
+  void read_global_config();
+
+  // Set the flags directly (for debug use only)
+  void set_flags(int flags_);
+
+  // Set the priority of the messages being put into the stream
+  void set_prio(int prio);
+
+  // Call after calling daemon()
+  // A change in the process ID sometimes requires us to change our output
+  // path name.
+  int handle_pid_change();
+
+  // Create a rank symlink to the log file
+  int create_rank_symlink(int n);
+
+  std::string config_to_str() const;
+
+protected:
+  // Called when the buffer fills up
+  virtual int_type overflow(int_type c);
+
+  // Called when the buffer is flushed
+  virtual int_type sync();
+
+  // Called when we try to read, but there are no more chars in the buffer
+  virtual int_type underflow();
+
+private:
+  void _clear_output_buffer();
+  std::string _calculate_opath() const;
+  std::string _get_symlink_dir() const;
+  bool _read_ofile_config();
+  int _rotate_files(const std::string &base);
+
+  // Output buffer
+  charT obuf[OBUF_SZ];
+
+  // Output flags
+  int flags;
+
+  // ofile stuff
+  int ofd;
+  std::string opath;
+
+  // symlinks
+  std::string isym_path;
+  std::string rsym_path;
+};
+
+#endif
diff --git a/src/common/Finisher.cc b/src/common/Finisher.cc
index 05fba0c0b3e..7315b9c4f48 100644
--- a/src/common/Finisher.cc
+++ b/src/common/Finisher.cc
@@ -5,7 +5,7 @@
 #include "common/debug.h"
 #define DOUT_SUBSYS finisher
 #undef dout_prefix
-#define dout_prefix *_dout << dbeginl << "finisher(" << this << ") "
+#define dout_prefix *_dout << "finisher(" << this << ") "
 
 void Finisher::start()
 {
diff --git a/src/common/LogClient.cc b/src/common/LogClient.cc
index 921adcc5485..26f054411d0 100644
--- a/src/common/LogClient.cc
+++ b/src/common/LogClient.cc
@@ -27,6 +27,7 @@
 #include <iostream>
 #include <errno.h>
 #include <sys/stat.h>
+#include <syslog.h>
 
 #ifdef DARWIN
 #include <sys/param.h>
@@ -37,25 +38,59 @@
 
 #include "config.h"
 
-void LogClient::log(log_type type, const char *s)
+/*
+ * Given a clog log_type, return the equivalent syslog priority
+ */
+static inline int clog_type_to_syslog_prio(clog_type t)
+{
+  switch (t) {
+    case CLOG_DEBUG:
+      return LOG_DEBUG;
+    case CLOG_INFO:
+      return LOG_INFO;
+    case CLOG_WARN:
+      return LOG_WARNING;
+    case CLOG_ERROR:
+      return LOG_ERR;
+    case CLOG_SEC:
+      return LOG_CRIT;
+    default:
+      assert(0);
+      return 0;
+  }
+}
+
+LogClientTemp::LogClientTemp(clog_type type_, LogClient &parent_)
+  : type(type_), parent(parent_)
 {
-  string str(s);
-  log(type, str);
 }
 
-void LogClient::log(log_type type, stringstream& ss)
+LogClientTemp::LogClientTemp(const LogClientTemp &rhs)
+  : type(rhs.type), parent(rhs.parent)
+{
+  // don't want to-- nor can we-- copy the ostringstream
+}
+
+LogClientTemp::~LogClientTemp()
+{
+  if (ss.peek() != EOF)
+    parent.do_log(type, ss);
+}
+
+void LogClient::do_log(clog_type type, std::stringstream& ss)
 {
   while (!ss.eof()) {
     string s;
     getline(ss, s);
-    log(type, s);
+    if (!s.empty())
+      do_log(type, s);
   }
 }
 
-void LogClient::log(log_type type, string& s)
+void LogClient::do_log(clog_type type, const std::string& s)
 {
   Mutex::Locker l(log_lock);
-  dout(0) << "log " << (log_type)type << " : " << s << dendl;
+  dout(0) << "log " << type << " : " << s << dendl;
   LogEntry e;
   e.who = messenger->get_myinst();
   e.stamp = g_clock.now();
@@ -76,6 +111,33 @@ void LogClient::send_log()
 
 void LogClient::_send_log()
 {
+  if (g_conf.clog_to_syslog)
+    _send_log_to_syslog();
+  if (g_conf.clog_to_monitors)
+    _send_log_to_monitors();
+}
+
+void LogClient::_send_log_to_syslog()
+{
+  std::deque<LogEntry>::const_reverse_iterator rbegin = log_queue.rbegin();
+  std::deque<LogEntry>::const_reverse_iterator rend = log_queue.rend();
+  for (std::deque<LogEntry>::const_reverse_iterator a = rbegin; a != rend; ++a) {
+    const LogEntry entry(*a);
+    if (entry.seq < last_syslog)
+      break;
+    ostringstream oss;
+    oss << entry;
+    string str(oss.str());
+    syslog(clog_type_to_syslog_prio(entry.type) | LOG_USER, "%s", str.c_str());
+  }
+  if (rbegin != rend) {
+    const LogEntry entry(*rbegin);
+    last_syslog = entry.seq;
+  }
+}
+
+void LogClient::_send_log_to_monitors()
+{
   if (log_queue.empty())
     return;
   MLog *log = new MLog(monmap->get_fsid(), log_queue);
@@ -96,9 +158,14 @@ void LogClient::handle_log_ack(MLogAck *m)
   dout(10) << "handle_log_ack " << *m << dendl;
 
   version_t last = m->last;
-  while (log_queue.size() && log_queue.begin()->seq <= last) {
-    dout(10) << " logged " << log_queue.front() << dendl;
-    log_queue.pop_front();
+
+  deque<LogEntry>::iterator q = log_queue.begin();
+  while (q != log_queue.end()) {
+    const LogEntry &entry(*q);
+    if (entry.seq > last)
+      break;
+    dout(10) << " logged " << entry << dendl;
+    q = log_queue.erase(q);
   }
   m->put();
 }
diff --git a/src/common/LogClient.h b/src/common/LogClient.h
index a6584739f64..c63bddd7302 100644
--- a/src/common/LogClient.h
+++ b/src/common/LogClient.h
@@ -15,51 +15,107 @@
 #ifndef CEPH_LOGCLIENT_H
 #define CEPH_LOGCLIENT_H
 
-#include "msg/Dispatcher.h"
-
+#include "common/LogEntry.h"
 #include "common/Mutex.h"
-#include "include/LogEntry.h"
+#include "msg/Dispatcher.h"
 
+#include <iosfwd>
 #include <sstream>
 
-class Messenger;
+class LogClient;
 class MLog;
 class MLogAck;
-class MonMap;
+class Messenger;
 class MonClient;
+class MonMap;
 
-class LogClient : public Dispatcher {
-  Messenger *messenger;
-  MonMap *monmap;
-  MonClient *monc;
+class LogClientTemp
+{
+public:
+  LogClientTemp(clog_type type_, LogClient &parent_);
+  LogClientTemp(const LogClientTemp &rhs);
+  ~LogClientTemp();
+
+  template<typename T>
+  std::ostream& operator<<(const T& rhs)
+  {
+    return ss << rhs;
+  }
+
+private:
+  clog_type type;
+  LogClient &parent;
+  stringstream ss;
+};
 
+class LogClient : public Dispatcher
+{
+public:
+  enum logclient_flag_t {
+    NO_FLAGS = 0,
+    FLAG_SYNC = 0x1,
+  };
+
+  LogClient(Messenger *m, MonMap *mm, MonClient *mc, enum logclient_flag_t flags) :
+    messenger(m), monmap(mm), monc(mc), is_synchronous(flags & FLAG_SYNC),
+    log_lock("LogClient::log_lock"), last_log(0), last_syslog(0) { }
+
+  void send_log();
+  void handle_log_ack(MLogAck *m);
+  void set_synchronous(bool sync) { is_synchronous = sync; }
+
+  LogClientTemp debug() {
+    return LogClientTemp(CLOG_DEBUG, *this);
+  }
+  void debug(std::stringstream &s) {
+    do_log(CLOG_DEBUG, s);
+  }
+  LogClientTemp info() {
+    return LogClientTemp(CLOG_INFO, *this);
+  }
+  void info(std::stringstream &s) {
+    do_log(CLOG_INFO, s);
+  }
+  LogClientTemp warn() {
+    return LogClientTemp(CLOG_WARN, *this);
+  }
+  void warn(std::stringstream &s) {
+    do_log(CLOG_WARN, s);
+  }
+  LogClientTemp error() {
+    return LogClientTemp(CLOG_ERROR, *this);
+  }
+  void error(std::stringstream &s) {
+    do_log(CLOG_ERROR, s);
+  }
+  LogClientTemp sec() {
+    return LogClientTemp(CLOG_SEC, *this);
+  }
+  void sec(std::stringstream &s) {
+    do_log(CLOG_SEC, s);
+  }
+
+private:
+  void do_log(clog_type type, std::stringstream& ss);
+  void do_log(clog_type type, const std::string& s);
   bool ms_dispatch(Message *m);
-  bool is_synchronous;
   void _send_log();
-
+  void _send_log_to_syslog();
+  void _send_log_to_monitors();
   void ms_handle_connect(Connection *con);
-
   bool ms_handle_reset(Connection *con) { return false; }
   void ms_handle_remote_reset(Connection *con) {}
 
-
- public:
-
-  // -- log --
+  Messenger *messenger;
+  MonMap *monmap;
+  MonClient *monc;
+  bool is_synchronous;
   Mutex log_lock;
-  deque<LogEntry> log_queue;
   version_t last_log;
+  uint64_t last_syslog;
+  std::deque<LogEntry> log_queue;
 
-  void log(log_type type, const char *s);
-  void log(log_type type, string& s);
-  void log(log_type type, stringstream& s);
-  void send_log();
-  void handle_log_ack(MLogAck *m);
-  void set_synchronous(bool sync) { is_synchronous = sync; }
-
-  LogClient(Messenger *m, MonMap *mm, MonClient *mc=0) : 
-    messenger(m), monmap(mm), monc(mc), is_synchronous(false),
-    log_lock("LogClient::log_lock"), last_log(0) { }
+  friend class LogClientTemp;
 };
 
 #endif
diff --git a/src/common/LogEntry.h b/src/common/LogEntry.h
new file mode 100644
index 00000000000..04c80b607a4
--- /dev/null
+++ b/src/common/LogEntry.h
@@ -0,0 +1,144 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2006 Sage Weil <sage@newdream.net>
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software 
+ * Foundation.  See file COPYING.
+ * 
+ */
+
+#ifndef CEPH_LOGENTRY_H
+#define CEPH_LOGENTRY_H
+
+#include "include/types.h"
+#include "include/encoding.h"
+
+typedef enum {
+  CLOG_DEBUG = 0,
+  CLOG_INFO = 1,
+  CLOG_SEC = 2,
+  CLOG_WARN = 3,
+  CLOG_ERROR = 4,
+} clog_type;
+
+struct LogEntryKey {
+  entity_inst_t who;
+  utime_t stamp;
+  uint64_t seq;
+
+  LogEntryKey() {}
+  LogEntryKey(entity_inst_t w, utime_t t, uint64_t s) : who(w), stamp(t), seq(s) {}
+
+  void encode(bufferlist& bl) const {
+    ::encode(who, bl);
+    ::encode(stamp, bl);
+    ::encode(seq, bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    ::decode(who, bl);
+    ::decode(stamp, bl);
+    ::decode(seq, bl);
+  }
+};
+WRITE_CLASS_ENCODER(LogEntryKey)
+
+static inline bool operator==(const LogEntryKey& l, const LogEntryKey& r) {
+  return l.who == r.who && l.stamp == r.stamp && l.seq == r.seq;
+}
+
+struct LogEntry {
+  entity_inst_t who;
+  utime_t stamp;
+  uint64_t seq;
+  clog_type type;
+  string msg;
+
+  LogEntryKey key() const { return LogEntryKey(who, stamp, seq); }
+
+  void encode(bufferlist& bl) const {
+    __u8 v = 1;
+    ::encode(v, bl);
+    __u16 t = type;
+    ::encode(who, bl);
+    ::encode(stamp, bl);
+    ::encode(seq, bl);
+    ::encode(t, bl);
+    ::encode(msg, bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    __u8 v;
+    ::decode(v, bl);
+    __u16 t;
+    ::decode(who, bl);
+    ::decode(stamp, bl);
+    ::decode(seq, bl);
+    ::decode(t, bl);
+    type = (clog_type)t;
+    ::decode(msg, bl);
+  }
+};
+WRITE_CLASS_ENCODER(LogEntry)
+
+struct LogSummary {
+  version_t version;
+  list<LogEntry> tail;
+
+  LogSummary() : version(0) {}
+
+  void add(const LogEntry& e) {
+    tail.push_back(e);
+    while (tail.size() > 50)
+      tail.pop_front();
+  }
+  bool contains(LogEntryKey k) const {
+    for (list<LogEntry>::const_iterator p = tail.begin();
+	 p != tail.end();
+	 p++)
+      if (p->key() == k) return true;
+    return false;
+  }
+
+  void encode(bufferlist& bl) const {
+    __u8 v = 1;
+    ::encode(v, bl);
+    ::encode(version, bl);
+    ::encode(tail, bl);
+  }
+  void decode(bufferlist::iterator& bl) {
+    __u8 v;
+    ::decode(v, bl);
+    ::decode(version, bl);
+    ::decode(tail, bl);
+  }
+};
+WRITE_CLASS_ENCODER(LogSummary)
+
+inline ostream& operator<<(ostream& out, clog_type t)
+{
+  switch (t) {
+  case CLOG_DEBUG:
+    return out << "[DBG]";
+  case CLOG_INFO:
+    return out << "[INF]";
+  case CLOG_WARN:
+    return out << "[WRN]";
+  case CLOG_ERROR:
+    return out << "[ERR]";
+  case CLOG_SEC:
+    return out << "[SEC]";
+  default:
+    return out << "[???]";
+  }
+}
+
+inline ostream& operator<<(ostream& out, const LogEntry& e)
+{
+  return out << e.stamp << " " << e.who << " " << e.seq << " : " << e.type << " " << e.msg;
+}
+
+#endif
diff --git a/src/common/Thread.h b/src/common/Thread.h
index a567d2dbac4..e7da795dbdd 100644
--- a/src/common/Thread.h
+++ b/src/common/Thread.h
@@ -75,7 +75,7 @@ class Thread {
 
     if (r) {
       char buf[80];
-      generic_derr(0) << "pthread_create failed with message: " << strerror_r(r, buf, sizeof(buf)) << dendl;
+      generic_dout(0) << "pthread_create failed with message: " << strerror_r(r, buf, sizeof(buf)) << dendl;
     } else {
       _num_threads.inc();
       generic_dout(10) << "thread " << thread_id << " start" << dendl;
@@ -84,7 +84,7 @@ class Thread {
   }
   int join(void **prval = 0) {
     if (thread_id == 0) {
-      generic_derr(0) << "WARNING: join on thread that was never started" << dendl;
+      generic_dout(0) << "WARNING: join on thread that was never started" << dendl;
       assert(0);
       return -EINVAL;   // never started.
     }
@@ -93,17 +93,17 @@ class Thread {
     if (status != 0) {
       switch (status) {
       case -EINVAL:
-	generic_derr(0) << "thread " << thread_id << " join status = EINVAL" << dendl;
+	generic_dout(0) << "thread " << thread_id << " join status = EINVAL" << dendl;
 	break;
       case -ESRCH:
-	generic_derr(0) << "thread " << thread_id << " join status = ESRCH" << dendl;
+	generic_dout(0) << "thread " << thread_id << " join status = ESRCH" << dendl;
 	assert(0);
 	break;
       case -EDEADLK:
-	generic_derr(0) << "thread " << thread_id << " join status = EDEADLK" << dendl;
+	generic_dout(0) << "thread " << thread_id << " join status = EDEADLK" << dendl;
 	break;
       default:
-	generic_derr(0) << "thread " << thread_id << " join status = " << status << dendl;
+	generic_dout(0) << "thread " << thread_id << " join status = " << status << dendl;
       }
       assert(0); // none of these should happen.
     }
diff --git a/src/common/Timer.cc b/src/common/Timer.cc
index 7a3439e37f1..9980790b36e 100644
--- a/src/common/Timer.cc
+++ b/src/common/Timer.cc
@@ -22,7 +22,7 @@
 
 #define DOUT_SUBSYS timer
 #undef dout_prefix
-#define dout_prefix *_dout << dbeginl << "timer(" << this << ")."
+#define dout_prefix *_dout << "timer(" << this << ")."
 
 #include <sstream>
 #include <signal.h>
diff --git a/src/common/WorkQueue.cc b/src/common/WorkQueue.cc
index cae4cef6c8f..a3f67e0a0d2 100644
--- a/src/common/WorkQueue.cc
+++ b/src/common/WorkQueue.cc
@@ -19,7 +19,7 @@
 
 #define DOUT_SUBSYS tp
 #undef dout_prefix
-#define dout_prefix *_dout << dbeginl << name << " "
+#define dout_prefix *_dout << name << " "
 
 
 void ThreadPool::worker()
diff --git a/src/common/common_init.cc b/src/common/common_init.cc
index f634247f8cb..a5c02b9fff1 100644
--- a/src/common/common_init.cc
+++ b/src/common/common_init.cc
@@ -44,13 +44,6 @@ void common_init(std::vector<const char*>& args, const char *module_type, bool i
   }
 #endif //HAVE_LIBTCMALLOC
 
-  if (g_conf.log_file && g_conf.log_file[0])
-    g_conf.log_to_stdout = false;
-
-  // open log file?
-  if (!g_conf.log_to_stdout)
-    _dout_open_log();
-
   if (init_keys && is_supported_auth(CEPH_AUTH_CEPHX)) {
     g_keyring.load(g_conf.keyring);
 
diff --git a/src/common/debug.cc b/src/common/debug.cc
index 86def0548a1..10cd6ceed9e 100644
--- a/src/common/debug.cc
+++ b/src/common/debug.cc
@@ -1,212 +1,70 @@
-
-#include "include/types.h"
-#include "config.h"
-#include "debug.h"
 #include "Mutex.h"
-#include "Clock.h"
-
 #include "ceph_ver.h"
+#include "common/DoutStreambuf.h"
+#include "config.h"
+#include "debug.h"
 
 #include <errno.h>
 #include <fstream>
 #include <iostream>
-using namespace std;
-
-#define _STR(x) #x
-#define STRINGIFY(x) _STR(x)
 
 // debug output
-Mutex _dout_lock("_dout_lock", false, false /* no lockdep */);  
-ostream *_dout = &std::cout;
-ostream *_derr = &std::cerr;
-char _dout_dir[PATH_MAX] = {0};
-char _dout_symlink_dir[PATH_MAX] = {0};
-char _dout_file[PATH_MAX] = {0};
-char _dout_path[PATH_MAX] = {0};
-char _dout_rank_symlink_path[PATH_MAX] = {0};
-char _dout_name_symlink_path[PATH_MAX] = {0};
-char *_dout_symlink_target = 0;   // _dout_path or _dout_file
-bool _dout_is_open = false;
+std::ostream *_dout = NULL;
+DoutStreambuf <char> *_doss = NULL;
 bool _dout_need_open = true;
-std::ofstream _dout_out;
-
-static void normalize_relative(const char *from, char *to, int tolen)
-{
-  if (from[0] == '/') 
-    strncpy(to, from, tolen);
-  else {
-    char *c = getcwd(to, tolen);
-    assert(c);
-    strncat(to, "/", tolen);
-    strncat(to, from, tolen);
-  }
-}
-
-static void build_log_paths()
-{
-  if (g_conf.log_file && g_conf.log_file[0]) {
-    normalize_relative(g_conf.log_file, _dout_path, sizeof(_dout_path));
-  } else {
-    if (g_conf.log_per_instance) {
-      char hostname[80];
-      gethostname(hostname, 79);
-      snprintf(_dout_file, sizeof(_dout_file), "%s.%d", hostname, getpid());
-    } else {
-      snprintf(_dout_file, sizeof(_dout_file), "%s.%s.log", g_conf.type, g_conf.id);
-    }
-    snprintf(_dout_path, sizeof(_dout_path), "%s/%s", _dout_dir, _dout_file);
-  }
-}
-
-static bool log_to_file()
-{
-  return (g_conf.log_dir || g_conf.log_file) && !g_conf.log_to_stdout;
-}
-
-static int create_symlink(const char *from)
-{
-  ::unlink(from);
-  int r = ::symlink(_dout_symlink_target, from);
-  if (r) {
-    char buf[80];
-    *_dout << "---- " << getpid() << " failed to symlink " << _dout_symlink_target
-	   << " from " << from
-	   << ": " << strerror_r(errno, buf, sizeof(buf)) << std::endl;
-  }
-  return r;
-}
-
-static void rotate_file(const char *fn, int max)
-{
-  char a[200], b[200];
-  // rotate out old
-  int n = 0;
-  while (1) {
-    struct stat st;
-    snprintf(a, sizeof(a), "%s.%lld", fn, (long long)n);
-    if (::lstat(a, &st) != 0)
-      break;
-    n++;
-  }
-  while (n >= 0) {
-    if (n)
-      snprintf(a, sizeof(a), "%s.%lld", fn, (long long)n-1);
-    else
-      snprintf(a, sizeof(a), "%s", fn);
-    if (n >= max) {
-      ::unlink(a);
-      *_dout << "---- " << getpid() << " removed " << a << " ----" << std::endl;
-    } else {
-      snprintf(b, sizeof(b), "%s.%lld", fn, (long long)n);
-      ::rename(a, b);
-      *_dout << "---- " << getpid() << " renamed " << a << " -> " << b << " ----" << std::endl;
-    }
-    n--;
-  }
-}
-
-static int create_name_symlink()
-{
-  int r = 0;
-  if (log_to_file() && g_conf.log_per_instance && !(g_conf.log_file && g_conf.log_file[0])) {
-    snprintf(_dout_name_symlink_path, sizeof(_dout_name_symlink_path),
-	     "%s/%s.%s", _dout_symlink_dir, g_conf.type, g_conf.id);
-
-    rotate_file(_dout_name_symlink_path, g_conf.log_sym_history);
-    r = create_symlink(_dout_name_symlink_path);
-  }
-  return r;
-}
+Mutex _dout_lock("_dout_lock", false, false /* no lockdep */);
 
+#define _STR(x) #x
+#define STRINGIFY(x) _STR(x)
 
 void _dout_open_log()
 {
-  bool need_symlink = false;
-
-  // logging enabled?
-  if (!log_to_file()) {
-    _dout_need_open = false;
-    return;
-  }
-
-  // calculate log dir, filename, etc.
-  // do this _once_.
-  if (!_dout_path[0]) {
-
-    // normalize paths
-    normalize_relative(g_conf.log_dir, _dout_dir, sizeof(_dout_dir));
-    if (!g_conf.log_sym_dir)
-      g_conf.log_sym_dir = strdup(g_conf.log_dir);
-    normalize_relative(g_conf.log_sym_dir, _dout_symlink_dir, sizeof(_dout_symlink_dir));
+  assert(_dout_need_open);
+  assert(_dout_lock.is_locked());
 
-    // make symlink targets absolute or relative?
-    if ((g_conf.log_file && g_conf.log_file[0]) ||
-	strcmp(_dout_symlink_dir, _dout_dir) == 0)
-      _dout_symlink_target = _dout_file;
-    else
-      _dout_symlink_target = _dout_path;
-
-    build_log_paths();
-
-    need_symlink = true;
+  if (!_doss) {
+    _doss = new DoutStreambuf <char>();
   }
-
-  _dout_out.close();
-  _dout_out.open(_dout_path, ios::out | ios::app);
-  if (!_dout_out.is_open()) {
-    std::cerr << "error opening output file " << _dout_path << std::endl;
-    _dout = &std::cout;
-  } else {
-    _dout_need_open = false;
-    _dout_is_open = true;
-    _dout = &_dout_out;
-    *_dout << g_clock.now() << " --- " << getpid() << " opened log "
-	   << _dout_path << " ---" << std::endl;
+  _doss->read_global_config();
+  if (!_dout) {
+    _dout = new std::ostream(_doss);
   }
-  *_dout << "ceph version " << VERSION << " (commit:" << STRINGIFY(CEPH_GIT_VER) << ")" << std::endl;
 
-  if (need_symlink)
-    create_name_symlink();
+  *_dout << "ceph version " << VERSION << " (commit:"
+	 << STRINGIFY(CEPH_GIT_VER) << ")" << std::endl;
+  _dout_need_open = false;
 }
 
-int dout_rename_output_file()  // after calling daemon()
+int dout_handle_daemonize()
 {
   Mutex::Locker l(_dout_lock);
-  if (log_to_file() && g_conf.log_per_instance) {
-    char oldpath[PATH_MAX];
-    char hostname[80];
-    gethostname(hostname, 79);
-
-    strcpy(oldpath, _dout_path);
-    
-    build_log_paths();
-
-    *_dout << "---- " << getpid() << " renamed log " << oldpath << " -> " << _dout_path << " ----" << std::endl;
-    ::rename(oldpath, _dout_path);
-
-    // $type.$id symlink
-    if (g_conf.log_per_instance && _dout_name_symlink_path[0])
-      create_symlink(_dout_name_symlink_path);
-    if (_dout_rank_symlink_path[0])
-      create_symlink(_dout_rank_symlink_path);
-  }
-  return 0;
+  _doss->handle_stdout_closed();
+  _doss->handle_stderr_closed();
+  return _doss->handle_pid_change();
 }
 
-int dout_create_rank_symlink(int64_t n)
+int dout_create_rank_symlink(int n)
 {
   Mutex::Locker l(_dout_lock);
-  int r = 0;
-  if (log_to_file() && !(g_conf.log_file && g_conf.log_file[0])) {
-    if (_dout_need_open)
-      _dout_open_log();
+  return _doss->create_rank_symlink(n);
+}
 
-    snprintf(_dout_rank_symlink_path, sizeof(_dout_rank_symlink_path),
-	     "%s/%s%lld", _dout_symlink_dir, g_conf.type, (long long)n);
-    r = create_symlink(_dout_rank_symlink_path);
+void hex2str(const char *s, int len, char *buf, int dest_len)
+{
+  int pos = 0;
+  for (int i=0; i<len && pos<dest_len; i++) {
+    if (i && !(i%8))
+      pos += snprintf(&buf[pos], dest_len-pos, " ");
+    if (i && !(i%16))
+      pos += snprintf(&buf[pos], dest_len-pos, "\n");
+    pos += snprintf(&buf[pos], dest_len-pos, "%.2x ", (int)(unsigned char)s[i]);
   }
-  return r;
 }
 
-
-
+void hexdump(string msg, const char *s, int len)
+{
+  int buf_len = len*4;
+  char buf[buf_len];
+  hex2str(s, len, buf, buf_len);
+  generic_dout(0) << msg << ":\n" << buf << dendl;
+}
diff --git a/src/common/debug.h b/src/common/debug.h
index 7b60c38abe8..af6fe9957e1 100644
--- a/src/common/debug.h
+++ b/src/common/debug.h
@@ -1,117 +1,87 @@
-// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*- 
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
 // vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2004-2010 Sage Weil <sage@newdream.net>
+ * Copyright (C) 2010 Dreamhost
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
 
 #ifndef CEPH_DEBUG_H
 #define CEPH_DEBUG_H
 
-#include "include/assert.h"
-#include "Mutex.h"
 #include "Clock.h"
+#include "Mutex.h"
+#include "common/DoutStreambuf.h"
+#include "common/likely.h"
+#include "include/assert.h"
 
-#include <ostream>
-using std::ostream;
-
-// the streams
-extern ostream *_dout;
-extern ostream *_derr;
-
-extern Mutex _dout_lock;
+#include <iosfwd>
 
+extern std::ostream *_dout;
+extern DoutStreambuf <char> *_doss;
 extern bool _dout_need_open;
-extern bool _dout_is_open;
+extern Mutex _dout_lock;
 
 extern void _dout_open_log();
 
-static inline void dout_open_log() {
-  _dout_lock.Lock();
-  _dout_open_log();
-  _dout_lock.Unlock();
-}
-extern int dout_rename_output_file();  // after calling daemon()
-extern int dout_create_rank_symlink(int64_t n);
+extern int dout_handle_daemonize();
 
-static inline void _dout_check_log() {
-  _dout_lock.Lock();
-  if (_dout_need_open)
-    _dout_open_log();
-  _dout_lock.Unlock();
-}
+extern int dout_create_rank_symlink(int n);
 
-static inline void _dout_begin_line() {
+static inline void _dout_begin_line(int prio) {
   _dout_lock.Lock();
-  if (_dout_need_open)
+  if (unlikely(_dout_need_open))
     _dout_open_log();
-  *_dout << g_clock.now() << " " << std::hex << pthread_self() << std::dec << " ";
-}
-static void _dout_begin_line_static() {
-  _dout_begin_line();
-}
 
-static inline void _dout_end_line() {
-  _dout_lock.Unlock();
-}
+  // Put priority information into dout
+  _doss->sputc(1);
+  _doss->sputc(prio + 11);
 
-struct _dbeginl_t { _dbeginl_t(int) {} };
-static const _dbeginl_t dbeginl = 0;
-inline ostream& operator<<(ostream& out, _dbeginl_t) {
-  _dout_begin_line();
-  return out;
+  // Some information that goes in every dout message
+  *_dout << g_clock.now() << " " << std::hex << pthread_self()
+	 << std::dec << " ";
 }
 
-struct _dbeginlstatic_t { _dbeginlstatic_t(int) {} };
-static const _dbeginlstatic_t dbeginlstatic = 0;
-inline ostream& operator<<(ostream& out, _dbeginlstatic_t) {
-  _dout_begin_line_static();
-  return out;
+static inline void _dout_end_line() {
+  _dout_lock.Unlock();
 }
 
 // intentionally conflict with endl
 class _bad_endl_use_dendl_t { public: _bad_endl_use_dendl_t(int) {} };
 static const _bad_endl_use_dendl_t endl = 0;
-inline ostream& operator<<(ostream& out, _bad_endl_use_dendl_t) {
+inline std::ostream& operator<<(std::ostream& out, _bad_endl_use_dendl_t) {
   assert(0 && "you are using the wrong endl.. use std::endl or dendl");
   return out;
 }
 
-
 // generic macros
-#define generic_dout(x) do { if ((x) <= g_conf.debug) { *_dout << dbeginl
-#define generic_derr(x) do { if ((x) <= g_conf.debug) { *_derr << dbeginl
+#define generic_dout(x) do { if ((x) <= g_conf.debug) {\
+  _dout_begin_line(x); *_dout
 
-#define pdout(x,p) do { if ((x) <= (p)) { *_dout << dbeginl
+#define pdout(x,p) do { if ((x) <= (p)) {\
+  _dout_begin_line(x); *_dout
 
 #define debug_DOUT_SUBSYS debug
-#define dout_prefix *_dout << dbeginlstatic
+#define dout_prefix *_dout
 #define DOUT_CONDVAR(x) g_conf.debug_ ## x
 #define XDOUT_CONDVAR(x) DOUT_CONDVAR(x)
 #define DOUT_COND(l) l <= XDOUT_CONDVAR(DOUT_SUBSYS)
 
-#define dout(l) do { if (DOUT_COND(l)) { dout_prefix
-#define derr(l) do { if (DOUT_COND(l)) { dout_prefix
+#define dout(l) do { if (DOUT_COND(l)) {\
+  _dout_begin_line(l); dout_prefix
 
 #define dendl std::endl; _dout_end_line(); } } while (0)
 
 
-inline static void hex2str(const char *s, int len, char *buf, int dest_len)
-{
-  int pos = 0;
-  for (int i=0; i<len && pos<dest_len; i++) {
-    if (i && !(i%8))
-      pos += snprintf(&buf[pos], dest_len-pos, " ");
-    if (i && !(i%16))
-      pos += snprintf(&buf[pos], dest_len-pos, "\n");
-    pos += snprintf(&buf[pos], dest_len-pos, "%.2x ", (int)(unsigned char)s[i]);
-  }
-}
-
-inline static void hexdump(string msg, const char *s, int len)
-{
-  int buf_len = len*4;
-  char buf[buf_len];
-  hex2str(s, len, buf, buf_len);
-  generic_dout(0) << msg << ":\n" << buf << dendl;
-}
-
+extern void hex2str(const char *s, int len, char *buf, int dest_len);
 
+extern void hexdump(string msg, const char *s, int len);
 
 #endif
diff --git a/src/common/likely.h b/src/common/likely.h
new file mode 100644
index 00000000000..e8146a3f69e
--- /dev/null
+++ b/src/common/likely.h
@@ -0,0 +1,24 @@
+// -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
+// vim: ts=8 sw=2 smarttab
+/*
+ * Ceph - scalable distributed file system
+ *
+ * Copyright (C) 2010 Dreamhost
+ *
+ * This is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License version 2.1, as published by the Free Software
+ * Foundation.  See file COPYING.
+ *
+ */
+
+#ifndef CEPH_LIKELY_DOT_H
+#define CEPH_LIKELY_DOT_H
+
+/*
+ * Likely / Unlikely macros
+ */
+#define likely(x)       __builtin_expect((x),1)
+#define unlikely(x)     __builtin_expect((x),0)
+
+#endif