From 3f5a9c7ea03e5c6a62f41bb90f6b1655c59d3d75 Mon Sep 17 00:00:00 2001 From: Alfranio Correia Date: Fri, 5 Nov 2010 17:42:37 +0000 Subject: BUG#57275 binlog_cache_size affects trx- and stmt-cache and gets twice the expected memory After the WL#2687, the binlog_cache_size and max_binlog_cache_size affect both the stmt-cache and the trx-cache. This means that the resource used is twice the amount expected/defined by the user. The binlog_cache_use is incremented when the stmt-cache or the trx-cache is used and binlog_cache_disk_use is incremented when the disk space from the stmt-cache or the trx-cache is used. This behavior does not allow to distinguish which cache may be harming performance due to the extra disk accesses and needs to have its in-memory cache increased. To fix the problem, we introduced two new options and status variables related to the stmt-cache: Options: . binlog_stmt_cache_size . max_binlog_stmt_cache_size Status Variables: . binlog_stmt_cache_use . binlog_stmt_cache_disk_use So there are . binlog_cache_size that defines the size of the transactional cache for updates to transactional engines for the binary log. . binlog_stmt_cache_size that defines the size of the statement cache for updates to non-transactional engines for the binary log. . max_binlog_cache_size that sets the total size of the transactional cache. . max_binlog_stmt_cache_size that sets the total size of the statement cache. . binlog_cache_use that identifies the number of transactions that used the temporary transactional binary log cache. . binlog_cache_disk_use that identifies the number of transactions that used the temporary transactional binary log cache but that exceeded the value of binlog_cache_size. . binlog_stmt_cache_use that identifies the number of statements that used the temporary non-transactional binary log cache. . binlog_stmt_cache_disk_use that identifies the number of statements that used the temporary non-transactional binary log cache but that exceeded the value of binlog_stmt_cache_size. include/my_sys.h: Updated message on disk_writes' usage. mysql-test/extra/binlog_tests/binlog_cache_stat.test: Updated the test case and added code to check the new status variables binlog_stmt_cache_use and binlog_stmt_cache_disk_use. mysql-test/extra/rpl_tests/rpl_binlog_max_cache_size.test: Updated the test case to use the new system variables max_binlog_stmt_cache_size and binlog_stmt_cache_size. mysql-test/r/mysqld--help-notwin.result: Updated the result file. mysql-test/suite/binlog/r/binlog_mixed_cache_stat.result: Updated the result file. mysql-test/suite/binlog/r/binlog_row_cache_stat.result: Updated the result file. mysql-test/suite/binlog/r/binlog_stm_cache_stat.result: Updated the result file. mysql-test/suite/rpl/r/rpl_mixed_binlog_max_cache_size.result: Update the result file. mysql-test/suite/rpl/r/rpl_row_binlog_max_cache_size.result: Update the result file. mysql-test/suite/rpl/r/rpl_stm_binlog_max_cache_size.result: Updated the result file. mysql-test/suite/sys_vars/inc/binlog_stmt_cache_size_basic.inc: Added a test case to check the binlog_stmt_cache_size. mysql-test/suite/sys_vars/r/binlog_stmt_cache_size_basic_32.result: Updated the result file. mysql-test/suite/sys_vars/r/binlog_stmt_cache_size_basic_64.result: Updated the result file. mysql-test/suite/sys_vars/r/max_binlog_stmt_cache_size_basic.result: Updated the result file. mysql-test/suite/sys_vars/t/binlog_stmt_cache_size_basic_32.test: Added a test case to check the binlog_stmt_cache_size. mysql-test/suite/sys_vars/t/binlog_stmt_cache_size_basic_64.test: Added a test case to check the binlog_stmt_cache_size. mysql-test/suite/sys_vars/t/max_binlog_cache_size_func-master.opt: Removed because there is no test case max_binlog_cache_size_func. mysql-test/suite/sys_vars/t/max_binlog_stmt_cache_size_basic.test: Added a test case to check the system variable max_binlog_stmt_cache_size. sql/log.cc: There two main changes in here: . Changed the set_write_error() as an error message is set according to the type of the cache. . Created the function set_binlog_cache_info where references to the appropriate status and system variables are set and the server can smoothly compute statistics and set the maximum size for each cache. sql/log.h: Changed the signature of the function in order to identify the error message to be printed out as there is a different error code for each type of cache. sql/mysqld.cc: Added new status variables binlog_stmt_cache_use and binlog_stmt_cache_disk_use. sql/mysqld.h: Added new system variables max_binlog_stmt_cache_size and binlog_stmt_cache_size. sql/share/errmsg-utf8.txt: Added new error message related to the statement cache. sql/sys_vars.cc: Added new system variables max_binlog_stmt_cache_size and binlog_stmt_cache_size. --- sql/log.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'sql/log.h') diff --git a/sql/log.h b/sql/log.h index 89b3594cd1e..df7ed61a473 100644 --- a/sql/log.h +++ b/sql/log.h @@ -399,7 +399,7 @@ public: bool write_incident(THD *thd, bool lock); int write_cache(IO_CACHE *cache, bool lock_log, bool flush_and_sync); - void set_write_error(THD *thd); + void set_write_error(THD *thd, bool is_transactional); bool check_write_error(THD *thd); void start_union_events(THD *thd, query_id_t query_id_param); -- cgit v1.2.1 From 41dc34d60b7e924ace2c6a044189aaa345cf2820 Mon Sep 17 00:00:00 2001 From: Marc Alff Date: Fri, 12 Nov 2010 07:23:26 +0100 Subject: Bug#58052 Binary log IO not being accounted for properly Before this fix, file io for the binary log file was not accounted properly, and showed no io at all. This bug was due to the following issues: 1) file io for the binlog was instrumented: - sometime as "wait/io/file/sql/binlog" - sometime as "wait/io/file/sql/MYSQL_LOG" leading to inconsistent event_names. 2) the binlog file itself was using an IO_CACHE, but the IO_CACHE implementation in mysys/mf_iocache.c was not instrumented to make performance schema calls to record file io. 3) The "wait/io/file/sql/MYSQL_LOG" instrumentation was used for several log files, such as: - the binary log - the slow log - the query log which caused file io in these different log files to be accounted against the same instrument. The instrumentation needs to have a finer grain and report io in different event_names, because each file really serves a different purpose. With this fix: - the IO_CACHE implementation is now instrumented - the "wait/io/file/sql/MYSQL_LOG" instrument has been removed - binlog io is now always instrumented with "wait/io/file/sql/binlog" - the slow log is instrumented with a new name, "wait/io/file/sql/slow_log" - the query log is instrumented with a new name, "wait/io/file/sql/query_log" --- sql/log.h | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) (limited to 'sql/log.h') diff --git a/sql/log.h b/sql/log.h index 89b3594cd1e..d824d3afa26 100644 --- a/sql/log.h +++ b/sql/log.h @@ -196,7 +196,11 @@ public: MYSQL_LOG(); void init_pthread_objects(); void cleanup(); - bool open(const char *log_name, + bool open( +#ifdef HAVE_PSI_INTERFACE + PSI_file_key log_file_key, +#endif + const char *log_name, enum_log_type log_type, const char *new_name, enum cache_type io_cache_type_arg); @@ -223,6 +227,10 @@ public: volatile enum_log_state log_state; enum cache_type io_cache_type; friend class Log_event; +#ifdef HAVE_PSI_INTERFACE + /** Instrumentation key to use for file io in @c log_file */ + PSI_file_key m_log_file_key; +#endif }; class MYSQL_QUERY_LOG: public MYSQL_LOG @@ -241,14 +249,22 @@ public: bool open_slow_log(const char *log_name) { char buf[FN_REFLEN]; - return open(generate_name(log_name, "-slow.log", 0, buf), LOG_NORMAL, 0, - WRITE_CACHE); + return open( +#ifdef HAVE_PSI_INTERFACE + key_file_slow_log, +#endif + generate_name(log_name, "-slow.log", 0, buf), + LOG_NORMAL, 0, WRITE_CACHE); } bool open_query_log(const char *log_name) { char buf[FN_REFLEN]; - return open(generate_name(log_name, ".log", 0, buf), LOG_NORMAL, 0, - WRITE_CACHE); + return open( +#ifdef HAVE_PSI_INTERFACE + key_file_query_log, +#endif + generate_name(log_name, ".log", 0, buf), + LOG_NORMAL, 0, WRITE_CACHE); } private: -- cgit v1.2.1 From 647c619393f713ad6725d990128a8d1911e956f8 Mon Sep 17 00:00:00 2001 From: Luis Soares Date: Tue, 30 Nov 2010 23:32:51 +0000 Subject: BUG#46166: MYSQL_BIN_LOG::new_file_impl is not propagating error when generating new name. If find_uniq_filename returns an error, then this error is not being propagated upwards, and execution does not report error to the user (although a entry in the error log is generated). Additionally, some more errors were ignored in new_file_impl: - when writing the rotate event - when reopening the index and binary log file This patch addresses this by propagating the error up in the execution stack. Furthermore, when rotation of the binary log fails, an incident event is written, because there may be a chance that some changes for a given statement, were not properly logged. For example, in SBR, LOAD DATA INFILE statement requires more than one event to be logged, should rotation fail while logging part of the LOAD DATA events, then the logged data would become inconsistent with the data in the storage engine. mysql-test/include/restart_mysqld.inc: Refactored restart_mysqld so that it is not hardcoded for mysqld.1, but rather for the current server. mysql-test/suite/binlog/t/binlog_index.test: The error on open of index and binary log on new_file_impl is now caught. Thence the user will get an error message. We need to accomodate this change in the test case for the failing FLUSH LOGS. mysql-test/suite/rpl/t/rpl_binlog_errors-master.opt: Sets max_binlog_size to 4096. mysql-test/suite/rpl/t/rpl_binlog_errors.test: Added some test cases for asserting that the error is found and reported. sql/handler.cc: Catching error now returned by unlog (in ha_commit_trans) and returning it. sql/log.cc: Propagating errors from new_file_impl upwards. The errors that new_file_impl catches now are: - error on generate_new_name - error on writing the rotate event - error when opening the index or the binary log file. sql/log.h: Changing declaration of: - rotate_and_purge - new_file - new_file_without_locking - new_file_impl - unlog They now return int instead of void. sql/mysql_priv.h: Change signature of reload_acl_and_cache so that write_to_binlog is an int instead of bool. sql/mysqld.cc: Redeclaring not_used var as int instead of bool. sql/rpl_injector.cc: Changes to catch the return from rotate_and_purge. sql/slave.cc: Changes to catch the return values for new_file and rotate_relay_log. sql/slave.h: Changes to rotate_relay_log declaration (now returns int instead of void). sql/sql_load.cc: In SBR, some logging of LOAD DATA events goes through IO_CACHE_CALLBACK invocation at mf_iocache.c:_my_b_get. The IO_CACHE implementation is ignoring the return value for from these callbacks (pre_read and post_read), so we need to find out at the end of the execution if the error is set or not in THD. sql/sql_parse.cc: Catching the rotate_relay_log and rotate_and_purge return values. Semantic change in reload_acl_and_cache so that we report errors in binlog interactions through the write_to_binlog output parameter. If there was any failure while rotating the binary log, we should then report the error to the client when handling SQLCOMM_FLUSH. --- sql/log.h | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'sql/log.h') diff --git a/sql/log.h b/sql/log.h index 8f1ed7ee90c..2b0bc6111b3 100644 --- a/sql/log.h +++ b/sql/log.h @@ -39,7 +39,7 @@ class TC_LOG virtual int open(const char *opt_name)=0; virtual void close()=0; virtual int log_xid(THD *thd, my_xid xid)=0; - virtual void unlog(ulong cookie, my_xid xid)=0; + virtual int unlog(ulong cookie, my_xid xid)=0; }; class TC_LOG_DUMMY: public TC_LOG // use it to disable the logging @@ -49,7 +49,7 @@ public: int open(const char *opt_name) { return 0; } void close() { } int log_xid(THD *thd, my_xid xid) { return 1; } - void unlog(ulong cookie, my_xid xid) { } + int unlog(ulong cookie, my_xid xid) { return 0; } }; #ifdef HAVE_MMAP @@ -94,7 +94,7 @@ class TC_LOG_MMAP: public TC_LOG int open(const char *opt_name); void close(); int log_xid(THD *thd, my_xid xid); - void unlog(ulong cookie, my_xid xid); + int unlog(ulong cookie, my_xid xid); int recover(); private: @@ -283,8 +283,8 @@ class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG new_file() is locking. new_file_without_locking() does not acquire LOCK_log. */ - void new_file_without_locking(); - void new_file_impl(bool need_lock); + int new_file_without_locking(); + int new_file_impl(bool need_lock); public: MYSQL_LOG::generate_name; @@ -314,7 +314,7 @@ public: int open(const char *opt_name); void close(); int log_xid(THD *thd, my_xid xid); - void unlog(ulong cookie, my_xid xid); + int unlog(ulong cookie, my_xid xid); int recover(IO_CACHE *log, Format_description_log_event *fdle); #if !defined(MYSQL_CLIENT) int flush_and_set_pending_rows_event(THD *thd, Rows_log_event* event); @@ -354,7 +354,7 @@ public: bool open_index_file(const char *index_file_name_arg, const char *log_name, bool need_mutex); /* Use this to start writing a new log file */ - void new_file(); + int new_file(); void reset_gathered_updates(THD *thd); bool write(Log_event* event_info); // binary log write @@ -379,7 +379,7 @@ public: void make_log_name(char* buf, const char* log_ident); bool is_active(const char* log_file_name); int update_log_index(LOG_INFO* linfo, bool need_update_threads); - void rotate_and_purge(uint flags); + int rotate_and_purge(uint flags); bool flush_and_sync(); int purge_logs(const char *to_log, bool included, bool need_mutex, bool need_update_threads, -- cgit v1.2.1 From 14a46479b64e1c0ee0ced10905c65e8c7e276d8b Mon Sep 17 00:00:00 2001 From: Mikael Ronstrom Date: Mon, 17 Jan 2011 13:25:31 +0100 Subject: BUG#59549, fix compiler error on Windows --- sql/log.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'sql/log.h') diff --git a/sql/log.h b/sql/log.h index 05f8a4ce286..d287da0dcd1 100644 --- a/sql/log.h +++ b/sql/log.h @@ -63,9 +63,9 @@ class TC_LOG_MMAP: public TC_LOG { public: // only to keep Sun Forte on sol9x86 happy typedef enum { - POOL, // page is in pool - ERROR, // last sync failed - DIRTY // new xids added since last sync + PS_POOL, // page is in pool + PS_ERROR, // last sync failed + PS_DIRTY // new xids added since last sync } PAGE_STATE; private: -- cgit v1.2.1 From 5ee90018446bcb10c4a85f03eee968f99de4fa97 Mon Sep 17 00:00:00 2001 From: Marc Alff Date: Tue, 1 Mar 2011 17:39:28 +0100 Subject: Bug#11766528 PERFORMANCE_SCHEMA TRACKS BOTH BINARY AND RELAY LOGS IN THE SAME EVENTS Before this fix, all the performance schema instrumentation for both the binary log and the relay log would use the following instruments: - wait/io/file/sql/binlog - wait/io/file/sql/binlog_index - wait/synch/mutex/sql/MYSQL_BIN_LOG::LOCK_index - wait/synch/cond/sql/MYSQL_BIN_LOG::update_cond This instrumentation is too general and can be more specific. With this fix, the binlog instrumentation is identical, and the relay log instrumentation is changed to: - wait/io/file/sql/relaylog - wait/io/file/sql/relaylog_index - wait/synch/mutex/sql/MYSQL_RELAY_LOG::LOCK_index - wait/synch/cond/sql/MYSQL_RELAY_LOG::update_cond With this change, the performance instrumentation for the binary log and the relay log, which share the same structure but have different uses, is more detailed. This is especially important for hosts in the middle of a replication chain, that are both masters (binlog) and slaves (relaylog). --- sql/log.h | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'sql/log.h') diff --git a/sql/log.h b/sql/log.h index 05f8a4ce286..7f7d1a1cf3a 100644 --- a/sql/log.h +++ b/sql/log.h @@ -274,6 +274,16 @@ private: class MYSQL_BIN_LOG: public TC_LOG, private MYSQL_LOG { private: +#ifdef HAVE_PSI_INTERFACE + /** The instrumentation key to use for @ LOCK_index. */ + PSI_mutex_key m_key_LOCK_index; + /** The instrumentation key to use for @ update_cond. */ + PSI_cond_key m_key_update_cond; + /** The instrumentation key to use for opening the log file. */ + PSI_file_key m_key_file_log; + /** The instrumentation key to use for opening the log index file. */ + PSI_file_key m_key_file_log_index; +#endif /* LOCK_log and LOCK_index are inited by init_pthread_objects() */ mysql_mutex_t LOCK_index; mysql_mutex_t LOCK_prep_xids; @@ -362,6 +372,19 @@ public: on exit() - but only during the correct shutdown process */ +#ifdef HAVE_PSI_INTERFACE + void set_psi_keys(PSI_mutex_key key_LOCK_index, + PSI_cond_key key_update_cond, + PSI_file_key key_file_log, + PSI_file_key key_file_log_index) + { + m_key_LOCK_index= key_LOCK_index; + m_key_update_cond= key_update_cond; + m_key_file_log= key_file_log; + m_key_file_log_index= key_file_log_index; + } +#endif + int open(const char *opt_name); void close(); int log_xid(THD *thd, my_xid xid); -- cgit v1.2.1