diff options
| author | Marko Mäkelä <marko.makela@mariadb.com> | 2018-12-13 21:58:35 +0200 |
|---|---|---|
| committer | Marko Mäkelä <marko.makela@mariadb.com> | 2018-12-13 21:58:35 +0200 |
| commit | f6e16bdc62d80a1b26a955aafb1b60fafa912beb (patch) | |
| tree | bb2d6126379b0e0518d0064b584b8677d3df8582 | |
| parent | 839cf16bb2de078d5000bcb2f9b3151f1ebda708 (diff) | |
| parent | e3dda3d95ee4c09c2ed45ce886ecd25d1fc1ae92 (diff) | |
| download | mariadb-git-f6e16bdc62d80a1b26a955aafb1b60fafa912beb.tar.gz | |
Merge 10.2 into 10.3
32 files changed, 512 insertions, 821 deletions
diff --git a/client/mysqltest.cc b/client/mysqltest.cc index 79356108915..47e92eba7f8 100644 --- a/client/mysqltest.cc +++ b/client/mysqltest.cc @@ -6612,8 +6612,6 @@ static inline bool is_escape_char(char c, char in_string) SYNOPSIS read_line - buf buffer for the read line - size size of the buffer i.e max size to read DESCRIPTION This function actually reads several lines and adds them to the @@ -6631,10 +6629,15 @@ static inline bool is_escape_char(char c, char in_string) */ -int read_line(char *buf, int size) +static char *read_command_buf= NULL; +static size_t read_command_buflen= 0; +static const size_t max_multibyte_length= 6; + +int read_line() { char c, last_quote=0, last_char= 0; - char *p= buf, *buf_end= buf + size - 1; + char *p= read_command_buf; + char *buf_end= read_command_buf + read_command_buflen - max_multibyte_length; int skip_char= 0; my_bool have_slash= FALSE; @@ -6642,10 +6645,21 @@ int read_line(char *buf, int size) R_COMMENT, R_LINE_START} state= R_LINE_START; DBUG_ENTER("read_line"); + *p= 0; start_lineno= cur_file->lineno; DBUG_PRINT("info", ("Starting to read at lineno: %d", start_lineno)); - for (; p < buf_end ;) + while (1) { + if (p >= buf_end) + { + my_ptrdiff_t off= p - read_command_buf; + read_command_buf= (char*)my_realloc(read_command_buf, + read_command_buflen*2, MYF(MY_FAE)); + p= read_command_buf + off; + read_command_buflen*= 2; + buf_end= read_command_buf + read_command_buflen - max_multibyte_length; + } + skip_char= 0; c= my_getc(cur_file->file); if (feof(cur_file->file)) @@ -6681,7 +6695,7 @@ int read_line(char *buf, int size) cur_file->lineno++; /* Convert cr/lf to lf */ - if (p != buf && *(p-1) == '\r') + if (p != read_command_buf && *(p-1) == '\r') p--; } @@ -6696,9 +6710,9 @@ int read_line(char *buf, int size) } else if ((c == '{' && (!my_strnncoll_simple(charset_info, (const uchar*) "while", 5, - (uchar*) buf, MY_MIN(5, p - buf), 0) || + (uchar*) read_command_buf, MY_MIN(5, p - read_command_buf), 0) || !my_strnncoll_simple(charset_info, (const uchar*) "if", 2, - (uchar*) buf, MY_MIN(2, p - buf), 0)))) + (uchar*) read_command_buf, MY_MIN(2, p - read_command_buf), 0)))) { /* Only if and while commands can be terminated by { */ *p++= c; @@ -6830,8 +6844,6 @@ int read_line(char *buf, int size) } } } - die("The input buffer is too small for this query.\n" - "check your query or increase MAX_QUERY and recompile"); DBUG_RETURN(0); } @@ -6976,12 +6988,8 @@ bool is_delimiter(const char* p) terminated by new line '\n' regardless how many "delimiter" it contain. */ -#define MAX_QUERY (256*1024*2) /* 256K -- a test in sp-big is >128K */ -static char read_command_buf[MAX_QUERY]; - int read_command(struct st_command** command_ptr) { - char *p= read_command_buf; struct st_command* command; DBUG_ENTER("read_command"); @@ -6998,8 +7006,7 @@ int read_command(struct st_command** command_ptr) die("Out of memory"); command->type= Q_UNKNOWN; - read_command_buf[0]= 0; - if (read_line(read_command_buf, sizeof(read_command_buf))) + if (read_line()) { check_eol_junk(read_command_buf); DBUG_RETURN(1); @@ -7008,6 +7015,7 @@ int read_command(struct st_command** command_ptr) if (opt_result_format_version == 1) convert_to_format_v1(read_command_buf); + char *p= read_command_buf; DBUG_PRINT("info", ("query: '%s'", read_command_buf)); if (*p == '#') { @@ -9239,6 +9247,8 @@ int main(int argc, char **argv) init_win_path_patterns(); #endif + read_command_buf= (char*)my_malloc(read_command_buflen= 65536, MYF(MY_FAE)); + init_dynamic_string(&ds_res, "", 2048, 2048); init_alloc_root(&require_file_root, "require_file", 1024, 1024, MYF(0)); diff --git a/debian/additions/debian-start b/debian/additions/debian-start index 40c248fd81f..7940bbe68a5 100755 --- a/debian/additions/debian-start +++ b/debian/additions/debian-start @@ -15,7 +15,7 @@ fi MYSQL="/usr/bin/mysql --defaults-file=/etc/mysql/debian.cnf" MYADMIN="/usr/bin/mysqladmin --defaults-file=/etc/mysql/debian.cnf" -MYUPGRADE="/usr/bin/mysql_upgrade --defaults-extra-file=/etc/mysql/debian.cnf" +MYUPGRADE="/usr/bin/mysql_upgrade --defaults-extra-file=/etc/mysql/debian.cnf --version-check" MYCHECK="/usr/bin/mysqlcheck --defaults-file=/etc/mysql/debian.cnf" MYCHECK_SUBJECT="WARNING: mysqlcheck has found corrupt tables" MYCHECK_PARAMS="--all-databases --fast --silent" diff --git a/debian/mariadb-server-10.3.postinst b/debian/mariadb-server-10.3.postinst index dbe664f30fb..7874a691268 100644 --- a/debian/mariadb-server-10.3.postinst +++ b/debian/mariadb-server-10.3.postinst @@ -2,13 +2,16 @@ . /usr/share/debconf/confmodule +# assume the filename is /path/to/mariadb-server-##.#.postinst +VER=${0: -13:4} + if [ -n "$DEBIAN_SCRIPT_DEBUG" ]; then set -v -x; DEBIAN_SCRIPT_TRACE=1; fi ${DEBIAN_SCRIPT_TRACE:+ echo "#42#DEBUG# RUNNING $0 $*" 1>&2 } export PATH=$PATH:/sbin:/usr/sbin:/bin:/usr/bin # This command can be used as pipe to syslog. With "-s" it also logs to stderr. -ERR_LOGGER="logger -p daemon.err -t mysqld_safe -i" +ERR_LOGGER="logger -p daemon.err -t mariadb-server-$VER.postinst -i" # This will make an error in a logged command immediately apparent by aborting # the install, rather than failing silently and leaving a broken install. set -o pipefail @@ -147,6 +150,9 @@ EOF set -e + # To avoid downgrades. + touch $mysql_statedir/debian-$VER.flag + ## On every reconfiguration the maintenance user is recreated. # # - It is easier to regenerate the password every time but as people diff --git a/include/my_global.h b/include/my_global.h index 2922b74192a..37f78eabb4f 100644 --- a/include/my_global.h +++ b/include/my_global.h @@ -1073,7 +1073,7 @@ typedef ulong myf; /* Type of MyFlags in my_funcs */ static inline char *dlerror(void) { static char win_errormsg[2048]; - FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM, + FormatMessageA(FORMAT_MESSAGE_FROM_SYSTEM, 0, GetLastError(), 0, win_errormsg, 2048, NULL); return win_errormsg; } diff --git a/mysql-test/suite/encryption/r/innodb-checksum-algorithm.result b/mysql-test/suite/encryption/r/innodb-checksum-algorithm.result index cca3b9ad686..afb79dbad8a 100644 --- a/mysql-test/suite/encryption/r/innodb-checksum-algorithm.result +++ b/mysql-test/suite/encryption/r/innodb-checksum-algorithm.result @@ -58,31 +58,6 @@ tpe_crc32.cfg tpe_crc32.frm tpe_crc32.ibd UNLOCK TABLES; -SET GLOBAL innodb_checksum_algorithm=strict_crc32; -ALTER TABLE tce_crc32 DISCARD TABLESPACE; -ALTER TABLE tc_crc32 DISCARD TABLESPACE; -ALTER TABLE te_crc32 DISCARD TABLESPACE; -ALTER TABLE t_crc32 DISCARD TABLESPACE; -ALTER TABLE tpe_crc32 DISCARD TABLESPACE; -ALTER TABLE tp_crc32 DISCARD TABLESPACE; -restore: tce_crc32 .ibd and .cfg files -restore: tc_crc32 .ibd and .cfg files -restore: te_crc32 .ibd and .cfg files -restore: t_crc32 .ibd and .cfg files -restore: tpe_crc32 .ibd and .cfg files -restore: tp_crc32 .ibd and .cfg files -ALTER TABLE tce_crc32 IMPORT TABLESPACE; -update tce_crc32 set b=substr(b,1); -ALTER TABLE tc_crc32 IMPORT TABLESPACE; -update tc_crc32 set b=substr(b,1); -ALTER TABLE te_crc32 IMPORT TABLESPACE; -update te_crc32 set b=substr(b,1); -ALTER TABLE t_crc32 IMPORT TABLESPACE; -update t_crc32 set b=substr(b,1); -ALTER TABLE tpe_crc32 IMPORT TABLESPACE; -update tpe_crc32 set b=substr(b,1); -ALTER TABLE tp_crc32 IMPORT TABLESPACE; -update tp_crc32 set b=substr(b,1); SET GLOBAL innodb_checksum_algorithm=crc32; ALTER TABLE tce_crc32 DISCARD TABLESPACE; ALTER TABLE tc_crc32 DISCARD TABLESPACE; @@ -108,31 +83,6 @@ ALTER TABLE tpe_crc32 IMPORT TABLESPACE; update tpe_crc32 set b=substr(b,1); ALTER TABLE tp_crc32 IMPORT TABLESPACE; update tp_crc32 set b=substr(b,1); -SET GLOBAL innodb_checksum_algorithm=strict_innodb; -ALTER TABLE tce_crc32 DISCARD TABLESPACE; -ALTER TABLE tc_crc32 DISCARD TABLESPACE; -ALTER TABLE te_crc32 DISCARD TABLESPACE; -ALTER TABLE t_crc32 DISCARD TABLESPACE; -ALTER TABLE tpe_crc32 DISCARD TABLESPACE; -ALTER TABLE tp_crc32 DISCARD TABLESPACE; -restore: tce_crc32 .ibd and .cfg files -restore: tc_crc32 .ibd and .cfg files -restore: te_crc32 .ibd and .cfg files -restore: t_crc32 .ibd and .cfg files -restore: tpe_crc32 .ibd and .cfg files -restore: tp_crc32 .ibd and .cfg files -ALTER TABLE tce_crc32 IMPORT TABLESPACE; -update tce_crc32 set b=substr(b,1); -ALTER TABLE tc_crc32 IMPORT TABLESPACE; -update tc_crc32 set b=substr(b,1); -ALTER TABLE te_crc32 IMPORT TABLESPACE; -update te_crc32 set b=substr(b,1); -ALTER TABLE t_crc32 IMPORT TABLESPACE; -update t_crc32 set b=substr(b,1); -ALTER TABLE tpe_crc32 IMPORT TABLESPACE; -update tpe_crc32 set b=substr(b,1); -ALTER TABLE tp_crc32 IMPORT TABLESPACE; -update tp_crc32 set b=substr(b,1); SET GLOBAL innodb_checksum_algorithm=innodb; ALTER TABLE tce_crc32 DISCARD TABLESPACE; ALTER TABLE tc_crc32 DISCARD TABLESPACE; @@ -158,31 +108,6 @@ ALTER TABLE tpe_crc32 IMPORT TABLESPACE; update tpe_crc32 set b=substr(b,1); ALTER TABLE tp_crc32 IMPORT TABLESPACE; update tp_crc32 set b=substr(b,1); -SET GLOBAL innodb_checksum_algorithm=strict_none; -ALTER TABLE tce_crc32 DISCARD TABLESPACE; -ALTER TABLE tc_crc32 DISCARD TABLESPACE; -ALTER TABLE te_crc32 DISCARD TABLESPACE; -ALTER TABLE t_crc32 DISCARD TABLESPACE; -ALTER TABLE tpe_crc32 DISCARD TABLESPACE; -ALTER TABLE tp_crc32 DISCARD TABLESPACE; -restore: tce_crc32 .ibd and .cfg files -restore: tc_crc32 .ibd and .cfg files -restore: te_crc32 .ibd and .cfg files -restore: t_crc32 .ibd and .cfg files -restore: tpe_crc32 .ibd and .cfg files -restore: tp_crc32 .ibd and .cfg files -ALTER TABLE tce_crc32 IMPORT TABLESPACE; -update tce_crc32 set b=substr(b,1); -ALTER TABLE tc_crc32 IMPORT TABLESPACE; -update tc_crc32 set b=substr(b,1); -ALTER TABLE te_crc32 IMPORT TABLESPACE; -update te_crc32 set b=substr(b,1); -ALTER TABLE t_crc32 IMPORT TABLESPACE; -update t_crc32 set b=substr(b,1); -ALTER TABLE tpe_crc32 IMPORT TABLESPACE; -update tpe_crc32 set b=substr(b,1); -ALTER TABLE tp_crc32 IMPORT TABLESPACE; -update tp_crc32 set b=substr(b,1); SET GLOBAL innodb_checksum_algorithm=none; ALTER TABLE tce_crc32 DISCARD TABLESPACE; ALTER TABLE tc_crc32 DISCARD TABLESPACE; @@ -268,31 +193,6 @@ tpe_innodb.cfg tpe_innodb.frm tpe_innodb.ibd UNLOCK TABLES; -SET GLOBAL innodb_checksum_algorithm=strict_crc32; -ALTER TABLE tce_innodb DISCARD TABLESPACE; -ALTER TABLE tc_innodb DISCARD TABLESPACE; -ALTER TABLE te_innodb DISCARD TABLESPACE; -ALTER TABLE t_innodb DISCARD TABLESPACE; -ALTER TABLE tpe_innodb DISCARD TABLESPACE; -ALTER TABLE tp_innodb DISCARD TABLESPACE; -restore: tce_innodb .ibd and .cfg files -restore: tc_innodb .ibd and .cfg files -restore: te_innodb .ibd and .cfg files -restore: t_innodb .ibd and .cfg files -restore: tpe_innodb .ibd and .cfg files -restore: tp_innodb .ibd and .cfg files -ALTER TABLE tce_innodb IMPORT TABLESPACE; -update tce_innodb set b=substr(b,1); -ALTER TABLE tc_innodb IMPORT TABLESPACE; -update tc_innodb set b=substr(b,1); -ALTER TABLE te_innodb IMPORT TABLESPACE; -update te_innodb set b=substr(b,1); -ALTER TABLE t_innodb IMPORT TABLESPACE; -update t_innodb set b=substr(b,1); -ALTER TABLE tpe_innodb IMPORT TABLESPACE; -update tpe_innodb set b=substr(b,1); -ALTER TABLE tp_innodb IMPORT TABLESPACE; -update tp_innodb set b=substr(b,1); SET GLOBAL innodb_checksum_algorithm=crc32; ALTER TABLE tce_innodb DISCARD TABLESPACE; ALTER TABLE tc_innodb DISCARD TABLESPACE; @@ -318,31 +218,6 @@ ALTER TABLE tpe_innodb IMPORT TABLESPACE; update tpe_innodb set b=substr(b,1); ALTER TABLE tp_innodb IMPORT TABLESPACE; update tp_innodb set b=substr(b,1); -SET GLOBAL innodb_checksum_algorithm=strict_innodb; -ALTER TABLE tce_innodb DISCARD TABLESPACE; -ALTER TABLE tc_innodb DISCARD TABLESPACE; -ALTER TABLE te_innodb DISCARD TABLESPACE; -ALTER TABLE t_innodb DISCARD TABLESPACE; -ALTER TABLE tpe_innodb DISCARD TABLESPACE; -ALTER TABLE tp_innodb DISCARD TABLESPACE; -restore: tce_innodb .ibd and .cfg files -restore: tc_innodb .ibd and .cfg files -restore: te_innodb .ibd and .cfg files -restore: t_innodb .ibd and .cfg files -restore: tpe_innodb .ibd and .cfg files -restore: tp_innodb .ibd and .cfg files -ALTER TABLE tce_innodb IMPORT TABLESPACE; -update tce_innodb set b=substr(b,1); -ALTER TABLE tc_innodb IMPORT TABLESPACE; -update tc_innodb set b=substr(b,1); -ALTER TABLE te_innodb IMPORT TABLESPACE; -update te_innodb set b=substr(b,1); -ALTER TABLE t_innodb IMPORT TABLESPACE; -update t_innodb set b=substr(b,1); -ALTER TABLE tpe_innodb IMPORT TABLESPACE; -update tpe_innodb set b=substr(b,1); -ALTER TABLE tp_innodb IMPORT TABLESPACE; -update tp_innodb set b=substr(b,1); SET GLOBAL innodb_checksum_algorithm=innodb; ALTER TABLE tce_innodb DISCARD TABLESPACE; ALTER TABLE tc_innodb DISCARD TABLESPACE; @@ -368,31 +243,6 @@ ALTER TABLE tpe_innodb IMPORT TABLESPACE; update tpe_innodb set b=substr(b,1); ALTER TABLE tp_innodb IMPORT TABLESPACE; update tp_innodb set b=substr(b,1); -SET GLOBAL innodb_checksum_algorithm=strict_none; -ALTER TABLE tce_innodb DISCARD TABLESPACE; -ALTER TABLE tc_innodb DISCARD TABLESPACE; -ALTER TABLE te_innodb DISCARD TABLESPACE; -ALTER TABLE t_innodb DISCARD TABLESPACE; -ALTER TABLE tpe_innodb DISCARD TABLESPACE; -ALTER TABLE tp_innodb DISCARD TABLESPACE; -restore: tce_innodb .ibd and .cfg files -restore: tc_innodb .ibd and .cfg files -restore: te_innodb .ibd and .cfg files -restore: t_innodb .ibd and .cfg files -restore: tpe_innodb .ibd and .cfg files -restore: tp_innodb .ibd and .cfg files -ALTER TABLE tce_innodb IMPORT TABLESPACE; -update tce_innodb set b=substr(b,1); -ALTER TABLE tc_innodb IMPORT TABLESPACE; -update tc_innodb set b=substr(b,1); -ALTER TABLE te_innodb IMPORT TABLESPACE; -update te_innodb set b=substr(b,1); -ALTER TABLE t_innodb IMPORT TABLESPACE; -update t_innodb set b=substr(b,1); -ALTER TABLE tpe_innodb IMPORT TABLESPACE; -update tpe_innodb set b=substr(b,1); -ALTER TABLE tp_innodb IMPORT TABLESPACE; -update tp_innodb set b=substr(b,1); SET GLOBAL innodb_checksum_algorithm=none; ALTER TABLE tce_innodb DISCARD TABLESPACE; ALTER TABLE tc_innodb DISCARD TABLESPACE; @@ -478,31 +328,6 @@ tpe_none.cfg tpe_none.frm tpe_none.ibd UNLOCK TABLES; -SET GLOBAL innodb_checksum_algorithm=strict_crc32; -ALTER TABLE tce_none DISCARD TABLESPACE; -ALTER TABLE tc_none DISCARD TABLESPACE; -ALTER TABLE te_none DISCARD TABLESPACE; -ALTER TABLE t_none DISCARD TABLESPACE; -ALTER TABLE tpe_none DISCARD TABLESPACE; -ALTER TABLE tp_none DISCARD TABLESPACE; -restore: tce_none .ibd and .cfg files -restore: tc_none .ibd and .cfg files -restore: te_none .ibd and .cfg files -restore: t_none .ibd and .cfg files -restore: tpe_none .ibd and .cfg files -restore: tp_none .ibd and .cfg files -ALTER TABLE tce_none IMPORT TABLESPACE; -update tce_none set b=substr(b,1); -ALTER TABLE tc_none IMPORT TABLESPACE; -update tc_none set b=substr(b,1); -ALTER TABLE te_none IMPORT TABLESPACE; -update te_none set b=substr(b,1); -ALTER TABLE t_none IMPORT TABLESPACE; -update t_none set b=substr(b,1); -ALTER TABLE tpe_none IMPORT TABLESPACE; -update tpe_none set b=substr(b,1); -ALTER TABLE tp_none IMPORT TABLESPACE; -update tp_none set b=substr(b,1); SET GLOBAL innodb_checksum_algorithm=crc32; ALTER TABLE tce_none DISCARD TABLESPACE; ALTER TABLE tc_none DISCARD TABLESPACE; @@ -528,31 +353,6 @@ ALTER TABLE tpe_none IMPORT TABLESPACE; update tpe_none set b=substr(b,1); ALTER TABLE tp_none IMPORT TABLESPACE; update tp_none set b=substr(b,1); -SET GLOBAL innodb_checksum_algorithm=strict_innodb; -ALTER TABLE tce_none DISCARD TABLESPACE; -ALTER TABLE tc_none DISCARD TABLESPACE; -ALTER TABLE te_none DISCARD TABLESPACE; -ALTER TABLE t_none DISCARD TABLESPACE; -ALTER TABLE tpe_none DISCARD TABLESPACE; -ALTER TABLE tp_none DISCARD TABLESPACE; -restore: tce_none .ibd and .cfg files -restore: tc_none .ibd and .cfg files -restore: te_none .ibd and .cfg files -restore: t_none .ibd and .cfg files -restore: tpe_none .ibd and .cfg files -restore: tp_none .ibd and .cfg files -ALTER TABLE tce_none IMPORT TABLESPACE; -update tce_none set b=substr(b,1); -ALTER TABLE tc_none IMPORT TABLESPACE; -update tc_none set b=substr(b,1); -ALTER TABLE te_none IMPORT TABLESPACE; -update te_none set b=substr(b,1); -ALTER TABLE t_none IMPORT TABLESPACE; -update t_none set b=substr(b,1); -ALTER TABLE tpe_none IMPORT TABLESPACE; -update tpe_none set b=substr(b,1); -ALTER TABLE tp_none IMPORT TABLESPACE; -update tp_none set b=substr(b,1); SET GLOBAL innodb_checksum_algorithm=innodb; ALTER TABLE tce_none DISCARD TABLESPACE; ALTER TABLE tc_none DISCARD TABLESPACE; @@ -578,31 +378,6 @@ ALTER TABLE tpe_none IMPORT TABLESPACE; update tpe_none set b=substr(b,1); ALTER TABLE tp_none IMPORT TABLESPACE; update tp_none set b=substr(b,1); -SET GLOBAL innodb_checksum_algorithm=strict_none; -ALTER TABLE tce_none DISCARD TABLESPACE; -ALTER TABLE tc_none DISCARD TABLESPACE; -ALTER TABLE te_none DISCARD TABLESPACE; -ALTER TABLE t_none DISCARD TABLESPACE; -ALTER TABLE tpe_none DISCARD TABLESPACE; -ALTER TABLE tp_none DISCARD TABLESPACE; -restore: tce_none .ibd and .cfg files -restore: tc_none .ibd and .cfg files -restore: te_none .ibd and .cfg files -restore: t_none .ibd and .cfg files -restore: tpe_none .ibd and .cfg files -restore: tp_none .ibd and .cfg files -ALTER TABLE tce_none IMPORT TABLESPACE; -update tce_none set b=substr(b,1); -ALTER TABLE tc_none IMPORT TABLESPACE; -update tc_none set b=substr(b,1); -ALTER TABLE te_none IMPORT TABLESPACE; -update te_none set b=substr(b,1); -ALTER TABLE t_none IMPORT TABLESPACE; -update t_none set b=substr(b,1); -ALTER TABLE tpe_none IMPORT TABLESPACE; -update tpe_none set b=substr(b,1); -ALTER TABLE tp_none IMPORT TABLESPACE; -update tp_none set b=substr(b,1); SET GLOBAL innodb_checksum_algorithm=none; ALTER TABLE tce_none DISCARD TABLESPACE; ALTER TABLE tc_none DISCARD TABLESPACE; diff --git a/mysql-test/suite/encryption/t/innodb-checksum-algorithm.test b/mysql-test/suite/encryption/t/innodb-checksum-algorithm.test index 7eaa1bd64c6..c9a4a9e1b92 100644 --- a/mysql-test/suite/encryption/t/innodb-checksum-algorithm.test +++ b/mysql-test/suite/encryption/t/innodb-checksum-algorithm.test @@ -65,17 +65,14 @@ EOF --list_files $MYSQLD_DATADIR/test UNLOCK TABLES; -let $to = 6; +let $to = 3; while ($to) { dec $to; let $tocksum = `select case $to when 0 then 'none' - when 1 then 'strict_none' - when 2 then 'innodb' - when 3 then 'strict_innodb' - when 4 then 'crc32' - when 5 then 'strict_crc32' + when 1 then 'innodb' + when 2 then 'crc32' end`; eval SET GLOBAL innodb_checksum_algorithm=$tocksum; diff --git a/mysql-test/suite/innodb/r/default_row_format_create,redundant.rdiff b/mysql-test/suite/innodb/r/default_row_format_create,redundant.rdiff index 5258b7193fa..fbf1d914f5b 100644 --- a/mysql-test/suite/innodb/r/default_row_format_create,redundant.rdiff +++ b/mysql-test/suite/innodb/r/default_row_format_create,redundant.rdiff @@ -9,3 +9,14 @@ DROP TABLE t1; CREATE TABLE t1(c1 TEXT,c2 BLOB) ENGINE=InnoDB ROW_FORMAT=DYNAMIC; SHOW TABLE STATUS LIKE 't1'; +@@ -31,8 +31,9 @@ + CREATE TABLE t1 (c1 INT) ENGINE=InnoDB page_compressed=1; + SHOW TABLE STATUS LIKE 't1'; + Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment Max_index_length Temporary +-t1 InnoDB # Dynamic # # # # # # NULL # NULL NULL latin1_swedish_ci NULL `page_compressed`=1 0 N + DROP TABLE IF EXISTS t1; ++Warnings: ++Note 1051 Unknown table 'test.t1' + SET @save_format = @@GLOBAL.innodb_default_row_format; + SET GLOBAL innodb_default_row_format = redundant; + CREATE TABLE t1 (c1 INT) ENGINE=InnoDB; diff --git a/mysql-test/suite/innodb/r/default_row_format_create.result b/mysql-test/suite/innodb/r/default_row_format_create.result index 83d04b47bd0..262e8bc7f19 100644 --- a/mysql-test/suite/innodb/r/default_row_format_create.result +++ b/mysql-test/suite/innodb/r/default_row_format_create.result @@ -19,8 +19,26 @@ Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length I t1 InnoDB # Redundant # # # # # # NULL # NULL NULL latin1_swedish_ci NULL row_format=REDUNDANT 0 N DROP TABLE t1; CREATE TABLE t1(c1 TEXT,c2 BLOB) ENGINE=InnoDB -ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=1; +ROW_FORMAT=COMPRESSED; SHOW TABLE STATUS LIKE 't1'; Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment Max_index_length Temporary -t1 InnoDB # Compressed # # # # # # NULL # NULL NULL latin1_swedish_ci NULL row_format=COMPRESSED key_block_size=1 0 N +t1 InnoDB # Compressed # # # # # # NULL # NULL NULL latin1_swedish_ci NULL row_format=COMPRESSED 0 N +TRUNCATE TABLE t1; +SHOW TABLE STATUS LIKE 't1'; +Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment Max_index_length Temporary +t1 InnoDB # Compressed # # # # # # NULL # NULL NULL latin1_swedish_ci NULL row_format=COMPRESSED 0 N +DROP TABLE t1; +CREATE TABLE t1 (c1 INT) ENGINE=InnoDB page_compressed=1; +SHOW TABLE STATUS LIKE 't1'; +Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment Max_index_length Temporary +t1 InnoDB # Dynamic # # # # # # NULL # NULL NULL latin1_swedish_ci NULL `page_compressed`=1 0 N +DROP TABLE IF EXISTS t1; +SET @save_format = @@GLOBAL.innodb_default_row_format; +SET GLOBAL innodb_default_row_format = redundant; +CREATE TABLE t1 (c1 INT) ENGINE=InnoDB; +SET GLOBAL innodb_default_row_format = @save_format; +TRUNCATE TABLE t1; +SHOW TABLE STATUS LIKE 't1'; +Name Engine Version Row_format Rows Avg_row_length Data_length Max_data_length Index_length Data_free Auto_increment Create_time Update_time Check_time Collation Checksum Create_options Comment Max_index_length Temporary +t1 InnoDB # Redundant # # # # # # NULL # NULL NULL latin1_swedish_ci NULL 0 N DROP TABLE t1; diff --git a/mysql-test/suite/innodb/r/truncate.result b/mysql-test/suite/innodb/r/truncate.result index 52fe1e28948..0e5ffeea34f 100644 --- a/mysql-test/suite/innodb/r/truncate.result +++ b/mysql-test/suite/innodb/r/truncate.result @@ -30,3 +30,12 @@ SELECT * FROM t1; a 1 DROP TABLE t1; +# +# MDEV-17885 TRUNCATE on temporary table causes ER_GET_ERRNO +# +CREATE TEMPORARY TABLE t1 (a INT) ENCRYPTED=NO ENGINE=InnoDB; +INSERT INTO t1 VALUES(1); +TRUNCATE t1; +SELECT * FROM t1; +a +DROP TEMPORARY TABLE t1; diff --git a/mysql-test/suite/innodb/t/default_row_format_create.test b/mysql-test/suite/innodb/t/default_row_format_create.test index e0981abf7eb..03a7ebd3752 100644 --- a/mysql-test/suite/innodb/t/default_row_format_create.test +++ b/mysql-test/suite/innodb/t/default_row_format_create.test @@ -22,7 +22,25 @@ SHOW TABLE STATUS LIKE 't1'; DROP TABLE t1; CREATE TABLE t1(c1 TEXT,c2 BLOB) ENGINE=InnoDB -ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=1; +ROW_FORMAT=COMPRESSED; +--replace_column 3 # 5 # 6 # 7 # 8 # 9 # 10 # 12 # +SHOW TABLE STATUS LIKE 't1'; +TRUNCATE TABLE t1; +--replace_column 3 # 5 # 6 # 7 # 8 # 9 # 10 # 12 # +SHOW TABLE STATUS LIKE 't1'; +DROP TABLE t1; + +--error 0,ER_CANT_CREATE_TABLE +CREATE TABLE t1 (c1 INT) ENGINE=InnoDB page_compressed=1; +--replace_column 3 # 5 # 6 # 7 # 8 # 9 # 10 # 12 # +SHOW TABLE STATUS LIKE 't1'; +DROP TABLE IF EXISTS t1; + +SET @save_format = @@GLOBAL.innodb_default_row_format; +SET GLOBAL innodb_default_row_format = redundant; +CREATE TABLE t1 (c1 INT) ENGINE=InnoDB; +SET GLOBAL innodb_default_row_format = @save_format; +TRUNCATE TABLE t1; --replace_column 3 # 5 # 6 # 7 # 8 # 9 # 10 # 12 # SHOW TABLE STATUS LIKE 't1'; DROP TABLE t1; diff --git a/mysql-test/suite/innodb/t/innodb_zip_innochecksum.opt b/mysql-test/suite/innodb/t/innodb_zip_innochecksum.opt index 1dcd1367b3a..4ed6c2c8fd0 100644 --- a/mysql-test/suite/innodb/t/innodb_zip_innochecksum.opt +++ b/mysql-test/suite/innodb/t/innodb_zip_innochecksum.opt @@ -1,2 +1,3 @@ --skip-innodb-doublewrite --innodb-file-per-table +--innodb_checksum_algorithm=crc32 diff --git a/mysql-test/suite/innodb/t/innodb_zip_innochecksum.test b/mysql-test/suite/innodb/t/innodb_zip_innochecksum.test index 63a4b418677..fec8acf52c4 100644 --- a/mysql-test/suite/innodb/t/innodb_zip_innochecksum.test +++ b/mysql-test/suite/innodb/t/innodb_zip_innochecksum.test @@ -79,18 +79,22 @@ let SEARCH_PATTERN= Error: --strict-check option cannot be used together with -- --echo [9]: check the innochecksum with full form --strict-check=innodb # Server Default checksum = crc32 +--error 1 --exec $INNOCHECKSUM --strict-check=innodb $MYSQLD_DATADIR/test/tab1.ibd 2> $SEARCH_FILE --echo [10]: check the innochecksum with full form --strict-check=none --echo # when server Default checksum=crc32 +--error 1 --exec $INNOCHECKSUM --strict-check=none $MYSQLD_DATADIR/test/tab1.ibd 2> $SEARCH_FILE --echo [11]: check the innochecksum with short form -C innodb --echo # when server Default checksum=crc32 +--error 1 --exec $INNOCHECKSUM -C innodb $MYSQLD_DATADIR/test/tab1.ibd 2> $SEARCH_FILE --echo [12]: check the innochecksum with short form -C none --echo # when server Default checksum=crc32 +--error 1 --exec $INNOCHECKSUM -C none $MYSQLD_DATADIR/test/tab1.ibd 2> $SEARCH_FILE --echo [13]: check strict-check with invalid values diff --git a/mysql-test/suite/innodb/t/truncate.test b/mysql-test/suite/innodb/t/truncate.test index 6c573cf42d4..cd1d827e157 100644 --- a/mysql-test/suite/innodb/t/truncate.test +++ b/mysql-test/suite/innodb/t/truncate.test @@ -41,3 +41,12 @@ CREATE TABLE t1 (a INT) ENGINE=InnoDB; --move_file $MYSQLD_DATADIR/test/hidden.frm $MYSQLD_DATADIR/test/t1.frm SELECT * FROM t1; DROP TABLE t1; + +--echo # +--echo # MDEV-17885 TRUNCATE on temporary table causes ER_GET_ERRNO +--echo # +CREATE TEMPORARY TABLE t1 (a INT) ENCRYPTED=NO ENGINE=InnoDB; +INSERT INTO t1 VALUES(1); +TRUNCATE t1; +SELECT * FROM t1; +DROP TEMPORARY TABLE t1; diff --git a/mysql-test/suite/innodb_zip/t/innochecksum.test b/mysql-test/suite/innodb_zip/t/innochecksum.test index 63a4b418677..fec8acf52c4 100644 --- a/mysql-test/suite/innodb_zip/t/innochecksum.test +++ b/mysql-test/suite/innodb_zip/t/innochecksum.test @@ -79,18 +79,22 @@ let SEARCH_PATTERN= Error: --strict-check option cannot be used together with -- --echo [9]: check the innochecksum with full form --strict-check=innodb # Server Default checksum = crc32 +--error 1 --exec $INNOCHECKSUM --strict-check=innodb $MYSQLD_DATADIR/test/tab1.ibd 2> $SEARCH_FILE --echo [10]: check the innochecksum with full form --strict-check=none --echo # when server Default checksum=crc32 +--error 1 --exec $INNOCHECKSUM --strict-check=none $MYSQLD_DATADIR/test/tab1.ibd 2> $SEARCH_FILE --echo [11]: check the innochecksum with short form -C innodb --echo # when server Default checksum=crc32 +--error 1 --exec $INNOCHECKSUM -C innodb $MYSQLD_DATADIR/test/tab1.ibd 2> $SEARCH_FILE --echo [12]: check the innochecksum with short form -C none --echo # when server Default checksum=crc32 +--error 1 --exec $INNOCHECKSUM -C none $MYSQLD_DATADIR/test/tab1.ibd 2> $SEARCH_FILE --echo [13]: check strict-check with invalid values diff --git a/scripts/mytop.sh b/scripts/mytop.sh index 17a87e9efd2..3ef0a59f27f 100644 --- a/scripts/mytop.sh +++ b/scripts/mytop.sh @@ -437,7 +437,7 @@ while (1) if ($key eq 'C') { - if ( $HAS_COLOR ) + if ( $HAS_COLOR ) { $HAS_COLOR = 0; } @@ -817,11 +817,11 @@ sub GetData() if ($config{header}) { my @recs = ""; - if ( $db_release > 4 ) + if ( $db_release > 4 ) { @recs = Hashes("show global status"); - } - else + } + else { @recs = Hashes("show status"); } @@ -978,7 +978,7 @@ sub GetData() # print("q_diff: $STATUS{Questions} - $OLD_STATUS{Questions} / $t_delta = $q_diff\n"); printf(" Sorts: %5.0f qps now: %4.0f Slow qps: %3.1f Threads: %4.0f (%4.0f/%4.0f) %02.0f/%02.0f/%02.0f/%02.0f\n", - ( $STATUS{Sort_rows} - $OLD_STATUS{Sort_rows} ) / $t_delta, + ( $STATUS{Sort_rows} - $OLD_STATUS{Sort_rows} ) / $t_delta, ( $STATUS{Questions} - $OLD_STATUS{Questions} ) / $t_delta, ( # slow now (qps) ($STATUS{Slow_queries} ) ? @@ -989,7 +989,7 @@ sub GetData() $STATUS{Threads_running}, $STATUS{Threads_cached}, - (100 * ($STATUS{Com_select} - $OLD_STATUS{Com_select} + + (100 * ($STATUS{Com_select} - $OLD_STATUS{Com_select} + ($STATUS{Qcache_hits}||0) - ($OLD_STATUS{Qcache_hits}||0) ) ) / ($q_diff ), (100 * ($STATUS{Com_insert} - $OLD_STATUS{Com_insert} + @@ -1075,7 +1075,7 @@ sub GetData() $t_delta, ($STATUS{Rows_tmp_read} - $OLD_STATUS{Rows_tmp_read}) / $t_delta, - ($STATUS{Handler_tmp_write} + ($STATUS{Handler_tmp_write} -$OLD_STATUS{Handler_tmp_write})/$t_delta, ($STATUS{Handler_tmp_update} - $OLD_STATUS{Handler_tmp_update})/$t_delta); @@ -1119,6 +1119,7 @@ sub GetData() } } print " Replication "; + print "Master:$data->{Master_Host} "; print "IO:$data->{Slave_IO_Running} "; print "SQL:$data->{Slave_SQL_Running} "; print RESET() if ($HAS_COLOR); @@ -1225,9 +1226,9 @@ sub GetData() $thread->{State} ||= ""; $thread->{Progress} ||= 0; - ## alter double hyphen comments so they don't break + ## alter double hyphen comments so they don't break ## the query when newlines are removed - http://freshmeat.net/users/jerjones - $thread->{Info} =~ s~\s--(.*)$~ /* $1 */ ~mg; + $thread->{Info} =~ s~\s--(.*)$~ /* $1 */ ~mg; ## Normalize spaces -- mostly disabled for now. This can ## break EXPLAIN if you try to explain a mangled query. It diff --git a/sql/item.cc b/sql/item.cc index 529abad08cc..71a9afd6a9d 100644 --- a/sql/item.cc +++ b/sql/item.cc @@ -10652,6 +10652,9 @@ const char *dbug_print_unit(SELECT_LEX_UNIT *un) return "Couldn't fit into buffer"; } +const char *dbug_print(Item *x) { return dbug_print_item(x); } +const char *dbug_print(SELECT_LEX *x) { return dbug_print_select(x); } +const char *dbug_print(SELECT_LEX_UNIT *x) { return dbug_print_unit(x); } #endif /*DBUG_OFF*/ diff --git a/sql/sql_statistics.cc b/sql/sql_statistics.cc index 8a0c1554869..2384c68115b 100644 --- a/sql/sql_statistics.cc +++ b/sql/sql_statistics.cc @@ -3737,10 +3737,10 @@ void set_statistics_for_table(THD *thd, TABLE *table) Ideally, EITS should provide per-partition statistics but this is not implemented currently. */ - #ifdef WITH_PARTITION_STORAGE_ENGINE +#ifdef WITH_PARTITION_STORAGE_ENGINE if (table->part_info) table->used_stat_records= table->file->stats.records; - #endif +#endif KEY *key_info, *key_info_end; for (key_info= table->key_info, key_info_end= key_info+table->s->keys; @@ -4070,10 +4070,6 @@ bool is_stat_table(const LEX_CSTRING *db, LEX_CSTRING *table) bool is_eits_usable(Field *field) { - partition_info *part_info= NULL; - #ifdef WITH_PARTITION_STORAGE_ENGINE - part_info= field->table->part_info; - #endif /* (1): checks if we have EITS statistics for a particular column (2): Don't use EITS for GEOMETRY columns @@ -4082,9 +4078,11 @@ bool is_eits_usable(Field *field) such columns would be handled during partition pruning. */ Column_statistics* col_stats= field->read_stats; - if (col_stats && !col_stats->no_stat_values_provided() && //(1) - field->type() != MYSQL_TYPE_GEOMETRY && //(2) - (!part_info || !part_info->field_in_partition_expr(field))) //(3) - return TRUE; - return FALSE; + return col_stats && !col_stats->no_stat_values_provided() && //(1) + field->type() != MYSQL_TYPE_GEOMETRY && //(2) +#ifdef WITH_PARTITION_STORAGE_ENGINE + (!field->table->part_info || + !field->table->part_info->field_in_partition_expr(field)) && //(3) +#endif + true; } diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index dd9dca496fb..7133f3cfd7e 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -754,17 +754,14 @@ buf_page_is_zeroes( @param[in] read_buf database page @param[in] checksum_field1 new checksum field @param[in] checksum_field2 old checksum field -@param[in] use_legacy_big_endian use legacy big endian algorithm @return true if the page is in crc32 checksum format. */ bool buf_page_is_checksum_valid_crc32( const byte* read_buf, ulint checksum_field1, - ulint checksum_field2, - bool use_legacy_big_endian) + ulint checksum_field2) { - const uint32_t crc32 = buf_calc_page_crc32(read_buf, - use_legacy_big_endian); + const uint32_t crc32 = buf_calc_page_crc32(read_buf); #ifdef UNIV_INNOCHECKSUM if (log_file @@ -778,25 +775,14 @@ buf_page_is_checksum_valid_crc32( #endif /* UNIV_INNOCHECKSUM */ if (checksum_field1 != checksum_field2) { - goto invalid; - } - - if (checksum_field1 == crc32) { - return(true); - } else { - const uint32_t crc32_legacy = buf_calc_page_crc32(read_buf, true); - - if (checksum_field1 == crc32_legacy) { - return(true); - } + return false; } -invalid: - DBUG_LOG("checksum", "Page checksum crc32 not valid" - << " field1 " << checksum_field1 - << " field2 " << checksum_field2 - << " crc32 " << crc32); - return(false); + return checksum_field1 == crc32 +#ifdef INNODB_BUG_ENDIAN_CRC32 + || checksum_field1 == buf_calc_page_crc32(read_buf, true) +#endif + ; } /** Checks if the page is in innodb checksum format. @@ -925,6 +911,29 @@ buf_page_is_checksum_valid_none( && checksum_field1 == BUF_NO_CHECKSUM_MAGIC); } +#ifdef INNODB_BUG_ENDIAN_CRC32 +/** Validate the CRC-32C checksum of a page. +@param[in] page buffer page (srv_page_size bytes) +@param[in] checksum CRC-32C checksum stored on page +@return computed checksum */ +static uint32_t buf_page_check_crc32(const byte* page, uint32_t checksum) +{ + uint32_t crc32 = buf_calc_page_crc32(page); + + if (checksum != crc32) { + crc32 = buf_calc_page_crc32(page, true); + } + + return crc32; +} +#else /* INNODB_BUG_ENDIAN_CRC32 */ +/** Validate the CRC-32C checksum of a page. +@param[in] page buffer page (srv_page_size bytes) +@param[in] checksum CRC-32C checksum stored on page +@return computed checksum */ +# define buf_page_check_crc32(page, checksum) buf_calc_page_crc32(page) +#endif /* INNODB_BUG_ENDIAN_CRC32 */ + /** Check if a page is corrupt. @param[in] check_lsn whether the LSN should be checked @param[in] read_buf database page @@ -942,18 +951,21 @@ buf_page_is_corrupted( const void* space) #endif { - size_t checksum_field1 = 0; - size_t checksum_field2 = 0; #ifndef UNIV_INNOCHECKSUM DBUG_EXECUTE_IF("buf_page_import_corrupt_failure", return(true); ); #endif + size_t checksum_field1 = 0; + size_t checksum_field2 = 0; + uint32_t crc32 = 0; + bool crc32_inited = false; + ulint page_type = mach_read_from_2(read_buf + FIL_PAGE_TYPE); /* We can trust page type if page compression is set on tablespace flags because page compression flag means file must have been created with 10.1 (later than 5.5 code base). In 10.1 page compressed tables do not contain post compression checksum and - FIL_PAGE_END_LSN_OLD_CHKSUM field stored. Note that space can + FIL_PAGE_END_LSN_OLD_CHKSUM field stored. Note that space can be null if we are in fil_check_first_page() and first page is not compressed or encrypted. Page checksum is verified after decompression (i.e. normally pages are already @@ -974,13 +986,7 @@ buf_page_is_corrupted( /* Stored log sequence numbers at the start and the end of page do not match */ -#ifndef UNIV_INNOCHECKSUM - ib::info() << "Log sequence number at the start " - << mach_read_from_4(read_buf + FIL_PAGE_LSN + 4) - << " and the end " - << mach_read_from_4(read_buf + srv_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM + 4) - << " do not match"; -#endif /* UNIV_INNOCHECKSUM */ + return(true); } @@ -1041,73 +1047,39 @@ buf_page_is_corrupted( && *reinterpret_cast<const ib_uint64_t*>( read_buf + FIL_PAGE_LSN) == 0) { - ulint i; - /* make sure that the page is really empty */ - for (i = 0; i < page_size.logical(); ++i) { - - /* The FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID has been - repurposed for page compression. It can be - set for uncompressed empty pages. */ - - if ((i < FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION - || i >= FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID) - && read_buf[i] != 0) { - -#ifndef UNIV_INNOCHECKSUM - ib::info() << "Checksum fields zero but page is not empty."; -#endif - - break; + for (ulint i = 0; i < page_size.logical(); i++) { + if (read_buf[i] != 0) { + return(true); } } #ifdef UNIV_INNOCHECKSUM - if (i >= page_size.logical()) { - if (log_file) { - fprintf(log_file, "Page::%llu" - " is empty and uncorrupted\n", - cur_page_num); - } - return(false); + if (log_file) { + fprintf(log_file, "Page::%llu" + " is empty and uncorrupted\n", + cur_page_num); } -#else - return(i < page_size.logical()); #endif /* UNIV_INNOCHECKSUM */ + return(false); } -#ifndef UNIV_INNOCHECKSUM - const page_id_t page_id(mach_read_from_4( - read_buf + FIL_PAGE_SPACE_ID), - mach_read_from_4( - read_buf + FIL_PAGE_OFFSET)); -#endif /* UNIV_INNOCHECKSUM */ - const srv_checksum_algorithm_t curr_algo = static_cast<srv_checksum_algorithm_t>(srv_checksum_algorithm); - bool legacy_checksum_checked = false; - switch (curr_algo) { - case SRV_CHECKSUM_ALGORITHM_CRC32: case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: - - if (buf_page_is_checksum_valid_crc32(read_buf, - checksum_field1, checksum_field2, false)) { - return(false); - } - + return !buf_page_is_checksum_valid_crc32( + read_buf, checksum_field1, checksum_field2); + case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: + return !buf_page_is_checksum_valid_innodb( + read_buf, checksum_field1, checksum_field2); + case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: + return !buf_page_is_checksum_valid_none( + read_buf, checksum_field1, checksum_field2); + case SRV_CHECKSUM_ALGORITHM_CRC32: + case SRV_CHECKSUM_ALGORITHM_INNODB: if (buf_page_is_checksum_valid_none(read_buf, checksum_field1, checksum_field2)) { - if (curr_algo - == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { -#ifndef UNIV_INNOCHECKSUM - page_warn_strict_checksum( - curr_algo, - SRV_CHECKSUM_ALGORITHM_NONE, - page_id); -#endif /* !UNIV_INNOCHECKSUM */ - } - #ifdef UNIV_INNOCHECKSUM if (log_file) { fprintf(log_file, "page::%llu;" @@ -1125,173 +1097,97 @@ buf_page_is_corrupted( checksum_field1); } #endif /* UNIV_INNOCHECKSUM */ - - return(false); - } - - /* We need to check whether the stored checksum matches legacy - big endian checksum or Innodb checksum. We optimize the order - based on earlier results. if earlier we have found pages - matching legacy big endian checksum, we try to match it first. - Otherwise we check innodb checksum first. */ - if (legacy_big_endian_checksum) { - if (buf_page_is_checksum_valid_crc32(read_buf, - checksum_field1, checksum_field2, true)) { - - return(false); - } - legacy_checksum_checked = true; + return false; } - if (buf_page_is_checksum_valid_innodb(read_buf, - checksum_field1, checksum_field2)) { - if (curr_algo - == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { -#ifndef UNIV_INNOCHECKSUM - page_warn_strict_checksum( - curr_algo, - SRV_CHECKSUM_ALGORITHM_INNODB, - page_id); -#endif - } - - return(false); - } + /* Very old versions of InnoDB only stored 8 byte lsn to the + start and the end of the page. */ - /* If legacy checksum is not checked, do it now. */ - if (!legacy_checksum_checked && buf_page_is_checksum_valid_crc32( - read_buf, checksum_field1, checksum_field2, true)) { + /* Since innodb_checksum_algorithm is not strict_* allow + any of the algos to match for the old field */ - legacy_big_endian_checksum = true; - return(false); - } + if (checksum_field2 + != mach_read_from_4(read_buf + FIL_PAGE_LSN) + && checksum_field2 != BUF_NO_CHECKSUM_MAGIC) { -#ifdef UNIV_INNOCHECKSUM - if (log_file) { - fprintf(log_file, "Fail; page::%llu;" - " invalid (fails crc32 checksum)\n", - cur_page_num); - } -#endif /* UNIV_INNOCHECKSUM */ - return(true); + if (srv_checksum_algorithm + == SRV_CHECKSUM_ALGORITHM_CRC32) { + crc32 = buf_page_check_crc32(read_buf, + checksum_field2); + crc32_inited = true; - case SRV_CHECKSUM_ALGORITHM_INNODB: - case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: + if (checksum_field2 != crc32 + && checksum_field2 + != buf_calc_page_old_checksum(read_buf)) { + return true; + } + } else { + ut_ad(srv_checksum_algorithm + == SRV_CHECKSUM_ALGORITHM_INNODB); - if (buf_page_is_checksum_valid_innodb(read_buf, - checksum_field1, checksum_field2)) { - return(false); - } + if (checksum_field2 + != buf_calc_page_old_checksum(read_buf)) { + crc32 = buf_page_check_crc32( + read_buf, checksum_field2); + crc32_inited = true; - if (buf_page_is_checksum_valid_none(read_buf, - checksum_field1, checksum_field2)) { - if (curr_algo - == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { -#ifndef UNIV_INNOCHECKSUM - page_warn_strict_checksum( - curr_algo, - SRV_CHECKSUM_ALGORITHM_NONE, - page_id); -#endif - } -#ifdef UNIV_INNOCHECKSUM - if (log_file) { - fprintf(log_file, "page::%llu;" - " old style: calculated = %u;" - " recorded = %zu;\n", cur_page_num, - buf_calc_page_old_checksum(read_buf), - checksum_field2); - fprintf(log_file, "page::%llu;" - " new style: calculated = %u;" - " crc32 = %u; recorded = %zu;\n", - cur_page_num, - buf_calc_page_new_checksum(read_buf), - buf_calc_page_crc32(read_buf), - checksum_field1); + if (checksum_field2 != crc32) { + return true; + } + } } -#endif /* UNIV_INNOCHECKSUM */ - - return(false); } - if (buf_page_is_checksum_valid_crc32(read_buf, - checksum_field1, checksum_field2, false) - || buf_page_is_checksum_valid_crc32(read_buf, - checksum_field1, checksum_field2, true)) { + if (checksum_field1 == 0 + || checksum_field1 == BUF_NO_CHECKSUM_MAGIC) { + } else if (srv_checksum_algorithm + == SRV_CHECKSUM_ALGORITHM_CRC32) { - if (curr_algo - == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { -#ifndef UNIV_INNOCHECKSUM - page_warn_strict_checksum( - curr_algo, - SRV_CHECKSUM_ALGORITHM_CRC32, - page_id); -#endif + if (!crc32_inited) { + crc32 = buf_page_check_crc32( + read_buf, checksum_field2); + crc32_inited = true; } - return(false); - } - -#ifdef UNIV_INNOCHECKSUM - if (log_file) { - fprintf(log_file, "Fail; page::%llu;" - " invalid (fails innodb checksum)\n", - cur_page_num); - } -#endif /* UNIV_INNOCHECKSUM */ - - return(true); + if (checksum_field1 != crc32 + && checksum_field1 + != buf_calc_page_new_checksum(read_buf)) { + return true; + } + } else { + ut_ad(srv_checksum_algorithm + == SRV_CHECKSUM_ALGORITHM_INNODB); - case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: + if (checksum_field1 + != buf_calc_page_new_checksum(read_buf)) { - if (buf_page_is_checksum_valid_none(read_buf, - checksum_field1, checksum_field2)) { - return(false); - } - - if (buf_page_is_checksum_valid_crc32(read_buf, - checksum_field1, checksum_field2, false) - || buf_page_is_checksum_valid_crc32(read_buf, - checksum_field1, checksum_field2, true)) { -#ifndef UNIV_INNOCHECKSUM - page_warn_strict_checksum( - curr_algo, - SRV_CHECKSUM_ALGORITHM_CRC32, - page_id); -#endif /* !UNIV_INNOCHECKSUM */ - return(false); - } + if (!crc32_inited) { + crc32 = buf_page_check_crc32( + read_buf, checksum_field2); + crc32_inited = true; + } - if (buf_page_is_checksum_valid_innodb(read_buf, - checksum_field1, checksum_field2)) { -#ifndef UNIV_INNOCHECKSUM - page_warn_strict_checksum( - curr_algo, - SRV_CHECKSUM_ALGORITHM_INNODB, - page_id); -#endif /* !UNIV_INNOCHECKSUM */ - return(false); + if (checksum_field1 != crc32) { + return true; + } + } } -#ifdef UNIV_INNOCHECKSUM - if (log_file) { - fprintf(log_file, "Fail; page::%llu;" - " invalid (fails none checksum)\n", - cur_page_num); + if (crc32_inited + && ((checksum_field1 == crc32 + && checksum_field2 != crc32) + || (checksum_field1 != crc32 + && checksum_field2 == crc32))) { + return true; } -#endif /* UNIV_INNOCHECKSUM */ - - return(true); + break; case SRV_CHECKSUM_ALGORITHM_NONE: /* should have returned false earlier */ break; - /* no default so the compiler will emit a warning if new enum - is added and not handled here */ } - ut_error; - return(false); + return false; } #ifndef UNIV_INNOCHECKSUM @@ -1374,10 +1270,12 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size) << page_zip_calc_checksum( read_buf, page_size.physical(), SRV_CHECKSUM_ALGORITHM_CRC32) +#ifdef INNODB_BUG_ENDIAN_CRC32 << "/" << page_zip_calc_checksum( read_buf, page_size.physical(), SRV_CHECKSUM_ALGORITHM_CRC32, true) +#endif << ", " << buf_checksum_algorithm_name( SRV_CHECKSUM_ALGORITHM_INNODB) @@ -1403,9 +1301,10 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size) } else { const uint32_t crc32 = buf_calc_page_crc32(read_buf); - +#ifdef INNODB_BUG_ENDIAN_CRC32 const uint32_t crc32_legacy = buf_calc_page_crc32(read_buf, true); +#endif /* INNODB_BUG_ENDIAN_CRC32 */ ulint page_type = fil_page_get_type(read_buf); ib::info() << "Uncompressed page, stored checksum in field1 " @@ -1414,7 +1313,10 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size) << ", calculated checksums for field1: " << buf_checksum_algorithm_name( SRV_CHECKSUM_ALGORITHM_CRC32) << " " - << crc32 << "/" << crc32_legacy + << crc32 +#ifdef INNODB_BUG_ENDIAN_CRC32 + << "/" << crc32_legacy +#endif << ", " << buf_checksum_algorithm_name( SRV_CHECKSUM_ALGORITHM_INNODB) << " " @@ -1431,7 +1333,10 @@ buf_page_print(const byte* read_buf, const page_size_t& page_size) << ", calculated checksums for field2: " << buf_checksum_algorithm_name( SRV_CHECKSUM_ALGORITHM_CRC32) << " " - << crc32 << "/" << crc32_legacy + << crc32 +#ifdef INNODB_BUG_ENDIAN_CRC32 + << "/" << crc32_legacy +#endif << ", " << buf_checksum_algorithm_name( SRV_CHECKSUM_ALGORITHM_INNODB) << " " @@ -4068,10 +3973,12 @@ buf_zip_decompress( << ", crc32: " << page_zip_calc_checksum( frame, size, SRV_CHECKSUM_ALGORITHM_CRC32) +#ifdef INNODB_BUG_ENDIAN_CRC32 << "/" << page_zip_calc_checksum( frame, size, SRV_CHECKSUM_ALGORITHM_CRC32, true) +#endif << " innodb: " << page_zip_calc_checksum( frame, size, SRV_CHECKSUM_ALGORITHM_INNODB) diff --git a/storage/innobase/buf/buf0checksum.cc b/storage/innobase/buf/buf0checksum.cc index 74b182affbf..6c103c4f44b 100644 --- a/storage/innobase/buf/buf0checksum.cc +++ b/storage/innobase/buf/buf0checksum.cc @@ -39,44 +39,56 @@ ha_innodb.cc:12251: error: cannot convert 'srv_checksum_algorithm_t*' to 'long unsigned int*' in initialization */ ulong srv_checksum_algorithm = SRV_CHECKSUM_ALGORITHM_INNODB; -/** set if we have found pages matching legacy big endian checksum */ -bool legacy_big_endian_checksum = false; -/** Calculates the CRC32 checksum of a page. The value is stored to the page +#ifdef INNODB_BUG_ENDIAN_CRC32 +/** Calculate the CRC32 checksum of a page. The value is stored to the page when it is written to a file and also checked for a match when reading from -the file. When reading we allow both normal CRC32 and CRC-legacy-big-endian -variants. Note that we must be careful to calculate the same value on 32-bit -and 64-bit architectures. +the file. Note that we must be careful to calculate the same value on all +architectures. +@param[in] page buffer page (srv_page_size bytes) +@param[in] bug_endian whether to use big endian byteorder +when converting byte strings to integers, for bug-compatibility with +big-endian architecture running MySQL 5.6, MariaDB 10.0 or MariaDB 10.1 +@return CRC-32C */ +uint32_t buf_calc_page_crc32(const byte* page, bool bug_endian) +{ + return bug_endian + ? ut_crc32_legacy_big_endian( + page + FIL_PAGE_OFFSET, + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + - FIL_PAGE_OFFSET) + ^ ut_crc32_legacy_big_endian(page + FIL_PAGE_DATA, + srv_page_size + - (FIL_PAGE_DATA + + FIL_PAGE_END_LSN_OLD_CHKSUM)) + : ut_crc32(page + FIL_PAGE_OFFSET, + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + - FIL_PAGE_OFFSET) + ^ ut_crc32(page + FIL_PAGE_DATA, + srv_page_size + - (FIL_PAGE_DATA + FIL_PAGE_END_LSN_OLD_CHKSUM)); +} +#else +/** Calculate the CRC32 checksum of a page. The value is stored to the page +when it is written to a file and also checked for a match when reading from +the file. Note that we must be careful to calculate the same value on all +architectures. @param[in] page buffer page (srv_page_size bytes) -@param[in] use_legacy_big_endian if true then use big endian -byteorder when converting byte strings to integers -@return checksum */ -uint32_t -buf_calc_page_crc32( - const byte* page, - bool use_legacy_big_endian /* = false */) +@return CRC-32C */ +uint32_t buf_calc_page_crc32(const byte* page) { - /* Since the field FIL_PAGE_FILE_FLUSH_LSN, and in versions <= 4.1.x - FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, are written outside the buffer pool - to the first pages of data files, we have to skip them in the page - checksum calculation. - We must also skip the field FIL_PAGE_SPACE_OR_CHKSUM where the - checksum is stored, and also the last 8 bytes of page because - there we store the old formula checksum. */ - - ut_crc32_func_t crc32_func = use_legacy_big_endian - ? ut_crc32_legacy_big_endian - : ut_crc32; - - const uint32_t c1 = crc32_func( - page + FIL_PAGE_OFFSET, - FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION - FIL_PAGE_OFFSET); - - const uint32_t c2 = crc32_func( - page + FIL_PAGE_DATA, - srv_page_size - FIL_PAGE_DATA - FIL_PAGE_END_LSN_OLD_CHKSUM); - - return(c1 ^ c2); + /* Note: innodb_checksum_algorithm=crc32 could and should have + included the entire page in the checksum, and CRC-32 values + should be combined with the CRC-32 function, not with + exclusive OR. We stick to the current algorithm in order to + remain compatible with old data files. */ + return ut_crc32(page + FIL_PAGE_OFFSET, + FIL_PAGE_FILE_FLUSH_LSN_OR_KEY_VERSION + - FIL_PAGE_OFFSET) + ^ ut_crc32(page + FIL_PAGE_DATA, + srv_page_size + - (FIL_PAGE_DATA + FIL_PAGE_END_LSN_OLD_CHKSUM)); } +#endif /** Calculate a checksum which is stored to the page when it is written to a file. Note that we must be careful to calculate the same value on diff --git a/storage/innobase/fil/fil0crypt.cc b/storage/innobase/fil/fil0crypt.cc index 727264573da..637cf6f6173 100644 --- a/storage/innobase/fil/fil0crypt.cc +++ b/storage/innobase/fil/fil0crypt.cc @@ -2615,7 +2615,7 @@ fil_space_verify_crypt_checksum( uint32_t checksum1 = mach_read_from_4(page + FIL_PAGE_SPACE_OR_CHKSUM); uint32_t checksum2; - bool valid; + bool valid = false; if (page_size.is_compressed()) { valid = checksum1 == cchecksum1; @@ -2623,12 +2623,32 @@ fil_space_verify_crypt_checksum( } else { checksum2 = mach_read_from_4( page + srv_page_size - FIL_PAGE_END_LSN_OLD_CHKSUM); - valid = buf_page_is_checksum_valid_crc32( - page, checksum1, checksum2, false - /* FIXME: also try the original crc32 that was - buggy on big-endian architectures? */) - || buf_page_is_checksum_valid_innodb( + + srv_checksum_algorithm_t algorithm = + static_cast<srv_checksum_algorithm_t>( + srv_checksum_algorithm); + switch (algorithm) { + case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: + valid = buf_page_is_checksum_valid_crc32( + page, checksum1, checksum2); + break; + case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: + valid = buf_page_is_checksum_valid_innodb( page, checksum1, checksum2); + break; + case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: + case SRV_CHECKSUM_ALGORITHM_CRC32: + case SRV_CHECKSUM_ALGORITHM_INNODB: + case SRV_CHECKSUM_ALGORITHM_NONE: + /* never supported + innodb_checksum_algorithm=none or strict_none + for encrypted pages. */ + valid = buf_page_is_checksum_valid_crc32( + page, checksum1, checksum2) + || buf_page_is_checksum_valid_innodb( + page, checksum1, checksum2); + break; + } } if (encrypted && valid) { @@ -2653,8 +2673,11 @@ fil_space_verify_crypt_checksum( ib::info() << "If unencrypted: stored checksum [" << checksum1 << ":" << checksum2 << "] calculated crc32 [" - << buf_calc_page_crc32(page, false) << ":" - << buf_calc_page_crc32(page, true) << "] innodb [" + << buf_calc_page_crc32(page) +# ifdef INNODB_BUG_ENDIAN_CRC32 + << ":" << buf_calc_page_crc32(page, true) +# endif /* INNODB_BUG_ENDIAN_CRC32 */ + << "] innodb [" << buf_calc_page_old_checksum(page) << ":" << buf_calc_page_new_checksum(page) << "] LSN " << mach_read_from_4(page + FIL_PAGE_LSN); diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index eaa2ee504aa..d90088f2c5e 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -5447,6 +5447,26 @@ normalize_table_name_c_low( } } +create_table_info_t::create_table_info_t( + THD* thd, + TABLE* form, + HA_CREATE_INFO* create_info, + char* table_name, + char* remote_path, + bool file_per_table, + trx_t* trx) + : m_thd(thd), + m_trx(trx), + m_form(form), + m_default_row_format(innodb_default_row_format), + m_create_info(create_info), + m_table_name(table_name), m_table(NULL), + m_drop_before_rollback(false), + m_remote_path(remote_path), + m_innodb_file_per_table(file_per_table) +{ +} + /** Normalizes a table name string. A normalized name consists of the database name catenated to '/' and table name. For example: test/mytable. @@ -11474,7 +11494,7 @@ Check engine specific table options not handled by SQL-parser. const char* create_table_info_t::check_table_options() { - enum row_type row_format = m_form->s->row_type; + enum row_type row_format = m_create_info->row_type; ha_table_option_struct *options= m_form->s->option_struct; fil_encryption_t encrypt = (fil_encryption_t)options->encryption; bool should_encrypt = (encrypt == FIL_ENCRYPTION_ON); @@ -11521,7 +11541,16 @@ create_table_info_t::check_table_options() return "PAGE_COMPRESSED"; } - if (row_format == ROW_TYPE_REDUNDANT) { + switch (row_format) { + default: + break; + case ROW_TYPE_DEFAULT: + if (m_default_row_format + != DEFAULT_ROW_FORMAT_REDUNDANT) { + break; + } + /* fall through */ + case ROW_TYPE_REDUNDANT: push_warning( m_thd, Sql_condition::WARN_LEVEL_WARN, HA_WRONG_CREATE_OPTION, @@ -11727,9 +11756,9 @@ create_table_info_t::parse_table_name( /** Determine InnoDB table flags. If strict_mode=OFF, this will adjust the flags to what should be assumed. -@retval true if successful, false if error */ -bool -create_table_info_t::innobase_table_flags() +@retval true on success +@retval false on error */ +bool create_table_info_t::innobase_table_flags() { DBUG_ENTER("innobase_table_flags"); @@ -11737,7 +11766,7 @@ create_table_info_t::innobase_table_flags() ulint zip_ssize = 0; enum row_type row_type; rec_format_t innodb_row_format = - get_row_format(innodb_default_row_format); + get_row_format(m_default_row_format); const bool is_temp = m_create_info->options & HA_LEX_CREATE_TMP_TABLE; bool zip_allowed @@ -11838,7 +11867,7 @@ index_bad: } } - row_type = m_form->s->row_type; + row_type = m_create_info->row_type; if (zip_ssize && zip_allowed) { /* if ROW_FORMAT is set to default, @@ -12174,8 +12203,6 @@ create_table_info_t::initialize() DBUG_RETURN(HA_ERR_WRONG_INDEX); } - ut_ad(m_form->s->row_type == m_create_info->row_type); - /* Get the transaction associated with the current thd, or create one if not yet created */ @@ -12219,8 +12246,6 @@ int create_table_info_t::prepare_create_table(const char* name, bool strict) ut_ad(m_thd != NULL); ut_ad(m_create_info != NULL); - ut_ad(m_form->s->row_type == m_create_info->row_type); - set_tablespace_type(false); normalize_table_name(m_table_name, name); @@ -13227,8 +13252,24 @@ int ha_innobase::truncate() trx_rollback_for_mysql(trx); row_mysql_unlock_data_dictionary(trx); } else { + switch (dict_tf_get_rec_format(ib_table->flags)) { + case REC_FORMAT_REDUNDANT: + info.row_type = ROW_TYPE_REDUNDANT; + break; + case REC_FORMAT_COMPACT: + info.row_type = ROW_TYPE_COMPACT; + break; + case REC_FORMAT_COMPRESSED: + info.row_type = ROW_TYPE_COMPRESSED; + break; + case REC_FORMAT_DYNAMIC: + info.row_type = ROW_TYPE_DYNAMIC; + break; + } + err = create(name, table, &info, - dict_table_is_file_per_table(ib_table), trx); + ib_table->is_temporary() + || dict_table_is_file_per_table(ib_table), trx); } trx_free(trx); diff --git a/storage/innobase/handler/ha_innodb.h b/storage/innobase/handler/ha_innodb.h index ed7d380db8b..6f0c5b535fb 100644 --- a/storage/innobase/handler/ha_innodb.h +++ b/storage/innobase/handler/ha_innodb.h @@ -651,15 +651,7 @@ public: char* table_name, char* remote_path, bool file_per_table, - trx_t* trx = NULL) - :m_thd(thd), - m_trx(trx), - m_form(form), - m_create_info(create_info), - m_table_name(table_name), m_table(NULL), m_drop_before_rollback(false), - m_remote_path(remote_path), - m_innodb_file_per_table(file_per_table) - {} + trx_t* trx = NULL); /** Initialize the object. */ int initialize(); @@ -768,6 +760,9 @@ private: /** Information on table columns and indexes. */ const TABLE* m_form; + /** Value of innodb_default_row_format */ + const ulong m_default_row_format; + /** Create options. */ HA_CREATE_INFO* m_create_info; diff --git a/storage/innobase/include/buf0buf.h b/storage/innobase/include/buf0buf.h index 8ca9ddd28fe..86a433d36e3 100644 --- a/storage/innobase/include/buf0buf.h +++ b/storage/innobase/include/buf0buf.h @@ -716,14 +716,12 @@ buf_block_unfix( @param[in] read_buf database page @param[in] checksum_field1 new checksum field @param[in] checksum_field2 old checksum field -@param[in] use_legacy_big_endian use legacy big endian algorithm @return true if the page is in crc32 checksum format. */ bool buf_page_is_checksum_valid_crc32( const byte* read_buf, ulint checksum_field1, - ulint checksum_field2, - bool use_legacy_big_endian) + ulint checksum_field2) MY_ATTRIBUTE((nonnull(1), warn_unused_result)); /** Checks if the page is in innodb checksum format. diff --git a/storage/innobase/include/buf0checksum.h b/storage/innobase/include/buf0checksum.h index 98c6ff16a6a..06eb37906d2 100644 --- a/storage/innobase/include/buf0checksum.h +++ b/storage/innobase/include/buf0checksum.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -29,19 +29,26 @@ Created Aug 11, 2011 Vasil Dimov #include "buf0types.h" +#ifdef INNODB_BUG_ENDIAN_CRC32 /** Calculate the CRC32 checksum of a page. The value is stored to the page when it is written to a file and also checked for a match when reading from -the file. When reading we allow both normal CRC32 and CRC-legacy-big-endian -variants. Note that we must be careful to calculate the same value on 32-bit -and 64-bit architectures. +the file. Note that we must be careful to calculate the same value on all +architectures. +@param[in] page buffer page (srv_page_size bytes) +@param[in] bug_endian whether to use big endian byteorder +when converting byte strings to integers, for bug-compatibility with +big-endian architecture running MySQL 5.6, MariaDB 10.0 or MariaDB 10.1 +@return CRC-32C */ +uint32_t buf_calc_page_crc32(const byte* page, bool bug_endian = false); +#else +/** Calculate the CRC32 checksum of a page. The value is stored to the page +when it is written to a file and also checked for a match when reading from +the file. Note that we must be careful to calculate the same value on all +architectures. @param[in] page buffer page (srv_page_size bytes) -@param[in] use_legacy_big_endian if true then use big endian -byteorder when converting byte strings to integers -@return checksum */ -uint32_t -buf_calc_page_crc32( - const byte* page, - bool use_legacy_big_endian = false); +@return CRC-32C */ +uint32_t buf_calc_page_crc32(const byte* page); +#endif /** Calculate a checksum which is stored to the page when it is written to a file. Note that we must be careful to calculate the same value on @@ -69,6 +76,5 @@ const char* buf_checksum_algorithm_name(srv_checksum_algorithm_t algo); extern ulong srv_checksum_algorithm; -extern bool legacy_big_endian_checksum; #endif /* buf0checksum_h */ diff --git a/storage/innobase/include/dict0dict.ic b/storage/innobase/include/dict0dict.ic index 03b842e041d..78e16774a09 100644 --- a/storage/innobase/include/dict0dict.ic +++ b/storage/innobase/include/dict0dict.ic @@ -685,31 +685,30 @@ dict_tf_set( bool page_compressed, ulint page_compression_level) { + *flags = use_data_dir ? 1 << DICT_TF_POS_DATA_DIR : 0; + switch (format) { case REC_FORMAT_REDUNDANT: - *flags = 0; ut_ad(zip_ssize == 0); - break; + /* no other options are allowed */ + ut_ad(!page_compressed); + return; case REC_FORMAT_COMPACT: - *flags = DICT_TF_COMPACT; + *flags |= DICT_TF_COMPACT; ut_ad(zip_ssize == 0); break; case REC_FORMAT_COMPRESSED: - *flags = DICT_TF_COMPACT + *flags |= DICT_TF_COMPACT | (1 << DICT_TF_POS_ATOMIC_BLOBS) | (zip_ssize << DICT_TF_POS_ZIP_SSIZE); break; case REC_FORMAT_DYNAMIC: - *flags = DICT_TF_COMPACT + *flags |= DICT_TF_COMPACT | (1 << DICT_TF_POS_ATOMIC_BLOBS); ut_ad(zip_ssize == 0); break; } - if (use_data_dir) { - *flags |= (1 << DICT_TF_POS_DATA_DIR); - } - if (page_compressed) { *flags |= (1 << DICT_TF_POS_ATOMIC_BLOBS) | (1 << DICT_TF_POS_PAGE_COMPRESSION) diff --git a/storage/innobase/include/page0page.h b/storage/innobase/include/page0page.h index 0ff63f8047f..58115b767d2 100644 --- a/storage/innobase/include/page0page.h +++ b/storage/innobase/include/page0page.h @@ -1351,17 +1351,6 @@ const rec_t* page_find_rec_max_not_deleted( const page_t* page); -/** Issue a warning when the checksum that is stored in the page is valid, -but different than the global setting innodb_checksum_algorithm. -@param[in] current_algo current checksum algorithm -@param[in] page_checksum page valid checksum -@param[in] page_id page identifier */ -void -page_warn_strict_checksum( - srv_checksum_algorithm_t curr_algo, - srv_checksum_algorithm_t page_checksum, - const page_id_t page_id); - #ifdef UNIV_MATERIALIZE #undef UNIV_INLINE #define UNIV_INLINE UNIV_INLINE_ORIGINAL diff --git a/storage/innobase/include/page0zip.h b/storage/innobase/include/page0zip.h index a2910a73634..bb9bb049c22 100644 --- a/storage/innobase/include/page0zip.h +++ b/storage/innobase/include/page0zip.h @@ -513,16 +513,17 @@ page_zip_parse_compress( @param[in] data compressed page @param[in] size size of compressed page @param[in] algo algorithm to use -@param[in] use_legacy_big_endian only used if algo is -SRV_CHECKSUM_ALGORITHM_CRC32 or SRV_CHECKSUM_ALGORITHM_STRICT_CRC32 - if true -then use big endian byteorder when converting byte strings to integers. @return page checksum */ uint32_t page_zip_calc_checksum( const void* data, ulint size, - srv_checksum_algorithm_t algo, - bool use_legacy_big_endian = false); + srv_checksum_algorithm_t algo +#ifdef INNODB_BUG_ENDIAN_CRC32 + /** for crc32, use the big-endian bug-compatible crc32 variant */ + , bool use_legacy_big_endian = false +#endif +); /**********************************************************************//** Verify a compressed page's checksum. diff --git a/storage/innobase/include/ut0crc32.h b/storage/innobase/include/ut0crc32.h index 32ad066f85a..b8fc4168dfd 100644 --- a/storage/innobase/include/ut0crc32.h +++ b/storage/innobase/include/ut0crc32.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2011, 2015, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2016, MariaDB Corporation. +Copyright (c) 2016, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -47,9 +47,11 @@ typedef uint32_t (*ut_crc32_func_t)(const byte* ptr, ulint len); /** Pointer to CRC32 calculation function. */ extern ut_crc32_func_t ut_crc32; -/** CRC32 calculation function, which uses big-endian byte order +#ifdef INNODB_BUG_ENDIAN_CRC32 +/** Pointer to CRC32 calculation function, which uses big-endian byte order when converting byte strings to integers internally. */ extern uint32_t ut_crc32_legacy_big_endian(const byte* buf, ulint len); +#endif /* INNODB_BUG_ENDIAN_CRC32 */ /** Text description of CRC32 implementation */ extern const char* ut_crc32_implementation; diff --git a/storage/innobase/innodb.cmake b/storage/innobase/innodb.cmake index a728dd08c0d..33b73813b0e 100644 --- a/storage/innobase/innodb.cmake +++ b/storage/innobase/innodb.cmake @@ -25,6 +25,7 @@ INCLUDE(lzma.cmake) INCLUDE(bzip2.cmake) INCLUDE(snappy.cmake) INCLUDE(numa) +INCLUDE(TestBigEndian) MYSQL_CHECK_LZ4() MYSQL_CHECK_LZO() @@ -32,6 +33,7 @@ MYSQL_CHECK_LZMA() MYSQL_CHECK_BZIP2() MYSQL_CHECK_SNAPPY() MYSQL_CHECK_NUMA() +TEST_BIG_ENDIAN(IS_BIG_ENDIAN) INCLUDE(${MYSQL_CMAKE_SCRIPT_DIR}/compile_flags.cmake) @@ -119,6 +121,11 @@ ELSEIF(WITH_INNODB_ROOT_GUESS) ADD_DEFINITIONS(-DBTR_CUR_ADAPT) ENDIF() +OPTION(WITH_INNODB_BUG_ENDIAN_CRC32 "Weaken innodb_checksum_algorithm=crc32 by supporting upgrade from big-endian systems running 5.6/10.0/10.1" ${IS_BIG_ENDIAN}) +IF(WITH_INNODB_BUG_ENDIAN_CRC32) + ADD_DEFINITIONS(-DINNODB_BUG_ENDIAN_CRC32) +ENDIF() + OPTION(WITH_INNODB_EXTRA_DEBUG "Enable extra InnoDB debug checks" OFF) IF(WITH_INNODB_EXTRA_DEBUG) ADD_DEFINITIONS(-DUNIV_ZIP_DEBUG) diff --git a/storage/innobase/page/page0page.cc b/storage/innobase/page/page0page.cc index 36a4cb46cf7..f56d944697f 100644 --- a/storage/innobase/page/page0page.cc +++ b/storage/innobase/page/page0page.cc @@ -2836,42 +2836,3 @@ page_find_rec_max_not_deleted( } return(prev_rec); } - -/** Issue a warning when the checksum that is stored in the page is valid, -but different than the global setting innodb_checksum_algorithm. -@param[in] current_algo current checksum algorithm -@param[in] page_checksum page valid checksum -@param[in] page_id page identifier */ -void -page_warn_strict_checksum( - srv_checksum_algorithm_t curr_algo, - srv_checksum_algorithm_t page_checksum, - const page_id_t page_id) -{ - srv_checksum_algorithm_t curr_algo_nonstrict; - switch (curr_algo) { - case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: - curr_algo_nonstrict = SRV_CHECKSUM_ALGORITHM_CRC32; - break; - case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: - curr_algo_nonstrict = SRV_CHECKSUM_ALGORITHM_INNODB; - break; - case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: - curr_algo_nonstrict = SRV_CHECKSUM_ALGORITHM_NONE; - break; - default: - ut_error; - } - - ib::warn() << "innodb_checksum_algorithm is set to \"" - << buf_checksum_algorithm_name(curr_algo) << "\"" - << " but the page " << page_id << " contains a valid checksum \"" - << buf_checksum_algorithm_name(page_checksum) << "\". " - << " Accepting the page as valid. Change" - << " innodb_checksum_algorithm to \"" - << buf_checksum_algorithm_name(curr_algo_nonstrict) - << "\" to silently accept such pages or rewrite all pages" - << " so that they contain \"" - << buf_checksum_algorithm_name(curr_algo_nonstrict) - << "\" checksum."; -} diff --git a/storage/innobase/page/page0zip.cc b/storage/innobase/page/page0zip.cc index 4b611baefae..05889b617bc 100644 --- a/storage/innobase/page/page0zip.cc +++ b/storage/innobase/page/page0zip.cc @@ -4980,18 +4980,17 @@ corrupt: @param[in] data compressed page @param[in] size size of compressed page @param[in] algo algorithm to use -@param[in] use_legacy_big_endian only used if algo is -SRV_CHECKSUM_ALGORITHM_CRC32 or SRV_CHECKSUM_ALGORITHM_STRICT_CRC32 - if true -then use big endian byteorder when converting byte strings to integers. -SRV_CHECKSUM_ALGORITHM_CRC32 or SRV_CHECKSUM_ALGORITHM_STRICT_CRC32 - if true -then use big endian byteorder when converting byte strings to integers. @return page checksum */ uint32_t page_zip_calc_checksum( const void* data, ulint size, - srv_checksum_algorithm_t algo, - bool use_legacy_big_endian /* = false */) + srv_checksum_algorithm_t algo +#ifdef INNODB_BUG_ENDIAN_CRC32 + /** for crc32, use the big-endian bug-compatible crc32 variant */ + , bool use_legacy_big_endian +#endif +) { uLong adler; const Bytef* s = static_cast<const byte*>(data); @@ -5002,25 +5001,25 @@ page_zip_calc_checksum( switch (algo) { case SRV_CHECKSUM_ALGORITHM_CRC32: case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: - { - ut_ad(size > FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - - ut_crc32_func_t crc32_func = use_legacy_big_endian - ? ut_crc32_legacy_big_endian - : ut_crc32; - - const uint32_t crc32 - = crc32_func( - s + FIL_PAGE_OFFSET, - FIL_PAGE_LSN - FIL_PAGE_OFFSET) - ^ crc32_func( + ut_ad(size > FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); +#ifdef INNODB_BUG_ENDIAN_CRC32 + if (use_legacy_big_endian) { + return ut_crc32_legacy_big_endian(s + FIL_PAGE_OFFSET, + FIL_PAGE_LSN + - FIL_PAGE_OFFSET) + ^ ut_crc32_legacy_big_endian( s + FIL_PAGE_TYPE, 2) - ^ crc32_func( + ^ ut_crc32_legacy_big_endian( s + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, - size - FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); - - return(crc32); + size + - FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); } +#endif + return ut_crc32(s + FIL_PAGE_OFFSET, + FIL_PAGE_LSN - FIL_PAGE_OFFSET) + ^ ut_crc32(s + FIL_PAGE_TYPE, 2) + ^ ut_crc32(s + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, + size - FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); case SRV_CHECKSUM_ALGORITHM_INNODB: case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: ut_ad(size > FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); @@ -5055,19 +5054,8 @@ page_zip_verify_checksum( const void* data, /*!< in: compressed page */ ulint size) /*!< in: size of compressed page */ { - ib_uint32_t stored; - ib_uint32_t calc; - - stored = static_cast<ib_uint32_t>(mach_read_from_4( - static_cast<const unsigned char*>(data) + FIL_PAGE_SPACE_OR_CHKSUM)); - - ulint page_no MY_ATTRIBUTE((unused)) = - mach_read_from_4(static_cast<const unsigned char*> - (data) + FIL_PAGE_OFFSET); - ulint space_id MY_ATTRIBUTE((unused)) = - mach_read_from_4(static_cast<const unsigned char*> - (data) + FIL_PAGE_SPACE_ID); - const page_id_t page_id(space_id, page_no); + const uint32_t stored = mach_read_from_4( + static_cast<const byte*>(data) + FIL_PAGE_SPACE_OR_CHKSUM); compile_time_assert(!(FIL_PAGE_LSN % 8)); @@ -5109,8 +5097,7 @@ page_zip_verify_checksum( return(TRUE); } - calc = static_cast<ib_uint32_t>(page_zip_calc_checksum( - data, size, curr_algo)); + uint32_t calc = page_zip_calc_checksum(data, size, curr_algo); #ifdef UNIV_INNOCHECKSUM if (log_file) { @@ -5143,149 +5130,44 @@ page_zip_verify_checksum( return(TRUE); } - bool legacy_checksum_checked = false; - switch (curr_algo) { case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: - case SRV_CHECKSUM_ALGORITHM_CRC32: { - - if (stored == BUF_NO_CHECKSUM_MAGIC) { -#ifndef UNIV_INNOCHECKSUM - if (curr_algo - == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { - page_warn_strict_checksum( - curr_algo, - SRV_CHECKSUM_ALGORITHM_NONE, - page_id); - } -#endif /* UNIV_INNOCHECKSUM */ - - return(TRUE); - } - - /* We need to check whether the stored checksum matches legacy - big endian checksum or Innodb checksum. We optimize the order - based on earlier results. if earlier we have found pages - matching legacy big endian checksum, we try to match it first. - Otherwise we check innodb checksum first. */ - if (legacy_big_endian_checksum) { - const uint32_t calculated = - page_zip_calc_checksum(data, size, curr_algo, true); - if (stored == calculated) { - - return(TRUE); - } - legacy_checksum_checked = true; - } - - uint32_t calculated = - page_zip_calc_checksum(data, size, SRV_CHECKSUM_ALGORITHM_INNODB); - - if (stored == calculated) { - -#ifndef UNIV_INNOCHECKSUM - if (curr_algo - == SRV_CHECKSUM_ALGORITHM_STRICT_CRC32) { - page_warn_strict_checksum( - curr_algo, - SRV_CHECKSUM_ALGORITHM_INNODB, - page_id); - } -#endif /* UNIV_INNOCHECKSUM */ - - return(TRUE); - } - - calculated = page_zip_calc_checksum( - data, size, curr_algo, true); - - /* If legacy checksum is not checked, do it now. */ - if ((legacy_checksum_checked - && stored == calculated)) { - legacy_big_endian_checksum = true; - return(TRUE); - } - - break; - } +#ifdef INNODB_BUG_ENDIAN_CRC32 + return stored == page_zip_calc_checksum(data, size, curr_algo, + true); +#endif + /* fall through */ case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: - case SRV_CHECKSUM_ALGORITHM_INNODB: { - + case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: + return FALSE; + case SRV_CHECKSUM_ALGORITHM_CRC32: if (stored == BUF_NO_CHECKSUM_MAGIC) { -#ifndef UNIV_INNOCHECKSUM - if (curr_algo - == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { - page_warn_strict_checksum( - curr_algo, - SRV_CHECKSUM_ALGORITHM_NONE, - page_id); - } -#endif /* UNIV_INNOCHECKSUM */ - - return(TRUE); - } - - const uint32_t calculated = page_zip_calc_checksum( - data, size, SRV_CHECKSUM_ALGORITHM_CRC32); - uint32_t calculated1; - - if (stored == calculated - || stored == (calculated1 = - page_zip_calc_checksum(data, size, SRV_CHECKSUM_ALGORITHM_CRC32, true)) - ) { -#ifndef UNIV_INNOCHECKSUM - if (curr_algo - == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB) { - page_warn_strict_checksum( - curr_algo, - SRV_CHECKSUM_ALGORITHM_CRC32, - page_id); - } -#endif /* UNIV_INNOCHECKSUM */ - return(TRUE); - } - - break; - } - case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: { - - uint32_t calculated = page_zip_calc_checksum( - data, size, SRV_CHECKSUM_ALGORITHM_CRC32); - const uint32_t calculated1 = page_zip_calc_checksum( - data, size, SRV_CHECKSUM_ALGORITHM_CRC32, true); - - if (stored == calculated - || stored == calculated1) { -#ifndef UNIV_INNOCHECKSUM - page_warn_strict_checksum( - curr_algo, - SRV_CHECKSUM_ALGORITHM_CRC32, - page_id); -#endif /* UNIV_INNOCHECKSUM */ return(TRUE); } - calculated = page_zip_calc_checksum( - data, size, SRV_CHECKSUM_ALGORITHM_INNODB); - - if (stored == calculated) { - -#ifndef UNIV_INNOCHECKSUM - page_warn_strict_checksum( - curr_algo, - SRV_CHECKSUM_ALGORITHM_INNODB, - page_id); -#endif /* UNIV_INNOCHECKSUM */ - return(TRUE); + return +#ifdef INNODB_BUG_ENDIAN_CRC32 + stored == page_zip_calc_checksum(data, size, curr_algo, + true) || +#endif + stored == page_zip_calc_checksum( + data, size, SRV_CHECKSUM_ALGORITHM_INNODB); + case SRV_CHECKSUM_ALGORITHM_INNODB: + if (stored == BUF_NO_CHECKSUM_MAGIC) { + return TRUE; } - break; - } + return stored == page_zip_calc_checksum( + data, size, SRV_CHECKSUM_ALGORITHM_CRC32) +#ifdef INNODB_BUG_ENDIAN_CRC32 + || stored == page_zip_calc_checksum( + data, size, + SRV_CHECKSUM_ALGORITHM_CRC32, true) +#endif + ; case SRV_CHECKSUM_ALGORITHM_NONE: - ut_error; - /* no default so the compiler will emit a warning if new enum - is added and not handled here */ + return TRUE; } - return(FALSE); + return FALSE; } diff --git a/storage/innobase/ut/ut0crc32.cc b/storage/innobase/ut/ut0crc32.cc index 5d95202b7c9..0b1c1b3991a 100644 --- a/storage/innobase/ut/ut0crc32.cc +++ b/storage/innobase/ut/ut0crc32.cc @@ -2,7 +2,7 @@ Copyright (c) 2009, 2010 Facebook, Inc. All Rights Reserved. Copyright (c) 2011, 2015, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2016, MariaDB Corporation. +Copyright (c) 2016, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -469,6 +469,7 @@ ut_crc32_64_sw( *len -= 8; } +#ifdef INNODB_BUG_ENDIAN_CRC32 /** Calculate CRC32 over 64-bit byte string using a software implementation. The byte string is converted to a 64-bit integer using big endian byte order. @param[in,out] crc crc32 checksum so far when this function is called, @@ -494,6 +495,7 @@ ut_crc32_64_legacy_big_endian_sw( *data += 8; *len -= 8; } +#endif /* INNODB_BUG_ENDIAN_CRC32 */ /** Calculates CRC32 in software, without using CPU instructions. @param[in] buf data over which to calculate CRC32 @@ -545,16 +547,14 @@ ut_crc32_sw( return(~crc); } +#ifdef INNODB_BUG_ENDIAN_CRC32 /** Calculates CRC32 in software, without using CPU instructions. This function uses big endian byte ordering when converting byte sequence to integers. @param[in] buf data over which to calculate CRC32 @param[in] len data length @return CRC-32C (polynomial 0x11EDC6F41) */ -uint32_t -ut_crc32_legacy_big_endian( - const byte* buf, - ulint len) +uint32_t ut_crc32_legacy_big_endian(const byte* buf, ulint len) { uint32_t crc = 0xFFFFFFFFU; @@ -596,6 +596,7 @@ ut_crc32_legacy_big_endian( return(~crc); } +#endif /* INNODB_BUG_ENDIAN_CRC32 */ /********************************************************************//** Initializes the data structures used by ut_crc32*(). Does not do any @@ -636,6 +637,9 @@ ut_crc32_init() if (features_ecx & 1 << 20) { ut_crc32 = ut_crc32_hw; +#ifdef INNODB_BUG_ENDIAN_CRC32 + ut_crc32_legacy_big_endian = ut_crc32_legacy_big_endian_hw; +#endif /* INNODB_BUG_ENDIAN_CRC32 */ ut_crc32_implementation = "Using SSE2 crc32 instructions"; } #endif |