diff options
author | Julius Goryavsky <julius.goryavsky@mariadb.com> | 2022-01-17 03:00:42 +0100 |
---|---|---|
committer | Julius Goryavsky <julius.goryavsky@mariadb.com> | 2022-01-17 03:00:42 +0100 |
commit | d0280be10ffb15a023132509decbbe60a7cfae28 (patch) | |
tree | 839851d5aca0d3909a4c149997cb7acf6c71dfd2 | |
parent | 746050d02d6b927c59767c8ff121e0d674520f90 (diff) | |
download | mariadb-git-bb-10.2-sysprg-binlog.tar.gz |
Experimental changebb-10.2-sysprg-binlog
19 files changed, 940 insertions, 338 deletions
diff --git a/extra/mariabackup/backup_copy.cc b/extra/mariabackup/backup_copy.cc index c46c32213d6..967da80591a 100644 --- a/extra/mariabackup/backup_copy.cc +++ b/extra/mariabackup/backup_copy.cc @@ -582,7 +582,6 @@ datafile_read(datafile_cur_t *cursor) Check to see if a file exists. Takes name of the file to check. @return true if file exists. */ -static bool file_exists(const char *filename) { @@ -1547,13 +1546,14 @@ bool backup_start(CorruptedPages &corrupted_pages) if (!write_galera_info(mysql_connection)) { return(false); } - write_current_binlog_file(mysql_connection); } - if (opt_binlog_info == BINLOG_INFO_ON) { + bool with_binlogs = opt_binlog_info == BINLOG_INFO_ON; - lock_binlog_maybe(mysql_connection); - write_binlog_info(mysql_connection); + if (with_binlogs || opt_galera_info) { + if (!write_current_binlog_file(mysql_connection, with_binlogs)) { + return(false); + } } if (have_flush_engine_logs && !opt_no_lock) { diff --git a/extra/mariabackup/backup_copy.h b/extra/mariabackup/backup_copy.h index 62b2b1bc232..858182001ce 100644 --- a/extra/mariabackup/backup_copy.h +++ b/extra/mariabackup/backup_copy.h @@ -32,6 +32,13 @@ copy_file(ds_ctxt_t *datasink, const char *dst_file_path, uint thread_n); +/************************************************************************ +Check to see if a file exists. +Takes name of the file to check. +@return true if file exists. */ +bool +file_exists(const char *filename); + /** Start --backup */ bool backup_start(CorruptedPages &corrupted_pages); /** Release resources after backup_start() */ diff --git a/extra/mariabackup/backup_mysql.cc b/extra/mariabackup/backup_mysql.cc index 824b8b6d5f3..d953cb23b3d 100644 --- a/extra/mariabackup/backup_mysql.cc +++ b/extra/mariabackup/backup_mysql.cc @@ -1337,25 +1337,12 @@ cleanup: Flush and copy the current binary log file into the backup, if GTID is enabled */ bool -write_current_binlog_file(MYSQL *connection) +write_current_binlog_file(MYSQL *connection, bool write_binlogs) { - char *executed_gtid_set = NULL; char *gtid_binlog_state = NULL; - char *log_bin_file = NULL; char *log_bin_dir = NULL; - bool gtid_exists; + bool gtid_exists = false; bool result = true; - char filepath[FN_REFLEN]; - - mysql_variable status[] = { - {"Executed_Gtid_Set", &executed_gtid_set}, - {NULL, NULL} - }; - - mysql_variable status_after_flush[] = { - {"File", &log_bin_file}, - {NULL, NULL} - }; mysql_variable vars[] = { {"gtid_binlog_state", >id_binlog_state}, @@ -1363,13 +1350,36 @@ write_current_binlog_file(MYSQL *connection) {NULL, NULL} }; - read_mysql_variables(connection, "SHOW MASTER STATUS", status, false); read_mysql_variables(connection, "SHOW VARIABLES", vars, true); - gtid_exists = (executed_gtid_set && *executed_gtid_set) - || (gtid_binlog_state && *gtid_binlog_state); + if (!write_binlogs) { + + char *executed_gtid_set = NULL; + + mysql_variable status[] = { + {"Executed_Gtid_Set", &executed_gtid_set}, + {NULL, NULL} + }; + + read_mysql_variables(connection, "SHOW MASTER STATUS", status, false); + + gtid_exists = (executed_gtid_set && *executed_gtid_set) + || (gtid_binlog_state && *gtid_binlog_state); + + free_mysql_variables(status); + + } + + if (write_binlogs || gtid_exists) { + + char *log_bin_file = NULL; + char filepath[FN_REFLEN]; + + mysql_variable status_after_flush[] = { + {"File", &log_bin_file}, + {NULL, NULL} + }; - if (gtid_exists) { size_t log_bin_dir_length; lock_binlog_maybe(connection); @@ -1405,14 +1415,31 @@ write_current_binlog_file(MYSQL *connection) goto cleanup; } - snprintf(filepath, sizeof(filepath), "%s%c%s", - log_bin_dir, FN_LIBCHAR, log_bin_file); - result = copy_file(ds_data, filepath, log_bin_file, 0); - } + MYSQL_RES *mysql_result; + MYSQL_ROW row; + + mysql_result = xb_mysql_query(connection, "SHOW BINARY LOGS", true); + + ut_ad(mysql_num_fields(mysql_result) >= 2); + + while ((row = mysql_fetch_row(mysql_result))) { + const char *binlog_name = row[0]; + snprintf(filepath, sizeof(filepath), "%s%c%s", + log_bin_dir, FN_LIBCHAR, binlog_name); + if (file_exists(filepath)) { + result = copy_file(ds_data, filepath, binlog_name, 0); + if (!result) break; + } + } + mysql_free_result(mysql_result); + + write_binlog_info(connection, log_bin_dir); cleanup: - free_mysql_variables(status_after_flush); - free_mysql_variables(status); + free_mysql_variables(status_after_flush); + + } + free_mysql_variables(vars); return(result); @@ -1423,7 +1450,7 @@ cleanup: Retrieves MySQL binlog position and saves it in a file. It also prints it to stdout. */ bool -write_binlog_info(MYSQL *connection) +write_binlog_info(MYSQL *connection, char *log_bin_dir) { char *filename = NULL; char *position = NULL; @@ -1431,9 +1458,16 @@ write_binlog_info(MYSQL *connection) char *gtid_current_pos = NULL; char *gtid_executed = NULL; char *gtid = NULL; + char *buffer; + char *buf; + int total; + int position_length; + int bytes; bool result; bool mysql_gtid; bool mariadb_gtid; + bool with_gtid; + MYSQL_ROW_OFFSET start; mysql_variable status[] = { {"File", &filename}, @@ -1461,24 +1495,96 @@ write_binlog_info(MYSQL *connection) mysql_gtid = ((gtid_mode != NULL) && (strcmp(gtid_mode, "ON") == 0)); mariadb_gtid = (gtid_current_pos != NULL); + with_gtid = mariadb_gtid || mysql_gtid; + gtid = (gtid_executed != NULL ? gtid_executed : gtid_current_pos); - if (mariadb_gtid || mysql_gtid) { + if (with_gtid) { ut_a(asprintf(&mysql_binlog_position, "filename '%s', position '%s', " "GTID of the last change '%s'", filename, position, gtid) != -1); - result = backup_file_printf(XTRABACKUP_BINLOG_INFO, - "%s\t%s\t%s\n", filename, position, - gtid); } else { ut_a(asprintf(&mysql_binlog_position, "filename '%s', position '%s'", filename, position) != -1); - result = backup_file_printf(XTRABACKUP_BINLOG_INFO, - "%s\t%s\n", filename, position); } + MYSQL_RES *mysql_result; + MYSQL_ROW row; + + mysql_result = xb_mysql_query(connection, "SHOW BINARY LOGS", true); + + ut_ad(mysql_num_fields(mysql_result) >= 2); + start = mysql_row_tell(mysql_result); + + position_length = strlen(position) + 2; + if (with_gtid) { + position_length += strlen(gtid) + 1; + } + + total = strlen(filename) + position_length + 1; + while ((row = mysql_fetch_row(mysql_result))) { + const char *binlog_name = row[0]; + if (strcmp(binlog_name, filename) == 0) { + continue; + } + total += strlen(binlog_name) + position_length; + } + + buffer = static_cast<char*>(malloc(total)); + if (!buffer) { + msg("Failed to allocate memory for temporary buffer"); + result = false; + goto cleanup2; + } + + mysql_row_seek(mysql_result, start); + + buf = buffer; + while ((row = mysql_fetch_row(mysql_result))) { + const char *binlog_name = row[0]; + if (strcmp(binlog_name, filename) == 0) { + continue; + } + char filepath[FN_REFLEN]; + snprintf(filepath, sizeof(filepath), "%s%c%s", + log_bin_dir, FN_LIBCHAR, binlog_name); + if (file_exists(filepath)) { + if (with_gtid) { + bytes = snprintf(buf, total, "%s\t%s\t%s\n", binlog_name, position, gtid); + } else { + bytes = snprintf(buf, total, "%s\t%s\n", binlog_name, position); + } + if (bytes <= 0) { + msg("Buffer overflow in write_binlog_info"); + result = false; + goto cleanup3; + } + buf += bytes; + total -= bytes; + } + } + + if (with_gtid) { + bytes = snprintf(buf, total, "%s\t%s\t%s\n", filename, position, gtid); + } else { + bytes = snprintf(buf, total, "%s\t%s\n", filename, position); + } + if (bytes <= 0) { + msg("Buffer overflow in write_binlog_info"); + result = false; + goto cleanup3; + } + + result = backup_file_printf(XTRABACKUP_BINLOG_INFO, "%s", buffer); + +cleanup3: + free(buffer); + +cleanup2: + mysql_free_result(mysql_result); + cleanup: free_mysql_variables(status); free_mysql_variables(vars); diff --git a/extra/mariabackup/backup_mysql.h b/extra/mariabackup/backup_mysql.h index b61fa2362c6..8d8dd998bae 100644 --- a/extra/mariabackup/backup_mysql.h +++ b/extra/mariabackup/backup_mysql.h @@ -62,10 +62,10 @@ void unlock_all(MYSQL *connection); bool -write_current_binlog_file(MYSQL *connection); +write_current_binlog_file(MYSQL *connection, bool write_binlogs); bool -write_binlog_info(MYSQL *connection); +write_binlog_info(MYSQL *connection, char *log_bin_dir); bool write_xtrabackup_info(MYSQL *connection, const char * filename, bool history, diff --git a/mysql-test/suite/galera/r/galera_log_bin_ext.result b/mysql-test/suite/galera/r/galera_log_bin_ext.result index f5276b7d1ac..c5ff66d17b3 100644 --- a/mysql-test/suite/galera/r/galera_log_bin_ext.result +++ b/mysql-test/suite/galera/r/galera_log_bin_ext.result @@ -1,4 +1,6 @@ connection node_1; +connection node_2; +connection node_1; reset master; connection node_2; reset master; @@ -40,6 +42,12 @@ hostname1-bin.000001 # Xid # # COMMIT /* XID */ hostname1-bin.000001 # Gtid # # GTID #-#-# hostname1-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN f2 INTEGER connection node_2; +Shutting down server ... +connection node_1; +Cleaning var directory ... +connection node_2; +Starting server ... +connection node_2; SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; COUNT(*) = 2 1 @@ -66,6 +74,7 @@ hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F hostname1-bin.000001 # Xid # # COMMIT /* XID */ hostname1-bin.000001 # Gtid # # GTID #-#-# hostname1-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN f2 INTEGER +hostname1-bin.000001 # Rotate # # hostname1-bin.000002;pos=4 DROP TABLE t1; DROP TABLE t2; #cleanup diff --git a/mysql-test/suite/galera/r/galera_log_bin_ext_mariabackup.result b/mysql-test/suite/galera/r/galera_log_bin_ext_mariabackup.result new file mode 100644 index 00000000000..c5ff66d17b3 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_log_bin_ext_mariabackup.result @@ -0,0 +1,82 @@ +connection node_1; +connection node_2; +connection node_1; +reset master; +connection node_2; +reset master; +CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1); +CREATE TABLE t2 (id INT) ENGINE=InnoDB; +INSERT INTO t2 VALUES (1); +INSERT INTO t2 VALUES (1); +connection node_2; +SELECT COUNT(*) = 1 FROM t1; +COUNT(*) = 1 +1 +SELECT COUNT(*) = 2 FROM t2; +COUNT(*) = 2 +1 +connection node_1; +ALTER TABLE t1 ADD COLUMN f2 INTEGER; +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t1 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t1) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; CREATE TABLE t2 (id INT) ENGINE=InnoDB +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t2 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t2) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t2 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t2) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN f2 INTEGER +connection node_2; +Shutting down server ... +connection node_1; +Cleaning var directory ... +connection node_2; +Starting server ... +connection node_2; +SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +COUNT(*) = 2 +1 +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t1 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t1) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; CREATE TABLE t2 (id INT) ENGINE=InnoDB +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t2 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t2) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t2 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t2) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN f2 INTEGER +hostname1-bin.000001 # Rotate # # hostname1-bin.000002;pos=4 +DROP TABLE t1; +DROP TABLE t2; +#cleanup +connection node_1; +RESET MASTER; diff --git a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2,debug.rdiff b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2,debug.rdiff index 8b091eb370a..71f27a8f7cd 100644 --- a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2,debug.rdiff +++ b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2,debug.rdiff @@ -1,5 +1,5 @@ ---- galera_sst_mariabackup.result -+++ galera_sst_mariabackup,debug.reject +--- galera_sst_xtrabackup-v2.result ++++ galera_sst_xtrabackup-v2,debug.reject @@ -286,5 +286,113 @@ DROP TABLE t1; COMMIT; diff --git a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir,debug.rdiff b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir,debug.rdiff index ac232020037..d287a3bfcaf 100644 --- a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir,debug.rdiff +++ b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir,debug.rdiff @@ -1,11 +1,12 @@ ---- r/galera_sst_xtrabackup-v2_data_dir.result 2018-11-19 12:27:24.795221479 +0200 -+++ r/galera_sst_xtrabackup-v2_data_dir.reject 2018-11-19 19:15:38.774008404 +0200 -@@ -260,3 +260,100 @@ +--- galera_sst_xtrabackup-v2_data_dir.result ++++ galera_sst_xtrabackup-v2_data_dir,debug.reject +@@ -286,5 +286,113 @@ DROP TABLE t1; COMMIT; SET AUTOCOMMIT=ON; +Performing State Transfer on a server that has been killed and restarted +while a DDL was in progress on it ++connection node_1; +CREATE TABLE t1 (f1 CHAR(255)) ENGINE=InnoDB; +SET AUTOCOMMIT=OFF; +START TRANSACTION; @@ -14,6 +15,7 @@ +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); +INSERT INTO t1 VALUES ('node1_committed_before'); ++connection node_2; +START TRANSACTION; +INSERT INTO t1 VALUES ('node2_committed_before'); +INSERT INTO t1 VALUES ('node2_committed_before'); @@ -22,9 +24,12 @@ +INSERT INTO t1 VALUES ('node2_committed_before'); +COMMIT; +SET GLOBAL debug_dbug = 'd,sync.alter_opened_table'; ++connection node_1; +ALTER TABLE t1 ADD COLUMN f2 INTEGER; ++connection node_2; +SET wsrep_sync_wait = 0; +Killing server ... ++connection node_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 (f1) VALUES ('node1_committed_during'); @@ -39,6 +44,7 @@ +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); ++connect node_1a_galera_st_kill_slave_ddl, 127.0.0.1, root, , test, $NODE_MYPORT_1; +SET AUTOCOMMIT=OFF; +START TRANSACTION; +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); @@ -46,7 +52,9 @@ +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); ++connection node_2; +Performing --wsrep-recover ... ++connection node_2; +Starting server ... +Using --wsrep-start-position when starting mysqld ... +SET AUTOCOMMIT=OFF; @@ -57,6 +65,7 @@ +INSERT INTO t1 (f1) VALUES ('node2_committed_after'); +INSERT INTO t1 (f1) VALUES ('node2_committed_after'); +COMMIT; ++connection node_1; +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_committed_after'); @@ -71,6 +80,7 @@ +INSERT INTO t1 (f1) VALUES ('node1_committed_after'); +INSERT INTO t1 (f1) VALUES ('node1_committed_after'); +COMMIT; ++connection node_1a_galera_st_kill_slave_ddl; +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); +INSERT INTO t1 (f1) VALUES ('node1_to_be_rollbacked_after'); @@ -88,6 +98,7 @@ +1 +COMMIT; +SET AUTOCOMMIT=ON; ++connection node_1; +SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +COUNT(*) = 2 +1 @@ -101,3 +112,5 @@ +COMMIT; +SET AUTOCOMMIT=ON; +SET GLOBAL debug_dbug = $debug_orig; + disconnect node_2; + disconnect node_1; diff --git a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir.result b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir.result index ff85a7d6c0f..fdb5883b590 100644 --- a/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir.result +++ b/mysql-test/suite/galera/r/galera_sst_xtrabackup-v2_data_dir.result @@ -286,3 +286,5 @@ COUNT(*) = 0 DROP TABLE t1; COMMIT; SET AUTOCOMMIT=ON; +disconnect node_2; +disconnect node_1; diff --git a/mysql-test/suite/galera/t/galera_log_bin_ext.cnf b/mysql-test/suite/galera/t/galera_log_bin_ext.cnf index 012209610ea..8837138f59a 100644 --- a/mysql-test/suite/galera/t/galera_log_bin_ext.cnf +++ b/mysql-test/suite/galera/t/galera_log_bin_ext.cnf @@ -9,3 +9,6 @@ log-slave-updates log-bin = hostname2-bin log-bin-index = hostname2.bdx log-slave-updates + +[sst] +sst_rsync_max_binlogs= diff --git a/mysql-test/suite/galera/t/galera_log_bin_ext.test b/mysql-test/suite/galera/t/galera_log_bin_ext.test index 923bd623a8a..752073aecdb 100644 --- a/mysql-test/suite/galera/t/galera_log_bin_ext.test +++ b/mysql-test/suite/galera/t/galera_log_bin_ext.test @@ -1 +1 @@ ---source galera_log_bin.inc +--source galera_log_bin_sst.inc diff --git a/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.cnf b/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.cnf new file mode 100644 index 00000000000..c22fb723d73 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.cnf @@ -0,0 +1,18 @@ +!include ../galera_2nodes.cnf + +[mysqld] +wsrep_sst_method=mariabackup +wsrep_sst_auth="root:" + +[mysqld.1] +log-bin=@ENV.MYSQLTEST_VARDIR/mysqld.1/data/hostname1-bin +log-bin-index = hostname1.bdx +log-slave-updates + +[mysqld.2] +log-bin=@ENV.MYSQLTEST_VARDIR/mysqld.2/data/hostname2-bin +log-bin-index = hostname2.bdx +log-slave-updates + +[sst] +transferfmt=@ENV.MTR_GALERA_TFMT diff --git a/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.test b/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.test new file mode 100644 index 00000000000..47df45b4c71 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.test @@ -0,0 +1,2 @@ +--source include/have_mariabackup.inc +--source galera_log_bin_sst.inc diff --git a/mysql-test/suite/galera/t/galera_log_bin_sst.inc b/mysql-test/suite/galera/t/galera_log_bin_sst.inc new file mode 100644 index 00000000000..5d543f6f8b6 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_log_bin_sst.inc @@ -0,0 +1,84 @@ +--source include/galera_cluster.inc +--source include/force_restart.inc + +# Save original auto_increment_offset values. +--let $node_1=node_1 +--let $node_2=node_2 +--source include/auto_increment_offset_save.inc + +--connection node_1 +reset master; +--connection node_2 +reset master; + +# +# Test Galera with --log-bin --log-slave-updates . +# This way the actual MySQL binary log is used, +# rather than Galera's own implementation +# + +CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1); + +CREATE TABLE t2 (id INT) ENGINE=InnoDB; +INSERT INTO t2 VALUES (1); +INSERT INTO t2 VALUES (1); + +--connection node_2 +SELECT COUNT(*) = 1 FROM t1; +SELECT COUNT(*) = 2 FROM t2; + +--connection node_1 +ALTER TABLE t1 ADD COLUMN f2 INTEGER; +--let $MASTER_MYPORT=$NODE_MYPORT_1 +--source include/show_binlog_events.inc + +--connection node_2 + +#--connection node_2 +#--source suite/galera/include/galera_stop_replication.inc + +--echo Shutting down server ... +--source include/shutdown_mysqld.inc + +--connection node_1 +--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size' +--source include/wait_condition.inc + +# +# Force SST +# +--echo Cleaning var directory ... +--remove_file $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/mtr +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/performance_schema +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/test +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/mysql +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data + +--connection node_2 + +--echo Starting server ... +let $restart_noprint=2; +--source include/start_mysqld.inc + +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +--let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready'; +--source include/wait_condition.inc + +--connection node_2 +SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +--let $MASTER_MYPORT=$NODE_MYPORT_2 +--source include/show_binlog_events.inc + +DROP TABLE t1; +DROP TABLE t2; + +--echo #cleanup +--connection node_1 +RESET MASTER; + +# Restore original auto_increment_offset values. +--source include/auto_increment_offset_restore.inc diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index deebe7cf820..daa5d9d9055 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -22,6 +22,45 @@ set -ue # Setting the path for some utilities on CentOS export PATH="$PATH:/usr/sbin:/usr/bin:/sbin:/bin" +trim_string() +{ + if [ -n "$BASH_VERSION" ]; then + local pattern="[![:space:]${2:-}]" + local x="${1#*$pattern}" + local z=${#1} + x=${#x} + if [ $x -ne $z ]; then + local y="${1%$pattern*}" + y=${#y} + x=$(( z-x-1 )) + y=$(( y-x+1 )) + printf '%s' "${1:$x:$y}" + else + printf '' + fi + else + local pattern="[[:space:]${2:-}]" + echo "$1" | sed -E "s/^$pattern+|$pattern+\$//g" + fi +} + +trim_dir() +{ + local t=$(trim_string "$1") + if [ "$t" != '/' ]; then + if [ "${t%/}" != "$t" ]; then + t=$(trim_string "${t%/}") + fi + else + t='.' + fi + if [ -n "$BASH_VERSION" ]; then + printf '%s' "$t" + else + echo "$t" + fi +} + WSREP_SST_OPT_BYPASS=0 WSREP_SST_OPT_BINLOG="" WSREP_SST_OPT_BINLOG_INDEX="" @@ -44,9 +83,9 @@ WSREP_SST_OPT_ADDR_PORT="" WSREP_SST_OPT_HOST="" WSREP_SST_OPT_HOST_UNESCAPED="" WSREP_SST_OPT_HOST_ESCAPED="" -INNODB_DATA_HOME_DIR="${INNODB_DATA_HOME_DIR:-}" -INNODB_LOG_GROUP_HOME="${INNODB_LOG_GROUP_HOME:-}" -INNODB_UNDO_DIR="${INNODB_UNDO_DIR:-}" +INNODB_DATA_HOME_DIR=$(trim_dir "${INNODB_DATA_HOME_DIR:-}") +INNODB_LOG_GROUP_HOME=$(trim_dir "${INNODB_LOG_GROUP_HOME:-}") +INNODB_UNDO_DIR=$(trim_dir "${INNODB_UNDO_DIR:-}") INNODB_FORCE_RECOVERY="" INNOEXTRA="" @@ -138,22 +177,22 @@ case "$1" in ;; '--datadir') # Let's remove the trailing slash: - readonly WSREP_SST_OPT_DATA="${2%/}" + readonly WSREP_SST_OPT_DATA=$(trim_dir "$2") shift ;; '--innodb-data-home-dir') # Let's remove the trailing slash: - readonly INNODB_DATA_HOME_DIR="${2%/}" + readonly INNODB_DATA_HOME_DIR=$(trim_dir "$2") shift ;; '--innodb-log-group-home-dir') # Let's remove the trailing slash: - readonly INNODB_LOG_GROUP_HOME="${2%/}" + readonly INNODB_LOG_GROUP_HOME=$(trim_dir "$2") shift ;; '--innodb-undo-directory') # Let's remove the trailing slash: - readonly INNODB_UNDO_DIR="${2%/}" + readonly INNODB_UNDO_DIR=$(trim_dir "$2") shift ;; '--defaults-file') @@ -247,6 +286,7 @@ case "$1" in '--mysqld-args') original_cmd="" shift + cmd_tail=0 while [ $# -gt 0 ]; do lname="${1#--}" # "--" is interpreted as the end of the list of options: @@ -261,7 +301,7 @@ case "$1" in shift done fi - break; + break fi # Make sure the argument does not start with "--", otherwise it # is a long option, which is processed after this "if": @@ -301,15 +341,25 @@ case "$1" in if [ "${2#-}" = "$2" ]; then shift value="$1" + elif [ "$2" = '--' ]; then + shift + if [ $# -gt 1 ]; then + cmd_tail=1 + shift + value="$1" + fi fi fi if [ $option = 'h' ]; then if [ -z "$WSREP_SST_OPT_DATA" ]; then - MYSQLD_OPT_DATADIR="${value%/}" + MYSQLD_OPT_DATADIR=$(trim_dir "$value") fi elif [ $option != 'u' -a \ $option != 'P' ] then + if [ $cmd_tail -ne 0 ]; then + option="$option --" + fi if [ -z "$value" ]; then slist="$slist$option" elif [ -z "$slist" ]; then @@ -317,9 +367,16 @@ case "$1" in else slist="$slist -$option '$value'" fi + break + fi + if [ $cmd_tail -ne 0 ]; then + if [ -n "$slist" ]; then + slist="$slist --" + else + slist='-' + fi fi break - else slist="$slist$option" fi @@ -329,7 +386,7 @@ case "$1" in original_cmd="$original_cmd -$slist" fi elif [ -z "$options" ]; then - # We found an equal sign without any characters after it: + # We found an minus sign without any characters after it: original_cmd="$original_cmd -" else # We found a value that does not start with a minus - @@ -338,7 +395,15 @@ case "$1" in original_cmd="$original_cmd '$1'" fi shift - continue; + if [ $cmd_tail -ne 0 ]; then + # All other arguments must be copied unchanged: + while [ $# -gt 0 ]; do + original_cmd="$original_cmd '$1'" + shift + done + break + fi + continue fi # Now we are sure that we are working with an option # that has a "long" name, so remove all characters after @@ -370,19 +435,19 @@ case "$1" in case "$option" in '--innodb-data-home-dir') if [ -z "$INNODB_DATA_HOME_DIR" ]; then - MYSQLD_OPT_INNODB_DATA_HOME_DIR="${value%/}" + MYSQLD_OPT_INNODB_DATA_HOME_DIR=$(trim_dir "$value") fi skip_mysqld_arg=1 ;; '--innodb-log-group-home-dir') if [ -z "$INNODB_LOG_GROUP_HOME" ]; then - MYSQLD_OPT_INNODB_LOG_GROUP_HOME="${value%/}" + MYSQLD_OPT_INNODB_LOG_GROUP_HOME=$(trim_dir "$value") fi skip_mysqld_arg=1 ;; '--innodb-undo-directory') if [ -z "$INNODB_UNDO_DIR" ]; then - MYSQLD_OPT_INNODB_UNDO_DIR="${value%/}" + MYSQLD_OPT_INNODB_UNDO_DIR=$(trim_dir "$value") fi skip_mysqld_arg=1 ;; @@ -412,7 +477,7 @@ case "$1" in ;; '--datadir') if [ -z "$WSREP_SST_OPT_DATA" ]; then - MYSQLD_OPT_DATADIR="${value%/}" + MYSQLD_OPT_DATADIR=$(trim_dir "$value") fi skip_mysqld_arg=1 ;; @@ -530,12 +595,10 @@ get_binlog() WSREP_SST_OPT_LOG_BASENAME=$(parse_cnf '--mysqld' 'log-basename') fi if [ -z "$WSREP_SST_OPT_BINLOG" ]; then - # If the --log-bin option is specified without a parameter, + # If the log-bin option is specified without a parameter, # then we need to build the name of the index file according # to the rules described in the server documentation: - if [ -n "${MYSQLD_OPT_LOG_BIN+x}" -o \ - $(in_config '--mysqld' 'log-bin') -eq 1 ] - then + if [ $(in_config '--mysqld' 'log-bin') -eq 1 ]; then if [ -n "$WSREP_SST_OPT_LOG_BASENAME" ]; then # If the WSREP_SST_OPT_BINLOG variable is not set, but # --log-basename is present among the arguments of mysqld, @@ -543,8 +606,7 @@ get_binlog() # the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG="$WSREP_SST_OPT_LOG_BASENAME-bin" else - # If the --log-bin option is present without a value, then - # we take the default name: + # Take the default name: readonly WSREP_SST_OPT_BINLOG='mysql-bin' fi fi @@ -554,10 +616,10 @@ get_binlog() # it according to the specifications for the server: if [ -z "$WSREP_SST_OPT_BINLOG_INDEX" ]; then if [ -n "$WSREP_SST_OPT_LOG_BASENAME" ]; then - # If the WSREP_SST_OPT_BINLOG variable is not set, but + # If the WSREP_SST_OPT_BINLOG_INDEX variable is not set, but # --log-basename is present among the arguments of mysqld, - # then set WSREP_SST_OPT_BINLOG equal to the base name with - # the "-bin" suffix: + # then set WSREP_SST_OPT_BINLOG_INDEX equal to the base name + # with the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_LOG_BASENAME-bin.index" else # the default name (note that base of this name @@ -754,7 +816,11 @@ parse_cnf() if [ -z "$reval" ]; then [ -n "${3:-}" ] && reval="$3" fi - echo "$reval" + if [ -n "$BASH_VERSION" ]; then + printf '%s' "$reval" + else + echo "$reval" + fi } # @@ -804,7 +870,11 @@ in_config() break fi done - echo $found + if [ -n "$BASH_VERSION" ]; then + printf '%s' $found + else + echo $found + fi } wsrep_auth_not_set() @@ -937,11 +1007,22 @@ wsrep_gen_secret() { get_openssl if [ -n "$OPENSSL_BINARY" ]; then - echo $("$OPENSSL_BINARY" rand -hex 16) - else - printf "%04x%04x%04x%04x%04x%04x%04x%04x" \ + "$OPENSSL_BINARY" rand -hex 16 + elif [ -n "$BASH_VERSION" ]; then + printf '%04x%04x%04x%04x%04x%04x%04x%04x' \ $RANDOM $RANDOM $RANDOM $RANDOM \ $RANDOM $RANDOM $RANDOM $RANDOM + elif [ -n $(commandex 'cksum') -a \ + -n $(commandex 'printf') ] + then + printf '%08x%08x%08x%08x' \ + $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) \ + $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) \ + $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) \ + $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) + else + wsrep_log_error "Unable to generate 16-byte secret" + exit 22 fi } @@ -979,14 +1060,14 @@ is_local_ip() if [ -n "$ip_util" ]; then # ip address show ouput format is " inet[6] <address>/<mask>": "$ip_util" address show \ - | grep -E "^[[:space:]]*inet.? [^[:space:]]+/" -o \ + | grep -E '^[[:space:]]*inet.? [^[:space:]]+/' -o \ | grep -F " $1/" >/dev/null && return 0 else local ifconfig_util=$(commandex 'ifconfig') if [ -n "$ifconfig_util" ]; then # ifconfig output format is " inet[6] <address> ...": "$ifconfig_util" \ - | grep -E "^[[:space:]]*inet.? [^[:space:]]+ " -o \ + | grep -E '^[[:space:]]*inet.? [^[:space:]]+ ' -o \ | grep -F " $1 " >/dev/null && return 0 fi fi @@ -1220,28 +1301,6 @@ check_for_version() return 0 } -trim_string() -{ - if [ -n "$BASH_VERSION" ]; then - local pattern="[![:space:]${2:-}]" - local x="${1#*$pattern}" - local z=${#1} - x=${#x} - if [ $x -ne $z ]; then - local y="${1%$pattern*}" - y=${#y} - x=$(( z-x-1 )) - y=$(( y-x+1 )) - printf '%s' "${1:$x:$y}" - else - printf '' - fi - else - local pattern="[[:space:]${2:-}]" - echo "$1" | sed -E "s/^$pattern+|$pattern+\$//g" - fi -} - # # Check whether process is still running. # The first parameter contains the name of the PID file. diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index aa9442b0601..af4a23bc6d9 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -37,7 +37,7 @@ tcert="" tcap="" tpem="" tkey="" -tmode="DISABLED" +tmode='DISABLED' sockopt="" progress="" ttime=0 @@ -82,13 +82,13 @@ backup_threads="" encrypt_threads="" encrypt_chunk="" -readonly SECRET_TAG="secret" +readonly SECRET_TAG='secret' # Required for backup locks # For backup locks it is 1 sent by joiner sst_ver=1 -if [ -n "$(commandex pv)" ] && pv --help | grep -qw -- '-F'; then +if [ -n "$(commandex pv)" ] && pv --help | grep -qw -F -- '-F'; then pvopts="$pvopts $pvformat" fi pcmd="pv $pvopts" @@ -101,17 +101,14 @@ if [ -z "$BACKUP_BIN" ]; then fi DATA="$WSREP_SST_OPT_DATA" -INFO_FILE="xtrabackup_galera_info" -IST_FILE="xtrabackup_ist" +INFO_FILE='xtrabackup_galera_info' +IST_FILE='xtrabackup_ist' MAGIC_FILE="$DATA/$INFO_FILE" INNOAPPLYLOG="$DATA/mariabackup.prepare.log" INNOMOVELOG="$DATA/mariabackup.move.log" INNOBACKUPLOG="$DATA/mariabackup.backup.log" -# Setting the path for ss and ip -export PATH="/usr/sbin:/sbin:$PATH" - timeit() { local stage="$1" @@ -181,11 +178,11 @@ get_keys() exit 2 fi ecmd="'$OPENSSL_BINARY' enc -$ealgo" - if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-pbkdf2'; then + if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-pbkdf2'; then ecmd="$ecmd -pbkdf2" - elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-iter'; then + elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-iter'; then ecmd="$ecmd -iter 1" - elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-md'; then + elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-md'; then ecmd="$ecmd -md sha256" fi if [ -z "$ekey" ]; then @@ -229,12 +226,12 @@ get_transfer() if [ $tfmt = 'nc' ]; then wsrep_log_info "Using netcat as streamer" wsrep_check_programs nc - tcmd="nc" + tcmd='nc' if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then - if nc -h 2>&1 | grep -q 'ncat'; then + if nc -h 2>&1 | grep -q -F 'ncat'; then wsrep_log_info "Using Ncat as streamer" tcmd="$tcmd -l" - elif nc -h 2>&1 | grep -qw -- '-d'; then + elif nc -h 2>&1 | grep -qw -F -- '-d'; then wsrep_log_info "Using Debian netcat as streamer" tcmd="$tcmd -dl" if [ $WSREP_SST_OPT_HOST_IPv6 -eq 1 ]; then @@ -256,14 +253,14 @@ get_transfer() # transfer and cause the command to timeout. # Older versions of netcat did not need this flag and will # return an error if the flag is used. - if nc -h 2>&1 | grep -qw -- '-N'; then + if nc -h 2>&1 | grep -qw -F -- '-N'; then tcmd="$tcmd -N" wsrep_log_info "Using nc -N" fi # netcat doesn't understand [] around IPv6 address - if nc -h 2>&1 | grep -q ncat; then + if nc -h 2>&1 | grep -q -F 'ncat'; then wsrep_log_info "Using Ncat as streamer" - elif nc -h 2>&1 | grep -qw -- '-d'; then + elif nc -h 2>&1 | grep -qw -F -- '-d'; then wsrep_log_info "Using Debian netcat as streamer" else wsrep_log_info "Using traditional netcat as streamer" @@ -453,7 +450,7 @@ adjust_progress() fi elif [ -z "$progress" -a -n "$rlimit" ]; then # When rlimit is non-zero - pcmd="pv -q" + pcmd='pv -q' fi if [ -n "$rlimit" -a "$WSREP_SST_OPT_ROLE" = 'donor' ]; then @@ -580,8 +577,14 @@ get_stream() sig_joiner_cleanup() { + local estatus=$? + if [ $estatus -ne 0 ]; then + wsrep_log_error "Cleanup after exit with status: $estatus" + fi wsrep_log_error "Removing $MAGIC_FILE file due to signal" + [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" + exit $estatus } cleanup_at_exit() @@ -592,6 +595,8 @@ cleanup_at_exit() wsrep_log_error "Cleanup after exit with status: $estatus" fi + [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" + if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then wsrep_log_info "Removing the sst_in_progress file" wsrep_cleanup_progress_file @@ -621,7 +626,7 @@ cleanup_at_exit() fi # Final cleanup - pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o '[0-9]*' || :) + pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o -E '[0-9]*' || :) # This means no setsid done in mysqld. # We don't want to kill mysqld here otherwise. @@ -712,7 +717,7 @@ recv_joiner() local ltcmd="$tcmd" if [ $tmt -gt 0 ]; then if [ -n "$(commandex timeout)" ]; then - if timeout --help | grep -qw -- '-k'; then + if timeout --help | grep -qw -F -- '-k'; then ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd" else ltcmd="timeout -s9 $tmt $tcmd" @@ -812,7 +817,9 @@ monitor_process() [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" -if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a "$WSREP_SST_OPT_ROLE" != 'donor' ]; then +if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a \ + "$WSREP_SST_OPT_ROLE" != 'donor' ] +then wsrep_log_error "Invalid role '$WSREP_SST_OPT_ROLE'" exit 22 fi @@ -820,25 +827,17 @@ fi read_cnf setup_ports -if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then +if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -F -- '--version-check'; then disver=' --no-version-check' fi -# if no command line argument and INNODB_DATA_HOME_DIR environment variable -# is not set, try to get it from my.cnf: -if [ -z "$INNODB_DATA_HOME_DIR" ]; then - INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') -fi - OLD_PWD="$(pwd)" -cd "$WSREP_SST_OPT_DATA" -if [ -n "$INNODB_DATA_HOME_DIR" ]; then - # handle both relative and absolute paths - [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" - cd "$INNODB_DATA_HOME_DIR" +if [ -n "$DATA" -a "$DATA" != '.' ]; then + [ ! -d "$DATA" ] && mkdir -p "$DATA" + cd "$DATA" fi -INNODB_DATA_HOME_DIR=$(pwd -P) +DATA_DIR="$(pwd -P)" cd "$OLD_PWD" @@ -866,7 +865,7 @@ if [ $ssyslog -eq 1 ]; then else if [ $sstlogarchive -eq 1 ] then - ARCHIVETIMESTAMP=$(date "+%Y.%m.%d-%H.%M.%S.%N") + ARCHIVETIMESTAMP=$(date '+%Y.%m.%d-%H.%M.%S.%N') if [ -n "$sstlogarchivedir" ]; then if [ ! -d "$sstlogarchivedir" ]; then @@ -1100,22 +1099,53 @@ then wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" [ -n "$SST_PROGRESS_FILE" ] && touch "$SST_PROGRESS_FILE" - ib_home_dir="$INNODB_DATA_HOME_DIR" + # if no command line argument and INNODB_DATA_HOME_DIR environment + # variable is not set, try to get it from the my.cnf: + if [ -z "$INNODB_DATA_HOME_DIR" ]; then + INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') + INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") + fi + + if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" + cd "$INNODB_DATA_HOME_DIR" + ib_home_dir="$(pwd -P)" + cd "$OLD_PWD" + fi # if no command line argument and INNODB_LOG_GROUP_HOME is not set, - # try to get it from my.cnf: + # then try to get it from the my.cnf: if [ -z "$INNODB_LOG_GROUP_HOME" ]; then INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') + INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") fi - ib_log_dir="$INNODB_LOG_GROUP_HOME" + if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" + cd "$INNODB_LOG_GROUP_HOME" + ib_log_dir="$(pwd -P)" + cd "$OLD_PWD" + fi - # if no command line argument then try to get it from my.cnf: + # if no command line argument and INNODB_UNDO_DIR is not set, + # then try to get it from the my.cnf: if [ -z "$INNODB_UNDO_DIR" ]; then INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') + INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") fi - ib_undo_dir="$INNODB_UNDO_DIR" + if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" + cd "$INNODB_UNDO_DIR" + ib_undo_dir="$(pwd -P)" + cd "$OLD_PWD" + fi if [ -n "$backup_threads" ]; then impts="--parallel=$backup_threads${impts:+ }$impts" @@ -1139,7 +1169,7 @@ then stagemsg='Joiner-Recv' - MODULE="xtrabackup_sst" + MODULE='xtrabackup_sst' [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" @@ -1162,7 +1192,7 @@ then exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$tpem" | \ - tr "," "\n" | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ + tr ',' '\n' | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ sed s/\ %//) fi MY_SECRET="$(wsrep_gen_secret)" @@ -1236,14 +1266,29 @@ then if [ -n "$WSREP_SST_OPT_BINLOG" ]; then binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") - if [ -d "$binlog_dir" ]; then - cd "$binlog_dir" - wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" - rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || : - [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ] && \ - rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+ - cd "$OLD_PWD" + binlog_base=$(basename "$WSREP_SST_OPT_BINLOG") + binlog_index="$WSREP_SST_OPT_BINLOG_INDEX" + binlog_explicit=0 + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' ]; then + binlog_explicit=1 + if [ -d "$binlog_dir" ]; then + cd "$binlog_dir" + wsrep_log_info \ + "Cleaning the binlog directory $binlog_dir as well" + if [ -f "$binlog_index" ]; then + binlogs=$(cat "$binlog_index") + rm -fv "$binlog_index" 1>&2 + cd "$DATA_DIR" + echo "$binlogs" | \ + while read bin_file || [ -n "$bin_file" ]; do + rm -fv "$bin_file" 1>&2 || : + done + else + rm -fv "$binlog_base".[0-9]* 1>&2 || : + fi + fi fi + cd "$OLD_PWD" fi TDATA="$DATA" @@ -1279,11 +1324,13 @@ then dcmd="xargs -n 2 qpress -dT$nproc" - if [ -n "$progress" ] && pv --help | grep -qw -- '--line-mode'; then + if [ -n "$progress" ] && \ + pv --help | grep -qw -F -- '--line-mode' + then count=$(find "$DATA" -type f -name '*.qp' | wc -l) count=$(( count*2 )) pvopts="-f -s $count -l -N Decompression" - if pv --help | grep -qw -- '-F'; then + if pv --help | grep -qw -F -- '-F'; then pvopts="$pvopts -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'" fi pcmd="pv $pvopts" @@ -1293,8 +1340,9 @@ then # Decompress the qpress files wsrep_log_info "Decompression with $nproc threads" - timeit "Joiner-Decompression" \ - "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" + timeit 'Joiner-Decompression' \ + "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | \ + $dcmd" extcode=$? if [ $extcode -eq 0 ]; then @@ -1312,24 +1360,36 @@ then fi if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - - BINLOG_DIRNAME=$(dirname "$WSREP_SST_OPT_BINLOG") - BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG") - - # To avoid comparing data directory and BINLOG_DIRNAME - mv "$DATA/$BINLOG_FILENAME".* "$BINLOG_DIRNAME/" 2>/dev/null || : - - cd "$BINLOG_DIRNAME" - for bfile in $(ls -1 "$BINLOG_FILENAME".[0-9]*); do - echo "$BINLOG_DIRNAME/$bfile" >> "$WSREP_SST_OPT_BINLOG_INDEX" - done + cd "$DATA" + binlogs="" + if [ -f 'xtrabackup_binlog_info' ]; then + NL=$'\n' + while read bin_string || [ -n "$bin_string" ]; do + bin_file=$(echo "$bin_string" | cut -f1) + if [ -f "$bin_file" ]; then + binlogs="$binlogs${binlogs:+$NL}$bin_file" + fi + done < 'xtrabackup_binlog_info' + else + binlogs=$(ls -d -1 "$binlog_base".[0-9]* 2>/dev/null || :) + fi + if [ -n "$binlogs" ]; then + cd "$DATA_DIR" + if [ $binlog_explicit -ne 0 ]; then + [ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir" + cd "$binlog_dir" + fi + echo "$binlogs" | \ + while read bin_file || [ -n "$bin_file" ]; do + echo "$binlog_dir${binlog_dir:+/}$bin_file" >> "$binlog_index" + done + fi cd "$OLD_PWD" - fi wsrep_log_info "Preparing the backup at $DATA" setup_commands - timeit "mariabackup prepare stage" "$INNOAPPLY" + timeit 'mariabackup prepare stage' "$INNOAPPLY" if [ $? -ne 0 ]; then wsrep_log_error "mariabackup apply finished with errors." \ @@ -1340,7 +1400,7 @@ then MAGIC_FILE="$TDATA/$INFO_FILE" wsrep_log_info "Moving the backup to $TDATA" - timeit "mariabackup move stage" "$INNOMOVE" + timeit 'mariabackup move stage' "$INNOMOVE" if [ $? -eq 0 ]; then wsrep_log_info "Move successful, removing $DATA" rm -rf "$DATA" diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh index bed2cac0a9a..134695bbbbd 100644 --- a/scripts/wsrep_sst_mysqldump.sh +++ b/scripts/wsrep_sst_mysqldump.sh @@ -37,7 +37,7 @@ then fi # Check client version -if ! $MYSQL_CLIENT --version | grep 'Distrib 10\.[1-9]' >/dev/null +if ! $MYSQL_CLIENT --version | grep -q -E 'Distrib 10\.[1-9]' then $MYSQL_CLIENT --version >&2 wsrep_log_error "this operation requires MySQL client version 10.1 or newer" diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 28dfed18218..c82290b21a2 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -34,6 +34,8 @@ cleanup_joiner() { local failure=0 + [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" + wsrep_log_info "Joiner cleanup: rsync PID=$RSYNC_REAL_PID," \ "stunnel PID=$STUNNEL_REAL_PID" @@ -56,6 +58,7 @@ cleanup_joiner() if [ $failure -eq 0 ]; then if cleanup_pid $RSYNC_REAL_PID "$RSYNC_PID" "$RSYNC_CONF"; then [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" + [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" else wsrep_log_warning "rsync cleanup failed." fi @@ -138,66 +141,77 @@ STUNNEL_PID="$WSREP_SST_OPT_DATA/stunnel.pid" MAGIC_FILE="$WSREP_SST_OPT_DATA/rsync_sst_complete" -BINLOG_TAR_FILE="$WSREP_SST_OPT_DATA/wsrep_sst_binlog.tar" -BINLOG_N_FILES=1 - get_binlog if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - BINLOG_DIRNAME=$(dirname "$WSREP_SST_OPT_BINLOG") - BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG") + binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") + binlog_base=$(basename "$WSREP_SST_OPT_BINLOG") fi -# if no command line argument and INNODB_LOG_GROUP_HOME is not set, -# try to get it from my.cnf: -if [ -z "$INNODB_LOG_GROUP_HOME" ]; then - INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') +OLD_PWD="$(pwd)" + +DATA="$WSREP_SST_OPT_DATA" +if [ -n "$DATA" -a "$DATA" != '.' ]; then + [ ! -d "$DATA" ] && mkdir -p "$DATA" + cd "$DATA" fi +DATA_DIR="$(pwd -P)" -OLD_PWD="$(pwd)" +cd "$OLD_PWD" + +BINLOG_TAR_FILE="$DATA_DIR/wsrep_sst_binlog.tar" -WSREP_LOG_DIR="$INNODB_LOG_GROUP_HOME" +ib_log_dir="$DATA_DIR" +ib_home_dir="$DATA_DIR" +ib_undo_dir="$DATA_DIR" -cd "$WSREP_SST_OPT_DATA" -if [ -n "$WSREP_LOG_DIR" ]; then - # handle both relative and absolute paths - [ ! -d "$WSREP_LOG_DIR" ] && mkdir -p "$WSREP_LOG_DIR" - cd "$WSREP_LOG_DIR" +# if no command line argument and INNODB_LOG_GROUP_HOME is not set, +# then try to get it from the my.cnf: +if [ -z "$INNODB_LOG_GROUP_HOME" ]; then + INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') + INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") fi -WSREP_LOG_DIR=$(pwd -P) -cd "$OLD_PWD" +if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" + cd "$INNODB_LOG_GROUP_HOME" + ib_log_dir="$(pwd -P)" + cd "$OLD_PWD" +fi -# if no command line argument and INNODB_DATA_HOME_DIR environment variable -# is not set, try to get it from my.cnf: +# if no command line argument and INNODB_DATA_HOME_DIR environment +# variable is not set, try to get it from the my.cnf: if [ -z "$INNODB_DATA_HOME_DIR" ]; then INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') + INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") fi -cd "$WSREP_SST_OPT_DATA" -if [ -n "$INNODB_DATA_HOME_DIR" ]; then - # handle both relative and absolute paths +if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" cd "$INNODB_DATA_HOME_DIR" + ib_home_dir="$(pwd -P)" + cd "$OLD_PWD" fi -INNODB_DATA_HOME_DIR=$(pwd -P) -cd "$OLD_PWD" - -# if no command line argument then try to get it from my.cnf: +# if no command line argument and INNODB_UNDO_DIR is not set, +# then try to get it from the my.cnf: if [ -z "$INNODB_UNDO_DIR" ]; then INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') + INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") fi -cd "$WSREP_SST_OPT_DATA" -if [ -n "$INNODB_UNDO_DIR" ]; then - # handle both relative and absolute paths +if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" cd "$INNODB_UNDO_DIR" + ib_undo_dir="$(pwd -P)" + cd "$OLD_PWD" fi -INNODB_UNDO_DIR=$(pwd -P) - -cd "$OLD_PWD" encgroups='--mysqld|sst' @@ -274,7 +288,7 @@ if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ]; then CHECK_OPT="checkHost = $WSREP_SST_OPT_HOST" fi if is_local_ip "$WSREP_SST_OPT_HOST_UNESCAPED"; then - CHECK_OPT_LOCAL="checkHost = localhost" + CHECK_OPT_LOCAL='checkHost = localhost' fi fi fi @@ -291,7 +305,7 @@ if [ -n "$SSLMODE" -a "$SSLMODE" != 'DISABLED' ]; then fi fi -readonly SECRET_TAG="secret" +readonly SECRET_TAG='secret' if [ "$WSREP_SST_OPT_ROLE" = 'donor' ] then @@ -331,7 +345,7 @@ EOF [ -f "$FLUSHED" ] && rm -f "$FLUSHED" [ -f "$ERROR" ] && rm -f "$ERROR" - echo "flush tables" + echo 'flush tables' # Wait for : # (a) Tables to be flushed, AND @@ -355,32 +369,80 @@ EOF sync - if [ -n "$WSREP_SST_OPT_BINLOG" -a -d "${BINLOG_DIRNAME:-}" ] - then - # Prepare binlog files - cd "$BINLOG_DIRNAME" - - binlog_files_full=$(tail -n $BINLOG_N_FILES \ - "$WSREP_SST_OPT_BINLOG_INDEX") - binlog_files="" - for file in $binlog_files_full; do - binlog_file=$(basename "$file") - binlog_files="$binlog_files${binlog_files:+ }'$binlog_file'" - done - - if [ -n "$binlog_files" ]; then - wsrep_log_info "Preparing binlog files for transfer:" - eval tar -cvf "'$BINLOG_TAR_FILE'" $binlog_files >&2 + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + # Change the directory to binlog base (if possible): + cd "$DATA" + if [ -n "$binlog_dir" -a \ + "$binlog_dir" != '.' -a \ + -d "$binlog_dir" ] + then + cd "$binlog_dir" + fi + # Let's check the existence of the file with the index: + if [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ]; then + # Let's read the binlog index: + BINLOG_N_FILES=$(parse_cnf "$encgroups" \ + 'sst-rsync-max-binlogs') + if [ -n "$BINLOG_N_FILES" ]; then + binlog_files="" + if [ $BINLOG_N_FILES -gt 0 ]; then + binlog_files=$(tail -n $BINLOG_N_FILES \ + "$WSREP_SST_OPT_BINLOG_INDEX") + fi + else + binlog_files=$(cat "$WSREP_SST_OPT_BINLOG_INDEX") + fi + if [ -n "$binlog_files" ]; then + # Preparing binlog files for transfer: + wsrep_log_info "Preparing binlog files for transfer:" + tar_type=0 + if tar --help | grep -qw -F -- '--transform'; then + tar_type=1 + elif tar --help | grep -qw -E -- '-s[[:space:]]pattern' + then + tar_type=2 + fi + if [ $tar_type -ne 0 ]; then + # Preparing list of the binlog file names: + echo "$binlog_files" | { + binlogs="" + while read bin_file || [ -n "$bin_file" ]; do + [ ! -f "$bin_file" ] && continue + binlogs="$binlogs${binlogs:+ }'$bin_file'" + done + if [ -n "$binlogs" ]; then + tar_options='/^.*\///g' + if [ $tar_type -eq 1 ]; then + tar_options="--transform='s$tar_options'" + else + tar_options="-s '$tar_options'" + fi + eval tar $tar_options \ + -cvf "'$BINLOG_TAR_FILE'" $binlogs >&2 + fi + } + else + tar_options='-cvf' + echo "$binlog_files" | \ + while read bin_file || [ -n "$bin_file" ]; do + [ ! -f "$bin_file" ] && continue + bin_dir=$(dirname "$bin_file") + bin_base=$(basename "$bin_file") + tar $tar_options \ + "$BINLOG_TAR_FILE" -C "$bin_dir" "$bin_base" + tar_options='-rvf' + done + fi + fi fi - cd "$OLD_PWD" fi - # Use deltaxfer only for WAN + # Use deltaxfer only for WAN: inv=$(basename "$0") WHOLE_FILE_OPT="" if [ "${inv%wsrep_sst_rsync_wan*}" = "$inv" ]; then - WHOLE_FILE_OPT="--whole-file" + WHOLE_FILE_OPT='--whole-file' fi # Old filter - include everything except selected @@ -397,9 +459,9 @@ FILTER="-f '- /lost+found' -f '- /.pid' -f '- /.conf' -f '+ /wsrep_sst_binlog.tar' - -f '- $INNODB_DATA_HOME_DIR/ib_lru_dump' - -f '- $INNODB_DATA_HOME_DIR/ibdata*' - -f '+ $INNODB_UNDO_DIR/undo*' + -f '- $ib_home_dir/ib_lru_dump' + -f '- $ib_home_dir/ibdata*' + -f '+ $ib_undo_dir/undo*' -f '+ /*/' -f '- /*'" @@ -433,7 +495,7 @@ FILTER="-f '- /lost+found' --owner --group --perms --links --specials \ --ignore-times --inplace --dirs --delete --quiet \ $WHOLE_FILE_OPT -f '+ /ibdata*' -f '+ /ib_lru_dump' \ - -f '- **' "$INNODB_DATA_HOME_DIR/" \ + -f '- **' "$ib_home_dir/" \ "rsync://$WSREP_SST_OPT_ADDR-data_dir" >&2 || RC=$? if [ $RC -ne 0 ]; then @@ -446,7 +508,7 @@ FILTER="-f '- /lost+found' --owner --group --perms --links --specials \ --ignore-times --inplace --dirs --delete --quiet \ $WHOLE_FILE_OPT -f '+ /ib_logfile[0-9]*' -f '+ /aria_log.*' \ - -f '+ /aria_log_control' -f '- **' "$WSREP_LOG_DIR/" \ + -f '+ /aria_log_control' -f '- **' "$ib_log_dir/" \ "rsync://$WSREP_SST_OPT_ADDR-log_dir" >&2 || RC=$? if [ $RC -ne 0 ]; then @@ -549,7 +611,7 @@ then sleep 1 done - MODULE="rsync_sst" + MODULE='rsync_sst' RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid" RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf" @@ -576,8 +638,8 @@ then RSYNC_ADDR="$WSREP_SST_OPT_HOST" RSYNC_ADDR_UNESCAPED="$WSREP_SST_OPT_HOST_UNESCAPED" - trap "exit 32" HUP PIPE - trap "exit 3" INT TERM ABRT + trap 'exit 32' HUP PIPE + trap 'exit 3' INT TERM ABRT trap cleanup_joiner EXIT touch "$SST_PROGRESS_FILE" @@ -598,9 +660,9 @@ $SILENT path = $WSREP_SST_OPT_DATA exclude = .zfs [$MODULE-log_dir] - path = $WSREP_LOG_DIR + path = $ib_log_dir [$MODULE-data_dir] - path = $INNODB_DATA_HOME_DIR + path = $ib_home_dir EOF # rm -rf "$DATA/ib_logfile"* # we don't want old logs around @@ -615,7 +677,7 @@ EOF RSYNC_EXTRA_ARGS="" STUNNEL_ACCEPT="$RSYNC_PORT" # Overwrite address with all: - RSYNC_ADDR="*" + RSYNC_ADDR='*' fi if [ -z "$STUNNEL" ]; then @@ -686,7 +748,7 @@ EOF exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$SSTCERT" | \ - tr "," "\n" | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ + tr ',' '\n' | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ sed s/\ %//) fi MY_SECRET="$(wsrep_gen_secret)" @@ -723,16 +785,60 @@ EOF exit 32 fi - if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - if [ -f "$BINLOG_TAR_FILE" ]; then - cd "$BINLOG_DIRNAME" + if [ -r "$MAGIC_FILE" ]; then + if [ -n "$MY_SECRET" ]; then + # Check donor supplied secret: + SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ + cut -d ' ' -f 2) + if [ "$SECRET" != "$MY_SECRET" ]; then + wsrep_log_error "Donor does not know my secret!" + wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" + exit 32 + fi + fi + else + # This message should cause joiner to abort: + wsrep_log_info "rsync process ended without creating magic file" + echo "rsync process ended without creating '$MAGIC_FILE'" + exit 32 + fi + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + binlog_tar_present=0 + [ -f "$BINLOG_TAR_FILE" ] && binlog_tar_present=1 + # If it is SST (not an IST) or tar with binlogs is present + # among the transferred files, then we need to remove the + # old binlogs: + if [ $WSREP_SST_OPT_BYPASS -eq 0 -o $binlog_tar_present -ne 0 ]; then + # Change the directory to binlog base (if possible): + cd "$DATA" + binlog_cd=0 + if [ -n "$binlog_dir" -a \ + "$binlog_dir" != '.' -a \ + -d "$binlog_dir" ] + then + binlog_cd=1 + cd "$binlog_dir" + fi binlog_index="$WSREP_SST_OPT_BINLOG_INDEX" - - # Clean up old binlog files first - rm -f "$BINLOG_FILENAME".[0-9]* - [ -f "$binlog_index" ] && rm -f "$binlog_index" - + # Clean up old binlog files first: + if [ -f "$binlog_index" ]; then + binlogs=$(cat "$binlog_index") + rm -f "$binlog_index" + if [ $binlog_cd -ne 0 ]; then + cd "$DATA_DIR" + fi + echo "$binlogs" | \ + while read bin_file || [ -n "$bin_file" ]; do + rm -f "$bin_file" || : + done + else + rm -f "$binlog_base".[0-9]* || : + fi + cd "$OLD_PWD" + fi + if [ $binlog_tar_present -ne 0 ]; then + cd "$DATA" # Create a temporary file: tmpdir=$(parse_cnf '--mysqld|sst' 'tmpdir') if [ -z "$tmpdir" ]; then @@ -742,44 +848,34 @@ EOF else tmpfile=$(TMPDIR="$tmpdir"; mktemp '-d') fi - + # Extracting binlog files: wsrep_log_info "Extracting binlog files:" if ! tar -xvf "$BINLOG_TAR_FILE" > "$tmpfile"; then - wsrep_log_error "Error unpacking tar file with binlog files" rm -f "$tmpfile" + wsrep_log_error "Error unpacking tar file with binlog files" exit 32 fi - - # Rebuild binlog index: - while read bin_file; do - echo "$BINLOG_DIRNAME/$bin_file" >> "$binlog_index" - done < "$tmpfile" + binlogs=$(cat "$tmpfile") rm -f "$tmpfile" - + # Rebuild binlog index: + if [ $binlog_cd -ne 0 ]; then + cd "$binlog_dir" + fi + echo "$binlogs" | \ + while read bin_file || [ -n "$bin_file" ]; do + echo "$binlog_dir${binlog_dir:+/}$bin_file" >> "$binlog_index" + done cd "$OLD_PWD" fi fi - if [ -r "$MAGIC_FILE" ]; then - if [ -n "$MY_SECRET" ]; then - # check donor supplied secret - SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ - cut -d ' ' -f 2) - if [ "$SECRET" != "$MY_SECRET" ]; then - wsrep_log_error "Donor does not know my secret!" - wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" - exit 32 - fi - # remove secret from the magic file, and output - # the UUID:seqno & wsrep_gtid_domain_id: - grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE" - else - # Output the UUID:seqno and wsrep_gtid_domain_id: - cat "$MAGIC_FILE" - fi + if [ -n "$MY_SECRET" ]; then + # remove secret from the magic file, and output + # the UUID:seqno & wsrep_gtid_domain_id: + grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE" else - # this message should cause joiner to abort - echo "rsync process ended without creating '$MAGIC_FILE'" + # Output the UUID:seqno and wsrep_gtid_domain_id: + cat "$MAGIC_FILE" fi # wsrep_cleanup_progress_file diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index 569d5084c98..cf6da9a6992 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -38,7 +38,7 @@ tcert="" tcap="" tpem="" tkey="" -tmode="DISABLED" +tmode='DISABLED' sockopt="" progress="" ttime=0 @@ -85,11 +85,11 @@ backup_threads="" encrypt_threads="" encrypt_chunk="" -readonly SECRET_TAG="secret" +readonly SECRET_TAG='secret' sst_ver=-1 -if [ -n "$(commandex pv)" ] && pv --help | grep -qw -- '-F'; then +if [ -n "$(commandex pv)" ] && pv --help | grep -qw -F -- '-F'; then pvopts="$pvopts $pvformat" fi pcmd="pv $pvopts" @@ -102,17 +102,14 @@ if [ -z "$BACKUP_BIN" ]; then fi DATA="$WSREP_SST_OPT_DATA" -INFO_FILE="xtrabackup_galera_info" -IST_FILE="xtrabackup_ist" +INFO_FILE='xtrabackup_galera_info' +IST_FILE='xtrabackup_ist' MAGIC_FILE="$DATA/$INFO_FILE" INNOAPPLYLOG="$DATA/innobackupex.prepare.log" INNOMOVELOG="$DATA/innobackupex.move.log" INNOBACKUPLOG="$DATA/innobackupex.backup.log" -# Setting the path for ss and ip -export PATH="/usr/sbin:/sbin:$PATH" - timeit() { local stage="$1" @@ -182,11 +179,11 @@ get_keys() exit 2 fi ecmd="'$OPENSSL_BINARY' enc -$ealgo" - if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-pbkdf2'; then + if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-pbkdf2'; then ecmd="$ecmd -pbkdf2" - elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-iter'; then + elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-iter'; then ecmd="$ecmd -iter 1" - elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-md'; then + elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -F -- '-md'; then ecmd="$ecmd -md sha256" fi if [ -z "$ekey" ]; then @@ -234,12 +231,12 @@ get_transfer() if [ $tfmt = 'nc' ]; then wsrep_log_info "Using netcat as streamer" wsrep_check_programs nc - tcmd="nc" + tcmd='nc' if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then - if nc -h 2>&1 | grep -q 'ncat'; then + if nc -h 2>&1 | grep -q -F 'ncat'; then wsrep_log_info "Using Ncat as streamer" tcmd="$tcmd -l" - elif nc -h 2>&1 | grep -qw -- '-d'; then + elif nc -h 2>&1 | grep -qw -F -- '-d'; then wsrep_log_info "Using Debian netcat as streamer" tcmd="$tcmd -dl" if [ $WSREP_SST_OPT_HOST_IPv6 -eq 1 ]; then @@ -261,14 +258,14 @@ get_transfer() # transfer and cause the command to timeout. # Older versions of netcat did not need this flag and will # return an error if the flag is used. - if nc -h 2>&1 | grep -qw -- '-N'; then + if nc -h 2>&1 | grep -qw -F -- '-N'; then tcmd="$tcmd -N" wsrep_log_info "Using nc -N" fi # netcat doesn't understand [] around IPv6 address - if nc -h 2>&1 | grep -q ncat; then + if nc -h 2>&1 | grep -q -F 'ncat'; then wsrep_log_info "Using Ncat as streamer" - elif nc -h 2>&1 | grep -qw -- '-d'; then + elif nc -h 2>&1 | grep -qw -F -- '-d'; then wsrep_log_info "Using Debian netcat as streamer" else wsrep_log_info "Using traditional netcat as streamer" @@ -458,7 +455,7 @@ adjust_progress() fi elif [ -z "$progress" -a -n "$rlimit" ]; then # When rlimit is non-zero - pcmd="pv -q" + pcmd='pv -q' fi if [ -n "$rlimit" -a "$WSREP_SST_OPT_ROLE" = 'donor' ]; then @@ -592,8 +589,14 @@ get_stream() sig_joiner_cleanup() { + local estatus=$? + if [ $estatus -ne 0 ]; then + wsrep_log_error "Cleanup after exit with status: $estatus" + fi wsrep_log_error "Removing $MAGIC_FILE file due to signal" + [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" + exit $estatus } cleanup_at_exit() @@ -604,6 +607,8 @@ cleanup_at_exit() wsrep_log_error "Cleanup after exit with status: $estatus" fi + [ "$(pwd)" != "$OLD_PWD" ] && cd "$OLD_PWD" + if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then wsrep_log_info "Removing the sst_in_progress file" wsrep_cleanup_progress_file @@ -633,7 +638,7 @@ cleanup_at_exit() fi # Final cleanup - pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o '[0-9]*' || :) + pgid=$(ps -o pgid= $$ 2>/dev/null | grep -o -E '[0-9]*' || :) # This means no setsid done in mysqld. # We don't want to kill mysqld here otherwise. @@ -724,7 +729,7 @@ recv_joiner() local ltcmd="$tcmd" if [ $tmt -gt 0 ]; then if [ -n "$(commandex timeout)" ]; then - if timeout --help | grep -qw -- '-k'; then + if timeout --help | grep -qw -F -- '-k'; then ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd" else ltcmd="timeout -s9 $tmt $tcmd" @@ -824,9 +829,10 @@ monitor_process() # check the version, we require XB-2.3.5 to ensure that we can pass the # datadir via the command-line option -XB_REQUIRED_VERSION="2.3.5" +XB_REQUIRED_VERSION='2.3.5' -XB_VERSION=`$BACKUP_BIN --version 2>&1 | grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1` +XB_VERSION=$($BACKUP_BIN --version 2>&1 | \ + grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1) if [ -z "$XB_VERSION" ]; then wsrep_log_error "FATAL: Cannot determine the $BACKUP_BIN version." \ "Needs xtrabackup-$XB_REQUIRED_VERSION or higher to" \ @@ -843,7 +849,9 @@ fi [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE" -if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a "$WSREP_SST_OPT_ROLE" != 'donor' ]; then +if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a \ + "$WSREP_SST_OPT_ROLE" != 'donor' ] +then wsrep_log_error "Invalid role '$WSREP_SST_OPT_ROLE'" exit 22 fi @@ -851,25 +859,17 @@ fi read_cnf setup_ports -if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then +if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -F -- '--version-check'; then disver=' --no-version-check' fi -# if no command line argument and INNODB_DATA_HOME_DIR environment variable -# is not set, try to get it from my.cnf: -if [ -z "$INNODB_DATA_HOME_DIR" ]; then - INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') -fi - OLD_PWD="$(pwd)" -cd "$WSREP_SST_OPT_DATA" -if [ -n "$INNODB_DATA_HOME_DIR" ]; then - # handle both relative and absolute paths - [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" - cd "$INNODB_DATA_HOME_DIR" +if [ -n "$DATA" -a "$DATA" != '.' ]; then + [ ! -d "$DATA" ] && mkdir -p "$DATA" + cd "$DATA" fi -INNODB_DATA_HOME_DIR=$(pwd -P) +DATA_DIR="$(pwd -P)" cd "$OLD_PWD" @@ -897,7 +897,7 @@ if [ $ssyslog -eq 1 ]; then else if [ $sstlogarchive -eq 1 ] then - ARCHIVETIMESTAMP=$(date "+%Y.%m.%d-%H.%M.%S.%N") + ARCHIVETIMESTAMP=$(date '+%Y.%m.%d-%H.%M.%S.%N') if [ -n "$sstlogarchivedir" ]; then if [ ! -d "$sstlogarchivedir" ]; then @@ -1131,22 +1131,53 @@ then wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" [ -n "$SST_PROGRESS_FILE" ] && touch "$SST_PROGRESS_FILE" - ib_home_dir="$INNODB_DATA_HOME_DIR" + # if no command line argument and INNODB_DATA_HOME_DIR environment + # variable is not set, try to get it from the my.cnf: + if [ -z "$INNODB_DATA_HOME_DIR" ]; then + INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') + INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") + fi + + if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" + cd "$INNODB_DATA_HOME_DIR" + ib_home_dir="$(pwd -P)" + cd "$OLD_PWD" + fi # if no command line argument and INNODB_LOG_GROUP_HOME is not set, - # try to get it from my.cnf: + # then try to get it from the my.cnf: if [ -z "$INNODB_LOG_GROUP_HOME" ]; then INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') + INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") fi - ib_log_dir="$INNODB_LOG_GROUP_HOME" + if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" + cd "$INNODB_LOG_GROUP_HOME" + ib_log_dir="$(pwd -P)" + cd "$OLD_PWD" + fi - # if no command line argument then try to get it from my.cnf: + # if no command line argument and INNODB_UNDO_DIR is not set, + # then try to get it from the my.cnf: if [ -z "$INNODB_UNDO_DIR" ]; then INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') + INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") fi - ib_undo_dir="$INNODB_UNDO_DIR" + if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" + cd "$INNODB_UNDO_DIR" + ib_undo_dir="$(pwd -P)" + cd "$OLD_PWD" + fi if [ -n "$backup_threads" ]; then impts="--parallel=$backup_threads${impts:+ }$impts" @@ -1170,7 +1201,7 @@ then stagemsg='Joiner-Recv' - MODULE="xtrabackup_sst" + MODULE='xtrabackup_sst' [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" @@ -1193,7 +1224,7 @@ then exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$tpem" | \ - tr "," "\n" | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ + tr ',' '\n' | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ sed s/\ %//) fi MY_SECRET="$(wsrep_gen_secret)" @@ -1267,14 +1298,29 @@ then if [ -n "$WSREP_SST_OPT_BINLOG" ]; then binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") - if [ -d "$binlog_dir" ]; then - cd "$binlog_dir" - wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" - rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || : - [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ] && \ - rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+ - cd "$OLD_PWD" + binlog_base=$(basename "$WSREP_SST_OPT_BINLOG") + binlog_index="$WSREP_SST_OPT_BINLOG_INDEX" + binlog_explicit=0 + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' ]; then + binlog_explicit=1 + if [ -d "$binlog_dir" ]; then + cd "$binlog_dir" + wsrep_log_info \ + "Cleaning the binlog directory $binlog_dir as well" + if [ -f "$binlog_index" ]; then + binlogs=$(cat "$binlog_index") + rm -fv "$binlog_index" 1>&2 + cd "$DATA_DIR" + echo "$binlogs" | \ + while read bin_file || [ -n "$bin_file" ]; do + rm -fv "$bin_file" 1>&2 || : + done + else + rm -fv "$binlog_base".[0-9]* 1>&2 || : + fi + fi fi + cd "$OLD_PWD" fi TDATA="$DATA" @@ -1312,11 +1358,13 @@ then dcmd="xargs -n 2 qpress -dT$nproc" - if [ -n "$progress" ] && pv --help | grep -qw -- '--line-mode'; then + if [ -n "$progress" ] && \ + pv --help | grep -qw -F -- '--line-mode' + then count=$(find "$DATA" -type f -name '*.qp' | wc -l) count=$(( count*2 )) pvopts="-f -s $count -l -N Decompression" - if pv --help | grep -qw -- '-F'; then + if pv --help | grep -qw -F -- '-F'; then pvopts="$pvopts -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'" fi pcmd="pv $pvopts" @@ -1326,8 +1374,9 @@ then # Decompress the qpress files wsrep_log_info "Decompression with $nproc threads" - timeit "Joiner-Decompression" \ - "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" + timeit 'Joiner-Decompression' \ + "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | \ + $dcmd" extcode=$? if [ $extcode -eq 0 ]; then @@ -1345,24 +1394,36 @@ then fi if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - - BINLOG_DIRNAME=$(dirname "$WSREP_SST_OPT_BINLOG") - BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG") - - # To avoid comparing data directory and BINLOG_DIRNAME - mv "$DATA/$BINLOG_FILENAME".* "$BINLOG_DIRNAME/" 2>/dev/null || : - - cd "$BINLOG_DIRNAME" - for bfile in $(ls -1 "$BINLOG_FILENAME".[0-9]*); do - echo "$BINLOG_DIRNAME/$bfile" >> "$WSREP_SST_OPT_BINLOG_INDEX" - done + cd "$DATA" + binlogs="" + if [ -f 'xtrabackup_binlog_info' ]; then + NL=$'\n' + while read bin_string || [ -n "$bin_string" ]; do + bin_file=$(echo "$bin_string" | cut -f1) + if [ -f "$bin_file" ]; then + binlogs="$binlogs${binlogs:+$NL}$bin_file" + fi + done < 'xtrabackup_binlog_info' + else + binlogs=$(ls -d -1 "$binlog_base".[0-9]* 2>/dev/null || :) + fi + if [ -n "$binlogs" ]; then + cd "$DATA_DIR" + if [ $binlog_explicit -ne 0 ]; then + [ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir" + cd "$binlog_dir" + fi + echo "$binlogs" | \ + while read bin_file || [ -n "$bin_file" ]; do + echo "$binlog_dir${binlog_dir:+/}$bin_file" >> "$binlog_index" + done + fi cd "$OLD_PWD" - fi wsrep_log_info "Preparing the backup at $DATA" setup_commands - timeit "Xtrabackup prepare stage" "$INNOAPPLY" + timeit 'Xtrabackup prepare stage' "$INNOAPPLY" if [ $? -ne 0 ]; then wsrep_log_error "xtrabackup apply finished with errors." \ @@ -1373,7 +1434,7 @@ then MAGIC_FILE="$TDATA/$INFO_FILE" wsrep_log_info "Moving the backup to $TDATA" - timeit "Xtrabackup move stage" "$INNOMOVE" + timeit 'Xtrabackup move stage' "$INNOMOVE" if [ $? -eq 0 ]; then wsrep_log_info "Move successful, removing $DATA" rm -rf "$DATA" |