diff options
author | Julius Goryavsky <julius.goryavsky@mariadb.com> | 2022-01-14 04:10:48 +0100 |
---|---|---|
committer | Julius Goryavsky <julius.goryavsky@mariadb.com> | 2022-01-14 04:10:48 +0100 |
commit | 27ff5e949b1599bbb2bfc039dd2a12ad0c76f018 (patch) | |
tree | 2e8157a1b49cb549edc768d94ab620f57a6d81fc | |
parent | a3267c11fa6dfd950f8da0266b8afe8691809e82 (diff) | |
download | mariadb-git-10.2-sysprg-binlog.tar.gz |
Experimental change10.2-sysprg-binlog
-rw-r--r-- | mysql-test/suite/galera/r/galera_log_bin_ext.result | 9 | ||||
-rw-r--r-- | mysql-test/suite/galera/r/galera_log_bin_ext_mariabackup.result | 82 | ||||
-rw-r--r-- | mysql-test/suite/galera/t/galera_log_bin_ext.cnf | 3 | ||||
-rw-r--r-- | mysql-test/suite/galera/t/galera_log_bin_ext.test | 2 | ||||
-rw-r--r-- | mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.cnf | 18 | ||||
-rw-r--r-- | mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.test | 2 | ||||
-rw-r--r-- | mysql-test/suite/galera/t/galera_log_bin_sst.inc | 84 | ||||
-rw-r--r-- | scripts/wsrep_sst_common.sh | 137 | ||||
-rw-r--r-- | scripts/wsrep_sst_mariabackup.sh | 147 | ||||
-rw-r--r-- | scripts/wsrep_sst_rsync.sh | 291 | ||||
-rw-r--r-- | scripts/wsrep_sst_xtrabackup-v2.sh | 151 |
11 files changed, 669 insertions, 257 deletions
diff --git a/mysql-test/suite/galera/r/galera_log_bin_ext.result b/mysql-test/suite/galera/r/galera_log_bin_ext.result index f5276b7d1ac..c5ff66d17b3 100644 --- a/mysql-test/suite/galera/r/galera_log_bin_ext.result +++ b/mysql-test/suite/galera/r/galera_log_bin_ext.result @@ -1,4 +1,6 @@ connection node_1; +connection node_2; +connection node_1; reset master; connection node_2; reset master; @@ -40,6 +42,12 @@ hostname1-bin.000001 # Xid # # COMMIT /* XID */ hostname1-bin.000001 # Gtid # # GTID #-#-# hostname1-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN f2 INTEGER connection node_2; +Shutting down server ... +connection node_1; +Cleaning var directory ... +connection node_2; +Starting server ... +connection node_2; SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; COUNT(*) = 2 1 @@ -66,6 +74,7 @@ hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F hostname1-bin.000001 # Xid # # COMMIT /* XID */ hostname1-bin.000001 # Gtid # # GTID #-#-# hostname1-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN f2 INTEGER +hostname1-bin.000001 # Rotate # # hostname1-bin.000002;pos=4 DROP TABLE t1; DROP TABLE t2; #cleanup diff --git a/mysql-test/suite/galera/r/galera_log_bin_ext_mariabackup.result b/mysql-test/suite/galera/r/galera_log_bin_ext_mariabackup.result new file mode 100644 index 00000000000..c5ff66d17b3 --- /dev/null +++ b/mysql-test/suite/galera/r/galera_log_bin_ext_mariabackup.result @@ -0,0 +1,82 @@ +connection node_1; +connection node_2; +connection node_1; +reset master; +connection node_2; +reset master; +CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1); +CREATE TABLE t2 (id INT) ENGINE=InnoDB; +INSERT INTO t2 VALUES (1); +INSERT INTO t2 VALUES (1); +connection node_2; +SELECT COUNT(*) = 1 FROM t1; +COUNT(*) = 1 +1 +SELECT COUNT(*) = 2 FROM t2; +COUNT(*) = 2 +1 +connection node_1; +ALTER TABLE t1 ADD COLUMN f2 INTEGER; +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t1 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t1) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; CREATE TABLE t2 (id INT) ENGINE=InnoDB +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t2 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t2) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t2 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t2) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN f2 INTEGER +connection node_2; +Shutting down server ... +connection node_1; +Cleaning var directory ... +connection node_2; +Starting server ... +connection node_2; +SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +COUNT(*) = 2 +1 +include/show_binlog_events.inc +Log_name Pos Event_type Server_id End_log_pos Info +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t1 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t1) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; CREATE TABLE t2 (id INT) ENGINE=InnoDB +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t2 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t2) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # BEGIN GTID #-#-# +hostname1-bin.000001 # Annotate_rows # # INSERT INTO t2 VALUES (1) +hostname1-bin.000001 # Table_map # # table_id: # (test.t2) +hostname1-bin.000001 # Write_rows_v1 # # table_id: # flags: STMT_END_F +hostname1-bin.000001 # Xid # # COMMIT /* XID */ +hostname1-bin.000001 # Gtid # # GTID #-#-# +hostname1-bin.000001 # Query # # use `test`; ALTER TABLE t1 ADD COLUMN f2 INTEGER +hostname1-bin.000001 # Rotate # # hostname1-bin.000002;pos=4 +DROP TABLE t1; +DROP TABLE t2; +#cleanup +connection node_1; +RESET MASTER; diff --git a/mysql-test/suite/galera/t/galera_log_bin_ext.cnf b/mysql-test/suite/galera/t/galera_log_bin_ext.cnf index 012209610ea..8837138f59a 100644 --- a/mysql-test/suite/galera/t/galera_log_bin_ext.cnf +++ b/mysql-test/suite/galera/t/galera_log_bin_ext.cnf @@ -9,3 +9,6 @@ log-slave-updates log-bin = hostname2-bin log-bin-index = hostname2.bdx log-slave-updates + +[sst] +sst_rsync_max_binlogs= diff --git a/mysql-test/suite/galera/t/galera_log_bin_ext.test b/mysql-test/suite/galera/t/galera_log_bin_ext.test index 923bd623a8a..752073aecdb 100644 --- a/mysql-test/suite/galera/t/galera_log_bin_ext.test +++ b/mysql-test/suite/galera/t/galera_log_bin_ext.test @@ -1 +1 @@ ---source galera_log_bin.inc +--source galera_log_bin_sst.inc diff --git a/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.cnf b/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.cnf new file mode 100644 index 00000000000..c22fb723d73 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.cnf @@ -0,0 +1,18 @@ +!include ../galera_2nodes.cnf + +[mysqld] +wsrep_sst_method=mariabackup +wsrep_sst_auth="root:" + +[mysqld.1] +log-bin=@ENV.MYSQLTEST_VARDIR/mysqld.1/data/hostname1-bin +log-bin-index = hostname1.bdx +log-slave-updates + +[mysqld.2] +log-bin=@ENV.MYSQLTEST_VARDIR/mysqld.2/data/hostname2-bin +log-bin-index = hostname2.bdx +log-slave-updates + +[sst] +transferfmt=@ENV.MTR_GALERA_TFMT diff --git a/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.test b/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.test new file mode 100644 index 00000000000..47df45b4c71 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_log_bin_ext_mariabackup.test @@ -0,0 +1,2 @@ +--source include/have_mariabackup.inc +--source galera_log_bin_sst.inc diff --git a/mysql-test/suite/galera/t/galera_log_bin_sst.inc b/mysql-test/suite/galera/t/galera_log_bin_sst.inc new file mode 100644 index 00000000000..5d543f6f8b6 --- /dev/null +++ b/mysql-test/suite/galera/t/galera_log_bin_sst.inc @@ -0,0 +1,84 @@ +--source include/galera_cluster.inc +--source include/force_restart.inc + +# Save original auto_increment_offset values. +--let $node_1=node_1 +--let $node_2=node_2 +--source include/auto_increment_offset_save.inc + +--connection node_1 +reset master; +--connection node_2 +reset master; + +# +# Test Galera with --log-bin --log-slave-updates . +# This way the actual MySQL binary log is used, +# rather than Galera's own implementation +# + +CREATE TABLE t1 (id INT PRIMARY KEY) ENGINE=InnoDB; +INSERT INTO t1 VALUES (1); + +CREATE TABLE t2 (id INT) ENGINE=InnoDB; +INSERT INTO t2 VALUES (1); +INSERT INTO t2 VALUES (1); + +--connection node_2 +SELECT COUNT(*) = 1 FROM t1; +SELECT COUNT(*) = 2 FROM t2; + +--connection node_1 +ALTER TABLE t1 ADD COLUMN f2 INTEGER; +--let $MASTER_MYPORT=$NODE_MYPORT_1 +--source include/show_binlog_events.inc + +--connection node_2 + +#--connection node_2 +#--source suite/galera/include/galera_stop_replication.inc + +--echo Shutting down server ... +--source include/shutdown_mysqld.inc + +--connection node_1 +--let $wait_condition = SELECT VARIABLE_VALUE = 1 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size' +--source include/wait_condition.inc + +# +# Force SST +# +--echo Cleaning var directory ... +--remove_file $MYSQLTEST_VARDIR/mysqld.2/data/grastate.dat +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/mtr +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/performance_schema +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/test +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data/mysql +--remove_files_wildcard $MYSQLTEST_VARDIR/mysqld.2/data + +--connection node_2 + +--echo Starting server ... +let $restart_noprint=2; +--source include/start_mysqld.inc + +--let $wait_condition = SELECT VARIABLE_VALUE = 2 FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_cluster_size'; +--source include/wait_condition.inc + +--let $wait_condition = SELECT VARIABLE_VALUE = 'ON' FROM INFORMATION_SCHEMA.GLOBAL_STATUS WHERE VARIABLE_NAME = 'wsrep_ready'; +--source include/wait_condition.inc + +--connection node_2 +SELECT COUNT(*) = 2 FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_NAME = 't1'; +--let $MASTER_MYPORT=$NODE_MYPORT_2 +--source include/show_binlog_events.inc + +DROP TABLE t1; +DROP TABLE t2; + +--echo #cleanup +--connection node_1 +RESET MASTER; + +# Restore original auto_increment_offset values. +--source include/auto_increment_offset_restore.inc diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh index deebe7cf820..9a6a35c1403 100644 --- a/scripts/wsrep_sst_common.sh +++ b/scripts/wsrep_sst_common.sh @@ -22,6 +22,45 @@ set -ue # Setting the path for some utilities on CentOS export PATH="$PATH:/usr/sbin:/usr/bin:/sbin:/bin" +trim_string() +{ + if [ -n "$BASH_VERSION" ]; then + local pattern="[![:space:]${2:-}]" + local x="${1#*$pattern}" + local z=${#1} + x=${#x} + if [ $x -ne $z ]; then + local y="${1%$pattern*}" + y=${#y} + x=$(( z-x-1 )) + y=$(( y-x+1 )) + printf '%s' "${1:$x:$y}" + else + printf '' + fi + else + local pattern="[[:space:]${2:-}]" + echo "$1" | sed -E "s/^$pattern+|$pattern+\$//g" + fi +} + +trim_dir() +{ + local t=$(trim_string "$1") + if [ "$t" != '/' ]; then + if [ "${t%/}" != "$t" ]; then + t=$(trim_string "${t%/}") + fi + else + t='.' + fi + if [ -n "$BASH_VERSION" ]; then + printf '%s' "$t" + else + echo "$t" + fi +} + WSREP_SST_OPT_BYPASS=0 WSREP_SST_OPT_BINLOG="" WSREP_SST_OPT_BINLOG_INDEX="" @@ -44,9 +83,9 @@ WSREP_SST_OPT_ADDR_PORT="" WSREP_SST_OPT_HOST="" WSREP_SST_OPT_HOST_UNESCAPED="" WSREP_SST_OPT_HOST_ESCAPED="" -INNODB_DATA_HOME_DIR="${INNODB_DATA_HOME_DIR:-}" -INNODB_LOG_GROUP_HOME="${INNODB_LOG_GROUP_HOME:-}" -INNODB_UNDO_DIR="${INNODB_UNDO_DIR:-}" +INNODB_DATA_HOME_DIR=$(trim_dir "${INNODB_DATA_HOME_DIR:-}") +INNODB_LOG_GROUP_HOME=$(trim_dir "${INNODB_LOG_GROUP_HOME:-}") +INNODB_UNDO_DIR=$(trim_dir "${INNODB_UNDO_DIR:-}") INNODB_FORCE_RECOVERY="" INNOEXTRA="" @@ -138,22 +177,22 @@ case "$1" in ;; '--datadir') # Let's remove the trailing slash: - readonly WSREP_SST_OPT_DATA="${2%/}" + readonly WSREP_SST_OPT_DATA=$(trim_dir "$2") shift ;; '--innodb-data-home-dir') # Let's remove the trailing slash: - readonly INNODB_DATA_HOME_DIR="${2%/}" + readonly INNODB_DATA_HOME_DIR=$(trim_dir "$2") shift ;; '--innodb-log-group-home-dir') # Let's remove the trailing slash: - readonly INNODB_LOG_GROUP_HOME="${2%/}" + readonly INNODB_LOG_GROUP_HOME=$(trim_dir "$2") shift ;; '--innodb-undo-directory') # Let's remove the trailing slash: - readonly INNODB_UNDO_DIR="${2%/}" + readonly INNODB_UNDO_DIR=$(trim_dir "$2") shift ;; '--defaults-file') @@ -261,7 +300,7 @@ case "$1" in shift done fi - break; + break fi # Make sure the argument does not start with "--", otherwise it # is a long option, which is processed after this "if": @@ -305,7 +344,7 @@ case "$1" in fi if [ $option = 'h' ]; then if [ -z "$WSREP_SST_OPT_DATA" ]; then - MYSQLD_OPT_DATADIR="${value%/}" + MYSQLD_OPT_DATADIR=$(trim_dir "$value") fi elif [ $option != 'u' -a \ $option != 'P' ] @@ -338,7 +377,7 @@ case "$1" in original_cmd="$original_cmd '$1'" fi shift - continue; + continue fi # Now we are sure that we are working with an option # that has a "long" name, so remove all characters after @@ -370,19 +409,19 @@ case "$1" in case "$option" in '--innodb-data-home-dir') if [ -z "$INNODB_DATA_HOME_DIR" ]; then - MYSQLD_OPT_INNODB_DATA_HOME_DIR="${value%/}" + MYSQLD_OPT_INNODB_DATA_HOME_DIR=$(trim_dir "$value") fi skip_mysqld_arg=1 ;; '--innodb-log-group-home-dir') if [ -z "$INNODB_LOG_GROUP_HOME" ]; then - MYSQLD_OPT_INNODB_LOG_GROUP_HOME="${value%/}" + MYSQLD_OPT_INNODB_LOG_GROUP_HOME=$(trim_dir "$value") fi skip_mysqld_arg=1 ;; '--innodb-undo-directory') if [ -z "$INNODB_UNDO_DIR" ]; then - MYSQLD_OPT_INNODB_UNDO_DIR="${value%/}" + MYSQLD_OPT_INNODB_UNDO_DIR=$(trim_dir "$value") fi skip_mysqld_arg=1 ;; @@ -412,7 +451,7 @@ case "$1" in ;; '--datadir') if [ -z "$WSREP_SST_OPT_DATA" ]; then - MYSQLD_OPT_DATADIR="${value%/}" + MYSQLD_OPT_DATADIR=$(trim_dir "$value") fi skip_mysqld_arg=1 ;; @@ -530,12 +569,10 @@ get_binlog() WSREP_SST_OPT_LOG_BASENAME=$(parse_cnf '--mysqld' 'log-basename') fi if [ -z "$WSREP_SST_OPT_BINLOG" ]; then - # If the --log-bin option is specified without a parameter, + # If the log-bin option is specified without a parameter, # then we need to build the name of the index file according # to the rules described in the server documentation: - if [ -n "${MYSQLD_OPT_LOG_BIN+x}" -o \ - $(in_config '--mysqld' 'log-bin') -eq 1 ] - then + if [ $(in_config '--mysqld' 'log-bin') -eq 1 ]; then if [ -n "$WSREP_SST_OPT_LOG_BASENAME" ]; then # If the WSREP_SST_OPT_BINLOG variable is not set, but # --log-basename is present among the arguments of mysqld, @@ -543,8 +580,7 @@ get_binlog() # the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG="$WSREP_SST_OPT_LOG_BASENAME-bin" else - # If the --log-bin option is present without a value, then - # we take the default name: + # Take the default name: readonly WSREP_SST_OPT_BINLOG='mysql-bin' fi fi @@ -554,10 +590,10 @@ get_binlog() # it according to the specifications for the server: if [ -z "$WSREP_SST_OPT_BINLOG_INDEX" ]; then if [ -n "$WSREP_SST_OPT_LOG_BASENAME" ]; then - # If the WSREP_SST_OPT_BINLOG variable is not set, but + # If the WSREP_SST_OPT_BINLOG_INDEX variable is not set, but # --log-basename is present among the arguments of mysqld, - # then set WSREP_SST_OPT_BINLOG equal to the base name with - # the "-bin" suffix: + # then set WSREP_SST_OPT_BINLOG_INDEX equal to the base name + # with the "-bin" suffix: readonly WSREP_SST_OPT_BINLOG_INDEX="$WSREP_SST_OPT_LOG_BASENAME-bin.index" else # the default name (note that base of this name @@ -754,7 +790,11 @@ parse_cnf() if [ -z "$reval" ]; then [ -n "${3:-}" ] && reval="$3" fi - echo "$reval" + if [ -n "$BASH_VERSION" ]; then + printf '%s' "$reval" + else + echo "$reval" + fi } # @@ -804,7 +844,11 @@ in_config() break fi done - echo $found + if [ -n "$BASH_VERSION" ]; then + printf '%s' $found + else + echo $found + fi } wsrep_auth_not_set() @@ -937,11 +981,22 @@ wsrep_gen_secret() { get_openssl if [ -n "$OPENSSL_BINARY" ]; then - echo $("$OPENSSL_BINARY" rand -hex 16) - else - printf "%04x%04x%04x%04x%04x%04x%04x%04x" \ + "$OPENSSL_BINARY" rand -hex 16 + elif [ -n "$BASH_VERSION" ]; then + printf '%04x%04x%04x%04x%04x%04x%04x%04x' \ $RANDOM $RANDOM $RANDOM $RANDOM \ $RANDOM $RANDOM $RANDOM $RANDOM + elif [ -n $(commandex 'cksum') -a \ + -n $(commandex 'printf') ] + then + printf '%08x%08x%08x%08x' \ + $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) \ + $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) \ + $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) \ + $(head -8 /dev/urandom | cksum | cut -d ' ' -f1) + else + wsrep_log_error "Unable to generate 16-byte secret" + exit 22 fi } @@ -979,14 +1034,14 @@ is_local_ip() if [ -n "$ip_util" ]; then # ip address show ouput format is " inet[6] <address>/<mask>": "$ip_util" address show \ - | grep -E "^[[:space:]]*inet.? [^[:space:]]+/" -o \ + | grep -E '^[[:space:]]*inet.? [^[:space:]]+/' -o \ | grep -F " $1/" >/dev/null && return 0 else local ifconfig_util=$(commandex 'ifconfig') if [ -n "$ifconfig_util" ]; then # ifconfig output format is " inet[6] <address> ...": "$ifconfig_util" \ - | grep -E "^[[:space:]]*inet.? [^[:space:]]+ " -o \ + | grep -E '^[[:space:]]*inet.? [^[:space:]]+ ' -o \ | grep -F " $1 " >/dev/null && return 0 fi fi @@ -1220,28 +1275,6 @@ check_for_version() return 0 } -trim_string() -{ - if [ -n "$BASH_VERSION" ]; then - local pattern="[![:space:]${2:-}]" - local x="${1#*$pattern}" - local z=${#1} - x=${#x} - if [ $x -ne $z ]; then - local y="${1%$pattern*}" - y=${#y} - x=$(( z-x-1 )) - y=$(( y-x+1 )) - printf '%s' "${1:$x:$y}" - else - printf '' - fi - else - local pattern="[[:space:]${2:-}]" - echo "$1" | sed -E "s/^$pattern+|$pattern+\$//g" - fi -} - # # Check whether process is still running. # The first parameter contains the name of the PID file. diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh index aa9442b0601..26c723c55b7 100644 --- a/scripts/wsrep_sst_mariabackup.sh +++ b/scripts/wsrep_sst_mariabackup.sh @@ -37,7 +37,7 @@ tcert="" tcap="" tpem="" tkey="" -tmode="DISABLED" +tmode='DISABLED' sockopt="" progress="" ttime=0 @@ -82,7 +82,7 @@ backup_threads="" encrypt_threads="" encrypt_chunk="" -readonly SECRET_TAG="secret" +readonly SECRET_TAG='secret' # Required for backup locks # For backup locks it is 1 sent by joiner @@ -101,17 +101,14 @@ if [ -z "$BACKUP_BIN" ]; then fi DATA="$WSREP_SST_OPT_DATA" -INFO_FILE="xtrabackup_galera_info" -IST_FILE="xtrabackup_ist" +INFO_FILE='xtrabackup_galera_info' +IST_FILE='xtrabackup_ist' MAGIC_FILE="$DATA/$INFO_FILE" INNOAPPLYLOG="$DATA/mariabackup.prepare.log" INNOMOVELOG="$DATA/mariabackup.move.log" INNOBACKUPLOG="$DATA/mariabackup.backup.log" -# Setting the path for ss and ip -export PATH="/usr/sbin:/sbin:$PATH" - timeit() { local stage="$1" @@ -229,7 +226,7 @@ get_transfer() if [ $tfmt = 'nc' ]; then wsrep_log_info "Using netcat as streamer" wsrep_check_programs nc - tcmd="nc" + tcmd='nc' if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then if nc -h 2>&1 | grep -q 'ncat'; then wsrep_log_info "Using Ncat as streamer" @@ -453,7 +450,7 @@ adjust_progress() fi elif [ -z "$progress" -a -n "$rlimit" ]; then # When rlimit is non-zero - pcmd="pv -q" + pcmd='pv -q' fi if [ -n "$rlimit" -a "$WSREP_SST_OPT_ROLE" = 'donor' ]; then @@ -824,21 +821,13 @@ if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then disver=' --no-version-check' fi -# if no command line argument and INNODB_DATA_HOME_DIR environment variable -# is not set, try to get it from my.cnf: -if [ -z "$INNODB_DATA_HOME_DIR" ]; then - INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') -fi - OLD_PWD="$(pwd)" -cd "$WSREP_SST_OPT_DATA" -if [ -n "$INNODB_DATA_HOME_DIR" ]; then - # handle both relative and absolute paths - [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" - cd "$INNODB_DATA_HOME_DIR" +if [ -n "$DATA" -a "$DATA" != '.' ]; then + [ ! -d "$DATA" ] && mkdir -p "$DATA" + cd "$DATA" fi -INNODB_DATA_HOME_DIR=$(pwd -P) +DATA_DIR="$(pwd -P)" cd "$OLD_PWD" @@ -866,7 +855,7 @@ if [ $ssyslog -eq 1 ]; then else if [ $sstlogarchive -eq 1 ] then - ARCHIVETIMESTAMP=$(date "+%Y.%m.%d-%H.%M.%S.%N") + ARCHIVETIMESTAMP=$(date '+%Y.%m.%d-%H.%M.%S.%N') if [ -n "$sstlogarchivedir" ]; then if [ ! -d "$sstlogarchivedir" ]; then @@ -1100,22 +1089,53 @@ then wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" [ -n "$SST_PROGRESS_FILE" ] && touch "$SST_PROGRESS_FILE" - ib_home_dir="$INNODB_DATA_HOME_DIR" + # if no command line argument and INNODB_DATA_HOME_DIR environment + # variable is not set, try to get it from the my.cnf: + if [ -z "$INNODB_DATA_HOME_DIR" ]; then + INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') + INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") + fi + + if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" + cd "$INNODB_DATA_HOME_DIR" + ib_home_dir="$(pwd -P)" + cd "$OLD_PWD" + fi # if no command line argument and INNODB_LOG_GROUP_HOME is not set, - # try to get it from my.cnf: + # then try to get it from the my.cnf: if [ -z "$INNODB_LOG_GROUP_HOME" ]; then INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') + INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") fi - ib_log_dir="$INNODB_LOG_GROUP_HOME" + if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" + cd "$INNODB_LOG_GROUP_HOME" + ib_log_dir="$(pwd -P)" + cd "$OLD_PWD" + fi - # if no command line argument then try to get it from my.cnf: + # if no command line argument and INNODB_UNDO_DIR is not set, + # then try to get it from the my.cnf: if [ -z "$INNODB_UNDO_DIR" ]; then INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') + INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") fi - ib_undo_dir="$INNODB_UNDO_DIR" + if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" + cd "$INNODB_UNDO_DIR" + ib_undo_dir="$(pwd -P)" + cd "$OLD_PWD" + fi if [ -n "$backup_threads" ]; then impts="--parallel=$backup_threads${impts:+ }$impts" @@ -1162,7 +1182,7 @@ then exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$tpem" | \ - tr "," "\n" | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ + tr ',' '\n' | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ sed s/\ %//) fi MY_SECRET="$(wsrep_gen_secret)" @@ -1236,14 +1256,29 @@ then if [ -n "$WSREP_SST_OPT_BINLOG" ]; then binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") - if [ -d "$binlog_dir" ]; then - cd "$binlog_dir" - wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" - rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || : - [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ] && \ - rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+ - cd "$OLD_PWD" + binlog_base=$(basename "$WSREP_SST_OPT_BINLOG") + binlog_index="$WSREP_SST_OPT_BINLOG_INDEX" + binlog_explicit=0 + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' ]; then + binlog_explicit=1 + if [ -d "$binlog_dir" ]; then + cd "$binlog_dir" + wsrep_log_info \ + "Cleaning the binlog directory $binlog_dir as well" + if [ -f "$binlog_index" ]; then + binlogs=$(cat "$binlog_index") + rm -fv "$binlog_index" 1>&2 \+ + cd "$DATA_DIR" + echo "$binlogs" | \ + while read bin_file || [ -n "$bin_file" ]; do + rm -fv "$bin_file" 1>&2 \+ || : + done + else + rm -fv "$binlog_base".[0-9]* 1>&2 \+ || : + fi + fi fi + cd "$OLD_PWD" fi TDATA="$DATA" @@ -1293,7 +1328,7 @@ then # Decompress the qpress files wsrep_log_info "Decompression with $nproc threads" - timeit "Joiner-Decompression" \ + timeit 'Joiner-Decompression' \ "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" extcode=$? @@ -1312,24 +1347,36 @@ then fi if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - - BINLOG_DIRNAME=$(dirname "$WSREP_SST_OPT_BINLOG") - BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG") - - # To avoid comparing data directory and BINLOG_DIRNAME - mv "$DATA/$BINLOG_FILENAME".* "$BINLOG_DIRNAME/" 2>/dev/null || : - - cd "$BINLOG_DIRNAME" - for bfile in $(ls -1 "$BINLOG_FILENAME".[0-9]*); do - echo "$BINLOG_DIRNAME/$bfile" >> "$WSREP_SST_OPT_BINLOG_INDEX" - done + cd "$DATA" + binlogs="" + if [ -f 'xtrabackup_binlog_info' ]; then + NL=$'\n' + while read bin_string || [ -n "$bin_string" ]; do + bin_file=$(echo "$bin_string" | cut -f1) + if [ -f "$bin_file" ]; then + binlogs="$binlogs${binlogs:+$NL}$bin_file" + fi + done < 'xtrabackup_binlog_info' + else + binlogs=$(ls -d -1 "$binlog_base".[0-9]* 2>/dev/null || :) + fi + if [ -n "$binlogs" ]; then + cd "$DATA_DIR" + if [ $binlog_explicit -ne 0 ]; then + [ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir" + cd "$binlog_dir" + fi + echo "$binlogs" | \ + while read bin_file || [ -n "$bin_file" ]; do + echo "$binlog_dir${binlog_dir:+/}$bin_file" >> "$binlog_index" + done + fi cd "$OLD_PWD" - fi wsrep_log_info "Preparing the backup at $DATA" setup_commands - timeit "mariabackup prepare stage" "$INNOAPPLY" + timeit 'mariabackup prepare stage' "$INNOAPPLY" if [ $? -ne 0 ]; then wsrep_log_error "mariabackup apply finished with errors." \ @@ -1340,7 +1387,7 @@ then MAGIC_FILE="$TDATA/$INFO_FILE" wsrep_log_info "Moving the backup to $TDATA" - timeit "mariabackup move stage" "$INNOMOVE" + timeit 'mariabackup move stage' "$INNOMOVE" if [ $? -eq 0 ]; then wsrep_log_info "Move successful, removing $DATA" rm -rf "$DATA" diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh index 28dfed18218..8620f5f8001 100644 --- a/scripts/wsrep_sst_rsync.sh +++ b/scripts/wsrep_sst_rsync.sh @@ -138,66 +138,77 @@ STUNNEL_PID="$WSREP_SST_OPT_DATA/stunnel.pid" MAGIC_FILE="$WSREP_SST_OPT_DATA/rsync_sst_complete" -BINLOG_TAR_FILE="$WSREP_SST_OPT_DATA/wsrep_sst_binlog.tar" -BINLOG_N_FILES=1 - get_binlog if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - BINLOG_DIRNAME=$(dirname "$WSREP_SST_OPT_BINLOG") - BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG") + binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") + binlog_base=$(basename "$WSREP_SST_OPT_BINLOG") fi -# if no command line argument and INNODB_LOG_GROUP_HOME is not set, -# try to get it from my.cnf: -if [ -z "$INNODB_LOG_GROUP_HOME" ]; then - INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') +OLD_PWD="$(pwd)" + +DATA="$WSREP_SST_OPT_DATA" +if [ -n "$DATA" -a "$DATA" != '.' ]; then + [ ! -d "$DATA" ] && mkdir -p "$DATA" + cd "$DATA" fi +DATA_DIR="$(pwd -P)" -OLD_PWD="$(pwd)" +cd "$OLD_PWD" -WSREP_LOG_DIR="$INNODB_LOG_GROUP_HOME" +BINLOG_TAR_FILE="$DATA_DIR/wsrep_sst_binlog.tar" -cd "$WSREP_SST_OPT_DATA" -if [ -n "$WSREP_LOG_DIR" ]; then - # handle both relative and absolute paths - [ ! -d "$WSREP_LOG_DIR" ] && mkdir -p "$WSREP_LOG_DIR" - cd "$WSREP_LOG_DIR" +ib_log_dir="$DATA_DIR" +ib_home_dir="$DATA_DIR" +ib_undo_dir="$DATA_DIR" + +# if no command line argument and INNODB_LOG_GROUP_HOME is not set, +# then try to get it from the my.cnf: +if [ -z "$INNODB_LOG_GROUP_HOME" ]; then + INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') + INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") fi -WSREP_LOG_DIR=$(pwd -P) -cd "$OLD_PWD" +if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" + cd "$INNODB_LOG_GROUP_HOME" + ib_log_dir="$(pwd -P)" + cd "$OLD_PWD" +fi -# if no command line argument and INNODB_DATA_HOME_DIR environment variable -# is not set, try to get it from my.cnf: +# if no command line argument and INNODB_DATA_HOME_DIR environment +# variable is not set, try to get it from the my.cnf: if [ -z "$INNODB_DATA_HOME_DIR" ]; then INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') + INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") fi -cd "$WSREP_SST_OPT_DATA" -if [ -n "$INNODB_DATA_HOME_DIR" ]; then - # handle both relative and absolute paths +if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" cd "$INNODB_DATA_HOME_DIR" + ib_home_dir="$(pwd -P)" + cd "$OLD_PWD" fi -INNODB_DATA_HOME_DIR=$(pwd -P) - -cd "$OLD_PWD" -# if no command line argument then try to get it from my.cnf: +# if no command line argument and INNODB_UNDO_DIR is not set, +# then try to get it from the my.cnf: if [ -z "$INNODB_UNDO_DIR" ]; then INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') + INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") fi -cd "$WSREP_SST_OPT_DATA" -if [ -n "$INNODB_UNDO_DIR" ]; then - # handle both relative and absolute paths +if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" cd "$INNODB_UNDO_DIR" + ib_undo_dir="$(pwd -P)" + cd "$OLD_PWD" fi -INNODB_UNDO_DIR=$(pwd -P) - -cd "$OLD_PWD" encgroups='--mysqld|sst' @@ -274,7 +285,7 @@ if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ]; then CHECK_OPT="checkHost = $WSREP_SST_OPT_HOST" fi if is_local_ip "$WSREP_SST_OPT_HOST_UNESCAPED"; then - CHECK_OPT_LOCAL="checkHost = localhost" + CHECK_OPT_LOCAL='checkHost = localhost' fi fi fi @@ -291,7 +302,7 @@ if [ -n "$SSLMODE" -a "$SSLMODE" != 'DISABLED' ]; then fi fi -readonly SECRET_TAG="secret" +readonly SECRET_TAG='secret' if [ "$WSREP_SST_OPT_ROLE" = 'donor' ] then @@ -331,7 +342,7 @@ EOF [ -f "$FLUSHED" ] && rm -f "$FLUSHED" [ -f "$ERROR" ] && rm -f "$ERROR" - echo "flush tables" + echo 'flush tables' # Wait for : # (a) Tables to be flushed, AND @@ -355,32 +366,74 @@ EOF sync - if [ -n "$WSREP_SST_OPT_BINLOG" -a -d "${BINLOG_DIRNAME:-}" ] - then - # Prepare binlog files - cd "$BINLOG_DIRNAME" - - binlog_files_full=$(tail -n $BINLOG_N_FILES \ - "$WSREP_SST_OPT_BINLOG_INDEX") - binlog_files="" - for file in $binlog_files_full; do - binlog_file=$(basename "$file") - binlog_files="$binlog_files${binlog_files:+ }'$binlog_file'" - done - - if [ -n "$binlog_files" ]; then - wsrep_log_info "Preparing binlog files for transfer:" - eval tar -cvf "'$BINLOG_TAR_FILE'" $binlog_files >&2 + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + # Change the directory to binlog base (if possible): + cd "$DATA" + if [ -n "$binlog_dir" -a \ + "$binlog_dir" != '.' -a \ + -d "$binlog_dir" ] + then + cd "$binlog_dir" + fi + # Let's check the existence of the file with the index: + if [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ]; then + # Let's read the binlog index: + BINLOG_N_FILES=$(parse_cnf "$encgroups" \ + 'sst-rsync-max-binlogs') + if [ -n "$BINLOG_N_FILES" ]; then + binlog_files_full="" + if [ $BINLOG_N_FILES -gt 0 ]; then + binlog_files_full=$(tail -n $BINLOG_N_FILES \ + "$WSREP_SST_OPT_BINLOG_INDEX") + fi + else + binlog_files_full=$(cat "$WSREP_SST_OPT_BINLOG_INDEX") + fi + if [ -n "$binlog_files_full" ]; then + # Preparing binlog files for transfer: + wsrep_log_info "Preparing binlog files for transfer:" + tar_type=0 + if tar --help | grep -qW -F '--transform'; then + tar_type=1 + elif tar --help | grep -qW -E '-s[[:space:]]pattern'; then + tar_type=2 + fi + if [ $tar_type -gt 0 ]; then + # Preparing list of the binlog file names: + binlogs="" + echo "$binlog_files_full" | \ + while read bin_file || [ -n "$bin_file" ]; do + binlogs="$binlogs${binlogs:+ }'$bin_file'" + done + tar_options='/^.*\///g' + if [ $tar_type -eq 1 ]; then + tar_options="--transform 's$tar_options'" + else + tar_options="-s '$tar_options'" + fi + eval tar $tar_options \ + -cvf "'$BINLOG_TAR_FILE'" $binlogs >&2 + else + tar_options='-cvf' + echo "$binlog_files_full" | \ + while read bin_file || [ -n "$bin_file" ]; do + bin_dir=$(dirname "$bin_file") + bin_base=$(basename "$bin_file") + tar $tar_options \ + "$BINLOG_TAR_FILE" -C "$bin_dir" "$bin_base" + tar_options='-rvf' + done + fi + fi fi - cd "$OLD_PWD" fi - # Use deltaxfer only for WAN + # Use deltaxfer only for WAN: inv=$(basename "$0") WHOLE_FILE_OPT="" if [ "${inv%wsrep_sst_rsync_wan*}" = "$inv" ]; then - WHOLE_FILE_OPT="--whole-file" + WHOLE_FILE_OPT='--whole-file' fi # Old filter - include everything except selected @@ -397,9 +450,9 @@ FILTER="-f '- /lost+found' -f '- /.pid' -f '- /.conf' -f '+ /wsrep_sst_binlog.tar' - -f '- $INNODB_DATA_HOME_DIR/ib_lru_dump' - -f '- $INNODB_DATA_HOME_DIR/ibdata*' - -f '+ $INNODB_UNDO_DIR/undo*' + -f '- $ib_home_dir/ib_lru_dump' + -f '- $ib_home_dir/ibdata*' + -f '+ $ib_undo_dir/undo*' -f '+ /*/' -f '- /*'" @@ -433,7 +486,7 @@ FILTER="-f '- /lost+found' --owner --group --perms --links --specials \ --ignore-times --inplace --dirs --delete --quiet \ $WHOLE_FILE_OPT -f '+ /ibdata*' -f '+ /ib_lru_dump' \ - -f '- **' "$INNODB_DATA_HOME_DIR/" \ + -f '- **' "$ib_home_dir/" \ "rsync://$WSREP_SST_OPT_ADDR-data_dir" >&2 || RC=$? if [ $RC -ne 0 ]; then @@ -446,7 +499,7 @@ FILTER="-f '- /lost+found' --owner --group --perms --links --specials \ --ignore-times --inplace --dirs --delete --quiet \ $WHOLE_FILE_OPT -f '+ /ib_logfile[0-9]*' -f '+ /aria_log.*' \ - -f '+ /aria_log_control' -f '- **' "$WSREP_LOG_DIR/" \ + -f '+ /aria_log_control' -f '- **' "$ib_log_dir/" \ "rsync://$WSREP_SST_OPT_ADDR-log_dir" >&2 || RC=$? if [ $RC -ne 0 ]; then @@ -549,7 +602,7 @@ then sleep 1 done - MODULE="rsync_sst" + MODULE='rsync_sst' RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid" RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf" @@ -576,8 +629,8 @@ then RSYNC_ADDR="$WSREP_SST_OPT_HOST" RSYNC_ADDR_UNESCAPED="$WSREP_SST_OPT_HOST_UNESCAPED" - trap "exit 32" HUP PIPE - trap "exit 3" INT TERM ABRT + trap 'exit 32' HUP PIPE + trap 'exit 3' INT TERM ABRT trap cleanup_joiner EXIT touch "$SST_PROGRESS_FILE" @@ -598,9 +651,9 @@ $SILENT path = $WSREP_SST_OPT_DATA exclude = .zfs [$MODULE-log_dir] - path = $WSREP_LOG_DIR + path = $ib_log_dir [$MODULE-data_dir] - path = $INNODB_DATA_HOME_DIR + path = $ib_home_dir EOF # rm -rf "$DATA/ib_logfile"* # we don't want old logs around @@ -615,7 +668,7 @@ EOF RSYNC_EXTRA_ARGS="" STUNNEL_ACCEPT="$RSYNC_PORT" # Overwrite address with all: - RSYNC_ADDR="*" + RSYNC_ADDR='*' fi if [ -z "$STUNNEL" ]; then @@ -686,7 +739,7 @@ EOF exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$SSTCERT" | \ - tr "," "\n" | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ + tr ',' '\n' | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ sed s/\ %//) fi MY_SECRET="$(wsrep_gen_secret)" @@ -723,16 +776,60 @@ EOF exit 32 fi - if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - if [ -f "$BINLOG_TAR_FILE" ]; then - cd "$BINLOG_DIRNAME" + if [ -r "$MAGIC_FILE" ]; then + if [ -n "$MY_SECRET" ]; then + # Check donor supplied secret: + SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ + cut -d ' ' -f 2) + if [ "$SECRET" != "$MY_SECRET" ]; then + wsrep_log_error "Donor does not know my secret!" + wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" + exit 32 + fi + fi + else + [ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE" + # this message should cause joiner to abort + echo "rsync process ended without creating '$MAGIC_FILE'" + exit 32 + fi + if [ -n "$WSREP_SST_OPT_BINLOG" ]; then + binlog_tar_present=0 + [ -f "$BINLOG_TAR_FILE" ] && binlog_tar_present=1 + # If it is SST (not an IST) or tar with binlogs is present + # among the transferred files, then we need to remove the + # old binlogs: + if [ $WSREP_SST_OPT_BYPASS -eq 0 -o $binlog_tar_present -ne 0 ]; then + # Change the directory to binlog base (if possible): + cd "$DATA" + binlog_cd=0 + if [ -n "$binlog_dir" -a \ + "$binlog_dir" != '.' -a \ + -d "$binlog_dir" ] + then + binlog_cd=1 + cd "$binlog_dir" + fi binlog_index="$WSREP_SST_OPT_BINLOG_INDEX" - - # Clean up old binlog files first - rm -f "$BINLOG_FILENAME".[0-9]* - [ -f "$binlog_index" ] && rm -f "$binlog_index" - + # Clean up old binlog files first: + if [ -f "$binlog_index" ]; then + binlogs=$(cat "$binlog_index") + rm -fv "$binlog_index" + if [ $binlog_cd -ne 0 ]; then + cd "$DATA_DIR" + fi + echo "$binlogs" | \ + while read bin_file || [ -n "$bin_file" ]; do + rm -f "$bin_file" || : + done + else + rm -f "$binlog_base".[0-9]* || : + fi + cd "$OLD_PWD" + fi + if [ $binlog_tar_present -ne 0 ]; then + cd "$DATA" # Create a temporary file: tmpdir=$(parse_cnf '--mysqld|sst' 'tmpdir') if [ -z "$tmpdir" ]; then @@ -742,44 +839,34 @@ EOF else tmpfile=$(TMPDIR="$tmpdir"; mktemp '-d') fi - + # Extracting binlog files: wsrep_log_info "Extracting binlog files:" if ! tar -xvf "$BINLOG_TAR_FILE" > "$tmpfile"; then - wsrep_log_error "Error unpacking tar file with binlog files" rm -f "$tmpfile" + wsrep_log_error "Error unpacking tar file with binlog files" exit 32 fi - - # Rebuild binlog index: - while read bin_file; do - echo "$BINLOG_DIRNAME/$bin_file" >> "$binlog_index" - done < "$tmpfile" + binlogs=$(cat "$tmpfile") rm -f "$tmpfile" - + # Rebuild binlog index: + if [ $binlog_cd -ne 0 ]; then + cd "$binlog_dir" + fi + echo "$binlogs" | \ + while read bin_file || [ -n "$bin_file" ]; do + echo "$binlog_dir${binlog_dir:+/}$bin_file" >> "$binlog_index" + done cd "$OLD_PWD" fi fi - if [ -r "$MAGIC_FILE" ]; then - if [ -n "$MY_SECRET" ]; then - # check donor supplied secret - SECRET=$(grep -F -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | \ - cut -d ' ' -f 2) - if [ "$SECRET" != "$MY_SECRET" ]; then - wsrep_log_error "Donor does not know my secret!" - wsrep_log_info "Donor: '$SECRET', my: '$MY_SECRET'" - exit 32 - fi - # remove secret from the magic file, and output - # the UUID:seqno & wsrep_gtid_domain_id: - grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE" - else - # Output the UUID:seqno and wsrep_gtid_domain_id: - cat "$MAGIC_FILE" - fi + if [ -n "$MY_SECRET" ]; then + # remove secret from the magic file, and output + # the UUID:seqno & wsrep_gtid_domain_id: + grep -v -F -- "$SECRET_TAG " "$MAGIC_FILE" else - # this message should cause joiner to abort - echo "rsync process ended without creating '$MAGIC_FILE'" + # Output the UUID:seqno and wsrep_gtid_domain_id: + cat "$MAGIC_FILE" fi # wsrep_cleanup_progress_file diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh index 569d5084c98..48d9cffb956 100644 --- a/scripts/wsrep_sst_xtrabackup-v2.sh +++ b/scripts/wsrep_sst_xtrabackup-v2.sh @@ -38,7 +38,7 @@ tcert="" tcap="" tpem="" tkey="" -tmode="DISABLED" +tmode='DISABLED' sockopt="" progress="" ttime=0 @@ -85,7 +85,7 @@ backup_threads="" encrypt_threads="" encrypt_chunk="" -readonly SECRET_TAG="secret" +readonly SECRET_TAG='secret' sst_ver=-1 @@ -102,17 +102,14 @@ if [ -z "$BACKUP_BIN" ]; then fi DATA="$WSREP_SST_OPT_DATA" -INFO_FILE="xtrabackup_galera_info" -IST_FILE="xtrabackup_ist" +INFO_FILE='xtrabackup_galera_info' +IST_FILE='xtrabackup_ist' MAGIC_FILE="$DATA/$INFO_FILE" INNOAPPLYLOG="$DATA/innobackupex.prepare.log" INNOMOVELOG="$DATA/innobackupex.move.log" INNOBACKUPLOG="$DATA/innobackupex.backup.log" -# Setting the path for ss and ip -export PATH="/usr/sbin:/sbin:$PATH" - timeit() { local stage="$1" @@ -234,7 +231,7 @@ get_transfer() if [ $tfmt = 'nc' ]; then wsrep_log_info "Using netcat as streamer" wsrep_check_programs nc - tcmd="nc" + tcmd='nc' if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then if nc -h 2>&1 | grep -q 'ncat'; then wsrep_log_info "Using Ncat as streamer" @@ -458,7 +455,7 @@ adjust_progress() fi elif [ -z "$progress" -a -n "$rlimit" ]; then # When rlimit is non-zero - pcmd="pv -q" + pcmd='pv -q' fi if [ -n "$rlimit" -a "$WSREP_SST_OPT_ROLE" = 'donor' ]; then @@ -824,7 +821,7 @@ monitor_process() # check the version, we require XB-2.3.5 to ensure that we can pass the # datadir via the command-line option -XB_REQUIRED_VERSION="2.3.5" +XB_REQUIRED_VERSION='2.3.5' XB_VERSION=`$BACKUP_BIN --version 2>&1 | grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1` if [ -z "$XB_VERSION" ]; then @@ -855,21 +852,13 @@ if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then disver=' --no-version-check' fi -# if no command line argument and INNODB_DATA_HOME_DIR environment variable -# is not set, try to get it from my.cnf: -if [ -z "$INNODB_DATA_HOME_DIR" ]; then - INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') -fi - OLD_PWD="$(pwd)" -cd "$WSREP_SST_OPT_DATA" -if [ -n "$INNODB_DATA_HOME_DIR" ]; then - # handle both relative and absolute paths - [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" - cd "$INNODB_DATA_HOME_DIR" +if [ -n "$DATA" -a "$DATA" != '.' ]; then + [ ! -d "$DATA" ] && mkdir -p "$DATA" + cd "$DATA" fi -INNODB_DATA_HOME_DIR=$(pwd -P) +DATA_DIR="$(pwd -P)" cd "$OLD_PWD" @@ -897,7 +886,7 @@ if [ $ssyslog -eq 1 ]; then else if [ $sstlogarchive -eq 1 ] then - ARCHIVETIMESTAMP=$(date "+%Y.%m.%d-%H.%M.%S.%N") + ARCHIVETIMESTAMP=$(date '+%Y.%m.%d-%H.%M.%S.%N') if [ -n "$sstlogarchivedir" ]; then if [ ! -d "$sstlogarchivedir" ]; then @@ -1131,22 +1120,53 @@ then wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE" [ -n "$SST_PROGRESS_FILE" ] && touch "$SST_PROGRESS_FILE" - ib_home_dir="$INNODB_DATA_HOME_DIR" + # if no command line argument and INNODB_DATA_HOME_DIR environment + # variable is not set, try to get it from the my.cnf: + if [ -z "$INNODB_DATA_HOME_DIR" ]; then + INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir') + INNODB_DATA_HOME_DIR=$(trim_dir "$INNODB_DATA_HOME_DIR") + fi + + if [ -n "$INNODB_DATA_HOME_DIR" -a "$INNODB_DATA_HOME_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR" + cd "$INNODB_DATA_HOME_DIR" + ib_home_dir="$(pwd -P)" + cd "$OLD_PWD" + fi # if no command line argument and INNODB_LOG_GROUP_HOME is not set, - # try to get it from my.cnf: + # then try to get it from the my.cnf: if [ -z "$INNODB_LOG_GROUP_HOME" ]; then INNODB_LOG_GROUP_HOME=$(parse_cnf '--mysqld' 'innodb-log-group-home-dir') + INNODB_LOG_GROUP_HOME=$(trim_dir "$INNODB_LOG_GROUP_HOME") fi - ib_log_dir="$INNODB_LOG_GROUP_HOME" + if [ -n "$INNODB_LOG_GROUP_HOME" -a "$INNODB_LOG_GROUP_HOME" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_LOG_GROUP_HOME" ] && mkdir -p "$INNODB_LOG_GROUP_HOME" + cd "$INNODB_LOG_GROUP_HOME" + ib_log_dir="$(pwd -P)" + cd "$OLD_PWD" + fi - # if no command line argument then try to get it from my.cnf: + # if no command line argument and INNODB_UNDO_DIR is not set, + # then try to get it from the my.cnf: if [ -z "$INNODB_UNDO_DIR" ]; then INNODB_UNDO_DIR=$(parse_cnf '--mysqld' 'innodb-undo-directory') + INNODB_UNDO_DIR=$(trim_dir "$INNODB_UNDO_DIR") fi - ib_undo_dir="$INNODB_UNDO_DIR" + if [ -n "$INNODB_UNDO_DIR" -a "$INNODB_UNDO_DIR" != '.' ]; then + # handle both relative and absolute paths: + cd "$DATA" + [ ! -d "$INNODB_UNDO_DIR" ] && mkdir -p "$INNODB_UNDO_DIR" + cd "$INNODB_UNDO_DIR" + ib_undo_dir="$(pwd -P)" + cd "$OLD_PWD" + fi if [ -n "$backup_threads" ]; then impts="--parallel=$backup_threads${impts:+ }$impts" @@ -1170,7 +1190,7 @@ then stagemsg='Joiner-Recv' - MODULE="xtrabackup_sst" + MODULE='xtrabackup_sst' [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE" @@ -1193,7 +1213,7 @@ then exit 42 fi CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$tpem" | \ - tr "," "\n" | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ + tr ',' '\n' | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \ sed s/\ %//) fi MY_SECRET="$(wsrep_gen_secret)" @@ -1267,14 +1287,29 @@ then if [ -n "$WSREP_SST_OPT_BINLOG" ]; then binlog_dir=$(dirname "$WSREP_SST_OPT_BINLOG") - if [ -d "$binlog_dir" ]; then - cd "$binlog_dir" - wsrep_log_info "Cleaning the binlog directory $binlog_dir as well" - rm -fv "$WSREP_SST_OPT_BINLOG".[0-9]* 1>&2 \+ || : - [ -f "$WSREP_SST_OPT_BINLOG_INDEX" ] && \ - rm -fv "$WSREP_SST_OPT_BINLOG_INDEX" 1>&2 \+ - cd "$OLD_PWD" + binlog_base=$(basename "$WSREP_SST_OPT_BINLOG") + binlog_index="$WSREP_SST_OPT_BINLOG_INDEX" + binlog_explicit=0 + if [ -n "$binlog_dir" -a "$binlog_dir" != '.' ]; then + binlog_explicit=1 + if [ -d "$binlog_dir" ]; then + cd "$binlog_dir" + wsrep_log_info \ + "Cleaning the binlog directory $binlog_dir as well" + if [ -f "$binlog_index" ]; then + binlogs=$(cat "$binlog_index") + rm -fv "$binlog_index" 1>&2 \+ + cd "$DATA_DIR" + echo "$binlogs" | \ + while read bin_file || [ -n "$bin_file" ]; do + rm -fv "$bin_file" 1>&2 \+ || : + done + else + rm -fv "$binlog_base".[0-9]* 1>&2 \+ || : + fi + fi fi + cd "$OLD_PWD" fi TDATA="$DATA" @@ -1326,7 +1361,7 @@ then # Decompress the qpress files wsrep_log_info "Decompression with $nproc threads" - timeit "Joiner-Decompression" \ + timeit 'Joiner-Decompression' \ "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd" extcode=$? @@ -1345,24 +1380,36 @@ then fi if [ -n "$WSREP_SST_OPT_BINLOG" ]; then - - BINLOG_DIRNAME=$(dirname "$WSREP_SST_OPT_BINLOG") - BINLOG_FILENAME=$(basename "$WSREP_SST_OPT_BINLOG") - - # To avoid comparing data directory and BINLOG_DIRNAME - mv "$DATA/$BINLOG_FILENAME".* "$BINLOG_DIRNAME/" 2>/dev/null || : - - cd "$BINLOG_DIRNAME" - for bfile in $(ls -1 "$BINLOG_FILENAME".[0-9]*); do - echo "$BINLOG_DIRNAME/$bfile" >> "$WSREP_SST_OPT_BINLOG_INDEX" - done + cd "$DATA" + binlogs="" + if [ -f 'xtrabackup_binlog_info' ]; then + NL=$'\n' + while read bin_string || [ -n "$bin_string" ]; do + bin_file=$(echo "$bin_string" | cut -f1) + if [ -f "$bin_file" ]; then + binlogs="$binlogs${binlogs:+$NL}$bin_file" + fi + done < 'xtrabackup_binlog_info' + else + binlogs=$(ls -d -1 "$binlog_base".[0-9]* 2>/dev/null || :) + fi + if [ -n "$binlogs" ]; then + cd "$DATA_DIR" + if [ $binlog_explicit -ne 0 ]; then + [ ! -d "$binlog_dir" ] && mkdir -p "$binlog_dir" + cd "$binlog_dir" + fi + echo "$binlogs" | \ + while read bin_file || [ -n "$bin_file" ]; do + echo "$binlog_dir${binlog_dir:+/}$bin_file" >> "$binlog_index" + done + fi cd "$OLD_PWD" - fi wsrep_log_info "Preparing the backup at $DATA" setup_commands - timeit "Xtrabackup prepare stage" "$INNOAPPLY" + timeit 'Xtrabackup prepare stage' "$INNOAPPLY" if [ $? -ne 0 ]; then wsrep_log_error "xtrabackup apply finished with errors." \ @@ -1373,7 +1420,7 @@ then MAGIC_FILE="$TDATA/$INFO_FILE" wsrep_log_info "Moving the backup to $TDATA" - timeit "Xtrabackup move stage" "$INNOMOVE" + timeit 'Xtrabackup move stage' "$INNOMOVE" if [ $? -eq 0 ]; then wsrep_log_info "Move successful, removing $DATA" rm -rf "$DATA" |