summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJulius Goryavsky <julius.goryavsky@mariadb.com>2021-05-29 19:54:25 +0200
committerJulius Goryavsky <julius.goryavsky@mariadb.com>2021-05-29 19:54:25 +0200
commit3bfbd805adf4c0504f230b673fa213ed97301e94 (patch)
treedebcefe000173faca4520426ace047de6f9d8caf
parentd06205ba3713da6c5875f124d5e431d3704aad1d (diff)
downloadmariadb-git-bb-10.2-MDEV-25818.tar.gz
MDEV-25818: RSYNC SST failed due to busy portbb-10.2-MDEV-25818
This commit reduces the likelihood of getting a busy port on quick restarts with rsync SST (problem MDEV-25818) and fixes a number of other flaws in SST scripts, adds new functionality, and also synchronizes the xtrabackup-v2 script with the mariabackup script (the latter applies only to the 10.2 branch): 1) SST via rsync: rsync and stunnel does not always get the right time to complete by correctly handling SIGTERM. These utilities are now given more time to complete normally (via normal SIGTERM processing) before we move on to using "kill -9"; 2) SST via rsync: attempts to terminate an rsync or stunnel process (via "kill" utility) are only made if it did not terminated on its own; 3) SST via rsync: if a combination of stunnel and rsync is used, then we need to wait for both utilities to finish or stop, not just one of them; 4) The config file and pid file for stunnel are now deleted after successful completion of SST on the donor node; 5) The configs and pid files from rsync and stunnel should not be deleted unless these utilities succeed (or are sucessfully terminated) on the joiner node; 6) The configs and pid files now excluded from transfer via rsync; 7) Spaces in paths are now valid for config files as well (when used with SST via rsync or mariabackup / xtrabackup[-v2]); 8) SST via mariabackup: added preliminary verification of keys and certificates that are used when establishing a connection using SSL (to avoid long timeouts and improve diagnostics) - by analogy with how it is done for the xtrabackup-v2 (plus check for CA file), while that check is skipped if the user does not have openssl installed (or does not have diff utility); 9) Added backup-threads=<n> configuration option which adds "--parallel=<n>" for mariabackup / xtrabackup at backup and move-back stages; 10) Added encrypt-threads and encrypt-chunk-size configuration options for xbcrypt management (when xbcrypt is used); 11) Small optimization: checking the socat version and adding a file with parameters for 2048-bit Diffie-Hellman (if necessary) is done only if the user has not specified "dhparam=" in the "sockopt" option value; 12) SST via rsync now supports "backup-threads" configuration option (in server-related sections or in the "[sst]"); 13) Determining the number of available processors is now supported for FreeBSD + mariabackup/xtrabackup: before that we might have problems with "--compact" (rebuild indexes) or qpress on FreeBSD; 14) The check_pid() function should not raise an error state in the rare cases when the pid file was created, but it is empty, or if it is deleted right during the check, or when zero is read from the pid file; 15) Iproved templates that are used to check if a requested socket is "listening" when using the ss utility; 16) Shortened some other templates for socket state utilities; 17) Temporary files created by mariabackup / xtrabackup are moved to a separate subdirectory inside tmpdir (so they don't get mixed with other temporary files, which can make debugging more difficult); 18) 10.2 only: the script for SST via xtrabackup-v2 has been brought in full compliance with all the bugfixes made for mariabackup (as it previously contained many flaws compared to the updated script for mariabackup).
-rw-r--r--mysql-test/suite/galera/t/galera_autoinc_sst_xtrabackup.cnf1
-rw-r--r--mysql-test/suite/galera/t/galera_ist_mariabackup.cnf1
-rw-r--r--mysql-test/suite/galera/t/galera_ist_mariabackup_innodb_flush_logs.cnf1
-rw-r--r--mysql-test/suite/galera/t/galera_ist_mysqldump.cnf2
-rw-r--r--mysql-test/suite/galera/t/galera_ist_xtrabackup-v2.cnf2
-rw-r--r--mysql-test/suite/galera/t/galera_sst_mariabackup.cnf2
-rw-r--r--mysql-test/suite/galera/t/galera_sst_mariabackup_qpress.cnf1
-rw-r--r--mysql-test/suite/galera/t/galera_sst_mariabackup_table_options.cnf2
-rw-r--r--mysql-test/suite/galera/t/galera_sst_rsync.cnf1
-rw-r--r--mysql-test/suite/galera/t/galera_sst_rsync2.cnf1
-rw-r--r--mysql-test/suite/galera/t/galera_sst_rsync_data_dir.cnf3
-rw-r--r--scripts/wsrep_sst_common.sh361
-rw-r--r--scripts/wsrep_sst_mariabackup.sh292
-rw-r--r--scripts/wsrep_sst_mysqldump.sh4
-rw-r--r--scripts/wsrep_sst_rsync.sh255
-rw-r--r--scripts/wsrep_sst_xtrabackup-v2.sh1034
-rw-r--r--scripts/wsrep_sst_xtrabackup.sh23
17 files changed, 1169 insertions, 817 deletions
diff --git a/mysql-test/suite/galera/t/galera_autoinc_sst_xtrabackup.cnf b/mysql-test/suite/galera/t/galera_autoinc_sst_xtrabackup.cnf
index 7da7dd4e73b..2e25ecae317 100644
--- a/mysql-test/suite/galera/t/galera_autoinc_sst_xtrabackup.cnf
+++ b/mysql-test/suite/galera/t/galera_autoinc_sst_xtrabackup.cnf
@@ -10,4 +10,3 @@ wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore
[mysqld.2]
wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true'
-
diff --git a/mysql-test/suite/galera/t/galera_ist_mariabackup.cnf b/mysql-test/suite/galera/t/galera_ist_mariabackup.cnf
index 75dff78149d..3dc62546591 100644
--- a/mysql-test/suite/galera/t/galera_ist_mariabackup.cnf
+++ b/mysql-test/suite/galera/t/galera_ist_mariabackup.cnf
@@ -9,4 +9,3 @@ wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true'
[mysqld.2]
wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true'
-
diff --git a/mysql-test/suite/galera/t/galera_ist_mariabackup_innodb_flush_logs.cnf b/mysql-test/suite/galera/t/galera_ist_mariabackup_innodb_flush_logs.cnf
index 1542376b2b8..7979b23e085 100644
--- a/mysql-test/suite/galera/t/galera_ist_mariabackup_innodb_flush_logs.cnf
+++ b/mysql-test/suite/galera/t/galera_ist_mariabackup_innodb_flush_logs.cnf
@@ -11,4 +11,3 @@ wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true'
[mysqld.2]
wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true'
-
diff --git a/mysql-test/suite/galera/t/galera_ist_mysqldump.cnf b/mysql-test/suite/galera/t/galera_ist_mysqldump.cnf
index 357c8678658..807829f047f 100644
--- a/mysql-test/suite/galera/t/galera_ist_mysqldump.cnf
+++ b/mysql-test/suite/galera/t/galera_ist_mysqldump.cnf
@@ -8,5 +8,3 @@ wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true'
[mysqld.2]
wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true'
-
-
diff --git a/mysql-test/suite/galera/t/galera_ist_xtrabackup-v2.cnf b/mysql-test/suite/galera/t/galera_ist_xtrabackup-v2.cnf
index 52df3814129..5b35701aeab 100644
--- a/mysql-test/suite/galera/t/galera_ist_xtrabackup-v2.cnf
+++ b/mysql-test/suite/galera/t/galera_ist_xtrabackup-v2.cnf
@@ -5,10 +5,8 @@ wsrep_sst_method=xtrabackup-v2
wsrep_sst_auth=root:
innodb_safe_truncate=OFF
-
[mysqld.1]
wsrep_provider_options='base_port=@mysqld.1.#galera_port;pc.ignore_sb=true'
[mysqld.2]
wsrep_provider_options='base_port=@mysqld.2.#galera_port;pc.ignore_sb=true'
-
diff --git a/mysql-test/suite/galera/t/galera_sst_mariabackup.cnf b/mysql-test/suite/galera/t/galera_sst_mariabackup.cnf
index 336296e9bfe..4110192b947 100644
--- a/mysql-test/suite/galera/t/galera_sst_mariabackup.cnf
+++ b/mysql-test/suite/galera/t/galera_sst_mariabackup.cnf
@@ -13,4 +13,4 @@ wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore
[sst]
transferfmt=@ENV.MTR_GALERA_TFMT
-streamfmt=xbstream
+streamfmt=mbstream
diff --git a/mysql-test/suite/galera/t/galera_sst_mariabackup_qpress.cnf b/mysql-test/suite/galera/t/galera_sst_mariabackup_qpress.cnf
index 25a9d5fc0b2..f58d22cba52 100644
--- a/mysql-test/suite/galera/t/galera_sst_mariabackup_qpress.cnf
+++ b/mysql-test/suite/galera/t/galera_sst_mariabackup_qpress.cnf
@@ -10,3 +10,4 @@ transferfmt=@ENV.MTR_GALERA_TFMT
compress=quicklz
compress-threads=2
compress-chunk-size=32768
+backup-threads=2
diff --git a/mysql-test/suite/galera/t/galera_sst_mariabackup_table_options.cnf b/mysql-test/suite/galera/t/galera_sst_mariabackup_table_options.cnf
index 5e6913d4d2b..13cb1aa5366 100644
--- a/mysql-test/suite/galera/t/galera_sst_mariabackup_table_options.cnf
+++ b/mysql-test/suite/galera/t/galera_sst_mariabackup_table_options.cnf
@@ -15,4 +15,4 @@ wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore
[sst]
transferfmt=@ENV.MTR_GALERA_TFMT
-streamfmt=xbstream
+streamfmt=mbstream
diff --git a/mysql-test/suite/galera/t/galera_sst_rsync.cnf b/mysql-test/suite/galera/t/galera_sst_rsync.cnf
index 93981d9daa7..29d3a1e6219 100644
--- a/mysql-test/suite/galera/t/galera_sst_rsync.cnf
+++ b/mysql-test/suite/galera/t/galera_sst_rsync.cnf
@@ -8,4 +8,3 @@ wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore
[mysqld.2]
wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true'
-
diff --git a/mysql-test/suite/galera/t/galera_sst_rsync2.cnf b/mysql-test/suite/galera/t/galera_sst_rsync2.cnf
index 34e67c66403..0159596f99b 100644
--- a/mysql-test/suite/galera/t/galera_sst_rsync2.cnf
+++ b/mysql-test/suite/galera/t/galera_sst_rsync2.cnf
@@ -12,4 +12,3 @@ log_bin_index=@ENV.MYSQLTEST_VARDIR/tmp/server1_binlog_index.index
wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true'
log_bin=@ENV.MYSQLTEST_VARDIR/server2_binlog
log_bin_index=@ENV.MYSQLTEST_VARDIR/tmp/server2_binlog_index.index
-
diff --git a/mysql-test/suite/galera/t/galera_sst_rsync_data_dir.cnf b/mysql-test/suite/galera/t/galera_sst_rsync_data_dir.cnf
index afe9796a11a..4cc3442bd87 100644
--- a/mysql-test/suite/galera/t/galera_sst_rsync_data_dir.cnf
+++ b/mysql-test/suite/galera/t/galera_sst_rsync_data_dir.cnf
@@ -9,3 +9,6 @@ wsrep_provider_options='base_port=@mysqld.1.#galera_port;gcache.size=1;pc.ignore
[mysqld.2]
innodb_data_home_dir=@ENV.MYSQL_TMP_DIR/rsync_test_2
wsrep_provider_options='base_port=@mysqld.2.#galera_port;gcache.size=1;pc.ignore_sb=true'
+
+[sst]
+backup_threads=2
diff --git a/scripts/wsrep_sst_common.sh b/scripts/wsrep_sst_common.sh
index 261912b86ad..05944ef6035 100644
--- a/scripts/wsrep_sst_common.sh
+++ b/scripts/wsrep_sst_common.sh
@@ -29,7 +29,9 @@ WSREP_SST_OPT_USER="${WSREP_SST_OPT_USER:-}"
WSREP_SST_OPT_PSWD="${WSREP_SST_OPT_PSWD:-}"
WSREP_SST_OPT_REMOTE_AUTH="${WSREP_SST_OPT_REMOTE_AUTH:-}"
WSREP_SST_OPT_DEFAULT=""
+WSREP_SST_OPT_DEFAULTS=""
WSREP_SST_OPT_EXTRA_DEFAULT=""
+WSREP_SST_OPT_EXTRA_DEFAULTS=""
WSREP_SST_OPT_SUFFIX_DEFAULT=""
WSREP_SST_OPT_SUFFIX_VALUE=""
WSREP_SST_OPT_MYSQLD=""
@@ -152,10 +154,12 @@ case "$1" in
;;
'--defaults-file')
readonly WSREP_SST_OPT_DEFAULT="$1=$2"
+ readonly WSREP_SST_OPT_DEFAULTS="$1='$2'"
shift
;;
'--defaults-extra-file')
readonly WSREP_SST_OPT_EXTRA_DEFAULT="$1=$2"
+ readonly WSREP_SST_OPT_EXTRA_DEFAULTS="$1='$2'"
shift
;;
'--defaults-group-suffix')
@@ -611,24 +615,54 @@ else
MYSQLDUMP="$(command -v mysqldump)"
fi
+wsrep_log()
+{
+ # echo everything to stderr so that it gets into common error log
+ # deliberately made to look different from the rest of the log
+ local readonly tst="$(date +%Y%m%d\ %H:%M:%S.%N | cut -b -21)"
+ echo "WSREP_SST: $* ($tst)" >&2
+}
+
+wsrep_log_error()
+{
+ wsrep_log "[ERROR] $*"
+}
+
+wsrep_log_warning()
+{
+ wsrep_log "[WARNING] $*"
+}
+
+wsrep_log_info()
+{
+ wsrep_log "[INFO] $*"
+}
+
if [ -x "$SCRIPTS_DIR/my_print_defaults" ]; then
MY_PRINT_DEFAULTS="$SCRIPTS_DIR/my_print_defaults"
elif [ -x "$EXTRA_DIR/my_print_defaults" ]; then
MY_PRINT_DEFAULTS="$EXTRA_DIR/my_print_defaults"
else
MY_PRINT_DEFAULTS="$(command -v my_print_defaults)"
+ if [ -z "$MY_PRINT_DEFAULTS" ]; then
+ wsrep_log_error "my_print_defaults not found in path"
+ exit 2
+ fi
fi
+readonly MY_PRINT_DEFAULTS
+
+wsrep_defaults="$WSREP_SST_OPT_DEFAULTS"
+wsrep_defaults="$wsrep_defaults${wsrep_defaults:+ }$WSREP_SST_OPT_EXTRA_DEFAULTS"
+wsrep_defaults="$wsrep_defaults${wsrep_defaults:+ }$WSREP_SST_OPT_SUFFIX_DEFAULT"
+
+readonly WSREP_SST_OPT_CONF="$wsrep_defaults"
+
wsrep_defaults="$WSREP_SST_OPT_DEFAULT"
-if [ -n "$wsrep_defaults" ]; then
- wsrep_defaults="$wsrep_defaults "
-fi
-wsrep_defaults="$wsrep_defaults$WSREP_SST_OPT_EXTRA_DEFAULT"
-if [ -n "$wsrep_defaults" ]; then
- wsrep_defaults="$wsrep_defaults "
-fi
-readonly WSREP_SST_OPT_CONF="$wsrep_defaults$WSREP_SST_OPT_SUFFIX_DEFAULT"
-readonly MY_PRINT_DEFAULTS="$MY_PRINT_DEFAULTS $WSREP_SST_OPT_CONF"
+wsrep_defaults="$wsrep_defaults${wsrep_defaults:+ }$WSREP_SST_OPT_EXTRA_DEFAULT"
+wsrep_defaults="$wsrep_defaults${wsrep_defaults:+ }$WSREP_SST_OPT_SUFFIX_DEFAULT"
+
+readonly WSREP_SST_OPT_CONF_UNQUOTED="$wsrep_defaults"
#
# User can specify mariabackup specific settings that will be used during sst
@@ -663,13 +697,21 @@ parse_cnf()
# If the group name is the same as the "mysqld" without "--" prefix,
# then try to use it together with the group suffix:
if [ "$group" = 'mysqld' -a -n "$WSREP_SST_OPT_SUFFIX_VALUE" ]; then
- reval=$($MY_PRINT_DEFAULTS "mysqld$WSREP_SST_OPT_SUFFIX_VALUE" | awk "$pattern")
+ reval=$("$MY_PRINT_DEFAULTS" \
+ ${WSREP_SST_OPT_DEFAULT:+"$WSREP_SST_OPT_DEFAULT"} \
+ ${WSREP_SST_OPT_EXTRA_DEFAULT:+"$WSREP_SST_OPT_EXTRA_DEFAULT"} \
+ ${WSREP_SST_OPT_SUFFIX_DEFAULT:+"$WSREP_SST_OPT_SUFFIX_DEFAULT"} \
+ "mysqld$WSREP_SST_OPT_SUFFIX_VALUE" | awk "$pattern")
if [ -n "$reval" ]; then
break
fi
fi
# Let's try to use the group name as it is:
- reval=$($MY_PRINT_DEFAULTS "$group" | awk "$pattern")
+ reval=$("$MY_PRINT_DEFAULTS" \
+ ${WSREP_SST_OPT_DEFAULT:+"$WSREP_SST_OPT_DEFAULT"} \
+ ${WSREP_SST_OPT_EXTRA_DEFAULT:+"$WSREP_SST_OPT_EXTRA_DEFAULT"} \
+ ${WSREP_SST_OPT_SUFFIX_DEFAULT:+"$WSREP_SST_OPT_SUFFIX_DEFAULT"} \
+ "$group" | awk "$pattern")
if [ -n "$reval" ]; then
break
fi
@@ -710,13 +752,21 @@ in_config()
# If the group name is the same as the "mysqld" without "--" prefix,
# then try to use it together with the group suffix:
if [ "$group" = 'mysqld' -a -n "$WSREP_SST_OPT_SUFFIX_VALUE" ]; then
- found=$($MY_PRINT_DEFAULTS "mysqld$WSREP_SST_OPT_SUFFIX_VALUE" | awk "$pattern")
+ found=$("$MY_PRINT_DEFAULTS" \
+ ${WSREP_SST_OPT_DEFAULT:+"$WSREP_SST_OPT_DEFAULT"} \
+ ${WSREP_SST_OPT_EXTRA_DEFAULT:+"$WSREP_SST_OPT_EXTRA_DEFAULT"} \
+ ${WSREP_SST_OPT_SUFFIX_DEFAULT:+"$WSREP_SST_OPT_SUFFIX_DEFAULT"} \
+ "mysqld$WSREP_SST_OPT_SUFFIX_VALUE" | awk "$pattern")
if [ $found -ne 0 ]; then
break
fi
fi
# Let's try to use the group name as it is:
- found=$($MY_PRINT_DEFAULTS "$group" | awk "$pattern")
+ found=$($MY_PRINT_DEFAULTS \
+ ${WSREP_SST_OPT_DEFAULT:+"$WSREP_SST_OPT_DEFAULT"} \
+ ${WSREP_SST_OPT_EXTRA_DEFAULT:+"$WSREP_SST_OPT_EXTRA_DEFAULT"} \
+ ${WSREP_SST_OPT_SUFFIX_DEFAULT:+"$WSREP_SST_OPT_SUFFIX_DEFAULT"} \
+ "$group" | awk "$pattern")
if [ $found -ne 0 ]; then
break
fi
@@ -797,29 +847,6 @@ else
SST_PROGRESS_FILE=""
fi
-wsrep_log()
-{
- # echo everything to stderr so that it gets into common error log
- # deliberately made to look different from the rest of the log
- local readonly tst="$(date +%Y%m%d\ %H:%M:%S.%N | cut -b -21)"
- echo "WSREP_SST: $* ($tst)" >&2
-}
-
-wsrep_log_error()
-{
- wsrep_log "[ERROR] $*"
-}
-
-wsrep_log_warning()
-{
- wsrep_log "[WARNING] $*"
-}
-
-wsrep_log_info()
-{
- wsrep_log "[INFO] $*"
-}
-
wsrep_cleanup_progress_file()
{
[ -n "$SST_PROGRESS_FILE" -a \
@@ -960,35 +987,139 @@ check_sockets_utils()
}
#
+# Check if the port is in the "listen" state.
+# The first parameter is the PID of the process that should
+# listen on the port - if it is not known, you can specify
+# an empty string or zero.
+# The second parameter is the port number.
+# The third parameter is a list of the names of utilities
+# (via "|") that can listen on this port during the state
+# transfer.
+#
+check_port()
+{
+ local pid="$1"
+ local port="$2"
+ local utils="$3"
+
+ [ -z "$pid" ] || [ $pid -eq 0 ] && pid='[0-9]+'
+
+ local rc=1
+
+ if [ $lsof_available -ne 0 ]; then
+ lsof -Pnl -i ":$port" 2>/dev/null | \
+ grep -q -E "^($utils)[^[:space:]]*[[:space:]]+$pid[[:space:]].*\\(LISTEN\\)" && rc=0
+ elif [ $sockstat_available -ne 0 ]; then
+ sockstat -p "$port" 2>/dev/null | \
+ grep -q -E "[[:space:]]+($utils)[^[:space:]]*[[:space:]]+$pid[[:space:]].*[[:space:]]LISTEN" && rc=0
+ elif [ $ss_available -ne 0 ]; then
+ ss -nlpH "( sport = :$port )" 2>/dev/null | \
+ grep -q -E "users:\\(.*\\(\"($utils)[^[:space:]]*\"[^)]*,pid=$pid(,[^)]*)?\\)" && rc=0
+ else
+ wsrep_log_error "unknown sockets utility"
+ exit 2 # ENOENT
+ fi
+
+ return $rc
+}
+
+#
# If the ssl_dhparams variable is already set, uses that as a source
# of dh parameters for OpenSSL. Otherwise, looks for dhparams.pem in
# the datadir, and creates it there if it can't find the file.
#
check_for_dhparams()
{
- if [ -z "$ssl_dhparams" ]; then
- ssl_dhparams="$DATA/dhparams.pem"
- if [ ! -r "$ssl_dhparams" ]; then
- get_openssl
- if [ -n "$OPENSSL_BINARY" ]; then
- wsrep_log_info "Could not find dhparams file, creating $ssl_dhparams"
- if ! "$OPENSSL_BINARY" dhparam -out "$ssl_dhparams" 2048 >/dev/null 2>&1
- then
- wsrep_log_error "******** ERROR *****************************************"
- wsrep_log_error "* Could not create the dhparams.pem file with OpenSSL. *"
- wsrep_log_error "********************************************************"
- ssl_dhparams=""
- fi
- else
- # Rollback: if openssl is not installed, then use
- # the default parameters:
+ ssl_dhparams="$DATA/dhparams.pem"
+ if [ ! -r "$ssl_dhparams" ]; then
+ get_openssl
+ if [ -n "$OPENSSL_BINARY" ]; then
+ wsrep_log_info "Could not find dhparams file, creating $ssl_dhparams"
+ if ! "$OPENSSL_BINARY" dhparam -out "$ssl_dhparams" 2048 >/dev/null 2>&1
+ then
+ wsrep_log_error "******** ERROR *****************************************"
+ wsrep_log_error "* Could not create the dhparams.pem file with OpenSSL. *"
+ wsrep_log_error "********************************************************"
ssl_dhparams=""
- fi
+ fi
+ else
+ # Rollback: if openssl is not installed, then use
+ # the default parameters:
+ ssl_dhparams=""
fi
fi
}
#
+# Verifies that the CA file verifies the certificate.
+# Doing this here lets us generate better error messages.
+#
+# 1st param: path to the CA file.
+# 2nd param: path to the certificate.
+#
+verify_ca_matches_cert()
+{
+ local ca_path="$1"
+ local cert_path="$2"
+
+ # If the openssl utility is not installed, then
+ # we will not do this certificate check:
+ get_openssl
+ if [ -z "$OPENSSL_BINARY" ]; then
+ return
+ fi
+
+ if ! "$OPENSSL_BINARY" verify -verbose -CAfile "$ca_path" "$cert_path" >/dev/null 2>&1
+ then
+ wsrep_log_error "******** FATAL ERROR ********************************************"
+ wsrep_log_error "* The certifcate and CA (certificate authority) do not match. *"
+ wsrep_log_error "* It does not appear that the certificate was issued by the CA. *"
+ wsrep_log_error "* Please check your certificate and CA files. *"
+ wsrep_log_error "*****************************************************************"
+ exit 22
+ fi
+}
+
+#
+# Verifies that the certificate matches the private key.
+# Doing this will save us having to wait for a timeout that would
+# otherwise occur.
+#
+# 1st param: path to the certificate.
+# 2nd param: path to the private key.
+#
+verify_cert_matches_key()
+{
+ local cert_path="$1"
+ local key_path="$2"
+
+ # If the diff utility is not installed, then
+ # we will not do this certificate check:
+ if [ -z "$(command -v diff)" ]; then
+ return
+ fi
+
+ # If the openssl utility is not installed, then
+ # we will not do this certificate check:
+ get_openssl
+ if [ -z "$OPENSSL_BINARY" ]; then
+ return
+ fi
+
+ # Generate the public key from the cert and the key.
+ # They should match (otherwise we can't create an SSL connection).
+ if ! diff <("$OPENSSL_BINARY" x509 -in "$cert_path" -pubkey -noout 2>/dev/null) \
+ <("$OPENSSL_BINARY" pkey -in "$key_path" -pubout 2>/dev/null) >/dev/null 2>&1
+ then
+ wsrep_log_error "******************* FATAL ERROR ****************"
+ wsrep_log_error "* The certifcate and private key do not match. *"
+ wsrep_log_error "* Please check your certificate and key files. *"
+ wsrep_log_error "************************************************"
+ exit 22
+ fi
+}
+
+#
# Compares two version strings.
# The first parameter is the version to be checked;
# The second parameter is the minimum version required;
@@ -996,22 +1127,22 @@ check_for_dhparams()
#
check_for_version()
{
- y1=${1#*.}
+ y1="${1#*.}"
[ "$y1" = "$1" ] && y1=""
z1=${y1#*.}
[ "$z1" = "$y1" ] && z1=""
- x1=${1%%.*}
- y1=${y1%%.*}
- z1=${z1%%.*}
+ x1="${1%%.*}"
+ y1="${y1%%.*}"
+ z1="${z1%%.*}"
[ -z "$y1" ] && y1=0
[ -z "$z1" ] && z1=0
- y2=${2#*.}
+ y2="${2#*.}"
[ "$y2" = "$2" ] && y2=""
- z2=${y2#*.}
+ z2="${y2#*.}"
[ "$z2" = "$y2" ] && z2=""
- x2=${2%%.*}
- y2=${y2%%.*}
- z2=${z2%%.*}
+ x2="${2%%.*}"
+ y2="${y2%%.*}"
+ z2="${z2%%.*}"
[ -z "$y2" ] && y2=0
[ -z "$z2" ] && z2=0
[ $x1 -lt $x2 ] && return 1
@@ -1032,8 +1163,8 @@ trim_string()
if [ $x -ne $z ]; then
local y="${1%$pattern*}"
y=${#y}
- x=$(( $z-$x-1 ))
- y=$(( $y-$x+1 ))
+ x=$(( z-x-1 ))
+ y=$(( y-x+1 ))
printf '%s' "${1:$x:$y}"
else
printf ''
@@ -1043,3 +1174,105 @@ trim_string()
echo "$1" | sed -E "s/^$pattern+|$pattern+\$//g"
fi
}
+
+#
+# Check whether process is still running.
+# The first parameter contains the name of the PID file.
+# The second parameter is the flag of the need to delete
+# the PID file.
+# If the second parameter is not zero and not empty,
+# then if the process terminates, the corresponding
+# PID file will be deleted.
+# This function also sets the CHECK_PID variable to zero
+# if the process has already exited, or writes the PID
+# of the process there if it is still running.
+#
+check_pid()
+{
+ local pid_file="$1"
+ local remove=${2:-0}
+ if [ -r "$pid_file" ]; then
+ local pid=$(cat "$pid_file" 2>/dev/null)
+ if [ -n "$pid" ]; then
+ if [ $pid -ne 0 ]; then
+ if ps -p "$pid" >/dev/null 2>&1; then
+ CHECK_PID=$pid
+ return 0
+ fi
+ fi
+ fi
+ if [ $remove -eq 1 ]; then
+ rm -f "$pid_file"
+ fi
+ fi
+ CHECK_PID=0
+ return 1
+}
+
+#
+# Checking that the process with the specified PID is still
+# running and killing it in this case by sending SIGTERM
+# (using the "kill" operation).
+# The first parameter contains PID of the process.
+# The second and third parameters (both optional) are the names
+# of the PID and the configuration files, which should be removed
+# after the process ends.
+# If the first parameter (PID of the process) is zero, then
+# the function immediately deletes the PID and the configuration
+# files (if specified), without any additional checks.
+#
+cleanup_pid()
+{
+ local pid="$1"
+ local pid_file="${2:-}"
+ local config="${3:-}"
+
+ if [ $pid -ne 0 ]; then
+ if ps -p $pid >/dev/null 2>&1; then
+ if kill $pid >/dev/null 2>&1; then
+ sleep 0.5
+ local round=0
+ local force=0
+ while ps -p $pid >/dev/null 2>&1; do
+ sleep 1
+ round=$(( round+1 ))
+ if [ $round -eq 16 ]; then
+ if [ $force -eq 0 ]; then
+ round=8
+ force=1
+ kill -9 $pid >/dev/null 2>&1
+ else
+ return 1;
+ fi
+ fi
+ done
+ elif ps -p $pid >/dev/null 2>&1; then
+ wsrep_log_warning "Unable to kill PID=$pid ($pid_file)"
+ return 1
+ fi
+ fi
+ fi
+
+ [ -n "$pid_file" ] && [ -f "$pid_file" ] && rm -f "$pid_file"
+ [ -n "$config" ] && [ -f "$config" ] && rm -f "$config"
+
+ return 0
+}
+
+nproc=""
+
+get_proc()
+{
+ if [ -z "$nproc" ]; then
+ set +e
+ if [ "$OS" = 'Linux' ]; then
+ nproc=$(grep -c processor /proc/cpuinfo 2>/dev/null)
+ elif [ "$OS" = 'Darwin' -o "$OS" = 'FreeBSD' ]; then
+ nproc=$(sysctl -n hw.ncpu)
+ fi
+ if [ -z "$nproc" ] || [ $nproc -eq 0 ]; then
+ nproc=1
+ fi
+ set -e
+ fi
+}
diff --git a/scripts/wsrep_sst_mariabackup.sh b/scripts/wsrep_sst_mariabackup.sh
index 0fdc765ff8b..7f97d9e8dea 100644
--- a/scripts/wsrep_sst_mariabackup.sh
+++ b/scripts/wsrep_sst_mariabackup.sh
@@ -29,11 +29,10 @@ eformat=""
ekey=""
ekeyfile=""
encrypt=0
-nproc=1
ecode=0
ssyslog=""
ssystag=""
-MARIABACKUP_PID=""
+BACKUP_PID=""
tcert=""
tpem=""
tkey=""
@@ -77,6 +76,11 @@ compress='none'
compress_chunk=""
compress_threads=""
+backup_threads=""
+
+encrypt_threads=""
+encrypt_chunk=""
+
readonly SECRET_TAG="secret"
# Required for backup locks
@@ -90,8 +94,8 @@ fi
pcmd="pv $pvopts"
declare -a RC
-MARIABACKUP_BIN="$(command -v mariabackup)"
-if [ ! -x "$MARIABACKUP_BIN" ]; then
+BACKUP_BIN="$(command -v mariabackup)"
+if [ ! -x "$BACKUP_BIN" ]; then
wsrep_log_error 'mariabackup binary not found in path'
exit 42
fi
@@ -108,7 +112,8 @@ INNOBACKUPLOG="$DATA/mariabackup.backup.log"
# Setting the path for ss and ip
export PATH="/usr/sbin:/sbin:$PATH"
-timeit(){
+timeit()
+{
local stage="$1"
shift
local cmd="$@"
@@ -198,6 +203,12 @@ get_keys()
else
ecmd="xbcrypt --encrypt-algo='$ealgo' --encrypt-key='$ekey'"
fi
+ if [ -n "$encrypt_threads" ]; then
+ ecmd="$ecmd --encrypt-threads=$encrypt_threads"
+ fi
+ if [ -n "$encrypt_chunk" ]; then
+ ecmd="$ecmd --encrypt-chunk-size=$encrypt_chunk"
+ fi
else
wsrep_log_error "Unknown encryption format='$eformat'"
exit 2
@@ -298,15 +309,6 @@ get_transfer()
exit 2
fi
- # Determine the socat version
- SOCAT_VERSION=$(socat -V 2>&1 | grep -m1 -oe '[0-9]\.[0-9][\.0-9]*')
- if [ -z "$SOCAT_VERSION" ]; then
- wsrep_log_error "******** FATAL ERROR ******************"
- wsrep_log_error "* Cannot determine the socat version. *"
- wsrep_log_error "***************************************"
- exit 2
- fi
-
local action='Decrypting'
if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
tcmd="socat -u openssl-listen:$SST_PORT,reuseaddr"
@@ -315,10 +317,22 @@ get_transfer()
action='Encrypting'
fi
- if ! check_for_version "$SOCAT_VERSION" '1.7.3'; then
- # socat versions < 1.7.3 will have 512-bit dhparams (too small)
- # so create 2048-bit dhparams and send that as a parameter:
- check_for_dhparams
+ if [ "${sockopt#*,dhparam=}" != "$sockopt" ]; then
+ if [ -z "$ssl_dhparams" ]; then
+ # Determine the socat version
+ SOCAT_VERSION=$(socat -V 2>&1 | grep -m1 -oe '[0-9]\.[0-9][\.0-9]*')
+ if [ -z "$SOCAT_VERSION" ]; then
+ wsrep_log_error "******** FATAL ERROR ******************"
+ wsrep_log_error "* Cannot determine the socat version. *"
+ wsrep_log_error "***************************************"
+ exit 2
+ fi
+ if ! check_for_version "$SOCAT_VERSION" '1.7.3'; then
+ # socat versions < 1.7.3 will have 512-bit dhparams (too small)
+ # so create 2048-bit dhparams and send that as a parameter:
+ check_for_dhparams
+ fi
+ fi
if [ -n "$ssl_dhparams" ]; then
tcmd="$tcmd,dhparam='$ssl_dhparams'"
fi
@@ -330,6 +344,11 @@ get_transfer()
wsrep_log_error "Both PEM and CRT files required"
exit 22
fi
+ if [ ! -r "$tpem" -o ! -r "$tcert" ]; then
+ wsrep_log_error "Both PEM and CRT files must be readable"
+ exit 22
+ fi
+ verify_ca_matches_cert "$tcert" "$tpem"
tcmd="$tcmd,cert='$tpem',cafile='$tcert'$sockopt"
stagemsg="$stagemsg-OpenSSL-Encrypted-2"
wsrep_log_info "$action with cert=$tpem, cafile=$tcert"
@@ -339,6 +358,11 @@ get_transfer()
wsrep_log_error "Both certificate and key files required"
exit 22
fi
+ if [ ! -r "$tpem" -o ! -r "$tkey" ]; then
+ wsrep_log_error "Both certificate and key files must be readable"
+ exit 22
+ fi
+ verify_cert_matches_key "$tpem" "$tkey"
stagemsg="$stagemsg-OpenSSL-Encrypted-3"
if [ -z "$tcert" ]; then
if [ $encrypt -eq 4 ]; then
@@ -350,6 +374,11 @@ get_transfer()
wsrep_log_info "$action with cert=$tpem, key=$tkey, verify=0"
else
# CA verification
+ if [ ! -r "$tcert" ]; then
+ wsrep_log_error "Certificate file must be readable"
+ exit 22
+ fi
+ verify_ca_matches_cert "$tcert" "$tpem"
if [ -n "$WSREP_SST_OPT_REMOTE_USER" ]; then
CN_option=",commonname='$WSREP_SST_OPT_REMOTE_USER'"
elif [ $encrypt -eq 4 ]; then
@@ -440,10 +469,10 @@ read_cnf()
tpem=$(parse_cnf 'sst' 'tcert')
tkey=$(parse_cnf 'sst' 'tkey')
fi
- if [ "$tmode" != 'DISABLED' ]
- then # backward-incompatible behavior
- if [ -z "$tpem" -a -z "$tkey" -a -z "$tcert" ]
- then # no old-style SSL config in [sst]
+ if [ "$tmode" != 'DISABLED' ]; then
+ # backward-incompatible behavior
+ if [ -z "$tpem" -a -z "$tkey" -a -z "$tcert" ]; then
+ # no old-style SSL config in [sst]
check_server_ssl_config
fi
if [ 0 -eq $encrypt -a -n "$tpem" -a -n "$tkey" ]
@@ -504,21 +533,28 @@ read_cnf()
compress_threads=$(parse_cnf "$encgroups" 'compress-threads')
fi
fi
+
+ backup_threads=$(parse_cnf "$encgroups" 'backup-threads')
+
+ if [ "$eformat" = 'xbcrypt' ]; then
+ encrypt_threads=$(parse_cnf "$encgroups" 'encrypt-threads')
+ encrypt_chunk=$(parse_cnf "$encgroups" 'encrypt-chunk-size')
+ fi
}
get_stream()
{
if [ "$sfmt" = 'mbstream' -o "$sfmt" = 'xbstream' ]; then
sfmt='mbstream'
- MBSTREAM_BIN="$(command -v mbstream)"
- if [ -z "$MBSTREAM_BIN" ]; then
+ STREAM_BIN="$(command -v mbstream)"
+ if [ -z "$STREAM_BIN" ]; then
wsrep_log_error "Streaming with $sfmt, but $sfmt not found in path"
exit 42
fi
if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
- strmcmd="'$MBSTREAM_BIN' -x"
+ strmcmd="'$STREAM_BIN' -x"
else
- strmcmd="'$MBSTREAM_BIN' -c '$INFO_FILE'"
+ strmcmd="'$STREAM_BIN' -c '$INFO_FILE'"
fi
else
sfmt='tar'
@@ -531,79 +567,33 @@ get_stream()
wsrep_log_info "Streaming with $sfmt"
}
-get_proc()
-{
- set +e
- nproc=$(grep -c processor /proc/cpuinfo)
- [ -z $nproc -o $nproc -eq 0 ] && nproc=1
- set -e
-}
-
sig_joiner_cleanup()
{
wsrep_log_error "Removing $MAGIC_FILE file due to signal"
[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
}
-cleanup_joiner()
+cleanup_at_exit()
{
# Since this is invoked just after exit NNN
local estatus=$?
if [ $estatus -ne 0 ]; then
wsrep_log_error "Cleanup after exit with status:$estatus"
- elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
- wsrep_log_info "Removing the sst_in_progress file"
- wsrep_cleanup_progress_file
- fi
- if [ -n "$progress" -a -p "$progress" ]; then
- wsrep_log_info "Cleaning up fifo file $progress"
- rm "$progress"
- fi
-
- if [ -n "$STATDIR" ]; then
- [ -d "$STATDIR" ] && rm -rf "$STATDIR"
fi
- # Final cleanup
- pgid=$(ps -o pgid= $$ | grep -o '[0-9]*')
-
- # This means no setsid done in mysqld.
- # We don't want to kill mysqld here otherwise.
- if [ $$ -eq $pgid ]; then
- # This means a signal was delivered to the process.
- # So, more cleanup.
- if [ $estatus -ge 128 ]; then
- kill -KILL -$$ || true
- fi
- fi
-
- exit $estatus
-}
-
-check_pid()
-{
- local pid_file="$1"
- [ -r "$pid_file" ] && ps -p $(cat "$pid_file") 2>&1 >/dev/null
-}
-
-cleanup_donor()
-{
- # Since this is invoked just after exit NNN
- local estatus=$?
- if [ $estatus -ne 0 ]; then
- wsrep_log_error "Cleanup after exit with status:$estatus"
- fi
-
- if [ -n "$MARIABACKUP_PID" ]; then
- if check_pid $MARIABACKUP_PID
- then
- wsrep_log_error "mariabackup process is still running. Killing..."
- kill_mariabackup
+ if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
+ wsrep_log_info "Removing the sst_in_progress file"
+ wsrep_cleanup_progress_file
+ else
+ if [ -n "$BACKUP_PID" ]; then
+ if check_pid "$BACKUP_PID" 1; then
+ wsrep_log_error "mariabackup process is still running. Killing..."
+ cleanup_pid $CHECK_PID "$BACKUP_PID"
+ fi
fi
+ [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE"
fi
- [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE"
-
if [ -n "$progress" -a -p "$progress" ]; then
wsrep_log_info "Cleaning up fifo file $progress"
rm -f "$progress" || true
@@ -611,8 +601,14 @@ cleanup_donor()
wsrep_log_info "Cleaning up temporary directories"
- [ -n "$xtmpdir" -a -d "$xtmpdir" ] && rm -rf "$xtmpdir" || true
- [ -n "$itmpdir" -a -d "$itmpdir" ] && rm -rf "$itmpdir" || true
+ if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
+ if [ -n "$STATDIR" ]; then
+ [ -d "$STATDIR" ] && rm -rf "$STATDIR"
+ fi
+ else
+ [ -n "$xtmpdir" -a -d "$xtmpdir" ] && rm -rf "$xtmpdir" || true
+ [ -n "$itmpdir" -a -d "$itmpdir" ] && rm -rf "$itmpdir" || true
+ fi
# Final cleanup
pgid=$(ps -o pgid= $$ | grep -o '[0-9]*')
@@ -623,21 +619,13 @@ cleanup_donor()
# This means a signal was delivered to the process.
# So, more cleanup.
if [ $estatus -ge 128 ]; then
- kill -KILL -$$ || true
+ kill -KILL -- -$$ || true
fi
fi
exit $estatus
}
-kill_mariabackup()
-{
- local PID=$(cat "$MARIABACKUP_PID")
- [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || :
- wsrep_log_info "Removing mariabackup pid file ($MARIABACKUP_PID)"
- rm -f "$MARIABACKUP_PID" || true
-}
-
setup_ports()
{
SST_PORT="$WSREP_SST_OPT_PORT"
@@ -648,51 +636,17 @@ setup_ports()
fi
}
-check_port()
-{
- local PORT="$1"
- local UTILS="$2"
-
- local port_info is_util
-
- if [ $lsof_available -ne 0 ]; then
- port_info=$(lsof -i ":$PORT" -Pn 2>/dev/null | \
- grep -F '(LISTEN)')
- is_util=$(echo "$port_info" | \
- grep -E "^($UTILS)[^[:space:]]*[[:space:]]+[0-9]+[[:space:]]+")
- elif [ $sockstat_available -ne 0 ]; then
- port_info=$(sockstat -p "$PORT" 2>/dev/null | \
- grep -F 'LISTEN')
- is_util=$(echo "$port_info" | \
- grep -E "[[:space:]]+($UTILS)[^[:space:]]*[[:space:]]+[0-9]+[[:space:]]+")
- elif [ $ss_available -ne 0 ]; then
- port_info=$(ss -H -p -n -l "( sport = :$PORT )" 2>/dev/null)
- is_util=$(echo "$port_info" | \
- grep -E "users:\\(.*\\(\"($UTILS)[^[:space:]]*\".*\<pid=[0-9]+\>.*\\)")
- else
- wsrep_log_error "unknown sockets utility"
- exit 2 # ENOENT
- fi
-
- if [ -z "$is_util" ]; then
- return 1
- fi
-
- return 0
-}
-
-# waits ~10 seconds for nc to open the port and then reports ready
-# (regardless of timeout)
+#
+# Waits ~30 seconds for socat or nc to open the port and
+# then reports ready, regardless of timeout.
+#
wait_for_listen()
{
local PORT="$1"
local ADDR="$2"
local MODULE="$3"
-
- for i in {1..50}
- do
- if check_port "$PORT" 'socat|nc'
- then
+ for i in {1..150}; do
+ if check_port "" "$PORT" 'socat|nc'; then
break
fi
sleep 0.2
@@ -708,8 +662,8 @@ check_extra()
if [ "$thread_handling" = 'pool-of-threads' ]; then
local eport=$(parse_cnf '--mysqld' 'extra-port')
if [ -n "$eport" ]; then
- # mariabackup works only locally, hence,
- # setting host to 127.0.0.1 unconditionally:
+ # mariabackup works only locally.
+ # Hence, setting host to 127.0.0.1 unconditionally:
wsrep_log_info "SST through extra_port $eport"
INNOEXTRA="$INNOEXTRA --host=127.0.0.1 --port=$eport"
use_socket=0
@@ -825,11 +779,12 @@ monitor_process()
local sst_stream_pid=$1
while true ; do
- if ! ps -p "$WSREP_SST_OPT_PARENT" &>/dev/null; then
+ if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1; then
wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT) terminated unexpectedly."
+ kill -- -"$WSREP_SST_OPT_PARENT"
exit 32
fi
- if ! ps -p "$sst_stream_pid" &>/dev/null; then
+ if ! ps -p "$sst_stream_pid" >/dev/null 2>&1; then
break
fi
sleep 0.1
@@ -839,14 +794,14 @@ monitor_process()
[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a "$WSREP_SST_OPT_ROLE" != 'donor' ]; then
- wsrep_log_error "Invalid role ${WSREP_SST_OPT_ROLE}"
+ wsrep_log_error "Invalid role '$WSREP_SST_OPT_ROLE'"
exit 22
fi
read_cnf
setup_ports
-if "$MARIABACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then
+if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then
disver='--no-version-check'
fi
@@ -942,8 +897,8 @@ else
gzip "$newfile"
fi
fi
- INNOAPPLY="&> '$INNOAPPLYLOG'"
- INNOMOVE="&> '$INNOMOVELOG'"
+ INNOAPPLY="> '$INNOAPPLYLOG' 2>&1"
+ INNOMOVE="> '$INNOMOVELOG' 2>&1"
INNOBACKUP="2> '$INNOBACKUPLOG'"
fi
@@ -953,9 +908,9 @@ setup_commands()
if [ -n "$WSREP_SST_OPT_MYSQLD" ]; then
mysqld_args="--mysqld-args $WSREP_SST_OPT_MYSQLD"
fi
- INNOAPPLY="$MARIABACKUP_BIN --prepare $disver $iapts $INNOEXTRA --target-dir='$DATA' --datadir='$DATA' $mysqld_args $INNOAPPLY"
- INNOMOVE="$MARIABACKUP_BIN $WSREP_SST_OPT_CONF --move-back $disver $impts --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE"
- INNOBACKUP="$MARIABACKUP_BIN $WSREP_SST_OPT_CONF --backup $disver $iopts $tmpopts $INNOEXTRA --galera-info --stream=$sfmt --target-dir='$itmpdir' --datadir='$DATA' $mysqld_args $INNOBACKUP"
+ INNOAPPLY="$BACKUP_BIN --prepare $disver $iapts $INNOEXTRA --target-dir='$DATA' --datadir='$DATA' $mysqld_args $INNOAPPLY"
+ INNOMOVE="$BACKUP_BIN $WSREP_SST_OPT_CONF --move-back $disver $impts --force-non-empty-directories --target-dir='$DATA' --datadir='${TDATA:-$DATA}' $INNOMOVE"
+ INNOBACKUP="$BACKUP_BIN $WSREP_SST_OPT_CONF --backup $disver $iopts $tmpopts $INNOEXTRA --galera-info --stream=$sfmt --target-dir='$itmpdir' --datadir='$DATA' $mysqld_args $INNOBACKUP"
}
get_stream
@@ -963,7 +918,7 @@ get_transfer
if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]
then
- trap cleanup_donor EXIT
+ trap cleanup_at_exit EXIT
if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
then
@@ -976,12 +931,15 @@ then
tmpdir=$(parse_cnf "$encgroups" 'tmpdir')
if [ -z "$tmpdir" ]; then
xtmpdir="$(mktemp -d)"
- tmpopts="--tmpdir='$xtmpdir'"
- wsrep_log_info "Using $xtmpdir as mariabackup temporary directory"
+ else
+ xtmpdir=$(mktemp '-d' "--tmpdir=$tmpdir")
fi
+ wsrep_log_info "Using '$xtmpdir' as mariabackup temporary directory"
+ tmpopts="--tmpdir='$xtmpdir'"
+
itmpdir="$(mktemp -d)"
- wsrep_log_info "Using $itmpdir as mariabackup temporary directory"
+ wsrep_log_info "Using '$itmpdir' as mariabackup working directory"
usrst=0
if [ -n "$WSREP_SST_OPT_USER" ]; then
@@ -1048,25 +1006,29 @@ then
tcmd="$ecmd | $tcmd"
fi
- iopts="$iopts --databases-exclude='lost+found'"
+ iopts="--databases-exclude='lost+found' $iopts"
if [ ${FORCE_FTWRL:-0} -eq 1 ]; then
wsrep_log_info "Forcing FTWRL due to environment variable FORCE_FTWRL equal to $FORCE_FTWRL"
- iopts="$iopts --no-backup-locks"
+ iopts="--no-backup-locks $iopts"
fi
# if compression is enabled for backup files, then add the
# appropriate options to the mariabackup command line:
if [ "$compress" != 'none' ]; then
- iopts="$iopts --compress${compress:+=$compress}"
+ iopts="--compress${compress:+=$compress} $iopts"
if [ -n "$compress_threads" ]; then
- iopts="$iopts --compress-threads=$compress_threads"
+ iopts="--compress-threads=$compress_threads $iopts"
fi
if [ -n "$compress_chunk" ]; then
- iopts="$iopts --compress-chunk-size=$compress_chunk"
+ iopts="--compress-chunk-size=$compress_chunk $iopts"
fi
fi
+ if [ -n "$backup_threads" ]; then
+ iopts="--parallel=$backup_threads $iopts"
+ fi
+
setup_commands
set +e
timeit "$stagemsg-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )"
@@ -1082,7 +1044,7 @@ then
fi
# mariabackup implicitly writes PID to fixed location in $xtmpdir
- MARIABACKUP_PID="$xtmpdir/xtrabackup_pid"
+ BACKUP_PID="$xtmpdir/xtrabackup_pid"
else # BYPASS FOR IST
@@ -1134,6 +1096,10 @@ then
ib_undo_dir="$INNODB_UNDO_DIR"
+ if [ -n "$backup_threads" ]; then
+ impts="--parallel=$backup_threads $impts"
+ fi
+
stagemsg='Joiner-Recv'
sencrypted=1
@@ -1173,7 +1139,7 @@ then
fi
trap sig_joiner_cleanup HUP PIPE INT TERM
- trap cleanup_joiner EXIT
+ trap cleanup_at_exit EXIT
if [ -n "$progress" ]; then
adjust_progress
@@ -1196,7 +1162,7 @@ then
recv_joiner "$STATDIR" "$stagemsg-gtid" $stimeout 1 1
- if ! ps -p "$WSREP_SST_OPT_PARENT" &>/dev/null
+ if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1
then
wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT) terminated unexpectedly."
exit 32
@@ -1316,7 +1282,7 @@ then
fi
- wsrep_log_info "Preparing the backup at ${DATA}"
+ wsrep_log_info "Preparing the backup at $DATA"
setup_commands
timeit "mariabackup prepare stage" "$INNOAPPLY"
@@ -1327,26 +1293,26 @@ then
MAGIC_FILE="$TDATA/$INFO_FILE"
- wsrep_log_info "Moving the backup to ${TDATA}"
+ wsrep_log_info "Moving the backup to $TDATA"
timeit "mariabackup move stage" "$INNOMOVE"
if [ $? -eq 0 ]; then
- wsrep_log_info "Move successful, removing ${DATA}"
+ wsrep_log_info "Move successful, removing $DATA"
rm -rf "$DATA"
DATA="$TDATA"
else
- wsrep_log_error "Move failed, keeping ${DATA} for further diagnosis"
+ wsrep_log_error "Move failed, keeping '$DATA' for further diagnosis"
wsrep_log_error "Check syslog or '$INNOMOVELOG' for details"
exit 22
fi
else
- wsrep_log_info "${IST_FILE} received from donor: Running IST"
+ wsrep_log_info "'$IST_FILE' received from donor: Running IST"
fi
if [ ! -r "$MAGIC_FILE" ]; then
- wsrep_log_error "SST magic file ${MAGIC_FILE} not found/readable"
+ wsrep_log_error "SST magic file '$MAGIC_FILE' not found/readable"
exit 2
fi
diff --git a/scripts/wsrep_sst_mysqldump.sh b/scripts/wsrep_sst_mysqldump.sh
index 4aa3f8e63d8..798bee1ac10 100644
--- a/scripts/wsrep_sst_mysqldump.sh
+++ b/scripts/wsrep_sst_mysqldump.sh
@@ -103,7 +103,7 @@ then
DROP PREPARE stmt;"
fi
-MYSQL="$MYSQL_CLIENT $WSREP_SST_OPT_CONF "\
+MYSQL="$MYSQL_CLIENT $WSREP_SST_OPT_CONF_UNQUOTED "\
"$AUTH -h$WSREP_SST_OPT_HOST_UNESCAPED "\
"-P$WSREP_SST_OPT_PORT --disable-reconnect --connect_timeout=10"
@@ -140,7 +140,7 @@ then
fi
# NOTE: we don't use --routines here because we're dumping mysql.proc table
-MYSQLDUMP="$MYSQLDUMP $WSREP_SST_OPT_CONF $AUTH -S$WSREP_SST_OPT_SOCKET \
+MYSQLDUMP="$MYSQLDUMP $WSREP_SST_OPT_CONF_UNQUOTED $AUTH -S$WSREP_SST_OPT_SOCKET \
--add-drop-database --add-drop-table --skip-add-locks --create-options \
--disable-keys --extended-insert --skip-lock-tables --quick --set-charset \
--skip-comments --flush-privileges --all-databases --events"
diff --git a/scripts/wsrep_sst_rsync.sh b/scripts/wsrep_sst_rsync.sh
index 0cc13ee4191..19a4d19fded 100644
--- a/scripts/wsrep_sst_rsync.sh
+++ b/scripts/wsrep_sst_rsync.sh
@@ -1,7 +1,7 @@
#!/bin/bash -ue
-# Copyright (C) 2010-2014 Codership Oy
# Copyright (C) 2017-2021 MariaDB
+# Copyright (C) 2010-2014 Codership Oy
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -19,9 +19,8 @@
# This is a reference script for rsync-based state snapshot tansfer
-RSYNC_PID= # rsync pid file
-RSYNC_CONF= # rsync configuration file
-RSYNC_REAL_PID= # rsync process id
+RSYNC_REAL_PID=0 # rsync process id
+STUNNEL_REAL_PID=0 # stunnel process id
OS="$(uname)"
[ "$OS" = 'Darwin' ] && export -n LD_LIBRARY_PATH
@@ -36,95 +35,95 @@ wsrep_check_programs rsync
cleanup_joiner()
{
- wsrep_log_info "Joiner cleanup. rsync PID: $RSYNC_REAL_PID"
- [ "0" != "$RSYNC_REAL_PID" ] && \
- kill $RSYNC_REAL_PID && \
- sleep 0.5 && \
- kill -9 $RSYNC_REAL_PID >/dev/null 2>&1 || :
- [ -f "$RSYNC_CONF" ] && rm -f "$RSYNC_CONF"
- [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF"
- [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID"
- [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
- [ -f "$RSYNC_PID" ] && rm -f "$RSYNC_PID"
+ local failure=0
+
+ wsrep_log_info "Joiner cleanup: rsync PID=$RSYNC_REAL_PID, stunnel PID=$STUNNEL_REAL_PID"
+
+ if [ -n "$STUNNEL" ]; then
+ if cleanup_pid $STUNNEL_REAL_PID "$STUNNEL_PID" "$STUNNEL_CONF"; then
+ if [ $RSYNC_REAL_PID -eq 0 ]; then
+ if [ -r "$RSYNC_PID" ]; then
+ RSYNC_REAL_PID=$(cat "$RSYNC_PID" 2>/dev/null)
+ if [ -z "$RSYNC_REAL_PID" ]; then
+ RSYNC_REAL_PID=0
+ fi
+ fi
+ fi
+ else
+ wsrep_log_warning "stunnel cleanup failed."
+ failure=1
+ fi
+ fi
+
+ if [ $failure -eq 0 ]; then
+ if cleanup_pid $RSYNC_REAL_PID "$RSYNC_PID" "$RSYNC_CONF"; then
+ [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
+ else
+ wsrep_log_warning "rsync cleanup failed."
+ fi
+ fi
+
wsrep_log_info "Joiner cleanup done."
+
if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
wsrep_cleanup_progress_file
fi
}
-# Check whether rsync process is still running.
-check_pid()
-{
- local pid_file="$1"
- [ -r "$pid_file" ] && ps -p $(cat "$pid_file") 2>&1 >/dev/null
-}
-
check_pid_and_port()
{
local pid_file="$1"
- local rsync_pid=$2
- local rsync_addr="$3"
- local rsync_port="$4"
+ local pid=$2
+ local addr="$3"
+ local port="$4"
- if [ -z "$rsync_port" -o -z "$rsync_addr" -o -z "$rsync_pid" ]; then
- wsrep_log_error "check_pid_and_port(): bad arguments"
- exit 2 # ENOENT
- fi
+ local utils='rsync|stunnel'
- local port_info is_rsync
-
- if [ $lsof_available -ne 0 ]; then
- port_info=$(lsof -i ":$rsync_port" -Pn 2>/dev/null | \
- grep -F '(LISTEN)')
- is_rsync=$(echo "$port_info" | \
- grep -E "^(rsync|stunnel)[^[:space:]]*[[:space:]]+$rsync_pid[[:space:]]+")
- elif [ $sockstat_available -ne 0 ]; then
- port_info=$(sockstat -p "$rsync_port" 2>/dev/null | \
- grep -F 'LISTEN')
- is_rsync=$(echo "$port_info" | \
- grep -E "[[:space:]]+(rsync|stunnel)[^[:space:]]*[[:space:]]+$rsync_pid[[:space:]]+")
- elif [ $ss_available -ne 0 ]; then
- port_info=$(ss -H -p -n -l "( sport = :$rsync_port )" 2>/dev/null)
- is_rsync=$(echo "$port_info" | \
- grep -E "users:\\(.*\\(\"(rsync|stunnel)[^[:space:]]*\".*\<pid=$rsync_pid\>.*\\)")
- else
- wsrep_log_error "unknown sockets utility"
- exit 2 # ENOENT
- fi
+ if ! check_port "$pid" "$port" "$utils"; then
+ local port_info
+ local busy=0
- if [ -z "$is_rsync" ]; then
- local is_listening_all
if [ $lsof_available -ne 0 ]; then
- is_listening_all=$(echo "$port_info" | \
- grep -E "[[:space:]](\\*|\\[?::\\]?):$rsync_port[[:space:]]")
+ port_info=$(lsof -Pnl -i ":$port" 2>/dev/null | \
+ grep -F '(LISTEN)')
+ echo "$port_info" | \
+ grep -q -E "[[:space:]](\\*|\\[?::\\]?):$port[[:space:]]" && busy=1
else
- if [ $sockstat_available -eq 0 ]; then
- port_info=$(echo "$port_info" | grep -q -F 'users:(')
+ local filter='([^[:space:]]+[[:space:]]+){4}[^[:space:]]+'
+ if [ $sockstat_available -eq 1 ]; then
+ port_info=$(sockstat -p "$port" 2>/dev/null | \
+ grep -E '[[:space:]]LISTEN' | grep -o -E "$filter")
+ else
+ port_info=$(ss -nlpH "( sport = :$port )" 2>/dev/null | \
+ grep -F 'users:(' | grep -o -E "$filter")
fi
- port_info=$(echo "$port_info" | \
- grep -E "[^[:space:]]+[[:space:]]+[^[:space:]]+[[:space:]]+[^[:space:]]+[[:space:]]+[^[:space:]]+[[:space:]]+[^[:space:]]+" -o)
- is_listening_all=$(echo "$port_info" | \
- grep -E "[[:space:]](\\*|\\[?::\\]?):$rsync_port\$")
+ echo "$port_info" | \
+ grep -q -E "[[:space:]](\\*|\\[?::\\]?):$port\$" && busy=1
fi
- local is_listening_addr=$(echo "$port_info" | \
- grep -w -F -- "$rsync_addr:$rsync_port")
- if [ -z "$is_listening_addr" ]; then
- is_listening_addr=$(echo "$port_info" | \
- grep -w -F "[$rsync_addr]:$rsync_port")
+
+ if [ $busy -eq 0 ]; then
+ if echo "$port_info" | grep -qw -F "[$addr]:$port" || \
+ echo "$port_info" | grep -qw -F -- "$addr:$port"
+ then
+ busy=1
+ fi
fi
- if [ -n "$is_listening_all" -o -n "$is_listening_addr" ]; then
- wsrep_log_error "rsync or stunnel daemon port '$rsync_port' " \
+
+ if [ $busy -eq 0 ]; then
+ return 1
+ fi
+
+ if ! check_port "$pid" "$port" "$utils"; then
+ wsrep_log_error "rsync or stunnel daemon port '$port' " \
"has been taken by another program"
exit 16 # EBUSY
fi
- return 1
fi
- check_pid "$pid_file" && [ $(cat "$pid_file") -eq $rsync_pid ]
+ check_pid "$pid_file" && [ $CHECK_PID -eq $pid ]
}
STUNNEL_CONF="$WSREP_SST_OPT_DATA/stunnel.conf"
-
STUNNEL_PID="$WSREP_SST_OPT_DATA/stunnel.pid"
MAGIC_FILE="$WSREP_SST_OPT_DATA/rsync_sst_complete"
@@ -201,6 +200,8 @@ FILTER="-f '- /lost+found'
-f '- /.zfs'
-f '- /.fseventsd'
-f '- /.Trashes'
+ -f '- /.pid'
+ -f '- /.conf'
-f '+ /wsrep_sst_binlog.tar'
-f '- $INNODB_DATA_HOME_DIR/ib_lru_dump'
-f '- $INNODB_DATA_HOME_DIR/ibdata*'
@@ -266,7 +267,7 @@ then
else
# check if the address is an ip-address (v4 or v6):
if echo "$WSREP_SST_OPT_HOST_UNESCAPED" | \
- grep -q -E '^([0-9]+(\.[0-9]+){3,3}|[0-9a-fA-F]*(\:[0-9a-fA-F]*)+)$'
+ grep -q -E '^([0-9]+(\.[0-9]+){3}|[0-9a-fA-F]*(\:[0-9a-fA-F]*)+)$'
then
CHECK_OPT="checkIP = $WSREP_SST_OPT_HOST_UNESCAPED"
else
@@ -303,10 +304,10 @@ then
[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
[ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE"
+ [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID"
if [ -n "$STUNNEL" ]
then
- [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID"
cat << EOF > "$STUNNEL_CONF"
key = $SSTKEY
cert = $SSTCERT
@@ -321,6 +322,8 @@ ${VERIFY_OPT}
${CHECK_OPT}
${CHECK_OPT_LOCAL}
EOF
+ else
+ [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF"
fi
if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
@@ -329,13 +332,8 @@ EOF
FLUSHED="$WSREP_SST_OPT_DATA/tables_flushed"
ERROR="$WSREP_SST_OPT_DATA/sst_error"
- rm -rf "$FLUSHED"
- rm -rf "$ERROR"
-
- # Use deltaxfer only for WAN
- inv=$(basename "$0")
- [ "$inv" = "wsrep_sst_rsync_wan" ] && WHOLE_FILE_OPT="" \
- || WHOLE_FILE_OPT="--whole-file"
+ [ -f "$FLUSHED" ] && rm -f "$FLUSHED"
+ [ -f "$ERROR" ] && rm -f "$ERROR"
echo "flush tables"
@@ -350,15 +348,14 @@ EOF
if [ -f "$ERROR" ]
then
# Flush tables operation failed.
- rm -rf "$ERROR"
+ rm -f "$ERROR"
exit 255
fi
-
sleep 0.2
done
STATE=$(cat "$FLUSHED")
- rm -rf "$FLUSHED"
+ rm -f "$FLUSHED"
sync
@@ -385,6 +382,13 @@ EOF
cd "$OLD_PWD"
fi
+ # Use deltaxfer only for WAN
+ inv=$(basename "$0")
+ WHOLE_FILE_OPT=""
+ if [ "${inv%wsrep_sst_rsync_wan*}" != "$inv" ]; then
+ WHOLE_FILE_OPT="--whole-file"
+ fi
+
# first, the normal directories, so that we can detect incompatible protocol
RC=0
eval rsync ${STUNNEL:+"'--rsh=$STUNNEL'"} \
@@ -436,16 +440,18 @@ EOF
fi
# then, we parallelize the transfer of database directories,
- # use . so that path concatenation works:
+ # use '.' so that path concatenation works:
cd "$WSREP_SST_OPT_DATA"
- count=1
- [ "$OS" = 'Linux' ] && count=$(grep -c processor /proc/cpuinfo)
- [ "$OS" = 'Darwin' -o "$OS" = 'FreeBSD' ] && count=$(sysctl -n hw.ncpu)
+ backup_threads=$(parse_cnf "--mysqld|sst" 'backup-threads')
+ if [ -z "$backup_threads" ]; then
+ get_proc
+ backup_threads=$nproc
+ fi
find . -maxdepth 1 -mindepth 1 -type d -not -name 'lost+found' \
- -not -name '.zfs' -print0 | xargs -I{} -0 -P $count \
+ -not -name '.zfs' -print0 | xargs -I{} -0 -P $backup_threads \
rsync ${STUNNEL:+--rsh="$STUNNEL"} \
--owner --group --perms --links --specials \
--ignore-times --inplace --recursive --delete --quiet \
@@ -484,35 +490,52 @@ EOF
echo "done $STATE"
+ if [ -n "$STUNNEL" ]; then
+ [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF"
+ [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID"
+ fi
+
elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]
then
check_sockets_utils
- touch "$SST_PROGRESS_FILE"
- MYSQLD_PID="$WSREP_SST_OPT_PARENT"
+ # give some time for lingering stunnel from previous SST to complete
+ check_round=0
+ while check_pid "$STUNNEL_PID" 1
+ do
+ wsrep_log_info "lingering stunnel daemon found at startup, waiting for it to exit"
+ check_round=$(( check_round + 1 ))
+ if [ $check_round -eq 10 ]; then
+ wsrep_log_error "stunnel daemon already running."
+ exit 114 # EALREADY
+ fi
+ sleep 1
+ done
MODULE="rsync_sst"
-
RSYNC_PID="$WSREP_SST_OPT_DATA/$MODULE.pid"
+ RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf"
+
# give some time for lingering rsync from previous SST to complete
check_round=0
- while check_pid "$RSYNC_PID" && [ $check_round -lt 10 ]
+ while check_pid "$RSYNC_PID" 1
do
wsrep_log_info "lingering rsync daemon found at startup, waiting for it to exit"
check_round=$(( check_round + 1 ))
+ if [ $check_round -eq 10 ]; then
+ wsrep_log_error "rsync daemon already running."
+ exit 114 # EALREADY
+ fi
sleep 1
done
- if check_pid "$RSYNC_PID"
- then
- wsrep_log_error "rsync daemon already running."
- exit 114 # EALREADY
- fi
-
- [ -f "$RSYNC_PID" ] && rm -f "$RSYNC_PID"
[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
[ -f "$BINLOG_TAR_FILE" ] && rm -f "$BINLOG_TAR_FILE"
+ if [ -z "$STUNNEL" ]; then
+ [ -f "$STUNNEL_CONF" ] && rm -f "$STUNNEL_CONF"
+ fi
+
ADDR="$WSREP_SST_OPT_ADDR"
RSYNC_PORT="$WSREP_SST_OPT_PORT"
RSYNC_ADDR="$WSREP_SST_OPT_HOST"
@@ -522,7 +545,7 @@ then
trap "exit 3" INT TERM ABRT
trap cleanup_joiner EXIT
- RSYNC_CONF="$WSREP_SST_OPT_DATA/$MODULE.conf"
+ touch "$SST_PROGRESS_FILE"
if [ -n "${MYSQL_TMP_DIR:-}" ]; then
SILENT="log file = $MYSQL_TMP_DIR/rsyncd.log"
@@ -545,18 +568,18 @@ $SILENT
path = $INNODB_DATA_HOME_DIR
EOF
-# rm -rf "$DATA"/ib_logfile* # we don't want old logs around
+# rm -rf "$DATA/ib_logfile"* # we don't want old logs around
- # If the IP is local listen only in it
+ # If the IP is local, listen only on it:
if is_local_ip "$RSYNC_ADDR_UNESCAPED"
then
RSYNC_EXTRA_ARGS="--address $RSYNC_ADDR_UNESCAPED"
STUNNEL_ACCEPT="$RSYNC_ADDR_UNESCAPED:$RSYNC_PORT"
else
- # Not local, possibly a NAT, listen on all interfaces
+ # Not local, possibly a NAT, listen on all interfaces:
RSYNC_EXTRA_ARGS=""
STUNNEL_ACCEPT="$RSYNC_PORT"
- # Overwrite address with all
+ # Overwrite address with all:
RSYNC_ADDR="*"
fi
@@ -564,8 +587,9 @@ EOF
then
rsync --daemon --no-detach --port "$RSYNC_PORT" --config "$RSYNC_CONF" $RSYNC_EXTRA_ARGS &
RSYNC_REAL_PID=$!
+ TRANSFER_REAL_PID="$RSYNC_REAL_PID"
+ TRANSFER_PID=$RSYNC_PID
else
- [ -f "$STUNNEL_PID" ] && rm -f "$STUNNEL_PID"
# Let's check if the path to the config file contains a space?
if [ "${RSYNC_CONF#* }" = "$RSYNC_CONF" ]; then
cat << EOF > "$STUNNEL_CONF"
@@ -606,15 +630,11 @@ execargs = $SHELL -c \$RSYNC_CMD
EOF
fi
stunnel "$STUNNEL_CONF" &
- RSYNC_REAL_PID=$!
- RSYNC_PID="$STUNNEL_PID"
+ STUNNEL_REAL_PID=$!
+ TRANSFER_REAL_PID="$STUNNEL_REAL_PID"
+ TRANSFER_PID=$STUNNEL_PID
fi
- until check_pid_and_port "$RSYNC_PID" "$RSYNC_REAL_PID" "$RSYNC_ADDR_UNESCAPED" "$RSYNC_PORT"
- do
- sleep 0.2
- done
-
if [ "${SSLMODE#VERIFY}" != "$SSLMODE" ]
then # backward-incompatible behavior
CN=""
@@ -635,19 +655,26 @@ EOF
ADDR="$CN:$MY_SECRET@$WSREP_SST_OPT_HOST"
else
MY_SECRET="" # for check down in recv_joiner()
- ADDR=$WSREP_SST_OPT_HOST
+ ADDR="$WSREP_SST_OPT_HOST"
fi
+ until check_pid_and_port "$TRANSFER_PID" $TRANSFER_REAL_PID "$RSYNC_ADDR_UNESCAPED" "$RSYNC_PORT"
+ do
+ sleep 0.2
+ done
+
echo "ready $ADDR:$RSYNC_PORT/$MODULE"
+ MYSQLD_PID="$WSREP_SST_OPT_PARENT"
+
# wait for SST to complete by monitoring magic file
- while [ ! -r "$MAGIC_FILE" ] && check_pid "$RSYNC_PID" && \
- ps -p $MYSQLD_PID >/dev/null
+ while [ ! -r "$MAGIC_FILE" ] && check_pid "$TRANSFER_PID" && \
+ ps -p $MYSQLD_PID >/dev/null 2>&1
do
sleep 1
done
- if ! ps -p $MYSQLD_PID >/dev/null
+ if ! ps -p $MYSQLD_PID >/dev/null 2>&1
then
wsrep_log_error \
"Parent mysqld process (PID: $MYSQLD_PID) terminated unexpectedly."
@@ -698,7 +725,7 @@ EOF
echo "rsync process ended without creating '$MAGIC_FILE'"
fi
- wsrep_cleanup_progress_file
+# wsrep_cleanup_progress_file
# cleanup_joiner
else
wsrep_log_error "Unrecognized role: '$WSREP_SST_OPT_ROLE'"
diff --git a/scripts/wsrep_sst_xtrabackup-v2.sh b/scripts/wsrep_sst_xtrabackup-v2.sh
index b5ede2b29cb..24bff12219d 100644
--- a/scripts/wsrep_sst_xtrabackup-v2.sh
+++ b/scripts/wsrep_sst_xtrabackup-v2.sh
@@ -1,6 +1,6 @@
#!/bin/bash -ue
-# Copyright (C) 2013 Percona Inc
# Copyright (C) 2017-2021 MariaDB
+# Copyright (C) 2013 Percona Inc
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -23,27 +23,31 @@
. $(dirname "$0")/wsrep_sst_common
wsrep_check_datadir
+OS="$(uname)"
ealgo=""
+eformat=""
ekey=""
ekeyfile=""
encrypt=0
-nproc=1
ecode=0
ssyslog=""
ssystag=""
-XTRABACKUP_PID=""
+BACKUP_PID=""
tcert=""
tpem=""
tkey=""
+tmode="DISABLED"
sockopt=""
progress=""
ttime=0
totime=0
+lsn=""
ecmd=""
rlimit=""
# Initially
-stagemsg="${WSREP_SST_OPT_ROLE}"
+stagemsg="$WSREP_SST_OPT_ROLE"
cpat=""
+speciald=1
ib_home_dir=""
ib_log_dir=""
ib_undo_dir=""
@@ -67,41 +71,54 @@ xtmpdir=""
scomp=""
sdecomp=""
+
ssl_dhparams=""
-ssl_cert=""
-ssl_ca=""
-ssl_key=""
+compress='none'
+compress_chunk=""
+compress_threads=""
+
+backup_threads=""
+
+encrypt_threads=""
+encrypt_chunk=""
+
+readonly SECRET_TAG="secret"
+
+sst_ver=-1
if [ -n "$(command -v pv)" ] && pv --help | grep -qw -- '-F'; then
- pvopts+=$pvformat
+ pvopts="$pvopts $pvformat"
fi
pcmd="pv $pvopts"
declare -a RC
-INNOBACKUPEX_BIN='innobackupex'
-DATA="${WSREP_SST_OPT_DATA}"
+BACKUP_BIN="$(command -v innobackupex)"
+if [ ! -x "$BACKUP_BIN" ]; then
+ wsrep_log_error 'innobackupex binary not found in path'
+ exit 42
+fi
+
+DATA="$WSREP_SST_OPT_DATA"
INFO_FILE="xtrabackup_galera_info"
IST_FILE="xtrabackup_ist"
-MAGIC_FILE="${DATA}/${INFO_FILE}"
+MAGIC_FILE="$DATA/$INFO_FILE"
+
+INNOAPPLYLOG="$DATA/innobackupex.prepare.log"
+INNOMOVELOG="$DATA/innobackupex.move.log"
+INNOBACKUPLOG="$DATA/innobackupex.backup.log"
# Setting the path for ss and ip
export PATH="/usr/sbin:/sbin:$PATH"
-OS="$(uname)"
-
-if [ -z "$(command -v lsof)" ]; then
- wsrep_log_error "lsof tool not found in PATH! Make sure you have it installed."
- exit 2 # ENOENT
-fi
-
-timeit(){
+timeit()
+{
local stage="$1"
shift
local cmd="$@"
local x1 x2 took extcode
- if [[ $ttime -eq 1 ]];then
+ if [ $ttime -eq 1 ]; then
x1=$(date +%s)
wsrep_log_info "Evaluating $cmd"
eval "$cmd"
@@ -121,105 +138,89 @@ timeit(){
get_keys()
{
# $encrypt -eq 1 is for internal purposes only
- if [[ $encrypt -ge 2 || $encrypt -eq -1 ]];then
+ if [ $encrypt -ge 2 -o $encrypt -eq -1 ]; then
return
fi
- if [[ $encrypt -eq 0 ]];then
- if $MY_PRINT_DEFAULTS xtrabackup | grep -q encrypt;then
- wsrep_log_error "Unexpected option combination. SST may fail. Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html "
+ if [ $encrypt -eq 0 ]; then
+ if [ -n "$ealgo" -o -n "$ekey" -o -n "$ekeyfile" ]; then
+ wsrep_log_error "Options for encryption are specified, " \
+ "but encryption itself is disabled. SST may fail."
fi
return
fi
- if [[ "$sfmt" == 'tar' ]];then
- wsrep_log_info "NOTE: Xtrabackup-based encryption - encrypt=1 - cannot be enabled with tar format"
+ if [ $sfmt = 'tar' ]; then
+ wsrep_log_info "NOTE: key-based encryption (encrypt=1) " \
+ "cannot be enabled with tar format"
encrypt=-1
return
fi
- wsrep_log_info "Xtrabackup based encryption enabled in my.cnf - Supported only from Xtrabackup 2.1.4"
+ wsrep_log_info "Key based encryption enabled in my.cnf - supported only from Xtrabackup 2.1.4"
- if [[ -z "$ealgo" ]];then
+ if [ -z "$ealgo" ]; then
wsrep_log_error "FATAL: Encryption algorithm empty from my.cnf, bailing out"
exit 3
fi
- if [[ -z "$ekey" && ! -r "$ekeyfile" ]];then
+ if [ -z "$ekey" -a ! -r "$ekeyfile" ]; then
wsrep_log_error "FATAL: Either key or keyfile must be readable"
exit 3
fi
- if [[ -z "$ekey" ]];then
- ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key-file=$ekeyfile"
+ if [ "$eformat" = 'openssl' ]; then
+ get_openssl
+ if [ -z "$OPENSSL_BINARY" ]; then
+ wsrep_log_error "If encryption using the openssl is enabled, " \
+ "then you need to install openssl"
+ exit 2
+ fi
+ ecmd="'$OPENSSL_BINARY' enc -$ealgo"
+ if "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-pbkdf2'; then
+ ecmd="$ecmd -pbkdf2"
+ elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-iter'; then
+ ecmd="$ecmd -iter 1"
+ elif "$OPENSSL_BINARY" enc -help 2>&1 | grep -qw -- '-md'; then
+ ecmd="$ecmd -md sha256"
+ fi
+ if [ -z "$ekey" ]; then
+ ecmd="$ecmd -kfile '$ekeyfile'"
+ else
+ ecmd="$ecmd -k '$ekey'"
+ fi
+ elif [ "$eformat" = 'xbcrypt' ]; then
+ if [ -z "$(command -v xbcrypt)" ]; then
+ wsrep_log_error "If encryption using the xbcrypt is enabled, " \
+ "then you need to install xbcrypt"
+ exit 2
+ fi
+ wsrep_log_info "NOTE: xbcrypt-based encryption, " \
+ "supported only from Xtrabackup 2.1.4"
+ if [ -z "$ekey" ]; then
+ ecmd="xbcrypt --encrypt-algo='$ealgo' --encrypt-key-file='$ekeyfile'"
+ else
+ wsrep_log_warning "Using the 'encrypt-key' option causes the encryption key"
+ wsrep_log_warning "to be set via the command-line and is considered insecure."
+ wsrep_log_warning "It is recommended to use the 'encrypt-key-file' option instead."
+ ecmd="xbcrypt --encrypt-algo='$ealgo' --encrypt-key='$ekey'"
+ fi
+ if [ -n "$encrypt_threads" ]; then
+ ecmd="$ecmd --encrypt-threads=$encrypt_threads"
+ fi
+ if [ -n "$encrypt_chunk" ]; then
+ ecmd="$ecmd --encrypt-chunk-size=$encrypt_chunk"
+ fi
else
- wsrep_log_warning "Using the 'encrypt-key' option causes the encryption key"
- wsrep_log_warning "to be set via the command-line and is considered insecure."
- wsrep_log_warning "It is recommended to use the 'encrypt-key-file' option instead."
-
- ecmd="xbcrypt --encrypt-algo=$ealgo --encrypt-key=$ekey"
+ wsrep_log_error "Unknown encryption format='$eformat'"
+ exit 2
fi
- if [[ "$WSREP_SST_OPT_ROLE" == "joiner" ]];then
- ecmd+=" -d"
+ if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
+ ecmd="$ecmd -d"
fi
- stagemsg+="-XB-Encrypted"
-}
-
-#
-# verifies that the certificate matches the private key
-# doing this will save us having to wait for a timeout that would
-# otherwise occur.
-#
-# 1st param: path to the cert
-# 2nd param: path to the private key
-#
-verify_cert_matches_key()
-{
- local cert_path="$1"
- local key_path="$2"
-
- wsrep_check_programs openssl diff
-
- # generate the public key from the cert and the key
- # they should match (otherwise we can't create an SSL connection)
- if ! diff <(openssl x509 -in "$cert_path" -pubkey -noout) <(openssl rsa -in "$key_path" -pubout 2>/dev/null) >/dev/null 2>&1
- then
- wsrep_log_error "******** FATAL ERROR ************************* "
- wsrep_log_error "* The certifcate and private key do not match. "
- wsrep_log_error "* Please check your certificate and key files. "
- wsrep_log_error "********************************************** "
- exit 22
- fi
-}
-
-# Checks to see if the file exists
-# If the file does not exist (or cannot be read), issues an error
-# and exits
-#
-# 1st param: file name to be checked (for read access)
-# 2nd param: 1st error message (header)
-# 3rd param: 2nd error message (footer, optional)
-#
-verify_file_exists()
-{
- local file_path="$1"
- local error_message1="$2"
- local error_message2="$3"
-
- if ! [[ -r "$file_path" ]]; then
- wsrep_log_error "******** FATAL ERROR ************************* "
- wsrep_log_error "* $error_message1 "
- wsrep_log_error "* Could not find/access : $file_path "
-
- if ! [[ -z "$error_message2" ]]; then
- wsrep_log_error "* $error_message2 "
- fi
-
- wsrep_log_error "********************************************** "
- exit 22
- fi
+ stagemsg="$stagemsg-XB-Encrypted"
}
get_transfer()
@@ -310,15 +311,6 @@ get_transfer()
exit 2
fi
- # Determine the socat version
- SOCAT_VERSION=$(socat -V 2>&1 | grep -m1 -oe '[0-9]\.[0-9][\.0-9]*')
- if [ -z "$SOCAT_VERSION" ]; then
- wsrep_log_error "******** FATAL ERROR ******************"
- wsrep_log_error "* Cannot determine the socat version. *"
- wsrep_log_error "***************************************"
- exit 2
- fi
-
local action='Decrypting'
if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
tcmd="socat -u openssl-listen:$SST_PORT,reuseaddr"
@@ -327,11 +319,25 @@ get_transfer()
action='Encrypting'
fi
- if ! check_for_version "$SOCAT_VERSION" '1.7.3'; then
- # socat versions < 1.7.3 will have 512-bit dhparams (too small)
- # so create 2048-bit dhparams and send that as a parameter:
- check_for_dhparams
- tcmd="$tcmd,dhparam='$ssl_dhparams'"
+ if [ "${sockopt#*,dhparam=}" != "$sockopt" ]; then
+ if [ -z "$ssl_dhparams" ]; then
+ # Determine the socat version
+ SOCAT_VERSION=$(socat -V 2>&1 | grep -m1 -oe '[0-9]\.[0-9][\.0-9]*')
+ if [ -z "$SOCAT_VERSION" ]; then
+ wsrep_log_error "******** FATAL ERROR ******************"
+ wsrep_log_error "* Cannot determine the socat version. *"
+ wsrep_log_error "***************************************"
+ exit 2
+ fi
+ if ! check_for_version "$SOCAT_VERSION" '1.7.3'; then
+ # socat versions < 1.7.3 will have 512-bit dhparams (too small)
+ # so create 2048-bit dhparams and send that as a parameter:
+ check_for_dhparams
+ fi
+ fi
+ if [ -n "$ssl_dhparams" ]; then
+ tcmd="$tcmd,dhparam='$ssl_dhparams'"
+ fi
fi
if [ $encrypt -eq 2 ]; then
@@ -340,6 +346,11 @@ get_transfer()
wsrep_log_error "Both PEM and CRT files required"
exit 22
fi
+ if [ ! -r "$tpem" -o ! -r "$tcert" ]; then
+ wsrep_log_error "Both PEM and CRT files must be readable"
+ exit 22
+ fi
+ verify_ca_matches_cert "$tcert" "$tpem"
tcmd="$tcmd,cert='$tpem',cafile='$tcert'$sockopt"
stagemsg="$stagemsg-OpenSSL-Encrypted-2"
wsrep_log_info "$action with cert=$tpem, cafile=$tcert"
@@ -349,6 +360,11 @@ get_transfer()
wsrep_log_error "Both certificate and key files required"
exit 22
fi
+ if [ ! -r "$tpem" -o ! -r "$tkey" ]; then
+ wsrep_log_error "Both certificate and key files must be readable"
+ exit 22
+ fi
+ verify_cert_matches_key "$tpem" "$tkey"
stagemsg="$stagemsg-OpenSSL-Encrypted-3"
if [ -z "$tcert" ]; then
if [ $encrypt -eq 4 ]; then
@@ -360,6 +376,11 @@ get_transfer()
wsrep_log_info "$action with cert=$tpem, key=$tkey, verify=0"
else
# CA verification
+ if [ ! -r "$tcert" ]; then
+ wsrep_log_error "Certificate file must be readable"
+ exit 22
+ fi
+ verify_ca_matches_cert "$tcert" "$tpem"
if [ -n "$WSREP_SST_OPT_REMOTE_USER" ]; then
CN_option=",commonname='$WSREP_SST_OPT_REMOTE_USER'"
elif [ $encrypt -eq 4 ]; then
@@ -387,13 +408,13 @@ get_footprint()
{
pushd "$WSREP_SST_OPT_DATA" 1>/dev/null
payload=$(find . -regex '.*\.ibd$\|.*\.MYI$\|.*\.MYD$\|.*ibdata1$' -type f -print0 | du --files0-from=- --block-size=1 -c -s | awk 'END { print $1 }')
- if [ $(in_config 'xtrabackup' 'compress') -eq 1 ]; then
+ if [ "$compress" != 'none' ]; then
# QuickLZ has around 50% compression ratio
# When compression/compaction used, the progress is only an approximate.
payload=$(( payload*1/2 ))
fi
popd 1>/dev/null
- pcmd+=" -s $payload"
+ pcmd="$pcmd -s $payload"
adjust_progress
}
@@ -408,93 +429,141 @@ adjust_progress()
return
fi
- if [[ -n "$progress" && "$progress" != '1' ]];then
- if [[ -e "$progress" ]];then
- pcmd+=" 2>>$progress"
+ if [ -n "$progress" -a "$progress" != '1' ]; then
+ if [ -e "$progress" ]; then
+ pcmd="$pcmd 2>>'$progress'"
else
- pcmd+=" 2>$progress"
+ pcmd="$pcmd 2>'$progress'"
fi
- elif [[ -z "$progress" && -n "$rlimit" ]];then
+ elif [ -z "$progress" -a -n "$rlimit" ]; then
# When rlimit is non-zero
pcmd="pv -q"
fi
- if [[ -n "$rlimit" && "$WSREP_SST_OPT_ROLE" == "donor" ]];then
+ if [ -n "$rlimit" -a "$WSREP_SST_OPT_ROLE" = 'donor' ]; then
wsrep_log_info "Rate-limiting SST to $rlimit"
- pcmd+=" -L \$rlimit"
+ pcmd="$pcmd -L \$rlimit"
fi
}
+encgroups='--mysqld|sst|xtrabackup'
+
+check_server_ssl_config()
+{
+ tcert=$(parse_cnf "$encgroups" 'ssl-ca')
+ tpem=$(parse_cnf "$encgroups" 'ssl-cert')
+ tkey=$(parse_cnf "$encgroups" 'ssl-key')
+}
+
read_cnf()
{
- sfmt=$(parse_cnf sst streamfmt "xbstream")
- tfmt=$(parse_cnf sst transferfmt "socat")
- tcert=$(parse_cnf sst tca "")
- tpem=$(parse_cnf sst tcert "")
- tkey=$(parse_cnf sst tkey "")
- encrypt=$(parse_cnf sst encrypt 0)
+ sfmt=$(parse_cnf sst streamfmt 'xbstream')
+ tfmt=$(parse_cnf sst transferfmt 'socat')
+
+ encrypt=$(parse_cnf "$encgroups" 'encrypt' 0)
+ tmode=$(parse_cnf "$encgroups" 'ssl-mode' 'DISABLED' | tr [:lower:] [:upper:])
+
+ if [ $encrypt -eq 0 -o $encrypt -ge 2 ]
+ then
+ if [ "$tmode" != 'DISABLED' -o $encrypt -ge 2 ]
+ then
+ tcert=$(parse_cnf 'sst' 'tca')
+ tpem=$(parse_cnf 'sst' 'tcert')
+ tkey=$(parse_cnf 'sst' 'tkey')
+ fi
+ if [ "$tmode" != 'DISABLED' ]; then
+ # backward-incompatible behavior
+ if [ -z "$tpem" -a -z "$tkey" -a -z "$tcert" ]; then
+ # no old-style SSL config in [sst]
+ check_server_ssl_config
+ fi
+ if [ 0 -eq $encrypt -a -n "$tpem" -a -n "$tkey" ]
+ then
+ encrypt=3 # enable cert/key SSL encyption
+
+ # avoid CA verification if not set explicitly:
+ # nodes may happen to have different CA if self-generated
+ # zeroing up tcert does the trick
+ [ "${tmode#VERIFY}" != "$tmode" ] || tcert=""
+ fi
+ fi
+ elif [ $encrypt -eq 1 ]; then
+ ealgo=$(parse_cnf "$encgroups" 'encrypt-algo')
+ eformat=$(parse_cnf "$encgroups" 'encrypt-format' 'xbcrypt')
+ ekey=$(parse_cnf "$encgroups" 'encrypt-key')
+ ekeyfile=$(parse_cnf "$encgroups" 'encrypt-key-file')
+ fi
+
+ wsrep_log_info "SSL configuration: CA='$tcert', CERT='$tpem'," \
+ "KEY='$tkey', MODE='$tmode', encrypt='$encrypt'"
+
sockopt=$(parse_cnf sst sockopt "")
progress=$(parse_cnf sst progress "")
rebuild=$(parse_cnf sst rebuild 0)
ttime=$(parse_cnf sst time 0)
cpat='.*\.pem$\|.*init\.ok$\|.*galera\.cache$\|.*sst_in_progress$\|.*\.sst$\|.*gvwstate\.dat$\|.*grastate\.dat$\|.*\.err$\|.*\.log$\|.*RPM_UPGRADE_MARKER$\|.*RPM_UPGRADE_HISTORY$'
[ "$OS" = 'FreeBSD' ] && cpat=$(echo "$cpat" | sed 's/\\|/|/g')
- ealgo=$(parse_cnf xtrabackup encrypt "")
- ekey=$(parse_cnf xtrabackup encrypt-key "")
- ekeyfile=$(parse_cnf xtrabackup encrypt-key-file "")
+ cpat=$(parse_cnf sst cpat "$cpat")
scomp=$(parse_cnf sst compressor "")
sdecomp=$(parse_cnf sst decompressor "")
- # Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html
- if [[ -z "$ealgo" ]];then
- ealgo=$(parse_cnf sst encrypt-algo "")
- ekey=$(parse_cnf sst encrypt-key "")
- ekeyfile=$(parse_cnf sst encrypt-key-file "")
- fi
-
- # Pull the parameters needed for encrypt=4
- ssl_ca=$(parse_cnf sst ssl-ca "")
- if [[ -z "$ssl_ca" ]]; then
- ssl_ca=$(parse_cnf --mysqld ssl-ca "")
- fi
- ssl_cert=$(parse_cnf sst ssl-cert "")
- if [[ -z "$ssl_cert" ]]; then
- ssl_cert=$(parse_cnf --mysqld ssl-cert "")
- fi
- ssl_key=$(parse_cnf sst ssl-key "")
- if [[ -z "$ssl_key" ]]; then
- ssl_key=$(parse_cnf --mysqld ssl-key "")
- fi
-
rlimit=$(parse_cnf sst rlimit "")
uextra=$(parse_cnf sst use-extra 0)
+ speciald=$(parse_cnf sst sst-special-dirs 1)
iopts=$(parse_cnf sst inno-backup-opts "")
iapts=$(parse_cnf sst inno-apply-opts "")
impts=$(parse_cnf sst inno-move-opts "")
- stimeout=$(parse_cnf sst sst-initial-timeout 100)
+ stimeout=$(parse_cnf sst sst-initial-timeout 300)
ssyslog=$(parse_cnf sst sst-syslog 0)
ssystag=$(parse_cnf mysqld_safe syslog-tag "${SST_SYSLOG_TAG:-}")
- ssystag+="-"
+ ssystag="$ssystag-"
+ sstlogarchive=$(parse_cnf sst sst-log-archive 1)
+ sstlogarchivedir=$(parse_cnf sst sst-log-archive-dir '/tmp/sst_log_archive')
+
+ if [ $speciald -eq 0 ]; then
+ wsrep_log_error "sst-special-dirs equal to 0 is not supported, falling back to 1"
+ speciald=1
+ fi
if [ $ssyslog -ne -1 ]; then
ssyslog=$(in_config 'mysqld_safe' 'syslog')
fi
+
+ if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then
+ compress=$(parse_cnf "$encgroups" 'compress' 'none')
+ if [ "$compress" != 'none' ]; then
+ compress_chunk=$(parse_cnf "$encgroups" 'compress-chunk-size')
+ compress_threads=$(parse_cnf "$encgroups" 'compress-threads')
+ fi
+ fi
+
+ backup_threads=$(parse_cnf "$encgroups" 'backup-threads')
+
+ if [ "$eformat" = 'xbcrypt' ]; then
+ encrypt_threads=$(parse_cnf "$encgroups" 'encrypt-threads')
+ encrypt_chunk=$(parse_cnf "$encgroups" 'encrypt-chunk-size')
+ fi
}
get_stream()
{
if [ "$sfmt" = 'mbstream' -o "$sfmt" = 'xbstream' ]; then
- XBSTREAM_BIN=$(command -v "$sfmt")
- if [ -z "$XBSTREAM_BIN" ]; then
- if [ -z "$XBSTREAM_BIN" ]; then
- wsrep_log_error "Streaming with $sfmt, but $sfmt not found in path"
- exit 42
+ STREAM_BIN=$(command -v "$sfmt")
+ if [ -z "$STREAM_BIN" ]; then
+ if [ "$sfmt" = 'xbstream' ]; then
+ STREAM_BIN="$(command -v mbstream)"
+ else
+ STREAM_BIN="$(command -v xbstream)"
fi
fi
+ if [ -z "$STREAM_BIN" ]; then
+ wsrep_log_error "Streaming with $sfmt, but $sfmt not found in path"
+ exit 42
+ fi
if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
- strmcmd="'$XBSTREAM_BIN' -x"
+ strmcmd="'$STREAM_BIN' -x"
else
- strmcmd="'$XBSTREAM_BIN' -c '$INFO_FILE'"
+ strmcmd="'$STREAM_BIN' -c '$INFO_FILE'"
fi
else
sfmt='tar'
@@ -507,91 +576,47 @@ get_stream()
wsrep_log_info "Streaming with $sfmt"
}
-get_proc()
-{
- set +e
- nproc=$(grep -c processor /proc/cpuinfo)
- [[ -z $nproc || $nproc -eq 0 ]] && nproc=1
- set -e
-}
-
sig_joiner_cleanup()
{
wsrep_log_error "Removing $MAGIC_FILE file due to signal"
- rm -f "$MAGIC_FILE"
-}
-
-cleanup_joiner()
-{
- # Since this is invoked just after exit NNN
- local estatus=$?
- if [[ $estatus -ne 0 ]];then
- wsrep_log_error "Cleanup after exit with status:$estatus"
- elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ];then
- wsrep_log_info "Removing the sst_in_progress file"
- wsrep_cleanup_progress_file
- fi
- if [[ -n "$progress" && -p "$progress" ]];then
- wsrep_log_info "Cleaning up fifo file $progress"
- rm $progress
- fi
- if [[ -n "${STATDIR:-}" ]];then
- [[ -d "$STATDIR" ]] && rm -rf "$STATDIR"
- fi
-
- # Final cleanup
- pgid=$(ps -o pgid= $$ | grep -o '[0-9]*')
-
- # This means no setsid done in mysqld.
- # We don't want to kill mysqld here otherwise.
- if [[ $$ -eq $pgid ]];then
- # This means a signal was delivered to the process.
- # So, more cleanup.
- if [[ $estatus -ge 128 ]];then
- kill -KILL -$$ || true
- fi
- fi
-
- exit $estatus
-}
-
-check_pid()
-{
- local pid_file="$1"
- [ -r "$pid_file" ] && ps -p $(cat "$pid_file") >/dev/null 2>&1
+ [ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
}
-cleanup_donor()
+cleanup_at_exit()
{
# Since this is invoked just after exit NNN
local estatus=$?
- if [[ $estatus -ne 0 ]];then
+ if [ $estatus -ne 0 ]; then
wsrep_log_error "Cleanup after exit with status:$estatus"
fi
- if [[ -n "${XTRABACKUP_PID:-}" ]];then
- if check_pid "$XTRABACKUP_PID"
- then
- wsrep_log_error "xtrabackup process is still running. Killing... "
- kill_xtrabackup
+ if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
+ wsrep_log_info "Removing the sst_in_progress file"
+ wsrep_cleanup_progress_file
+ else
+ if [ -n "$BACKUP_PID" ]; then
+ if check_pid "$BACKUP_PID" 1; then
+ wsrep_log_error "xtrabackup process is still running. Killing..."
+ cleanup_pid $CHECK_PID "$BACKUP_PID"
+ fi
fi
+ [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE"
fi
- rm -f "${DATA}/${IST_FILE}" || true
-
- if [[ -n "$progress" && -p "$progress" ]];then
+ if [ -n "$progress" -a -p "$progress" ]; then
wsrep_log_info "Cleaning up fifo file $progress"
rm -f "$progress" || true
fi
wsrep_log_info "Cleaning up temporary directories"
- if [[ -n "$xtmpdir" ]];then
- [[ -d "$xtmpdir" ]] && rm -rf "$xtmpdir" || true
- fi
-
- if [[ -n "$itmpdir" ]];then
- [[ -d "$itmpdir" ]] && rm -rf "$itmpdir" || true
+ if [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]; then
+ if [ -n "$STATDIR" ]; then
+ [ -d "$STATDIR" ] && rm -rf "$STATDIR"
+ fi
+ else
+ [ -n "$xtmpdir" -a -d "$xtmpdir" ] && rm -rf "$xtmpdir" || true
+ [ -n "$itmpdir" -a -d "$itmpdir" ] && rm -rf "$itmpdir" || true
fi
# Final cleanup
@@ -599,44 +624,43 @@ cleanup_donor()
# This means no setsid done in mysqld.
# We don't want to kill mysqld here otherwise.
- if [[ $$ -eq $pgid ]];then
-
+ if [ $$ -eq $pgid ]; then
# This means a signal was delivered to the process.
# So, more cleanup.
- if [[ $estatus -ge 128 ]];then
- kill -KILL -$$ || true
+ if [ $estatus -ge 128 ]; then
+ kill -KILL -- -$$ || true
fi
-
fi
exit $estatus
}
-kill_xtrabackup()
+setup_ports()
{
- local PID=$(cat "$XTRABACKUP_PID")
- [ -n "$PID" -a "0" != "$PID" ] && kill $PID && (kill $PID && kill -9 $PID) || :
- wsrep_log_info "Removing xtrabackup pid file $XTRABACKUP_PID"
- rm -f "$XTRABACKUP_PID" || true
+ SST_PORT="$WSREP_SST_OPT_PORT"
+ if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]; then
+ REMOTEIP="$WSREP_SST_OPT_HOST"
+ lsn="$WSREP_SST_OPT_LSN"
+ sst_ver="$WSREP_SST_OPT_SST_VER"
+ fi
}
-# waits ~1 minute for nc/socat to open the port and then reports ready
-# (regardless of timeout)
+#
+# Waits ~30 seconds for socat or nc to open the port and
+# then reports ready, regardless of timeout.
+#
wait_for_listen()
{
- local HOST="$1"
- local PORT="$2"
+ local PORT="$1"
+ local ADDR="$2"
local MODULE="$3"
- local LSOF_OUT
-
- for i in {1..300}
- do
- LSOF_OUT=$(lsof -sTCP:LISTEN -i TCP:${PORT} -a -c nc -c socat -F c 2> /dev/null || :)
- [ -n "${LSOF_OUT}" ] && break
+ for i in {1..150}; do
+ if check_port "" "$PORT" 'socat|nc'; then
+ break
+ fi
sleep 0.2
done
-
- echo "ready ${HOST}:${PORT}/${MODULE}//${WSREP_SST_OPT_SST_VER:-1}"
+ echo "ready $ADDR/$MODULE//$sst_ver"
}
check_extra()
@@ -648,9 +672,9 @@ check_extra()
local eport=$(parse_cnf '--mysqld' 'extra-port')
if [ -n "$eport" ]; then
# Xtrabackup works only locally.
- # Hence, setting host to 127.0.0.1 unconditionally.
+ # Hence, setting host to 127.0.0.1 unconditionally:
wsrep_log_info "SST through extra_port $eport"
- INNOEXTRA+=" --host=127.0.0.1 --port=$eport"
+ INNOEXTRA="$INNOEXTRA --host=127.0.0.1 --port=$eport"
use_socket=0
else
wsrep_log_error "Extra port $eport null, failing"
@@ -661,7 +685,7 @@ check_extra()
fi
fi
if [ $use_socket -eq 1 -a -n "$WSREP_SST_OPT_SOCKET" ]; then
- INNOEXTRA+=" --socket='$WSREP_SST_OPT_SOCKET'"
+ INNOEXTRA="$INNOEXTRA --socket='$WSREP_SST_OPT_SOCKET'"
fi
}
@@ -671,49 +695,71 @@ recv_joiner()
local msg="$2"
local tmt=$3
local checkf=$4
- local ltcmd
+ local wait=$5
- if [[ ! -d "${dir}" ]];then
+ if [ ! -d "$dir" ]; then
# This indicates that IST is in progress
return
fi
- pushd "${dir}" 1>/dev/null
+ local ltcmd="$tcmd"
+ if [ $tmt -gt 0 ]; then
+ if [ -n "$(command -v timeout)" ]; then
+ if timeout --help | grep -qw -- '-k'; then
+ ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd"
+ else
+ ltcmd="timeout -s9 $tmt $tcmd"
+ fi
+ fi
+ fi
+
+ pushd "$dir" 1>/dev/null
set +e
- if [ $tmt -gt 0 -a -n "$(command -v timeout)" ]; then
- if timeout --help | grep -qw -- '-k';then
- ltcmd="timeout -k $(( tmt+10 )) $tmt $tcmd"
- else
- ltcmd="timeout -s9 $tmt $tcmd"
- fi
- timeit "$msg" "$ltcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )"
- else
- timeit "$msg" "$tcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )"
+ if [ $wait -ne 0 ]; then
+ wait_for_listen "$SST_PORT" "$ADDR" "$MODULE" &
fi
+ timeit "$msg" "$ltcmd | $strmcmd; RC=( "\${PIPESTATUS[@]}" )"
+
set -e
popd 1>/dev/null
- if [[ ${RC[0]} -eq 124 ]];then
- wsrep_log_error "Possible timeout in receving first data from donor in gtid stage"
+ if [ ${RC[0]} -eq 124 ]; then
+ wsrep_log_error "Possible timeout in receiving first data from " \
+ "donor in gtid stage: exit codes: ${RC[@]}"
exit 32
fi
- for ecode in "${RC[@]}";do
- if [[ $ecode -ne 0 ]];then
+ for ecode in "${RC[@]}"; do
+ if [ $ecode -ne 0 ]; then
wsrep_log_error "Error while getting data from donor node: " \
"exit codes: ${RC[@]}"
exit 32
fi
done
- if [[ $checkf -eq 1 && ! -r "${MAGIC_FILE}" ]];then
- # this message should cause joiner to abort
- wsrep_log_error "xtrabackup process ended without creating '${MAGIC_FILE}'"
- wsrep_log_info "Contents of datadir"
- wsrep_log_info $(ls -l "$dir/"*)
- exit 32
+ if [ $checkf -eq 1 ]; then
+ if [ ! -r "$MAGIC_FILE" ]; then
+ # this message should cause joiner to abort
+ wsrep_log_error "receiving process ended without creating " \
+ "'$MAGIC_FILE'"
+ wsrep_log_info "Contents of datadir"
+ wsrep_log_info $(ls -l "$dir/"*)
+ exit 32
+ fi
+
+ # check donor supplied secret
+ SECRET=$(grep -- "$SECRET_TAG " "$MAGIC_FILE" 2>/dev/null | cut -d ' ' -f 2)
+ if [ "$SECRET" != "$MY_SECRET" ]; then
+ wsrep_log_error "Donor does not know my secret!"
+ wsrep_log_info "Donor:'$SECRET', my:'$MY_SECRET'"
+ exit 32
+ fi
+
+ # remove secret from magic file
+ grep -v -- "$SECRET_TAG " "$MAGIC_FILE" > "$MAGIC_FILE.new"
+ mv "$MAGIC_FILE.new" "$MAGIC_FILE"
fi
}
@@ -722,14 +768,14 @@ send_donor()
local dir="$1"
local msg="$2"
- pushd "${dir}" 1>/dev/null
+ pushd "$dir" 1>/dev/null
set +e
timeit "$msg" "$strmcmd | $tcmd; RC=( "\${PIPESTATUS[@]}" )"
set -e
popd 1>/dev/null
- for ecode in "${RC[@]}";do
- if [[ $ecode -ne 0 ]];then
+ for ecode in "${RC[@]}"; do
+ if [ $ecode -ne 0 ]; then
wsrep_log_error "Error while sending data to joiner node: " \
"exit codes: ${RC[@]}"
exit 32
@@ -742,68 +788,68 @@ monitor_process()
local sst_stream_pid=$1
while true ; do
-
- if ! ps --pid "${WSREP_SST_OPT_PARENT}" &>/dev/null; then
+ if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1; then
wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT) terminated unexpectedly."
- kill -- -"${WSREP_SST_OPT_PARENT}"
+ kill -- -"$WSREP_SST_OPT_PARENT"
exit 32
fi
-
- if ! ps --pid "${sst_stream_pid}" &>/dev/null; then
+ if ! ps -p "$sst_stream_pid" >/dev/null 2>&1; then
break
fi
-
sleep 0.1
-
done
}
-innobackup=$(command -v "$INNOBACKUPEX_BIN")
-if [ ! -x "$innobackup" ]; then
- wsrep_log_error "innobackupex not in path: $PATH"
- exit 2
-fi
-
# check the version, we require XB-2.4 to ensure that we can pass the
# datadir via the command-line option
XB_REQUIRED_VERSION="2.3.5"
-XB_VERSION=`$INNOBACKUPEX_BIN --version 2>&1 | grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1`
+XB_VERSION=`$BACKUP_BIN --version 2>&1 | grep -oe '[0-9]\.[0-9][\.0-9]*' | head -n1`
if [[ -z "$XB_VERSION" ]]; then
- wsrep_log_error "FATAL: Cannot determine the $INNOBACKUPEX_BIN version. Needs xtrabackup-$XB_REQUIRED_VERSION or higher to perform SST"
+ wsrep_log_error "FATAL: Cannot determine the $BACKUP_BIN version. Needs xtrabackup-$XB_REQUIRED_VERSION or higher to perform SST"
exit 2
fi
if ! check_for_version "$XB_VERSION" "$XB_REQUIRED_VERSION"; then
- wsrep_log_error "FATAL: The $INNOBACKUPEX_BIN version is $XB_VERSION. Needs xtrabackup-$XB_REQUIRED_VERSION or higher to perform SST"
+ wsrep_log_error "FATAL: The $BACKUP_BIN version is $XB_VERSION. Needs xtrabackup-$XB_REQUIRED_VERSION or higher to perform SST"
exit 2
fi
-rm -f "${MAGIC_FILE}"
+[ -f "$MAGIC_FILE" ] && rm -f "$MAGIC_FILE"
-if [[ ! "${WSREP_SST_OPT_ROLE}" == 'joiner' && ! "${WSREP_SST_OPT_ROLE}" == 'donor' ]];then
- wsrep_log_error "Invalid role ${WSREP_SST_OPT_ROLE}"
+if [ "$WSREP_SST_OPT_ROLE" != 'joiner' -a "$WSREP_SST_OPT_ROLE" != 'donor' ]; then
+ wsrep_log_error "Invalid role '$WSREP_SST_OPT_ROLE'"
exit 22
fi
read_cnf
+setup_ports
-if ${INNOBACKUPEX_BIN} /tmp --help 2>/dev/null | grep -qw -- '--version-check'; then
+if "$BACKUP_BIN" --help 2>/dev/null | grep -qw -- '--version-check'; then
disver='--no-version-check'
fi
-iopts+=" --databases-exclude='lost+found'"
+# if no command line argument and INNODB_DATA_HOME_DIR environment variable
+# is not set, try to get it from my.cnf:
+if [ -z "$INNODB_DATA_HOME_DIR" ]; then
+ INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir')
+fi
+
+OLD_PWD="$(pwd)"
-if [ ${FORCE_FTWRL:-0} -eq 1 ]; then
- wsrep_log_info "Forcing FTWRL due to environment variable FORCE_FTWRL equal to $FORCE_FTWRL"
- iopts+=' --no-backup-locks'
+cd "$WSREP_SST_OPT_DATA"
+if [ -n "$INNODB_DATA_HOME_DIR" ]; then
+ # handle both relative and absolute paths
+ [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR"
+ cd "$INNODB_DATA_HOME_DIR"
fi
+INNODB_DATA_HOME_DIR=$(pwd -P)
-if [[ $ssyslog -eq 1 ]];then
- if [ -z "$(command -v logger)" ]; then
- wsrep_log_error "logger not in path: $PATH. Ignoring"
- else
- wsrep_log_info "Logging all stderr of SST/Innobackupex to syslog"
+cd "$OLD_PWD"
+
+if [ $ssyslog -eq 1 ]; then
+ if [ -n "$(command -v logger)" ]; then
+ wsrep_log_info "Logging all stderr of SST/xtrabackup to syslog"
exec 2> >(logger -p daemon.err -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE)
@@ -816,69 +862,108 @@ if [[ $ssyslog -eq 1 ]];then
{
logger -p daemon.info -t ${ssystag}wsrep-sst-$WSREP_SST_OPT_ROLE "$@"
}
-
- INNOAPPLY="2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-apply"
- INNOMOVE="2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-move"
- INNOBACKUP="2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)"
+ else
+ wsrep_log_error "logger not in path: $PATH. Ignoring"
fi
+ INNOAPPLY="2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-apply"
+ INNOMOVE="2>&1 | logger -p daemon.err -t ${ssystag}innobackupex-move"
+ INNOBACKUP="2> >(logger -p daemon.err -t ${ssystag}innobackupex-backup)"
else
- INNOAPPLY="&>'$DATA/innobackup.prepare.log'"
- INNOMOVE="&>'$DATA/innobackup.move.log'"
- INNOBACKUP="2>'$DATA/innobackup.backup.log'"
-fi
+ if [ $sstlogarchive -eq 1 ]
+ then
+ ARCHIVETIMESTAMP=$(date "+%Y.%m.%d-%H.%M.%S.%N")
-get_stream
-get_transfer
+ if [ -n "$sstlogarchivedir" ]; then
+ if [ ! -d "$sstlogarchivedir" ]; then
+ mkdir -p "$sstlogarchivedir"
+ fi
+ fi
-# if no command line argument and INNODB_DATA_HOME_DIR environment variable
-# is not set, try to get it from my.cnf:
-if [ -z "$INNODB_DATA_HOME_DIR" ]; then
- INNODB_DATA_HOME_DIR=$(parse_cnf '--mysqld' 'innodb-data-home-dir')
-fi
+ if [ -e "$INNOAPPLYLOG" ]
+ then
+ if [ -n "$sstlogarchivedir" ]
+ then
+ newfile=$(basename "$INNOAPPLYLOG")
+ newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP"
+ else
+ newfile="$INNOAPPLYLOG.$ARCHIVETIMESTAMP"
+ fi
+ wsrep_log_info "Moving '$INNOAPPLYLOG' to '$newfile'"
+ mv "$INNOAPPLYLOG" "$newfile"
+ gzip "$newfile"
+ fi
-OLD_PWD="$(pwd)"
+ if [ -e "$INNOMOVELOG" ]
+ then
+ if [ -n "$sstlogarchivedir" ]
+ then
+ newfile=$(basename "$INNOMOVELOG")
+ newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP"
+ else
+ newfile="$INNOMOVELOG.$ARCHIVETIMESTAMP"
+ fi
+ wsrep_log_info "Moving '$INNOMOVELOG' to '$newfile'"
+ mv "$INNOMOVELOG" "$newfile"
+ gzip "$newfile"
+ fi
-cd "$WSREP_SST_OPT_DATA"
-if [ -n "$INNODB_DATA_HOME_DIR" ]; then
- # handle both relative and absolute paths
- [ ! -d "$INNODB_DATA_HOME_DIR" ] && mkdir -p "$INNODB_DATA_HOME_DIR"
- cd "$INNODB_DATA_HOME_DIR"
+ if [ -e "$INNOBACKUPLOG" ]
+ then
+ if [ -n "$sstlogarchivedir" ]
+ then
+ newfile=$(basename "$INNOBACKUPLOG")
+ newfile="$sstlogarchivedir/$newfile.$ARCHIVETIMESTAMP"
+ else
+ newfile="$INNOBACKUPLOG.$ARCHIVETIMESTAMP"
+ fi
+ wsrep_log_info "Moving '$INNOBACKUPLOG' to '$newfile'"
+ mv "$INNOBACKUPLOG" "$newfile"
+ gzip "$newfile"
+ fi
+ fi
+ INNOAPPLY="> '$INNOAPPLYLOG' 2>&1"
+ INNOMOVE="> '$INNOMOVELOG' 2>&1"
+ INNOBACKUP="2> '$INNOBACKUPLOG'"
fi
-INNODB_DATA_HOME_DIR=$(pwd -P)
-
-cd "$OLD_PWD"
-setup_commands () {
- INNOAPPLY="$INNOBACKUPEX_BIN $disver $iapts $INNOEXTRA --apply-log $rebuildcmd '$DATA' $INNOAPPLY"
- INNOMOVE="$INNOBACKUPEX_BIN $WSREP_SST_OPT_CONF $disver $impts --move-back --force-non-empty-directories '$DATA' $INNOMOVE"
+setup_commands()
+{
+ INNOAPPLY="$BACKUP_BIN $disver $iapts $INNOEXTRA --apply-log $rebuildcmd '$DATA' $INNOAPPLY"
+ INNOMOVE="$BACKUP_BIN $WSREP_SST_OPT_CONF --move-back $disver $impts --force-non-empty-directories '$DATA' $INNOMOVE"
sfmt_work="$sfmt"
if [ "$sfmt" = 'mbstream' ]; then
sfmt_work='xbstream'
fi
- INNOBACKUP="$INNOBACKUPEX_BIN $WSREP_SST_OPT_CONF $disver $iopts $tmpopts $INNOEXTRA --galera-info --stream=$sfmt_work '$itmpdir' $INNOBACKUP"
+ INNOBACKUP="$BACKUP_BIN $WSREP_SST_OPT_CONF $disver $iopts $tmpopts $INNOEXTRA --galera-info --stream=$sfmt_work '$itmpdir' $INNOBACKUP"
}
-if [ "$WSREP_SST_OPT_ROLE" = "donor" ]
+get_stream
+get_transfer
+
+if [ "$WSREP_SST_OPT_ROLE" = 'donor' ]
then
- trap cleanup_donor EXIT
+ trap cleanup_at_exit EXIT
if [ $WSREP_SST_OPT_BYPASS -eq 0 ]
then
- if [ -z "$WSREP_SST_OPT_SST_VER" ]; then
+ if [ -z "$sst_ver" ]; then
wsrep_log_error "Upgrade joiner to 5.6.21 or higher for backup locks support"
wsrep_log_error "The joiner is not supported for this version of donor"
exit 93
fi
- tmpdir=$(parse_cnf "--mysqld|sst|xtrabackup" 'tmpdir')
+ tmpdir=$(parse_cnf "$encgroups" 'tmpdir')
if [ -z "$tmpdir" ]; then
- xtmpdir=$(mktemp -d)
- tmpopts=" --tmpdir='$xtmpdir'"
- wsrep_log_info "Using $xtmpdir as xtrabackup temporary directory"
+ xtmpdir="$(mktemp -d)"
+ else
+ xtmpdir=$(mktemp '-d' "--tmpdir=$tmpdir")
fi
- itmpdir=$(mktemp -d)
- wsrep_log_info "Using $itmpdir as innobackupex temporary directory"
+ wsrep_log_info "Using '$xtmpdir' as xtrabackup temporary directory"
+ tmpopts="--tmpdir='$xtmpdir'"
+
+ itmpdir="$(mktemp -d)"
+ wsrep_log_info "Using '$itmpdir' as xtrabackup working directory"
usrst=0
if [ -n "$WSREP_SST_OPT_USER" ]; then
@@ -887,37 +972,41 @@ then
fi
if [ -n "$WSREP_SST_OPT_PSWD" ]; then
- export MYSQL_PWD="$WSREP_SST_OPT_PSWD"
+ INNOEXTRA="$INNOEXTRA --password='$WSREP_SST_OPT_PSWD'"
elif [ $usrst -eq 1 ]; then
# Empty password, used for testing, debugging etc.
- unset MYSQL_PWD
+ INNOEXTRA="$INNOEXTRA --password="
fi
- get_keys
check_extra
wsrep_log_info "Streaming GTID file before SST"
# Store donor's wsrep GTID (state ID) and wsrep_gtid_domain_id
# (separated by a space).
- echo "${WSREP_SST_OPT_GTID} ${WSREP_SST_OPT_GTID_DOMAIN_ID}" > "${MAGIC_FILE}"
+ echo "$WSREP_SST_OPT_GTID $WSREP_SST_OPT_GTID_DOMAIN_ID" > "$MAGIC_FILE"
+
+ if [ -n "$WSREP_SST_OPT_REMOTE_PSWD" ]; then
+ # Let joiner know that we know its secret
+ echo "$SECRET_TAG $WSREP_SST_OPT_REMOTE_PSWD" >> "$MAGIC_FILE"
+ fi
ttcmd="$tcmd"
- if [ $encrypt -eq 1 ]; then
- if [ -n "$scomp" ]; then
- tcmd="\$ecmd | $scomp | $tcmd"
- else
- tcmd="\$ecmd | $tcmd"
- fi
- elif [ -n "$scomp" ]; then
+ if [ -n "$scomp" ]; then
tcmd="$scomp | $tcmd"
fi
- send_donor $DATA "${stagemsg}-gtid"
+ get_keys
+ if [ $encrypt -eq 1 ]; then
+ tcmd="$ecmd | $tcmd"
+ fi
+
+ send_donor "$DATA" "$stagemsg-gtid"
# Restore the transport commmand to its original state
tcmd="$ttcmd"
+
if [ -n "$progress" ]; then
get_footprint
tcmd="$pcmd | $tcmd"
@@ -929,34 +1018,57 @@ then
wsrep_log_info "Sleeping before data transfer for SST"
sleep 10
- wsrep_log_info "Streaming the backup to joiner at ${WSREP_SST_OPT_HOST}:${WSREP_SST_OPT_PORT}"
+ wsrep_log_info "Streaming the backup to joiner at $REMOTEIP:$SST_PORT"
# Add compression to the head of the stream (if specified)
- if [[ -n "$scomp" ]]; then
+ if [ -n "$scomp" ]; then
tcmd="$scomp | $tcmd"
fi
# Add encryption to the head of the stream (if specified)
- if [[ $encrypt -eq 1 ]]; then
- tcmd="\$ecmd | $tcmd"
+ if [ $encrypt -eq 1 ]; then
+ tcmd="$ecmd | $tcmd"
+ fi
+
+ iopts="--databases-exclude='lost+found' $iopts"
+
+ if [ ${FORCE_FTWRL:-0} -eq 1 ]; then
+ wsrep_log_info "Forcing FTWRL due to environment variable FORCE_FTWRL equal to $FORCE_FTWRL"
+ iopts="--no-backup-locks $iopts"
+ fi
+
+ # if compression is enabled for backup files, then add the
+ # appropriate options to the innobackupex command line:
+ if [ "$compress" != 'none' ]; then
+ iopts="--compress${compress:+=$compress} $iopts"
+ if [ -n "$compress_threads" ]; then
+ iopts="--compress-threads=$compress_threads $iopts"
+ fi
+ if [ -n "$compress_chunk" ]; then
+ iopts="--compress-chunk-size=$compress_chunk $iopts"
+ fi
+ fi
+
+ if [ -n "$backup_threads" ]; then
+ iopts="--parallel=$backup_threads $iopts"
fi
setup_commands
set +e
- timeit "${stagemsg}-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )"
+ timeit "$stagemsg-SST" "$INNOBACKUP | $tcmd; RC=( "\${PIPESTATUS[@]}" )"
set -e
if [ ${RC[0]} -ne 0 ]; then
- wsrep_log_error "${INNOBACKUPEX_BIN} finished with error: ${RC[0]}. " \
- "Check ${DATA}/innobackup.backup.log"
- exit 22
- elif [[ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]];then
- wsrep_log_error "$tcmd finished with error: ${RC[1]}"
- exit 22
+ wsrep_log_error "innobackupex finished with error: ${RC[0]}. " \
+ "Check syslog or '$INNOBACKUPLOG' for details"
+ exit 22
+ elif [ ${RC[$(( ${#RC[@]}-1 ))]} -eq 1 ]; then
+ wsrep_log_error "$tcmd finished with error: ${RC[1]}"
+ exit 22
fi
# innobackupex implicitly writes PID to fixed location in $xtmpdir
- XTRABACKUP_PID="$xtmpdir/xtrabackup_pid"
+ BACKUP_PID="$xtmpdir/xtrabackup_pid"
else # BYPASS FOR IST
@@ -965,31 +1077,31 @@ then
# Store donor's wsrep GTID (state ID) and wsrep_gtid_domain_id
# (separated by a space).
- echo "${WSREP_SST_OPT_GTID} ${WSREP_SST_OPT_GTID_DOMAIN_ID}" > "${MAGIC_FILE}"
- echo "1" > "${DATA}/${IST_FILE}"
+ echo "$WSREP_SST_OPT_GTID $WSREP_SST_OPT_GTID_DOMAIN_ID" > "$MAGIC_FILE"
+ echo "1" > "$DATA/$IST_FILE"
+
+ if [ -n "$scomp" ]; then
+ tcmd="$scomp | $tcmd"
+ fi
+
get_keys
if [ $encrypt -eq 1 ]; then
- if [ -n "$scomp" ]; then
- tcmd="\$ecmd | $scomp | $tcmd"
- else
- tcmd="\$ecmd | $tcmd"
- fi
- elif [ -n "$scomp" ]; then
- tcmd="$scomp | $tcmd"
+ tcmd="$ecmd | $tcmd"
fi
- strmcmd+=" \${IST_FILE}"
- send_donor "$DATA" "${stagemsg}-IST"
+ strmcmd="$strmcmd '$IST_FILE'"
+
+ send_donor "$DATA" "$stagemsg-IST"
fi
- echo "done ${WSREP_SST_OPT_GTID}"
+ echo "done $WSREP_SST_OPT_GTID"
wsrep_log_info "Total time on donor: $totime seconds"
-elif [ "${WSREP_SST_OPT_ROLE}" = "joiner" ]
+elif [ "$WSREP_SST_OPT_ROLE" = 'joiner' ]
then
- [[ -e "$SST_PROGRESS_FILE" ]] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE"
- [[ -n "$SST_PROGRESS_FILE" ]] && touch "$SST_PROGRESS_FILE"
+ [ -e "$SST_PROGRESS_FILE" ] && wsrep_log_info "Stale sst_in_progress file: $SST_PROGRESS_FILE"
+ [ -n "$SST_PROGRESS_FILE" ] && touch "$SST_PROGRESS_FILE"
ib_home_dir="$INNODB_DATA_HOME_DIR"
@@ -1008,66 +1120,101 @@ then
ib_undo_dir="$INNODB_UNDO_DIR"
- stagemsg="Joiner-Recv"
+ if [ -n "$backup_threads" ]; then
+ impts="--parallel=$backup_threads $impts"
+ fi
+
+ stagemsg='Joiner-Recv'
sencrypted=1
nthreads=1
MODULE="xtrabackup_sst"
- rm -f "${DATA}/${IST_FILE}"
+ [ -f "$DATA/$IST_FILE" ] && rm -f "$DATA/$IST_FILE"
# May need xtrabackup_checkpoints later on
- rm -f "${DATA}/xtrabackup_binary" "${DATA}/xtrabackup_galera_info" "${DATA}/xtrabackup_logfile"
+ [ -f "$DATA/xtrabackup_binary" ] && rm -f "$DATA/xtrabackup_binary"
+ [ -f "$DATA/xtrabackup_galera_info" ] && rm -f "$DATA/xtrabackup_galera_info"
+ [ -f "$DATA/ib_logfile0" ] && rm -f "$DATA/ib_logfile0"
+
+ ADDR="$WSREP_SST_OPT_ADDR"
- wait_for_listen ${WSREP_SST_OPT_HOST} ${WSREP_SST_OPT_PORT} ${MODULE} &
+ if [ "${tmode#VERIFY}" != "$tmode" ]
+ then # backward-incompatible behavior
+ CN=""
+ if [ -n "$tpem" ]
+ then
+ # find out my Common Name
+ get_openssl
+ if [ -z "$OPENSSL_BINARY" ]; then
+ wsrep_log_error 'openssl not found but it is required for authentication'
+ exit 42
+ fi
+ CN=$("$OPENSSL_BINARY" x509 -noout -subject -in "$tpem" | \
+ tr "," "\n" | grep -F 'CN =' | cut -d= -f2 | sed s/^\ // | \
+ sed s/\ %//)
+ fi
+ MY_SECRET="$(wsrep_gen_secret)"
+ # Add authentication data to address
+ ADDR="$CN:$MY_SECRET@$ADDR"
+ else
+ MY_SECRET="" # for check down in recv_joiner()
+ fi
trap sig_joiner_cleanup HUP PIPE INT TERM
- trap cleanup_joiner EXIT
+ trap cleanup_at_exit EXIT
- if [[ -n "$progress" ]];then
+ if [ -n "$progress" ]; then
adjust_progress
- tcmd+=" | $pcmd"
+ tcmd="$tcmd | $pcmd"
fi
get_keys
if [ $encrypt -eq 1 -a $sencrypted -eq 1 ]; then
- if [ -n "$sdecomp" ]; then
- strmcmd="$sdecomp | \$ecmd | $strmcmd"
- else
- strmcmd="\$ecmd | $strmcmd"
- fi
- elif [ -n "$sdecomp" ]; then
- strmcmd="$sdecomp | $strmcmd"
+ strmcmd="$ecmd | $strmcmd"
fi
- STATDIR=$(mktemp -d)
- MAGIC_FILE="${STATDIR}/${INFO_FILE}"
- recv_joiner "$STATDIR" "${stagemsg}-gtid" $stimeout 1
+ if [ -n "$sdecomp" ]; then
+ strmcmd="$sdecomp | $strmcmd"
+ fi
+
+ check_sockets_utils
+
+ STATDIR="$(mktemp -d)"
+ MAGIC_FILE="$STATDIR/$INFO_FILE"
- if ! ps -p ${WSREP_SST_OPT_PARENT} &>/dev/null
+ recv_joiner "$STATDIR" "$stagemsg-gtid" $stimeout 1 1
+
+ if ! ps -p "$WSREP_SST_OPT_PARENT" >/dev/null 2>&1
then
wsrep_log_error "Parent mysqld process (PID: $WSREP_SST_OPT_PARENT) terminated unexpectedly."
exit 32
fi
- if [ ! -r "${STATDIR}/${IST_FILE}" ]
+ if [ ! -r "$STATDIR/$IST_FILE" ]
then
- if [[ -d "${DATA}/.sst" ]];then
- wsrep_log_info "WARNING: Stale temporary SST directory: ${DATA}/.sst from previous state transfer. Removing"
- rm -rf "${DATA}/.sst"
+ if [ -d "$DATA/.sst" ]; then
+ wsrep_log_info "WARNING: Stale temporary SST directory: '$DATA/.sst' from previous state transfer. Removing"
+ rm -rf "$DATA/.sst"
fi
- mkdir -p "${DATA}/.sst"
- (recv_joiner "$DATA/.sst" "${stagemsg}-SST" 0 0) &
+ mkdir -p "$DATA/.sst"
+ (recv_joiner "$DATA/.sst" "$stagemsg-SST" 0 0 0) &
jpid=$!
wsrep_log_info "Proceeding with SST"
wsrep_log_info "Cleaning the existing datadir and innodb-data/log directories"
- if [ "${OS}" = "FreeBSD" ]; then
- find -E $ib_home_dir $ib_log_dir $ib_undo_dir $DATA -mindepth 1 -prune -regex $cpat -o -exec rm -rfv {} 1>&2 \+
+ if [ "$OS" = 'FreeBSD' ]; then
+ find -E ${ib_home_dir:+"$ib_home_dir"} \
+ ${ib_undo_dir:+"$ib_undo_dir"} \
+ ${ib_log_dir:+"$ib_log_dir"} \
+ "$DATA" -mindepth 1 -prune -regex "$cpat" -o -exec rm -rfv {} 1>&2 \+
else
- find $ib_home_dir $ib_log_dir $ib_undo_dir $DATA -mindepth 1 -prune -regex $cpat -o -exec rm -rfv {} 1>&2 \+
+ find ${ib_home_dir:+"$ib_home_dir"} \
+ ${ib_undo_dir:+"$ib_undo_dir"} \
+ ${ib_log_dir:+"$ib_log_dir"} \
+ "$DATA" -mindepth 1 -prune -regex "$cpat" -o -exec rm -rfv {} 1>&2 \+
fi
get_binlog
@@ -1082,50 +1229,46 @@ then
cd "$OLD_PWD"
fi
- TDATA="${DATA}"
- DATA="${DATA}/.sst"
+ TDATA="$DATA"
+ DATA="$DATA/.sst"
- MAGIC_FILE="${DATA}/${INFO_FILE}"
+ MAGIC_FILE="$DATA/$INFO_FILE"
wsrep_log_info "Waiting for SST streaming to complete!"
monitor_process $jpid
- get_proc
-
- if [[ ! -s "${DATA}/xtrabackup_checkpoints" ]];then
- wsrep_log_error "xtrabackup_checkpoints missing, failed innobackupex/SST on donor"
+ if [ ! -s "$DATA/xtrabackup_checkpoints" ]; then
+ wsrep_log_error "xtrabackup_checkpoints missing, failed xtrabackup/SST on donor"
exit 2
fi
# Rebuild indexes for compact backups
- if grep -q 'compact = 1' "${DATA}/xtrabackup_checkpoints";then
+ if grep -q -F 'compact = 1' "$DATA/xtrabackup_checkpoints"; then
wsrep_log_info "Index compaction detected"
- rebuild=1
- fi
-
- if [[ $rebuild -eq 1 ]];then
- nthreads=$(parse_cnf xtrabackup rebuild-threads $nproc)
+ get_proc
+ nthreads=$(parse_cnf "$encgroups" 'rebuild-threads' $nproc)
wsrep_log_info "Rebuilding during prepare with $nthreads threads"
- rebuildcmd="--rebuild-indexes --rebuild-threads='$nthreads'"
+ rebuildcmd="--rebuild-indexes --rebuild-threads=$nthreads"
fi
- qpfiles=$(find "${DATA}" -maxdepth 1 -type f -name '*.qp' -print -quit)
+ qpfiles=$(find "$DATA" -maxdepth 1 -type f -name '*.qp' -print -quit)
if [ -n "$qpfiles" ]; then
wsrep_log_info "Compressed qpress files found"
if [ -z "$(command -v qpress)" ]; then
- wsrep_log_error "qpress not found in path: $PATH"
+ wsrep_log_error "qpress utility not found in the path"
exit 22
fi
+ get_proc
+
dcmd="xargs -n 2 qpress -dT$nproc"
- if [[ -n "$progress" ]] && pv --help | grep -qw '--line-mode';then
- count=$(find "${DATA}" -type f -name '*.qp' | wc -l)
+ if [ -n "$progress" ] && pv --help | grep -qw -- '--line-mode'; then
+ count=$(find "$DATA" -type f -name '*.qp' | wc -l)
count=$(( count*2 ))
- if pv --help | grep -qw -F '-F';then
- pvopts="-f -s $count -l -N Decompression -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'"
- else
- pvopts="-f -s $count -l -N Decompression"
+ pvopts="-f -s $count -l -N Decompression"
+ if pv --help | grep -qw -- '-F'; then
+ pvopts="$pvopts -F '%N => Rate:%r Elapsed:%t %e Progress: [%b/$count]'"
fi
pcmd="pv $pvopts"
adjust_progress
@@ -1134,13 +1277,13 @@ then
# Decompress the qpress files
wsrep_log_info "Decompression with $nproc threads"
- timeit "Joiner-Decompression" "find ${DATA} -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd"
+ timeit "Joiner-Decompression" "find '$DATA' -type f -name '*.qp' -printf '%p\n%h\n' | $dcmd"
extcode=$?
- if [[ $extcode -eq 0 ]];then
+ if [ $extcode -eq 0 ]; then
wsrep_log_info "Removing qpress files after decompression"
find "$DATA" -type f -name '*.qp' -delete
- if [[ $? -ne 0 ]];then
+ if [ $? -ne 0 ]; then
wsrep_log_error "Something went wrong with deletion of qpress files. Investigate"
fi
else
@@ -1165,38 +1308,39 @@ then
fi
- wsrep_log_info "Preparing the backup at ${DATA}"
+ wsrep_log_info "Preparing the backup at $DATA"
setup_commands
timeit "Xtrabackup prepare stage" "$INNOAPPLY"
- if [ $? -ne 0 ];
- then
- wsrep_log_error "${INNOBACKUPEX_BIN} apply finished with errors. Check ${DATA}/innobackup.prepare.log"
+ if [ $? -ne 0 ]; then
+ wsrep_log_error "xtrabackup apply finished with errors. Check '$INNOAPPLYLOG' for details"
exit 22
fi
- MAGIC_FILE="${TDATA}/${INFO_FILE}"
- set +e
- rm "$TDATA/innobackup.prepare.log" "$TDATA/innobackup.move.log"
- set -e
- wsrep_log_info "Moving the backup to ${TDATA}"
+ # [ -f "$INNOAPPLYLOG" ] && rm "$INNOAPPLYLOG"
+
+ MAGIC_FILE="$TDATA/$INFO_FILE"
+
+ wsrep_log_info "Moving the backup to $TDATA"
timeit "Xtrabackup move stage" "$INNOMOVE"
- if [[ $? -eq 0 ]];then
- wsrep_log_info "Move successful, removing ${DATA}"
+ if [ $? -eq 0 ]; then
+ wsrep_log_info "Move successful, removing $DATA"
rm -rf "$DATA"
- DATA="${TDATA}"
+ DATA="$TDATA"
else
- wsrep_log_error "Move failed, keeping ${DATA} for further diagnosis"
- wsrep_log_error "Check ${DATA}/innobackup.move.log for details"
+ wsrep_log_error "Move failed, keeping '$DATA' for further diagnosis"
+ wsrep_log_error "Check syslog or '$INNOMOVELOG' for details"
exit 22
fi
else
- wsrep_log_info "${IST_FILE} received from donor: Running IST"
+
+ wsrep_log_info "'$IST_FILE' received from donor: Running IST"
+
fi
- if [[ ! -r "${MAGIC_FILE}" ]];then
- wsrep_log_error "SST magic file ${MAGIC_FILE} not found/readable"
+ if [ ! -r "$MAGIC_FILE" ]; then
+ wsrep_log_error "SST magic file '$MAGIC_FILE' not found/readable"
exit 2
fi
diff --git a/scripts/wsrep_sst_xtrabackup.sh b/scripts/wsrep_sst_xtrabackup.sh
index c6dad8d1791..b84aca4866e 100644
--- a/scripts/wsrep_sst_xtrabackup.sh
+++ b/scripts/wsrep_sst_xtrabackup.sh
@@ -97,8 +97,9 @@ get_keys()
fi
if [[ $encrypt -eq 0 ]];then
- if $MY_PRINT_DEFAULTS xtrabackup | grep -q encrypt;then
- wsrep_log_error "Unexpected option combination. SST may fail. Refer to http://www.percona.com/doc/percona-xtradb-cluster/manual/xtrabackup_sst.html "
+ if [ -n "$ealgo" -o -n "$ekey" -o -n "$ekeyfile" ]; then
+ wsrep_log_error "Options for encryption are specified, " \
+ "but encryption itself is disabled. SST may fail."
fi
return
fi
@@ -324,12 +325,6 @@ cleanup_joiner()
fi
}
-check_pid()
-{
- local pid_file="$1"
- [ -r "$pid_file" ] && ps -p $(cat "$pid_file") >/dev/null 2>&1
-}
-
cleanup_donor()
{
# Since this is invoked just after exit NNN
@@ -339,12 +334,11 @@ cleanup_donor()
fi
if [[ -n "$XTRABACKUP_PID" ]];then
- if check_pid "$XTRABACKUP_PID"
+ if check_pid "$XTRABACKUP_PID" 1
then
wsrep_log_error "xtrabackup process is still running. Killing... "
- kill_xtrabackup
+ cleanup_pid $CHECK_PID "$XTRABACKUP_PID"
fi
- rm -f "$XTRABACKUP_PID"
fi
rm -f "${DATA}/${IST_FILE}"
@@ -355,13 +349,6 @@ cleanup_donor()
fi
}
-kill_xtrabackup()
-{
- local PID=$(cat "$XTRABACKUP_PID")
- [ -n "$PID" -a $PID -ne 0 ] && kill $PID && (kill $PID && kill -9 $PID) || :
- rm -f "$XTRABACKUP_PID"
-}
-
# waits ~10 seconds for nc to open the port and then reports ready
# (regardless of timeout)
wait_for_listen()