blob: d8096ee46c60d9c0d81d8c8a5f4eb1034abd6a62 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
|
#!/bin/sh
# Event script for ctdb-specific setup and other things that don't fit
# elsewhere.
[ -n "$CTDB_BASE" ] || \
export CTDB_BASE=$(cd -P $(dirname "$0") ; dirname "$PWD")
. $CTDB_BASE/functions
loadconfig
ctdb_setup_service_state_dir "ctdb"
############################################################
select_tdb_checker ()
{
# Find the best TDB consistency check available.
use_tdb_tool_check=false
which tdbtool >/dev/null 2>&1 && found_tdbtool=true
which tdbdump >/dev/null 2>&1 && found_tdbdump=true
if $found_tdbtool && echo "help" | tdbtool | grep -q check ; then
use_tdb_tool_check=true
elif $found_tdbtool && $found_tdbdump ; then
cat <<EOF
WARNING: The installed 'tdbtool' does not offer the 'check' subcommand.
Using 'tdbdump' for database checks.
Consider updating 'tdbtool' for better checks!
EOF
elif $found_tdbdump ; then
cat <<EOF
WARNING: 'tdbtool' is not available.
Using 'tdbdump' to check the databases.
Consider installing a recent 'tdbtool' for better checks!
EOF
else
cat <<EOF
WARNING: Cannot check databases since neither
'tdbdump' nor 'tdbtool check' is available.
Consider installing tdbtool or at least tdbdump!
EOF
return 1
fi
}
check_tdb ()
{
_db="$1"
if $use_tdb_tool_check ; then
# tdbtool always exits with 0 :-(
if timeout 10 tdbtool "$_db" check 2>/dev/null |
grep -q "Database integrity is OK" ; then
return 0
else
return 1
fi
else
timeout 10 tdbdump "$_db" >/dev/null 2>/dev/null
return $?
fi
}
check_persistent_databases ()
{
_dir="${CTDB_DBDIR_PERSISTENT:-${CTDB_DBDIR:-${CTDB_VARDIR}}/persistent}"
mkdir -p "$_dir" 2>/dev/null
[ "${CTDB_MAX_PERSISTENT_CHECK_ERRORS:-0}" = "0" ] || return 0
for _db in $(ls "$_dir/"*.tdb.*[0-9] 2>/dev/null) ; do
check_tdb $_db || {
echo "Persistent database $_db is corrupted! CTDB will not start."
return 1
}
done
}
check_non_persistent_databases ()
{
_dir="${CTDB_DBDIR:-${CTDB_VARDIR}}"
mkdir -p "$_dir" 2>/dev/null
for _db in $(ls "${_dir}/"*.tdb.*[0-9] 2>/dev/null) ; do
check_tdb $_db || {
_backup="${_db}.$(date +'%Y%m%d.%H%M%S.%N').corrupt"
cat <<EOF
WARNING: database ${_db} is corrupted.
Moving to backup ${_backup} for later analysis.
EOF
mv "$_db" "$_backup"
# Now remove excess backups
ls -td "${_db}."*".corrupt" |
tail -n +$((${CTDB_MAX_CORRUPT_DB_BACKUPS:-10} + 1)) |
xargs rm -f
}
done
}
update_config_from_tdb() {
# Pull optional ctdb configuration data out of config.tdb
ctdb_get_pnn
_key="public_addresses:node#${pnn}"
_t="$service_state_dir/public_addresses"
rm -f "$_t"
if ctdb pfetch config.tdb "$_key" "$_t" 2>/dev/null && \
[ -s "$_t" -a -n "$CTDB_PUBLIC_ADDRESSES"] && \
! cmp -s "$_t" "$CTDB_PUBLIC_ADDRESSES" ; then
echo "CTDB public address configuration has changed."
echo "Extracting new configuration from database."
diff "$_t" "$CTDB_PUBLIC_ADDRESSES"
cp "$_t" "$CTDB_PUBLIC_ADDRESSES"
echo "Restarting CTDB"
service ctdb restart &
fi
}
set_ctdb_variables ()
{
# set any tunables from the config file
set | sed -n '/^CTDB_SET_/s/=.*//p' |
while read v; do
varname="${v#CTDB_SET_}"
value=$(eval echo "\$$v")
if ctdb setvar $varname $value ; then
echo "Set $varname to $value"
else
echo "Invalid configuration: CTDB_SET_${varname}=${value}"
return 1
fi
done
}
monitor_system_memory ()
{
# If monitoring free memory then calculate how much there is
if [ -n "$CTDB_MONITOR_FREE_MEMORY_WARN" -o \
-n "$CTDB_MONITOR_FREE_MEMORY" ] ; then
free_mem=$(free -m | awk '$2 == "buffers/cache:" { print $4 }')
fi
# Shutdown CTDB when memory is below the configured limit
if [ -n "$CTDB_MONITOR_FREE_MEMORY" ] ; then
if [ $free_mem -le $CTDB_MONITOR_FREE_MEMORY ] ; then
echo "CRITICAL: OOM - ${free_mem}MB free <= ${CTDB_MONITOR_FREE_MEMORY}MB (CTDB threshold)"
echo "CRITICAL: Shutting down CTDB!!!"
get_proc "meminfo"
ps auxfww
set_proc "sysrq-trigger" "m"
ctdb disable
sleep 3
ctdb shutdown
fi
fi
# Warn when low on memory
if [ -n "$CTDB_MONITOR_FREE_MEMORY_WARN" ] ; then
if [ $free_mem -le $CTDB_MONITOR_FREE_MEMORY_WARN ] ; then
echo "WARNING: free memory is low - ${free_mem}MB free <= ${CTDB_MONITOR_FREE_MEMORY_WARN}MB (CTDB threshold)"
fi
fi
# We should never enter swap, so SwapTotal == SwapFree.
if [ "$CTDB_CHECK_SWAP_IS_NOT_USED" = "yes" ] ; then
set -- $(get_proc "meminfo" | awk '$1 ~ /Swap(Total|Free):/ { print $2 }')
if [ "$1" != "$2" ] ; then
echo We are swapping:
get_proc "meminfo"
ps auxfww
fi
fi
}
############################################################
ctdb_check_args "$@"
case "$1" in
init)
# make sure we have a blank state directory for the scripts to work with
rm -rf $CTDB_VARDIR/state
# Look at the pattern - this should not be -rf!!!
rm -f $ctdb_managed_dir/*
mkdir -p $CTDB_VARDIR/state || {
ret=$?
echo "mkdir -p $CTDB_VARDIR/state - failed - $ret"
exit $ret
}
# make sure we drop any ips that might still be held if
# previous instance of ctdb got killed with -9 or similar
drop_all_public_ips
if select_tdb_checker ; then
check_persistent_databases || exit $?
check_non_persistent_databases
fi
;;
setup)
# Set any tunables from the config file
set_ctdb_variables || \
die "Aborting setup due to invalid configuration - fix typos, remove unknown tunables"
;;
startup)
ctdb attach ctdb.tdb persistent
update_config_from_tdb &
;;
monitor)
monitor_system_memory
;;
*)
ctdb_standard_event_handler "$@"
;;
esac
# all OK
exit 0
|
