ctdb-scripts: New consistent system memory and swap monitoring

New variables CTDB_MONITOR_MEMORY_USAGE and CTDB_MONITOR_SWAP_USAGE.
Both take a pair of <warn_threshold>:<unhealthy_threshold> where each
theshold is specified as a percentage.

This adds a callout to check_thresholds() that is run when the
unhealthy threshold is reached.

Add some combination tests.

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>

8 files changed, 78 insertions, 24 deletions

diff --git a/ctdb/tests/eventscripts/05.system.monitor.011.sh b/ctdb/tests/eventscripts/05.system.monitor.011.sh
index 79f55f06a91..79ceb90b768 100755
--- a/ctdb/tests/eventscripts/05.system.monitor.011.sh
+++ b/ctdb/tests/eventscripts/05.system.monitor.011.sh
@@ -6,9 +6,8 @@ define_test "Memory check, bad situation, no checks enabled"
 
 setup_memcheck 100 100
 
-CTDB_MONITOR_FREE_MEMORY=""
-CTDB_MONITOR_FREE_MEMORY_WARN=""
-CTDB_CHECK_SWAP_IS_NOT_USED="no"
+CTDB_MONITOR_MEMORY_USAGE=""
+CTDB_MONITOR_SWAP_USAGE=""
 
 ok_null
 
diff --git a/ctdb/tests/eventscripts/05.system.monitor.012.sh b/ctdb/tests/eventscripts/05.system.monitor.012.sh
index 6c06480824a..bb2c7b57811 100755
--- a/ctdb/tests/eventscripts/05.system.monitor.012.sh
+++ b/ctdb/tests/eventscripts/05.system.monitor.012.sh
@@ -6,9 +6,8 @@ define_test "Memory check, good situation, all enabled"
 
 setup_memcheck
 
-CTDB_MONITOR_FREE_MEMORY="90"
-CTDB_MONITOR_FREE_MEMORY_WARN="80"
-CTDB_CHECK_SWAP_IS_NOT_USED="yes"
+CTDB_MONITOR_MEMORY_USAGE="80:90"
+CTDB_MONITOR_SWAP_USAGE="1:50"
 
 ok_null
 
diff --git a/ctdb/tests/eventscripts/05.system.monitor.013.sh b/ctdb/tests/eventscripts/05.system.monitor.013.sh
index dc3d40d0fa9..25fa780d89e 100755
--- a/ctdb/tests/eventscripts/05.system.monitor.013.sh
+++ b/ctdb/tests/eventscripts/05.system.monitor.013.sh
@@ -4,16 +4,17 @@
 
 define_test "Memory check, bad situation, only swap check"
 
-setup_memcheck 100 10
+setup_memcheck 100 90
 
-CTDB_MONITOR_FREE_MEMORY=""
-CTDB_MONITOR_FREE_MEMORY_WARN=""
-CTDB_CHECK_SWAP_IS_NOT_USED="yes"
+CTDB_MONITOR_MEMORY_USAGE=""
+CTDB_MONITOR_SWAP_USAGE=":50"
 
-ok <<EOF
-We are swapping:
+required_result 1 <<EOF
+ERROR: System swap utilization 90% >= threshold 50%
+CRITICAL: Shutting down CTDB!!!
 $FAKE_PROC_MEMINFO
 $(ps foobar)
+CTDB says BYE!
 EOF
 
 simple_test
diff --git a/ctdb/tests/eventscripts/05.system.monitor.014.sh b/ctdb/tests/eventscripts/05.system.monitor.014.sh
index 64c07416445..46955f34020 100755
--- a/ctdb/tests/eventscripts/05.system.monitor.014.sh
+++ b/ctdb/tests/eventscripts/05.system.monitor.014.sh
@@ -6,12 +6,11 @@ define_test "Memory check, bad situation, only memory warning"
 
 setup_memcheck 90 10
 
-CTDB_MONITOR_FREE_MEMORY=""
-CTDB_MONITOR_FREE_MEMORY_WARN="85"
-CTDB_CHECK_SWAP_IS_NOT_USED="no"
+CTDB_MONITOR_MEMORY_USAGE="85:"
+CTDB_MONITOR_SWAP_USAGE=""
 
 ok <<EOF
-WARNING: memory usage is excessive - 90% >=  85% (CTDB threshold)
+WARNING: System memory utilization 90% >= threshold 85%
 EOF
 
 simple_test
diff --git a/ctdb/tests/eventscripts/05.system.monitor.015.sh b/ctdb/tests/eventscripts/05.system.monitor.015.sh
index e950bbd276e..3beac4cc91d 100755
--- a/ctdb/tests/eventscripts/05.system.monitor.015.sh
+++ b/ctdb/tests/eventscripts/05.system.monitor.015.sh
@@ -6,12 +6,11 @@ define_test "Memory check, bad situation, only memory critical"
 
 setup_memcheck 90 0
 
-CTDB_MONITOR_FREE_MEMORY="85"
-CTDB_MONITOR_FREE_MEMORY_WARN=""
-CTDB_CHECK_SWAP_IS_NOT_USED="no"
+CTDB_MONITOR_MEMORY_USAGE=":85"
+CTDB_MONITOR_SWAP_USAGE=""
 
-ok <<EOF
-CRITICAL: OOM - 90% usage >= 85% (CTDB threshold)
+required_result 1 <<EOF
+ERROR: System memory utilization 90% >= threshold 85%
 CRITICAL: Shutting down CTDB!!!
 $FAKE_PROC_MEMINFO
 $(ps foobar)
diff --git a/ctdb/tests/eventscripts/05.system.monitor.016.sh b/ctdb/tests/eventscripts/05.system.monitor.016.sh
new file mode 100755
index 00000000000..44dddc688a9
--- /dev/null
+++ b/ctdb/tests/eventscripts/05.system.monitor.016.sh
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "Memory check, bad situation, both memory checks, causes warning"
+
+setup_memcheck 87 0
+
+CTDB_MONITOR_MEMORY_USAGE="80:90"
+CTDB_MONITOR_SWAP_USAGE=""
+
+ok <<EOF
+WARNING: System memory utilization 87% >= threshold 80%
+EOF
+
+simple_test
diff --git a/ctdb/tests/eventscripts/05.system.monitor.017.sh b/ctdb/tests/eventscripts/05.system.monitor.017.sh
new file mode 100755
index 00000000000..f1b6a26ec2e
--- /dev/null
+++ b/ctdb/tests/eventscripts/05.system.monitor.017.sh
@@ -0,0 +1,42 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "Memory check, bad situation, both memory checks, causes unhealthy"
+
+setup_memcheck 87 0
+
+CTDB_MONITOR_MEMORY_USAGE="70:80"
+CTDB_MONITOR_SWAP_USAGE=""
+
+required_result 1 <<EOF
+ERROR: System memory utilization 87% >= threshold 80%
+CRITICAL: Shutting down CTDB!!!
+MemTotal:        3940712 kB
+MemFree:          225268 kB
+Buffers:          146120 kB
+Cached:          140904 kB
+SwapCached:        56016 kB
+Active:          2422104 kB
+Inactive:        1019928 kB
+Active(anon):    1917580 kB
+Inactive(anon):   523080 kB
+Active(file):     504524 kB
+Inactive(file):   496848 kB
+Unevictable:        4844 kB
+Mlocked:            4844 kB
+SwapTotal:       5857276 kB
+SwapFree:        5857276 kB
+...
+USER       PID %CPU %MEM    VSZ   RSS TTY      STAT START   TIME COMMAND
+root         2  0.0  0.0      0     0 ?        S    Aug28   0:00 [kthreadd]
+root         3  0.0  0.0      0     0 ?        S    Aug28   0:43  \_ [ksoftirqd/0]
+...
+root         1  0.0  0.0   2976   624 ?        Ss   Aug28   0:07 init [2]
+root       495  0.0  0.0   3888  1640 ?        Ss   Aug28   0:00 udevd --daemon
+...
+[MORE FAKE ps OUTPUT]
+CTDB says BYE!
+EOF
+
+simple_test
diff --git a/ctdb/tests/eventscripts/scripts/local.sh b/ctdb/tests/eventscripts/scripts/local.sh
index ce1c2510bd0..57e022536cd 100644
--- a/ctdb/tests/eventscripts/scripts/local.sh
+++ b/ctdb/tests/eventscripts/scripts/local.sh
@@ -369,9 +369,8 @@ SwapTotal:       ${_swap_total} kB
 SwapFree:        ${_swap_free} kB
 ..."
 
-    export CTDB_MONITOR_FREE_MEMORY
-    export CTDB_MONITOR_FREE_MEMORY_WARN
-    export CTDB_CHECK_SWAP_IS_NOT_USED
+    export CTDB_MONITOR_MEMORY_USAGE
+    export CTDB_MONITOR_SWAP_USAGE
 }
 
 setup_fscheck ()