ctdb-scripts: Compact server-end TCP connection killing output

When thousands of connections are being killed the logs are flooded
with information about connections that should be killed.  When some
connections are not killed then the number not killed is printed.
This is the wrong way around!  When debugging "fail-back" problems, it
is important to know details of connections that were *not* killed.
It is almost never important to know the full list of all connections
that were *supposed* to be killed.

Instead, print a summary showing how many connections of the total
were killed.  If any were not killed then print a list of remaining
connections.

Update unit tests: infrastructure for fake TCP connections, existing,
test cases, add new test cases.

Signed-off-by: Martin Schwenke <martin@meltin.net>
Reviewed-by: Amitay Isaacs <amitay@gmail.com>

6 files changed, 127 insertions, 44 deletions

diff --git a/ctdb/tests/eventscripts/10.interface.releaseip.010.sh b/ctdb/tests/eventscripts/10.interface.releaseip.010.sh
index b6d9c7a8bd6..095e85c9b7c 100755
--- a/ctdb/tests/eventscripts/10.interface.releaseip.010.sh
+++ b/ctdb/tests/eventscripts/10.interface.releaseip.010.sh
@@ -8,25 +8,15 @@ setup_ctdb
 
 ctdb_get_1_public_address |
 while read dev ip bits ; do
-    ip addr add "${ip}/${bits}" dev "$dev"
+	ip addr add "${ip}/${bits}" dev "$dev"
 
-    # Setup 10 fake connections...
-    count=10
-    out=""
-    nl="
-"
-    i=0
-    while [ $i -lt $count ] ; do
-	echo "${ip}:445 10.254.254.1:1230${i}"
-	# Expected output for killing this connection
-	out="${out}${out:+${nl}}Killing TCP connection 10.254.254.1:1230${i} ${ip}:445"
-	i=$(($i + 1))
-    done >"$FAKE_NETSTAT_TCP_ESTABLISHED_FILE"
+	count=10
+	setup_tcp_connections $count \
+			      "$ip" 445 10.254.254.0 12300
 
-    ok <<EOF
-$out
-Killed $count TCP connections to released IP $ip
+	ok <<EOF
+Killed ${count}/${count} TCP connections to released IP $ip
 EOF
 
-    simple_test $dev $ip $bits
+	simple_test $dev $ip $bits
 done
diff --git a/ctdb/tests/eventscripts/10.interface.releaseip.011.sh b/ctdb/tests/eventscripts/10.interface.releaseip.011.sh
index 6203b97e2f5..c129346db8e 100755
--- a/ctdb/tests/eventscripts/10.interface.releaseip.011.sh
+++ b/ctdb/tests/eventscripts/10.interface.releaseip.011.sh
@@ -8,31 +8,20 @@ setup_ctdb
 
 ctdb_get_1_public_address |
 while read dev ip bits ; do
-    ip addr add "${ip}/${bits}" dev "$dev"
-
-    # Setup 10 fake connections...
-    count=10
-    out=""
-    nl="
-"
-    i=0
-    while [ $i -lt $count ] ; do
-	echo "${ip}:445 10.254.254.1:1230${i}"
-	# Expected output for killing this connection
-	out="${out}${out:+${nl}}Killing TCP connection 10.254.254.1:1230${i} ${ip}:445"
-	i=$(($i + 1))
-    done >"$FAKE_NETSTAT_TCP_ESTABLISHED_FILE"
-
-    # Note that the fake TCP killing done by the "ctdb killtcp" stub
-    # can only kill conections in the file, so killing this connection
-    # will never succeed so it will look like a time out.
-    FAKE_NETSTAT_TCP_ESTABLISHED="${ip}:445|10.254.254.1:43210"
-
-    ok <<EOF
-Killing TCP connection 10.254.254.1:43210 ${ip}:445
-$out
-Failed to kill TCP connections for IP 10.0.0.3 (1/11 remaining)
+	ip addr add "${ip}/${bits}" dev "$dev"
+
+	count=10
+	setup_tcp_connections $count \
+			      "$ip" 445 10.254.254.0 12300
+
+	setup_tcp_connections_unkillable 1 \
+					 "$ip" 445 10.254.254.0 43210
+
+	ok <<EOF
+Killed 10/11 TCP connections to released IP 10.0.0.3
+Remaining connections:
+  10.0.0.3:445 10.254.254.1:43211
 EOF
 
-    simple_test $dev $ip $bits
+	simple_test $dev $ip $bits
 done
diff --git a/ctdb/tests/eventscripts/10.interface.releaseip.012.sh b/ctdb/tests/eventscripts/10.interface.releaseip.012.sh
new file mode 100755
index 00000000000..53971fec88d
--- /dev/null
+++ b/ctdb/tests/eventscripts/10.interface.releaseip.012.sh
@@ -0,0 +1,30 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "Release 1 IP, 10 connections killed, 3 fail"
+
+setup_ctdb
+
+ctdb_get_1_public_address |
+while read dev ip bits ; do
+	ip addr add "${ip}/${bits}" dev "$dev"
+
+	count=10
+
+	setup_tcp_connections $count \
+			      "$ip" 445 10.254.254.0 12300
+
+	setup_tcp_connections_unkillable 3 \
+					 "$ip" 445 10.254.254.0 43210
+
+	ok <<EOF
+Killed 10/13 TCP connections to released IP 10.0.0.3
+Remaining connections:
+  10.0.0.3:445 10.254.254.1:43211
+  10.0.0.3:445 10.254.254.2:43212
+  10.0.0.3:445 10.254.254.3:43213
+EOF
+
+	simple_test $dev $ip $bits
+done
diff --git a/ctdb/tests/eventscripts/10.interface.releaseip.013.sh b/ctdb/tests/eventscripts/10.interface.releaseip.013.sh
new file mode 100755
index 00000000000..91393d27ed9
--- /dev/null
+++ b/ctdb/tests/eventscripts/10.interface.releaseip.013.sh
@@ -0,0 +1,35 @@
+#!/bin/sh
+
+. "${TEST_SCRIPTS_DIR}/unit.sh"
+
+define_test "Release 1 IP, all 10 connections kills fail"
+
+setup_ctdb
+
+ctdb_get_1_public_address |
+while read dev ip bits ; do
+	ip addr add "${ip}/${bits}" dev "$dev"
+
+	setup_tcp_connections 0
+
+	count=10
+	setup_tcp_connections_unkillable $count \
+					 "$ip" 445 10.254.254.0 43210
+
+	ok <<EOF
+Killed 0/$count TCP connections to released IP 10.0.0.3
+Remaining connections:
+  10.0.0.3:445 10.254.254.1:43211
+  10.0.0.3:445 10.254.254.2:43212
+  10.0.0.3:445 10.254.254.3:43213
+  10.0.0.3:445 10.254.254.4:43214
+  10.0.0.3:445 10.254.254.5:43215
+  10.0.0.3:445 10.254.254.6:43216
+  10.0.0.3:445 10.254.254.7:43217
+  10.0.0.3:445 10.254.254.8:43218
+  10.0.0.3:445 10.254.254.9:43219
+  10.0.0.3:445 10.254.254.10:43220
+EOF
+
+    simple_test $dev $ip $bits
+done
diff --git a/ctdb/tests/eventscripts/scripts/local.sh b/ctdb/tests/eventscripts/scripts/local.sh
index 5a638dfa707..0e16f5b1b1d 100644
--- a/ctdb/tests/eventscripts/scripts/local.sh
+++ b/ctdb/tests/eventscripts/scripts/local.sh
@@ -124,8 +124,7 @@ setup_generic ()
     export FAKE_TDB_IS_OK
     export FAKE_DATE_OUTPUT
 
-    export FAKE_NETSTAT_TCP_ESTABLISHED FAKE_TCP_LISTEN FAKE_NETSTAT_UNIX_LISTEN
-    export FAKE_NETSTAT_TCP_ESTABLISHED_FILE=$(mktemp --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR")
+    export FAKE_TCP_LISTEN FAKE_NETSTAT_UNIX_LISTEN
 }
 
 tcp_port_down ()
@@ -136,6 +135,42 @@ tcp_port_down ()
     done
 }
 
+_tcp_connections ()
+{
+	_count="$1"
+	_sip="$2"
+	_sport="$3"
+	_cip_base="$4"
+	_cport_base="$5"
+
+	_cip_prefix="${_cip_base%.*}"
+	_cip_suffix="${_cip_base##*.}"
+
+	for _i in $(seq 1 $_count) ; do
+		_cip_last=$((_cip_suffix + _i))
+		_cip="${_cip_prefix}.${_cip_last}"
+		_cport=$((_cport_base + _i))
+		echo "${_sip}:${_sport} ${_cip}:${_cport}"
+	done
+}
+
+setup_tcp_connections ()
+{
+	_t==$(mktemp --tmpdir="$EVENTSCRIPTS_TESTS_VAR_DIR")
+	export FAKE_NETSTAT_TCP_ESTABLISHED_FILE"$_t"
+	_tcp_connections "$@" >"$FAKE_NETSTAT_TCP_ESTABLISHED_FILE"
+}
+
+setup_tcp_connections_unkillable ()
+{
+	# These connections are listed by the "ss" stub but are not
+	# killed by the "ctdb killtcp" stub.  So killing these
+	# connections will never succeed... and will look like a time
+	# out.
+	_t=$(_tcp_connections "$@" | sed -e 's/ /|/g')
+	export FAKE_NETSTAT_TCP_ESTABLISHED="$_t"
+}
+
 shares_missing ()
 {
     _fmt="$1" ; shift
diff --git a/ctdb/tests/eventscripts/stubs/ss b/ctdb/tests/eventscripts/stubs/ss
index e8d804481df..1a3db0d0467 100755
--- a/ctdb/tests/eventscripts/stubs/ss
+++ b/ctdb/tests/eventscripts/stubs/ss
@@ -77,6 +77,10 @@ ss_tcp_established ()
 	    echo 0 0 "$src" "$dst"
 	fi
     done
+
+    if [ -z "$FAKE_NETSTAT_TCP_ESTABLISHED_FILE" ] ; then
+	    return
+    fi
     while read src dst ; do
 	if filter_socket "$srcs" "$sports" "$src" ; then
 	    echo 0 0 "$src" "$dst"