path: root/ctdb/config/functions

# Hey Emacs, this is a -*- shell-script -*- !!!

# utility functions for ctdb event scripts

if [ -z "$CTDB_BASE" ] ; then
    echo 'CTDB_BASE unset in CTDB functions file'
    exit 1
fi
export CTDB_BASE

# CTDB_VARDIR is used elsewhere
# shellcheck disable=SC2034
CTDB_VARDIR="/usr/local/var/lib/ctdb"
ctdb_rundir="/usr/local/var/run/ctdb"

CTDB="${CTDB:-/usr/local/bin/ctdb}"

# Only (and always) override these variables in test code

if [ -z "$CTDB_SCRIPT_VARDIR" ] ; then
    CTDB_SCRIPT_VARDIR="/usr/local/var/lib/ctdb/state"
fi

if [ -z "$CTDB_SYS_ETCDIR" ] ; then
    CTDB_SYS_ETCDIR="/etc"
fi

if [ -z "$CTDB_HELPER_BINDIR" ] ; then
    CTDB_HELPER_BINDIR="/usr/local/libexec/ctdb"
fi

#######################################
# pull in a system config file, if any

rewrite_ctdb_options ()
{
    case "$CTDB_DBDIR" in
	tmpfs|tmpfs:*)
	    _opts_defaults="mode=700"
	    # Get any extra options specified after colon
	    if [ "$CTDB_DBDIR" = "tmpfs" ] ; then
		_opts=""
	    else
		_opts="${CTDB_DBDIR#tmpfs:}"
	    fi
	    # It is OK to repeat mount options - last value wins.
	    # CTDB_DBDIR_TMPFS_OPTIONS is used by ctdbd_wrapper
	    # shellcheck disable=SC2034
	    CTDB_DBDIR_TMPFS_OPTIONS="${_opts_defaults}${_opts:+,}${_opts}"

	    CTDB_DBDIR="${ctdb_rundir}/CTDB_DBDIR"
	    ;;
	*)
	    # shellcheck disable=SC2034
	    CTDB_DBDIR_TMPFS_OPTIONS=""
    esac
}

_loadconfig() {

    if [ -z "$1" ] ; then
	foo="${service_config:-${service_name}}"
	if [ -n "$foo" ] ; then
	    loadconfig "$foo"
	    return
	fi
    fi

    if [ "$1" != "ctdb" ] ; then
	loadconfig "ctdb"
    fi

    if [ -z "$1" ] ; then
	return
    fi

    if [ -f "${CTDB_SYS_ETCDIR}/sysconfig/$1" ]; then
	. "${CTDB_SYS_ETCDIR}/sysconfig/$1"
    elif [ -f "${CTDB_SYS_ETCDIR}/default/$1" ]; then
	. "${CTDB_SYS_ETCDIR}/default/$1"
    elif [ -f "${CTDB_BASE}/sysconfig/$1" ]; then
	. "${CTDB_BASE}/sysconfig/$1"
    fi

    if [ "$1" = "ctdb" ] ; then
	_config="${CTDBD_CONF:-${CTDB_BASE}/ctdbd.conf}"
	if [ -r "$_config" ] ; then
	    . "$_config"
	fi
	rewrite_ctdb_options
    fi
}

loadconfig () {
    _loadconfig "$@"
}

##############################################################

die ()
{
    _msg="$1"
    _rc="${2:-1}"

    echo "$_msg" >&2
    exit "$_rc"
}

# Log given message or stdin to either syslog or a CTDB log file
# $1 is the tag passed to logger if syslog is in use.
script_log ()
{
    _tag="$1" ; shift

    case "$CTDB_LOGGING" in
	file:*|"")
	    if [ -n "$CTDB_LOGGING" ] ; then
		_file="${CTDB_LOGGING#file:}"
	    else
		_file="/usr/local/var/log/log.ctdb"
	    fi
	    {
		if [ -n "$*" ] ; then
		    echo "$*"
		else
		    cat
		fi
	    } >>"$_file"
	    ;;
	*)
	    # Handle all syslog:* variants here too.  There's no tool to do
	    # the lossy things, so just use logger.
	    logger -t "ctdbd: ${_tag}" "$*"
	    ;;
    esac
}

# When things are run in the background in an eventscript then logging
# output might get lost.  This is the "solution".  :-)
background_with_logging ()
{
    (
	"$@" 2>&1 </dev/null |
	script_log "${script_name}&"
    )&

    return 0
}

##############################################################
# check number of args for different events
ctdb_check_args ()
{
    case "$1" in
	takeip|releaseip)
	    if [ $# != 4 ]; then
		echo "ERROR: must supply interface, IP and maskbits"
		exit 1
	    fi
	    ;;
	updateip)
	    if [ $# != 5 ]; then
		echo "ERROR: must supply old interface, new interface, IP and maskbits"
		exit 1
	    fi
	    ;;
    esac
}

##############################################################
# determine on what type of system (init style) we are running
detect_init_style()
{
    # only do detection if not already set:
    [ -z "$CTDB_INIT_STYLE" ] || return

    if [ -x /sbin/startproc ]; then
        CTDB_INIT_STYLE="suse"
    elif [ -x /sbin/start-stop-daemon ]; then
        CTDB_INIT_STYLE="debian"
    else
        CTDB_INIT_STYLE="redhat"
    fi
}

######################################################
# simulate /sbin/service on platforms that don't have it
# _service() makes it easier to hook the service() function for
# testing.
_service ()
{
  _service_name="$1"
  _op="$2"

  # do nothing, when no service was specified
  [ -z "$_service_name" ] && return

  if [ -x /sbin/service ]; then
      $_nice /sbin/service "$_service_name" "$_op"
  elif [ -x /usr/sbin/service ]; then
      $_nice /usr/sbin/service "$_service_name" "$_op"
  elif [ -x /bin/systemctl ]; then
      $_nice /bin/systemctl "$_op" "$_service_name"
  elif [ -x "${CTDB_SYS_ETCDIR}/init.d/${_service_name}" ]; then
      $_nice "${CTDB_SYS_ETCDIR}/init.d/${_service_name}" "$_op"
  elif [ -x "${CTDB_SYS_ETCDIR}/rc.d/init.d/${_service_name}" ]; then
      $_nice "${CTDB_SYS_ETCDIR}/rc.d/init.d/${_service_name}" "$_op"
  fi
}

service()
{
    _nice=""
    _service "$@"
}

######################################################
# simulate /sbin/service (niced) on platforms that don't have it
nice_service()
{
    _nice="nice"
    _service "$@"
}

######################################################
# Cached retrieval of PNN from local node.  This never changes so why
# open a client connection to the server each time this is needed?
ctdb_get_pnn ()
{
    _pnn_file="${CTDB_SCRIPT_VARDIR}/my-pnn"
    if [ ! -f "$_pnn_file" ] ; then
	$CTDB pnn >"$_pnn_file"
    fi

    cat "$_pnn_file"
}

# Cached retrieval of private IP address from local node.  This never
# changes.
ctdb_get_ip_address ()
{
    _ip_addr_file="${CTDB_SCRIPT_VARDIR}/my-ip-address"
    if [ ! -f "$_ip_addr_file" ] ; then
	$CTDB -X nodestatus |
	    awk -F '|' 'NR == 2 { print $3 }' >"$_ip_addr_file"
    fi

    # ip_address is used by caller
    # shellcheck disable=SC2034
    cat "$_ip_addr_file"
}

######################################################
# wrapper around /proc/ settings to allow them to be hooked
# for testing
# 1st arg is relative path under /proc/, 2nd arg is value to set
set_proc ()
{
    echo "$2" >"/proc/$1"
}

set_proc_maybe ()
{
    if [ -w "/proc/$1" ] ; then
	set_proc "$1" "$2"
    fi
}

######################################################
# wrapper around getting file contents from /proc/ to allow
# this to be hooked for testing
# 1st arg is relative path under /proc/
get_proc ()
{
    cat "/proc/$1"
}

######################################################
# Print up to $_max kernel stack traces for processes named $_program
program_stack_traces ()
{
    _prog="$1"
    _max="${2:-1}"

    _count=1
    for _pid in $(pidof "$_prog") ; do
	[ "$_count" -le "$_max" ] || break

	# Do this first to avoid racing with process exit
	_stack=$(get_proc "${_pid}/stack" 2>/dev/null)
	if [ -n "$_stack" ] ; then
	    echo "Stack trace for ${_prog}[${_pid}]:"
	    echo "$_stack"
	    _count=$((_count + 1))
	fi
    done
}

######################################################
# Ensure $service_name is set
assert_service_name ()
{
    [ -n "$service_name" ] || die "INTERNAL ERROR: \$service_name not set"
}

######################################################
# check a set of directories is available
# return 1 on a missing directory
# directories are read from stdin
######################################################
ctdb_check_directories_probe()
{
    while IFS="" read d ; do
	case "$d" in
	    *%*)
		continue
		;;
	    *)
		[ -d "${d}/." ] || return 1
	esac
    done
}

######################################################
# check a set of directories is available
# directories are read from stdin
######################################################
ctdb_check_directories()
{
    ctdb_check_directories_probe || {
	echo "ERROR: $service_name directory \"$d\" not available"
	exit 1
    }
}

######################################################
# check a set of tcp ports
# usage: ctdb_check_tcp_ports <ports...>
######################################################

# Check whether something is listening on all of the given TCP ports
# using the "ctdb checktcpport" command.
ctdb_check_tcp_ports()
{
	if [ -z "$1" ] ; then
		echo "INTERNAL ERROR: ctdb_check_tcp_ports - no ports specified"
		exit 1
	fi

	for _p ; do  # process each function argument (port)
		_cmd="$CTDB checktcpport $_p"
		_out=$($_cmd 2>&1)
		_ret=$?
		case "$_ret" in
		0)
			echo "$service_name not listening on TCP port $_p"
			return 1
			;;
		98)
			# Couldn't bind, something already listening, next port
			continue
			;;
		*)
			echo "unexpected error (${_ret}) running \"${_cmd}\""
			if [ -n "$_out" ] ; then
				echo "$_out"
			fi
			return $_ret
			;;
		esac
	done

	# All ports listening
	return 0
}

######################################################
# check a unix socket
# usage: ctdb_check_unix_socket SERVICE_NAME <socket_path>
######################################################
ctdb_check_unix_socket() {
    socket_path="$1"
    [ -z "$socket_path" ] && return

    if ! netstat --unix -a -n | grep -q "^unix.*LISTEN.*${socket_path}$"; then
        echo "ERROR: $service_name socket $socket_path not found"
        return 1
    fi
}

################################################
# kill off any TCP connections with the given IP
################################################
kill_tcp_connections ()
{
    _iface="$1"
    _ip="$2"

    _oneway=false
    if [ "$3" = "oneway" ] ; then
	_oneway=true
    fi

    get_tcp_connections_for_ip "$_ip" | {
	_killcount=0
	_connections=""
	_nl="
"
	while read _dst _src; do
	    _destport="${_dst##*:}"
	    __oneway=$_oneway
	    case $_destport in
		# we only do one-way killtcp for CIFS
		139|445) __oneway=true ;;
	    esac

	    echo "Killing TCP connection $_src $_dst"
	    _connections="${_connections}${_nl}${_src} ${_dst}"
	    if ! $__oneway ; then
		_connections="${_connections}${_nl}${_dst} ${_src}"
	    fi

	    _killcount=$((_killcount + 1))
	done

	if [ $_killcount -eq 0 ] ; then
	    return
	fi

	echo "$_connections" | \
		"${CTDB_HELPER_BINDIR}/ctdb_killtcp" "$_iface" || {
		echo "Failed to kill TCP connections"
		return
	}

	_remaining=$(get_tcp_connections_for_ip "$_ip" | wc -l)

	if [ "$_remaining" -eq 0 ] ; then
		echo "Killed $_killcount TCP connections to released IP $_ip"
		return
	fi

	_t="${_remaining}/${_killcount}"
	echo "Failed to kill TCP connections for IP $_ip (${_t} remaining)"
    }
}

##################################################################
# kill off the local end for any TCP connections with the given IP
##################################################################
kill_tcp_connections_local_only ()
{
    kill_tcp_connections "$@" "oneway"
}

##################################################################
# tickle any TCP connections with the given IP
##################################################################
tickle_tcp_connections ()
{
    _ip="$1"

    # Get connections, both directions
    _conns=$(get_tcp_connections_for_ip "$_ip" | \
		    awk '{ print $1, $2 ; print $2, $1 }')

    echo "$_conns" | awk '{ print "Tickle TCP connection", $1, $2 }'
    echo "$_conns" | ctdb tickle
}

get_tcp_connections_for_ip ()
{
    _ip="$1"

    ss -tn state established "src [$_ip]" | awk 'NR > 1 {print $3, $4}'
}

########################################################

add_ip_to_iface ()
{
    _iface=$1
    _ip=$2
    _maskbits=$3

    # Ensure interface is up
    ip link set "$_iface" up || \
	die "Failed to bringup interface $_iface"

    # Only need to define broadcast for IPv4
    case "$_ip" in
        *:*) _bcast=""      ;;
	*)   _bcast="brd +" ;;
    esac

    # Intentionally unquoted multi-word value here
    # shellcheck disable=SC2086
    ip addr add "$_ip/$_maskbits" $_bcast dev "$_iface" || {
	echo "Failed to add $_ip/$_maskbits on dev $_iface"
	return 1
    }

    # Wait 5 seconds for IPv6 addresses to stop being tentative...
    if [ -z "$_bcast" ] ; then
	for _x in $(seq 1 10) ; do
	    ip addr show to "${_ip}/128" | grep -q "tentative" || break
	    sleep 0.5
	done

	# If the address was a duplicate then it won't be on the
	# interface so flag an error.
	_t=$(ip addr show to "${_ip}/128")
	case "$_t" in
	    "")
		echo "Failed to add $_ip/$_maskbits on dev $_iface"
		return 1
		;;
	    *tentative*|*dadfailed*)
		echo "Failed to add $_ip/$_maskbits on dev $_iface"
		ip addr del "$_ip/$_maskbits" dev "$_iface"
		return 1
		;;
	esac
    fi
}

delete_ip_from_iface()
{
    _iface=$1
    _ip=$2
    _maskbits=$3

    # This could be set globally for all interfaces but it is probably
    # better to avoid surprises, so limit it the interfaces where CTDB
    # has public IP addresses.  There isn't anywhere else convenient
    # to do this so just set it each time.  This is much cheaper than
    # remembering and re-adding secondaries.
    set_proc "sys/net/ipv4/conf/${_iface}/promote_secondaries" 1

    ip addr del "$_ip/$_maskbits" dev "$_iface" || {
	echo "Failed to del $_ip on dev $_iface"
	return 1
    }
}

# If the given IP is hosted then print 2 items: maskbits and iface
ip_maskbits_iface ()
{
    _addr="$1"

    case "$_addr" in
	*:*) _bits=128 ;;
	*)   _bits=32  ;;
    esac
    ip addr show to "${_addr}/${_bits}" 2>/dev/null | \
	awk 'NR == 1 { iface = $2; sub(":$", "", iface) ;
		       sub("@.*", "", iface) }
             $1 ~ /inet/ { mask = $2; sub(".*/", "", mask);
                           print mask, iface }'
}

drop_ip ()
{
    _addr="${1%/*}"  # Remove optional maskbits

    # Intentional word splitting here
    # shellcheck disable=SC2046
    set -- $(ip_maskbits_iface "$_addr")
    if [ -n "$1" ] ; then
	_maskbits="$1"
	_iface="$2"
	echo "Removing public address $_addr/$_maskbits from device $_iface"
	delete_ip_from_iface "$_iface" "$_addr" "$_maskbits" >/dev/null 2>&1
    fi
}

drop_all_public_ips ()
{
	# _x is intentionally ignored
	# shellcheck disable=SC2034
	while read _ip _x ; do
		drop_ip "$_ip"
	done <"${CTDB_PUBLIC_ADDRESSES:-/dev/null}"
}

flush_route_cache ()
{
    set_proc_maybe sys/net/ipv4/route/flush 1
    set_proc_maybe sys/net/ipv6/route/flush 1
}

########################################################
# Interface monitoring

# If the interface is a virtual one (e.g. VLAN) then get the
# underlying interface
interface_get_real ()
{
    # Output of "ip link show <iface>"
    _iface_info="$1"

    # Extract the full interface description to see if it is a VLAN
    _t=$(echo "$_iface_info" |
		awk 'NR == 1 { iface = $2; sub(":$", "", iface) ;
			       print iface }')
    case "$_t" in
	*@*)
	    # VLAN: use the underlying interface, after the '@'
	    echo "${_t##*@}"
	    ;;
	*)
	    # Not a regular VLAN.  For backward compatibility, assume
	    # there is some other sort of VLAN that doesn't have the
	    # '@' in the output and only use what is before a '.'.  If
	    # there is no '.' then this will be the whole interface
	    # name.
	    echo "${_t%%.*}"
    esac
}

# Check whether an interface is operational
interface_monitor ()
{
    _iface="$1"

    _iface_info=$(ip link show "$_iface" 2>&1) || {
	echo "ERROR: Monitored interface ${_iface} does not exist"
	return 1
    }


    # If the interface is a virtual one (e.g. VLAN) then get the
    # underlying interface.
    _realiface=$(interface_get_real "$_iface_info")

    if _bi=$(get_proc "net/bonding/${_realiface}" 2>/dev/null) ; then
	# This is a bond: various monitoring strategies
	echo "$_bi" | grep -q 'Currently Active Slave: None' && {
	    echo "ERROR: No active slaves for bond device ${_realiface}"
	    return 1
	}
	echo "$_bi" | grep -q '^MII Status: up' || {
	    echo "ERROR: public network interface ${_realiface} is down"
	    return 1
	}
	echo "$_bi" | grep -q '^Bonding Mode: IEEE 802.3ad Dynamic link aggregation' && {
	    # This works around a bug in the driver where the
	    # overall bond status can be up but none of the actual
	    # physical interfaces have a link.
	    echo "$_bi" | grep 'MII Status:' | tail -n +2 | grep -q '^MII Status: up' || {
		echo "ERROR: No active slaves for 802.ad bond device ${_realiface}"
		return 1
	    }
	}

	return 0
    else
	# Not a bond
	case "$_iface" in
	    lo*)
		# loopback is always working
		return 0
		;;
	    ib*)
		# we don't know how to test ib links
		return 0
		;;
	    *)
		ethtool "$_iface" | grep -q 'Link detected: yes' || {
		    # On some systems, this is not successful when a
		    # cable is plugged but the interface has not been
		    # brought up previously. Bring the interface up
		    # and try again...
		    ip link set "$_iface" up
		    ethtool "$_iface" | grep -q 'Link detected: yes' || {
			echo "ERROR: No link on the public network interface ${_iface}"
			return 1
		    }
		}
		return 0
		;;
	esac
    fi
}

########################################################
# Simple counters
_ctdb_counter_common () {
    _service_name="${1:-${service_name:-${script_name}}}"
    _counter_file="${CTDB_SCRIPT_VARDIR}/failcount/${_service_name}"
    mkdir -p "${_counter_file%/*}" # dirname
}
# Some code passes an argument
# shellcheck disable=SC2120
ctdb_counter_init () {
    _ctdb_counter_common "$1"

    >"$_counter_file"
}
ctdb_counter_incr () {
    _ctdb_counter_common "$1"

    # unary counting using newlines!
    echo >>"$_counter_file"
}
ctdb_counter_get () {
    _ctdb_counter_common "$1"
    # unary counting!
    stat -c "%s" "$_counter_file" 2>/dev/null || echo 0
}

########################################################

ctdb_setup_service_state_dir ()
{
	_s="${1:-${service_name}}"

	_service_state_dir="${CTDB_SCRIPT_VARDIR}/service_state/${_s}"
	mkdir -p "$_service_state_dir" ||
		die "Error creating state dir \"${_service_state_dir}\""

	echo "$_service_state_dir"
}

##################################################################
# Reconfigure a service on demand

_ctdb_service_reconfigure_common ()
{
    _d="${CTDB_SCRIPT_VARDIR}/service_status/${service_name}"
    mkdir -p "$_d"
    _ctdb_service_reconfigure_flag="$_d/reconfigure"
}

ctdb_service_needs_reconfigure ()
{
    _ctdb_service_reconfigure_common
    [ -e "$_ctdb_service_reconfigure_flag" ]
}

ctdb_service_set_reconfigure ()
{
    _ctdb_service_reconfigure_common
    >"$_ctdb_service_reconfigure_flag"
}

ctdb_service_unset_reconfigure ()
{
    _ctdb_service_reconfigure_common
    rm -f "$_ctdb_service_reconfigure_flag"
}

ctdb_service_reconfigure ()
{
    echo "Reconfiguring service \"${service_name}\"..."
    ctdb_service_unset_reconfigure
    service_reconfigure || return $?
    # Intentionally have this use $service_name as default
    # shellcheck disable=SC2119
    ctdb_counter_init
}

# Default service_reconfigure() function does nothing.
service_reconfigure ()
{
    :
}

##################################################################
# Does CTDB manage this service?

ctdb_compat_managed_service ()
{
    if [ "$1" = "yes" -a "$2" = "$service_name" ] ; then
	CTDB_MANAGED_SERVICES="$CTDB_MANAGED_SERVICES $2"
    fi
}

is_ctdb_managed_service ()
{
    assert_service_name

    # $t is used just for readability and to allow better accurate
    # matching via leading/trailing spaces
    t=" $CTDB_MANAGED_SERVICES "

    # Return 0 if "<space>$service_name<space>" appears in $t
    if [ "${t#* ${service_name} }" != "${t}" ] ; then
	return 0
    fi

    # If above didn't match then update $CTDB_MANAGED_SERVICES for
    # backward compatibility and try again.
    ctdb_compat_managed_service "$CTDB_MANAGES_VSFTPD"   "vsftpd"
    ctdb_compat_managed_service "$CTDB_MANAGES_SAMBA"    "samba"
    ctdb_compat_managed_service "$CTDB_MANAGES_WINBIND"  "winbind"
    ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "apache2"
    ctdb_compat_managed_service "$CTDB_MANAGES_HTTPD"    "httpd"
    ctdb_compat_managed_service "$CTDB_MANAGES_ISCSI"    "iscsi"
    ctdb_compat_managed_service "$CTDB_MANAGES_CLAMD"    "clamd"
    ctdb_compat_managed_service "$CTDB_MANAGES_NFS"      "nfs"

    t=" $CTDB_MANAGED_SERVICES "

    # Return 0 if "<space>$service_name<space>" appears in $t
    [ "${t#* ${service_name} }" != "${t}" ]
}

# Default service_start() and service_stop() functions.
 
# These may be overridden in an eventscript.
service_start ()
{
    service "$service_name" start
}

service_stop ()
{
    service "$service_name" stop
}

##################################################################

# This exists only for backward compatibility with 3rd party scripts
# that call it
ctdb_standard_event_handler ()
{
    :
}

iptables_wrapper ()
{
    _family="$1" ; shift
    if [ "$_family" = "inet6" ] ; then
	_iptables_cmd="ip6tables"
    else
	_iptables_cmd="iptables"
    fi

    # iptables doesn't like being re-entered, so flock-wrap it.
    flock -w 30 "${CTDB_SCRIPT_VARDIR}/iptables.flock" "$_iptables_cmd" "$@"
}

# AIX (and perhaps others?) doesn't have mktemp
# type is commonly supported and more portable than which(1)
# shellcheck disable=SC2039
if ! type mktemp >/dev/null 2>&1 ; then
    mktemp ()
    {
	_dir=false
	if [ "$1" = "-d" ] ; then
	    _dir=true
	    shift
	fi
	_d="${TMPDIR:-/tmp}"
	_hex10=$(dd if=/dev/urandom count=20 2>/dev/null | \
	    md5sum | \
	    sed -e 's@\(..........\).*@\1@')
	_t="${_d}/tmp.${_hex10}"
	(
	    umask 077
	    if $_dir ; then
		mkdir "$_t"
	    else
		>"$_t"
	    fi
	)
	echo "$_t"
    }
fi

######################################################################
# NFS callout handling

nfs_callout_init ()
{
	_state_dir="$1"

	if [ -z "$CTDB_NFS_CALLOUT" ] ; then
		CTDB_NFS_CALLOUT="${CTDB_BASE}/nfs-linux-kernel-callout"
	fi
	# Always export, for statd callout
	export CTDB_NFS_CALLOUT

	# If the callout wants to use this then it must create it
	export CTDB_NFS_CALLOUT_STATE_DIR="${_state_dir}/callout-state"

	# Export, if set, for use by clustered NFS callouts
	if [ -n "$CTDB_NFS_STATE_FS_TYPE" ] ; then
		export CTDB_NFS_STATE_FS_TYPE
	fi
	if [ -n "$CTDB_NFS_STATE_MNT" ] ; then
		export CTDB_NFS_STATE_MNT
	fi

	nfs_callout_cache="${_state_dir}/nfs_callout_cache"
	nfs_callout_cache_callout="${nfs_callout_cache}/CTDB_NFS_CALLOUT"
	nfs_callout_cache_ops="${nfs_callout_cache}/ops"
}

nfs_callout_register ()
{
    mkdir -p "$nfs_callout_cache_ops"
    rm -f "$nfs_callout_cache_ops"/*

    echo "$CTDB_NFS_CALLOUT" >"$nfs_callout_cache_callout"

    _t=$(eval "$CTDB_NFS_CALLOUT" "register")
    if [ -n "$_t" ] ; then
	echo "$_t" |
	    while IFS="" read _op ; do
		touch "${nfs_callout_cache_ops}/${_op}"
	    done
    else
	touch "${nfs_callout_cache_ops}/ALL"
    fi
}

nfs_callout ()
{
    # Re-run registration if $CTDB_NFS_CALLOUT has changed
    _prev=""
    if [ -r "$nfs_callout_cache_callout" ] ; then
	read _prev <"$nfs_callout_cache_callout"
    fi
    if [ "$CTDB_NFS_CALLOUT" != "$_prev" ] ; then
	nfs_callout_register
    fi

    # Run the operation if it is registered...
    if [ -e "${nfs_callout_cache_ops}/${1}" ] || \
	   [ -e "${nfs_callout_cache_ops}/ALL" ]; then
	eval "$CTDB_NFS_CALLOUT" "$@"
    fi
}

########################################################
# tickle handling
########################################################

update_tickles ()
{
	_port="$1"

	tickledir="${CTDB_SCRIPT_VARDIR}/tickles"
	mkdir -p "$tickledir"

	# What public IPs do I hold?
	_pnn=$(ctdb_get_pnn)
	_ips=$($CTDB -X ip | awk -F'|' -v pnn="$_pnn" '$3 == pnn {print $2}')

	# IPs and port as ss filters
	_ip_filter=""
	for _ip in $_ips ; do
	    _ip_filter="${_ip_filter}${_ip_filter:+ || }src [${_ip}]"
	done
	_port_filter="sport == :${_port}"

	# Record connections to our public IPs in a temporary file.
	# This temporary file is in CTDB's private state directory and
	# $$ is used to avoid a very rare race involving CTDB's script
	# debugging.  No security issue, nothing to see here...
	_my_connections="${tickledir}/${_port}.connections.$$"
	# Parentheses are needed around the filters for precedence but
	# the parentheses can't be empty!
	ss -tn state established \
	   "${_ip_filter:+( ${_ip_filter} )}" \
	   "${_port_filter:+( ${_port_filter} )}" |
	awk 'NR > 1 {print $4, $3}' |
	sort >"$_my_connections"

	# Record our current tickles in a temporary file
	_my_tickles="${tickledir}/${_port}.tickles.$$"
	for _i in $_ips ; do
		$CTDB -X gettickles "$_i" "$_port" |
		awk -F'|' 'NR > 1 { printf "%s:%s %s:%s\n", $2, $3, $4, $5 }'
	done |
	sort >"$_my_tickles"

	# Add tickles for connections that we haven't already got tickles for
	comm -23 "$_my_connections" "$_my_tickles" | \
		$CTDB addtickle

	# Remove tickles for connections that are no longer there
	comm -13 "$_my_connections" "$_my_tickles" | \
		$CTDB deltickle

	rm -f "$_my_connections" "$_my_tickles"

	# Remove stale files from killed scripts
	# Files can't have spaces in name, more portable than -print0/-0
	# shellcheck disable=SC2038
	(cd "$tickledir" && find . -type f -mmin +10 | xargs -r rm)
}

########################################################
# load a site local config file
########################################################

[ -n "$CTDB_RC_LOCAL" -a -x "$CTDB_RC_LOCAL" ] && {
	. "$CTDB_RC_LOCAL"
}

[ -x "${CTDB_BASE}/rc.local" ] && {
	. "${CTDB_BASE}/rc.local"
}

[ -d "${CTDB_BASE}/rc.local.d" ] && {
	for i in "${CTDB_BASE}/rc.local.d"/* ; do
		[ -x "$i" ] && . "$i"
	done
}

script_name="${0##*/}"       # basename