ctdb: Add new helper ctdb_etcd_lock

This introduces a mutex helper called ctdb_etcd_lock, which allows CTDB to
use an existing etcd cluster to provide the functionality of a recovery lock
using the API outlined in ctdb/doc/cluster_mutex_helper.txt.

Signed-off-by: Jose A. Rivera <jarrpa@samba.org>
Reviewed-by: David Disseldorp <ddiss@samba.org>

Autobuild-User(master): José A. Rivera <jarrpa@samba.org>
Autobuild-Date(master): Mon Dec  5 19:39:10 CET 2016 on sn-devel-144

4 files changed, 335 insertions, 1 deletions

diff --git a/ctdb/doc/Makefile b/ctdb/doc/Makefile
index f0f82153aa6..50ab7199995 100644
--- a/ctdb/doc/Makefile
+++ b/ctdb/doc/Makefile
@@ -8,6 +8,7 @@ DOCS = ctdb.1 ctdb.1.html \
 	ctdbd.conf.5 ctdbd.conf.5.html \
 	ctdb.7 ctdb.7.html \
 	ctdb-statistics.7 ctdb-statistics.7.html \
+	ctdb-etcd.7 ctdb-etcd.7.html \
 	ctdb-tunables.7 ctdb-tunables.7.html
 
 all: $(DOCS)
diff --git a/ctdb/doc/ctdb-etcd.7.xml b/ctdb/doc/ctdb-etcd.7.xml
new file mode 100644
index 00000000000..59acece321b
--- /dev/null
+++ b/ctdb/doc/ctdb-etcd.7.xml
@@ -0,0 +1,117 @@
+<?xml version="1.0" encoding="iso-8859-1"?>
+<!DOCTYPE refentry
+	PUBLIC "-//OASIS//DTD DocBook XML V4.5//EN"
+	"http://www.oasis-open.org/docbook/xml/4.5/docbookx.dtd">
+<refentry id="ctdb-etcd.7">
+
+  <refentryinfo>
+    <author>
+      <contrib>
+	This documentation was written by
+	Jose A. Rivera
+      </contrib>
+    </author>
+
+    <copyright>
+      <year>2016</year>
+      <holder>Jose A. Rivera</holder>
+    </copyright>
+    <legalnotice>
+      <para>
+	This program is free software; you can redistribute it and/or
+	modify it under the terms of the GNU General Public License as
+	published by the Free Software Foundation; either version 3 of
+	the License, or (at your option) any later version.
+      </para>
+      <para>
+	This program is distributed in the hope that it will be
+	useful, but WITHOUT ANY WARRANTY; without even the implied
+	warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
+	PURPOSE.  See the GNU General Public License for more details.
+      </para>
+      <para>
+	You should have received a copy of the GNU General Public
+	License along with this program; if not, see
+	<ulink url="http://www.gnu.org/licenses"/>.
+      </para>
+    </legalnotice>
+  </refentryinfo>
+
+  <refmeta>
+    <refentrytitle>ctdb-etcd</refentrytitle>
+    <manvolnum>7</manvolnum>
+    <refmiscinfo class="source">ctdb</refmiscinfo>
+    <refmiscinfo class="manual">CTDB - clustered TDB database</refmiscinfo>
+  </refmeta>
+
+  <refnamediv>
+    <refname>ctdb-etcd</refname>
+    <refpurpose>CTDB etcd integration</refpurpose>
+  </refnamediv>
+
+  <refsynopsisdiv>
+    <cmdsynopsis>
+      <command>ctdb_etcd_lock</command>
+    </cmdsynopsis>
+  </refsynopsisdiv>
+
+  <refsect1>
+    <title>DESCRIPTION</title>
+    <para>
+      ctdb_etcd_lock is intended to be run as a mutex helper for CTDB. It
+      will try to connect to an existing etcd cluster and grab a lock in that
+      cluster to function as CTDB's recovery lock. Please see
+      <emphasis>ctdb/doc/cluster_mutex_helper.txt</emphasis> for details on
+      the mutex helper API. To use this, include the following line in your
+      CTDB config file:
+    </para>
+    <screen format="linespecific">
+CTDB_RECOVERY_LOCK="!/usr/local/usr/libexec/ctdb/ctdb_etcd_lock"
+    </screen>
+    <para>
+      You can also pass "-v", "-vv", or "-vvv" to include verbose output in
+      the CTDB log. Additional "v"s indicate increases in verbosity.
+    </para>
+    <para>
+      This mutex helper expects the system Python interpreter to have access
+      to the etcd Python module. It also expects an etcd cluster to be
+      configured and running. To integrate with this, there is an optional
+      config file of the following format:
+    </para>
+    <screen format="linespecific">
+key = value
+    </screen>
+    <para>
+      The following configuration parameters (and their defaults) are defined
+      for use by ctdb_etcd_lock:
+    </para>
+    <screen format="linespecific">
+port      = 2379   # connecting port for the etcd cluster
+lock_ttl  = 9      # seconds for TTL
+refresh   = 2      # seconds between attempts to maintain lock
+locks_dir = _ctdb  # where to store CTDB locks in etcd
+                   # The final etcd directory for any given lock looks like:
+                   #   /_locks/{locks_dir}/{netbios name}/
+    </screen>
+    <para>
+      In addition, any keyword parameter that can be used to configure an
+      etcd client may be specified and modified here. For more documentation
+      on these parameters, see here: https://github.com/jplana/python-etcd/
+    </para>
+  </refsect1>
+
+  <refsect1>
+    <title>SEE ALSO</title>
+    <para>
+      <citerefentry><refentrytitle>ctdb</refentrytitle>
+      <manvolnum>7</manvolnum></citerefentry>,
+
+      <citerefentry><refentrytitle>ctdbd</refentrytitle>
+      <manvolnum>1</manvolnum></citerefentry>,
+
+      <ulink url="http://ctdb.samba.org/"/>
+    </para>
+  </refsect1>
+
+
+</refentry>
diff --git a/ctdb/tools/ctdb_etcd_lock b/ctdb/tools/ctdb_etcd_lock
new file mode 100755
index 00000000000..3e7e2bffca4
--- /dev/null
+++ b/ctdb/tools/ctdb_etcd_lock
@@ -0,0 +1,208 @@
+#!/usr/bin/python
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, either version 3 of the License, or
+#    (at your option) any later version.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <http://www.gnu.org/licenses/>.
+#
+#    Copyright (C) 2016 Jose A. Rivera <jarrpa@samba.org>
+#    Copyright (C) 2016 Ira Cooper <ira@samba.org>
+"""CTDB mutex helper using etcd.
+
+This script is intended to be run as a mutex helper for CTDB. It will try to
+connect to an existing etcd cluster and grab an etcd.Lock() to function as
+CTDB's recovery lock. Please see ctdb/doc/cluster_mutex_helper.txt for
+details on what we're SUPPOSED to be doing. :) To use this, include the
+following line in your CTDB config file:
+
+CTDB_RECOVERY_LOCK="!/path/to/script"
+
+You can also pass "-v", "-vv", or "-vvv" to include verbose output in the
+CTDB log. Additional "v"s indicate increases in verbosity.
+
+This mutex helper expects the system Python interpreter to have access to the
+etcd Python module. It also expects an etcd cluster to be configured and
+running. To integrate with this, there is an optional config file of the
+following format:
+
+key = value
+
+The following configuration variables (and their defaults) are defined for
+use by this script:
+
+port      = 2379   # connecting port for the etcd cluster
+lock_ttl  = 9      # seconds for TTL
+refresh   = 2      # seconds between attempts to maintain lock
+locks_dir = _ctdb  # where to store CTDB locks in etcd
+                   # The final etcd directory for any given lock looks like:
+                   #   /_locks/{locks_dir}/{netbios name}/
+
+In addition, any keyword parameter that can be used to configure an etcd
+client may be specified and modified here. For more documentation on these
+parameters, see here: https://github.com/jplana/python-etcd/
+
+"""
+import signal
+import time
+import etcd
+import sys
+import os
+import argparse
+import logging
+import subprocess
+
+# Globals ---------------------------------------------------------------------
+#
+defaults = { 'config': os.path.join(
+                         os.getenv('CTDB_BASE', '/usr/local/etc/ctdb'),
+                         'etcd'),
+             'verbose' : 0,
+           }
+helpmsg  = { 'config': 'Configuration file to use. The default behavior ' + \
+                       'is to look is the base CTDB configuration ' + \
+                       'directory, which can be overwritten by setting the' + \
+                       'CTDB_BASE environment variable, for a file called' + \
+                       '\'etcd\'. Default value is ' + defaults['config'],
+             'verbose' : 'Display verbose output to stderr. Default is no output.',
+           }
+
+log_levels = { 0: logging.ERROR,
+               1: logging.WARNING,
+               2: logging.DEBUG,
+             }
+
+config_file = defaults['config']
+verbose = defaults['verbose']
+
+# Helper Functions ------------------------------------------------------------
+#
+def sigterm_handler(signum, frame):
+  """Handler for SIGTERM signals.
+  """
+  sys.exit()
+
+def print_nonl(out):
+  """Dumb shortcut for printing to stdout with no newline.
+  """
+  sys.stdout.write(str(out))
+  sys.stdout.flush()
+
+def int_or_not(s):
+  """Try to convert input to an integer.
+  """
+  try:
+    return int(s)
+  except ValueError:
+    return s
+
+# Mainline --------------------------------------------------------------------
+#
+def main():
+  global config_file
+  global verbose
+
+  logging.basicConfig(level=log_levels[verbose])
+
+  # etcd config defaults
+  etcd_config = {
+    'port'        : 2379,
+    'locks_dir'   : '_ctdb',
+    'lock_ttl'    : 9,
+    'lock_refresh': 2,
+  }
+  # Find and read etcd config file
+  etcd_client_params = (
+    'host',
+    'port',
+    'srv_domain',
+    'version_prefix',
+    'read_timeout',
+    'allow_redirect',
+    'protocol',
+    'cert',
+    'ca_cert',
+    'username',
+    'password',
+    'allow_reconnect',
+    'use_proxies',
+    'expected_cluster_id',
+    'per_host_pool_size',
+  )
+  if os.path.isfile(config_file):
+    f = open(config_file, 'r')
+    for line in f:
+      (key, value) = line.split("=",1)
+      etcd_config[key.strip()] = int_or_not(value.strip())
+
+  # Minor hack: call out to shell to retrieve CTDB netbios name and PNN.
+  tmp = subprocess.Popen("testparm -s --parameter-name 'netbios name'; \
+                          ctdb pnn",
+                         shell=True,
+                         universal_newlines=True,
+                         stdout=subprocess.PIPE
+                        ).stdout.read().strip()
+  nb_name, pnn = tmp.split()
+
+  # Try to get and hold the lock
+  try:
+    client = etcd.Client(**{k: etcd_config[k] for k in \
+                         set(etcd_client_params).intersection(etcd_config)})
+    lock = etcd.Lock(client, etcd_config['locks_dir'] + "/" + nb_name)
+    lock._uuid = lock._uuid + "_" + pnn
+    logging.debug("Updated lock UUID: " + lock.uuid)
+    ppid = os.getppid()
+    while True:
+      lock.acquire(blocking=False, lock_ttl=etcd_config['lock_ttl'])
+      if lock.is_acquired:
+        print_nonl(0)
+      else:
+        locks = "No locks found."
+        if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
+          keys = client.read(lock.path, recursive=True)
+          if keys is not None:
+            locks = "Existing locks:\n  "
+            locks += '\n  '.join((child.key + ": " + child.value  for child in keys.children))
+        logging.debug("Lock contention. " + locks)
+        print_nonl(1)
+        break
+      os.kill(ppid, 0)
+      time.sleep(etcd_config['lock_refresh'])
+  except (OSError, SystemExit) as e:
+    if lock is not None and lock.is_acquired:
+      lock.release()
+  except:
+    print_nonl(3)
+    if logging.getLogger().getEffectiveLevel() == logging.DEBUG:
+      raise
+
+if __name__== "__main__":
+  signal.signal(signal.SIGTERM, sigterm_handler)
+
+  parser = argparse.ArgumentParser(
+                          description=__doc__,
+                          epilog='',
+                          formatter_class=argparse.RawDescriptionHelpFormatter )
+  parser.add_argument( '-v', '--verbose',
+                       action='count',
+                       help=helpmsg['verbose'],
+                       default=defaults['verbose'],
+                     )
+  parser.add_argument( '-c', '--config',
+                       action='store',
+                       help=helpmsg['config'],
+                       default=defaults['config'],
+                     )
+  args = parser.parse_args()
+
+  config_file = args.config
+  verbose     = args.verbose if args.verbose <= 2 else 2
+
+  main()
diff --git a/ctdb/wscript b/ctdb/wscript
index 85437350463..6137f92d15a 100644
--- a/ctdb/wscript
+++ b/ctdb/wscript
@@ -44,6 +44,7 @@ manpages = [
     'ctdbd.1',
     'ctdbd.conf.5',
     'ctdbd_wrapper.1',
+    'ctdb-etcd.7',
     'ctdb-statistics.7',
     'ctdb-tunables.7',
     'ltdbtool.1',
@@ -536,7 +537,7 @@ def build(bld):
     if bld.env.ctdb_generate_manpages:
         bld.MANPAGES('''onnode.1 ctdbd_wrapper.1 ctdbd.conf.5
                         ctdb.7 ctdb-statistics.7 ctdb-tunables.7
-                        ctdb_diagnostics.1''',
+                        ctdb_diagnostics.1 ctdb-etcd.7''',
                       True)
     else:
         for m in bld.env.ctdb_prebuilt_manpages:
@@ -560,6 +561,13 @@ def build(bld):
     bld.INSTALL_FILES('${BINDIR}', 'ctdb_diagnostics',
                       destname='ctdb_diagnostics', chmod=0755)
 
+    bld.SAMBA_GENERATOR('ctdb-etcd-lock',
+                        source='tools/ctdb_etcd_lock',
+                        target='ctdb_etcd_lock',
+                        rule='sed %s ${SRC} > ${TGT}' % (sed_cmdline))
+    bld.INSTALL_FILES('${CTDB_HELPER_BINDIR}', 'ctdb_etcd_lock',
+                      destname='ctdb_etcd_lock', chmod=0744)
+
     bld.SAMBA_GENERATOR('ctdb-natgw',
                         source='tools/ctdb_natgw',
                         target='ctdb_natgw',