Graceful shutdown WSGI/RPC server

Currently, termination of WSGI application or RPC server immediately stops
service and so interrupts in-progress request.

Graceful handler for SIGTERM signal was added.
SIGINT signal handler was removed to allow instantaneous termination of
service.

DocImpact: graceful termination of process can be done by sending SIGTERM
signal to parent WSGI process. Graceful termination is not instantaneous.
To force instantaneous termination SIGINT signal must be sent.

Change-Id: If25f9565d832d1c36642ec3b1921563ef02890aa
Closes-bug: #1382390
Co-Author: Tiantian Gao <gtt116@gmail.com>
(cherry picked from commit fa9aa6b665f75e610f2b91a7d310f6499bd71770)

2 files changed, 68 insertions, 17 deletions

diff --git a/openstack/common/service.py b/openstack/common/service.py
index 9375bdc6..47b4887b 100644
--- a/openstack/common/service.py
+++ b/openstack/common/service.py
@@ -89,7 +89,6 @@ def _signo_to_signame(signo):
 
 def _set_signals_handler(handler):
     signal.signal(signal.SIGTERM, handler)
-    signal.signal(signal.SIGINT, handler)
     if _sighup_supported():
         signal.signal(signal.SIGHUP, handler)
 
@@ -216,6 +215,7 @@ class ProcessLauncher(object):
         self.sigcaught = None
         self.running = True
         self.wait_interval = wait_interval
+        self.launcher = None
         rfd, self.writepipe = os.pipe()
         self.readpipe = eventlet.greenio.GreenPipe(rfd, 'r')
         self.handle_signal()
@@ -238,20 +238,26 @@ class ProcessLauncher(object):
 
         LOG.info(_LI('Parent process has died unexpectedly, exiting'))
 
+        if self.launcher:
+            self.launcher.stop()
+
         sys.exit(1)
 
     def _child_process_handle_signal(self):
         # Setup child signal handlers differently
+
+        def _sigterm(*args):
+            signal.signal(signal.SIGTERM, signal.SIG_DFL)
+            self.launcher.stop()
+
         def _sighup(*args):
             signal.signal(signal.SIGHUP, signal.SIG_DFL)
             raise SignalExit(signal.SIGHUP)
 
         # Parent signals with SIGTERM when it wants us to go away.
-        signal.signal(signal.SIGTERM, signal.SIG_DFL)
+        signal.signal(signal.SIGTERM, _sigterm)
         if _sighup_supported():
             signal.signal(signal.SIGHUP, _sighup)
-        # Block SIGINT and let the parent send us a SIGTERM
-        signal.signal(signal.SIGINT, signal.SIG_IGN)
 
     def _child_wait_for_exit_or_signal(self, launcher):
         status = 0
@@ -272,8 +278,6 @@ class ProcessLauncher(object):
         except BaseException:
             LOG.exception(_LE('Unhandled exception'))
             status = 2
-        finally:
-            launcher.stop()
 
         return status, signo
 
@@ -312,13 +316,15 @@ class ProcessLauncher(object):
 
         pid = os.fork()
         if pid == 0:
-            launcher = self._child_process(wrap.service)
+            self.launcher = self._child_process(wrap.service)
             while True:
                 self._child_process_handle_signal()
-                status, signo = self._child_wait_for_exit_or_signal(launcher)
+                status, signo = self._child_wait_for_exit_or_signal(
+                    self.launcher)
                 if not _is_sighup_and_daemon(signo):
+                    self.launcher.wait()
                     break
-                launcher.restart()
+                self.launcher.restart()
 
             os._exit(status)
 
@@ -414,6 +420,13 @@ class ProcessLauncher(object):
     def stop(self):
         """Terminate child processes and wait on each."""
         self.running = False
+
+        LOG.debug("Stop services.")
+        for service in set(
+                [wrap.service for wrap in self.children.values()]):
+            service.stop()
+
+        LOG.debug("Killing children.")
         for pid in self.children:
             try:
                 os.kill(pid, signal.SIGTERM)
@@ -470,7 +483,6 @@ class Services(object):
         # wait for graceful shutdown of services:
         for service in self.services:
             service.stop()
-            service.wait()
 
         # Each service has performed cleanup, now signal that the run_service
         # wrapper threads can now die:
@@ -481,6 +493,8 @@ class Services(object):
         self.tg.stop()
 
     def wait(self):
+        for service in self.services:
+            service.wait()
         self.tg.wait()
 
     def restart(self):
diff --git a/tests/unit/test_service.py b/tests/unit/test_service.py
index f0fb5753..acfc9f31 100644
--- a/tests/unit/test_service.py
+++ b/tests/unit/test_service.py
@@ -382,17 +382,29 @@ class ProcessLauncherTest(test_base.BaseTestCase):
         launcher = service.ProcessLauncher()
         self.assertTrue(launcher.running)
 
-        launcher.children = [22, 222]
+        pid_nums = [22, 222]
+        fakeServiceWrapper = service.ServiceWrapper(service.Service(), 1)
+        launcher.children = {pid_nums[0]: fakeServiceWrapper,
+                             pid_nums[1]: fakeServiceWrapper}
         with mock.patch('openstack.common.service.os.kill') as mock_kill:
             with mock.patch.object(launcher, '_wait_child') as _wait_child:
-                _wait_child.side_effect = lambda: launcher.children.pop()
-                launcher.stop()
+
+                def fake_wait_child():
+                    pid = pid_nums.pop()
+                    return launcher.children.pop(pid)
+
+                _wait_child.side_effect = fake_wait_child
+                with mock.patch('openstack.common.service.Service.stop') as \
+                        mock_service_stop:
+                    mock_service_stop.side_effect = lambda: None
+                    launcher.stop()
 
         self.assertFalse(launcher.running)
         self.assertFalse(launcher.children)
-        self.assertEqual([mock.call(22, signal_mock.SIGTERM),
-                          mock.call(222, signal_mock.SIGTERM)],
+        self.assertEqual([mock.call(222, signal_mock.SIGTERM),
+                          mock.call(22, signal_mock.SIGTERM)],
                          mock_kill.mock_calls)
+        mock_service_stop.assert_called_once_with()
 
     @mock.patch(
         "openstack.common.service.ProcessLauncher._signal_handlers_set",
@@ -485,7 +497,7 @@ class ServiceTest(test_base.BaseTestCase):
 
 
 class EventletServerTest(test_base.BaseTestCase):
-    def test_shuts_down_on_sigterm_when_client_connected(self):
+    def run_server(self):
 
         server_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                    'eventlet_service.py')
@@ -524,10 +536,35 @@ class EventletServerTest(test_base.BaseTestCase):
         # server or signal handlers.
         time.sleep(1)
 
+        return (server, conn)
+
+    def test_shuts_down_on_sigint_when_client_connected(self):
+        server, conn = self.run_server()
+
+        # send SIGINT to the server and wait for it to exit while client still
+        # connected.
+        server.send_signal(signal.SIGINT)
+        server.wait()
+        conn.close()
+
+    def test_graceful_shuts_down_on_sigterm_when_client_connected(self):
+        server, conn = self.run_server()
+
         # send SIGTERM to the server and wait for it to exit while client still
         # connected.
         server.send_signal(signal.SIGTERM)
 
-        server.wait()
+        server_wait_thread = threading.Thread(
+            target=lambda server: server.wait(), args=(server,))
+        server_wait_thread.start()
+
+        # server with graceful shutdown must wait forewer
+        # for closing connection by client
+        # but for test 3 seconds is enough
+        time.sleep(3)
+
+        self.assertEqual(True, server_wait_thread.is_alive())
 
         conn.close()
+
+        server_wait_thread.join()