New option for num_threads for state change serverliberty-eol7.2.0stable/liberty

Currently max number of client connections(i.e greenlets spawned at
a time) opened at any time by the WSGI server is set to 100 with
wsgi_default_pool_size[1].

This configuration may be fine for neutron api server. But with
wsgi_default_pool_size(=100) requests, state change server
is creating heavy cpu load on agent.
So this server(which run on agents) need lesser value i.e
can be configured to half the number of cpu on agent

We use "ha_keepalived_state_change_server_threads" config option
to configure number of threads in state change server instead of
wsgi_default_pool_size.

[1] https://review.openstack.org/#/c/278007/

DocImpact: Add new config option -
ha_keepalived_state_change_server_threads, to configure number
of threads in state change server.

Closes-Bug: #1581580
Change-Id: I822ea3844792a7731fd24419b7e90e5aef141993
(cherry picked from commit 70ea188f5d87c45fb60ace8b8405274e5f6dd489)

4 files changed, 38 insertions, 4 deletions

diff --git a/neutron/agent/l3/ha.py b/neutron/agent/l3/ha.py
index 2caa999491..01d408e869 100644
--- a/neutron/agent/l3/ha.py
+++ b/neutron/agent/l3/ha.py
@@ -45,6 +45,13 @@ OPTS = [
     cfg.IntOpt('ha_vrrp_advert_int',
                default=2,
                help=_('The advertisement interval in seconds')),
+    cfg.IntOpt('ha_keepalived_state_change_server_threads',
+               default=(1 + common_utils.cpu_count()) // 2,
+               min=1,
+               help=_('Number of concurrent threads for '
+                      'keepalived server connection requests.'
+                      'More threads create a higher CPU load '
+                      'on the agent node.')),
 ]
 
 
@@ -79,7 +86,8 @@ class L3AgentKeepalivedStateChangeServer(object):
 
     def run(self):
         server = agent_utils.UnixDomainWSGIServer(
-            'neutron-keepalived-state-change')
+            'neutron-keepalived-state-change',
+            num_threads=self.conf.ha_keepalived_state_change_server_threads)
         server.start(KeepalivedStateChangeHandler(self.agent),
                      self.get_keepalived_state_change_socket_path(self.conf),
                      workers=0,
diff --git a/neutron/agent/linux/utils.py b/neutron/agent/linux/utils.py
index 93524b92a8..d66a73966f 100644
--- a/neutron/agent/linux/utils.py
+++ b/neutron/agent/linux/utils.py
@@ -402,11 +402,12 @@ class UnixDomainHttpProtocol(eventlet.wsgi.HttpProtocol):
 
 
 class UnixDomainWSGIServer(wsgi.Server):
-    def __init__(self, name):
+    def __init__(self, name, num_threads=None):
         self._socket = None
         self._launcher = None
         self._server = None
-        super(UnixDomainWSGIServer, self).__init__(name, disable_ssl=True)
+        super(UnixDomainWSGIServer, self).__init__(name, disable_ssl=True,
+                                                   num_threads=num_threads)
 
     def start(self, application, file_socket, workers, backlog, mode=None):
         self._socket = eventlet.listen(file_socket,
diff --git a/neutron/tests/unit/agent/linux/test_utils.py b/neutron/tests/unit/agent/linux/test_utils.py
index 15c4af9f67..b6dbde64bc 100644
--- a/neutron/tests/unit/agent/linux/test_utils.py
+++ b/neutron/tests/unit/agent/linux/test_utils.py
@@ -411,9 +411,9 @@ class TestUnixDomainWSGIServer(base.BaseTestCase):
         super(TestUnixDomainWSGIServer, self).setUp()
         self.eventlet_p = mock.patch.object(utils, 'eventlet')
         self.eventlet = self.eventlet_p.start()
-        self.server = utils.UnixDomainWSGIServer('test')
 
     def test_start(self):
+        self.server = utils.UnixDomainWSGIServer('test')
         mock_app = mock.Mock()
         with mock.patch.object(self.server, '_launch') as launcher:
             self.server.start(mock_app, '/the/path', workers=5, backlog=128)
@@ -427,6 +427,7 @@ class TestUnixDomainWSGIServer(base.BaseTestCase):
             launcher.assert_called_once_with(mock_app, workers=5)
 
     def test_run(self):
+        self.server = utils.UnixDomainWSGIServer('test')
         self.server._run('app', 'sock')
 
         self.eventlet.wsgi.server.assert_called_once_with(
@@ -436,3 +437,17 @@ class TestUnixDomainWSGIServer(base.BaseTestCase):
             log=mock.ANY,
             max_size=self.server.num_threads
         )
+
+    def test_num_threads(self):
+        num_threads = 8
+        self.server = utils.UnixDomainWSGIServer('test',
+                                                 num_threads=num_threads)
+        self.server._run('app', 'sock')
+
+        self.eventlet.wsgi.server.assert_called_once_with(
+            'sock',
+            'app',
+            protocol=utils.UnixDomainHttpProtocol,
+            log=mock.ANY,
+            max_size=num_threads
+        )
diff --git a/releasenotes/notes/keepalived-state-change-server-threads-9ed775e7533dd1a0.yaml b/releasenotes/notes/keepalived-state-change-server-threads-9ed775e7533dd1a0.yaml
new file mode 100644
index 0000000000..b2d3bd1eaa
--- /dev/null
+++ b/releasenotes/notes/keepalived-state-change-server-threads-9ed775e7533dd1a0.yaml
@@ -0,0 +1,10 @@
+---
+upgrade:
+  - A new option ``ha_keepalived_state_change_server_threads`` has been
+    added to configure the number of concurrent threads spawned for
+    keepalived server connection requests. Higher values increase the
+    CPU load on the agent nodes. The default value is half of the number
+    of CPUs present on the node. This allows operators to tune the
+    number of threads to suit their environment. With more threads,
+    simultaneous requests for multiple HA routers state change can be
+    handled faster.