diff options
author | Eric Fried <openstack@fried.cc> | 2019-09-16 16:00:39 -0500 |
---|---|---|
committer | Matt Riedemann <mriedem.os@gmail.com> | 2019-10-30 19:44:17 +0000 |
commit | fe05d004b51dc9801749b0f5572e1a2392004830 (patch) | |
tree | c4b536e00f49cceda0af22b08da196abbdbe8b7c | |
parent | f33f8c2ba21dec59a460c4cce19559cd2996c791 (diff) | |
download | nova-fe05d004b51dc9801749b0f5572e1a2392004830.tar.gz |
Only allow one scheduler service in tests
There have been two recent issues [1][2] caused by starting multiple
instances of the same service in tests. This can cause races when the
services' (shared) state conflicts.
With this patch, the nexus of nova service starting,
nova.test.TestCase.start_service, is instrumented to keep track of how
many of each service we are running. If we try to run the scheduler
service more than once, we fail.
We could probably do the same thing for conductor, though that's less
important (for now) because conductor is stateless (for now).
[1] https://bugs.launchpad.net/nova/+bug/1844174
[2] https://review.opendev.org/#/c/681059/ (not a nova service, but same
class of problem)
Change-Id: I56d3cb17260dad8b88f03c0a7b9688efb3258d6f
-rw-r--r-- | nova/test.py | 21 |
1 files changed, 21 insertions, 0 deletions
diff --git a/nova/test.py b/nova/test.py index 6885943428..937bdfd279 100644 --- a/nova/test.py +++ b/nova/test.py @@ -24,6 +24,7 @@ inline callbacks. import nova.monkey_patch # noqa import abc +import collections import copy import datetime import inspect @@ -213,6 +214,9 @@ class TestCase(testtools.TestCase): os.environ.get('OS_TEST_TIMEOUT', 0), self.TIMEOUT_SCALING_FACTOR)) + # How many of which service we've started. {$service-name: $count} + self._service_fixture_count = collections.defaultdict(int) + self.useFixture(nova_fixtures.OpenStackSDKFixture()) self.useFixture(fixtures.NestedTempfile()) @@ -434,6 +438,10 @@ class TestCase(testtools.TestCase): CONF.set_override(k, v, group) def start_service(self, name, host=None, **kwargs): + # Disallow starting multiple scheduler services + if name == 'scheduler' and self._service_fixture_count[name]: + raise TestingException("Duplicate start_service(%s)!" % name) + cell = None # if the host is None then the CONF.host remains defaulted to # 'fake-mini' (originally done in ConfFixture) @@ -459,6 +467,19 @@ class TestCase(testtools.TestCase): svc = self.useFixture( nova_fixtures.ServiceFixture(name, host, cell=cell, **kwargs)) + # Keep track of how many instances of this service are running. + self._service_fixture_count[name] += 1 + real_stop = svc.service.stop + + # Make sure stopping the service decrements the active count, so that + # start,stop,start doesn't trigger the "Duplicate start_service" + # exception. + def patch_stop(*a, **k): + self._service_fixture_count[name] -= 1 + return real_stop(*a, **k) + self.useFixture(fixtures.MockPatchObject( + svc.service, 'stop', patch_stop)) + return svc.service def restart_compute_service(self, compute, keep_hypervisor_state=True): |