diff options
| author | Julia Kreger <juliaashleykreger@gmail.com> | 2022-06-23 11:04:20 -0700 |
|---|---|---|
| committer | Ruby Loo <opensrloo@gmail.com> | 2022-12-19 19:55:34 +0000 |
| commit | 8e6a337f2f6e9e2c3bbddad934ee9cd712b58c25 (patch) | |
| tree | 59b55f896070403ed4d835ae42809e751c731418 | |
| parent | 793b981320d9d5bd034397bffcfb949b55506027 (diff) | |
| download | ironic-8e6a337f2f6e9e2c3bbddad934ee9cd712b58c25.tar.gz | |
Prevent pxe retry when agent token exists
A race condition can be observed in CI under heavy load where the
conductor triggers are boot of the agent before it is fully online
based upon state, but not considering the existence of an agent
token. As a result, agent is never able to check in with Ironic
and the overall operation fails.
We now consider agent token's existence before retrying PXE as
it is the very earliest indicator of a starting agent.
Change-Id: Ice764866a08647031d16570860ec384204269501
Story: 2010107
Task: 45674
(cherry picked from commit d75424b5e5685a9cf04b30a5b0555efd1313e9c3)
| -rw-r--r-- | ironic/drivers/modules/pxe_base.py | 6 | ||||
| -rw-r--r-- | ironic/tests/unit/drivers/modules/test_pxe.py | 12 | ||||
| -rw-r--r-- | releasenotes/notes/prevent-pxe-retry-when-token-exists-a4f38f7da56c1397.yaml | 7 |
3 files changed, 25 insertions, 0 deletions
diff --git a/ironic/drivers/modules/pxe_base.py b/ironic/drivers/modules/pxe_base.py index 7434e179b..8da196dae 100644 --- a/ironic/drivers/modules/pxe_base.py +++ b/ironic/drivers/modules/pxe_base.py @@ -505,6 +505,12 @@ class PXEBaseMixin(object): def _should_retry_boot(node): # NOTE(dtantsur): this assumes IPA, do we need to make it generic? for field in ('agent_last_heartbeat', 'last_power_state_change'): + if node.driver_internal_info.get('agent_secret_token', False): + LOG.debug('Not retrying PXE boot for node %(node)s; an agent ' + 'token has been identified, meaning the agent ' + 'has started.', + {'node': node.uuid}) + return False if manager_utils.value_within_timeout( node.driver_internal_info.get(field), CONF.pxe.boot_retry_timeout): diff --git a/ironic/tests/unit/drivers/modules/test_pxe.py b/ironic/tests/unit/drivers/modules/test_pxe.py index fecc0b374..d8ad8d586 100644 --- a/ironic/tests/unit/drivers/modules/test_pxe.py +++ b/ironic/tests/unit/drivers/modules/test_pxe.py @@ -1152,6 +1152,18 @@ class PXEBootRetryTestCase(db_base.DbTestCase): mock_boot_dev.assert_called_once_with(task, 'pxe', persistent=False) + def test_check_boot_status_not_retry_with_token(self, mock_power, + mock_boot_dev): + with task_manager.acquire(self.context, self.node.uuid, + shared=True) as task: + task.node.driver_internal_info = { + 'agent_secret_token': 'xyz' + } + task.driver.boot._check_boot_status(task) + self.assertTrue(task.shared) + mock_power.assert_not_called() + mock_boot_dev.assert_not_called() + class iPXEBootRetryTestCase(PXEBootRetryTestCase): diff --git a/releasenotes/notes/prevent-pxe-retry-when-token-exists-a4f38f7da56c1397.yaml b/releasenotes/notes/prevent-pxe-retry-when-token-exists-a4f38f7da56c1397.yaml new file mode 100644 index 000000000..5db6db6ec --- /dev/null +++ b/releasenotes/notes/prevent-pxe-retry-when-token-exists-a4f38f7da56c1397.yaml @@ -0,0 +1,7 @@ +--- +fixes: + - | + Fixes a race condition in PXE initialization where logic to retry + what we suspect as potentially failed PXE boot operations was not + consulting if an ``agent token`` had been established, which is the + very first step in agent initialization. |