lxd: Retry if the server isn't ready (#2025)

2 files changed, 57 insertions, 2 deletions

diff --git a/cloudinit/sources/DataSourceLXD.py b/cloudinit/sources/DataSourceLXD.py
index eea1e294..ab440cc8 100644
--- a/cloudinit/sources/DataSourceLXD.py
+++ b/cloudinit/sources/DataSourceLXD.py
@@ -11,6 +11,7 @@ Notes:
 import os
 import socket
 import stat
+import time
 from enum import Flag, auto
 from json.decoder import JSONDecodeError
 from typing import Any, Dict, List, Optional, Union, cast
@@ -299,7 +300,20 @@ def _get_json_response(
 def _do_request(
     session: requests.Session, url: str, do_raise: bool = True
 ) -> requests.Response:
-    response = session.get(url)
+    for retries in range(30, 0, -1):
+        response = session.get(url)
+        if 500 == response.status_code:
+            # retry every 0.1 seconds for 3 seconds in the case of 500 error
+            # tis evil, but it also works around a bug
+            time.sleep(0.1)
+            LOG.warning(
+                "[GET] [HTTP:%d] %s, retrying %d more time(s)",
+                response.status_code,
+                url,
+                retries,
+            )
+        else:
+            break
     LOG.debug("[GET] [HTTP:%d] %s", response.status_code, url)
     if do_raise and not response.ok:
         raise sources.InvalidMetaDataException(
diff --git a/tests/unittests/sources/test_lxd.py b/tests/unittests/sources/test_lxd.py
index 8f6db3fe..efc24883 100644
--- a/tests/unittests/sources/test_lxd.py
+++ b/tests/unittests/sources/test_lxd.py
@@ -697,5 +697,46 @@ class TestReadMetadata:
             == m_session_get.call_args_list
         )
 
+    @mock.patch.object(lxd.requests.Session, "get")
+    @mock.patch.object(lxd.time, "sleep")
+    def test_socket_retry(self, m_session_get, m_sleep):
+        """validate socket retry logic"""
+
+        def generate_return_codes():
+            """
+            [200]
+            [500, 200]
+            [500, 500, 200]
+            [500, 500, ..., 200]
+            """
+            five_hundreds = []
+
+            # generate a couple of longer ones to assert timeout condition
+            for _ in range(33):
+                five_hundreds.append(500)
+                yield [*five_hundreds, 200]
+
+        for return_codes in generate_return_codes():
+            m = mock.Mock(
+                get=mock.Mock(
+                    side_effect=[
+                        mock.MagicMock(
+                            ok=mock.PropertyMock(return_value=True),
+                            status_code=code,
+                            text=mock.PropertyMock(
+                                return_value="properly formatted http response"
+                            ),
+                        )
+                        for code in return_codes
+                    ]
+                )
+            )
+            resp = lxd._do_request(m, "http://agua/")
 
-# vi: ts=4 expandtab
+            # assert that 30 iterations or the first 200 code is the final
+            # attempt, whichever comes first
+            assert min(len(return_codes), 30) == m.get.call_count
+            if len(return_codes) < 31:
+                assert 200 == resp.status_code
+            else:
+                assert 500 == resp.status_code