Use btrfs enquque when available (#1926)

btrfs has operations that are blocking and when we try to resize a btrfs
filesystem we may be in a race condition with blocking operations. Use the
enqueue feature introduced in btrfs 5.10 to queue our resize request until
resize if possible.

Before this commit, hitting this race would cause the command to
immediately fail. With this change, the resize is queued and the command
blocks until resize has completed (event driven, with a poll loop of 1m).

2 files changed, 37 insertions, 5 deletions

diff --git a/cloudinit/config/cc_resizefs.py b/cloudinit/config/cc_resizefs.py
index 7a0ecf96..0e6197a2 100644
--- a/cloudinit/config/cc_resizefs.py
+++ b/cloudinit/config/cc_resizefs.py
@@ -60,15 +60,28 @@ def _resize_btrfs(mount_point, devpth):
     if not util.mount_is_read_write(mount_point) and os.path.isdir(
         "%s/.snapshots" % mount_point
     ):
-        return (
+        cmd = [
             "btrfs",
             "filesystem",
             "resize",
             "max",
             "%s/.snapshots" % mount_point,
-        )
+        ]
     else:
-        return ("btrfs", "filesystem", "resize", "max", mount_point)
+        cmd = ["btrfs", "filesystem", "resize", "max", mount_point]
+
+    # btrfs has exclusive operations and resize may fail if btrfs is busy
+    # doing one of the operations that prevents resize. As of btrfs 5.10
+    # the resize operation can be queued
+    btrfs_with_queue = util.Version.from_str("5.10")
+    system_btrfs_ver = util.Version.from_str(
+        subp.subp(["btrfs", "--version"])[0].split("v")[-1].strip()
+    )
+    if system_btrfs_ver >= btrfs_with_queue:
+        idx = cmd.index("resize")
+        cmd.insert(idx + 1, "--enqueue")
+
+    return tuple(cmd)
 
 
 def _resize_ext(mount_point, devpth):
diff --git a/tests/unittests/config/test_cc_resizefs.py b/tests/unittests/config/test_cc_resizefs.py
index b46fab51..d7fda1b8 100644
--- a/tests/unittests/config/test_cc_resizefs.py
+++ b/tests/unittests/config/test_cc_resizefs.py
@@ -444,10 +444,12 @@ class TestMaybeGetDevicePathAsWritableBlock(CiTestCase):
 
     @mock.patch("cloudinit.util.mount_is_read_write")
     @mock.patch("cloudinit.config.cc_resizefs.os.path.isdir")
-    def test_resize_btrfs_mount_is_ro(self, m_is_dir, m_is_rw):
+    @mock.patch("cloudinit.subp.subp")
+    def test_resize_btrfs_mount_is_ro(self, m_subp, m_is_dir, m_is_rw):
         """Do not resize / directly if it is read-only. (LP: #1734787)."""
         m_is_rw.return_value = False
         m_is_dir.return_value = True
+        m_subp.return_value = ("btrfs-progs v4.19 \n", "")
         self.assertEqual(
             ("btrfs", "filesystem", "resize", "max", "//.snapshots"),
             _resize_btrfs("/", "/dev/sda1"),
@@ -455,15 +457,32 @@ class TestMaybeGetDevicePathAsWritableBlock(CiTestCase):
 
     @mock.patch("cloudinit.util.mount_is_read_write")
     @mock.patch("cloudinit.config.cc_resizefs.os.path.isdir")
-    def test_resize_btrfs_mount_is_rw(self, m_is_dir, m_is_rw):
+    @mock.patch("cloudinit.subp.subp")
+    def test_resize_btrfs_mount_is_rw(self, m_subp, m_is_dir, m_is_rw):
         """Do not resize / directly if it is read-only. (LP: #1734787)."""
         m_is_rw.return_value = True
         m_is_dir.return_value = True
+        m_subp.return_value = ("btrfs-progs v4.19 \n", "")
         self.assertEqual(
             ("btrfs", "filesystem", "resize", "max", "/"),
             _resize_btrfs("/", "/dev/sda1"),
         )
 
+    @mock.patch("cloudinit.util.mount_is_read_write")
+    @mock.patch("cloudinit.config.cc_resizefs.os.path.isdir")
+    @mock.patch("cloudinit.subp.subp")
+    def test_resize_btrfs_mount_is_rw_has_queue(
+        self, m_subp, m_is_dir, m_is_rw
+    ):
+        """Queue the resize request if btrfs >= 5.10"""
+        m_is_rw.return_value = True
+        m_is_dir.return_value = True
+        m_subp.return_value = ("btrfs-progs v5.10 \n", "")
+        self.assertEqual(
+            ("btrfs", "filesystem", "resize", "--enqueue", "max", "/"),
+            _resize_btrfs("/", "/dev/sda1"),
+        )
+
     @mock.patch("cloudinit.util.is_container", return_value=True)
     @mock.patch("cloudinit.util.is_FreeBSD")
     def test_maybe_get_writable_device_path_zfs_freebsd(