_artifactcache.py: Refactored to use utils._get_volume_size()

This will benefit from a better UtilError being raised, and
and turns the artifact cache's local function into a one liner.

The loop which finds the first existing directory in the
given path has been removed, being meaningless due to the
call to os.makedirs() in ArtifactCache.__init__().

The local function was renamed to _get_cache_volume_size() and
no longer takes any arguments, which is more suitable for the
function as it serves as a testing override surface for
unittest.mock().

The following test cases which use the function to override
the ArtifactCache behavior have been updated to use the new
overridable function name:

   tests/artifactcache/cache_size.py
   tests/artifactcache/expiry.py

3 files changed, 76 insertions, 11 deletions

diff --git a/buildstream/_artifactcache/artifactcache.py b/buildstream/_artifactcache/artifactcache.py
index 2b1b8f1c7..53f7708d6 100644
--- a/buildstream/_artifactcache/artifactcache.py
+++ b/buildstream/_artifactcache/artifactcache.py
@@ -333,7 +333,7 @@ class ArtifactCache():
     # it is greater than the actual cache size.
     #
     # Returns:
-    #     (int) An approximation of the artifact cache size.
+    #     (int) An approximation of the artifact cache size, in bytes.
     #
     def get_cache_size(self):
 
@@ -674,21 +674,16 @@ class ArtifactCache():
         else:
             headroom = 2e9
 
-        artifactdir_volume = self.context.artifactdir
-        while not os.path.exists(artifactdir_volume):
-            artifactdir_volume = os.path.dirname(artifactdir_volume)
-
         try:
-            cache_quota = utils._parse_size(self.context.config_cache_quota, artifactdir_volume)
+            cache_quota = utils._parse_size(self.context.config_cache_quota,
+                                            self.context.artifactdir)
         except utils.UtilError as e:
             raise LoadError(LoadErrorReason.INVALID_DATA,
                             "{}\nPlease specify the value in bytes or as a % of full disk space.\n"
                             "\nValid values are, for example: 800M 10G 1T 50%\n"
                             .format(str(e))) from e
 
-        stat = os.statvfs(artifactdir_volume)
-        available_space = (stat.f_bsize * stat.f_bavail)
-
+        total_size, available_space = self._get_cache_volume_size()
         cache_size = self.get_cache_size()
 
         # Ensure system has enough storage for the cache_quota
@@ -724,6 +719,22 @@ class ArtifactCache():
         self._cache_quota = cache_quota - headroom
         self._cache_lower_threshold = self._cache_quota / 2
 
+    # _get_cache_volume_size()
+    #
+    # Get the available space and total space for the volume on
+    # which the artifact cache is located.
+    #
+    # Returns:
+    #    (int): The total number of bytes on the volume
+    #    (int): The number of available bytes on the volume
+    #
+    # NOTE: We use this stub to allow the test cases
+    #       to override what an artifact cache thinks
+    #       about it's disk size and available bytes.
+    #
+    def _get_cache_volume_size(self):
+        return utils._get_volume_size(self.context.artifactdir)
+
 
 # _configured_remote_artifact_cache_specs():
 #
diff --git a/tests/artifactcache/cache_size.py b/tests/artifactcache/cache_size.py
index 0d12cda8c..8e47f9fb6 100644
--- a/tests/artifactcache/cache_size.py
+++ b/tests/artifactcache/cache_size.py
@@ -60,3 +60,29 @@ def test_cache_size_write(cli, tmpdir):
     with open(sizefile, "r") as f:
         size_data = f.read()
     size = int(size_data)
+
+
+def test_quota_over_1024T(cli, tmpdir):
+    KiB = 1024
+    MiB = (KiB * 1024)
+    GiB = (MiB * 1024)
+    TiB = (GiB * 1024)
+
+    cli.configure({
+        'cache': {
+            'quota': 2048 * TiB
+        }
+    })
+    project = tmpdir.join("main")
+    os.makedirs(str(project))
+    _yaml.dump({'name': 'main'}, str(project.join("project.conf")))
+
+    volume_space_patch = mock.patch(
+        "buildstream._artifactcache.ArtifactCache._get_cache_volume_size",
+        autospec=True,
+        return_value=(1025 * TiB, 1025 * TiB)
+    )
+
+    with volume_space_patch:
+        result = cli.run(project, args=["build", "file.bst"])
+        result.assert_main_error(ErrorDomain.ARTIFACT, 'insufficient-storage-for-quota')
diff --git a/tests/artifactcache/expiry.py b/tests/artifactcache/expiry.py
index f8b928cbf..68a1c87cd 100644
--- a/tests/artifactcache/expiry.py
+++ b/tests/artifactcache/expiry.py
@@ -303,8 +303,36 @@ def test_invalid_cache_quota(cli, datafiles, tmpdir, quota, success):
         }
     })
 
-    res = cli.run(project=project, args=['workspace', 'list'])
-    if success:
+    # We patch how we get space information
+    # Ideally we would instead create a FUSE device on which we control
+    # everything.
+    # If the value is a percentage, we fix the current values to take into
+    # account the block size, since this is important in how we compute the size
+
+    if quota.endswith("%"):  # We set the used space at 60% of total space
+        stats = os.statvfs(".")
+        free_space = 0.6 * stats.f_bsize * stats.f_blocks
+        total_space = stats.f_bsize * stats.f_blocks
+    else:
+        free_space = 6000
+        total_space = 10000
+
+    volume_space_patch = mock.patch(
+        "buildstream._artifactcache.ArtifactCache._get_cache_volume_size",
+        autospec=True,
+        return_value=(total_space, free_space),
+    )
+
+    cache_size_patch = mock.patch(
+        "buildstream._artifactcache.ArtifactCache.get_cache_size",
+        autospec=True,
+        return_value=0,
+    )
+
+    with volume_space_patch, cache_size_patch:
+        res = cli.run(project=project, args=['workspace', 'list'])
+
+    if err_domain == 'success':
         res.assert_success()
     else:
         res.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA)