Use ArtifactCache's get_cache_size when calculating the quota

It would have been more convenient to store the quota size in the
ArtifactCache, except that the scheduler needs the quota size and
doesn't have access to the ArtifactCache.

2 files changed, 57 insertions, 52 deletions

diff --git a/buildstream/_context.py b/buildstream/_context.py
index 8ebb61d23..cb19ddc45 100644
--- a/buildstream/_context.py
+++ b/buildstream/_context.py
@@ -124,6 +124,8 @@ class Context():
         self._workspaces = None
         self._log_handle = None
         self._log_filename = None
+        self._config_cache_quota = None
+        self._artifactdir_volume = None
 
     # load()
     #
@@ -187,67 +189,19 @@ class Context():
         while not os.path.exists(artifactdir_volume):
             artifactdir_volume = os.path.dirname(artifactdir_volume)
 
+        self._artifactdir_volume = artifactdir_volume
+
         # We read and parse the cache quota as specified by the user
         cache_quota = _yaml.node_get(cache, str, 'quota', default_value='infinity')
         try:
-            cache_quota = utils._parse_size(cache_quota, artifactdir_volume)
+            cache_quota = utils._parse_size(cache_quota, self._artifactdir_volume)
         except utils.UtilError as e:
             raise LoadError(LoadErrorReason.INVALID_DATA,
                             "{}\nPlease specify the value in bytes or as a % of full disk space.\n"
                             "\nValid values are, for example: 800M 10G 1T 50%\n"
                             .format(str(e))) from e
 
-        # Headroom intended to give BuildStream a bit of leeway.
-        # This acts as the minimum size of cache_quota and also
-        # is taken from the user requested cache_quota.
-        #
-        if 'BST_TEST_SUITE' in os.environ:
-            headroom = 0
-        else:
-            headroom = 2e9
-
-        stat = os.statvfs(artifactdir_volume)
-        available_space = (stat.f_bsize * stat.f_bavail)
-
-        # Again, the artifact directory may not yet have been created yet
-        #
-        if not os.path.exists(self.artifactdir):
-            cache_size = 0
-        else:
-            cache_size = utils._get_dir_size(self.artifactdir)
-
-        # Ensure system has enough storage for the cache_quota
-        #
-        # If cache_quota is none, set it to the maximum it could possibly be.
-        #
-        # Also check that cache_quota is atleast as large as our headroom.
-        #
-        if cache_quota is None:  # Infinity, set to max system storage
-            cache_quota = cache_size + available_space
-        if cache_quota < headroom:  # Check minimum
-            raise LoadError(LoadErrorReason.INVALID_DATA,
-                            "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
-                            "BuildStream requires a minimum cache quota of 2G.")
-        elif cache_quota > cache_size + available_space:  # Check maximum
-            raise LoadError(LoadErrorReason.INVALID_DATA,
-                            ("Your system does not have enough available " +
-                             "space to support the cache quota specified.\n" +
-                             "You currently have:\n" +
-                             "- {used} of cache in use at {local_cache_path}\n" +
-                             "- {available} of available system storage").format(
-                                 used=utils._pretty_size(cache_size),
-                                 local_cache_path=self.artifactdir,
-                                 available=utils._pretty_size(available_space)))
-
-        # Place a slight headroom (2e9 (2GB) on the cache_quota) into
-        # cache_quota to try and avoid exceptions.
-        #
-        # Of course, we might still end up running out during a build
-        # if we end up writing more than 2G, but hey, this stuff is
-        # already really fuzzy.
-        #
-        self.cache_quota = cache_quota - headroom
-        self.cache_lower_threshold = self.cache_quota / 2
+        self._config_cache_quota = cache_quota
 
         # Load artifact share configuration
         self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults)
@@ -571,6 +525,53 @@ class Context():
     def get_log_filename(self):
         return self._log_filename
 
+    def set_cache_quota(self, cache_size):
+        # Headroom intended to give BuildStream a bit of leeway.
+        # This acts as the minimum size of cache_quota and also
+        # is taken from the user requested cache_quota.
+        #
+        if 'BST_TEST_SUITE' in os.environ:
+            headroom = 0
+        else:
+            headroom = 2e9
+
+        stat = os.statvfs(self._artifactdir_volume)
+        available_space = (stat.f_bsize * stat.f_bavail)
+
+        # Ensure system has enough storage for the cache_quota
+        #
+        # If cache_quota is none, set it to the maximum it could possibly be.
+        #
+        # Also check that cache_quota is atleast as large as our headroom.
+        #
+        cache_quota = self._config_cache_quota
+        if cache_quota is None:  # Infinity, set to max system storage
+            cache_quota = cache_size + available_space
+        if cache_quota < headroom:  # Check minimum
+            raise LoadError(LoadErrorReason.INVALID_DATA,
+                            "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) +
+                            "BuildStream requires a minimum cache quota of 2G.")
+        elif cache_quota > cache_size + available_space:  # Check maximum
+            raise LoadError(LoadErrorReason.INVALID_DATA,
+                            ("Your system does not have enough available " +
+                             "space to support the cache quota specified.\n" +
+                             "You currently have:\n" +
+                             "- {used} of cache in use at {local_cache_path}\n" +
+                             "- {available} of available system storage").format(
+                                 used=utils._pretty_size(cache_size),
+                                 local_cache_path=self.artifactdir,
+                                 available=utils._pretty_size(available_space)))
+
+        # Place a slight headroom (2e9 (2GB) on the cache_quota) into
+        # cache_quota to try and avoid exceptions.
+        #
+        # Of course, we might still end up running out during a build
+        # if we end up writing more than 2G, but hey, this stuff is
+        # already really fuzzy.
+        #
+        self.cache_quota = cache_quota - headroom
+        self.cache_lower_threshold = self.cache_quota / 2
+
     # _record_message()
     #
     # Records the message if recording is enabled
diff --git a/buildstream/_frontend/app.py b/buildstream/_frontend/app.py
index 1550fbcb3..eb8d6849c 100644
--- a/buildstream/_frontend/app.py
+++ b/buildstream/_frontend/app.py
@@ -201,6 +201,10 @@ class App():
 
         Platform.create_instance(self.context)
 
+        platform = Platform.get_platform()
+        cache_size = platform._artifact_cache.calculate_cache_size()
+        self.context.set_cache_quota(cache_size)
+
         # Create the logger right before setting the message handler
         self.logger = LogLine(self.context,
                               self._content_profile,