diff options
author | Jonathan Maw <jonathan.maw@codethink.co.uk> | 2018-08-15 18:08:00 +0100 |
---|---|---|
committer | Valentin David <valentin.david@gmail.com> | 2018-08-22 13:22:27 +0000 |
commit | 6a7fdf12da800d91d3e752aacb8af321f48a7e40 (patch) | |
tree | b2b6e87cb081c10c924247d7a99a23c3a6df2af6 | |
parent | b567666d622173ed9f97f7a0c2d6da4f01a5e070 (diff) | |
download | buildstream-valentindavid/faster_startup-1.2.tar.gz |
Use ArtifactCache's get_cache_size when calculating the quotavalentindavid/faster_startup-1.2
It would have been more convenient to store the quota size in the
ArtifactCache, except that the scheduler needs the quota size and
doesn't have access to the ArtifactCache.
-rw-r--r-- | buildstream/_context.py | 105 | ||||
-rw-r--r-- | buildstream/_frontend/app.py | 4 |
2 files changed, 57 insertions, 52 deletions
diff --git a/buildstream/_context.py b/buildstream/_context.py index 8ebb61d23..cb19ddc45 100644 --- a/buildstream/_context.py +++ b/buildstream/_context.py @@ -124,6 +124,8 @@ class Context(): self._workspaces = None self._log_handle = None self._log_filename = None + self._config_cache_quota = None + self._artifactdir_volume = None # load() # @@ -187,67 +189,19 @@ class Context(): while not os.path.exists(artifactdir_volume): artifactdir_volume = os.path.dirname(artifactdir_volume) + self._artifactdir_volume = artifactdir_volume + # We read and parse the cache quota as specified by the user cache_quota = _yaml.node_get(cache, str, 'quota', default_value='infinity') try: - cache_quota = utils._parse_size(cache_quota, artifactdir_volume) + cache_quota = utils._parse_size(cache_quota, self._artifactdir_volume) except utils.UtilError as e: raise LoadError(LoadErrorReason.INVALID_DATA, "{}\nPlease specify the value in bytes or as a % of full disk space.\n" "\nValid values are, for example: 800M 10G 1T 50%\n" .format(str(e))) from e - # Headroom intended to give BuildStream a bit of leeway. - # This acts as the minimum size of cache_quota and also - # is taken from the user requested cache_quota. - # - if 'BST_TEST_SUITE' in os.environ: - headroom = 0 - else: - headroom = 2e9 - - stat = os.statvfs(artifactdir_volume) - available_space = (stat.f_bsize * stat.f_bavail) - - # Again, the artifact directory may not yet have been created yet - # - if not os.path.exists(self.artifactdir): - cache_size = 0 - else: - cache_size = utils._get_dir_size(self.artifactdir) - - # Ensure system has enough storage for the cache_quota - # - # If cache_quota is none, set it to the maximum it could possibly be. - # - # Also check that cache_quota is atleast as large as our headroom. - # - if cache_quota is None: # Infinity, set to max system storage - cache_quota = cache_size + available_space - if cache_quota < headroom: # Check minimum - raise LoadError(LoadErrorReason.INVALID_DATA, - "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) + - "BuildStream requires a minimum cache quota of 2G.") - elif cache_quota > cache_size + available_space: # Check maximum - raise LoadError(LoadErrorReason.INVALID_DATA, - ("Your system does not have enough available " + - "space to support the cache quota specified.\n" + - "You currently have:\n" + - "- {used} of cache in use at {local_cache_path}\n" + - "- {available} of available system storage").format( - used=utils._pretty_size(cache_size), - local_cache_path=self.artifactdir, - available=utils._pretty_size(available_space))) - - # Place a slight headroom (2e9 (2GB) on the cache_quota) into - # cache_quota to try and avoid exceptions. - # - # Of course, we might still end up running out during a build - # if we end up writing more than 2G, but hey, this stuff is - # already really fuzzy. - # - self.cache_quota = cache_quota - headroom - self.cache_lower_threshold = self.cache_quota / 2 + self._config_cache_quota = cache_quota # Load artifact share configuration self.artifact_cache_specs = ArtifactCache.specs_from_config_node(defaults) @@ -571,6 +525,53 @@ class Context(): def get_log_filename(self): return self._log_filename + def set_cache_quota(self, cache_size): + # Headroom intended to give BuildStream a bit of leeway. + # This acts as the minimum size of cache_quota and also + # is taken from the user requested cache_quota. + # + if 'BST_TEST_SUITE' in os.environ: + headroom = 0 + else: + headroom = 2e9 + + stat = os.statvfs(self._artifactdir_volume) + available_space = (stat.f_bsize * stat.f_bavail) + + # Ensure system has enough storage for the cache_quota + # + # If cache_quota is none, set it to the maximum it could possibly be. + # + # Also check that cache_quota is atleast as large as our headroom. + # + cache_quota = self._config_cache_quota + if cache_quota is None: # Infinity, set to max system storage + cache_quota = cache_size + available_space + if cache_quota < headroom: # Check minimum + raise LoadError(LoadErrorReason.INVALID_DATA, + "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) + + "BuildStream requires a minimum cache quota of 2G.") + elif cache_quota > cache_size + available_space: # Check maximum + raise LoadError(LoadErrorReason.INVALID_DATA, + ("Your system does not have enough available " + + "space to support the cache quota specified.\n" + + "You currently have:\n" + + "- {used} of cache in use at {local_cache_path}\n" + + "- {available} of available system storage").format( + used=utils._pretty_size(cache_size), + local_cache_path=self.artifactdir, + available=utils._pretty_size(available_space))) + + # Place a slight headroom (2e9 (2GB) on the cache_quota) into + # cache_quota to try and avoid exceptions. + # + # Of course, we might still end up running out during a build + # if we end up writing more than 2G, but hey, this stuff is + # already really fuzzy. + # + self.cache_quota = cache_quota - headroom + self.cache_lower_threshold = self.cache_quota / 2 + # _record_message() # # Records the message if recording is enabled diff --git a/buildstream/_frontend/app.py b/buildstream/_frontend/app.py index e32b87020..2dd22dc9d 100644 --- a/buildstream/_frontend/app.py +++ b/buildstream/_frontend/app.py @@ -202,6 +202,10 @@ class App(): Platform.create_instance(self.context) + platform = Platform.get_platform() + cache_size = platform._artifact_cache.calculate_cache_size() + self.context.set_cache_quota(cache_size) + # Create the logger right before setting the message handler self.logger = LogLine(self.context, self._content_profile, |