From fe33e328fac88aeb94855d6ddf1d4d76c37415b6 Mon Sep 17 00:00:00 2001 From: James Ennis Date: Wed, 7 Nov 2018 17:51:49 +0000 Subject: using_config.rst: Add documentation to showing how to impose quotas on the local cache This patch partially resolves #700 --- doc/source/using_config.rst | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/doc/source/using_config.rst b/doc/source/using_config.rst index 8835c23db..67574e5c4 100644 --- a/doc/source/using_config.rst +++ b/doc/source/using_config.rst @@ -147,6 +147,44 @@ The default mirror is defined by its name, e.g. ``--default-mirror`` command-line option. +Local cache expiry +~~~~~~~~~~~~~~~~~~ +BuildStream locally caches artifacts, build trees, log files and sources within a +cache located at ``~/.cache/buildstream`` (unless a $XDG_CACHE_HOME environment +variable exists). When building large projects, this cache can get very large, +thus BuildStream will attempt to clean up the cache automatically by expiring the least +recently *used* artifacts. + +By default, cache expiry will begin once the file system which contains the cache +approaches maximum usage. However, it is also possible to impose a quota on the local +cache in the user configuration. This can be done in two ways: + +1. By restricting the maximum size of the cache directory itself. + +For example, to ensure that BuildStream's cache does not grow beyond 100 GB, +simply declare the following in your user configuration (``~/.config/buildstream.conf``): + +.. code:: yaml + + cache: + quota: 100G + +This quota defines the maximum size of the artifact cache in bytes. +Other accepted values are: K, M, G or T (or you can simply declare the value in bytes, without the suffix). +This uses the same format as systemd's +`resource-control `_. + +2. By expiring artifacts once the file system which contains the cache exceeds a specified usage. + +To ensure that we start cleaning the cache once we've used 80% of local disk space (on the file system +which mounts the cache): + +.. code:: yaml + + cache: + quota: 80% + + Default configuration --------------------- The default BuildStream configuration is specified here for reference: -- cgit v1.2.1 From 09faf0029d02cd2792b007041e6e4a793cf6bd3c Mon Sep 17 00:00:00 2001 From: James Ennis Date: Thu, 8 Nov 2018 11:32:59 +0000 Subject: artifactcache.py: Fix misleading error message when using % cache quota Due to the changed Exception message, this patch also changes the test_parse_size_over_1024T test in misc.py --- buildstream/_artifactcache/artifactcache.py | 19 +++++++++++++------ tests/utils/misc.py | 3 ++- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/buildstream/_artifactcache/artifactcache.py b/buildstream/_artifactcache/artifactcache.py index 06a2b84e0..b0324462c 100644 --- a/buildstream/_artifactcache/artifactcache.py +++ b/buildstream/_artifactcache/artifactcache.py @@ -937,15 +937,22 @@ class ArtifactCache(): "Invalid cache quota ({}): ".format(utils._pretty_size(cache_quota)) + "BuildStream requires a minimum cache quota of 2G.") elif cache_quota > cache_size + available_space: # Check maximum + if '%' in self.context.config_cache_quota: + available = (available_space / (stat.f_blocks * stat.f_bsize)) * 100 + available = '{}% of total disk space'.format(round(available, 1)) + else: + available = utils._pretty_size(available_space) + raise LoadError(LoadErrorReason.INVALID_DATA, ("Your system does not have enough available " + "space to support the cache quota specified.\n" + - "You currently have:\n" + - "- {used} of cache in use at {local_cache_path}\n" + - "- {available} of available system storage").format( - used=utils._pretty_size(cache_size), - local_cache_path=self.context.artifactdir, - available=utils._pretty_size(available_space))) + "\nYou have specified a quota of {quota} total disk space.\n" + + "- The filesystem containing {local_cache_path} only " + + "has: {available_size} available.") + .format( + quota=self.context.config_cache_quota, + local_cache_path=self.context.artifactdir, + available_size=available)) # Place a slight headroom (2e9 (2GB) on the cache_quota) into # cache_quota to try and avoid exceptions. diff --git a/tests/utils/misc.py b/tests/utils/misc.py index ae584e4d5..7df08aec5 100644 --- a/tests/utils/misc.py +++ b/tests/utils/misc.py @@ -27,4 +27,5 @@ def test_parse_size_over_1024T(cli, tmpdir): patched_statvfs = mock_os.mock_statvfs(f_bavail=bavail, f_bsize=BLOCK_SIZE) with mock_os.monkey_patch("statvfs", patched_statvfs): result = cli.run(project, args=["build", "file.bst"]) - assert "1025T of available system storage" in result.stderr + failure_msg = 'Your system does not have enough available space to support the cache quota specified.' + assert failure_msg in result.stderr -- cgit v1.2.1