diff options
author | Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> | 2018-09-08 16:34:00 +0900 |
---|---|---|
committer | Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> | 2018-09-10 15:56:12 +0900 |
commit | 879d511732224d0789c35e2c0a8707b7096b3379 (patch) | |
tree | f23ee9482f126bca76b33f29e0fd9339dbd91f78 | |
parent | 8293d8da1ce481eb80f4b137a995d3bf30253ad0 (diff) | |
download | buildstream-879d511732224d0789c35e2c0a8707b7096b3379.tar.gz |
_scheduler: API fixes; _check_cache_size_real() -> check_cache_size()
Here we have a very private looking _check_cache_size_real() function
which no-one would ever want to call from outside of the _scheduler,
especially given it's `_real()` prefix we should look for another
outward facing API to use.
However this is not private to the scheduler, and is intended to
be called by the `Queue` implementations.
o Renamed this to check_cache_size()
o Moved it to the public API section of the Scheduler object
o Added the missing API documenting comment
o Also added the missing API documenting comment to the private
`_run_cleanup()` callback which runs in response to completion
of the cache size calculation job.
o Also place the cleanup job logs into a cleanup subdirectory,
for better symmetry with the cache_size jobs which now have
their own subdirectory
-rw-r--r-- | buildstream/_scheduler/queues/buildqueue.py | 2 | ||||
-rw-r--r-- | buildstream/_scheduler/queues/pullqueue.py | 2 | ||||
-rw-r--r-- | buildstream/_scheduler/scheduler.py | 40 |
3 files changed, 34 insertions, 10 deletions
diff --git a/buildstream/_scheduler/queues/buildqueue.py b/buildstream/_scheduler/queues/buildqueue.py index d5a5872e4..e867ef010 100644 --- a/buildstream/_scheduler/queues/buildqueue.py +++ b/buildstream/_scheduler/queues/buildqueue.py @@ -62,7 +62,7 @@ class BuildQueue(Queue): cache.add_artifact_size(artifact_size) if cache.get_approximate_cache_size() > cache.cache_quota: - self._scheduler._check_cache_size_real() + self._scheduler.check_cache_size() def done(self, job, element, result, success): diff --git a/buildstream/_scheduler/queues/pullqueue.py b/buildstream/_scheduler/queues/pullqueue.py index 2097d95af..e18967cf4 100644 --- a/buildstream/_scheduler/queues/pullqueue.py +++ b/buildstream/_scheduler/queues/pullqueue.py @@ -62,7 +62,7 @@ class PullQueue(Queue): # Build jobs will check the "approximate" size first. Since we # do not get an artifact size from pull jobs, we have to # actually check the cache size. - self._scheduler._check_cache_size_real() + self._scheduler.check_cache_size() # Element._pull() returns True if it downloaded an artifact, # here we want to appear skipped if we did not download. diff --git a/buildstream/_scheduler/scheduler.py b/buildstream/_scheduler/scheduler.py index 651d2f486..3874d6f99 100644 --- a/buildstream/_scheduler/scheduler.py +++ b/buildstream/_scheduler/scheduler.py @@ -239,6 +239,25 @@ class Scheduler(): self._schedule_queue_jobs() self._sched() + # check_cache_size(): + # + # Queues a cache size calculation job, after the cache + # size is calculated, a cleanup job will be run automatically + # if needed. + # + # FIXME: This should ensure that only one cache size job + # is ever pending at a given time. If a cache size + # job is already running, it is correct to queue + # a new one, it is incorrect to have more than one + # of these jobs pending at a given time, though. + # + def check_cache_size(self): + job = CacheSizeJob(self, 'cache_size', 'cache_size/cache_size', + resources=[ResourceType.CACHE, + ResourceType.PROCESS], + complete_cb=self._run_cleanup) + self.schedule_jobs([job]) + ####################################################### # Local Private Methods # ####################################################### @@ -314,25 +333,30 @@ class Scheduler(): self.schedule_jobs(ready) self._sched() + # _run_cleanup() + # + # Schedules the cache cleanup job if the passed size + # exceeds the cache quota. + # + # Args: + # cache_size (int): The calculated cache size + # + # NOTE: This runs in response to completion of the cache size + # calculation job lauched by Scheduler.check_cache_size(), + # which will report the calculated cache size. + # def _run_cleanup(self, cache_size): platform = Platform.get_platform() if cache_size and cache_size < platform.artifactcache.cache_quota: return - job = CleanupJob(self, 'cleanup', 'cleanup', + job = CleanupJob(self, 'cleanup', 'cleanup/cleanup', resources=[ResourceType.CACHE, ResourceType.PROCESS], exclusive_resources=[ResourceType.CACHE], complete_cb=None) self.schedule_jobs([job]) - def _check_cache_size_real(self): - job = CacheSizeJob(self, 'cache_size', 'cache_size/cache_size', - resources=[ResourceType.CACHE, - ResourceType.PROCESS], - complete_cb=self._run_cleanup) - self.schedule_jobs([job]) - # _suspend_jobs() # # Suspend all ongoing jobs. |