diff options
author | Daniel Silverstone <daniel.silverstone@codethink.co.uk> | 2014-04-02 18:34:34 +0300 |
---|---|---|
committer | Sam Thursfield <sam.thursfield@codethink.co.uk> | 2014-04-14 07:56:11 +0000 |
commit | 5450ad0cad99d30fe2c06fd574ffd7bccb1c2937 (patch) | |
tree | cdaa2e0e4c07246cc04c155c6cfe7244cea24cde | |
parent | 8518071959d0ccc1f2cdc76925e83a28c1179818 (diff) | |
download | morph-5450ad0cad99d30fe2c06fd574ffd7bccb1c2937.tar.gz |
distbuild: Stop workers blowing their entire cache away every time
-rw-r--r-- | morphlib/plugins/distbuild_plugin.py | 20 |
1 files changed, 14 insertions, 6 deletions
diff --git a/morphlib/plugins/distbuild_plugin.py b/morphlib/plugins/distbuild_plugin.py index 27d87e35..375e70f1 100644 --- a/morphlib/plugins/distbuild_plugin.py +++ b/morphlib/plugins/distbuild_plugin.py @@ -18,6 +18,7 @@ import cliapp import logging +import re import sys import morphlib @@ -89,16 +90,23 @@ class WorkerBuild(cliapp.Plugin): artifact = distbuild.deserialise_artifact(serialized) bc = morphlib.buildcommand.BuildCommand(self.app) - - # We always, unconditionally clear the local artifact cache - # to avoid it growing boundlessly on a worker. Especially system - # artifacts are big (up to gigabytes), and having a new one for - # every build eats up a lot of disk space. - bc.lac.clear() + + # Now, before we start the build, we garbage collect the caches + # to ensure we have room. First we remove all system artifacts + # since we never need to recover those from workers post-hoc + for cachekey, artifacts, last_used in bc.lac.list_contents(): + if any(self.is_system_artifact(f) for f in artifacts): + logging.debug("Removing all artifacts for system %s" % + cachekey) + bc.lac.remove(cachekey) + + self.app.subcommands['gc']([]) arch = artifact.arch bc.build_artifact(artifact, bc.new_build_env(arch)) + def is_system_artifact(self, filename): + return re.match(r'^[0-9a-fA-F]{64}\.system\.', filename) class WorkerDaemon(cliapp.Plugin): |