1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
|
# Copyright (C) 2013 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import logging
import os
import shutil
import time
import fs.osfs
import cliapp
import morphlib
class GCPlugin(cliapp.Plugin):
def enable(self):
self.app.add_subcommand('gc', self.gc,
arg_synopsis='')
self.app.settings.integer(['cachedir-artifact-delete-older-than'],
'always delete artifacts older than this '
'period in seconds, (default: 1 week)',
metavar='PERIOD',
group="Storage Options",
default=(60*60*24*7))
self.app.settings.integer(['cachedir-artifact-keep-younger-than'],
'allow deletion of artifacts older than '
'this period in seconds, (default: 1 day)',
metavar='PERIOD',
group="Storage Options",
default=(60*60*24))
def disable(self):
pass
def gc(self, args):
'''Make space by removing unused files.
This removes all artifacts older than
--cachedir-artifact-delete-older-than, and may delete artifacts
older than --cachedir-artifact-keep-younger-than if it still
needs to make space.
This removes extracted chunks and staging areas for failed builds
from the directory specified by --tempdir.
'''
tempdir = self.app.settings['tempdir']
cachedir = self.app.settings['cachedir']
tempdir_min_space, cachedir_min_space = \
morphlib.util.unify_space_requirements(
tempdir, self.app.settings['tempdir-min-space'],
cachedir, self.app.settings['cachedir-min-space'])
self.cleanup_tempdir(tempdir, tempdir_min_space)
self.cleanup_cachedir(cachedir, cachedir_min_space)
def cleanup_tempdir(self, temp_path, min_space):
self.app.status(msg='Cleaning up temp dir %(temp_path)s',
temp_path=temp_path, chatty=True)
for subdir in ('failed', 'chunks'):
if morphlib.util.get_bytes_free_in_path(temp_path) >= min_space:
self.app.status(msg='Not Removing subdirectory '
'%(subdir)s, enough space already cleared',
subdir=os.path.join(temp_path, subdir),
chatty=True)
break
self.app.status(msg='Removing temp subdirectory: %(subdir)s',
subdir=subdir)
path = os.path.join(temp_path, subdir)
if os.path.exists(path):
shutil.rmtree(path)
def calculate_delete_range(self):
now = time.time()
always_delete_age = \
now - self.app.settings['cachedir-artifact-delete-older-than']
may_delete_age = \
now - self.app.settings['cachedir-artifact-keep-younger-than']
return always_delete_age, may_delete_age
def find_deletable_artifacts(self, lac, max_age, min_age):
'''Get a list of cache keys in order of how old they are.'''
contents = list(lac.list_contents())
always = set(cachekey
for cachekey, artifacts, mtime in contents
if mtime < max_age)
maybe = ((cachekey, mtime)
for cachekey, artifacts, mtime in contents
if max_age <= mtime < min_age)
return always, [cachekey for cachekey, mtime
in sorted(maybe, key=lambda x: x[1])]
def cleanup_cachedir(self, cache_path, min_space):
def sufficient_free():
free = morphlib.util.get_bytes_free_in_path(cache_path)
return (free >= min_space)
if sufficient_free():
self.app.status(msg='Not cleaning up cachedir, '
'sufficient space already cleared',
chatty=True)
return
lac = morphlib.localartifactcache.LocalArtifactCache(
fs.osfs.OSFS(os.path.join(cache_path, 'artifacts')))
max_age, min_age = self.calculate_delete_range()
logging.debug('Must remove artifacts older than timestamp %d'
% max_age)
always_delete, may_delete = \
self.find_deletable_artifacts(lac, max_age, min_age)
removed = 0
source_count = len(always_delete) + len(may_delete)
logging.debug('Must remove artifacts %s' % repr(always_delete))
logging.debug('Can remove artifacts %s' % repr(may_delete))
# Remove all old artifacts
for cachekey in always_delete:
self.app.status(msg='Removing source %(cachekey)s',
cachekey=cachekey, chatty=True)
lac.remove(cachekey)
removed += 1
# Maybe remove remaining middle-aged artifacts
for cachekey in may_delete:
if sufficient_free():
self.app.status(msg='Finished cleaning up cachedir with '
'%(remaining)d old sources remaining',
remaining=(source_count - removed),
chatty=True)
break
self.app.status(msg='Removing source %(cachekey)s',
cachekey=cachekey, chatty=True)
lac.remove(cachekey)
removed += 1
if sufficient_free():
self.app.status(msg='Made sufficient space in %(cache_path)s '
'after removing %(removed)d sources',
removed=removed, cache_path=cache_path)
return
self.app.status(msg='Unable to clear enough space in %(cache_path)s '
'after removing %(removed)d sources. Please '
'reduce cachedir-artifact-keep-younger-than, '
'clear space from elsewhere, enlarge the disk '
'or reduce cachedir-min-space.',
cache_path=cache_path, removed=removed,
error=True)
|
