1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
|
# Copyright (C) 2012,2013 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
import collections
import os
import morphlib
class LocalArtifactCache(object):
'''Abstraction over the local artifact cache
It provides methods for getting a file handle to cached artifacts
so that the layout of the cache need not be known.
It also updates modification times of artifacts so that it can track
when they were last used, so it can be requested to clean up if
disk space is low.
Modification time is updated in both the get and has methods.
NOTE: Parts of the build assume that every artifact of a source is
available, so all the artifacts of a source need to be removed together.
This complication needs to be handled either during the fetch logic, by
updating the mtime of every artifact belonging to a source, or at
cleanup time by only removing an artifact if every artifact belonging to
a source is too old, and then remove them all at once.
Since the cleanup logic will be complicated for other reasons it makes
sense to put the complication there.
'''
def __init__(self, cachedir):
self.cachedir = cachedir
def put(self, artifact):
filename = self.artifact_filename(artifact)
return morphlib.savefile.SaveFile(filename, mode='w')
def put_artifact_metadata(self, artifact, name):
filename = self._artifact_metadata_filename(artifact, name)
return morphlib.savefile.SaveFile(filename, mode='w')
def put_source_metadata(self, source, cachekey, name):
filename = self._source_metadata_filename(source, cachekey, name)
return morphlib.savefile.SaveFile(filename, mode='w')
def _has_file(self, filename):
if os.path.exists(filename):
os.utime(filename, None)
return True
return False
def has(self, artifact):
filename = self.artifact_filename(artifact)
return self._has_file(filename)
def has_artifact_metadata(self, artifact, name):
filename = self._artifact_metadata_filename(artifact, name)
return self._has_file(filename)
def has_source_metadata(self, source, cachekey, name):
filename = self._source_metadata_filename(source, cachekey, name)
return self._has_file(filename)
def get(self, artifact):
filename = self.artifact_filename(artifact)
os.utime(filename, None)
return open(filename)
def get_artifact_metadata(self, artifact, name):
filename = self._artifact_metadata_filename(artifact, name)
os.utime(filename, None)
return open(filename)
def get_source_metadata(self, source, cachekey, name):
filename = self._source_metadata_filename(source, cachekey, name)
os.utime(filename, None)
return open(filename)
def artifact_filename(self, artifact):
basename = artifact.basename()
return os.path.join(self.cachedir, basename)
def _artifact_metadata_filename(self, artifact, name):
basename = artifact.metadata_basename(name)
return os.path.join(self.cachedir, basename)
def _source_metadata_filename(self, source, cachekey, name):
basename = '%s.%s' % (cachekey, name)
return os.path.join(self.cachedir, basename)
def clear(self):
'''Clear everything from the artifact cache directory.
After calling this, the artifact cache will be entirely empty.
Caveat caller.
'''
for dirname, subdirs, basenames in os.walk(self.cachedir):
for basename in basenames:
os.remove(os.path.join(dirname, basename))
def list_contents(self):
'''Return the set of sources cached and related information.
returns a [(cache_key, set(artifacts), last_used)]
'''
CacheInfo = collections.namedtuple('CacheInfo', ('artifacts', 'mtime'))
contents = collections.defaultdict(lambda: CacheInfo(set(), 0))
for dirpath, dirnames, filenames in os.walk(self.cachedir):
for filename in filenames:
cachekey = filename[:63]
artifact = filename[65:]
artifacts, max_mtime = contents[cachekey]
artifacts.add(artifact)
this_mtime = os.stat(os.path.join(dirpath, filename)).st_mtime
contents[cachekey] = CacheInfo(artifacts,
max(max_mtime, this_mtime))
return ((cache_key, info.artifacts, info.mtime)
for cache_key, info in contents.iteritems())
def remove(self, cachekey):
'''Remove all artifacts associated with the given cachekey.'''
for dirpath, dirnames, filenames in os.walk(self.cachedir):
for filename in filenames:
if filename.startswith(cachekey):
os.remove(os.path.join(dirpath, filename))
|
