1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
|
# Copyright (C) 2015 Codethink Limited
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; version 2 of the License.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program. If not, see <http://www.gnu.org/licenses/>.
import yoyo
import yoyo.connections
import logging
import os
log = logging.getLogger(__name__)
class ArtifactDatabase(object):
'''A database to track artifact info.
This is so that we can have multiple builders submitting different builds
of the same artifact, then compare whether they are identical or not.
It uses the 'yoyo-migrations' Python package to set up the database schema,
so that hopefully the schema can be changed in future without enourmous
hassle.
'''
def __init__(self, database_file):
self.db, self.paramstyle = yoyo.connections.connect(
'sqlite:///' + database_file)
# SQLite is by default very slow to write due to fsync() calls. These
# settings make it faster, but at a slightly increased risk of data
# loss.
#
# See:
# - https://sqlite.org/pragma.html#pragma_synchronous
# - https://sqlite.org/pragma.html#pragma_journal_mode
cursor = self.db.cursor()
cursor.execute('PRAGMA journal_mode=WAL;')
cursor.execute('PRAGMA synchronous=NORMAL;')
self.apply_migrations(self.migrations_dir())
def migrations_dir(self):
return os.path.join(os.path.dirname(__file__), 'migrations')
def apply_migrations(self, migrations_path):
migrations = yoyo.read_migrations(
self.db, self.paramstyle, migrations_path,
migration_table='migrations')
to_apply = migrations.to_apply()
log.info('Found %i migrations, applying %i', len(migrations),
len(to_apply))
to_apply.apply()
self.db.commit()
def intern_artifact_file(self, cache_name):
'''Record that a Baserock artifact file is now in the cache directory.
The 'cache_name' variable is the SHA256 hash of the 'cache key', plus
a string.
The 'cache key' is a dictionary of values that should describe exactly
how the artifact was built. There is no standard for what a 'cache
key' should contain at present, different Baserock build tools can put
whatever info they want. This is not ideal, but hopefully use of SHA256
makes collisions unlikely in any case.
Artifacts can be split into multiple files, because one single build
operation can produce a bunch of different things. For example, running
'./configure; make; make install' in glibc.git produces not only the
GNU C library binaries, but also documentation, helper utilities and
other miscellanous things. Thus, the string that follows the SHA256
hash identifies which of these things we are talking about. There is
no standard for these strings, at present. It is up to the build tools
to make sense of them.
'''
cursor = self.db.cursor()
find_artifact_sql = 'SELECT internal_id FROM artifact_files WHERE ' \
'cache_name=?'
row = cursor.execute(find_artifact_sql, [cache_name]).fetchone()
if row is None:
log.debug('Recording new artifact file %s', cache_name)
cursor.execute(
'INSERT INTO artifact_files(cache_name) VALUES(?)',
[cache_name])
self.db.commit()
internal_id = cursor.lastrowid
else:
# If the artifact file was already known, no problem.
internal_id = row[0]
return internal_id
def record_build(self, cache_name, builder_name, build_datetime,
hash_sha1):
'''Record a build of a Baserock artifact.
The artifact file is identified by the 'cache name', which is a hash of
some information that describes how it is built.
It is up to the build tool and the build instructions to ensure that a
given 'cache key' produces a set of identical artifact files each time.
We record every build that we receive in order to detect and highlight
cases where the build output for a given 'cache key' is not
deterministic.
'''
self.intern_artifact_file(cache_name)
cursor = self.db.cursor()
log.debug('Recording new build of %s, %s, %s', cache_name,
builder_name, build_datetime)
cursor.execute(
'INSERT INTO builds(cache_name, builder_name, build_datetime, '
' hash_sha1) VALUES(?, ?, ?, ?)',
[cache_name, builder_name, build_datetime, hash_sha1])
self.db.commit()
def iter_builds_for_artifact_file(self, cache_name):
'''Yield info on each recorded build of a given artifact.'''
cursor = self.db.cursor()
cursor.execute(
'SELECT builder_name, build_datetime, hash_sha1 FROM builds WHERE '
' cache_name=?', [cache_name])
for item in cursor:
builder_name, build_datetime, hash_sha1 = item
yield {
'builder_name': builder_name,
'build_datetime': build_datetime,
'hash_sha1': hash_sha1
}
def count_artifacts(self):
'''Return the number of known artifacts.
This may include artifacts where the actual files aren't stored in the
cache, as we allow submitting just the hash in order to check
reproducibility.
'''
n_artifacts_sql = '''
SELECT COUNT(cache_name) FROM artifact_files
'''
cursor = self.db.cursor()
logging.debug('Running: %s', n_artifacts_sql)
cursor.execute(n_artifacts_sql)
return cursor.fetchone()[0]
def view_artifact_statistics(self, start=0, page_size=50):
'''Return information on number of builds for each artifact.
Returns a dict.
'''
n_builds_per_artifact_sql = '''
SELECT builds.cache_name,
COUNT(builds.hash_sha1) as n_builds,
COUNT(DISTINCT builds.hash_sha1)-1 as n_different_builds
FROM artifact_files INNER JOIN builds
ON builds.cache_name = artifact_files.cache_name
GROUP BY builds.cache_name
'''
# Return the artifacts with the highest number of mismatched builds
# first, they are the most important. Sort order should be configurable,
# really.
sql = n_builds_per_artifact_sql + ' ORDER BY n_different_builds DESC'
sql += ' LIMIT %i OFFSET %i' % (page_size, start)
cursor = self.db.cursor()
logging.debug('Running: %s', sql)
cursor.execute(sql)
result = []
for row in cursor:
result.append({
'cache_name': row[0],
'n_builds': row[1],
'n_different_builds': row[2],
})
return result
|
