Track source_repo and source_ref for a given artifact

This is done in a hacky way because right now YBD artifact metadata is
pretty lacking. The "proper" approach is to require all of the info that
went into the cache key, then recalculate the cache key on the server
and check that it matches against the hash in the cache name. But first
we need to fix YBD to actually store all that info in the artifact.

3 files changed, 77 insertions, 19 deletions

diff --git a/morph-cache-server b/morph-cache-server
index 4cbc20d..5fe6b0f 100755
--- a/morph-cache-server
+++ b/morph-cache-server
@@ -388,6 +388,8 @@ class MorphCacheServer(cliapp.Application):
                 - builder_name: URL identifying the build worker
                 - build_datetime: time artifact build was started
                 - hash_sha1: SHA1 of built artifact
+                - source_repo: (optional): repo this is built from
+                - source_ref: (optional): ref this is built from
 
             '''
             cache_name = self._unescape_parameter(request.query.cache_name)
@@ -396,7 +398,13 @@ class MorphCacheServer(cliapp.Application):
                 request.query.build_datetime)
             hash_sha1 = self._unescape_parameter(request.query.hash_sha1)
 
-            db.record_build(cache_name, builder_name, build_datetime, hash_sha1)
+            source_repo = self._unescape_parameter(
+                request.query.get('source_repo'))
+            source_ref = self._unescape_parameter(
+                request.query.get('source_ref'))
+
+            db.record_build(cache_name, builder_name, build_datetime,
+                            hash_sha1, source_repo, source_ref)
 
         @app.get('/builds')
         def get_builds():
diff --git a/morphcacheserver/artifact_database.py b/morphcacheserver/artifact_database.py
index b9550a1..21d28ef 100644
--- a/morphcacheserver/artifact_database.py
+++ b/morphcacheserver/artifact_database.py
@@ -64,7 +64,24 @@ class ArtifactDatabase(object):
         to_apply.apply()
         self.db.commit()
 
-    def intern_artifact_file(self, cache_name):
+    def query(self, sql, args=[]):
+        '''Run a SELECT query.'''
+
+        cursor = self.db.cursor()
+        logging.debug('Running: %s with args %s', sql, args)
+        cursor.execute(sql, args)
+        return cursor
+
+    def update(self, sql, args=[]):
+        '''Run an INSERT or UPDATE query.'''
+
+        cursor = self.db.cursor()
+        log.debug('Running %s with args %s', sql, args)
+        cursor.execute(sql, args)
+        self.db.commit()
+        return cursor.lastrowid
+
+    def intern_artifact_file(self, cache_name, source_repo, source_ref):
         '''Record that a Baserock artifact file is now in the cache directory.
 
         The 'cache_name' variable is the SHA256 hash of the 'cache key', plus
@@ -87,23 +104,33 @@ class ArtifactDatabase(object):
 
         '''
         cursor = self.db.cursor()
-        find_artifact_sql = 'SELECT internal_id FROM artifact_files WHERE ' \
-                            'cache_name=?'
+        find_artifact_sql = '''
+        SELECT internal_id, source_repo, source_ref
+        FROM artifact_files
+        WHERE cache_name=?
+        '''
         row = cursor.execute(find_artifact_sql, [cache_name]).fetchone()
+
         if row is None:
-            log.debug('Recording new artifact file %s', cache_name)
-            cursor.execute(
-                'INSERT INTO artifact_files(cache_name) VALUES(?)',
-                [cache_name])
-            self.db.commit()
-            internal_id = cursor.lastrowid
+            internal_id = self.update(
+                'INSERT INTO artifact_files(cache_name, source_repo, '
+                '    source_ref) VALUES(?,?,?)',
+                [cache_name, source_repo, source_ref])
         else:
-            # If the artifact file was already known, no problem.
             internal_id = row[0]
+            if row[1] is None and row[2] is None:
+                # Set repo and ref if not set. Currently we take it on trust
+                # that these are correct. Actually, we should take the entire
+                # contents of the cache key, and verify that it matches with
+                # the hash in the cache_name parameter.
+                self.update(
+                    'UPDATE artifact_files SET source_repo=?, source_ref=? '
+                    'WHERE cache_name=?',
+                    [source_repo, source_ref, cache_name])
         return internal_id
 
     def record_build(self, cache_name, builder_name, build_datetime,
-                     hash_sha1):
+                     hash_sha1, source_repo, source_ref):
         '''Record a build of a Baserock artifact.
 
         The artifact file is identified by the 'cache name', which is a hash of
@@ -116,7 +143,7 @@ class ArtifactDatabase(object):
         deterministic.
 
         '''
-        self.intern_artifact_file(cache_name)
+        self.intern_artifact_file(cache_name, source_repo, source_ref)
 
         cursor = self.db.cursor()
         log.debug('Recording new build of %s, %s, %s', cache_name,
@@ -178,12 +205,9 @@ class ArtifactDatabase(object):
         # first, they are the most important. Sort order should be configurable,
         # really.
         sql = n_builds_per_artifact_sql + ' ORDER BY n_different_builds DESC'
-
         sql += ' LIMIT %i OFFSET %i' % (page_size, start)
 
-        cursor = self.db.cursor()
-        logging.debug('Running: %s', sql)
-        cursor.execute(sql)
+        cursor = self.query(sql)
 
         result = []
         for row in cursor:
@@ -193,3 +217,17 @@ class ArtifactDatabase(object):
                 'n_different_builds': row[2],
             })
         return result
+
+    def view_artifact_info(self, cache_name):
+        sql = '''
+        SELECT source_repo, source_ref FROM artifact_files WHERE cache_name=?
+        '''
+        cursor = self.query(sql, [cache_name])
+
+        row = cursor.fetchone()
+        if row is None:
+            raise KeyError('No artifact with name %s' % cache_name)
+        else:
+            source_repo = row[0]
+            source_ref = row[1]
+            return source_repo, source_ref
diff --git a/morphcacheserver/frontend.py b/morphcacheserver/frontend.py
index 6eb0917..b463c33 100644
--- a/morphcacheserver/frontend.py
+++ b/morphcacheserver/frontend.py
@@ -31,7 +31,7 @@ def make_table(data, heading_ids, heading_titles, row_class_cb=None):
     %for row in rows:
     <tr class="{{ row_class_cb(row) }}" >
     %for col in heading_ids:
-        <td>{{row[col]}}</td>
+        <td>{{!row[col]}}</td>
     %end
     </tr>
     %end
@@ -66,9 +66,14 @@ def web_frontend(db):
 
         artifacts = db.view_artifact_statistics(
             start=(page-1)*page_size, page_size=page_size)
+
+        for item in artifacts:
+            item['cache_name_with_url'] = '<a href="/artifacts/%s">%s</a>' % (
+                item['cache_name'], item['cache_name'])
+
         content += make_table(
             artifacts,
-            ['cache_name', 'n_builds', 'n_different_builds'],
+            ['cache_name_with_url', 'n_builds', 'n_different_builds'],
             ['Artifact', 'Total builds', 'Mismatching builds'],
             row_class_cb=row_style_class_cb)
 
@@ -87,6 +92,13 @@ def web_frontend(db):
 
         return template('morphcacheserver/templates/base', base=content)
 
+    @app.get('/artifacts/<cache_name>')
+    def artifact_info(cache_name):
+        source_repo, source_ref = db.view_artifact_info(cache_name)
+        return template('morphcacheserver/templates/artifact_info',
+            cache_name=cache_name, source_repo=source_repo,
+            source_ref=source_ref)
+
     @app.get('/')
     def frontpage():
         '''A nice frontpage.'''