fuzzy image comparison + create diff images

6 files changed, 99 insertions, 48 deletions

diff --git a/.gitignore b/.gitignore
index 8e63281caf..5cf9666f55 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,7 @@
 *.xcodeproj
 *.o
 *.actual.png
+*.diff.png
 /node_modules
 /mapnik-packaging
 /macosx/build
diff --git a/.travis.yml b/.travis.yml
index dddd096c86..46f8aabe00 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -29,6 +29,7 @@ script:
 - make linux -j4 BUILDTYPE=${BUILDTYPE}
 - make test -j4 BUILDTYPE=${BUILDTYPE}
 - ./scripts/run_tests.sh
+- ./scripts/compare_images.js
 
 after_script:
 - ./scripts/deploy_results.sh
diff --git a/scripts/compare_images.js b/scripts/compare_images.js
new file mode 100755
index 0000000000..f243753c39
--- /dev/null
+++ b/scripts/compare_images.js
@@ -0,0 +1,95 @@
+#!/usr/bin/env node
+
+var fs = require('fs');
+var path = require('path');
+var spawn = require('child_process').spawn;
+
+var base_dir = path.join(path.resolve('.'), 'test/fixtures/styles');
+
+var files = fs.readdirSync(base_dir).filter(function(name) {
+    return name.match(/\.info\.json$/);
+});
+
+var html =
+    '<style>\n' +
+    '    body { font-family: Helvetica; }\n' +
+    '    h2 a { color:white; text-decoration:none; }\n' +
+    '</style>\n' +
+    '<table>\n' +
+    '<tr>\n' +
+    '    <th>Actual</th>\n' +
+    '    <th>Expected</th>\n' +
+    '    <th>Info</th>\n' +
+    '</tr>\n';
+
+var exitCode = 0;
+
+processFiles();
+
+function processFiles() {
+    if (!files.length) return done();
+
+    var name = files.shift();
+
+    var info = require(path.join(base_dir, name));
+    var base = path.basename(name, '.info.json');
+
+    var actual = path.join(base_dir, base + '.actual.png');
+    var expected = path.join(base_dir, base + '.expected.png');
+    var diff = path.join(base_dir, base + '.diff.png');
+
+    var compare = spawn('compare', ['-metric', 'MAE', actual, expected, diff ]);
+    var error = '';
+    compare.stderr.on('data', function(data) {
+        error += data.toString();
+    });
+    compare.on('exit', function(code, signal) {
+        // The compare program returns 2 on error otherwise 0 if the images are similar or 1 if they are dissimilar.
+        if (code == 2) {
+            writeResult(base, info, error.trim(), Infinity);
+            exitCode = 2;
+        } else {
+            var difference = parseFloat(error.match(/^\d+(?:\.\d+)?\s+\(([^\)]+)\)\s*$/)[1]);
+            writeResult(base, info, '', difference);
+
+        }
+        return processFiles();
+    });
+    compare.stdin.end();
+}
+
+function writeResult(base, info, error, difference) {
+    var color = 'green';
+    if (difference > 0.01 && exitCode < 1) {
+        exitCode = 1;
+        color = 'red';
+    }
+
+    html +=
+        '<tr>\n' +
+        '    <td><img src="' + base + '.actual.png" onmouseover="this.src=\'' + base + '.expected.png\'" onmouseout="this.src=\'' + base + '.actual.png\'"></td>\n' +
+        '    <td><img src="' + base + '.expected.png" onmouseover="this.src=\'' + base + '.diff.png\'" onmouseout="this.src=\'' + base + '.expected.png\'"></td>\n' +
+        '    <td>\n' +
+        '        <h2 style="text-align:center; background:' + color + '"><a href="' + base + '.style.json">' + base + '</a></h2>\n' +
+        (error ? '        <p>' + error + '</p>\n' : '') +
+        '        <ul>\n' +
+        '            <li>diff: <strong>' + difference + '</strong></li>\n' +
+        '            <li>zoom: <strong>' + (info.zoom || 0) + '</strong></li>\n' +
+        '            <li>latitude: <strong>' + (info.latitude || 0) + '</strong></li>\n' +
+        '            <li>longitude: <strong>' + (info.longitude || 0) + '</strong></li>\n' +
+        '            <li>angle: <strong>' + (info.angle || 0) + '</strong></li>\n' +
+        '            <li>width: <strong>' + (info.width || 0) + '</strong></li>\n' +
+        '            <li>height: <strong>' + (info.height || 0) + '</strong></li>\n' +
+        '        </ul>\n' +
+        '    </td>\n' +
+        '</tr>\n'
+    ;
+}
+
+function done() {
+    html += "</table>\n";
+
+    fs.writeFileSync(path.join(base_dir, 'index.html'), html);
+
+    process.exit(exitCode);
+}
\ No newline at end of file
diff --git a/scripts/create_result_table.js b/scripts/create_result_table.js
deleted file mode 100644
index ad604890f7..0000000000
--- a/scripts/create_result_table.js
+++ /dev/null
@@ -1,42 +0,0 @@
-#!/usr/bin/env node
-
-var fs = require('fs');
-var path = require('path');
-
-var html =
-    '<style>body { font-family: Helvetica; }</style>\n' +
-    '<table>\n' +
-    '<tr>\n' +
-    '    <th>Actual</th>\n' +
-    '    <th>Expected</th>\n' +
-    '    <th>Info</th>\n' +
-    '</tr>\n';
-
-var dir = fs.readdirSync('./test/fixtures/styles').filter(function(name) {
-    return name.match(/\.info\.json$/);
-}).forEach(function(name) {
-    var info = require(path.join(path.resolve('.'), './test/fixtures/styles', name));
-    var base = path.basename(name, '.info.json');
-
-    html +=
-        '<tr>\n' +
-        '    <td><img src="' + base + '.actual.png"></td>\n' +
-        '    <td><img src="https://raw.githubusercontent.com/mapbox/mapbox-gl-native/' + process.env.TRAVIS_COMMIT + '/test/fixtures/styles/' + base + '.expected.png"></td>\n' +
-        '    <td>\n' +
-        '        <h2 style="text-align:center"><a href="https://raw.githubusercontent.com/mapbox/mapbox-gl-native/' + process.env.TRAVIS_COMMIT + '/test/fixtures/styles/' + base + '.style.json">' + base + '</a></h2>\n' +
-        '        <ul>\n' +
-        '            <li>zoom: <strong>' + (info.zoom || 0) + '</strong></li>\n' +
-        '            <li>latitude: <strong>' + (info.latitude || 0) + '</strong></li>\n' +
-        '            <li>longitude: <strong>' + (info.longitude || 0) + '</strong></li>\n' +
-        '            <li>angle: <strong>' + (info.angle || 0) + '</strong></li>\n' +
-        '            <li>width: <strong>' + (info.width || 0) + '</strong></li>\n' +
-        '            <li>height: <strong>' + (info.height || 0) + '</strong></li>\n' +
-        '        </ul>\n' +
-        '    </td>\n' +
-        '</tr>\n'
-    ;
-});
-
-html += "</table\n";
-
-fs.writeFileSync('./test/fixtures/styles/index.html', html);
diff --git a/scripts/deploy_results.sh b/scripts/deploy_results.sh
index 98b564cc67..bf13def012 100755
--- a/scripts/deploy_results.sh
+++ b/scripts/deploy_results.sh
@@ -3,9 +3,6 @@
 set -e
 set -o pipefail
 
-node scripts/create_result_table.js
+aws s3 cp test/fixtures/styles/ s3://mapbox-gl-testing/headless/$TRAVIS_JOB_NUMBER/ --acl public-read --recursive
 
-aws s3 cp test/fixtures/styles/ s3://mapbox-gl-testing/headless/$TRAVIS_BUILD_NUMBER/$TRAVIS_JOB_NUMBER/ --acl public-read --recursive --exclude "*" --include "*.actual.png" --include "*.html"
-
-
-echo http://mapbox-gl-testing.s3.amazonaws.com/headless/$TRAVIS_BUILD_NUMBER/$TRAVIS_JOB_NUMBER/index.html
+echo http://mapbox-gl-testing.s3.amazonaws.com/headless/$TRAVIS_JOB_NUMBER/index.html
diff --git a/test/headless.cpp b/test/headless.cpp
index 0ed6e0a091..6f02191376 100644
--- a/test/headless.cpp
+++ b/test/headless.cpp
@@ -63,7 +63,6 @@ TEST_P(HeadlessTest, render) {
     const llmr::util::Image expected(expected_image_data, true);
     ASSERT_EQ(width, expected.getWidth());
     ASSERT_EQ(height, expected.getHeight());
-    ASSERT_EQ(0, std::memcmp(pixels.get(), expected.getData(), width * height * 4));
 }
 
 INSTANTIATE_TEST_CASE_P(Headless, HeadlessTest,