diff options
author | Konstantin Käfer <mail@kkaefer.com> | 2014-07-09 15:06:55 -0700 |
---|---|---|
committer | Konstantin Käfer <mail@kkaefer.com> | 2014-07-09 15:07:48 -0700 |
commit | ce541a962efc5d5f15aff28efcf0064d85f9bbb0 (patch) | |
tree | 7e5dac770f521ff6bdc293b8833e981ae1b9636d | |
parent | 035ec18ae55e4d344b73b1be572d3f84526799bd (diff) | |
download | qtlocation-mapboxgl-ce541a962efc5d5f15aff28efcf0064d85f9bbb0.tar.gz |
fuzzy image comparison + create diff images
-rw-r--r-- | .gitignore | 1 | ||||
-rw-r--r-- | .travis.yml | 1 | ||||
-rwxr-xr-x | scripts/compare_images.js | 95 | ||||
-rw-r--r-- | scripts/create_result_table.js | 42 | ||||
-rwxr-xr-x | scripts/deploy_results.sh | 7 | ||||
-rw-r--r-- | test/headless.cpp | 1 |
6 files changed, 99 insertions, 48 deletions
diff --git a/.gitignore b/.gitignore index 8e63281caf..5cf9666f55 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ *.xcodeproj *.o *.actual.png +*.diff.png /node_modules /mapnik-packaging /macosx/build diff --git a/.travis.yml b/.travis.yml index dddd096c86..46f8aabe00 100644 --- a/.travis.yml +++ b/.travis.yml @@ -29,6 +29,7 @@ script: - make linux -j4 BUILDTYPE=${BUILDTYPE} - make test -j4 BUILDTYPE=${BUILDTYPE} - ./scripts/run_tests.sh +- ./scripts/compare_images.js after_script: - ./scripts/deploy_results.sh diff --git a/scripts/compare_images.js b/scripts/compare_images.js new file mode 100755 index 0000000000..f243753c39 --- /dev/null +++ b/scripts/compare_images.js @@ -0,0 +1,95 @@ +#!/usr/bin/env node + +var fs = require('fs'); +var path = require('path'); +var spawn = require('child_process').spawn; + +var base_dir = path.join(path.resolve('.'), 'test/fixtures/styles'); + +var files = fs.readdirSync(base_dir).filter(function(name) { + return name.match(/\.info\.json$/); +}); + +var html = + '<style>\n' + + ' body { font-family: Helvetica; }\n' + + ' h2 a { color:white; text-decoration:none; }\n' + + '</style>\n' + + '<table>\n' + + '<tr>\n' + + ' <th>Actual</th>\n' + + ' <th>Expected</th>\n' + + ' <th>Info</th>\n' + + '</tr>\n'; + +var exitCode = 0; + +processFiles(); + +function processFiles() { + if (!files.length) return done(); + + var name = files.shift(); + + var info = require(path.join(base_dir, name)); + var base = path.basename(name, '.info.json'); + + var actual = path.join(base_dir, base + '.actual.png'); + var expected = path.join(base_dir, base + '.expected.png'); + var diff = path.join(base_dir, base + '.diff.png'); + + var compare = spawn('compare', ['-metric', 'MAE', actual, expected, diff ]); + var error = ''; + compare.stderr.on('data', function(data) { + error += data.toString(); + }); + compare.on('exit', function(code, signal) { + // The compare program returns 2 on error otherwise 0 if the images are similar or 1 if they are dissimilar. + if (code == 2) { + writeResult(base, info, error.trim(), Infinity); + exitCode = 2; + } else { + var difference = parseFloat(error.match(/^\d+(?:\.\d+)?\s+\(([^\)]+)\)\s*$/)[1]); + writeResult(base, info, '', difference); + + } + return processFiles(); + }); + compare.stdin.end(); +} + +function writeResult(base, info, error, difference) { + var color = 'green'; + if (difference > 0.01 && exitCode < 1) { + exitCode = 1; + color = 'red'; + } + + html += + '<tr>\n' + + ' <td><img src="' + base + '.actual.png" onmouseover="this.src=\'' + base + '.expected.png\'" onmouseout="this.src=\'' + base + '.actual.png\'"></td>\n' + + ' <td><img src="' + base + '.expected.png" onmouseover="this.src=\'' + base + '.diff.png\'" onmouseout="this.src=\'' + base + '.expected.png\'"></td>\n' + + ' <td>\n' + + ' <h2 style="text-align:center; background:' + color + '"><a href="' + base + '.style.json">' + base + '</a></h2>\n' + + (error ? ' <p>' + error + '</p>\n' : '') + + ' <ul>\n' + + ' <li>diff: <strong>' + difference + '</strong></li>\n' + + ' <li>zoom: <strong>' + (info.zoom || 0) + '</strong></li>\n' + + ' <li>latitude: <strong>' + (info.latitude || 0) + '</strong></li>\n' + + ' <li>longitude: <strong>' + (info.longitude || 0) + '</strong></li>\n' + + ' <li>angle: <strong>' + (info.angle || 0) + '</strong></li>\n' + + ' <li>width: <strong>' + (info.width || 0) + '</strong></li>\n' + + ' <li>height: <strong>' + (info.height || 0) + '</strong></li>\n' + + ' </ul>\n' + + ' </td>\n' + + '</tr>\n' + ; +} + +function done() { + html += "</table>\n"; + + fs.writeFileSync(path.join(base_dir, 'index.html'), html); + + process.exit(exitCode); +}
\ No newline at end of file diff --git a/scripts/create_result_table.js b/scripts/create_result_table.js deleted file mode 100644 index ad604890f7..0000000000 --- a/scripts/create_result_table.js +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env node - -var fs = require('fs'); -var path = require('path'); - -var html = - '<style>body { font-family: Helvetica; }</style>\n' + - '<table>\n' + - '<tr>\n' + - ' <th>Actual</th>\n' + - ' <th>Expected</th>\n' + - ' <th>Info</th>\n' + - '</tr>\n'; - -var dir = fs.readdirSync('./test/fixtures/styles').filter(function(name) { - return name.match(/\.info\.json$/); -}).forEach(function(name) { - var info = require(path.join(path.resolve('.'), './test/fixtures/styles', name)); - var base = path.basename(name, '.info.json'); - - html += - '<tr>\n' + - ' <td><img src="' + base + '.actual.png"></td>\n' + - ' <td><img src="https://raw.githubusercontent.com/mapbox/mapbox-gl-native/' + process.env.TRAVIS_COMMIT + '/test/fixtures/styles/' + base + '.expected.png"></td>\n' + - ' <td>\n' + - ' <h2 style="text-align:center"><a href="https://raw.githubusercontent.com/mapbox/mapbox-gl-native/' + process.env.TRAVIS_COMMIT + '/test/fixtures/styles/' + base + '.style.json">' + base + '</a></h2>\n' + - ' <ul>\n' + - ' <li>zoom: <strong>' + (info.zoom || 0) + '</strong></li>\n' + - ' <li>latitude: <strong>' + (info.latitude || 0) + '</strong></li>\n' + - ' <li>longitude: <strong>' + (info.longitude || 0) + '</strong></li>\n' + - ' <li>angle: <strong>' + (info.angle || 0) + '</strong></li>\n' + - ' <li>width: <strong>' + (info.width || 0) + '</strong></li>\n' + - ' <li>height: <strong>' + (info.height || 0) + '</strong></li>\n' + - ' </ul>\n' + - ' </td>\n' + - '</tr>\n' - ; -}); - -html += "</table\n"; - -fs.writeFileSync('./test/fixtures/styles/index.html', html); diff --git a/scripts/deploy_results.sh b/scripts/deploy_results.sh index 98b564cc67..bf13def012 100755 --- a/scripts/deploy_results.sh +++ b/scripts/deploy_results.sh @@ -3,9 +3,6 @@ set -e set -o pipefail -node scripts/create_result_table.js +aws s3 cp test/fixtures/styles/ s3://mapbox-gl-testing/headless/$TRAVIS_JOB_NUMBER/ --acl public-read --recursive -aws s3 cp test/fixtures/styles/ s3://mapbox-gl-testing/headless/$TRAVIS_BUILD_NUMBER/$TRAVIS_JOB_NUMBER/ --acl public-read --recursive --exclude "*" --include "*.actual.png" --include "*.html" - - -echo http://mapbox-gl-testing.s3.amazonaws.com/headless/$TRAVIS_BUILD_NUMBER/$TRAVIS_JOB_NUMBER/index.html +echo http://mapbox-gl-testing.s3.amazonaws.com/headless/$TRAVIS_JOB_NUMBER/index.html diff --git a/test/headless.cpp b/test/headless.cpp index 0ed6e0a091..6f02191376 100644 --- a/test/headless.cpp +++ b/test/headless.cpp @@ -63,7 +63,6 @@ TEST_P(HeadlessTest, render) { const llmr::util::Image expected(expected_image_data, true); ASSERT_EQ(width, expected.getWidth()); ASSERT_EQ(height, expected.getHeight()); - ASSERT_EQ(0, std::memcmp(pixels.get(), expected.getData(), width * height * 4)); } INSTANTIATE_TEST_CASE_P(Headless, HeadlessTest, |