From b87cff289c1b65757ab0d652f09a5a12470c9cc2 Mon Sep 17 00:00:00 2001 From: Mike Morris Date: Tue, 10 Feb 2015 19:18:32 -0800 Subject: add mapbox-gl-test-suite render tests --- Makefile | 11 ++- package.json | 2 + scripts/travis_test.sh | 2 +- test/file_source.test.js | 21 ----- test/js/file_source.test.js | 21 +++++ test/js/map.test.js | 187 ++++++++++++++++++++++++++++++++++++++++++++ test/map.test.js | 187 -------------------------------------------- test/render.test.js | 161 ++++++++++++++++++++++++++++++++++++++ 8 files changed, 381 insertions(+), 211 deletions(-) delete mode 100644 test/file_source.test.js create mode 100644 test/js/file_source.test.js create mode 100644 test/js/map.test.js delete mode 100644 test/map.test.js create mode 100644 test/render.test.js diff --git a/Makefile b/Makefile index 4f83d28679..3df03e09de 100644 --- a/Makefile +++ b/Makefile @@ -49,9 +49,16 @@ build/Makefile: $(MBGL)/config/$(HOST).gypi $(MBGL)/config/%.gypi: $(MBGL) $(MBGL)/configure make -C $(MBGL) config/$*.gypi +.PHONY: test-suite +test-suite: build + @(`npm bin`/tape test/render.test.js || true) + +.PHONY: test-js +test-js: build + @`npm bin`/tape test/js/**/*.test.js + .PHONY: test -test: build - @`npm bin`/tape test/**/*.test.js +test: test-js test-suite .PHONY: clean clean: diff --git a/package.json b/package.json index 0f53fe51ec..be66045298 100644 --- a/package.json +++ b/package.json @@ -6,6 +6,8 @@ "nan": "^1.4.1" }, "devDependencies": { + "aws-sdk": "^2.1.9", + "mapbox-gl-test-suite": "git://github.com/mapbox/mapbox-gl-test-suite", "mkdirp": "^0.5.0", "tape": "^3.5.0" }, diff --git a/scripts/travis_test.sh b/scripts/travis_test.sh index 707293505a..70fff88aa4 100755 --- a/scripts/travis_test.sh +++ b/scripts/travis_test.sh @@ -6,5 +6,5 @@ set -o pipefail # Travis OS X has no GPU if [[ ${TRAVIS_OS_NAME} == "linux" ]]; then npm test - # ./scripts/deploy_results.sh + ./scripts/deploy_results.sh fi diff --git a/test/file_source.test.js b/test/file_source.test.js deleted file mode 100644 index 90a91de970..0000000000 --- a/test/file_source.test.js +++ /dev/null @@ -1,21 +0,0 @@ -/* jshint node: true, unused: false */ -'use strict'; - -var test = require('tape').test; - -var mbgl = require('..'); - -test('FileSource', function(t) { - t.test('constructor', function(t) { - t.test('must be called with new', function(t) { - t.throws(function() { - mbgl.FileSource(); - }, /Use the new operator to create new FileSource objects/); - t.end(); - }); - - t.end(); - }); - - t.end(); -}); diff --git a/test/js/file_source.test.js b/test/js/file_source.test.js new file mode 100644 index 0000000000..72f2180f20 --- /dev/null +++ b/test/js/file_source.test.js @@ -0,0 +1,21 @@ +'use strict'; + +/* jshint node: true */ + +var test = require('tape'); +var mbgl = require('../..'); + +test('FileSource', function(t) { + t.test('constructor', function(t) { + t.test('must be called with new', function(t) { + t.throws(function() { + mbgl.FileSource(); + }, /Use the new operator to create new FileSource objects/); + t.end(); + }); + + t.end(); + }); + + t.end(); +}); diff --git a/test/js/map.test.js b/test/js/map.test.js new file mode 100644 index 0000000000..97daa8d524 --- /dev/null +++ b/test/js/map.test.js @@ -0,0 +1,187 @@ +'use strict'; + +/* jshint node: true */ + +var test = require('tape'); +var mbgl = require('../..'); +var fs = require('fs'); +var path = require('path'); +var mkdirp = require('mkdirp'); + +mkdirp.sync('test/results'); + +var style = require('../fixtures/style.json'); + +function setup(fileSource, callback) { + callback(new mbgl.Map(fileSource)); +} + +test('Map', function(t) { + t.test('constructor', function(t) { + t.test('must be called with new', function(t) { + t.throws(function() { + mbgl.Map(); + }, /Use the new operator to create new Map objects/); + + t.end(); + }); + + t.test('should require a FileSource object as first parameter', function(t) { + t.throws(function() { + new mbgl.Map(); + }, /Requires a FileSource as first argument/); + + t.throws(function() { + new mbgl.Map('fileSource'); + }, /Requires a FileSource as first argument/); + + t.throws(function() { + new mbgl.Map({}); + }, /Requires a FileSource as first argument/); + + t.end(); + }); + + t.test('should require the FileSource object to have request and cancel methods', function(t) { + var fileSource = new mbgl.FileSource(); + + t.throws(function() { + new mbgl.Map(fileSource); + }, /FileSource must have a request member function/); + + fileSource.request = 'test'; + t.throws(function() { + new mbgl.Map(fileSource); + }, /FileSource must have a request member function/); + + fileSource.request = function() {}; + t.throws(function() { + new mbgl.Map(fileSource); + }, /FileSource must have a cancel member function/); + + fileSource.cancel = 'test'; + t.throws(function() { + new mbgl.Map(fileSource); + }, /FileSource must have a cancel member function/); + + fileSource.cancel = function() {}; + t.doesNotThrow(function() { + new mbgl.Map(fileSource); + }); + + t.end(); + }); + + t.end(); + }); + + t.test('load styles', function(t) { + var fileSource = new mbgl.FileSource(); + fileSource.request = function() {}; + fileSource.cancel = function() {}; + + t.test('requires a string or object as the first parameter', function(t) { + setup(fileSource, function(map) { + t.throws(function() { + map.load(); + }, /Requires a map style as first argument/); + + t.throws(function() { + map.load('invalid'); + }, /Expect either an object or array at root/); + t.end(); + }); + }); + + t.test('accepts an empty stylesheet string', function(t) { + setup(fileSource, function(map) { + t.doesNotThrow(function() { + map.load('{}'); + }); + t.end(); + }); + }); + + t.test('accepts a JSON stylesheet', function(t) { + setup(fileSource, function(map) { + t.doesNotThrow(function() { + map.load(style); + }); + t.end(); + }); + }); + + t.test('accepts a stringified stylesheet', function(t) { + setup(fileSource, function(map) { + t.doesNotThrow(function() { + map.load(JSON.stringify(style)); + }); + t.end(); + }); + }); + + t.end(); + }); + + t.test('render argument requirements', function(t) { + var fileSource = new mbgl.FileSource(); + fileSource.request = function(req) { + fs.readFile(path.join('test', req.url), function(err, data) { + req.respond(err, { data: data }); + t.error(err); + }); + }; + fileSource.cancel = function() {}; + + t.test('requires an object as the first parameter', function(t) { + setup(fileSource, function(map) { + t.throws(function() { + map.render(); + }, /First argument must be an options object/); + + t.throws(function() { + map.render('invalid'); + }, /First argument must be an options object/); + + t.end(); + }); + }); + + t.test('requires a callback as the second parameter', function(t) { + setup(fileSource, function(map) { + t.throws(function() { + map.render({}); + }, /Second argument must be a callback function/); + + t.throws(function() { + map.render({}, 'invalid'); + }, /Second argument must be a callback function/); + + t.end(); + }); + }); + + t.test('requires a style to be set', function(t) { + setup(fileSource, function(map) { + map.render({}, function(err) { + t.ok(err); + t.equal(err.message, 'Style is not set'); + t.end(); + }); + }); + }); + + t.test('returns an image', function(t) { + setup(fileSource, function(map) { + map.load(style); + map.render({}, function(err, data) { + t.error(err); + fs.writeFileSync('test/results/image.png', data); + t.end(); + }); + }); + }); + }); + + t.end(); +}); diff --git a/test/map.test.js b/test/map.test.js deleted file mode 100644 index c612636949..0000000000 --- a/test/map.test.js +++ /dev/null @@ -1,187 +0,0 @@ -/* jshint node: true, unused: false */ -'use strict'; - -var test = require('tape').test; - -var mbgl = require('..'); -var fs = require('fs'); -var path = require('path'); -var mkdirp = require('mkdirp'); - -mkdirp.sync('test/results'); - -var style = require('./fixtures/style.json'); - -function setup(fileSource, callback) { - callback(new mbgl.Map(fileSource)); -} - -test('Map', function(t) { - t.test('constructor', function(t) { - t.test('must be called with new', function(t) { - t.throws(function() { - mbgl.Map(); - }, /Use the new operator to create new Map objects/); - - t.end(); - }); - - t.test('should require a FileSource object as first parameter', function(t) { - t.throws(function() { - new mbgl.Map(); - }, /Requires a FileSource as first argument/); - - t.throws(function() { - new mbgl.Map('fileSource'); - }, /Requires a FileSource as first argument/); - - t.throws(function() { - new mbgl.Map({}); - }, /Requires a FileSource as first argument/); - - t.end(); - }); - - t.test('should require the FileSource object to have request and cancel methods', function(t) { - var fileSource = new mbgl.FileSource(); - - t.throws(function() { - new mbgl.Map(fileSource); - }, /FileSource must have a request member function/); - - fileSource.request = 'test'; - t.throws(function() { - new mbgl.Map(fileSource); - }, /FileSource must have a request member function/); - - fileSource.request = function() {}; - t.throws(function() { - new mbgl.Map(fileSource); - }, /FileSource must have a cancel member function/); - - fileSource.cancel = 'test'; - t.throws(function() { - new mbgl.Map(fileSource); - }, /FileSource must have a cancel member function/); - - fileSource.cancel = function() {}; - t.doesNotThrow(function() { - new mbgl.Map(fileSource); - }); - - t.end(); - }); - - t.end(); - }); - - t.test('load styles', function(t) { - var fileSource = new mbgl.FileSource(); - fileSource.request = function() {}; - fileSource.cancel = function() {}; - - t.test('requires a string or object as the first parameter', function(t) { - setup(fileSource, function(map) { - t.throws(function() { - map.load(); - }, /Requires a map style as first argument/); - - t.throws(function() { - map.load('invalid'); - }, /Expect either an object or array at root/); - t.end(); - }); - }); - - t.test('accepts an empty stylesheet string', function(t) { - setup(fileSource, function(map) { - t.doesNotThrow(function() { - map.load('{}'); - }); - t.end(); - }); - }); - - t.test('accepts a JSON stylesheet', function(t) { - setup(fileSource, function(map) { - t.doesNotThrow(function() { - map.load(style); - }); - t.end(); - }); - }); - - t.test('accepts a stringified stylesheet', function(t) { - setup(fileSource, function(map) { - t.doesNotThrow(function() { - map.load(JSON.stringify(style)); - }); - t.end(); - }); - }); - - t.end(); - }); - - t.test('render argument requirements', function(t) { - var fileSource = new mbgl.FileSource(); - fileSource.request = function(req) { - fs.readFile(path.join('test', req.url), function(err, data) { - req.respond(err, { data: data }); - t.error(err); - }); - }; - fileSource.cancel = function() {}; - - t.test('requires an object as the first parameter', function(t) { - setup(fileSource, function(map) { - t.throws(function() { - map.render(); - }, /First argument must be an options object/); - - t.throws(function() { - map.render('invalid'); - }, /First argument must be an options object/); - - t.end(); - }); - }); - - t.test('requires a callback as the second parameter', function(t) { - setup(fileSource, function(map) { - t.throws(function() { - map.render({}); - }, /Second argument must be a callback function/); - - t.throws(function() { - map.render({}, 'invalid'); - }, /Second argument must be a callback function/); - - t.end(); - }); - }); - - t.test('requires a style to be set', function(t) { - setup(fileSource, function(map) { - map.render({}, function(err) { - t.ok(err); - t.equal(err.message, 'Style is not set'); - t.end(); - }); - }); - }); - - t.test('returns an image', function(t) { - setup(fileSource, function(map) { - map.load(style); - map.render({}, function(err, data) { - t.error(err); - fs.writeFileSync('test/results/image.png', data); - t.end(); - }); - }); - }); - }); - - t.end(); -}); diff --git a/test/render.test.js b/test/render.test.js new file mode 100644 index 0000000000..1329b7ef53 --- /dev/null +++ b/test/render.test.js @@ -0,0 +1,161 @@ +'use strict'; + +/* jshint node:true */ + +var test = require('tape'); +var mbgl = require('..'); +var fs = require('fs'); +var path = require('path'); +var mkdirp = require('mkdirp'); +var spawn = require('child_process').spawn; + +var suitePath = path.dirname(require.resolve('mapbox-gl-test-suite/package.json')); + +function template(name) { + return fs.readFileSync(require.resolve('mapbox-gl-test-suite/templates/' + name + '.html.tmpl')).toString(); +} + +var results = ''; +var resultTemplate = template('result'); + +function format(tmpl, kwargs) { + return tmpl.replace(/\{\{|\}\}|\{([^}]+)\}/g, function(match, key) { + if (match === '{{') return '{'; + if (match === '}}') return '}'; + return kwargs[key]; + }); +} + +function renderTest(style, info, base, key) { + var dir = path.join(suitePath, 'tests', base, key); + return function(t) { + var watchdog = setTimeout(function() { + t.fail('timed out after 20 seconds'); + }, 20000); + + t.once('end', function() { + clearTimeout(watchdog); + }); + + var fileSource = new mbgl.FileSource(); + fileSource.request = function(req) { + fs.readFile(path.join(suitePath, decodeURIComponent(req.url)), function(err, data) { + req.respond(err, { data: data }); + t.error(err); + }); + }; + fileSource.cancel = function() {}; + + var map = new mbgl.Map(fileSource); + map.load(style); + + map.render(info[key], function(err, image) { + t.error(err); + mkdirp.sync(dir); + + var expected = path.join(dir, 'expected.png'); + var actual = path.join(dir, 'actual.png'); + var diff = path.join(dir, 'diff.png'); + + if (process.env.UPDATE) { + fs.writeFile(expected, image, function(err) { + t.error(err); + t.end(); + }); + } else { + fs.writeFile(actual, image, function(err) { + t.error(err); + + var compare = spawn('compare', ['-metric', 'MAE', actual, expected, diff]); + var error = ''; + + compare.stderr.on('data', function (data) { + error += data.toString(); + }); + + compare.on('exit', function (code) { + // The compare program returns 2 on error otherwise 0 if the images are similar or 1 if they are dissimilar. + if (code === 2) { + writeResult(error.trim(), Infinity); + } else { + var match = error.match(/^\d+(?:\.\d+)?\s+\(([^\)]+)\)\s*$/); + var difference = match ? parseFloat(match[1]) : Infinity; + writeResult(match ? '' : error, difference); + } + }); + + compare.stdin.end(); + + function writeResult(error, difference) { + var allowedDifference = ('diff' in info) ? info.diff : 0.001; + var color = difference <= allowedDifference ? 'green' : 'red'; + + results += format(resultTemplate, { + name: base, + key: key, + color: color, + error: error ? '

' + error + '

' : '', + difference: difference, + zoom: info.zoom || 0, + center: info.center || [0, 0], + bearing: info.bearing || 0, + width: info.width || 512, + height: info.height || 512 + }); + + t.ok(difference <= allowedDifference); + t.end(); + } + }); + } + }); + }; +} + +function rewriteLocalSchema(uri) { + return uri.replace(/^local:\/\//, ''); +} + +var tests; + +if (process.argv[1] === __filename) { + tests = process.argv.slice(2); +} + +test('Render', function(t) { + fs.readdirSync(path.join(suitePath, 'tests')).forEach(function(dir) { + if (dir === 'index.html' || dir[0] === '.') return; + if (tests && tests.length && tests.indexOf(dir) < 0) return; + + var style = require(path.join(suitePath, 'tests', dir, 'style.json')), + info = require(path.join(suitePath, 'tests', dir, 'info.json')); + + for (var k in style.sources) { + var source = style.sources[k]; + + if (source.tiles) { + source.tiles = source.tiles.map(rewriteLocalSchema); + } + + if (source.url) { + source.url = rewriteLocalSchema(source.url); + } + } + + if (style.sprite) style.sprite = rewriteLocalSchema(style.sprite); + if (style.glyphs) style.glyphs = rewriteLocalSchema(style.glyphs); + + style = JSON.stringify(style); + + for (k in info) { + (info[k].native === false ? t.skip : t.test)(dir + ' ' + k, renderTest(style, info, dir, k)); + } + }); + + t.test('results', function(t) { + var p = path.join(suitePath, 'tests', 'index.html'); + fs.writeFileSync(p, format(template('results'), {results: results})); + console.warn('Results at: ' + p); + t.end(); + }); +}); -- cgit v1.2.1