Consolidate test-suite rendering harnesses (#2236)

9 files changed, 59 insertions, 430 deletions

diff --git a/platform/node/README.md b/platform/node/README.md
index 870151fd25..72a08e8539 100644
--- a/platform/node/README.md
+++ b/platform/node/README.md
@@ -92,6 +92,8 @@ The `kind` is an enum and defined in [`mbgl.Resource`](https://github.com/mapbox
 
 It has no significance for anything but serves as a hint to your implemention as to what sort of resource to expect. E.g., your implementation could choose caching strategies based on the expected file type.
 
+THe `request` implementation should pass uncompressed data to `req.respond`. If you are downloading assets from a source that applies gzip transport encoding, the implementation must decompress the results before passing them on.
+
 A sample implementation that reads files from disk would look like the following:
 
 ```js
diff --git a/platform/node/test/compare.js b/platform/node/test/compare.js
deleted file mode 100644
index 3e5221de55..0000000000
--- a/platform/node/test/compare.js
+++ /dev/null
@@ -1,27 +0,0 @@
-var spawn = require('child_process').spawn;
-
-module.exports = function compare(actual, expected, diff, t, callback) {
-    var compare = spawn('compare', ['-metric', 'MAE', actual, expected, diff]);
-    var error = '';
-
-    compare.stderr.on('data', function(data) {
-        error += data.toString();
-    });
-
-    compare.on('error', function(err) {
-        t.error(err);
-    });
-
-    compare.on('exit', function(code) {
-        // The compare program returns 2 on error otherwise 0 if the images are similar or 1 if they are dissimilar.
-        if (code === 2) {
-            callback(error.trim(), Infinity);
-        } else {
-            var match = error.match(/^\d+(?:\.\d+)?\s+\(([^\)]+)\)\s*$/);
-            var difference = match ? parseFloat(match[1]) : Infinity;
-            callback(match ? '' : error, difference);
-        }
-    });
-
-    compare.stdin.end();
-};
diff --git a/platform/node/test/expected/gzip/success.png b/platform/node/test/expected/gzip/success.png
deleted file mode 100644
index de41e0fe2b..0000000000
--- a/platform/node/test/expected/gzip/success.png
+++ /dev/null
diff --git a/platform/node/test/expected/gzip/unhandled.png b/platform/node/test/expected/gzip/unhandled.png
deleted file mode 100644
index ddb90d8b8f..0000000000
--- a/platform/node/test/expected/gzip/unhandled.png
+++ /dev/null
diff --git a/platform/node/test/expected/map/image.png b/platform/node/test/expected/map/image.png
deleted file mode 100644
index de41e0fe2b..0000000000
--- a/platform/node/test/expected/map/image.png
+++ /dev/null
diff --git a/platform/node/test/js/consecutive.test.js b/platform/node/test/js/consecutive.test.js
deleted file mode 100644
index 928ca1d837..0000000000
--- a/platform/node/test/js/consecutive.test.js
+++ /dev/null
@@ -1,82 +0,0 @@
-'use strict';
-
-/* jshint node:true */
-
-var test = require('tape');
-var mbgl = require('../../../..');
-var fs = require('fs');
-var path = require('path');
-
-var suitePath = path.join(__dirname, '../../../../test/suite');
-
-function renderTest(style, info, dir, key) {
-    return function (t) {
-        var completed = 0;
-        var remaining = 10;
-        var start = +new Date;
-
-        var options = {};
-        options.request = function(req) {
-            fs.readFile(path.join(suitePath, decodeURIComponent(req.url)), function(err, data) {
-                req.respond(err, { data: data });
-                t.error(err);
-            });
-        };
-        options.cancel = function() {};
-        options.ratio = 1.0;
-
-        var map = new mbgl.Map(options);
-        map.load(style);
-
-        function render() {
-            map.render(info[key], function(err, image) {
-                t.error(err);
-
-                t.ok(true, 'render @ ' + ((+new Date) - start) + 'ms');
-                if (++completed === remaining) {
-                    map.release();
-                    t.end();
-                } else {
-                    render();
-                }
-            });
-        }
-
-        render();
-    };
-}
-
-function rewriteLocalSchema(uri) {
-    return uri.replace(/^local:\/\//, '');
-}
-
-test('Consecutive', function(t) {
-    var dir = 'line-join';
-    var k = 'round';
-
-    var style = require(path.join(suitePath, 'tests', dir, 'style.json')),
-        info  = require(path.join(suitePath, 'tests', dir, 'info.json'));
-
-    for (var k in style.sources) {
-        var source = style.sources[k];
-
-        if (source.tiles) {
-            source.tiles = source.tiles.map(rewriteLocalSchema);
-        }
-
-        if (source.url) {
-            source.url = rewriteLocalSchema(source.url);
-        }
-    }
-
-    if (style.sprite) style.sprite = rewriteLocalSchema(style.sprite);
-    if (style.glyphs) style.glyphs = rewriteLocalSchema(style.glyphs);
-
-    style = JSON.stringify(style);
-
-    for (k in info) {
-        t.test(dir + ' ' + k, renderTest(style, info, dir, k));
-    }
-
-    t.end();
-});
diff --git a/platform/node/test/js/gzip.test.js b/platform/node/test/js/gzip.test.js
deleted file mode 100644
index 4545a18cf5..0000000000
--- a/platform/node/test/js/gzip.test.js
+++ /dev/null
@@ -1,125 +0,0 @@
-'use strict';
-
-/* jshint node: true */
-
-var test = require('tape').test;
-var mbgl = require('../../../..');
-var fs = require('fs');
-var path = require('path');
-var mkdirp = require('mkdirp');
-var http = require('http');
-var request = require('request');
-var st = require('st');
-var style = require('../fixtures/style.json');
-var PNG = require('pngjs').PNG;
-var compare = require('../compare.js');
-
-var server = http.createServer(st({ path: path.join(__dirname, '..') }));
-server.listen(0);
-
-function filePath(name) {
-    return ['expected', 'actual', 'diff'].reduce(function(prev, key) {
-        var dir = path.join(__dirname, '..', key, 'gzip');
-        mkdirp.sync(dir);
-        prev[key] = path.join(dir, name);
-        return prev;
-    }, {});
-}
-
-function setup(options, callback) {
-    callback(new mbgl.Map(options));
-}
-
-function getOptions(gzip, t) {
-    return {
-        request: function(req) {
-            var parts = req.url.split('.');
-            var filetype = parts[parts.length - 1];
-
-            request({
-                url: 'http://localhost:' + server.address().port + path.join('/', req.url),
-                encoding: null,
-                gzip: filetype === 'pbf' ? gzip : true,
-                headers: {
-                    'Accept-Encoding': 'gzip'
-                }
-            }, function (err, res, body) {
-                t.error(err);
-                var response = {};
-                response.data = res.body;
-                req.respond(null, response);
-            });
-        },
-        ratio: 1.0
-    };
-}
-
-test('gzip', function(t) {
-    t.test('success', function(t) {
-        mbgl.on('message', function(msg) {
-            if (msg.severity == 'ERROR') t.error(msg);
-        });
-
-        setup(getOptions(true, t), function(map) {
-            map.load(style);
-            map.render({}, function(err, data) {
-                mbgl.removeAllListeners('message');
-                map.release();
-
-                t.error(err);
-
-                var filename = filePath('success.png');
-
-                var png = new PNG({
-                    width: data.width,
-                    height: data.height
-                });
-
-                png.data = data.pixels;
-
-                if (process.env.UPDATE) {
-                    png.pack()
-                        .pipe(fs.createWriteStream(filename.expected))
-                        .on('finish', t.end);
-                } else {
-                    png.pack()
-                        .pipe(fs.createWriteStream(filename.actual))
-                        .on('finish', function() {
-                            compare(filename.actual, filename.expected, filename.diff, t, function(err, diff) {
-                                t.error(err);
-                                t.ok(diff <= 0.01, 'actual matches expected');
-                                t.end();
-                            });
-                        });
-                }
-            });
-        });
-    });
-
-    t.test('unhandled', function(t) {
-        mbgl.once('message', function(msg) {
-            if (msg.severity == 'ERROR') {
-                t.ok(msg, 'emits error');
-                t.equal(msg.class, 'Style');
-                t.equal(msg.severity, 'ERROR');
-                t.ok(msg.text.match(/pbf unknown field type exception/), 'error text matches');
-            }
-        });
-
-        setup(getOptions(false, t), function(map) {
-            map.load(style);
-            map.render({}, function(err, data) {
-                map.release();
-
-                t.ok(err, 'returns error');
-                t.ok(err.message.match(/Failed to parse/), 'error text matches');
-
-                t.end();
-            });
-        });
-    });
-
-    t.test('teardown', function(t) {
-        server.close(t.end);
-    });
-});
diff --git a/platform/node/test/js/map.test.js b/platform/node/test/js/map.test.js
index 09870abaaa..c4e3416322 100644
--- a/platform/node/test/js/map.test.js
+++ b/platform/node/test/js/map.test.js
@@ -1,24 +1,10 @@
 'use strict';
 
-/* jshint node: true */
-
 var test = require('tape');
 var mbgl = require('../../../..');
 var fs = require('fs');
 var path = require('path');
-var mkdirp = require('mkdirp');
 var style = require('../fixtures/style.json');
-var PNG = require('pngjs').PNG;
-var compare = require('../compare.js');
-
-function filePath(name) {
-    return ['expected', 'actual', 'diff'].reduce(function(prev, key) {
-        var dir = path.join(__dirname, '..', key, 'map');
-        mkdirp.sync(dir);
-        prev[key] = path.join(dir, name);
-        return prev;
-    }, {});
-}
 
 test('Map', function(t) {
     t.test('must be constructed with new', function(t) {
@@ -250,31 +236,53 @@ test('Map', function(t) {
 
                 map.release();
 
-                var filename = filePath('image.png');
+                t.ok(data.pixels);
+                t.equal(data.width, 512);
+                t.equal(data.height, 512);
+                t.end();
+            });
+        });
+
+        t.test('can be called several times in serial', function(t) {
+            var completed = 0;
+            var remaining = 10;
+            var start = +new Date;
 
-                var png = new PNG({
-                    width: data.width,
-                    height: data.height
+            var map = new mbgl.Map(options);
+            map.load(style);
+
+            function render() {
+                map.render({}, function(err, data) {
+                    t.error(err);
+
+                    t.ok(true, 'render @ ' + ((+new Date) - start) + 'ms');
+                    if (++completed === remaining) {
+                        map.release();
+                        t.end();
+                    } else {
+                        render();
+                    }
                 });
+            }
+
+            render();
+        });
 
-                png.data = data.pixels;
-
-                if (process.env.UPDATE) {
-                    png.pack()
-                        .pipe(fs.createWriteStream(filename.expected))
-                        .on('finish', t.end);
-                } else {
-                    png.pack()
-                        .pipe(fs.createWriteStream(filename.actual))
-                        .on('finish', function() {
-                            compare(filename.actual, filename.expected, filename.diff, t, function(err, diff) {
-                                t.error(err);
-                                t.ok(diff <= 0.01, 'actual matches expected');
-                                t.end();
-                            });
-                        });
-                }
+        t.skip('throws if called in parallel', function(t) {
+            var completed = 0;
+            var remaining = 10;
+            var start = +new Date;
+
+            var map = new mbgl.Map(options);
+            map.load(style);
+
+            t.throws(function() {
+                map.render({}, function() {});
+                map.render({}, function() {});
             });
+
+            map.release();
+            t.end();
         });
     });
 });
diff --git a/platform/node/test/render.test.js b/platform/node/test/render.test.js
index 10a7afcb52..b9f13a28fe 100644
--- a/platform/node/test/render.test.js
+++ b/platform/node/test/render.test.js
@@ -1,175 +1,28 @@
 'use strict';
 
-/* jshint node:true */
-
-var test = require('tape');
 var mbgl = require('../../..');
-var fs = require('fs');
-var path = require('path');
-var mkdirp = require('mkdirp');
-var PNG = require('pngjs').PNG;
-var compare = require('./compare.js');
-var suitePath = path.join(__dirname, '../../../test/suite');
-
-function template(name) {
-    return fs.readFileSync(path.join(suitePath, 'templates', name + '.html.tmpl')).toString();
-}
-
-var results = '';
-var resultTemplate = template('result');
-
-function format(tmpl, kwargs) {
-    return tmpl.replace(/\{\{|\}\}|\{([^}]+)\}/g, function(match, key) {
-        if (match === '{{') return '{';
-        if (match === '}}') return '}';
-        return kwargs[key];
-    });
-}
-
-function renderTest(style, info, base, key) {
-    var dir = path.join(suitePath, 'tests', base, key);
-    mkdirp.sync(dir);
-
-    return function(t) {
-        var watchdog = setTimeout(function() {
-            t.fail('timed out after 20 seconds');
-        }, 20000);
-
-        t.once('end', function() {
-            clearTimeout(watchdog);
-
-            if (map) {
-                map.release();
-                map = null;
-            }
-        });
-
-        var options = {};
-        options.request = function(req) {
-            var url =  decodeURIComponent(req.url);
-            fs.readFile(path.join(suitePath, url), function(err, data) {
-                req.respond(err, { data: data });
-            });
-        };
-        options.ratio = info[key].pixelRatio || 1;
-
-        var map = new mbgl.Map(options);
-        map.load(style);
-
-        map.render(info[key], function(err, data) {
-            if (err) {
-                t.error(err);
-                return t.end();
-            }
-
-            var expected = path.join(dir, 'expected.png');
-            var actual = path.join(dir, 'actual.png');
-            var diff = path.join(dir, 'diff.png');
-
-            var png = new PNG({
-                width: data.width,
-                height: data.height
-            });
-
-            png.data = data.pixels;
-
-            if (process.env.UPDATE) {
-                png.pack()
-                    .pipe(fs.createWriteStream(expected))
-                    .on('finish', t.end);
-            } else {
-                png.pack()
-                    .pipe(fs.createWriteStream(actual))
-                    .on('finish', function() {
-                        compare(actual, expected, diff, t, function(err, diff) {
-                            t.error(err);
-
-                            var allowed = 0.001;
-
-                            if ('diff' in info[key]) {
-                                if (typeof info[key].diff === 'number') {
-                                    allowed = info[key].diff;
-                                } else if ('native' in info[key].diff) {
-                                    allowed = info[key].diff.native;
-                                }
-                            }
-
-                            results += format(resultTemplate, {
-                                name: base,
-                                key: key,
-                                color: diff <= allowed ? 'green' : 'red',
-                                error: err ? '<p>' + err + '</p>' : '',
-                                difference: diff,
-                                zoom: info.zoom || 0,
-                                center: info.center || [0, 0],
-                                bearing: info.bearing || 0,
-                                width: info.width || 512,
-                                height: info.height || 512
-                            });
-
-                            if (!info[key].ignored || !('native' in info[key].ignored)) {
-                                t.ok(diff <= allowed, 'expected ' + diff + ' to be less than ' + allowed);
-                            }
-
-                            t.end();
-                        });
-                });
-            }
-        });
-    };
-}
-
-function rewriteLocalSchema(url) {
-    var regex = /^local:\/\//;
-    if (url instanceof Array) {
-        return url.map(function(str) {
-            return str.replace(regex, '');
-        });
-    } else if (typeof url === "string") {
-        return url.replace(regex, '');
-    }
-}
+var suite = require('mapbox-gl-test-suite');
+var request = require('request');
 
 var tests;
 
-if (process.argv[1] === __filename) {
+if (process.argv[1] === __filename && process.argv.length > 2) {
     tests = process.argv.slice(2);
 }
 
-test('Render', function(t) {
-    fs.readdirSync(path.join(suitePath, 'tests')).forEach(function(dir) {
-        if (dir === 'index.html' || dir[0] === '.') return;
-        if (tests && tests.length && tests.indexOf(dir) < 0) return;
-
-        var style = require(path.join(suitePath, 'tests', dir, 'style.json')),
-            info  = require(path.join(suitePath, 'tests', dir, 'info.json'));
-
-        for (var k in style.sources) {
-            var source = style.sources[k];
-
-            if (source.tiles) {
-                source.tiles = source.tiles.map(rewriteLocalSchema);
-            }
-
-            if (source.url) {
-                source.url = rewriteLocalSchema(source.url);
-            }
-        }
-
-        if (style.sprite) style.sprite = rewriteLocalSchema(style.sprite);
-        if (style.glyphs) style.glyphs = rewriteLocalSchema(style.glyphs);
-
-        style = JSON.stringify(style);
-
-        for (k in info) {
-            (info[k].native === false ? t.skip : t.test)(dir + ' ' + k, renderTest(style, info, dir, k));
+suite.run('native', {tests: tests}, function (style, options, callback) {
+    var map = new mbgl.Map({
+        ratio: options.pixelRatio,
+        request: function (req) {
+            request(req.url, {encoding: null}, function (err, response, body) {
+                req.respond(err, {data: body});
+            });
         }
     });
 
-    t.test('results', function(t) {
-        var p = path.join(suitePath, 'tests', 'index.html');
-        fs.writeFileSync(p, format(template('results'), {results: results}));
-        console.warn('Results at: ' + p);
-        t.end();
+    map.load(style);
+    map.render(options, function (err, result) {
+        map.release();
+        callback(err, result && result.pixels);
     });
 });