Merge pull request #79 from mapbox/new-arch-test

add tests to new-arch branch

9 files changed, 459 insertions, 190 deletions

diff --git a/Makefile b/Makefile
index b1756f017a..3df03e09de 100644
--- a/Makefile
+++ b/Makefile
@@ -49,9 +49,16 @@ build/Makefile: $(MBGL)/config/$(HOST).gypi
 $(MBGL)/config/%.gypi: $(MBGL) $(MBGL)/configure
 	make -C $(MBGL) config/$*.gypi
 
+.PHONY: test-suite
+test-suite: build
+	@(`npm bin`/tape test/render.test.js || true)
+
+.PHONY: test-js
+test-js: build
+	@`npm bin`/tape test/js/**/*.test.js
+
 .PHONY: test
-test: build
-	@`npm bin`/mocha -R spec
+test: test-js test-suite
 
 .PHONY: clean
 clean:
diff --git a/package.json b/package.json
index 7781e1b82c..0618acb105 100644
--- a/package.json
+++ b/package.json
@@ -17,12 +17,14 @@
     }
   ],
   "dependencies": {
+    "nan": "^1.4.1"
+  },
+  "devDependencies": {
+    "aws-sdk": "^2.1.9",
+    "mapbox-gl-test-suite": "git://github.com/mapbox/mapbox-gl-test-suite",
     "mkdirp": "^0.5.0",
-    "mocha": "^2.1.0",
-    "nan": "^1.4.1",
-    "request": "^2.51.0"
+    "tape": "^3.5.0"
   },
-  "devDependencies": {},
   "scripts": {
     "install": "make",
     "test": "make test"
diff --git a/scripts/travis_test.sh b/scripts/travis_test.sh
index 707293505a..70fff88aa4 100755
--- a/scripts/travis_test.sh
+++ b/scripts/travis_test.sh
@@ -6,5 +6,5 @@ set -o pipefail
 # Travis OS X has no GPU
 if [[ ${TRAVIS_OS_NAME} == "linux" ]]; then
     npm test
-    # ./scripts/deploy_results.sh
+    ./scripts/deploy_results.sh
 fi
diff --git a/test/file_source.test.js b/test/file_source.test.js
deleted file mode 100644
index 1d7d86fe9a..0000000000
--- a/test/file_source.test.js
+++ /dev/null
@@ -1,19 +0,0 @@
-/* jshint node: true, unused: false */
-/* global describe, it */
-'use strict';
-var assert = require('assert');
-
-var mbgl = require('..');
-
-describe('FileSource', function() {
-
-    describe('constructor', function() {
-        it('must be called with new', function() {
-            assert.throws(function() {
-                mbgl.FileSource();
-            }, /Use the new operator to create new FileSource objects/);
-        });
-
-    });
-
-});
diff --git a/test/js/concurrency.test.js b/test/js/concurrency.test.js
new file mode 100644
index 0000000000..6a2ee32ca6
--- /dev/null
+++ b/test/js/concurrency.test.js
@@ -0,0 +1,74 @@
+'use strict';
+
+/* jshint node:true */
+
+var test = require('tape');
+var mbgl = require('../..');
+var fs = require('fs');
+var path = require('path');
+
+var suitePath = path.dirname(require.resolve('mapbox-gl-test-suite/package.json'));
+
+
+function renderTest(style, info, dir, key) {
+    return function (t) {
+        var completed = 0;
+        var remaining = 10;
+        var start = +new Date;
+
+        var fileSource = new mbgl.FileSource();
+        fileSource.request = function(req) {
+            fs.readFile(path.join(suitePath, decodeURIComponent(req.url)), function(err, data) {
+                req.respond(err, { data: data });
+                t.error(err);
+            });
+        };
+        fileSource.cancel = function() {};
+
+        var map = new mbgl.Map(fileSource);
+        map.load(style);
+
+        for (var i = 0; i < remaining; i++) {
+            map.render(info[key], function(err, image) {
+                t.error(err);
+                t.ok(true, 'render @ ' + ((+new Date) - start) + 'ms');
+                if (++completed === remaining) t.end();
+            });
+        }
+    };
+}
+
+function rewriteLocalSchema(uri) {
+    return uri.replace(/^local:\/\//, '');
+}
+
+test('Concurrency', function(t) {
+    var dir = 'line-join';
+    var k = 'round';
+
+    var style = require(path.join(suitePath, 'tests', dir, 'style.json')),
+        info  = require(path.join(suitePath, 'tests', dir, 'info.json'));
+
+    for (var k in style.sources) {
+        var source = style.sources[k];
+
+        if (source.tiles) {
+            source.tiles = source.tiles.map(rewriteLocalSchema);
+        }
+
+        if (source.url) {
+            source.url = rewriteLocalSchema(source.url);
+        }
+    }
+
+    if (style.sprite) style.sprite = rewriteLocalSchema(style.sprite);
+    if (style.glyphs) style.glyphs = rewriteLocalSchema(style.glyphs);
+
+    style = JSON.stringify(style);
+
+    for (k in info) {
+        t.test(dir + ' ' + k, renderTest(style, info, dir, k));
+    }
+
+    t.end();
+});
diff --git a/test/js/file_source.test.js b/test/js/file_source.test.js
new file mode 100644
index 0000000000..72f2180f20
--- /dev/null
+++ b/test/js/file_source.test.js
@@ -0,0 +1,21 @@
+'use strict';
+
+/* jshint node: true */
+
+var test = require('tape');
+var mbgl = require('../..');
+
+test('FileSource', function(t) {
+    t.test('constructor', function(t) {
+        t.test('must be called with new', function(t) {
+            t.throws(function() {
+                mbgl.FileSource();
+            }, /Use the new operator to create new FileSource objects/);
+            t.end();
+        });
+
+        t.end();
+    });
+
+    t.end();
+});
diff --git a/test/js/map.test.js b/test/js/map.test.js
new file mode 100644
index 0000000000..97daa8d524
--- /dev/null
+++ b/test/js/map.test.js
@@ -0,0 +1,187 @@
+'use strict';
+
+/* jshint node: true */
+
+var test = require('tape');
+var mbgl = require('../..');
+var fs = require('fs');
+var path = require('path');
+var mkdirp = require('mkdirp');
+
+mkdirp.sync('test/results');
+
+var style = require('../fixtures/style.json');
+
+function setup(fileSource, callback) {
+    callback(new mbgl.Map(fileSource));
+}
+
+test('Map', function(t) {
+    t.test('constructor', function(t) {
+        t.test('must be called with new', function(t) {
+            t.throws(function() {
+                mbgl.Map();
+            }, /Use the new operator to create new Map objects/);
+
+            t.end();
+        });
+
+        t.test('should require a FileSource object as first parameter', function(t) {
+            t.throws(function() {
+                new mbgl.Map();
+            }, /Requires a FileSource as first argument/);
+
+            t.throws(function() {
+                new mbgl.Map('fileSource');
+            }, /Requires a FileSource as first argument/);
+
+            t.throws(function() {
+                new mbgl.Map({});
+            }, /Requires a FileSource as first argument/);
+
+            t.end();
+        });
+
+        t.test('should require the FileSource object to have request and cancel methods', function(t) {
+            var fileSource = new mbgl.FileSource();
+
+            t.throws(function() {
+                new mbgl.Map(fileSource);
+            }, /FileSource must have a request member function/);
+
+            fileSource.request = 'test';
+            t.throws(function() {
+                new mbgl.Map(fileSource);
+            }, /FileSource must have a request member function/);
+
+            fileSource.request = function() {};
+            t.throws(function() {
+                new mbgl.Map(fileSource);
+            }, /FileSource must have a cancel member function/);
+
+            fileSource.cancel = 'test';
+            t.throws(function() {
+                new mbgl.Map(fileSource);
+            }, /FileSource must have a cancel member function/);
+
+            fileSource.cancel = function() {};
+            t.doesNotThrow(function() {
+                new mbgl.Map(fileSource);
+            });
+
+            t.end();
+        });
+
+        t.end();
+    });
+
+    t.test('load styles', function(t) {
+        var fileSource = new mbgl.FileSource();
+        fileSource.request = function() {};
+        fileSource.cancel = function() {};
+
+        t.test('requires a string or object as the first parameter', function(t) {
+            setup(fileSource, function(map) {
+                t.throws(function() {
+                    map.load();
+                }, /Requires a map style as first argument/);
+
+                t.throws(function() {
+                    map.load('invalid');
+                }, /Expect either an object or array at root/);
+                t.end();
+            });
+        });
+
+        t.test('accepts an empty stylesheet string', function(t) {
+            setup(fileSource, function(map) {
+                t.doesNotThrow(function() {
+                    map.load('{}');
+                });
+                t.end();
+            });
+        });
+
+        t.test('accepts a JSON stylesheet', function(t) {
+            setup(fileSource, function(map) {
+                t.doesNotThrow(function() {
+                    map.load(style);
+                });
+                t.end();
+            });
+        });
+
+        t.test('accepts a stringified stylesheet', function(t) {
+            setup(fileSource, function(map) {
+                t.doesNotThrow(function() {
+                    map.load(JSON.stringify(style));
+                });
+                t.end();
+            });
+        });
+
+        t.end();
+    });
+
+    t.test('render argument requirements', function(t) {
+        var fileSource = new mbgl.FileSource();
+        fileSource.request = function(req) {
+            fs.readFile(path.join('test', req.url), function(err, data) {
+                req.respond(err, { data: data });
+                t.error(err);
+            });
+        };
+        fileSource.cancel = function() {};
+
+        t.test('requires an object as the first parameter', function(t) {
+            setup(fileSource, function(map) {
+                t.throws(function() {
+                    map.render();
+                }, /First argument must be an options object/);
+
+                t.throws(function() {
+                    map.render('invalid');
+                }, /First argument must be an options object/);
+
+                t.end();
+            });
+        });
+
+        t.test('requires a callback as the second parameter', function(t) {
+            setup(fileSource, function(map) {
+                t.throws(function() {
+                    map.render({});
+                }, /Second argument must be a callback function/);
+
+                t.throws(function() {
+                    map.render({}, 'invalid');
+                }, /Second argument must be a callback function/);
+
+                t.end();
+            });
+        });
+
+        t.test('requires a style to be set', function(t) {
+            setup(fileSource, function(map) {
+                map.render({}, function(err) {
+                    t.ok(err);
+                    t.equal(err.message, 'Style is not set');
+                    t.end();
+                });
+            });
+        });
+
+        t.test('returns an image', function(t) {
+            setup(fileSource, function(map) {
+                map.load(style);
+                map.render({}, function(err, data) {
+                    t.error(err);
+                    fs.writeFileSync('test/results/image.png', data);
+                    t.end();
+                });
+            });
+        });
+    });
+
+    t.end();
+});
diff --git a/test/map.test.js b/test/map.test.js
deleted file mode 100644
index 1d542c7b31..0000000000
--- a/test/map.test.js
+++ /dev/null
@@ -1,164 +0,0 @@
-/* jshint node: true, unused: false */
-/* global describe, it, beforeEach, afterEach */
-'use strict';
-var assert = require('assert');
-
-var mbgl = require('..');
-var fs = require('fs');
-var path = require('path');
-var mkdirp = require('mkdirp');
-
-mkdirp.sync('test/results');
-
-var style = require('./fixtures/style.json');
-
-describe('Map', function() {
-
-    describe('constructor', function() {
-        it('must be called with new', function() {
-            assert.throws(function() {
-                mbgl.Map();
-            }, /Use the new operator to create new Map objects/);
-        });
-
-        it('should require a FileSource object as first parameter', function() {
-            assert.throws(function() {
-                new mbgl.Map();
-            }, /Requires a FileSource as first argument/);
-
-            assert.throws(function() {
-                new mbgl.Map('fileSource');
-            }, /Requires a FileSource as first argument/);
-
-            assert.throws(function() {
-                new mbgl.Map({});
-            }, /Requires a FileSource as first argument/);
-        });
-
-        it('should require the FileSource object to have request and cancel methods', function() {
-            var fileSource = new mbgl.FileSource();
-
-            assert.throws(function() {
-                new mbgl.Map(fileSource);
-            }, /FileSource must have a request member function/);
-
-            fileSource.request = 'test';
-            assert.throws(function() {
-                new mbgl.Map(fileSource);
-            }, /FileSource must have a request member function/);
-
-            fileSource.request = function() {};
-            assert.throws(function() {
-                new mbgl.Map(fileSource);
-            }, /FileSource must have a cancel member function/);
-
-            fileSource.cancel = 'test';
-            assert.throws(function() {
-                new mbgl.Map(fileSource);
-            }, /FileSource must have a cancel member function/);
-
-            fileSource.cancel = function() {};
-            assert.doesNotThrow(function() {
-                new mbgl.Map(fileSource);
-            });
-        });
-
-    });
-
-    describe('load styles', function() {
-        var map;
-
-        var fileSource = new mbgl.FileSource();
-        fileSource.request = function() {};
-        fileSource.cancel = function() {};
-
-        beforeEach(function() {
-            map = new mbgl.Map(fileSource);
-        });
-
-        afterEach(function() {
-            map = null;
-        });
-
-        it('requires a string or object as the first parameter', function() {
-            assert.throws(function() {
-                map.load();
-            }, /Requires a map style as first argument/);
-
-            assert.throws(function() {
-                map.load('invalid');
-            }, /Expect either an object or array at root/);
-        });
-
-        it('accepts an empty stylesheet string', function() {
-            map.load('{}');
-        });
-
-        it('accepts a JSON stylesheet', function() {
-            map.load(style);
-        });
-
-        it('accepts a stringified stylesheet', function() {
-            map.load(JSON.stringify(style));
-        });
-    });
-
-    describe('render argument requirements', function() {
-        var map;
-
-        var fileSource = new mbgl.FileSource();
-        fileSource.request = function(req) {
-            fs.readFile(path.join('test', req.url), function(err, data) {
-                req.respond(err, { data: data });
-                assert.ifError(err);
-            });
-        };
-        fileSource.cancel = function() {};
-
-        beforeEach(function() {
-            map = new mbgl.Map(fileSource);
-        });
-
-        afterEach(function() {
-            map = null;
-        });
-
-        it('requires an object as the first parameter', function() {
-            assert.throws(function() {
-                map.render();
-            }, /First argument must be an options object/);
-
-            assert.throws(function() {
-                map.render('invalid');
-            }, /First argument must be an options object/);
-        });
-
-        it('requires a callback as the second parameter', function() {
-            assert.throws(function() {
-                map.render({});
-            }, /Second argument must be a callback function/);
-
-            assert.throws(function() {
-                map.render({}, 'invalid');
-            }, /Second argument must be a callback function/);
-        });
-
-        it('requires a style to be set', function(done) {
-            map.render({}, function(err) {
-                assert.ok(err);
-                assert.equal(err.message, 'Style is not set');
-                done();
-            });
-        });
-
-        it('returns an image', function(done) {
-            map.load(style);
-            map.render({}, function(err, data) {
-                assert.ifError(err);
-                fs.writeFileSync('test/results/image.png', data);
-                done();
-            });
-        });
-    });
-
-});
diff --git a/test/render.test.js b/test/render.test.js
new file mode 100644
index 0000000000..17f9bd228d
--- /dev/null
+++ b/test/render.test.js
@@ -0,0 +1,161 @@
+'use strict';
+
+/* jshint node:true */
+
+var test = require('tape');
+var mbgl = require('..');
+var fs = require('fs');
+var path = require('path');
+var mkdirp = require('mkdirp');
+var spawn = require('child_process').spawn;
+
+var suitePath = path.dirname(require.resolve('mapbox-gl-test-suite/package.json'));
+
+function template(name) {
+    return fs.readFileSync(require.resolve('mapbox-gl-test-suite/templates/' + name + '.html.tmpl')).toString();
+}
+
+var results = '';
+var resultTemplate = template('result');
+
+function format(tmpl, kwargs) {
+    return tmpl.replace(/\{\{|\}\}|\{([^}]+)\}/g, function(match, key) {
+        if (match === '{{') return '{';
+        if (match === '}}') return '}';
+        return kwargs[key];
+    });
+}
+
+function renderTest(style, info, base, key) {
+    var dir = path.join(suitePath, 'tests', base, key);
+    return function(t) {
+        var watchdog = setTimeout(function() {
+            t.fail('timed out after 20 seconds');
+        }, 20000);
+
+        t.once('end', function() {
+            clearTimeout(watchdog);
+        });
+
+        var fileSource = new mbgl.FileSource();
+        fileSource.request = function(req) {
+            fs.readFile(path.join(suitePath, decodeURIComponent(req.url)), function(err, data) {
+                req.respond(err, { data: data });
+                t.error(err);
+            });
+        };
+        fileSource.cancel = function() {};
+
+        var map = new mbgl.Map(fileSource);
+        map.load(style);
+
+        map.render(info[key], function(err, image) {
+            t.error(err);
+            mkdirp.sync(dir);
+
+            var expected = path.join(dir, 'expected.png');
+            var actual   = path.join(dir, 'actual.png');
+            var diff     = path.join(dir, 'diff.png');
+
+            if (process.env.UPDATE) {
+                fs.writeFile(expected, image, function(err) {
+                    t.error(err);
+                    t.end();
+                });
+            } else {
+                fs.writeFile(actual, image, function(err) {
+                    t.error(err);
+
+                    var compare = spawn('compare', ['-metric', 'MAE', actual, expected, diff]);
+                    var error = '';
+
+                    compare.stderr.on('data', function (data) {
+                        error += data.toString();
+                    });
+
+                    compare.on('exit', function (code) {
+                        // The compare program returns 2 on error otherwise 0 if the images are similar or 1 if they are dissimilar.
+                        if (code === 2) {
+                            writeResult(error.trim(), Infinity);
+                        } else {
+                            var match = error.match(/^\d+(?:\.\d+)?\s+\(([^\)]+)\)\s*$/);
+                            var difference = match ? parseFloat(match[1]) : Infinity;
+                            writeResult(match ? '' : error, difference);
+                        }
+                    });
+
+                    compare.stdin.end();
+
+                    function writeResult(error, difference) {
+                        var allowedDifference = ('diff' in info) ? info.diff : 0.001;
+                        var color = difference <= allowedDifference ? 'green' : 'red';
+
+                        results += format(resultTemplate, {
+                            name: base,
+                            key: key,
+                            color: color,
+                            error: error ? '<p>' + error + '</p>' : '',
+                            difference: difference,
+                            zoom: info.zoom || 0,
+                            center: info.center || [0, 0],
+                            bearing: info.bearing || 0,
+                            width: info.width || 512,
+                            height: info.height || 512
+                        });
+
+                        t.ok(difference <= allowedDifference, 'actual matches expected');
+                        t.end();
+                    }
+                });
+            }
+        });
+    };
+}
+
+function rewriteLocalSchema(uri) {
+    return uri.replace(/^local:\/\//, '');
+}
+
+var tests;
+
+if (process.argv[1] === __filename) {
+    tests = process.argv.slice(2);
+}
+
+test('Render', function(t) {
+    fs.readdirSync(path.join(suitePath, 'tests')).forEach(function(dir) {
+        if (dir === 'index.html' || dir[0] === '.') return;
+        if (tests && tests.length && tests.indexOf(dir) < 0) return;
+
+        var style = require(path.join(suitePath, 'tests', dir, 'style.json')),
+            info  = require(path.join(suitePath, 'tests', dir, 'info.json'));
+
+        for (var k in style.sources) {
+            var source = style.sources[k];
+
+            if (source.tiles) {
+                source.tiles = source.tiles.map(rewriteLocalSchema);
+            }
+
+            if (source.url) {
+                source.url = rewriteLocalSchema(source.url);
+            }
+        }
+
+        if (style.sprite) style.sprite = rewriteLocalSchema(style.sprite);
+        if (style.glyphs) style.glyphs = rewriteLocalSchema(style.glyphs);
+
+        style = JSON.stringify(style);
+
+        for (k in info) {
+            (info[k].native === false ? t.skip : t.test)(dir + ' ' + k, renderTest(style, info, dir, k));
+        }
+    });
+
+    t.test('results', function(t) {
+        var p = path.join(suitePath, 'tests', 'index.html');
+        fs.writeFileSync(p, format(template('results'), {results: results}));
+        console.warn('Results at: ' + p);
+        t.end();
+    });
+});