Consolidate test-suite rendering harnesses (#2236)

24 files changed, 78 insertions, 692 deletions

diff --git a/.gitmodules b/.gitmodules
index d84cfecbc2..2aa6d52be9 100644
--- a/.gitmodules
+++ b/.gitmodules
@@ -1,7 +1,3 @@
-[submodule "test/suite"]
-	path = test/suite
-	url = https://github.com/mapbox/mapbox-gl-test-suite.git
-
 [submodule ".mason"]
 	path = .mason
 	url = https://github.com/mapbox/mason.git
diff --git a/package.json b/package.json
index 3795ca8f58..0fd6887ba0 100644
--- a/package.json
+++ b/package.json
@@ -26,16 +26,14 @@
   ],
   "devDependencies": {
     "aws-sdk": "^2.1.47",
-    "mkdirp": "^0.5.1",
-    "pngjs": "^0.4.0",
+    "mapbox-gl-test-suite": "git+https://github.com/mapbox/mapbox-gl-test-suite.git#6cfd7eb20de77018a7f10770f56df9ce1f8a76ac",
     "request": "^2.61.0",
-    "st": "^0.5.5",
     "tape": "^4.2.0"
   },
   "scripts": {
     "install": "node-pre-gyp install --fallback-to-build=false || make node",
     "test": "tape platform/node/test/js/**/*.test.js",
-    "test-suite": "tape platform/node/test/render.test.js"
+    "test-suite": "node platform/node/test/render.test.js"
   },
   "gypfile": true,
   "binary": {
diff --git a/platform/node/README.md b/platform/node/README.md
index 870151fd25..72a08e8539 100644
--- a/platform/node/README.md
+++ b/platform/node/README.md
@@ -92,6 +92,8 @@ The `kind` is an enum and defined in [`mbgl.Resource`](https://github.com/mapbox
 
 It has no significance for anything but serves as a hint to your implemention as to what sort of resource to expect. E.g., your implementation could choose caching strategies based on the expected file type.
 
+THe `request` implementation should pass uncompressed data to `req.respond`. If you are downloading assets from a source that applies gzip transport encoding, the implementation must decompress the results before passing them on.
+
 A sample implementation that reads files from disk would look like the following:
 
 ```js
diff --git a/platform/node/test/compare.js b/platform/node/test/compare.js
deleted file mode 100644
index 3e5221de55..0000000000
--- a/platform/node/test/compare.js
+++ /dev/null
@@ -1,27 +0,0 @@
-var spawn = require('child_process').spawn;
-
-module.exports = function compare(actual, expected, diff, t, callback) {
-    var compare = spawn('compare', ['-metric', 'MAE', actual, expected, diff]);
-    var error = '';
-
-    compare.stderr.on('data', function(data) {
-        error += data.toString();
-    });
-
-    compare.on('error', function(err) {
-        t.error(err);
-    });
-
-    compare.on('exit', function(code) {
-        // The compare program returns 2 on error otherwise 0 if the images are similar or 1 if they are dissimilar.
-        if (code === 2) {
-            callback(error.trim(), Infinity);
-        } else {
-            var match = error.match(/^\d+(?:\.\d+)?\s+\(([^\)]+)\)\s*$/);
-            var difference = match ? parseFloat(match[1]) : Infinity;
-            callback(match ? '' : error, difference);
-        }
-    });
-
-    compare.stdin.end();
-};
diff --git a/platform/node/test/expected/gzip/success.png b/platform/node/test/expected/gzip/success.png
deleted file mode 100644
index de41e0fe2b..0000000000
--- a/platform/node/test/expected/gzip/success.png
+++ /dev/null
diff --git a/platform/node/test/expected/gzip/unhandled.png b/platform/node/test/expected/gzip/unhandled.png
deleted file mode 100644
index ddb90d8b8f..0000000000
--- a/platform/node/test/expected/gzip/unhandled.png
+++ /dev/null
diff --git a/platform/node/test/expected/map/image.png b/platform/node/test/expected/map/image.png
deleted file mode 100644
index de41e0fe2b..0000000000
--- a/platform/node/test/expected/map/image.png
+++ /dev/null
diff --git a/platform/node/test/js/consecutive.test.js b/platform/node/test/js/consecutive.test.js
deleted file mode 100644
index 928ca1d837..0000000000
--- a/platform/node/test/js/consecutive.test.js
+++ /dev/null
@@ -1,82 +0,0 @@
-'use strict';
-
-/* jshint node:true */
-
-var test = require('tape');
-var mbgl = require('../../../..');
-var fs = require('fs');
-var path = require('path');
-
-var suitePath = path.join(__dirname, '../../../../test/suite');
-
-function renderTest(style, info, dir, key) {
-    return function (t) {
-        var completed = 0;
-        var remaining = 10;
-        var start = +new Date;
-
-        var options = {};
-        options.request = function(req) {
-            fs.readFile(path.join(suitePath, decodeURIComponent(req.url)), function(err, data) {
-                req.respond(err, { data: data });
-                t.error(err);
-            });
-        };
-        options.cancel = function() {};
-        options.ratio = 1.0;
-
-        var map = new mbgl.Map(options);
-        map.load(style);
-
-        function render() {
-            map.render(info[key], function(err, image) {
-                t.error(err);
-
-                t.ok(true, 'render @ ' + ((+new Date) - start) + 'ms');
-                if (++completed === remaining) {
-                    map.release();
-                    t.end();
-                } else {
-                    render();
-                }
-            });
-        }
-
-        render();
-    };
-}
-
-function rewriteLocalSchema(uri) {
-    return uri.replace(/^local:\/\//, '');
-}
-
-test('Consecutive', function(t) {
-    var dir = 'line-join';
-    var k = 'round';
-
-    var style = require(path.join(suitePath, 'tests', dir, 'style.json')),
-        info  = require(path.join(suitePath, 'tests', dir, 'info.json'));
-
-    for (var k in style.sources) {
-        var source = style.sources[k];
-
-        if (source.tiles) {
-            source.tiles = source.tiles.map(rewriteLocalSchema);
-        }
-
-        if (source.url) {
-            source.url = rewriteLocalSchema(source.url);
-        }
-    }
-
-    if (style.sprite) style.sprite = rewriteLocalSchema(style.sprite);
-    if (style.glyphs) style.glyphs = rewriteLocalSchema(style.glyphs);
-
-    style = JSON.stringify(style);
-
-    for (k in info) {
-        t.test(dir + ' ' + k, renderTest(style, info, dir, k));
-    }
-
-    t.end();
-});
diff --git a/platform/node/test/js/gzip.test.js b/platform/node/test/js/gzip.test.js
deleted file mode 100644
index 4545a18cf5..0000000000
--- a/platform/node/test/js/gzip.test.js
+++ /dev/null
@@ -1,125 +0,0 @@
-'use strict';
-
-/* jshint node: true */
-
-var test = require('tape').test;
-var mbgl = require('../../../..');
-var fs = require('fs');
-var path = require('path');
-var mkdirp = require('mkdirp');
-var http = require('http');
-var request = require('request');
-var st = require('st');
-var style = require('../fixtures/style.json');
-var PNG = require('pngjs').PNG;
-var compare = require('../compare.js');
-
-var server = http.createServer(st({ path: path.join(__dirname, '..') }));
-server.listen(0);
-
-function filePath(name) {
-    return ['expected', 'actual', 'diff'].reduce(function(prev, key) {
-        var dir = path.join(__dirname, '..', key, 'gzip');
-        mkdirp.sync(dir);
-        prev[key] = path.join(dir, name);
-        return prev;
-    }, {});
-}
-
-function setup(options, callback) {
-    callback(new mbgl.Map(options));
-}
-
-function getOptions(gzip, t) {
-    return {
-        request: function(req) {
-            var parts = req.url.split('.');
-            var filetype = parts[parts.length - 1];
-
-            request({
-                url: 'http://localhost:' + server.address().port + path.join('/', req.url),
-                encoding: null,
-                gzip: filetype === 'pbf' ? gzip : true,
-                headers: {
-                    'Accept-Encoding': 'gzip'
-                }
-            }, function (err, res, body) {
-                t.error(err);
-                var response = {};
-                response.data = res.body;
-                req.respond(null, response);
-            });
-        },
-        ratio: 1.0
-    };
-}
-
-test('gzip', function(t) {
-    t.test('success', function(t) {
-        mbgl.on('message', function(msg) {
-            if (msg.severity == 'ERROR') t.error(msg);
-        });
-
-        setup(getOptions(true, t), function(map) {
-            map.load(style);
-            map.render({}, function(err, data) {
-                mbgl.removeAllListeners('message');
-                map.release();
-
-                t.error(err);
-
-                var filename = filePath('success.png');
-
-                var png = new PNG({
-                    width: data.width,
-                    height: data.height
-                });
-
-                png.data = data.pixels;
-
-                if (process.env.UPDATE) {
-                    png.pack()
-                        .pipe(fs.createWriteStream(filename.expected))
-                        .on('finish', t.end);
-                } else {
-                    png.pack()
-                        .pipe(fs.createWriteStream(filename.actual))
-                        .on('finish', function() {
-                            compare(filename.actual, filename.expected, filename.diff, t, function(err, diff) {
-                                t.error(err);
-                                t.ok(diff <= 0.01, 'actual matches expected');
-                                t.end();
-                            });
-                        });
-                }
-            });
-        });
-    });
-
-    t.test('unhandled', function(t) {
-        mbgl.once('message', function(msg) {
-            if (msg.severity == 'ERROR') {
-                t.ok(msg, 'emits error');
-                t.equal(msg.class, 'Style');
-                t.equal(msg.severity, 'ERROR');
-                t.ok(msg.text.match(/pbf unknown field type exception/), 'error text matches');
-            }
-        });
-
-        setup(getOptions(false, t), function(map) {
-            map.load(style);
-            map.render({}, function(err, data) {
-                map.release();
-
-                t.ok(err, 'returns error');
-                t.ok(err.message.match(/Failed to parse/), 'error text matches');
-
-                t.end();
-            });
-        });
-    });
-
-    t.test('teardown', function(t) {
-        server.close(t.end);
-    });
-});
diff --git a/platform/node/test/js/map.test.js b/platform/node/test/js/map.test.js
index 09870abaaa..c4e3416322 100644
--- a/platform/node/test/js/map.test.js
+++ b/platform/node/test/js/map.test.js
@@ -1,24 +1,10 @@
 'use strict';
 
-/* jshint node: true */
-
 var test = require('tape');
 var mbgl = require('../../../..');
 var fs = require('fs');
 var path = require('path');
-var mkdirp = require('mkdirp');
 var style = require('../fixtures/style.json');
-var PNG = require('pngjs').PNG;
-var compare = require('../compare.js');
-
-function filePath(name) {
-    return ['expected', 'actual', 'diff'].reduce(function(prev, key) {
-        var dir = path.join(__dirname, '..', key, 'map');
-        mkdirp.sync(dir);
-        prev[key] = path.join(dir, name);
-        return prev;
-    }, {});
-}
 
 test('Map', function(t) {
     t.test('must be constructed with new', function(t) {
@@ -250,31 +236,53 @@ test('Map', function(t) {
 
                 map.release();
 
-                var filename = filePath('image.png');
+                t.ok(data.pixels);
+                t.equal(data.width, 512);
+                t.equal(data.height, 512);
+                t.end();
+            });
+        });
+
+        t.test('can be called several times in serial', function(t) {
+            var completed = 0;
+            var remaining = 10;
+            var start = +new Date;
 
-                var png = new PNG({
-                    width: data.width,
-                    height: data.height
+            var map = new mbgl.Map(options);
+            map.load(style);
+
+            function render() {
+                map.render({}, function(err, data) {
+                    t.error(err);
+
+                    t.ok(true, 'render @ ' + ((+new Date) - start) + 'ms');
+                    if (++completed === remaining) {
+                        map.release();
+                        t.end();
+                    } else {
+                        render();
+                    }
                 });
+            }
+
+            render();
+        });
 
-                png.data = data.pixels;
-
-                if (process.env.UPDATE) {
-                    png.pack()
-                        .pipe(fs.createWriteStream(filename.expected))
-                        .on('finish', t.end);
-                } else {
-                    png.pack()
-                        .pipe(fs.createWriteStream(filename.actual))
-                        .on('finish', function() {
-                            compare(filename.actual, filename.expected, filename.diff, t, function(err, diff) {
-                                t.error(err);
-                                t.ok(diff <= 0.01, 'actual matches expected');
-                                t.end();
-                            });
-                        });
-                }
+        t.skip('throws if called in parallel', function(t) {
+            var completed = 0;
+            var remaining = 10;
+            var start = +new Date;
+
+            var map = new mbgl.Map(options);
+            map.load(style);
+
+            t.throws(function() {
+                map.render({}, function() {});
+                map.render({}, function() {});
             });
+
+            map.release();
+            t.end();
         });
     });
 });
diff --git a/platform/node/test/render.test.js b/platform/node/test/render.test.js
index 10a7afcb52..b9f13a28fe 100644
--- a/platform/node/test/render.test.js
+++ b/platform/node/test/render.test.js
@@ -1,175 +1,28 @@
 'use strict';
 
-/* jshint node:true */
-
-var test = require('tape');
 var mbgl = require('../../..');
-var fs = require('fs');
-var path = require('path');
-var mkdirp = require('mkdirp');
-var PNG = require('pngjs').PNG;
-var compare = require('./compare.js');
-var suitePath = path.join(__dirname, '../../../test/suite');
-
-function template(name) {
-    return fs.readFileSync(path.join(suitePath, 'templates', name + '.html.tmpl')).toString();
-}
-
-var results = '';
-var resultTemplate = template('result');
-
-function format(tmpl, kwargs) {
-    return tmpl.replace(/\{\{|\}\}|\{([^}]+)\}/g, function(match, key) {
-        if (match === '{{') return '{';
-        if (match === '}}') return '}';
-        return kwargs[key];
-    });
-}
-
-function renderTest(style, info, base, key) {
-    var dir = path.join(suitePath, 'tests', base, key);
-    mkdirp.sync(dir);
-
-    return function(t) {
-        var watchdog = setTimeout(function() {
-            t.fail('timed out after 20 seconds');
-        }, 20000);
-
-        t.once('end', function() {
-            clearTimeout(watchdog);
-
-            if (map) {
-                map.release();
-                map = null;
-            }
-        });
-
-        var options = {};
-        options.request = function(req) {
-            var url =  decodeURIComponent(req.url);
-            fs.readFile(path.join(suitePath, url), function(err, data) {
-                req.respond(err, { data: data });
-            });
-        };
-        options.ratio = info[key].pixelRatio || 1;
-
-        var map = new mbgl.Map(options);
-        map.load(style);
-
-        map.render(info[key], function(err, data) {
-            if (err) {
-                t.error(err);
-                return t.end();
-            }
-
-            var expected = path.join(dir, 'expected.png');
-            var actual = path.join(dir, 'actual.png');
-            var diff = path.join(dir, 'diff.png');
-
-            var png = new PNG({
-                width: data.width,
-                height: data.height
-            });
-
-            png.data = data.pixels;
-
-            if (process.env.UPDATE) {
-                png.pack()
-                    .pipe(fs.createWriteStream(expected))
-                    .on('finish', t.end);
-            } else {
-                png.pack()
-                    .pipe(fs.createWriteStream(actual))
-                    .on('finish', function() {
-                        compare(actual, expected, diff, t, function(err, diff) {
-                            t.error(err);
-
-                            var allowed = 0.001;
-
-                            if ('diff' in info[key]) {
-                                if (typeof info[key].diff === 'number') {
-                                    allowed = info[key].diff;
-                                } else if ('native' in info[key].diff) {
-                                    allowed = info[key].diff.native;
-                                }
-                            }
-
-                            results += format(resultTemplate, {
-                                name: base,
-                                key: key,
-                                color: diff <= allowed ? 'green' : 'red',
-                                error: err ? '<p>' + err + '</p>' : '',
-                                difference: diff,
-                                zoom: info.zoom || 0,
-                                center: info.center || [0, 0],
-                                bearing: info.bearing || 0,
-                                width: info.width || 512,
-                                height: info.height || 512
-                            });
-
-                            if (!info[key].ignored || !('native' in info[key].ignored)) {
-                                t.ok(diff <= allowed, 'expected ' + diff + ' to be less than ' + allowed);
-                            }
-
-                            t.end();
-                        });
-                });
-            }
-        });
-    };
-}
-
-function rewriteLocalSchema(url) {
-    var regex = /^local:\/\//;
-    if (url instanceof Array) {
-        return url.map(function(str) {
-            return str.replace(regex, '');
-        });
-    } else if (typeof url === "string") {
-        return url.replace(regex, '');
-    }
-}
+var suite = require('mapbox-gl-test-suite');
+var request = require('request');
 
 var tests;
 
-if (process.argv[1] === __filename) {
+if (process.argv[1] === __filename && process.argv.length > 2) {
     tests = process.argv.slice(2);
 }
 
-test('Render', function(t) {
-    fs.readdirSync(path.join(suitePath, 'tests')).forEach(function(dir) {
-        if (dir === 'index.html' || dir[0] === '.') return;
-        if (tests && tests.length && tests.indexOf(dir) < 0) return;
-
-        var style = require(path.join(suitePath, 'tests', dir, 'style.json')),
-            info  = require(path.join(suitePath, 'tests', dir, 'info.json'));
-
-        for (var k in style.sources) {
-            var source = style.sources[k];
-
-            if (source.tiles) {
-                source.tiles = source.tiles.map(rewriteLocalSchema);
-            }
-
-            if (source.url) {
-                source.url = rewriteLocalSchema(source.url);
-            }
-        }
-
-        if (style.sprite) style.sprite = rewriteLocalSchema(style.sprite);
-        if (style.glyphs) style.glyphs = rewriteLocalSchema(style.glyphs);
-
-        style = JSON.stringify(style);
-
-        for (k in info) {
-            (info[k].native === false ? t.skip : t.test)(dir + ' ' + k, renderTest(style, info, dir, k));
+suite.run('native', {tests: tests}, function (style, options, callback) {
+    var map = new mbgl.Map({
+        ratio: options.pixelRatio,
+        request: function (req) {
+            request(req.url, {encoding: null}, function (err, response, body) {
+                req.respond(err, {data: body});
+            });
         }
     });
 
-    t.test('results', function(t) {
-        var p = path.join(suitePath, 'tests', 'index.html');
-        fs.writeFileSync(p, format(template('results'), {results: results}));
-        console.warn('Results at: ' + p);
-        t.end();
+    map.load(style);
+    map.render(options, function (err, result) {
+        map.release();
+        callback(err, result && result.pixels);
     });
 });
diff --git a/scripts/compare_images.sh b/scripts/compare_images.sh
deleted file mode 100755
index 2407726469..0000000000
--- a/scripts/compare_images.sh
+++ /dev/null
@@ -1,6 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-set -o pipefail
-
-(cd ./test/suite/ && ./bin/compare_images.py native)
diff --git a/scripts/linux/after_script.sh b/scripts/linux/after_script.sh
deleted file mode 100755
index c38af0a797..0000000000
--- a/scripts/linux/after_script.sh
+++ /dev/null
@@ -1,15 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-set -o pipefail
-
-if [ ! -z "${AWS_ACCESS_KEY_ID}" ] && [ ! -z "${AWS_SECRET_ACCESS_KEY}" ] ; then
-    # Install and add awscli to PATH for uploading the results
-    mapbox_time "install_awscli" \
-    pip install --user awscli
-    export PATH="`python -m site --user-base`/bin:${PATH}"
-
-    mapbox_time_start "deploy_results"
-    (cd ./test/suite/ && ./bin/deploy_results.sh)
-    mapbox_time_finish
-fi
diff --git a/scripts/linux/run.sh b/scripts/linux/run.sh
index 42d2a16d90..31272ea1c2 100755
--- a/scripts/linux/run.sh
+++ b/scripts/linux/run.sh
@@ -27,11 +27,5 @@ make test -j${JOBS} BUILDTYPE=${BUILDTYPE}
 # Test
 ################################################################################
 
-mapbox_time "checkout_test_suite" \
-git submodule update --init test/suite
-
 mapbox_time "run_tests" \
 make test-* BUILDTYPE=${BUILDTYPE}
-
-mapbox_time "compare_results" \
-./scripts/compare_images.sh
diff --git a/scripts/main.mk b/scripts/main.mk
index 65f283b219..3639a780a7 100644
--- a/scripts/main.mk
+++ b/scripts/main.mk
@@ -46,10 +46,6 @@ SUBMODULES += src/mbgl/util/geojsonvt/geojsonvt.hpp
 src/mbgl/util/geojsonvt/geojsonvt.hpp:
 	./scripts/flock.py .git/Submodule.lock git submodule update --init src/mbgl/util/geojsonvt
 
-SUBMODULES += test/suite/package.json
-test/suite/package.json:
-	./scripts/flock.py .git/Submodule.lock git submodule update --init test/suite
-
 ifeq ($(HOST),ios)
 SUBMODULES += platform/ios/vendor/SMCalloutView/SMCalloutView.h
 platform/ios/vendor/SMCalloutView/SMCalloutView.h:
diff --git a/scripts/node/after_script.sh b/scripts/node/after_script.sh
index d78e820e92..237f9fe600 100755
--- a/scripts/node/after_script.sh
+++ b/scripts/node/after_script.sh
@@ -35,3 +35,16 @@ if test "${COMMIT_MESSAGE#*'[publish binary]'}" != "$COMMIT_MESSAGE"; then
         npm test
     fi
 fi
+
+if [ ! -z "${AWS_ACCESS_KEY_ID}" ] && [ ! -z "${AWS_SECRET_ACCESS_KEY}" ] ; then
+    # Install and add awscli to PATH for uploading the results
+    pip install --user awscli
+    export PATH="`python -m site --user-base`/bin:${PATH}"
+
+    REPO_NAME=$(basename $TRAVIS_REPO_SLUG)
+    gzip --stdout node_modules/mapbox-gl-test-suite/tests/index.html | \
+        aws s3 cp --acl public-read --content-encoding gzip --content-type text/html \
+            - s3://mapbox/$REPO_NAME/tests/$TRAVIS_JOB_NUMBER/index.html
+
+    echo http://mapbox.s3.amazonaws.com/$REPO_NAME/tests/$TRAVIS_JOB_NUMBER/index.html
+fi
diff --git a/scripts/node/run.sh b/scripts/node/run.sh
index 8566fc3b3a..22db3fdedd 100755
--- a/scripts/node/run.sh
+++ b/scripts/node/run.sh
@@ -26,24 +26,9 @@ npm install --build-from-source
 
 # Travis OS X has no GPU
 if [[ ${TRAVIS_OS_NAME} == "linux" ]]; then
-    mapbox_time "checkout_test_suite" \
-    git submodule update --init test/suite
-
     mapbox_time "run_tests" \
     npm test
 
     mapbox_time "run_render_tests" \
     npm run test-suite
-
-    if [ ! -z "${AWS_ACCESS_KEY_ID}" ] && [ ! -z "${AWS_SECRET_ACCESS_KEY}" ] ; then
-      # Install and add awscli to PATH for uploading the results
-        mapbox_time "install_awscli" \
-        pip install --user awscli
-        export PATH="`python -m site --user-base`/bin:${PATH}"
-
-        pushd test/suite
-        mapbox_time "deploy_results" \
-        ./bin/deploy_results.sh
-        popd
-    fi
 fi
diff --git a/test/api/repeated_render.cpp b/test/api/repeated_render.cpp
index f8b92b85b8..3317b4e3a4 100644
--- a/test/api/repeated_render.cpp
+++ b/test/api/repeated_render.cpp
@@ -25,7 +25,7 @@ TEST(API, RepeatedRender) {
     Map map(view, fileSource, MapMode::Still);
 
     {
-        map.setStyleJSON(style, "test/suite");
+        map.setStyleJSON(style, "");
         std::promise<std::unique_ptr<const StillImage>> promise;
         map.renderStill([&promise](std::exception_ptr, std::unique_ptr<const StillImage> image) {
             promise.set_value(std::move(image));
diff --git a/test/api/set_style.cpp b/test/api/set_style.cpp
index db31254dbb..c3e62def20 100644
--- a/test/api/set_style.cpp
+++ b/test/api/set_style.cpp
@@ -18,7 +18,7 @@ TEST(API, SetStyle) {
 
     {
         Map map(view, fileSource, MapMode::Still);
-        map.setStyleJSON("invalid", "test/suite");
+        map.setStyleJSON("invalid", "");
     }
 
     auto observer = Log::removeObserver();
diff --git a/test/headless/headless.cpp b/test/headless/headless.cpp
deleted file mode 100644
index 80951433ed..0000000000
--- a/test/headless/headless.cpp
+++ /dev/null
@@ -1,184 +0,0 @@
-#include "../fixtures/util.hpp"
-#include "../fixtures/fixture_log_observer.hpp"
-
-#include <mbgl/map/map.hpp>
-#include <mbgl/map/still_image.hpp>
-#include <mbgl/util/image.hpp>
-
-#include <mbgl/util/io.hpp>
-#include <rapidjson/document.h>
-#include <rapidjson/writer.h>
-#include <rapidjson/stringbuffer.h>
-
-#include <mbgl/platform/platform.hpp>
-#include <mbgl/platform/default/headless_view.hpp>
-#include <mbgl/platform/default/headless_display.hpp>
-#include <mbgl/storage/default_file_source.hpp>
-
-#include <dirent.h>
-
-#include <future>
-
-void rewriteLocalScheme(rapidjson::Value &value, rapidjson::Document::AllocatorType &allocator) {
-    ASSERT_TRUE(value.IsString());
-    auto string = std::string { value.GetString(),value.GetStringLength() };
-    if (string.compare(0, 8, "local://") == 0) {
-        string.replace(0, 8, "http://127.0.0.1:2900/");
-        value.SetString(string.data(), string.size(), allocator);
-    }
-}
-
-
-class HeadlessTest : public ::testing::TestWithParam<std::string> {
-public:
-    static void SetUpTestCase() {
-        const auto server = mbgl::platform::applicationRoot() + "/TEST_DATA/headless/server.js";
-        pid = mbgl::test::startServer(server.c_str());
-        display = std::make_shared<mbgl::HeadlessDisplay>();
-    }
-
-    static void TearDownTestCase() {
-        display.reset();
-        mbgl::test::stopServer(pid);
-    }
-
-protected:
-    static pid_t pid;
-    static std::shared_ptr<mbgl::HeadlessDisplay> display;
-};
-
-pid_t HeadlessTest::pid = 0;
-std::shared_ptr<mbgl::HeadlessDisplay> HeadlessTest::display;
-
-TEST_P(HeadlessTest, render) {
-    using namespace mbgl;
-
-    const std::string& base = GetParam();
-
-    std::string style = util::read_file("test/suite/tests/" + base + "/style.json");
-    std::string info = util::read_file("test/suite/tests/" + base + "/info.json");
-
-    // Parse style.
-    rapidjson::Document styleDoc;
-    styleDoc.Parse<0>((const char *const)style.c_str());
-    ASSERT_FALSE(styleDoc.HasParseError());
-    ASSERT_TRUE(styleDoc.IsObject());
-
-    // Rewrite "local://" to "http://127.0.0.1:2900/".
-    if (styleDoc.HasMember("sprite")) {
-        rewriteLocalScheme(styleDoc["sprite"], styleDoc.GetAllocator());
-    }
-
-    if (styleDoc.HasMember("glyphs")) {
-        rewriteLocalScheme(styleDoc["glyphs"], styleDoc.GetAllocator());
-    }
-
-    if (styleDoc.HasMember("sources")) {
-        auto &sources = styleDoc["sources"];
-        ASSERT_TRUE(sources.IsObject());
-        for (auto source = sources.MemberBegin(), end = sources.MemberEnd(); source != end; source++) {
-            if (source->value.HasMember("tiles")) {
-                auto &tiles = source->value["tiles"];
-                ASSERT_TRUE(tiles.IsArray());
-                for (rapidjson::SizeType i = 0; i < tiles.Size(); i++) {
-                    rewriteLocalScheme(tiles[i], styleDoc.GetAllocator());
-                }
-            }
-
-            if (source->value.HasMember("url") && source->value["url"].IsString()) {
-                rewriteLocalScheme(source->value["url"], styleDoc.GetAllocator());
-            }
-        }
-    }
-
-    // Convert the JSON object back into a stringified version.
-    rapidjson::StringBuffer buffer;
-    rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
-    styleDoc.Accept(writer);
-    style = buffer.GetString();
-
-    // Parse settings.
-    rapidjson::Document infoDoc;
-    infoDoc.Parse<0>((const char *const)info.c_str());
-    ASSERT_FALSE(infoDoc.HasParseError());
-    ASSERT_TRUE(infoDoc.IsObject());
-
-    for (auto it = infoDoc.MemberBegin(), end = infoDoc.MemberEnd(); it != end; it++) {
-        const std::string name {
-            it->name.GetString(), it->name.GetStringLength()
-        };
-
-        Log::Info(Event::General, "%s %s", base.c_str(), name.c_str());
-
-        const rapidjson::Value& value = it->value;
-        ASSERT_TRUE(value.IsObject());
-
-        if (value.HasMember("native") && !value["native"].GetBool())
-            continue;
-
-        if (value.HasMember("center")) ASSERT_TRUE(value["center"].IsArray());
-
-        const std::string actual_image = "test/suite/tests/" + base + "/" + name +  "/actual.png";
-
-        const double zoom = value.HasMember("zoom") ? value["zoom"].GetDouble() : 0;
-        const double bearing = value.HasMember("bearing") ? value["bearing"].GetDouble() : 0;
-        const double latitude = value.HasMember("center") ? value["center"][rapidjson::SizeType(0)].GetDouble() : 0;
-        const double longitude = value.HasMember("center") ? value["center"][rapidjson::SizeType(1)].GetDouble() : 0;
-        const unsigned int width = value.HasMember("width") ? value["width"].GetUint() : 512;
-        const unsigned int height = value.HasMember("height") ? value["height"].GetUint() : 512;
-        const unsigned int pixelRatio = value.HasMember("pixelRatio") ? value["pixelRatio"].GetUint() : 1;
-
-        std::vector<std::string> classes;
-        if (value.HasMember("classes")) {
-            const rapidjson::Value& js_classes = value["classes"];
-            ASSERT_TRUE(js_classes.IsArray());
-            for (rapidjson::SizeType i = 0; i < js_classes.Size(); i++) {
-                const rapidjson::Value& js_class = js_classes[i];
-                ASSERT_TRUE(js_class.IsString());
-                classes.push_back({ js_class.GetString(), js_class.GetStringLength() });
-            }
-        }
-
-        std::promise<void> promise;
-
-        HeadlessView view(display, pixelRatio, width, height);
-        DefaultFileSource fileSource(nullptr);
-        Map map(view, fileSource, MapMode::Still);
-
-        map.setClasses(classes);
-        map.setStyleJSON(style, "test/suite");
-        map.setLatLngZoom(mbgl::LatLng(latitude, longitude), zoom);
-        map.setBearing(bearing);
-
-        map.renderStill([&](std::exception_ptr error, std::unique_ptr<const StillImage> image) {
-            if (error) {
-                promise.set_exception(error);
-                return;
-            }
-            const std::string png = util::compress_png(image->width, image->height, image->pixels.get());
-            util::write_file("test/suite/tests/" + base + "/" + name +  "/actual.png", png);
-            promise.set_value();
-        });
-
-        promise.get_future().get();
-    }
-}
-
-INSTANTIATE_TEST_CASE_P(Headless, HeadlessTest, ::testing::ValuesIn([] {
-    std::vector<std::string> names;
-
-    const auto tests = mbgl::platform::applicationRoot() + "/TEST_DATA/suite/tests";
-    DIR *dir = opendir(tests.c_str());
-    if (dir != nullptr) {
-        for (dirent *dp = nullptr; (dp = readdir(dir)) != nullptr;) {
-            const std::string name = dp->d_name;
-            if (name != "index.html" && !(name.size() >= 1 && name[0] == '.')) {
-                names.push_back(name);
-            }
-        }
-        closedir(dir);
-    }
-
-    EXPECT_GT(names.size(), 0ul);
-    return names;
-}()));
diff --git a/test/headless/server.js b/test/headless/server.js
deleted file mode 100755
index ff349b050e..0000000000
--- a/test/headless/server.js
+++ /dev/null
@@ -1,19 +0,0 @@
-#!/usr/bin/env node
-/* jshint node: true */
-'use strict';
-
-var express = require('express');
-var app = express();
-
-app.use(express.static('test/suite'));
-
-var server = app.listen(2900, function () {
-    var host = server.address().address;
-    var port = server.address().port;
-    console.log('Test server listening at http://%s:%s', host, port);
-
-    if (process.argv[2]) {
-        // Allow the test to continue running.
-        process.stdin.write("Go!\n");
-    }
-});
diff --git a/test/headless/custom_sprites.cpp b/test/miscellaneous/custom_sprites.cpp
index e2c479a5c2..a72119ea61 100644
--- a/test/headless/custom_sprites.cpp
+++ b/test/miscellaneous/custom_sprites.cpp
@@ -30,7 +30,7 @@ TEST(Headless, CustomSpriteImages) {
 
     map.setLatLngZoom(LatLng{ 52.499167, 13.418056 }, 15);
 
-    map.setStyleJSON(style, "test/suite");
+    map.setStyleJSON(style, "");
     map.setSprite("cafe",
                   std::make_shared<SpriteImage>(12, 12, 1, std::string(12 * 12 * 4, '\xFF')));
     std::promise<std::unique_ptr<const StillImage>> promise;
diff --git a/test/suite b/test/suite
deleted file mode 160000
-Subproject 023eb9512a2f73389f00e38fd9da272833992b3
diff --git a/test/test.gypi b/test/test.gypi
index 4e669aee1d..a35b00133b 100644
--- a/test/test.gypi
+++ b/test/test.gypi
@@ -50,13 +50,12 @@
         'api/repeated_render.cpp',
         'api/set_style.cpp',
 
-        'headless/custom_sprites.cpp',
-        'headless/headless.cpp',
 
         'miscellaneous/clip_ids.cpp',
         'miscellaneous/binpack.cpp',
         'miscellaneous/bilinear.cpp',
         'miscellaneous/comparisons.cpp',
+        'miscellaneous/custom_sprites.cpp',
         'miscellaneous/enums.cpp',
         'miscellaneous/functions.cpp',
         'miscellaneous/geo.cpp',