[test runner] Split operations parsing and execution

5 files changed, 805 insertions, 663 deletions

diff --git a/render-test/metadata.hpp b/render-test/metadata.hpp
index ba4a875b0c..918758c3e8 100644
--- a/render-test/metadata.hpp
+++ b/render-test/metadata.hpp
@@ -9,14 +9,19 @@
 
 #include "filesystem.hpp"
 
+#include <list>
 #include <map>
 
 namespace mbgl {
+
+class Map;
+class HeadlessFrontend;
 namespace gfx {
 struct RenderingStats;
 }
 } // namespace mbgl
 
+class TestRunnerMapObserver;
 struct TestStatistics {
     TestStatistics() = default;
 
@@ -168,3 +173,21 @@ struct TestMetadata {
     TestMetrics metrics;
     TestMetrics expectedMetrics;
 };
+
+class TestContext {
+public:
+    virtual mbgl::HeadlessFrontend& getFrontend() = 0;
+    virtual mbgl::Map& getMap() = 0;
+    virtual TestRunnerMapObserver& getObserver() = 0;
+    virtual TestMetadata& getMetadata() = 0;
+
+    GfxProbe activeGfxProbe;
+    GfxProbe baselineGfxProbe;
+    bool gfxProbeActive = false;
+
+protected:
+    virtual ~TestContext() = default;
+};
+
+using TestOperation = std::function<bool(TestContext&)>;
+using TestOperations = std::list<TestOperation>;
\ No newline at end of file
diff --git a/render-test/parser.cpp b/render-test/parser.cpp
index 1c1f8b2798..2373413da5 100644
--- a/render-test/parser.cpp
+++ b/render-test/parser.cpp
@@ -1,26 +1,41 @@
-#include <mbgl/util/logging.hpp>
+#include "parser.hpp"
+
+#include "allocation_index.hpp"
+#include "file_source.hpp"
+#include "filesystem.hpp"
+#include "metadata.hpp"
+#include "runner.hpp"
+
+#include <mbgl/map/map.hpp>
+#include <mbgl/renderer/renderer.hpp>
+#include <mbgl/style/conversion/filter.hpp>
+#include <mbgl/style/conversion/json.hpp>
+#include <mbgl/style/conversion/layer.hpp>
+#include <mbgl/style/conversion/light.hpp>
+#include <mbgl/style/conversion/source.hpp>
+#include <mbgl/style/layer.hpp>
+#include <mbgl/style/light.hpp>
+#include <mbgl/style/source.hpp>
+#include <mbgl/style/style.hpp>
+#include <mbgl/util/compression.hpp>
 #include <mbgl/util/io.hpp>
+#include <mbgl/util/logging.hpp>
 #include <mbgl/util/rapidjson.hpp>
+#include <mbgl/util/run_loop.hpp>
 #include <mbgl/util/string.hpp>
+#include <mbgl/util/timer.hpp>
 
 #include <rapidjson/prettywriter.h>
 #include <rapidjson/stringbuffer.h>
 #include <rapidjson/writer.h>
 
 #include <mapbox/geojson_impl.hpp>
-#include <mbgl/style/conversion/filter.hpp>
-#include <mbgl/style/conversion/json.hpp>
 
 #include <boost/archive/iterators/base64_from_binary.hpp>
 #include <boost/archive/iterators/insert_linebreaks.hpp>
 #include <boost/archive/iterators/transform_width.hpp>
 #include <boost/archive/iterators/ostream_iterator.hpp>
 
-#include "filesystem.hpp"
-#include "metadata.hpp"
-#include "parser.hpp"
-#include "runner.hpp"
-
 #include <regex>
 #include <sstream>
 
@@ -571,6 +586,697 @@ TestMetadata parseTestMetadata(const TestPaths& paths, const Manifest& manifest)
     return metadata;
 }
 
+namespace TestOperationNames {
+const std::string waitOp("wait");
+const std::string sleepOp("sleep");
+const std::string addImageOp("addImage");
+const std::string updateImageOp("updateImage");
+const std::string removeImageOp("removeImage");
+const std::string setStyleOp("setStyle");
+const std::string setCenterOp("setCenter");
+const std::string setZoomOp("setZoom");
+const std::string setBearingOp("setBearing");
+const std::string setPitchOp("setPitch");
+const std::string setFilterOp("setFilter");
+const std::string setLayerZoomRangeOp("setLayerZoomRange");
+const std::string setLightOp("setLight");
+const std::string addLayerOp("addLayer");
+const std::string removeLayerOp("removeLayer");
+const std::string addSourceOp("addSource");
+const std::string removeSourceOp("removeSource");
+const std::string setPaintPropertyOp("setPaintProperty");
+const std::string setLayoutPropertyOp("setLayoutProperty");
+const std::string fileSizeProbeOp("probeFileSize");
+const std::string memoryProbeOp("probeMemory");
+const std::string memoryProbeStartOp("probeMemoryStart");
+const std::string memoryProbeEndOp("probeMemoryEnd");
+const std::string networkProbeOp("probeNetwork");
+const std::string networkProbeStartOp("probeNetworkStart");
+const std::string networkProbeEndOp("probeNetworkEnd");
+const std::string setFeatureStateOp("setFeatureState");
+const std::string getFeatureStateOp("getFeatureState");
+const std::string removeFeatureStateOp("removeFeatureState");
+const std::string panGestureOp("panGesture");
+const std::string gfxProbeOp("probeGFX");
+const std::string gfxProbeStartOp("probeGFXStart");
+const std::string gfxProbeEndOp("probeGFXEnd");
+} // namespace TestOperationNames
+
+using namespace TestOperationNames;
+
+TestOperations parseTestOperations(TestMetadata& metadata, const Manifest& manifest) {
+    TestOperations result;
+    if (!metadata.document.HasMember("metadata") || !metadata.document["metadata"].HasMember("test") ||
+        !metadata.document["metadata"]["test"].HasMember("operations")) {
+        return result;
+    }
+    assert(metadata.document["metadata"]["test"]["operations"].IsArray());
+
+    const auto& operationsArray = metadata.document["metadata"]["test"]["operations"].GetArray();
+    if (operationsArray.Empty()) {
+        return result;
+    }
+    for (auto& operationArray : operationsArray) {
+        assert(operationArray.Size() >= 1u);
+
+        if (operationArray[0].GetString() == waitOp) {
+            // wait
+            result.emplace_back([](TestContext& ctx) {
+                try {
+                    ctx.getFrontend().render(ctx.getMap());
+                    return true;
+                } catch (const std::exception&) {
+                    return false;
+                }
+            });
+        } else if (operationArray[0].GetString() == sleepOp) {
+            // sleep
+            mbgl::Duration duration = mbgl::Seconds(3);
+            if (operationArray.Size() >= 2u) {
+                duration = mbgl::Milliseconds(operationArray[1].GetUint());
+            }
+            result.emplace_back([duration](TestContext&) {
+                mbgl::util::Timer sleepTimer;
+                bool sleeping = true;
+
+                sleepTimer.start(duration, mbgl::Duration::zero(), [&]() { sleeping = false; });
+
+                while (sleeping) {
+                    mbgl::util::RunLoop::Get()->runOnce();
+                }
+                return true;
+            });
+
+        } else if (operationArray[0].GetString() == addImageOp || operationArray[0].GetString() == updateImageOp) {
+            // addImage | updateImage
+            assert(operationArray.Size() >= 3u);
+
+            float pixelRatio = 1.0f;
+            bool sdf = false;
+
+            if (operationArray.Size() == 4u) {
+                assert(operationArray[3].IsObject());
+                const auto& imageOptions = operationArray[3].GetObject();
+                if (imageOptions.HasMember("pixelRatio")) {
+                    pixelRatio = imageOptions["pixelRatio"].GetFloat();
+                }
+                if (imageOptions.HasMember("sdf")) {
+                    sdf = imageOptions["sdf"].GetBool();
+                }
+            }
+
+            std::string imageName = operationArray[1].GetString();
+            imageName.erase(std::remove(imageName.begin(), imageName.end(), '"'), imageName.end());
+
+            std::string imagePath = operationArray[2].GetString();
+            imagePath.erase(std::remove(imagePath.begin(), imagePath.end(), '"'), imagePath.end());
+
+            const mbgl::filesystem::path filePath = (mbgl::filesystem::path(manifest.getAssetPath()) / imagePath);
+
+            result.emplace_back([filePath = filePath.string(), imageName, sdf, pixelRatio](TestContext& ctx) {
+                mbgl::optional<std::string> maybeImage = mbgl::util::readFile(filePath);
+                if (!maybeImage) {
+                    ctx.getMetadata().errorMessage = std::string("Failed to load expected image ") + filePath;
+                    return false;
+                }
+
+                ctx.getMap().getStyle().addImage(
+                    std::make_unique<mbgl::style::Image>(imageName, mbgl::decodeImage(*maybeImage), pixelRatio, sdf));
+                return true;
+            });
+
+        } else if (operationArray[0].GetString() == removeImageOp) {
+            // removeImage
+            assert(operationArray.Size() >= 2u);
+            assert(operationArray[1].IsString());
+
+            std::string imageName{operationArray[1].GetString(), operationArray[1].GetStringLength()};
+            result.emplace_back([imageName](TestContext& ctx) {
+                ctx.getMap().getStyle().removeImage(imageName);
+                return true;
+            });
+        } else if (operationArray[0].GetString() == setStyleOp) {
+            // setStyle
+            assert(operationArray.Size() >= 2u);
+            std::string json;
+            if (operationArray[1].IsString()) {
+                std::string stylePath = manifest.localizeURL(operationArray[1].GetString());
+                auto maybeStyle = readJson(stylePath);
+                if (maybeStyle.is<mbgl::JSDocument>()) {
+                    auto& style = maybeStyle.get<mbgl::JSDocument>();
+                    manifest.localizeStyleURLs(static_cast<mbgl::JSValue&>(style), style);
+                    json = serializeJsonValue(style);
+                }
+            } else {
+                manifest.localizeStyleURLs(operationArray[1], metadata.document);
+                json = serializeJsonValue(operationArray[1]);
+            }
+            result.emplace_back([json](TestContext& ctx) {
+                ctx.getMap().getStyle().loadJSON(json);
+                return true;
+            });
+        } else if (operationArray[0].GetString() == setCenterOp) {
+            // setCenter
+            assert(operationArray.Size() >= 2u);
+            assert(operationArray[1].IsArray());
+
+            const auto& centerArray = operationArray[1].GetArray();
+            assert(centerArray.Size() == 2u);
+            mbgl::LatLng center{centerArray[1].GetDouble(), centerArray[0].GetDouble()};
+            result.emplace_back([center](TestContext& ctx) {
+                ctx.getMap().jumpTo(mbgl::CameraOptions().withCenter(center));
+                return true;
+            });
+        } else if (operationArray[0].GetString() == setZoomOp) {
+            // setZoom
+            assert(operationArray.Size() >= 2u);
+            assert(operationArray[1].IsNumber());
+            double zoom = operationArray[1].GetDouble();
+            result.emplace_back([zoom](TestContext& ctx) {
+                ctx.getMap().jumpTo(mbgl::CameraOptions().withZoom(zoom));
+                return true;
+            });
+        } else if (operationArray[0].GetString() == setBearingOp) {
+            // setBearing
+            assert(operationArray.Size() >= 2u);
+            assert(operationArray[1].IsNumber());
+            double bearing = operationArray[1].GetDouble();
+            result.emplace_back([bearing](TestContext& ctx) {
+                ctx.getMap().jumpTo(mbgl::CameraOptions().withBearing(bearing));
+                return true;
+            });
+        } else if (operationArray[0].GetString() == setPitchOp) {
+            // setPitch
+            assert(operationArray.Size() >= 2u);
+            assert(operationArray[1].IsNumber());
+            double pitch = operationArray[1].GetDouble();
+            result.emplace_back([pitch](TestContext& ctx) {
+                ctx.getMap().jumpTo(mbgl::CameraOptions().withPitch(pitch));
+                return true;
+            });
+        } else if (operationArray[0].GetString() == setFilterOp) {
+            // setFilter
+            assert(operationArray.Size() >= 3u);
+            assert(operationArray[1].IsString());
+
+            std::string layerName{operationArray[1].GetString(), operationArray[1].GetStringLength()};
+            mbgl::style::conversion::Error error;
+            auto converted = mbgl::style::conversion::convert<mbgl::style::Filter>(operationArray[2], error);
+            result.emplace_back([converted, layerName, error](TestContext& ctx) {
+                if (!converted) {
+                    ctx.getMetadata().errorMessage = std::string("Unable to convert filter: ") + error.message;
+                    return false;
+                }
+                auto layer = ctx.getMap().getStyle().getLayer(layerName);
+                if (!layer) {
+                    ctx.getMetadata().errorMessage = std::string("Layer not found: ") + layerName;
+                    return false;
+                }
+                layer->setFilter(std::move(*converted));
+                return true;
+            });
+
+        } else if (operationArray[0].GetString() == setLayerZoomRangeOp) {
+            // setLayerZoomRange
+            assert(operationArray.Size() >= 4u);
+            assert(operationArray[1].IsString());
+            assert(operationArray[2].IsNumber());
+            assert(operationArray[3].IsNumber());
+
+            std::string layerName{operationArray[1].GetString(), operationArray[1].GetStringLength()};
+            float minZoom = operationArray[2].GetFloat();
+            float maxZoom = operationArray[3].GetFloat();
+            result.emplace_back([layerName, minZoom, maxZoom](TestContext& ctx) {
+                auto layer = ctx.getMap().getStyle().getLayer(layerName);
+                if (!layer) {
+                    ctx.getMetadata().errorMessage = std::string("Layer not found: ") + layerName;
+                    return false;
+                }
+                layer->setMinZoom(minZoom);
+                layer->setMaxZoom(maxZoom);
+                return true;
+            });
+        } else if (operationArray[0].GetString() == setLightOp) {
+            // setLight
+            assert(operationArray.Size() >= 2u);
+            assert(operationArray[1].IsObject());
+            mbgl::style::conversion::Error error;
+            auto converted = mbgl::style::conversion::convert<mbgl::style::Light>(operationArray[1], error);
+            if (!converted) {
+                metadata.errorMessage = std::string("Unable to convert light: ") + error.message;
+                return {};
+            }
+            result.emplace_back([impl = converted->impl](TestContext& ctx) {
+                ctx.getMap().getStyle().setLight(std::make_unique<mbgl::style::Light>(impl));
+                return true;
+            });
+        } else if (operationArray[0].GetString() == addLayerOp) {
+            // addLayer
+            assert(operationArray.Size() >= 2u);
+            assert(operationArray[1].IsObject());
+            result.emplace_back([json = serializeJsonValue(operationArray[1])](TestContext& ctx) {
+                mbgl::style::conversion::Error error;
+                auto converted = mbgl::style::conversion::convertJSON<std::unique_ptr<mbgl::style::Layer>>(json, error);
+                if (!converted) {
+                    ctx.getMetadata().errorMessage = std::string("Unable to convert layer: ") + error.message;
+                    return false;
+                }
+                ctx.getMap().getStyle().addLayer(std::move(*converted));
+                return true;
+            });
+        } else if (operationArray[0].GetString() == removeLayerOp) {
+            // removeLayer
+            assert(operationArray.Size() >= 2u);
+            assert(operationArray[1].IsString());
+            std::string layerName = operationArray[1].GetString();
+            result.emplace_back(
+                [layerName](TestContext& ctx) { return bool(ctx.getMap().getStyle().removeLayer(layerName)); });
+        } else if (operationArray[0].GetString() == addSourceOp) {
+            // addSource
+            assert(operationArray.Size() >= 3u);
+            assert(operationArray[1].IsString());
+            assert(operationArray[2].IsObject());
+            std::string sourceName = operationArray[1].GetString();
+
+            manifest.localizeSourceURLs(operationArray[2], metadata.document);
+            result.emplace_back([sourceName, json = serializeJsonValue(operationArray[2])](TestContext& ctx) {
+                mbgl::style::conversion::Error error;
+                auto converted =
+                    mbgl::style::conversion::convertJSON<std::unique_ptr<mbgl::style::Source>>(json, error, sourceName);
+                if (!converted) {
+                    ctx.getMetadata().errorMessage = std::string("Unable to convert source: ") + error.message;
+                    return false;
+                }
+                ctx.getMap().getStyle().addSource(std::move(*converted));
+                return true;
+            });
+        } else if (operationArray[0].GetString() == removeSourceOp) {
+            // removeSource
+            assert(operationArray.Size() >= 2u);
+            assert(operationArray[1].IsString());
+            std::string sourceName = operationArray[1].GetString();
+            result.emplace_back(
+                [sourceName](TestContext& ctx) { return bool(ctx.getMap().getStyle().removeSource(sourceName)); });
+        } else if (operationArray[0].GetString() == setLayoutPropertyOp ||
+                   operationArray[0].GetString() == setPaintPropertyOp) {
+            // set{Paint|Layout}Property
+            assert(operationArray.Size() >= 4u);
+            assert(operationArray[1].IsString());
+            assert(operationArray[2].IsString());
+
+            std::string layerName{operationArray[1].GetString(), operationArray[1].GetStringLength()};
+            std::string propertyName{operationArray[2].GetString(), operationArray[2].GetStringLength()};
+            result.emplace_back(
+                [layerName, propertyName, json = serializeJsonValue(operationArray[3])](TestContext& ctx) {
+                    auto layer = ctx.getMap().getStyle().getLayer(layerName);
+                    if (!layer) {
+                        ctx.getMetadata().errorMessage = std::string("Layer not found: ") + layerName;
+                        return false;
+                    }
+                    mbgl::JSDocument d;
+                    d.Parse(json.c_str(), json.length());
+                    const mbgl::JSValue* propertyValue = &d;
+                    layer->setProperty(propertyName, propertyValue);
+                    return true;
+                });
+        } else if (operationArray[0].GetString() == fileSizeProbeOp) {
+            // probeFileSize
+            assert(operationArray.Size() >= 4u);
+            assert(operationArray[1].IsString());
+            assert(operationArray[2].IsString());
+            assert(operationArray[3].IsNumber());
+
+            std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength());
+            std::string path = std::string(operationArray[2].GetString(), operationArray[2].GetStringLength());
+            assert(!path.empty());
+
+            float tolerance = operationArray[3].GetDouble();
+            mbgl::filesystem::path filePath(path);
+
+            bool compressed = false;
+            if (operationArray.Size() == 5) {
+                assert(operationArray[4].IsString());
+                assert(std::string(operationArray[4].GetString(), operationArray[4].GetStringLength()) == "compressed");
+                compressed = true;
+            }
+
+            if (!filePath.is_absolute()) {
+                filePath = metadata.paths.defaultExpectations() / filePath;
+            }
+            result.emplace_back([filePath, path, mark, tolerance, compressed](TestContext& ctx) {
+                if (!mbgl::filesystem::exists(filePath)) {
+                    ctx.getMetadata().errorMessage = std::string("File not found: ") + path;
+                    return false;
+                }
+                size_t size = 0;
+                if (compressed) {
+                    size = mbgl::util::compress(*mbgl::util::readFile(filePath)).size();
+                } else {
+                    size = mbgl::filesystem::file_size(filePath);
+                }
+
+                ctx.getMetadata().metrics.fileSize.emplace(std::piecewise_construct,
+                                                           std::forward_as_tuple(std::move(mark)),
+                                                           std::forward_as_tuple(std::move(path), size, tolerance));
+                return true;
+            });
+        } else if (operationArray[0].GetString() == memoryProbeStartOp) {
+            // probeMemoryStart
+            result.emplace_back([](TestContext&) {
+                assert(!AllocationIndex::isActive());
+                AllocationIndex::setActive(true);
+                return true;
+            });
+        } else if (operationArray[0].GetString() == memoryProbeOp) {
+            // probeMemory
+            assert(AllocationIndex::isActive());
+            assert(operationArray.Size() >= 2u);
+            assert(operationArray[1].IsString());
+            std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength());
+            float tolerance = -1.0f;
+            if (operationArray.Size() >= 3u) {
+                assert(operationArray[2].IsNumber());
+                tolerance = float(operationArray[2].GetDouble());
+            }
+            result.emplace_back([mark, tolerance](TestContext& ctx) {
+                auto emplaced = ctx.getMetadata().metrics.memory.emplace(
+                    std::piecewise_construct,
+                    std::forward_as_tuple(std::move(mark)),
+                    std::forward_as_tuple(AllocationIndex::getAllocatedSizePeak(),
+                                          AllocationIndex::getAllocationsCount()));
+                if (tolerance >= 0.0f) emplaced.first->second.tolerance = tolerance;
+                return true;
+            });
+        } else if (operationArray[0].GetString() == memoryProbeEndOp) {
+            // probeMemoryEnd
+            result.emplace_back([](TestContext&) {
+                assert(AllocationIndex::isActive());
+                AllocationIndex::setActive(false);
+                AllocationIndex::reset();
+                return true;
+            });
+        } else if (operationArray[0].GetString() == networkProbeStartOp) {
+            // probeNetworkStart
+            result.emplace_back([](TestContext&) {
+                assert(!mbgl::ProxyFileSource::isTrackingActive());
+                mbgl::ProxyFileSource::setTrackingActive(true);
+                return true;
+            });
+        } else if (operationArray[0].GetString() == networkProbeOp) {
+            // probeNetwork
+            assert(operationArray.Size() >= 2u);
+            assert(operationArray[1].IsString());
+            std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength());
+            result.emplace_back([mark](TestContext& ctx) {
+                assert(mbgl::ProxyFileSource::isTrackingActive());
+                ctx.getMetadata().metrics.network.emplace(
+                    std::piecewise_construct,
+                    std::forward_as_tuple(std::move(mark)),
+                    std::forward_as_tuple(mbgl::ProxyFileSource::getRequestCount(),
+                                          mbgl::ProxyFileSource::getTransferredSize()));
+                return true;
+            });
+        } else if (operationArray[0].GetString() == networkProbeEndOp) {
+            // probeNetworkEnd
+            result.emplace_back([](TestContext&) {
+                assert(mbgl::ProxyFileSource::isTrackingActive());
+                mbgl::ProxyFileSource::setTrackingActive(false);
+                return true;
+            });
+        } else if (operationArray[0].GetString() == setFeatureStateOp) {
+            // setFeatureState
+            assert(operationArray.Size() >= 3u);
+            assert(operationArray[1].IsObject());
+            assert(operationArray[2].IsObject());
+
+            using namespace mbgl;
+            using namespace mbgl::style::conversion;
+
+            std::string sourceID;
+            mbgl::optional<std::string> sourceLayer;
+            std::string featureID;
+            std::string stateKey;
+            Value stateValue;
+            bool valueParsed = false;
+            FeatureState parsedState;
+
+            const auto& featureOptions = operationArray[1].GetObject();
+            if (featureOptions.HasMember("source")) {
+                sourceID = featureOptions["source"].GetString();
+            }
+            if (featureOptions.HasMember("sourceLayer")) {
+                sourceLayer = {featureOptions["sourceLayer"].GetString()};
+            }
+            if (featureOptions.HasMember("id")) {
+                if (featureOptions["id"].IsString()) {
+                    featureID = featureOptions["id"].GetString();
+                } else if (featureOptions["id"].IsNumber()) {
+                    featureID = mbgl::util::toString(featureOptions["id"].GetUint64());
+                }
+            }
+            const JSValue* state = &operationArray[2];
+
+            const std::function<optional<Error>(const std::string&, const Convertible&)> convertFn =
+                [&](const std::string& k, const Convertible& v) -> optional<Error> {
+                optional<Value> value = toValue(v);
+                if (value) {
+                    stateValue = std::move(*value);
+                    valueParsed = true;
+                } else if (isArray(v)) {
+                    std::vector<Value> array;
+                    std::size_t length = arrayLength(v);
+                    array.reserve(length);
+                    for (size_t i = 0; i < length; ++i) {
+                        optional<Value> arrayVal = toValue(arrayMember(v, i));
+                        if (arrayVal) {
+                            array.emplace_back(*arrayVal);
+                        }
+                    }
+                    std::unordered_map<std::string, Value> result;
+                    result[k] = std::move(array);
+                    stateValue = std::move(result);
+                    valueParsed = true;
+                    return nullopt;
+
+                } else if (isObject(v)) {
+                    eachMember(v, convertFn);
+                }
+
+                if (!valueParsed) {
+                    metadata.errorMessage = std::string("Could not get feature state value, state key: ") + k;
+                    return nullopt;
+                }
+                stateKey = k;
+                parsedState[stateKey] = stateValue;
+                return nullopt;
+            };
+
+            eachMember(state, convertFn);
+            result.emplace_back([sourceID, sourceLayer, featureID, parsedState](TestContext& ctx) {
+                auto& frontend = ctx.getFrontend();
+                try {
+                    frontend.render(ctx.getMap());
+                } catch (const std::exception&) {
+                    return false;
+                }
+                frontend.getRenderer()->setFeatureState(sourceID, sourceLayer, featureID, parsedState);
+                return true;
+            });
+        } else if (operationArray[0].GetString() == getFeatureStateOp) {
+            // getFeatureState
+            assert(operationArray.Size() >= 2u);
+            assert(operationArray[1].IsObject());
+
+            std::string sourceID;
+            mbgl::optional<std::string> sourceLayer;
+            std::string featureID;
+
+            const auto& featureOptions = operationArray[1].GetObject();
+            if (featureOptions.HasMember("source")) {
+                sourceID = featureOptions["source"].GetString();
+            }
+            if (featureOptions.HasMember("sourceLayer")) {
+                sourceLayer = {featureOptions["sourceLayer"].GetString()};
+            }
+            if (featureOptions.HasMember("id")) {
+                if (featureOptions["id"].IsString()) {
+                    featureID = featureOptions["id"].GetString();
+                } else if (featureOptions["id"].IsNumber()) {
+                    featureID = mbgl::util::toString(featureOptions["id"].GetUint64());
+                }
+            }
+            result.emplace_back([sourceID, sourceLayer, featureID](TestContext& ctx) {
+                auto& frontend = ctx.getFrontend();
+                try {
+                    frontend.render(ctx.getMap());
+                } catch (const std::exception&) {
+                    return false;
+                }
+                mbgl::FeatureState state;
+                frontend.getRenderer()->getFeatureState(state, sourceID, sourceLayer, featureID);
+                return true;
+            });
+        } else if (operationArray[0].GetString() == removeFeatureStateOp) {
+            // removeFeatureState
+            assert(operationArray.Size() >= 2u);
+            assert(operationArray[1].IsObject());
+
+            std::string sourceID;
+            mbgl::optional<std::string> sourceLayer;
+            std::string featureID;
+            mbgl::optional<std::string> stateKey;
+
+            const auto& featureOptions = operationArray[1].GetObject();
+            if (featureOptions.HasMember("source")) {
+                sourceID = featureOptions["source"].GetString();
+            }
+            if (featureOptions.HasMember("sourceLayer")) {
+                sourceLayer = {featureOptions["sourceLayer"].GetString()};
+            }
+            if (featureOptions.HasMember("id")) {
+                if (featureOptions["id"].IsString()) {
+                    featureID = featureOptions["id"].GetString();
+                } else if (featureOptions["id"].IsNumber()) {
+                    featureID = mbgl::util::toString(featureOptions["id"].GetUint64());
+                }
+            }
+
+            if (operationArray.Size() >= 3u) {
+                assert(operationArray[2].IsString());
+                stateKey = {operationArray[2].GetString()};
+            }
+
+            result.emplace_back([sourceID, sourceLayer, featureID, stateKey](TestContext& ctx) {
+                auto& frontend = ctx.getFrontend();
+                try {
+                    frontend.render(ctx.getMap());
+                } catch (const std::exception&) {
+                    return false;
+                }
+                frontend.getRenderer()->removeFeatureState(sourceID, sourceLayer, featureID, stateKey);
+                return true;
+            });
+        } else if (operationArray[0].GetString() == panGestureOp) {
+            // benchmarkPanGesture
+            assert(operationArray.Size() >= 4u);
+            assert(operationArray[1].IsString()); // identifier
+            assert(operationArray[2].IsNumber()); // duration
+            assert(operationArray[3].IsArray());  // start [lat, lng, zoom]
+            assert(operationArray[4].IsArray());  // end [lat, lng, zoom]
+
+            if (metadata.mapMode != mbgl::MapMode::Continuous) {
+                metadata.errorMessage = "Map mode must be Continous for " + panGestureOp + " operation";
+                return {};
+            }
+
+            std::string mark = operationArray[1].GetString();
+            int duration = operationArray[2].GetFloat();
+            mbgl::LatLng startPos, endPos;
+            double startZoom, endZoom;
+
+            auto parsePosition = [](auto arr) -> std::tuple<mbgl::LatLng, double> {
+                assert(arr.Size() >= 3);
+                return {{arr[1].GetDouble(), arr[0].GetDouble()}, arr[2].GetDouble()};
+            };
+
+            std::tie(startPos, startZoom) = parsePosition(operationArray[3].GetArray());
+            std::tie(endPos, endZoom) = parsePosition(operationArray[4].GetArray());
+
+            result.emplace_back([mark, duration, startPos, endPos, startZoom, endZoom](TestContext& ctx) {
+                auto& map = ctx.getMap();
+                auto& observer = ctx.getObserver();
+                auto& frontend = ctx.getFrontend();
+                std::vector<float> samples;
+                // Jump to the starting point of the segment and make sure there's something to render
+                map.jumpTo(mbgl::CameraOptions().withCenter(startPos).withZoom(startZoom));
+
+                observer.reset();
+                while (!observer.finishRenderingMap) {
+                    frontend.renderOnce(map);
+                }
+
+                if (observer.mapLoadFailure) return false;
+
+                size_t frames = 0;
+                float totalTime = 0.0;
+                bool transitionFinished = false;
+
+                mbgl::AnimationOptions animationOptions(mbgl::Milliseconds(duration * 1000));
+                animationOptions.minZoom = mbgl::util::min(startZoom, endZoom);
+                animationOptions.transitionFinishFn = [&]() { transitionFinished = true; };
+
+                map.flyTo(mbgl::CameraOptions().withCenter(endPos).withZoom(endZoom), animationOptions);
+
+                while (!transitionFinished) {
+                    frames++;
+                    frontend.renderOnce(map);
+                    float frameTime = (float)frontend.getFrameTime();
+                    totalTime += frameTime;
+
+                    samples.push_back(frameTime);
+                }
+
+                float averageFps = totalTime > 0.0 ? frames / totalTime : 0.0;
+                float minFrameTime = 0.0;
+
+                // Use 1% of the longest frames to compute the minimum fps
+                std::sort(samples.begin(), samples.end());
+
+                int sampleCount = mbgl::util::max(1, (int)samples.size() / 100);
+                for (auto it = samples.rbegin(); it != samples.rbegin() + sampleCount; it++) minFrameTime += *it;
+
+                float minOnePcFps = sampleCount / minFrameTime;
+
+                ctx.getMetadata().metrics.fps.insert({mark, {averageFps, minOnePcFps, 0.0f}});
+                return true;
+            });
+        } else if (operationArray[0].GetString() == gfxProbeStartOp) {
+            // probeGFXStart
+            result.emplace_back([](TestContext& ctx) {
+                assert(!ctx.gfxProbeActive);
+                ctx.gfxProbeActive = true;
+                ctx.baselineGfxProbe = ctx.activeGfxProbe;
+                return true;
+            });
+        } else if (operationArray[0].GetString() == gfxProbeEndOp) {
+            // probeGFXEnd
+            result.emplace_back([](TestContext& ctx) {
+                assert(ctx.gfxProbeActive);
+                ctx.gfxProbeActive = false;
+                return true;
+            });
+        } else if (operationArray[0].GetString() == gfxProbeOp) {
+            // probeGFX
+            assert(operationArray.Size() >= 2u);
+            assert(operationArray[1].IsString());
+
+            std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength());
+            result.emplace_back([mark](TestContext& ctx) {
+                auto& frontend = ctx.getFrontend();
+                // Render the map and fetch rendering stats
+                try {
+                    mbgl::gfx::RenderingStats stats = frontend.render(ctx.getMap()).stats;
+                    ctx.activeGfxProbe = GfxProbe(stats, ctx.activeGfxProbe);
+                } catch (const std::exception&) {
+                    return false;
+                }
+                // Compare memory allocations to the baseline probe
+                GfxProbe metricProbe = ctx.activeGfxProbe;
+                metricProbe.memIndexBuffers.peak -= ctx.baselineGfxProbe.memIndexBuffers.peak;
+                metricProbe.memVertexBuffers.peak -= ctx.baselineGfxProbe.memVertexBuffers.peak;
+                metricProbe.memTextures.peak -= ctx.baselineGfxProbe.memTextures.peak;
+                ctx.getMetadata().metrics.gfx.insert({mark, metricProbe});
+                return true;
+            });
+        } else {
+            metadata.errorMessage = std::string("Unsupported operation: ") + operationArray[0].GetString();
+            return {};
+        }
+    }
+
+    return result;
+}
+
 // https://stackoverflow.com/questions/7053538/how-do-i-encode-a-string-to-base64-using-only-boost
 std::string encodeBase64(const std::string& data) {
     using namespace boost::archive::iterators;
diff --git a/render-test/parser.hpp b/render-test/parser.hpp
index b7b4e9201d..905118b401 100644
--- a/render-test/parser.hpp
+++ b/render-test/parser.hpp
@@ -25,8 +25,45 @@ std::vector<std::string> readExpectedJSONEntries(const mbgl::filesystem::path& b
 TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path);
 
 TestMetadata parseTestMetadata(const TestPaths& paths, const Manifest& manifest);
+TestOperations parseTestOperations(TestMetadata& metadata, const Manifest& manifest);
 
 std::string createResultPage(const TestStatistics&, const std::vector<TestMetadata>&, bool shuffle, uint32_t seed);
 
 std::string toJSON(const mbgl::Value& value, unsigned indent, bool singleLine);
 std::string toJSON(const std::vector<mbgl::Feature>& features, unsigned indent, bool singleLine);
+
+namespace TestOperationNames {
+extern const std::string waitOp;
+extern const std::string sleepOp;
+extern const std::string addImageOp;
+extern const std::string updateImageOp;
+extern const std::string removeImageOp;
+extern const std::string setStyleOp;
+extern const std::string setCenterOp;
+extern const std::string setZoomOp;
+extern const std::string setBearingOp;
+extern const std::string setPitchOp;
+extern const std::string setFilterOp;
+extern const std::string setLayerZoomRangeOp;
+extern const std::string setLightOp;
+extern const std::string addLayerOp;
+extern const std::string removeLayerOp;
+extern const std::string addSourceOp;
+extern const std::string removeSourceOp;
+extern const std::string setPaintPropertyOp;
+extern const std::string setLayoutPropertyOp;
+extern const std::string fileSizeProbeOp;
+extern const std::string memoryProbeOp;
+extern const std::string memoryProbeStartOp;
+extern const std::string memoryProbeEndOp;
+extern const std::string networkProbeOp;
+extern const std::string networkProbeStartOp;
+extern const std::string networkProbeEndOp;
+extern const std::string setFeatureStateOp;
+extern const std::string getFeatureStateOp;
+extern const std::string removeFeatureStateOp;
+extern const std::string panGestureOp;
+extern const std::string gfxProbeOp;
+extern const std::string gfxProbeStartOp;
+extern const std::string gfxProbeEndOp;
+} // namespace TestOperationNames
\ No newline at end of file
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index 4037d42e45..8f47977f31 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -2,22 +2,16 @@
 #include <mbgl/map/map_observer.hpp>
 #include <mbgl/renderer/renderer.hpp>
 #include <mbgl/renderer/renderer_observer.hpp>
-#include <mbgl/style/conversion/filter.hpp>
-#include <mbgl/style/conversion/layer.hpp>
-#include <mbgl/style/conversion/light.hpp>
-#include <mbgl/style/conversion/source.hpp>
 #include <mbgl/style/image.hpp>
 #include <mbgl/style/layer.hpp>
 #include <mbgl/style/light.hpp>
 #include <mbgl/style/rapidjson_conversion.hpp>
 #include <mbgl/style/style.hpp>
 #include <mbgl/util/chrono.hpp>
-#include <mbgl/util/compression.hpp>
 #include <mbgl/util/image.hpp>
 #include <mbgl/util/io.hpp>
 #include <mbgl/util/run_loop.hpp>
 #include <mbgl/util/string.hpp>
-#include <mbgl/util/timer.hpp>
 
 #include <mapbox/pixelmatch.hpp>
 
@@ -34,42 +28,7 @@
 #include <sstream>
 
 using namespace mbgl;
-
-namespace {
-static const std::string waitOp("wait");
-static const std::string sleepOp("sleep");
-static const std::string addImageOp("addImage");
-static const std::string updateImageOp("updateImage");
-static const std::string removeImageOp("removeImage");
-static const std::string setStyleOp("setStyle");
-static const std::string setCenterOp("setCenter");
-static const std::string setZoomOp("setZoom");
-static const std::string setBearingOp("setBearing");
-static const std::string setPitchOp("setPitch");
-static const std::string setFilterOp("setFilter");
-static const std::string setLayerZoomRangeOp("setLayerZoomRange");
-static const std::string setLightOp("setLight");
-static const std::string addLayerOp("addLayer");
-static const std::string removeLayerOp("removeLayer");
-static const std::string addSourceOp("addSource");
-static const std::string removeSourceOp("removeSource");
-static const std::string setPaintPropertyOp("setPaintProperty");
-static const std::string setLayoutPropertyOp("setLayoutProperty");
-static const std::string fileSizeProbeOp("probeFileSize");
-static const std::string memoryProbeOp("probeMemory");
-static const std::string memoryProbeStartOp("probeMemoryStart");
-static const std::string memoryProbeEndOp("probeMemoryEnd");
-static const std::string networkProbeOp("probeNetwork");
-static const std::string networkProbeStartOp("probeNetworkStart");
-static const std::string networkProbeEndOp("probeNetworkEnd");
-static const std::string setFeatureStateOp("setFeatureState");
-static const std::string getFeatureStateOp("getFeatureState");
-static const std::string removeFeatureStateOp("removeFeatureState");
-static const std::string panGestureOp("panGesture");
-static const std::string gfxProbeOp("probeGFX");
-static const std::string gfxProbeStartOp("probeGFXStart");
-static const std::string gfxProbeEndOp("probeGFXEnd");
-} // namespace
+using namespace TestOperationNames;
 
 GfxProbe::GfxProbe(const mbgl::gfx::RenderingStats& stats, const GfxProbe& prev)
     : numBuffers(stats.numBuffers),
@@ -80,29 +39,6 @@ GfxProbe::GfxProbe(const mbgl::gfx::RenderingStats& stats, const GfxProbe& prev)
       memVertexBuffers(stats.memVertexBuffers, std::max(stats.memVertexBuffers, prev.memVertexBuffers.peak)),
       memTextures(stats.memTextures, std::max(stats.memTextures, prev.memTextures.peak)) {}
 
-class TestRunnerMapObserver : public MapObserver {
-public:
-    TestRunnerMapObserver() : mapLoadFailure(false), finishRenderingMap(false), idle(false) {}
-
-    void onDidFailLoadingMap(MapLoadError, const std::string&) override { mapLoadFailure = true; }
-
-    void onDidFinishRenderingMap(RenderMode mode) override final {
-        if (!finishRenderingMap) finishRenderingMap = mode == RenderMode::Full;
-    }
-
-    void onDidBecomeIdle() override final { idle = true; }
-
-    void reset() {
-        mapLoadFailure = false;
-        finishRenderingMap = false;
-        idle = false;
-    }
-
-    bool mapLoadFailure;
-    bool finishRenderingMap;
-    bool idle;
-};
-
 // static
 gfx::HeadlessBackend::SwapBehaviour swapBehavior(MapMode mode) {
     return mode == MapMode::Continuous ? gfx::HeadlessBackend::SwapBehaviour::Flush
@@ -580,578 +516,6 @@ bool TestRunner::checkProbingResults(TestMetadata& resultMetadata) {
     return checkResult;
 }
 
-bool TestRunner::runOperations(TestContext& ctx) {
-    TestMetadata& metadata = ctx.getMetadata();
-    if (!metadata.document.HasMember("metadata") || !metadata.document["metadata"].HasMember("test") ||
-        !metadata.document["metadata"]["test"].HasMember("operations")) {
-        return true;
-    }
-    assert(metadata.document["metadata"]["test"]["operations"].IsArray());
-
-    const auto& operationsArray = metadata.document["metadata"]["test"]["operations"].GetArray();
-    if (operationsArray.Empty()) {
-        return true;
-    }
-
-    const auto& operationIt = operationsArray.Begin();
-    assert(operationIt->IsArray());
-
-    const auto& operationArray = operationIt->GetArray();
-    assert(operationArray.Size() >= 1u);
-
-    HeadlessFrontend& frontend = ctx.getFrontend();
-    Map& map = ctx.getMap();
-    TestRunnerMapObserver& observer = ctx.getObserver();
-
-    if (operationArray[0].GetString() == waitOp) {
-        // wait
-        try {
-            frontend.render(map);
-        } catch (const std::exception&) {
-            return false;
-        }
-    } else if (operationArray[0].GetString() == sleepOp) {
-        // sleep
-        mbgl::util::Timer sleepTimer;
-        bool sleeping = true;
-
-        mbgl::Duration duration = mbgl::Seconds(3);
-        if (operationArray.Size() >= 2u) {
-            duration = mbgl::Milliseconds(operationArray[1].GetUint());
-        }
-
-        sleepTimer.start(duration, mbgl::Duration::zero(), [&]() {
-            sleeping = false;
-        });
-
-        while (sleeping) {
-            mbgl::util::RunLoop::Get()->runOnce();
-        }
-    } else if (operationArray[0].GetString() == addImageOp || operationArray[0].GetString() == updateImageOp) {
-        // addImage | updateImage
-        assert(operationArray.Size() >= 3u);
-
-        float pixelRatio = 1.0f;
-        bool sdf = false;
-
-        if (operationArray.Size() == 4u) {
-            assert(operationArray[3].IsObject());
-            const auto& imageOptions = operationArray[3].GetObject();
-            if (imageOptions.HasMember("pixelRatio")) {
-                pixelRatio = imageOptions["pixelRatio"].GetFloat();
-            }
-            if (imageOptions.HasMember("sdf")) {
-                sdf = imageOptions["sdf"].GetBool();
-            }
-        }
-
-        std::string imageName = operationArray[1].GetString();
-        imageName.erase(std::remove(imageName.begin(), imageName.end(), '"'), imageName.end());
-
-        std::string imagePath = operationArray[2].GetString();
-        imagePath.erase(std::remove(imagePath.begin(), imagePath.end(), '"'), imagePath.end());
-
-        const mbgl::filesystem::path filePath = (mbgl::filesystem::path(manifest.getAssetPath()) / imagePath);
-
-        mbgl::optional<std::string> maybeImage = mbgl::util::readFile(filePath.string());
-        if (!maybeImage) {
-            metadata.errorMessage = std::string("Failed to load expected image ") + filePath.string();
-            return false;
-        }
-
-        map.getStyle().addImage(std::make_unique<mbgl::style::Image>(imageName, mbgl::decodeImage(*maybeImage), pixelRatio, sdf));
-    } else if (operationArray[0].GetString() == removeImageOp) {
-        // removeImage
-        assert(operationArray.Size() >= 2u);
-        assert(operationArray[1].IsString());
-
-        const std::string imageName { operationArray[1].GetString(), operationArray[1].GetStringLength() };
-        map.getStyle().removeImage(imageName);
-    } else if (operationArray[0].GetString() == setStyleOp) {
-        // setStyle
-        assert(operationArray.Size() >= 2u);
-        if (operationArray[1].IsString()) {
-            std::string stylePath = manifest.localizeURL(operationArray[1].GetString());
-            auto maybeStyle = readJson(stylePath);
-            if (maybeStyle.is<mbgl::JSDocument>()) {
-                auto& style = maybeStyle.get<mbgl::JSDocument>();
-                manifest.localizeStyleURLs((mbgl::JSValue&)style, style);
-                map.getStyle().loadJSON(serializeJsonValue(style));
-            }
-        } else {
-            manifest.localizeStyleURLs(operationArray[1], metadata.document);
-            map.getStyle().loadJSON(serializeJsonValue(operationArray[1]));
-        }
-    } else if (operationArray[0].GetString() == setCenterOp) {
-        // setCenter
-        assert(operationArray.Size() >= 2u);
-        assert(operationArray[1].IsArray());
-
-        const auto& centerArray = operationArray[1].GetArray();
-        assert(centerArray.Size() == 2u);
-
-        map.jumpTo(mbgl::CameraOptions().withCenter(mbgl::LatLng(centerArray[1].GetDouble(), centerArray[0].GetDouble())));
-    } else if (operationArray[0].GetString() == setZoomOp) {
-        // setZoom
-        assert(operationArray.Size() >= 2u);
-        assert(operationArray[1].IsNumber());
-        map.jumpTo(mbgl::CameraOptions().withZoom(operationArray[1].GetDouble()));
-    } else if (operationArray[0].GetString() == setBearingOp) {
-        // setBearing
-        assert(operationArray.Size() >= 2u);
-        assert(operationArray[1].IsNumber());
-        map.jumpTo(mbgl::CameraOptions().withBearing(operationArray[1].GetDouble()));
-    } else if (operationArray[0].GetString() == setPitchOp) {
-        // setPitch
-        assert(operationArray.Size() >= 2u);
-        assert(operationArray[1].IsNumber());
-        map.jumpTo(mbgl::CameraOptions().withPitch(operationArray[1].GetDouble()));
-    } else if (operationArray[0].GetString() == setFilterOp) {
-        // setFilter
-        assert(operationArray.Size() >= 3u);
-        assert(operationArray[1].IsString());
-
-        const std::string layerName { operationArray[1].GetString(), operationArray[1].GetStringLength() };
-
-        mbgl::style::conversion::Error error;
-        auto converted = mbgl::style::conversion::convert<mbgl::style::Filter>(operationArray[2], error);
-        if (!converted) {
-            metadata.errorMessage = std::string("Unable to convert filter: ")  + error.message;
-            return false;
-        } else {
-            auto layer = map.getStyle().getLayer(layerName);
-            if (!layer) {
-                metadata.errorMessage = std::string("Layer not found: ")  + layerName;
-                return false;
-            } else {
-                layer->setFilter(std::move(*converted));
-            }
-        }
-    } else if (operationArray[0].GetString() == setLayerZoomRangeOp) {
-        // setLayerZoomRange
-        assert(operationArray.Size() >= 4u);
-        assert(operationArray[1].IsString());
-        assert(operationArray[2].IsNumber());
-        assert(operationArray[3].IsNumber());
-
-        const std::string layerName { operationArray[1].GetString(), operationArray[1].GetStringLength() };
-        auto layer = map.getStyle().getLayer(layerName);
-        if (!layer) {
-            metadata.errorMessage = std::string("Layer not found: ")  + layerName;
-            return false;
-        } else {
-            layer->setMinZoom(operationArray[2].GetFloat());
-            layer->setMaxZoom(operationArray[3].GetFloat());
-        }
-    } else if (operationArray[0].GetString() == setLightOp) {
-        // setLight
-        assert(operationArray.Size() >= 2u);
-        assert(operationArray[1].IsObject());
-
-        mbgl::style::conversion::Error error;
-        auto converted = mbgl::style::conversion::convert<mbgl::style::Light>(operationArray[1], error);
-        if (!converted) {
-            metadata.errorMessage = std::string("Unable to convert light: ")  + error.message;
-            return false;
-        } else {
-            map.getStyle().setLight(std::make_unique<mbgl::style::Light>(std::move(*converted)));
-        }
-    } else if (operationArray[0].GetString() == addLayerOp) {
-        // addLayer
-        assert(operationArray.Size() >= 2u);
-        assert(operationArray[1].IsObject());
-
-        mbgl::style::conversion::Error error;
-        auto converted = mbgl::style::conversion::convert<std::unique_ptr<mbgl::style::Layer>>(operationArray[1], error);
-        if (!converted) {
-            metadata.errorMessage = std::string("Unable to convert layer: ")  + error.message;
-            return false;
-        } else {
-            map.getStyle().addLayer(std::move(*converted));
-        }
-    } else if (operationArray[0].GetString() == removeLayerOp) {
-        // removeLayer
-        assert(operationArray.Size() >= 2u);
-        assert(operationArray[1].IsString());
-        map.getStyle().removeLayer(operationArray[1].GetString());
-    } else if (operationArray[0].GetString() == addSourceOp) {
-        // addSource
-        assert(operationArray.Size() >= 3u);
-        assert(operationArray[1].IsString());
-        assert(operationArray[2].IsObject());
-
-        manifest.localizeSourceURLs(operationArray[2], metadata.document);
-
-        mbgl::style::conversion::Error error;
-        auto converted = mbgl::style::conversion::convert<std::unique_ptr<mbgl::style::Source>>(operationArray[2], error, operationArray[1].GetString());
-        if (!converted) {
-            metadata.errorMessage = std::string("Unable to convert source: ")  + error.message;
-            return false;
-        } else {
-            map.getStyle().addSource(std::move(*converted));
-        }
-    } else if (operationArray[0].GetString() == removeSourceOp) {
-        // removeSource
-        assert(operationArray.Size() >= 2u);
-        assert(operationArray[1].IsString());
-        map.getStyle().removeSource(operationArray[1].GetString());
-    } else if (operationArray[0].GetString() == setLayoutPropertyOp ||
-               operationArray[0].GetString() == setPaintPropertyOp) {
-        // set{Paint|Layout}Property
-        assert(operationArray.Size() >= 4u);
-        assert(operationArray[1].IsString());
-        assert(operationArray[2].IsString());
-
-        const std::string layerName { operationArray[1].GetString(), operationArray[1].GetStringLength() };
-        const std::string propertyName { operationArray[2].GetString(), operationArray[2].GetStringLength() };
-
-        auto layer = map.getStyle().getLayer(layerName);
-        if (!layer) {
-            metadata.errorMessage = std::string("Layer not found: ")  + layerName;
-            return false;
-        } else {
-            const mbgl::JSValue* propertyValue = &operationArray[3];
-            layer->setProperty(propertyName, propertyValue);
-        }
-    } else if (operationArray[0].GetString() == fileSizeProbeOp) {
-        // probeFileSize
-        assert(operationArray.Size() >= 4u);
-        assert(operationArray[1].IsString());
-        assert(operationArray[2].IsString());
-        assert(operationArray[3].IsNumber());
-
-        std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength());
-        std::string path = std::string(operationArray[2].GetString(), operationArray[2].GetStringLength());
-        assert(!path.empty());
-
-        float tolerance = operationArray[3].GetDouble();
-        mbgl::filesystem::path filePath(path);
-
-        bool compressed = false;
-        if (operationArray.Size() == 5) {
-            assert(operationArray[4].IsString());
-            assert(std::string(operationArray[4].GetString(), operationArray[4].GetStringLength()) == "compressed");
-            compressed = true;
-        }
-
-        if (!filePath.is_absolute()) {
-            filePath = metadata.paths.defaultExpectations() / filePath;
-        }
-
-        if (mbgl::filesystem::exists(filePath)) {
-            size_t size = 0;
-            if (compressed) {
-                size = mbgl::util::compress(*mbgl::util::readFile(filePath)).size();
-            } else {
-                size = mbgl::filesystem::file_size(filePath);
-            }
-
-            metadata.metrics.fileSize.emplace(std::piecewise_construct,
-                                              std::forward_as_tuple(std::move(mark)),
-                                              std::forward_as_tuple(std::move(path), size, tolerance));
-        } else {
-            metadata.errorMessage = std::string("File not found: ") + path;
-            return false;
-        }
-    } else if (operationArray[0].GetString() == memoryProbeStartOp) {
-        // probeMemoryStart
-        assert(!AllocationIndex::isActive());
-        AllocationIndex::setActive(true);
-    } else if (operationArray[0].GetString() == memoryProbeOp) {
-        // probeMemory
-        assert(AllocationIndex::isActive());
-        assert(operationArray.Size() >= 2u);
-        assert(operationArray[1].IsString());
-        std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength());
-
-        auto emplaced = metadata.metrics.memory.emplace(
-            std::piecewise_construct,
-            std::forward_as_tuple(std::move(mark)),
-            std::forward_as_tuple(AllocationIndex::getAllocatedSizePeak(), AllocationIndex::getAllocationsCount()));
-        assert(emplaced.second);
-        if (operationArray.Size() >= 3u) {
-            assert(operationArray[2].IsNumber());
-            emplaced.first->second.tolerance = float(operationArray[2].GetDouble());
-        }
-    } else if (operationArray[0].GetString() == memoryProbeEndOp) {
-        // probeMemoryEnd
-        assert(AllocationIndex::isActive());
-        AllocationIndex::setActive(false);
-        AllocationIndex::reset();
-    } else if (operationArray[0].GetString() == networkProbeStartOp) {
-        // probeNetworkStart
-        assert(!ProxyFileSource::isTrackingActive());
-        ProxyFileSource::setTrackingActive(true);
-    } else if (operationArray[0].GetString() == networkProbeOp) {
-        // probeNetwork
-        assert(ProxyFileSource::isTrackingActive());
-        assert(operationArray.Size() >= 2u);
-        assert(operationArray[1].IsString());
-        std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength());
-
-        metadata.metrics.network.emplace(
-            std::piecewise_construct,
-            std::forward_as_tuple(std::move(mark)),
-            std::forward_as_tuple(ProxyFileSource::getRequestCount(), ProxyFileSource::getTransferredSize()));
-    } else if (operationArray[0].GetString() == networkProbeEndOp) {
-        // probeNetworkEnd
-        assert(ProxyFileSource::isTrackingActive());
-        ProxyFileSource::setTrackingActive(false);
-    } else if (operationArray[0].GetString() == setFeatureStateOp) {
-        // setFeatureState
-        assert(operationArray.Size() >= 3u);
-        assert(operationArray[1].IsObject());
-        assert(operationArray[2].IsObject());
-
-        using namespace mbgl;
-        using namespace mbgl::style::conversion;
-
-        std::string sourceID;
-        mbgl::optional<std::string> sourceLayer;
-        std::string featureID;
-        std::string stateKey;
-        Value stateValue;
-        bool valueParsed = false;
-        FeatureState parsedState;
-
-        const auto& featureOptions = operationArray[1].GetObject();
-        if (featureOptions.HasMember("source")) {
-            sourceID = featureOptions["source"].GetString();
-        }
-        if (featureOptions.HasMember("sourceLayer")) {
-            sourceLayer = {featureOptions["sourceLayer"].GetString()};
-        }
-        if (featureOptions.HasMember("id")) {
-            if (featureOptions["id"].IsString()) {
-                featureID = featureOptions["id"].GetString();
-            } else if (featureOptions["id"].IsNumber()) {
-                featureID = mbgl::util::toString(featureOptions["id"].GetUint64());
-            }
-        }
-        const JSValue* state = &operationArray[2];
-
-        const std::function<optional<Error>(const std::string&, const Convertible&)> convertFn =
-            [&](const std::string& k, const Convertible& v) -> optional<Error> {
-            optional<Value> value = toValue(v);
-            if (value) {
-                stateValue = std::move(*value);
-                valueParsed = true;
-            } else if (isArray(v)) {
-                std::vector<Value> array;
-                std::size_t length = arrayLength(v);
-                array.reserve(length);
-                for (size_t i = 0; i < length; ++i) {
-                    optional<Value> arrayVal = toValue(arrayMember(v, i));
-                    if (arrayVal) {
-                        array.emplace_back(*arrayVal);
-                    }
-                }
-                std::unordered_map<std::string, Value> result;
-                result[k] = std::move(array);
-                stateValue = std::move(result);
-                valueParsed = true;
-                return nullopt;
-
-            } else if (isObject(v)) {
-                eachMember(v, convertFn);
-            }
-
-            if (!valueParsed) {
-                metadata.errorMessage = std::string("Could not get feature state value, state key: ") + k;
-                return nullopt;
-            }
-            stateKey = k;
-            parsedState[stateKey] = stateValue;
-            return nullopt;
-        };
-
-        eachMember(state, convertFn);
-
-        try {
-            frontend.render(map);
-        } catch (const std::exception&) {
-            return false;
-        }
-        frontend.getRenderer()->setFeatureState(sourceID, sourceLayer, featureID, parsedState);
-    } else if (operationArray[0].GetString() == getFeatureStateOp) {
-        // getFeatureState
-        assert(operationArray.Size() >= 2u);
-        assert(operationArray[1].IsObject());
-
-        std::string sourceID;
-        mbgl::optional<std::string> sourceLayer;
-        std::string featureID;
-
-        const auto& featureOptions = operationArray[1].GetObject();
-        if (featureOptions.HasMember("source")) {
-            sourceID = featureOptions["source"].GetString();
-        }
-        if (featureOptions.HasMember("sourceLayer")) {
-            sourceLayer = {featureOptions["sourceLayer"].GetString()};
-        }
-        if (featureOptions.HasMember("id")) {
-            if (featureOptions["id"].IsString()) {
-                featureID = featureOptions["id"].GetString();
-            } else if (featureOptions["id"].IsNumber()) {
-                featureID = mbgl::util::toString(featureOptions["id"].GetUint64());
-            }
-        }
-
-        try {
-            frontend.render(map);
-        } catch (const std::exception&) {
-            return false;
-        }
-        mbgl::FeatureState state;
-        frontend.getRenderer()->getFeatureState(state, sourceID, sourceLayer, featureID);
-    } else if (operationArray[0].GetString() == removeFeatureStateOp) {
-        // removeFeatureState
-        assert(operationArray.Size() >= 2u);
-        assert(operationArray[1].IsObject());
-
-        std::string sourceID;
-        mbgl::optional<std::string> sourceLayer;
-        std::string featureID;
-        mbgl::optional<std::string> stateKey;
-
-        const auto& featureOptions = operationArray[1].GetObject();
-        if (featureOptions.HasMember("source")) {
-            sourceID = featureOptions["source"].GetString();
-        }
-        if (featureOptions.HasMember("sourceLayer")) {
-            sourceLayer = {featureOptions["sourceLayer"].GetString()};
-        }
-        if (featureOptions.HasMember("id")) {
-            if (featureOptions["id"].IsString()) {
-                featureID = featureOptions["id"].GetString();
-            } else if (featureOptions["id"].IsNumber()) {
-                featureID = mbgl::util::toString(featureOptions["id"].GetUint64());
-            }
-        }
-
-        if (operationArray.Size() >= 3u) {
-            assert(operationArray[2].IsString());
-            stateKey = {operationArray[2].GetString()};
-        }
-
-        try {
-            frontend.render(map);
-        } catch (const std::exception&) {
-            return false;
-        }
-        frontend.getRenderer()->removeFeatureState(sourceID, sourceLayer, featureID, stateKey);
-    } else if (operationArray[0].GetString() == panGestureOp) {
-        // benchmarkPanGesture
-        assert(operationArray.Size() >= 4u);
-        assert(operationArray[1].IsString()); // identifier
-        assert(operationArray[2].IsNumber()); // duration
-        assert(operationArray[3].IsArray());  // start [lat, lng, zoom]
-        assert(operationArray[4].IsArray());  // end [lat, lng, zoom]
-
-        if (metadata.mapMode != mbgl::MapMode::Continuous) {
-            metadata.errorMessage = "Map mode must be Continous for " + panGestureOp + " operation";
-            return false;
-        }
-
-        std::string mark = operationArray[1].GetString();
-        int duration = operationArray[2].GetFloat();
-        LatLng startPos, endPos;
-        double startZoom, endZoom;
-        std::vector<float> samples;
-
-        auto parsePosition = [](auto arr) -> std::tuple<LatLng, double> {
-            assert(arr.Size() >= 3);
-            return {{arr[1].GetDouble(), arr[0].GetDouble()}, arr[2].GetDouble()};
-        };
-
-        std::tie(startPos, startZoom) = parsePosition(operationArray[3].GetArray());
-        std::tie(endPos, endZoom) = parsePosition(operationArray[4].GetArray());
-
-        // Jump to the starting point of the segment and make sure there's something to render
-        map.jumpTo(mbgl::CameraOptions().withCenter(startPos).withZoom(startZoom));
-
-        observer.reset();
-        while (!observer.finishRenderingMap) {
-            frontend.renderOnce(map);
-        }
-
-        if (observer.mapLoadFailure) return false;
-
-        size_t frames = 0;
-        float totalTime = 0.0;
-        bool transitionFinished = false;
-
-        mbgl::AnimationOptions animationOptions(mbgl::Milliseconds(duration * 1000));
-        animationOptions.minZoom = util::min(startZoom, endZoom);
-        animationOptions.transitionFinishFn = [&]() { transitionFinished = true; };
-
-        map.flyTo(mbgl::CameraOptions().withCenter(endPos).withZoom(endZoom), animationOptions);
-
-        while (!transitionFinished) {
-            frames++;
-            frontend.renderOnce(map);
-            float frameTime = (float)frontend.getFrameTime();
-            totalTime += frameTime;
-
-            samples.push_back(frameTime);
-        }
-
-        float averageFps = totalTime > 0.0 ? frames / totalTime : 0.0;
-        float minFrameTime = 0.0;
-
-        // Use 1% of the longest frames to compute the minimum fps
-        std::sort(samples.begin(), samples.end());
-
-        int sampleCount = util::max(1, (int)samples.size() / 100);
-        for (auto it = samples.rbegin(); it != samples.rbegin() + sampleCount; it++) minFrameTime += *it;
-
-        float minOnePcFps = sampleCount / minFrameTime;
-
-        metadata.metrics.fps.insert({std::move(mark), {averageFps, minOnePcFps, 0.0f}});
-
-    } else if (operationArray[0].GetString() == gfxProbeStartOp) {
-        // probeGFXStart
-        assert(!ctx.gfxProbeActive);
-        ctx.gfxProbeActive = true;
-        ctx.baselineGfxProbe = ctx.activeGfxProbe;
-    } else if (operationArray[0].GetString() == gfxProbeEndOp) {
-        // probeGFXEnd
-        assert(ctx.gfxProbeActive);
-        ctx.gfxProbeActive = false;
-    } else if (operationArray[0].GetString() == gfxProbeOp) {
-        // probeGFX
-        assert(operationArray.Size() >= 2u);
-        assert(operationArray[1].IsString());
-
-        std::string mark = std::string(operationArray[1].GetString(), operationArray[1].GetStringLength());
-
-        // Render the map and fetch rendering stats
-        gfx::RenderingStats stats;
-
-        try {
-            stats = frontend.render(map).stats;
-        } catch (const std::exception&) {
-            return false;
-        }
-
-        ctx.activeGfxProbe = GfxProbe(stats, ctx.activeGfxProbe);
-
-        // Compare memory allocations to the baseline probe
-        GfxProbe metricProbe = ctx.activeGfxProbe;
-        metricProbe.memIndexBuffers.peak -= ctx.baselineGfxProbe.memIndexBuffers.peak;
-        metricProbe.memVertexBuffers.peak -= ctx.baselineGfxProbe.memVertexBuffers.peak;
-        metricProbe.memTextures.peak -= ctx.baselineGfxProbe.memTextures.peak;
-        metadata.metrics.gfx.insert({mark, metricProbe});
-
-    } else {
-        metadata.errorMessage = std::string("Unsupported operation: ") + operationArray[0].GetString();
-        return false;
-    }
-
-    operationsArray.Erase(operationIt);
-    return runOperations(ctx);
-}
-
 TestRunner::Impl::Impl(const TestMetadata& metadata)
     : observer(std::make_unique<TestRunnerMapObserver>()),
       frontend(metadata.size, metadata.pixelRatio, swapBehavior(metadata.mapMode)),
@@ -1197,9 +561,9 @@ bool TestRunner::run(TestMetadata& metadata) {
         return false;
     }
 
-    std::string key = mbgl::util::toString(uint32_t(metadata.mapMode))
-        + "/" + mbgl::util::toString(metadata.pixelRatio)
-        + "/" + mbgl::util::toString(uint32_t(metadata.crossSourceCollisions));
+    std::string key = mbgl::util::toString(uint32_t(metadata.mapMode)) + "/" +
+                      mbgl::util::toString(metadata.pixelRatio) + "/" +
+                      mbgl::util::toString(uint32_t(metadata.crossSourceCollisions));
 
     if (maps.find(key) == maps.end()) {
         maps[key] = std::make_unique<TestRunner::Impl>(metadata);
@@ -1212,13 +576,19 @@ bool TestRunner::run(TestMetadata& metadata) {
     frontend.setSize(metadata.size);
     map.setSize(metadata.size);
 
-    map.setProjectionMode(mbgl::ProjectionMode().withAxonometric(metadata.axonometric).withXSkew(metadata.xSkew).withYSkew(metadata.ySkew));
+    map.setProjectionMode(mbgl::ProjectionMode()
+                              .withAxonometric(metadata.axonometric)
+                              .withXSkew(metadata.xSkew)
+                              .withYSkew(metadata.ySkew));
     map.setDebug(metadata.debug);
 
     map.getStyle().loadJSON(serializeJsonValue(metadata.document));
     map.jumpTo(map.getStyle().getDefaultCamera());
 
-    if (!runOperations(ctx)) return false;
+    TestOperations operations = parseTestOperations(metadata, manifest);
+    for (const auto& operation : operations) {
+        if (!operation(ctx)) return false;
+    }
 
     HeadlessFrontend::RenderResult result;
     try {
@@ -1245,6 +615,7 @@ bool TestRunner::run(TestMetadata& metadata) {
     }
 }
 
+namespace {
 using InjectedProbeMap = std::map<std::string, TestOperation>;
 
 bool runInjectedProbe(const std::set<std::string>& probes, TestContext& ctx, const InjectedProbeMap& probeMap) {
@@ -1258,6 +629,7 @@ bool runInjectedProbe(const std::set<std::string>& probes, TestContext& ctx, con
     }
     return true;
 }
+} // namespace
 
 bool TestRunner::runInjectedProbesBegin(TestContext& ctx) {
     static const std::string mark = " - default - start";
diff --git a/render-test/runner.hpp b/render-test/runner.hpp
index 7076cdfd78..8a1d00bc97 100644
--- a/render-test/runner.hpp
+++ b/render-test/runner.hpp
@@ -11,22 +11,27 @@
 class TestRunnerMapObserver;
 struct TestMetadata;
 
-class TestContext {
+class TestRunnerMapObserver : public mbgl::MapObserver {
 public:
-    virtual mbgl::HeadlessFrontend& getFrontend() = 0;
-    virtual mbgl::Map& getMap() = 0;
-    virtual TestRunnerMapObserver& getObserver() = 0;
-    virtual TestMetadata& getMetadata() = 0;
+    TestRunnerMapObserver() = default;
+    void onDidFailLoadingMap(mbgl::MapLoadError, const std::string&) override { mapLoadFailure = true; }
 
-    GfxProbe activeGfxProbe;
-    GfxProbe baselineGfxProbe;
-    bool gfxProbeActive = false;
+    void onDidFinishRenderingMap(RenderMode mode) override final {
+        if (!finishRenderingMap) finishRenderingMap = mode == RenderMode::Full;
+    }
 
-protected:
-    virtual ~TestContext() = default;
-};
+    void onDidBecomeIdle() override final { idle = true; }
+
+    void reset() {
+        mapLoadFailure = false;
+        finishRenderingMap = false;
+        idle = false;
+    }
 
-using TestOperation = std::function<bool(TestContext&)>;
+    bool mapLoadFailure;
+    bool finishRenderingMap;
+    bool idle;
+};
 
 class TestRunner {
 public:
@@ -40,7 +45,6 @@ public:
     void doShuffle(uint32_t seed);
 
 private:
-    bool runOperations(TestContext&);
     bool runInjectedProbesBegin(TestContext&);
     bool runInjectedProbesEnd(TestContext&, mbgl::gfx::RenderingStats);