[render-test] Effectively use the offline cache for fixtures

9 files changed, 87 insertions, 267 deletions

diff --git a/render-test/file_source.cpp b/render-test/file_source.cpp
index 0968f1d2f0..04ebf43b1b 100644
--- a/render-test/file_source.cpp
+++ b/render-test/file_source.cpp
@@ -1,4 +1,5 @@
 #include <mbgl/storage/resource_options.hpp>
+#include <mbgl/util/logging.hpp>
 
 #include "file_source.hpp"
 
@@ -7,21 +8,53 @@ namespace mbgl {
 std::atomic_size_t requestCount{0};
 std::atomic_size_t transferredSize{0};
 std::atomic_bool active{false};
+std::atomic_bool offline{true};
 
-ProxyFileSource::ProxyFileSource(const std::string& cachePath,
-                                 const std::string& assetPath,
-                                 bool supportCacheOnlyRequests_)
-    : DefaultFileSource(cachePath, assetPath, supportCacheOnlyRequests_) {}
-
-ProxyFileSource::ProxyFileSource(const std::string& cachePath,
-                                 std::unique_ptr<FileSource>&& assetFileSource_,
-                                 bool supportCacheOnlyRequests_)
-    : DefaultFileSource(cachePath, std::move(assetFileSource_), supportCacheOnlyRequests_) {}
+ProxyFileSource::ProxyFileSource(const std::string& cachePath, const std::string& assetPath)
+    : DefaultFileSource(cachePath, assetPath, false) {}
 
 ProxyFileSource::~ProxyFileSource() = default;
 
 std::unique_ptr<AsyncRequest> ProxyFileSource::request(const Resource& resource, Callback callback) {
-    auto result = DefaultFileSource::request(resource, [=](Response response) {
+    auto transformed = resource;
+
+    // If offline, force always loading the resource from the cache
+    // so we don't make any network request.
+    if (offline) {
+        transformed.loadingMethod = Resource::LoadingMethod::CacheOnly;
+    }
+
+    // This is needed for compatibility with the style tests that
+    // are using local:// instead of http:// which is the schema
+    // we support for cached files.
+    if (transformed.url.compare(0, 8, "local://") == 0) {
+        transformed.url.replace(0, 8, "http://");
+
+        if (transformed.kind == Resource::Kind::Tile && transformed.tileData) {
+            transformed.tileData->urlTemplate.replace(0, 8, "http://");
+        }
+    }
+
+    if (transformed.url.compare(0, 22, "http://localhost:2900/") == 0) {
+        transformed.url.replace(0, 22, "http://");
+
+        if (transformed.kind == Resource::Kind::Tile && transformed.tileData) {
+            transformed.tileData->urlTemplate.replace(0, 22, "http://");
+        }
+    }
+
+    return DefaultFileSource::request(transformed, [=](Response response) {
+        if (transformed.loadingMethod == Resource::LoadingMethod::CacheOnly && response.noContent) {
+            if (transformed.kind == Resource::Kind::Tile && transformed.tileData) {
+                mbgl::Log::Info(mbgl::Event::Database,
+                                "Resource not found in cache: %s (%s)",
+                                transformed.url.c_str(),
+                                transformed.tileData->urlTemplate.c_str());
+            } else {
+                mbgl::Log::Info(mbgl::Event::Database, "Resource not found in cache: %s", transformed.url.c_str());
+            }
+        }
+
         std::size_t size = response.data != nullptr ? response.data->size() : 0;
         if (active) {
             requestCount++;
@@ -29,18 +62,21 @@ std::unique_ptr<AsyncRequest> ProxyFileSource::request(const Resource& resource,
         }
         callback(response);
     });
-    return result;
 }
 
 std::shared_ptr<FileSource> FileSource::createPlatformFileSource(const ResourceOptions& options) {
-    auto fileSource = std::make_shared<ProxyFileSource>(
-        options.cachePath(), options.assetPath(), options.supportsCacheOnlyRequests());
+    auto fileSource = std::make_shared<ProxyFileSource>(options.cachePath(), options.assetPath());
     fileSource->setAccessToken(options.accessToken());
     fileSource->setAPIBaseURL(options.baseURL());
     return fileSource;
 }
 
 // static
+void ProxyFileSource::setOffline(bool status) {
+    offline = status;
+}
+
+// static
 void ProxyFileSource::setTrackingActive(bool active_) {
     active = active_;
     requestCount = 0;
diff --git a/render-test/file_source.hpp b/render-test/file_source.hpp
index 58acf7b6ad..34ba739a22 100644
--- a/render-test/file_source.hpp
+++ b/render-test/file_source.hpp
@@ -6,22 +6,26 @@ namespace mbgl {
 
 class ProxyFileSource : public DefaultFileSource {
 public:
-    ProxyFileSource(const std::string& cachePath, const std::string& assetPath, bool supportCacheOnlyRequests = true);
-    ProxyFileSource(const std::string& cachePath,
-                    std::unique_ptr<FileSource>&& assetFileSource,
-                    bool supportCacheOnlyRequests = true);
+    ProxyFileSource(const std::string& cachePath, const std::string& assetPath);
     ~ProxyFileSource();
 
     std::unique_ptr<AsyncRequest> request(const Resource&, Callback);
 
     /**
+     * @brief Flag to change the networking mode of the file source.
+     */
+    static void setOffline(bool);
+
+    /**
      * @brief Starts/stops metrics tracking.
      */
     static void setTrackingActive(bool);
+
     /**
      * @brief Returns metrics tracking status.
      */
     static bool isTrackingActive();
+
     /**
      * @brief Returns the total amount of requests.
      *
diff --git a/render-test/manifest_parser.cpp b/render-test/manifest_parser.cpp
index e0e4946215..45fceebeba 100644
--- a/render-test/manifest_parser.cpp
+++ b/render-test/manifest_parser.cpp
@@ -7,21 +7,6 @@
 #include <algorithm>
 #include <random>
 
-namespace {
-std::string removeURLArguments(const std::string& url) {
-    std::string::size_type index = url.find('?');
-    if (index != std::string::npos) {
-        return url.substr(0, index);
-    }
-    return url;
-}
-
-std::string prependFileScheme(const std::string& url) {
-    static const std::string fileScheme("file://");
-    return fileScheme + url;
-}
-} // namespace
-
 Manifest::Manifest() {
     const char* envToken = getenv("MAPBOX_ACCESS_TOKEN");
     if (envToken != nullptr) {
@@ -40,9 +25,6 @@ const std::vector<std::pair<std::string, std::string>>& Manifest::getIgnores() c
 const std::string& Manifest::getTestRootPath() const {
     return testRootPath;
 }
-const std::string& Manifest::getAssetPath() const {
-    return assetPath;
-}
 const std::string& Manifest::getManifestPath() const {
     return manifestPath;
 }
@@ -68,161 +50,6 @@ void Manifest::doShuffle(uint32_t seed) {
     std::shuffle(testPaths.begin(), testPaths.end(), shuffler);
 }
 
-std::string Manifest::localizeURL(const std::string& url) const {
-    static const std::regex regex{"local://"};
-    if (auto path = getVendorPath(url, regex)) {
-        return *path;
-    }
-    return getIntegrationPath(url, "", regex).value_or(url);
-}
-
-void Manifest::localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document) const {
-    if (root.HasMember("urls") && root["urls"].IsArray()) {
-        for (auto& urlValue : root["urls"].GetArray()) {
-            const std::string path =
-                prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString())
-                                      .value_or(localizeLocalURL(urlValue.GetString()).value_or(urlValue.GetString())));
-            urlValue.Set<std::string>(path, document.GetAllocator());
-        }
-    }
-
-    if (root.HasMember("url")) {
-        static const std::string image("image");
-        static const std::string video("video");
-
-        mbgl::JSValue& urlValue = root["url"];
-        const std::string path =
-            prependFileScheme(localizeMapboxTilesetURL(urlValue.GetString())
-                                  .value_or(localizeLocalURL(urlValue.GetString()).value_or(urlValue.GetString())));
-        urlValue.Set<std::string>(path, document.GetAllocator());
-
-        if (root["type"].GetString() != image && root["type"].GetString() != video) {
-            const auto tilesetPath = std::string(urlValue.GetString()).erase(0u, 7u); // remove "file://"
-            auto maybeTileset = readJson(tilesetPath);
-            if (maybeTileset.is<mbgl::JSDocument>()) {
-                const auto& tileset = maybeTileset.get<mbgl::JSDocument>();
-                assert(tileset.HasMember("tiles"));
-                root.AddMember("tiles", (mbgl::JSValue&)tileset["tiles"], document.GetAllocator());
-                root.RemoveMember("url");
-            }
-        }
-    }
-
-    if (root.HasMember("tiles")) {
-        mbgl::JSValue& tilesValue = root["tiles"];
-        assert(tilesValue.IsArray());
-        for (auto& tileValue : tilesValue.GetArray()) {
-            const std::string path = prependFileScheme(
-                localizeMapboxTilesURL(tileValue.GetString())
-                    .value_or(localizeLocalURL(tileValue.GetString())
-                                  .value_or(localizeHttpURL(tileValue.GetString()).value_or(tileValue.GetString()))));
-            tileValue.Set<std::string>(path, document.GetAllocator());
-        }
-    }
-
-    if (root.HasMember("data") && root["data"].IsString()) {
-        mbgl::JSValue& dataValue = root["data"];
-        const std::string path =
-            prependFileScheme(localizeLocalURL(dataValue.GetString()).value_or(dataValue.GetString()));
-        dataValue.Set<std::string>(path, document.GetAllocator());
-    }
-}
-
-void Manifest::localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document) const {
-    if (root.HasMember("sources")) {
-        mbgl::JSValue& sourcesValue = root["sources"];
-        for (auto& sourceProperty : sourcesValue.GetObject()) {
-            localizeSourceURLs(sourceProperty.value, document);
-        }
-    }
-
-    if (root.HasMember("glyphs")) {
-        mbgl::JSValue& glyphsValue = root["glyphs"];
-        const std::string path = prependFileScheme(
-            localizeMapboxFontsURL(glyphsValue.GetString())
-                .value_or(localizeLocalURL(glyphsValue.GetString(), true).value_or(glyphsValue.GetString())));
-        glyphsValue.Set<std::string>(path, document.GetAllocator());
-    }
-
-    if (root.HasMember("sprite")) {
-        mbgl::JSValue& spriteValue = root["sprite"];
-        const std::string path = prependFileScheme(
-            localizeMapboxSpriteURL(spriteValue.GetString())
-                .value_or(localizeLocalURL(spriteValue.GetString()).value_or(spriteValue.GetString())));
-        spriteValue.Set<std::string>(path, document.GetAllocator());
-    }
-}
-
-mbgl::optional<std::string> Manifest::localizeLocalURL(const std::string& url, bool glyphsPath) const {
-    static const std::regex regex{"local://"};
-    if (auto path = getVendorPath(url, regex, glyphsPath)) {
-        return path;
-    }
-    return getIntegrationPath(url, "", regex, glyphsPath);
-}
-
-mbgl::optional<std::string> Manifest::localizeHttpURL(const std::string& url) const {
-    static const std::regex regex{"http://localhost:2900"};
-    if (auto path = getVendorPath(url, regex)) {
-        return path;
-    }
-    return getIntegrationPath(url, "", regex);
-}
-
-mbgl::optional<std::string> Manifest::localizeMapboxSpriteURL(const std::string& url) const {
-    static const std::regex regex{"mapbox://"};
-    return getIntegrationPath(url, "", regex);
-}
-
-mbgl::optional<std::string> Manifest::localizeMapboxFontsURL(const std::string& url) const {
-    static const std::regex regex{"mapbox://fonts"};
-    return getIntegrationPath(url, "glyphs/", regex, true);
-}
-
-mbgl::optional<std::string> Manifest::localizeMapboxTilesURL(const std::string& url) const {
-    static const std::regex regex{"mapbox://"};
-    if (auto path = getVendorPath(url, regex)) {
-        return path;
-    }
-    return getIntegrationPath(url, "tiles/", regex);
-}
-
-mbgl::optional<std::string> Manifest::localizeMapboxTilesetURL(const std::string& url) const {
-    static const std::regex regex{"mapbox://"};
-    return getIntegrationPath(url, "tilesets/", regex);
-}
-
-mbgl::optional<std::string> Manifest::getVendorPath(const std::string& url,
-                                                    const std::regex& regex,
-                                                    bool glyphsPath) const {
-    mbgl::filesystem::path file = std::regex_replace(url, regex, vendorPath);
-    if (mbgl::filesystem::exists(file.parent_path())) {
-        return removeURLArguments(file.string());
-    }
-
-    if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) {
-        return removeURLArguments(file.string());
-    }
-
-    return mbgl::nullopt;
-}
-
-mbgl::optional<std::string> Manifest::getIntegrationPath(const std::string& url,
-                                                         const std::string& parent,
-                                                         const std::regex& regex,
-                                                         bool glyphsPath) const {
-    mbgl::filesystem::path file = std::regex_replace(url, regex, assetPath + parent);
-    if (mbgl::filesystem::exists(file.parent_path())) {
-        return removeURLArguments(file.string());
-    }
-
-    if (glyphsPath && mbgl::filesystem::exists(file.parent_path().parent_path())) {
-        return removeURLArguments(file.string());
-    }
-
-    return mbgl::nullopt;
-}
-
 namespace {
 std::vector<std::pair<std::string, std::string>> parseIgnores(const std::vector<mbgl::filesystem::path>& ignoresPaths) {
     std::vector<std::pair<std::string, std::string>> ignores;
@@ -283,30 +110,6 @@ mbgl::optional<Manifest> ManifestParser::parseManifest(const std::string& manife
     }
 
     auto document = std::move(contents.get<mbgl::JSDocument>());
-    if (document.HasMember("asset_path")) {
-        const auto& assetPathValue = document["asset_path"];
-        if (!assetPathValue.IsString()) {
-            mbgl::Log::Warning(
-                mbgl::Event::General, "Invalid asset_path is provided inside the manifest file: %s", filePath.c_str());
-            return mbgl::nullopt;
-        }
-        manifest.assetPath = (getValidPath(manifest.manifestPath, assetPathValue.GetString()) / "").string();
-        if (manifest.assetPath.empty()) {
-            return mbgl::nullopt;
-        }
-    }
-    if (document.HasMember("vendor_path")) {
-        const auto& vendorPathValue = document["vendor_path"];
-        if (!vendorPathValue.IsString()) {
-            mbgl::Log::Warning(
-                mbgl::Event::General, "Invalid vendor_path is provided inside the manifest file: %s", filePath.c_str());
-            return mbgl::nullopt;
-        }
-        manifest.vendorPath = (getValidPath(manifest.manifestPath, vendorPathValue.GetString()) / "").string();
-        if (manifest.vendorPath.empty()) {
-            return mbgl::nullopt;
-        }
-    }
     if (document.HasMember("result_path")) {
         const auto& resultPathValue = document["result_path"];
         if (!resultPathValue.IsString()) {
diff --git a/render-test/manifest_parser.hpp b/render-test/manifest_parser.hpp
index 120fb9a2bd..28cae6f93e 100644
--- a/render-test/manifest_parser.hpp
+++ b/render-test/manifest_parser.hpp
@@ -18,7 +18,6 @@ public:
     const std::vector<std::pair<std::string, std::string>>& getIgnores() const;
     const std::vector<TestPaths>& getTestPaths() const;
     const std::string& getTestRootPath() const;
-    const std::string& getAssetPath() const;
     const std::string& getManifestPath() const;
     const std::string& getResultPath() const;
     const std::string& getCachePath() const;
@@ -26,29 +25,10 @@ public:
     const std::set<std::string>& getProbes() const;
     void doShuffle(uint32_t seed);
 
-    std::string localizeURL(const std::string& url) const;
-    void localizeSourceURLs(mbgl::JSValue& root, mbgl::JSDocument& document) const;
-    void localizeStyleURLs(mbgl::JSValue& root, mbgl::JSDocument& document) const;
-
 private:
     friend class ManifestParser;
-    mbgl::optional<std::string> localizeLocalURL(const std::string& url, bool glyphsPath = false) const;
-    mbgl::optional<std::string> localizeHttpURL(const std::string& url) const;
-    mbgl::optional<std::string> localizeMapboxSpriteURL(const std::string& url) const;
-    mbgl::optional<std::string> localizeMapboxFontsURL(const std::string& url) const;
-    mbgl::optional<std::string> localizeMapboxTilesURL(const std::string& url) const;
-    mbgl::optional<std::string> localizeMapboxTilesetURL(const std::string& url) const;
-    mbgl::optional<std::string> getVendorPath(const std::string& url,
-                                              const std::regex& regex,
-                                              bool glyphsPath = false) const;
-    mbgl::optional<std::string> getIntegrationPath(const std::string& url,
-                                                   const std::string& parent,
-                                                   const std::regex& regex,
-                                                   bool glyphsPath = false) const;
     std::string manifestPath;
     std::string testRootPath;
-    std::string vendorPath;
-    std::string assetPath;
     std::string resultPath;
     std::string cachePath;
     std::string accessToken;
diff --git a/render-test/parser.cpp b/render-test/parser.cpp
index e58187eb15..c5fca18d64 100644
--- a/render-test/parser.cpp
+++ b/render-test/parser.cpp
@@ -441,7 +441,7 @@ TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path) {
     return result;
 }
 
-TestMetadata parseTestMetadata(const TestPaths& paths, const Manifest& manifest) {
+TestMetadata parseTestMetadata(const TestPaths& paths) {
     TestMetadata metadata;
     metadata.paths = paths;
 
@@ -452,8 +452,6 @@ TestMetadata parseTestMetadata(const TestPaths& paths, const Manifest& manifest)
     }
 
     metadata.document = std::move(maybeJson.get<mbgl::JSDocument>());
-    manifest.localizeStyleURLs(metadata.document, metadata.document);
-
     if (!metadata.document.HasMember("metadata")) {
         mbgl::Log::Warning(mbgl::Event::ParseStyle, "Style has no 'metadata': %s", paths.stylePath.c_str());
         return metadata;
@@ -625,7 +623,7 @@ const std::string gfxProbeEndOp("probeGFXEnd");
 
 using namespace TestOperationNames;
 
-TestOperations parseTestOperations(TestMetadata& metadata, const Manifest& manifest) {
+TestOperations parseTestOperations(TestMetadata& metadata) {
     TestOperations result;
     if (!metadata.document.HasMember("metadata") || !metadata.document["metadata"].HasMember("test") ||
         !metadata.document["metadata"]["test"].HasMember("operations")) {
@@ -692,12 +690,10 @@ TestOperations parseTestOperations(TestMetadata& metadata, const Manifest& manif
             std::string imagePath = operationArray[2].GetString();
             imagePath.erase(std::remove(imagePath.begin(), imagePath.end(), '"'), imagePath.end());
 
-            const mbgl::filesystem::path filePath = (mbgl::filesystem::path(manifest.getAssetPath()) / imagePath);
-
-            result.emplace_back([filePath = filePath.string(), imageName, sdf, pixelRatio](TestContext& ctx) {
-                mbgl::optional<std::string> maybeImage = mbgl::util::readFile(filePath);
+            result.emplace_back([imagePath, imageName, sdf, pixelRatio](TestContext& ctx) {
+                mbgl::optional<std::string> maybeImage = mbgl::util::readFile(imagePath);
                 if (!maybeImage) {
-                    ctx.getMetadata().errorMessage = std::string("Failed to load expected image ") + filePath;
+                    ctx.getMetadata().errorMessage = std::string("Failed to load expected image ") + imagePath;
                     return false;
                 }
 
@@ -719,23 +715,21 @@ TestOperations parseTestOperations(TestMetadata& metadata, const Manifest& manif
         } else if (operationArray[0].GetString() == setStyleOp) {
             // setStyle
             assert(operationArray.Size() >= 2u);
-            std::string json;
             if (operationArray[1].IsString()) {
-                std::string stylePath = manifest.localizeURL(operationArray[1].GetString());
-                auto maybeStyle = readJson(stylePath);
-                if (maybeStyle.is<mbgl::JSDocument>()) {
-                    auto& style = maybeStyle.get<mbgl::JSDocument>();
-                    manifest.localizeStyleURLs(static_cast<mbgl::JSValue&>(style), style);
-                    json = serializeJsonValue(style);
-                }
+                std::string url = operationArray[1].GetString();
+
+                result.emplace_back([url](TestContext& ctx) {
+                    ctx.getMap().getStyle().loadURL(url);
+                    return true;
+                });
             } else {
-                manifest.localizeStyleURLs(operationArray[1], metadata.document);
-                json = serializeJsonValue(operationArray[1]);
+                std::string json = serializeJsonValue(operationArray[1]);
+
+                result.emplace_back([json](TestContext& ctx) {
+                    ctx.getMap().getStyle().loadJSON(json);
+                    return true;
+                });
             }
-            result.emplace_back([json](TestContext& ctx) {
-                ctx.getMap().getStyle().loadJSON(json);
-                return true;
-            });
         } else if (operationArray[0].GetString() == setCenterOp) {
             // setCenter
             assert(operationArray.Size() >= 2u);
@@ -859,7 +853,6 @@ TestOperations parseTestOperations(TestMetadata& metadata, const Manifest& manif
             assert(operationArray[2].IsObject());
             std::string sourceName = operationArray[1].GetString();
 
-            manifest.localizeSourceURLs(operationArray[2], metadata.document);
             result.emplace_back([sourceName, json = serializeJsonValue(operationArray[2])](TestContext& ctx) {
                 mbgl::style::conversion::Error error;
                 auto converted =
diff --git a/render-test/parser.hpp b/render-test/parser.hpp
index 905118b401..a54af34345 100644
--- a/render-test/parser.hpp
+++ b/render-test/parser.hpp
@@ -24,8 +24,8 @@ std::vector<std::string> readExpectedJSONEntries(const mbgl::filesystem::path& b
 
 TestMetrics readExpectedMetrics(const mbgl::filesystem::path& path);
 
-TestMetadata parseTestMetadata(const TestPaths& paths, const Manifest& manifest);
-TestOperations parseTestOperations(TestMetadata& metadata, const Manifest& manifest);
+TestMetadata parseTestMetadata(const TestPaths& paths);
+TestOperations parseTestOperations(TestMetadata& metadata);
 
 std::string createResultPage(const TestStatistics&, const std::vector<TestMetadata>&, bool shuffle, uint32_t seed);
 
@@ -66,4 +66,4 @@ extern const std::string panGestureOp;
 extern const std::string gfxProbeOp;
 extern const std::string gfxProbeStartOp;
 extern const std::string gfxProbeEndOp;
-} // namespace TestOperationNames
\ No newline at end of file
+} // namespace TestOperationNames
diff --git a/render-test/render_test.cpp b/render-test/render_test.cpp
index 32d7e51330..f140b7d5b3 100644
--- a/render-test/render_test.cpp
+++ b/render-test/render_test.cpp
@@ -8,6 +8,7 @@
 
 #include <args.hxx>
 
+#include "file_source.hpp"
 #include "manifest_parser.hpp"
 #include "metadata.hpp"
 #include "parser.hpp"
@@ -136,6 +137,9 @@ int runRenderTests(int argc, char** argv, std::function<void()> testStatus) {
 
     std::tie(recycleMap, shuffle, online, seed, manifestPath, updateResults, testNames, testFilter) =
         parseArguments(argc, argv);
+
+    ProxyFileSource::setOffline(!online);
+
     auto manifestData = ManifestParser::parseManifest(manifestPath, testNames, testFilter);
     if (!manifestData) {
         exit(5);
@@ -158,7 +162,7 @@ int runRenderTests(int argc, char** argv, std::function<void()> testStatus) {
     TestStatistics stats;
 
     for (auto& testPath : testPaths) {
-        TestMetadata metadata = parseTestMetadata(testPath, manifest);
+        TestMetadata metadata = parseTestMetadata(testPath);
 
         if (!recycleMap) {
             runner.reset();
diff --git a/render-test/runner.cpp b/render-test/runner.cpp
index 804ae08a9a..eaf1934125 100644
--- a/render-test/runner.cpp
+++ b/render-test/runner.cpp
@@ -651,7 +651,7 @@ uint32_t getImageTileOffset(const std::set<uint32_t>& dims, uint32_t dim) {
 
 } // namespace
 
-TestRunner::Impl::Impl(const TestMetadata& metadata)
+TestRunner::Impl::Impl(const TestMetadata& metadata, const Manifest& manifest)
     : observer(std::make_unique<TestRunnerMapObserver>()),
       frontend(metadata.size, metadata.pixelRatio, swapBehavior(metadata.mapMode)),
       map(frontend,
@@ -661,7 +661,7 @@ TestRunner::Impl::Impl(const TestMetadata& metadata)
               .withSize(metadata.size)
               .withPixelRatio(metadata.pixelRatio)
               .withCrossSourceCollisions(metadata.crossSourceCollisions),
-          mbgl::ResourceOptions().withCacheOnlyRequestsSupport(false)) {}
+          mbgl::ResourceOptions().withCachePath(manifest.getCachePath()).withAccessToken(manifest.getAccessToken())) {}
 
 TestRunner::Impl::~Impl() {}
 
@@ -703,7 +703,7 @@ void TestRunner::run(TestMetadata& metadata) {
                       mbgl::util::toString(uint32_t(metadata.crossSourceCollisions));
 
     if (maps.find(key) == maps.end()) {
-        maps[key] = std::make_unique<TestRunner::Impl>(metadata);
+        maps[key] = std::make_unique<TestRunner::Impl>(metadata, manifest);
     }
 
     ctx.runnerImpl = maps[key].get();
@@ -779,7 +779,7 @@ void TestRunner::run(TestMetadata& metadata) {
 
 mbgl::HeadlessFrontend::RenderResult TestRunner::runTest(TestMetadata& metadata, TestContext& ctx) {
     HeadlessFrontend::RenderResult result{};
-    for (const auto& operation : parseTestOperations(metadata, manifest)) {
+    for (const auto& operation : parseTestOperations(metadata)) {
         if (!operation(ctx)) return result;
     }
 
diff --git a/render-test/runner.hpp b/render-test/runner.hpp
index 6ad204428c..75e552ead7 100644
--- a/render-test/runner.hpp
+++ b/render-test/runner.hpp
@@ -54,7 +54,7 @@ private:
     void checkProbingResults(TestMetadata&);
 
     struct Impl {
-        Impl(const TestMetadata&);
+        Impl(const TestMetadata&, const Manifest&);
         ~Impl();
 
         std::unique_ptr<TestRunnerMapObserver> observer;