[core] Modularize FileSource codebase (#15768)

* [core] Introduce FileSourceManager and use it for default platform impl

  - Add `FileSourceManager` interface that provides access to `FileSource`
    instances and means of registering / unregistering `FileSource` factories
  - Split `DefaultFileSource` into smaller parts
  - Add `DatabaseFileSource` interface and it's default implementation
  - Remove inter-dependencies between concrete `FileSource` classes

* [build] Add files to next build system

* [core] Add generic property setters / getters

* [core] Remove setOnlineStatus from OnlineFileSource interface

* [core] Hide threading implementation details from DatabaseFileSource interface

* [core] Make DB file source methods virtual

* [core] Add documentation for DatabaseFileSource and rename one method

* [core] Use simple callback instead of ActorRef

* [core] Remove ActorRef from OnlineFileSource public header

* [core] Add callback to FileSource::forward async API

* [core] Pass OfflineRegionDefinition by value

* [core] Update tests to use modular file sources

* [core] Update unit tests

* [core] Update unit tests after rebase

* [core] Backport low prio fix for cached requests

* [core] Backport pack database

* [core] Return removed factory from unRegisterFileSourceFactory

* [core] Rename shadowed args in onlinefilesource

* [core] Remove simple std::function callback aliases

* [core] Expose online file source property keys in public header file

* [test-runner] Add proxy file source test runner

* [cache] Update mbgl-cache utility to use new file source

* [metrics] Rebaseline binary size metrics

* [offline] Update offline utility

* [core] Update changelog

1 files changed, 54 insertions, 48 deletions

diff --git a/test/map/map.test.cpp b/test/map/map.test.cpp
index 3eb01a7383..89be4ad73e 100644
--- a/test/map/map.test.cpp
+++ b/test/map/map.test.cpp
@@ -10,7 +10,8 @@
 #include <mbgl/gl/context.hpp>
 #include <mbgl/map/map_options.hpp>
 #include <mbgl/math/log2.hpp>
-#include <mbgl/storage/default_file_source.hpp>
+#include <mbgl/storage/file_source_manager.hpp>
+#include <mbgl/storage/main_resource_loader.hpp>
 #include <mbgl/storage/network_status.hpp>
 #include <mbgl/storage/online_file_source.hpp>
 #include <mbgl/storage/resource_options.hpp>
@@ -47,13 +48,17 @@ public:
               MapOptions().withMapMode(mode).withSize(frontend.getSize()).withPixelRatio(pixelRatio)) {}
 
     template <typename T = FileSource>
-    MapTest(const std::string& cachePath, const std::string& assetPath,
-            float pixelRatio = 1, MapMode mode = MapMode::Static,
-            typename std::enable_if<std::is_same<T, DefaultFileSource>::value>::type* = nullptr)
-            : fileSource(std::make_shared<T>(cachePath, assetPath))
-            , frontend(pixelRatio)
-            , map(frontend, observer, fileSource,
-                  MapOptions().withMapMode(mode).withSize(frontend.getSize()).withPixelRatio(pixelRatio)) {}
+    MapTest(const std::string& cachePath,
+            const std::string& assetPath,
+            float pixelRatio = 1,
+            MapMode mode = MapMode::Static,
+            typename std::enable_if<std::is_same<T, MainResourceLoader>::value>::type* = nullptr)
+        : fileSource(std::make_shared<T>(ResourceOptions().withCachePath(cachePath).withAssetPath(assetPath))),
+          frontend(pixelRatio),
+          map(frontend,
+              observer,
+              fileSource,
+              MapOptions().withMapMode(mode).withSize(frontend.getSize()).withPixelRatio(pixelRatio)) {}
 };
 
 TEST(Map, RendererState) {
@@ -295,7 +300,7 @@ TEST(Map, CameraToLatLngBoundsUnwrappedCrossDateLine) {
 }
 
 TEST(Map, Offline) {
-    MapTest<DefaultFileSource> test {":memory:", "."};
+    MapTest<MainResourceLoader> test{":memory:", "."};
 
     auto expiredItem = [] (const std::string& path) {
         Response response;
@@ -304,19 +309,21 @@ TEST(Map, Offline) {
         return response;
     };
 
-    const std::string prefix = "http://127.0.0.1:3000/";
-    test.fileSource->put(Resource::style(prefix + "style.json"), expiredItem("style.json"));
-    test.fileSource->put(Resource::source(prefix + "streets.json"), expiredItem("streets.json"));
-    test.fileSource->put(Resource::spriteJSON(prefix + "sprite", 1.0), expiredItem("sprite.json"));
-    test.fileSource->put(Resource::spriteImage(prefix + "sprite", 1.0), expiredItem("sprite.png"));
-    test.fileSource->put(Resource::tile(prefix + "{z}-{x}-{y}.vector.pbf", 1.0, 0, 0, 0, Tileset::Scheme::XYZ), expiredItem("0-0-0.vector.pbf"));
-    test.fileSource->put(Resource::glyphs(prefix + "{fontstack}/{range}.pbf", {{"Helvetica"}}, {0, 255}), expiredItem("glyph.pbf"));
     NetworkStatus::Set(NetworkStatus::Status::Offline);
-
-    test.map.getStyle().loadURL(prefix + "style.json");
+    const std::string prefix = "http://127.0.0.1:3000/";
+    auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{});
+    dbfs->forward(Resource::style(prefix + "style.json"), expiredItem("style.json"));
+    dbfs->forward(Resource::source(prefix + "streets.json"), expiredItem("streets.json"));
+    dbfs->forward(Resource::spriteJSON(prefix + "sprite", 1.0), expiredItem("sprite.json"));
+    dbfs->forward(Resource::spriteImage(prefix + "sprite", 1.0), expiredItem("sprite.png"));
+    dbfs->forward(Resource::tile(prefix + "{z}-{x}-{y}.vector.pbf", 1.0, 0, 0, 0, Tileset::Scheme::XYZ),
+                  expiredItem("0-0-0.vector.pbf"));
+    dbfs->forward(Resource::glyphs(prefix + "{fontstack}/{range}.pbf", {{"Helvetica"}}, {0, 255}),
+                  expiredItem("glyph.pbf"),
+                  [&] { test.map.getStyle().loadURL(prefix + "style.json"); });
 
 #if ANDROID
-    test::checkImage("test/fixtures/map/offline", test.frontend.render(test.map).image, 0.0045, 0.1);
+    test::checkImage("test/fixtures/map/offline", test.frontend.render(test.map).image, 0.0046, 0.1);
 #else
     test::checkImage("test/fixtures/map/offline", test.frontend.render(test.map).image, 0.0015, 0.1);
 #endif
@@ -672,7 +679,7 @@ TEST(Map, WithoutVAOExtension) {
         return;
     }
 
-    MapTest<DefaultFileSource> test { ":memory:", "test/fixtures/api/assets" };
+    MapTest<MainResourceLoader> test{":memory:", "test/fixtures/api/assets"};
 
     gfx::BackendScope scope { *test.frontend.getBackend() };
     static_cast<gl::Context&>(test.frontend.getBackend()->getContext()).disableVAOExtension = true;
@@ -836,7 +843,7 @@ TEST(Map, TEST_DISABLED_ON_CI(ContinuousRendering)) {
 }
 
 TEST(Map, NoContentTiles) {
-    MapTest<DefaultFileSource> test {":memory:", "."};
+    MapTest<MainResourceLoader> test{":memory:", "."};
 
     using namespace std::chrono_literals;
 
@@ -844,33 +851,32 @@ TEST(Map, NoContentTiles) {
     Response response;
     response.noContent = true;
     response.expires = util::now() + 1h;
-    test.fileSource->put(Resource::tile("http://example.com/{z}-{x}-{y}.vector.pbf", 1.0, 0, 0, 0,
-                                       Tileset::Scheme::XYZ),
-                        response);
-
-    test.map.getStyle().loadJSON(R"STYLE({
-      "version": 8,
-      "name": "Water",
-      "sources": {
-        "mapbox": {
-          "type": "vector",
-          "tiles": ["http://example.com/{z}-{x}-{y}.vector.pbf"]
-        }
-      },
-      "layers": [{
-        "id": "background",
-        "type": "background",
-        "paint": {
-          "background-color": "red"
-        }
-      }, {
-        "id": "water",
-        "type": "fill",
-        "source": "mapbox",
-        "source-layer": "water"
-      }]
-    })STYLE");
-
+    auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{});
+    dbfs->forward(
+        Resource::tile("http://example.com/{z}-{x}-{y}.vector.pbf", 1.0, 0, 0, 0, Tileset::Scheme::XYZ), response, [&] {
+            test.map.getStyle().loadJSON(R"STYLE({
+                        "version": 8,
+                        "name": "Water",
+                        "sources": {
+                            "mapbox": {
+                            "type": "vector",
+                            "tiles": ["http://example.com/{z}-{x}-{y}.vector.pbf"]
+                            }
+                        },
+                        "layers": [{
+                            "id": "background",
+                            "type": "background",
+                            "paint": {
+                            "background-color": "red"
+                            }
+                        }, {
+                            "id": "water",
+                            "type": "fill",
+                            "source": "mapbox",
+                            "source-layer": "water"
+                        }]
+                        })STYLE");
+        });
     test::checkImage("test/fixtures/map/nocontent", test.frontend.render(test.map).image, 0.0015, 0.1);
 }