From 879c44f661c5eb762c93a721b657859a71aabfc7 Mon Sep 17 00:00:00 2001 From: Alexander Shalamov Date: Fri, 4 Oct 2019 15:02:01 +0300 Subject: [core] Modularize FileSource codebase (#15768) * [core] Introduce FileSourceManager and use it for default platform impl - Add `FileSourceManager` interface that provides access to `FileSource` instances and means of registering / unregistering `FileSource` factories - Split `DefaultFileSource` into smaller parts - Add `DatabaseFileSource` interface and it's default implementation - Remove inter-dependencies between concrete `FileSource` classes * [build] Add files to next build system * [core] Add generic property setters / getters * [core] Remove setOnlineStatus from OnlineFileSource interface * [core] Hide threading implementation details from DatabaseFileSource interface * [core] Make DB file source methods virtual * [core] Add documentation for DatabaseFileSource and rename one method * [core] Use simple callback instead of ActorRef * [core] Remove ActorRef from OnlineFileSource public header * [core] Add callback to FileSource::forward async API * [core] Pass OfflineRegionDefinition by value * [core] Update tests to use modular file sources * [core] Update unit tests * [core] Update unit tests after rebase * [core] Backport low prio fix for cached requests * [core] Backport pack database * [core] Return removed factory from unRegisterFileSourceFactory * [core] Rename shadowed args in onlinefilesource * [core] Remove simple std::function callback aliases * [core] Expose online file source property keys in public header file * [test-runner] Add proxy file source test runner * [cache] Update mbgl-cache utility to use new file source * [metrics] Rebaseline binary size metrics * [offline] Update offline utility * [core] Update changelog --- test/map/map.test.cpp | 102 ++-- test/src/mbgl/test/fake_file_source.hpp | 13 +- test/src/mbgl/test/stub_file_source.cpp | 1 + test/src/mbgl/test/stub_file_source.hpp | 13 +- test/storage/asset_file_source.test.cpp | 18 +- test/storage/default_file_source.test.cpp | 738 ---------------------------- test/storage/http_file_source.test.cpp | 7 +- test/storage/local_file_source.test.cpp | 14 +- test/storage/main_resource_loader.test.cpp | 744 +++++++++++++++++++++++++++++ test/storage/offline_download.test.cpp | 6 +- test/storage/online_file_source.test.cpp | 89 ++-- test/storage/sync_file_source.test.cpp | 11 +- test/style/source.test.cpp | 2 +- test/style/style.test.cpp | 8 +- test/style/style_layer.test.cpp | 4 +- test/test-files.json | 2 +- test/tile/custom_geometry_tile.test.cpp | 2 +- test/tile/geojson_tile.test.cpp | 2 +- test/tile/raster_dem_tile.test.cpp | 2 +- test/tile/raster_tile.test.cpp | 2 +- test/tile/tile_cache.test.cpp | 2 +- test/tile/vector_tile.test.cpp | 2 +- 22 files changed, 911 insertions(+), 873 deletions(-) delete mode 100644 test/storage/default_file_source.test.cpp create mode 100644 test/storage/main_resource_loader.test.cpp (limited to 'test') diff --git a/test/map/map.test.cpp b/test/map/map.test.cpp index 3eb01a7383..89be4ad73e 100644 --- a/test/map/map.test.cpp +++ b/test/map/map.test.cpp @@ -10,7 +10,8 @@ #include #include #include -#include +#include +#include #include #include #include @@ -47,13 +48,17 @@ public: MapOptions().withMapMode(mode).withSize(frontend.getSize()).withPixelRatio(pixelRatio)) {} template - MapTest(const std::string& cachePath, const std::string& assetPath, - float pixelRatio = 1, MapMode mode = MapMode::Static, - typename std::enable_if::value>::type* = nullptr) - : fileSource(std::make_shared(cachePath, assetPath)) - , frontend(pixelRatio) - , map(frontend, observer, fileSource, - MapOptions().withMapMode(mode).withSize(frontend.getSize()).withPixelRatio(pixelRatio)) {} + MapTest(const std::string& cachePath, + const std::string& assetPath, + float pixelRatio = 1, + MapMode mode = MapMode::Static, + typename std::enable_if::value>::type* = nullptr) + : fileSource(std::make_shared(ResourceOptions().withCachePath(cachePath).withAssetPath(assetPath))), + frontend(pixelRatio), + map(frontend, + observer, + fileSource, + MapOptions().withMapMode(mode).withSize(frontend.getSize()).withPixelRatio(pixelRatio)) {} }; TEST(Map, RendererState) { @@ -295,7 +300,7 @@ TEST(Map, CameraToLatLngBoundsUnwrappedCrossDateLine) { } TEST(Map, Offline) { - MapTest test {":memory:", "."}; + MapTest test{":memory:", "."}; auto expiredItem = [] (const std::string& path) { Response response; @@ -304,19 +309,21 @@ TEST(Map, Offline) { return response; }; - const std::string prefix = "http://127.0.0.1:3000/"; - test.fileSource->put(Resource::style(prefix + "style.json"), expiredItem("style.json")); - test.fileSource->put(Resource::source(prefix + "streets.json"), expiredItem("streets.json")); - test.fileSource->put(Resource::spriteJSON(prefix + "sprite", 1.0), expiredItem("sprite.json")); - test.fileSource->put(Resource::spriteImage(prefix + "sprite", 1.0), expiredItem("sprite.png")); - test.fileSource->put(Resource::tile(prefix + "{z}-{x}-{y}.vector.pbf", 1.0, 0, 0, 0, Tileset::Scheme::XYZ), expiredItem("0-0-0.vector.pbf")); - test.fileSource->put(Resource::glyphs(prefix + "{fontstack}/{range}.pbf", {{"Helvetica"}}, {0, 255}), expiredItem("glyph.pbf")); NetworkStatus::Set(NetworkStatus::Status::Offline); - - test.map.getStyle().loadURL(prefix + "style.json"); + const std::string prefix = "http://127.0.0.1:3000/"; + auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); + dbfs->forward(Resource::style(prefix + "style.json"), expiredItem("style.json")); + dbfs->forward(Resource::source(prefix + "streets.json"), expiredItem("streets.json")); + dbfs->forward(Resource::spriteJSON(prefix + "sprite", 1.0), expiredItem("sprite.json")); + dbfs->forward(Resource::spriteImage(prefix + "sprite", 1.0), expiredItem("sprite.png")); + dbfs->forward(Resource::tile(prefix + "{z}-{x}-{y}.vector.pbf", 1.0, 0, 0, 0, Tileset::Scheme::XYZ), + expiredItem("0-0-0.vector.pbf")); + dbfs->forward(Resource::glyphs(prefix + "{fontstack}/{range}.pbf", {{"Helvetica"}}, {0, 255}), + expiredItem("glyph.pbf"), + [&] { test.map.getStyle().loadURL(prefix + "style.json"); }); #if ANDROID - test::checkImage("test/fixtures/map/offline", test.frontend.render(test.map).image, 0.0045, 0.1); + test::checkImage("test/fixtures/map/offline", test.frontend.render(test.map).image, 0.0046, 0.1); #else test::checkImage("test/fixtures/map/offline", test.frontend.render(test.map).image, 0.0015, 0.1); #endif @@ -672,7 +679,7 @@ TEST(Map, WithoutVAOExtension) { return; } - MapTest test { ":memory:", "test/fixtures/api/assets" }; + MapTest test{":memory:", "test/fixtures/api/assets"}; gfx::BackendScope scope { *test.frontend.getBackend() }; static_cast(test.frontend.getBackend()->getContext()).disableVAOExtension = true; @@ -836,7 +843,7 @@ TEST(Map, TEST_DISABLED_ON_CI(ContinuousRendering)) { } TEST(Map, NoContentTiles) { - MapTest test {":memory:", "."}; + MapTest test{":memory:", "."}; using namespace std::chrono_literals; @@ -844,33 +851,32 @@ TEST(Map, NoContentTiles) { Response response; response.noContent = true; response.expires = util::now() + 1h; - test.fileSource->put(Resource::tile("http://example.com/{z}-{x}-{y}.vector.pbf", 1.0, 0, 0, 0, - Tileset::Scheme::XYZ), - response); - - test.map.getStyle().loadJSON(R"STYLE({ - "version": 8, - "name": "Water", - "sources": { - "mapbox": { - "type": "vector", - "tiles": ["http://example.com/{z}-{x}-{y}.vector.pbf"] - } - }, - "layers": [{ - "id": "background", - "type": "background", - "paint": { - "background-color": "red" - } - }, { - "id": "water", - "type": "fill", - "source": "mapbox", - "source-layer": "water" - }] - })STYLE"); - + auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); + dbfs->forward( + Resource::tile("http://example.com/{z}-{x}-{y}.vector.pbf", 1.0, 0, 0, 0, Tileset::Scheme::XYZ), response, [&] { + test.map.getStyle().loadJSON(R"STYLE({ + "version": 8, + "name": "Water", + "sources": { + "mapbox": { + "type": "vector", + "tiles": ["http://example.com/{z}-{x}-{y}.vector.pbf"] + } + }, + "layers": [{ + "id": "background", + "type": "background", + "paint": { + "background-color": "red" + } + }, { + "id": "water", + "type": "fill", + "source": "mapbox", + "source-layer": "water" + }] + })STYLE"); + }); test::checkImage("test/fixtures/map/nocontent", test.frontend.render(test.map).image, 0.0015, 0.1); } diff --git a/test/src/mbgl/test/fake_file_source.hpp b/test/src/mbgl/test/fake_file_source.hpp index 8803e9576b..1faf4b7a18 100644 --- a/test/src/mbgl/test/fake_file_source.hpp +++ b/test/src/mbgl/test/fake_file_source.hpp @@ -2,6 +2,8 @@ #include #include +#include +#include #include #include @@ -42,6 +44,8 @@ public: return std::make_unique(resource, callback, requests); } + bool canRequest(const Resource&) const override { return true; } + bool respond(Resource::Kind kind, const Response& response) { auto it = std::find_if(requests.begin(), requests.end(), [&] (FakeFileRequest* fakeRequest) { return fakeRequest->resource.kind == kind; @@ -60,7 +64,7 @@ public: }; -class FakeOnlineFileSource : public OnlineFileSource, public FakeFileSource { +class FakeOnlineFileSource : public FakeFileSource { public: std::unique_ptr request(const Resource& resource, Callback callback) override { return FakeFileSource::request(resource, callback); @@ -69,7 +73,12 @@ public: bool respond(Resource::Kind kind, const Response& response) { return FakeFileSource::respond(kind, response); } -}; + mapbox::base::Value getProperty(const std::string& property) const override { + return onlineFs.getProperty(property); + } + + OnlineFileSource onlineFs; +}; } // namespace mbgl diff --git a/test/src/mbgl/test/stub_file_source.cpp b/test/src/mbgl/test/stub_file_source.cpp index 0bbff84ff3..8870a45bdc 100644 --- a/test/src/mbgl/test/stub_file_source.cpp +++ b/test/src/mbgl/test/stub_file_source.cpp @@ -1,4 +1,5 @@ #include +#include namespace mbgl { diff --git a/test/src/mbgl/test/stub_file_source.hpp b/test/src/mbgl/test/stub_file_source.hpp index 1135fa9a80..46bb33d5e2 100644 --- a/test/src/mbgl/test/stub_file_source.hpp +++ b/test/src/mbgl/test/stub_file_source.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -19,6 +20,7 @@ public: ~StubFileSource() override; std::unique_ptr request(const Resource&, Callback) override; + bool canRequest(const Resource&) const override { return true; } void remove(AsyncRequest*); using ResponseFunction = std::function (const Resource&)>; @@ -48,15 +50,4 @@ private: util::Timer timer; }; -class StubOnlineFileSource : public StubFileSource, public OnlineFileSource { -public: - - StubOnlineFileSource(ResponseType t = ResponseType::Asynchronous) : StubFileSource(t) {}; - ~StubOnlineFileSource() override = default; - - std::unique_ptr request(const Resource& r, Callback c) override { return StubFileSource::request(r, c); }; - void remove(AsyncRequest* r) { StubFileSource::remove(r); }; -}; - - } // namespace mbgl diff --git a/test/storage/asset_file_source.test.cpp b/test/storage/asset_file_source.test.cpp index 978a41a306..ac04bc7dc2 100644 --- a/test/storage/asset_file_source.test.cpp +++ b/test/storage/asset_file_source.test.cpp @@ -1,9 +1,10 @@ +#include #include -#include +#include #include +#include #include #include -#include #include #include @@ -70,12 +71,13 @@ TEST(AssetFileSource, Load) { } TEST(AssetFileSource, AcceptsURL) { - EXPECT_TRUE(AssetFileSource::acceptsURL("asset://empty")); - EXPECT_TRUE(AssetFileSource::acceptsURL("asset:///test")); - EXPECT_FALSE(AssetFileSource::acceptsURL("assds://foo")); - EXPECT_FALSE(AssetFileSource::acceptsURL("asset:")); - EXPECT_FALSE(AssetFileSource::acceptsURL("style.json")); - EXPECT_FALSE(AssetFileSource::acceptsURL("")); + AssetFileSource fs("test/fixtures/storage/assets"); + EXPECT_TRUE(fs.canRequest(Resource::style("asset://empty"))); + EXPECT_TRUE(fs.canRequest(Resource::style("asset:///test"))); + EXPECT_FALSE(fs.canRequest(Resource::style("assds://foo"))); + EXPECT_FALSE(fs.canRequest(Resource::style("asset:"))); + EXPECT_FALSE(fs.canRequest(Resource::style("style.json"))); + EXPECT_FALSE(fs.canRequest(Resource::style(""))); } TEST(AssetFileSource, EmptyFile) { diff --git a/test/storage/default_file_source.test.cpp b/test/storage/default_file_source.test.cpp deleted file mode 100644 index 52051ac839..0000000000 --- a/test/storage/default_file_source.test.cpp +++ /dev/null @@ -1,738 +0,0 @@ -#include -#include -#include -#include -#include -#include - -using namespace mbgl; - -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CacheResponse)) { - util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); - - const Resource resource { Resource::Unknown, "http://127.0.0.1:3000/cache" }; - Response response; - - std::unique_ptr req1; - std::unique_ptr req2; - - req1 = fs.request(resource, [&](Response res) { - req1.reset(); - EXPECT_EQ(nullptr, res.error); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Response 1", *res.data); - EXPECT_TRUE(bool(res.expires)); - EXPECT_FALSE(res.mustRevalidate); - EXPECT_FALSE(bool(res.modified)); - EXPECT_FALSE(bool(res.etag)); - response = res; - - // Now test that we get the same values as in the previous request. If we'd go to the server - // again, we'd get different values. - req2 = fs.request(resource, [&](Response res2) { - req2.reset(); - EXPECT_EQ(response.error, res2.error); - ASSERT_TRUE(res2.data.get()); - EXPECT_EQ(*response.data, *res2.data); - EXPECT_EQ(response.expires, res2.expires); - EXPECT_EQ(response.mustRevalidate, res2.mustRevalidate); - EXPECT_EQ(response.modified, res2.modified); - EXPECT_EQ(response.etag, res2.etag); - - loop.stop(); - }); - }); - - loop.run(); -} - -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CacheRevalidateSame)) { - util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); - - const Resource revalidateSame { Resource::Unknown, "http://127.0.0.1:3000/revalidate-same" }; - std::unique_ptr req1; - std::unique_ptr req2; - bool gotResponse = false; - - // First request causes the response to get cached. - req1 = fs.request(revalidateSame, [&](Response res) { - req1.reset(); - - EXPECT_EQ(nullptr, res.error); - EXPECT_FALSE(res.notModified); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Response", *res.data); - EXPECT_FALSE(bool(res.expires)); - EXPECT_TRUE(res.mustRevalidate); - EXPECT_FALSE(bool(res.modified)); - EXPECT_EQ("snowfall", *res.etag); - - // The first response is stored in the cache, but it has 'must-revalidate' set. This means - // it can't return the cached response right away and we must wait for the revalidation - // request to complete. We can distinguish the cached response from the revalided response - // because the latter has an expiration date, while the cached response doesn't. - req2 = fs.request(revalidateSame, [&](Response res2) { - if (!gotResponse) { - // Even though we could find the response in the database, we send a revalidation - // request and get a 304 response. Since we haven't sent a reply yet, we're forcing - // notModified to be false so that implementations can continue to use the - // notModified flag to skip parsing new data. - gotResponse = true; - EXPECT_EQ(nullptr, res2.error); - EXPECT_FALSE(res2.notModified); - ASSERT_TRUE(res2.data.get()); - EXPECT_EQ("Response", *res2.data); - EXPECT_TRUE(bool(res2.expires)); - EXPECT_TRUE(res2.mustRevalidate); - EXPECT_FALSE(bool(res2.modified)); - EXPECT_EQ("snowfall", *res2.etag); - } else { - // The test server sends a Cache-Control header with a max-age of 1 second. This - // means that our OnlineFileSource implementation will request the tile again after - // 1 second. This time, our request already had a prior response, so we don't need - // to send the data again, and instead can actually forward the notModified flag. - req2.reset(); - EXPECT_EQ(nullptr, res2.error); - EXPECT_TRUE(res2.notModified); - EXPECT_FALSE(res2.data.get()); - EXPECT_TRUE(bool(res2.expires)); - EXPECT_TRUE(res2.mustRevalidate); - EXPECT_FALSE(bool(res2.modified)); - EXPECT_EQ("snowfall", *res2.etag); - loop.stop(); - } - }); - }); - - loop.run(); -} - -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CacheRevalidateModified)) { - util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); - - const Resource revalidateModified{ Resource::Unknown, - "http://127.0.0.1:3000/revalidate-modified" }; - std::unique_ptr req1; - std::unique_ptr req2; - bool gotResponse = false; - - // First request causes the response to get cached. - req1 = fs.request(revalidateModified, [&](Response res) { - req1.reset(); - - EXPECT_EQ(nullptr, res.error); - EXPECT_FALSE(res.notModified); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Response", *res.data); - EXPECT_FALSE(bool(res.expires)); - EXPECT_TRUE(res.mustRevalidate); - EXPECT_EQ(Timestamp{ Seconds(1420070400) }, *res.modified); - EXPECT_FALSE(res.etag); - - // The first response is stored in the cache, but it has 'must-revalidate' set. This means - // it can't return the cached response right away and we must wait for the revalidation - // request to complete. We can distinguish the cached response from the revalided response - // because the latter has an expiration date, while the cached response doesn't. - req2 = fs.request(revalidateModified, [&, res](Response res2) { - if (!gotResponse) { - // Even though we could find the response in the database, we send a revalidation - // request and get a 304 response. Since we haven't sent a reply yet, we're forcing - // notModified to be false so that implementations can continue to use the - // notModified flag to skip parsing new data. - gotResponse = true; - EXPECT_EQ(nullptr, res2.error); - EXPECT_FALSE(res2.notModified); - ASSERT_TRUE(res2.data.get()); - EXPECT_EQ("Response", *res2.data); - EXPECT_TRUE(bool(res2.expires)); - EXPECT_TRUE(res2.mustRevalidate); - EXPECT_EQ(Timestamp{ Seconds(1420070400) }, *res2.modified); - EXPECT_FALSE(res2.etag); - } else { - // The test server sends a Cache-Control header with a max-age of 1 second. This - // means that our OnlineFileSource implementation will request the tile again after - // 1 second. This time, our request already had a prior response, so we don't need - // to send the data again, and instead can actually forward the notModified flag. - req2.reset(); - EXPECT_EQ(nullptr, res2.error); - EXPECT_TRUE(res2.notModified); - EXPECT_FALSE(res2.data.get()); - EXPECT_TRUE(bool(res2.expires)); - EXPECT_TRUE(res2.mustRevalidate); - EXPECT_EQ(Timestamp{ Seconds(1420070400) }, *res2.modified); - EXPECT_FALSE(res2.etag); - loop.stop(); - } - }); - }); - - loop.run(); -} - -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CacheRevalidateEtag)) { - util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); - - const Resource revalidateEtag { Resource::Unknown, "http://127.0.0.1:3000/revalidate-etag" }; - std::unique_ptr req1; - std::unique_ptr req2; - - // First request causes the response to get cached. - req1 = fs.request(revalidateEtag, [&](Response res) { - req1.reset(); - - EXPECT_EQ(nullptr, res.error); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Response 1", *res.data); - EXPECT_FALSE(bool(res.expires)); - EXPECT_TRUE(res.mustRevalidate); - EXPECT_FALSE(bool(res.modified)); - EXPECT_EQ("response-1", *res.etag); - - // Second request does not return the cached response, since it had Cache-Control: must-revalidate set. - req2 = fs.request(revalidateEtag, [&, res](Response res2) { - req2.reset(); - - EXPECT_EQ(nullptr, res2.error); - ASSERT_TRUE(res2.data.get()); - EXPECT_NE(res.data, res2.data); - EXPECT_EQ("Response 2", *res2.data); - EXPECT_FALSE(bool(res2.expires)); - EXPECT_TRUE(res2.mustRevalidate); - EXPECT_FALSE(bool(res2.modified)); - EXPECT_EQ("response-2", *res2.etag); - - loop.stop(); - }); - }); - - loop.run(); -} - -// Test for https://github.com/mapbox/mapbox-gl-native/issue/1369 -// -// A request for http://example.com is made. This triggers a cache get. While the cache get is -// pending, the request is canceled. This removes it from pending. Then, still while the cache get -// is pending, a second request is made for the same resource. This adds an entry back to pending -// and queues another cache request, even though the first one is still pending. Now both cache -// requests resolve to misses, resulting in two HTTP requests for the same resource. The first one -// will notify as expected, the second one will have bound a DefaultFileRequest* in the lambda that -// gets invalidated by the first notify's pending.erase, and when it gets notified, the crash -// occurs. - -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(HTTPIssue1369)) { - util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); - - const Resource resource { Resource::Unknown, "http://127.0.0.1:3000/test" }; - - auto req = fs.request(resource, [&](Response) { - ADD_FAILURE() << "Callback should not be called"; - }); - req.reset(); - req = fs.request(resource, [&](Response res) { - req.reset(); - EXPECT_EQ(nullptr, res.error); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Hello World!", *res.data); - EXPECT_FALSE(bool(res.expires)); - EXPECT_FALSE(res.mustRevalidate); - EXPECT_FALSE(bool(res.modified)); - EXPECT_FALSE(bool(res.etag)); - loop.stop(); - }); - - loop.run(); -} - -TEST(DefaultFileSource, OptionalNonExpired) { - util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); - - const Resource optionalResource { Resource::Unknown, "http://127.0.0.1:3000/test", {}, Resource::LoadingMethod::CacheOnly }; - - using namespace std::chrono_literals; - - Response response; - response.data = std::make_shared("Cached value"); - response.expires = util::now() + 1h; - fs.put(optionalResource, response); - - std::unique_ptr req; - req = fs.request(optionalResource, [&](Response res) { - req.reset(); - EXPECT_EQ(nullptr, res.error); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Cached value", *res.data); - ASSERT_TRUE(bool(res.expires)); - EXPECT_EQ(*response.expires, *res.expires); - EXPECT_FALSE(res.mustRevalidate); - EXPECT_FALSE(bool(res.modified)); - EXPECT_FALSE(bool(res.etag)); - loop.stop(); - }); - - loop.run(); -} - -TEST(DefaultFileSource, OptionalExpired) { - util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); - - const Resource optionalResource { Resource::Unknown, "http://127.0.0.1:3000/test", {}, Resource::LoadingMethod::CacheOnly }; - - using namespace std::chrono_literals; - - Response response; - response.data = std::make_shared("Cached value"); - response.expires = util::now() - 1h; - fs.put(optionalResource, response); - - std::unique_ptr req; - req = fs.request(optionalResource, [&](Response res) { - req.reset(); - EXPECT_EQ(nullptr, res.error); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Cached value", *res.data); - ASSERT_TRUE(bool(res.expires)); - EXPECT_EQ(*response.expires, *res.expires); - EXPECT_FALSE(res.mustRevalidate); - EXPECT_FALSE(bool(res.modified)); - EXPECT_FALSE(bool(res.etag)); - loop.stop(); - }); - - loop.run(); -} - -TEST(DefaultFileSource, GetBaseURLAndAccessTokenWhilePaused) { - util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); - - fs.pause(); - - auto baseURL = "http://url"; - auto accessToken = "access_token"; - - fs.setAPIBaseURL(baseURL); - fs.setAccessToken(accessToken); - - EXPECT_EQ(fs.getAPIBaseURL(), baseURL); - EXPECT_EQ(fs.getAccessToken(), accessToken); -} - -TEST(DefaultFileSource, OptionalNotFound) { - util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); - - const Resource optionalResource { Resource::Unknown, "http://127.0.0.1:3000/test", {}, Resource::LoadingMethod::CacheOnly }; - - using namespace std::chrono_literals; - - std::unique_ptr req; - req = fs.request(optionalResource, [&](Response res) { - req.reset(); - ASSERT_TRUE(res.error.get()); - EXPECT_EQ(Response::Error::Reason::NotFound, res.error->reason); - EXPECT_EQ("Not found in offline database", res.error->message); - EXPECT_FALSE(res.data); - EXPECT_FALSE(bool(res.expires)); - EXPECT_FALSE(res.mustRevalidate); - EXPECT_FALSE(bool(res.modified)); - EXPECT_FALSE(bool(res.etag)); - loop.stop(); - }); - - loop.run(); -} - -// Test that a network only request doesn't attempt to load data from the cache. -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(NoCacheRefreshEtagNotModified)) { - util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); - - Resource resource { Resource::Unknown, "http://127.0.0.1:3000/revalidate-same" }; - resource.loadingMethod = Resource::LoadingMethod::NetworkOnly; - resource.priorEtag.emplace("snowfall"); - - using namespace std::chrono_literals; - - // Put a fake value into the cache to make sure we're not retrieving anything from the cache. - Response response; - response.data = std::make_shared("Cached value"); - response.expires = util::now() + 1h; - fs.put(resource, response); - - std::unique_ptr req; - req = fs.request(resource, [&](Response res) { - req.reset(); - EXPECT_EQ(nullptr, res.error); - EXPECT_TRUE(res.notModified); - EXPECT_FALSE(res.data.get()); - ASSERT_TRUE(bool(res.expires)); - EXPECT_LT(util::now(), *res.expires); - EXPECT_TRUE(res.mustRevalidate); - EXPECT_FALSE(bool(res.modified)); - ASSERT_TRUE(bool(res.etag)); - EXPECT_EQ("snowfall", *res.etag); - loop.stop(); - }); - - loop.run(); -} - -// Test that a network only request doesn't attempt to load data from the cache. -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(NoCacheRefreshEtagModified)) { - util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); - - Resource resource { Resource::Unknown, "http://127.0.0.1:3000/revalidate-same" }; - resource.loadingMethod = Resource::LoadingMethod::NetworkOnly; - resource.priorEtag.emplace("sunshine"); - - using namespace std::chrono_literals; - - // Put a fake value into the cache to make sure we're not retrieving anything from the cache. - Response response; - response.data = std::make_shared("Cached value"); - response.expires = util::now() + 1h; - fs.put(resource, response); - - std::unique_ptr req; - req = fs.request(resource, [&](Response res) { - req.reset(); - EXPECT_EQ(nullptr, res.error); - EXPECT_FALSE(res.notModified); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Response", *res.data); - EXPECT_FALSE(bool(res.expires)); - EXPECT_TRUE(res.mustRevalidate); - EXPECT_FALSE(bool(res.modified)); - ASSERT_TRUE(bool(res.etag)); - EXPECT_EQ("snowfall", *res.etag); - loop.stop(); - }); - - loop.run(); -} - -// Test that a network only request doesn't attempt to load data from the cache. -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(NoCacheFull)) { - util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); - - Resource resource { Resource::Unknown, "http://127.0.0.1:3000/revalidate-same" }; - resource.loadingMethod = Resource::LoadingMethod::NetworkOnly; - - using namespace std::chrono_literals; - - // Put a fake value into the cache to make sure we're not retrieving anything from the cache. - Response response; - response.data = std::make_shared("Cached value"); - response.expires = util::now() + 1h; - fs.put(resource, response); - - std::unique_ptr req; - req = fs.request(resource, [&](Response res) { - req.reset(); - EXPECT_EQ(nullptr, res.error); - EXPECT_FALSE(res.notModified); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Response", *res.data); - EXPECT_FALSE(bool(res.expires)); - EXPECT_TRUE(res.mustRevalidate); - EXPECT_FALSE(bool(res.modified)); - ASSERT_TRUE(bool(res.etag)); - EXPECT_EQ("snowfall", *res.etag); - loop.stop(); - }); - - loop.run(); -} - -// Test that we can make a request with a Modified field that doesn't first try to load -// from cache like a regular request -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(NoCacheRefreshModifiedNotModified)) { - util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); - - Resource resource { Resource::Unknown, "http://127.0.0.1:3000/revalidate-modified" }; - resource.loadingMethod = Resource::LoadingMethod::NetworkOnly; - resource.priorModified.emplace(Seconds(1420070400)); // January 1, 2015 - - using namespace std::chrono_literals; - - // Put a fake value into the cache to make sure we're not retrieving anything from the cache. - Response response; - response.data = std::make_shared("Cached value"); - response.expires = util::now() + 1h; - fs.put(resource, response); - - std::unique_ptr req; - req = fs.request(resource, [&](Response res) { - req.reset(); - EXPECT_EQ(nullptr, res.error); - EXPECT_TRUE(res.notModified); - EXPECT_FALSE(res.data.get()); - ASSERT_TRUE(bool(res.expires)); - EXPECT_LT(util::now(), *res.expires); - EXPECT_TRUE(res.mustRevalidate); - ASSERT_TRUE(bool(res.modified)); - EXPECT_EQ(Timestamp{ Seconds(1420070400) }, *res.modified); - EXPECT_FALSE(bool(res.etag)); - loop.stop(); - }); - - loop.run(); -} - -// Test that we can make a request with a Modified field that doesn't first try to load -// from cache like a regular request -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(NoCacheRefreshModifiedModified)) { - util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); - - Resource resource { Resource::Unknown, "http://127.0.0.1:3000/revalidate-modified" }; - resource.loadingMethod = Resource::LoadingMethod::NetworkOnly; - resource.priorModified.emplace(Seconds(1417392000)); // December 1, 2014 - - using namespace std::chrono_literals; - - // Put a fake value into the cache to make sure we're not retrieving anything from the cache. - Response response; - response.data = std::make_shared("Cached value"); - response.expires = util::now() + 1h; - fs.put(resource, response); - - std::unique_ptr req; - req = fs.request(resource, [&](Response res) { - req.reset(); - EXPECT_EQ(nullptr, res.error); - EXPECT_FALSE(res.notModified); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Response", *res.data); - EXPECT_FALSE(bool(res.expires)); - EXPECT_TRUE(res.mustRevalidate); - EXPECT_EQ(Timestamp{ Seconds(1420070400) }, *res.modified); - EXPECT_FALSE(res.etag); - loop.stop(); - }); - - loop.run(); -} - -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(SetResourceTransform)) { - util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); - - // Translates the URL "localhost://test to http://127.0.0.1:3000/test - Actor transform(loop, [](Resource::Kind, const std::string& url) -> std::string { - if (url == "localhost://test") { - return "http://127.0.0.1:3000/test"; - } else { - return url; - } - }); - - fs.setResourceTransform(transform.self()); - const Resource resource1 { Resource::Unknown, "localhost://test" }; - - std::unique_ptr req; - req = fs.request(resource1, [&](Response res) { - req.reset(); - EXPECT_EQ(nullptr, res.error); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Hello World!", *res.data); - EXPECT_FALSE(bool(res.expires)); - EXPECT_FALSE(res.mustRevalidate); - EXPECT_FALSE(bool(res.modified)); - EXPECT_FALSE(bool(res.etag)); - loop.stop(); - }); - - loop.run(); - - fs.setResourceTransform({}); - const Resource resource2 { Resource::Unknown, "http://127.0.0.1:3000/test" }; - - req = fs.request(resource2, [&](Response res) { - req.reset(); - EXPECT_EQ(nullptr, res.error); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Hello World!", *res.data); - EXPECT_FALSE(bool(res.expires)); - EXPECT_FALSE(res.mustRevalidate); - EXPECT_FALSE(bool(res.modified)); - EXPECT_FALSE(bool(res.etag)); - loop.stop(); - }); - - loop.run(); -} - -TEST(DefaultFileSource, SetResourceCachePath) { - util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); - - Actor callback(loop, [&]() -> void { - loop.stop(); - }); - - fs.setResourceCachePath("./new_offline.db", callback.self()); - loop.run(); -} - -// Test that a stale cache file that has must-revalidate set will trigger a response. -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(RespondToStaleMustRevalidate)) { - util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); - - Resource resource { Resource::Unknown, "http://127.0.0.1:3000/revalidate-same" }; - resource.loadingMethod = Resource::LoadingMethod::CacheOnly; - - // using namespace std::chrono_literals; - - // Put an existing value in the cache that has expired, and has must-revalidate set. - Response response; - response.data = std::make_shared("Cached value"); - response.modified = Timestamp(Seconds(1417392000)); // December 1, 2014 - response.expires = Timestamp(Seconds(1417392000)); - response.mustRevalidate = true; - response.etag.emplace("snowfall"); - fs.put(resource, response); - - std::unique_ptr req; - req = fs.request(resource, [&](Response res) { - req.reset(); - ASSERT_TRUE(res.error.get()); - EXPECT_EQ(Response::Error::Reason::NotFound, res.error->reason); - EXPECT_EQ("Cached resource is unusable", res.error->message); - EXPECT_FALSE(res.notModified); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Cached value", *res.data); - ASSERT_TRUE(res.expires); - EXPECT_EQ(Timestamp{ Seconds(1417392000) }, *res.expires); - EXPECT_TRUE(res.mustRevalidate); - ASSERT_TRUE(res.modified); - EXPECT_EQ(Timestamp{ Seconds(1417392000) }, *res.modified); - ASSERT_TRUE(res.etag); - EXPECT_EQ("snowfall", *res.etag); - - resource.priorEtag = res.etag; - resource.priorModified = res.modified; - resource.priorExpires = res.expires; - resource.priorData = res.data; - - loop.stop(); - }); - - loop.run(); - - // Now run this request again, with the data we gathered from the previous stale/unusable - // request. We're replacing the data so that we can check that the DefaultFileSource doesn't - // attempt another database access if we already have the value. - resource.loadingMethod = Resource::LoadingMethod::NetworkOnly; - resource.priorData = std::make_shared("Prior value"); - - req = fs.request(resource, [&](Response res) { - req.reset(); - ASSERT_EQ(nullptr, res.error.get()); - // Since the data was found in the cache, we're doing a revalidation request. Yet, since - // this request hasn't returned data before, we're setting notModified to false in the - // OnlineFileSource to ensure that requestors know that this is the first time they're - // seeing this data. - EXPECT_FALSE(res.notModified); - ASSERT_TRUE(res.data.get()); - // Ensure that it's the value that we manually inserted into the cache rather than the value - // the server returns, since we should be executing a revalidation request which doesn't - // return new data, only a 304 Not Modified response. - EXPECT_EQ("Prior value", *res.data); - ASSERT_TRUE(res.expires); - EXPECT_LE(util::now(), *res.expires); - EXPECT_TRUE(res.mustRevalidate); - ASSERT_TRUE(res.modified); - EXPECT_EQ(Timestamp{ Seconds(1417392000) }, *res.modified); - ASSERT_TRUE(res.etag); - EXPECT_EQ("snowfall", *res.etag); - loop.stop(); - }); - - loop.run(); -} - -// Test that requests for expired resources have lower priority than requests for new resources -TEST(DefaultFileSource, TEST_REQUIRES_SERVER(CachedResourceLowPriority)) { - util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); - - Response response; - std::size_t online_response_counter = 0; - - using namespace std::chrono_literals; - response.expires = util::now() - 1h; - - // Put existing values into the cache. - Resource resource1{Resource::Unknown, "http://127.0.0.1:3000/load/3", {}, Resource::LoadingMethod::All}; - response.data = std::make_shared("Cached Request 3"); - fs.put(resource1, response); - - Resource resource2{Resource::Unknown, "http://127.0.0.1:3000/load/4", {}, Resource::LoadingMethod::All}; - response.data = std::make_shared("Cached Request 4"); - fs.put(resource2, response); - - fs.setMaximumConcurrentRequests(1); - NetworkStatus::Set(NetworkStatus::Status::Offline); - - // Ensure that the online requests for new resources are processed first. - Resource nonCached1{Resource::Unknown, "http://127.0.0.1:3000/load/1", {}, Resource::LoadingMethod::All}; - std::unique_ptr req1 = fs.request(nonCached1, [&](Response res) { - online_response_counter++; - req1.reset(); - EXPECT_EQ(online_response_counter, 1); // make sure this is responded first - EXPECT_EQ("Request 1", *res.data); - }); - - Resource nonCached2{Resource::Unknown, "http://127.0.0.1:3000/load/2", {}, Resource::LoadingMethod::All}; - std::unique_ptr req2 = fs.request(nonCached2, [&](Response res) { - online_response_counter++; - req2.reset(); - EXPECT_EQ(online_response_counter, 2); // make sure this is responded second - EXPECT_EQ("Request 2", *res.data); - }); - - bool req3CachedResponseReceived = false; - std::unique_ptr req3 = fs.request(resource1, [&](Response res) { - // Offline callback is received first - if (!req3CachedResponseReceived) { - EXPECT_EQ("Cached Request 3", *res.data); - req3CachedResponseReceived = true; - } else { - online_response_counter++; - req3.reset(); - EXPECT_EQ(online_response_counter, 3); - EXPECT_EQ("Request 3", *res.data); - } - }); - - bool req4CachedResponseReceived = false; - std::unique_ptr req4 = fs.request(resource2, [&](Response res) { - // Offline callback is received first - if (!req4CachedResponseReceived) { - EXPECT_EQ("Cached Request 4", *res.data); - req4CachedResponseReceived = true; - } else { - online_response_counter++; - req4.reset(); - EXPECT_EQ(online_response_counter, 4); - EXPECT_EQ("Request 4", *res.data); - loop.stop(); - } - }); - - NetworkStatus::Set(NetworkStatus::Status::Online); - - loop.run(); -} diff --git a/test/storage/http_file_source.test.cpp b/test/storage/http_file_source.test.cpp index 42b4174e69..37476c8e7c 100644 --- a/test/storage/http_file_source.test.cpp +++ b/test/storage/http_file_source.test.cpp @@ -1,9 +1,10 @@ -#include #include -#include +#include +#include #include -#include +#include #include +#include using namespace mbgl; diff --git a/test/storage/local_file_source.test.cpp b/test/storage/local_file_source.test.cpp index e1756f8e7d..45c8c54d91 100644 --- a/test/storage/local_file_source.test.cpp +++ b/test/storage/local_file_source.test.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -21,12 +22,13 @@ std::string toAbsoluteURL(const std::string& fileName) { using namespace mbgl; TEST(LocalFileSource, AcceptsURL) { - EXPECT_TRUE(LocalFileSource::acceptsURL("file://empty")); - EXPECT_TRUE(LocalFileSource::acceptsURL("file:///test")); - EXPECT_FALSE(LocalFileSource::acceptsURL("flie://foo")); - EXPECT_FALSE(LocalFileSource::acceptsURL("file:")); - EXPECT_FALSE(LocalFileSource::acceptsURL("style.json")); - EXPECT_FALSE(LocalFileSource::acceptsURL("")); + LocalFileSource fs; + EXPECT_TRUE(fs.canRequest(Resource::style("file://empty"))); + EXPECT_TRUE(fs.canRequest(Resource::style("file:///test"))); + EXPECT_FALSE(fs.canRequest(Resource::style("flie://foo"))); + EXPECT_FALSE(fs.canRequest(Resource::style("file:"))); + EXPECT_FALSE(fs.canRequest(Resource::style("style.json"))); + EXPECT_FALSE(fs.canRequest(Resource::style(""))); } TEST(LocalFileSource, EmptyFile) { diff --git a/test/storage/main_resource_loader.test.cpp b/test/storage/main_resource_loader.test.cpp new file mode 100644 index 0000000000..c5f1a9c707 --- /dev/null +++ b/test/storage/main_resource_loader.test.cpp @@ -0,0 +1,744 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace mbgl; + +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CacheResponse)) { + util::RunLoop loop; + MainResourceLoader fs(ResourceOptions{}); + + const Resource resource{Resource::Unknown, "http://127.0.0.1:3000/cache"}; + Response response; + + std::unique_ptr req1; + std::unique_ptr req2; + + req1 = fs.request(resource, [&](Response res) { + req1.reset(); + EXPECT_EQ(nullptr, res.error); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Response 1", *res.data); + EXPECT_TRUE(bool(res.expires)); + EXPECT_FALSE(res.mustRevalidate); + EXPECT_FALSE(bool(res.modified)); + EXPECT_FALSE(bool(res.etag)); + response = res; + + // Now test that we get the same values as in the previous request. If we'd go to the server + // again, we'd get different values. + req2 = fs.request(resource, [&](Response res2) { + req2.reset(); + EXPECT_EQ(response.error, res2.error); + ASSERT_TRUE(res2.data.get()); + EXPECT_EQ(*response.data, *res2.data); + EXPECT_EQ(response.expires, res2.expires); + EXPECT_EQ(response.mustRevalidate, res2.mustRevalidate); + EXPECT_EQ(response.modified, res2.modified); + EXPECT_EQ(response.etag, res2.etag); + + loop.stop(); + }); + }); + + loop.run(); +} + +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CacheRevalidateSame)) { + util::RunLoop loop; + MainResourceLoader fs(ResourceOptions{}); + + const Resource revalidateSame{Resource::Unknown, "http://127.0.0.1:3000/revalidate-same"}; + std::unique_ptr req1; + std::unique_ptr req2; + bool gotResponse = false; + + // First request causes the response to get cached. + req1 = fs.request(revalidateSame, [&](Response res) { + req1.reset(); + + EXPECT_EQ(nullptr, res.error); + EXPECT_FALSE(res.notModified); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Response", *res.data); + EXPECT_FALSE(bool(res.expires)); + EXPECT_TRUE(res.mustRevalidate); + EXPECT_FALSE(bool(res.modified)); + EXPECT_EQ("snowfall", *res.etag); + + // The first response is stored in the cache, but it has 'must-revalidate' set. This means + // it can't return the cached response right away and we must wait for the revalidation + // request to complete. We can distinguish the cached response from the revalided response + // because the latter has an expiration date, while the cached response doesn't. + req2 = fs.request(revalidateSame, [&](Response res2) { + if (!gotResponse) { + // Even though we could find the response in the database, we send a revalidation + // request and get a 304 response. Since we haven't sent a reply yet, we're forcing + // notModified to be false so that implementations can continue to use the + // notModified flag to skip parsing new data. + gotResponse = true; + EXPECT_EQ(nullptr, res2.error); + EXPECT_FALSE(res2.notModified); + ASSERT_TRUE(res2.data.get()); + EXPECT_EQ("Response", *res2.data); + EXPECT_TRUE(bool(res2.expires)); + EXPECT_TRUE(res2.mustRevalidate); + EXPECT_FALSE(bool(res2.modified)); + EXPECT_EQ("snowfall", *res2.etag); + } else { + // The test server sends a Cache-Control header with a max-age of 1 second. This + // means that our OnlineFileSource implementation will request the tile again after + // 1 second. This time, our request already had a prior response, so we don't need + // to send the data again, and instead can actually forward the notModified flag. + req2.reset(); + EXPECT_EQ(nullptr, res2.error); + EXPECT_TRUE(res2.notModified); + EXPECT_FALSE(res2.data.get()); + EXPECT_TRUE(bool(res2.expires)); + EXPECT_TRUE(res2.mustRevalidate); + EXPECT_FALSE(bool(res2.modified)); + EXPECT_EQ("snowfall", *res2.etag); + loop.stop(); + } + }); + }); + + loop.run(); +} + +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CacheRevalidateModified)) { + util::RunLoop loop; + MainResourceLoader fs(ResourceOptions{}); + + const Resource revalidateModified{Resource::Unknown, "http://127.0.0.1:3000/revalidate-modified"}; + std::unique_ptr req1; + std::unique_ptr req2; + bool gotResponse = false; + + // First request causes the response to get cached. + req1 = fs.request(revalidateModified, [&](Response res) { + req1.reset(); + + EXPECT_EQ(nullptr, res.error); + EXPECT_FALSE(res.notModified); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Response", *res.data); + EXPECT_FALSE(bool(res.expires)); + EXPECT_TRUE(res.mustRevalidate); + EXPECT_EQ(Timestamp{Seconds(1420070400)}, *res.modified); + EXPECT_FALSE(res.etag); + + // The first response is stored in the cache, but it has 'must-revalidate' set. This means + // it can't return the cached response right away and we must wait for the revalidation + // request to complete. We can distinguish the cached response from the revalided response + // because the latter has an expiration date, while the cached response doesn't. + req2 = fs.request(revalidateModified, [&, res](Response res2) { + if (!gotResponse) { + // Even though we could find the response in the database, we send a revalidation + // request and get a 304 response. Since we haven't sent a reply yet, we're forcing + // notModified to be false so that implementations can continue to use the + // notModified flag to skip parsing new data. + gotResponse = true; + EXPECT_EQ(nullptr, res2.error); + EXPECT_FALSE(res2.notModified); + ASSERT_TRUE(res2.data.get()); + EXPECT_EQ("Response", *res2.data); + EXPECT_TRUE(bool(res2.expires)); + EXPECT_TRUE(res2.mustRevalidate); + EXPECT_EQ(Timestamp{Seconds(1420070400)}, *res2.modified); + EXPECT_FALSE(res2.etag); + } else { + // The test server sends a Cache-Control header with a max-age of 1 second. This + // means that our OnlineFileSource implementation will request the tile again after + // 1 second. This time, our request already had a prior response, so we don't need + // to send the data again, and instead can actually forward the notModified flag. + req2.reset(); + EXPECT_EQ(nullptr, res2.error); + EXPECT_TRUE(res2.notModified); + EXPECT_FALSE(res2.data.get()); + EXPECT_TRUE(bool(res2.expires)); + EXPECT_TRUE(res2.mustRevalidate); + EXPECT_EQ(Timestamp{Seconds(1420070400)}, *res2.modified); + EXPECT_FALSE(res2.etag); + loop.stop(); + } + }); + }); + + loop.run(); +} + +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CacheRevalidateEtag)) { + util::RunLoop loop; + MainResourceLoader fs(ResourceOptions{}); + + const Resource revalidateEtag{Resource::Unknown, "http://127.0.0.1:3000/revalidate-etag"}; + std::unique_ptr req1; + std::unique_ptr req2; + + // First request causes the response to get cached. + req1 = fs.request(revalidateEtag, [&](Response res) { + req1.reset(); + + EXPECT_EQ(nullptr, res.error); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Response 1", *res.data); + EXPECT_FALSE(bool(res.expires)); + EXPECT_TRUE(res.mustRevalidate); + EXPECT_FALSE(bool(res.modified)); + EXPECT_EQ("response-1", *res.etag); + + // Second request does not return the cached response, since it had Cache-Control: must-revalidate set. + req2 = fs.request(revalidateEtag, [&, res](Response res2) { + req2.reset(); + + EXPECT_EQ(nullptr, res2.error); + ASSERT_TRUE(res2.data.get()); + EXPECT_NE(res.data, res2.data); + EXPECT_EQ("Response 2", *res2.data); + EXPECT_FALSE(bool(res2.expires)); + EXPECT_TRUE(res2.mustRevalidate); + EXPECT_FALSE(bool(res2.modified)); + EXPECT_EQ("response-2", *res2.etag); + + loop.stop(); + }); + }); + + loop.run(); +} + +// Test for https://github.com/mapbox/mapbox-gl-native/issue/1369 +// +// A request for http://example.com is made. This triggers a cache get. While the cache get is +// pending, the request is canceled. This removes it from pending. Then, still while the cache get +// is pending, a second request is made for the same resource. This adds an entry back to pending +// and queues another cache request, even though the first one is still pending. Now both cache +// requests resolve to misses, resulting in two HTTP requests for the same resource. The first one +// will notify as expected, the second one will have bound a DefaultFileRequest* in the lambda that +// gets invalidated by the first notify's pending.erase, and when it gets notified, the crash +// occurs. +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(HTTPIssue1369)) { + util::RunLoop loop; + MainResourceLoader fs(ResourceOptions{}); + + const Resource resource{Resource::Unknown, "http://127.0.0.1:3000/test"}; + + auto req = fs.request(resource, [&](Response) { ADD_FAILURE() << "Callback should not be called"; }); + req.reset(); + req = fs.request(resource, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Hello World!", *res.data); + EXPECT_FALSE(bool(res.expires)); + EXPECT_FALSE(res.mustRevalidate); + EXPECT_FALSE(bool(res.modified)); + EXPECT_FALSE(bool(res.etag)); + loop.stop(); + }); + + loop.run(); +} + +TEST(MainResourceLoader, OptionalNonExpired) { + util::RunLoop loop; + MainResourceLoader fs(ResourceOptions{}); + + const Resource optionalResource{ + Resource::Unknown, "http://127.0.0.1:3000/test", {}, Resource::LoadingMethod::CacheOnly}; + + using namespace std::chrono_literals; + + Response response; + response.data = std::make_shared("Cached value"); + response.expires = util::now() + 1h; + + std::unique_ptr req; + auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); + dbfs->forward(optionalResource, response, [&] { + req = fs.request(optionalResource, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Cached value", *res.data); + ASSERT_TRUE(bool(res.expires)); + EXPECT_EQ(*response.expires, *res.expires); + EXPECT_FALSE(res.mustRevalidate); + EXPECT_FALSE(bool(res.modified)); + EXPECT_FALSE(bool(res.etag)); + loop.stop(); + }); + }); + + loop.run(); +} + +TEST(MainResourceLoader, OptionalExpired) { + util::RunLoop loop; + MainResourceLoader fs(ResourceOptions{}); + + const Resource optionalResource{ + Resource::Unknown, "http://127.0.0.1:3000/test", {}, Resource::LoadingMethod::CacheOnly}; + + using namespace std::chrono_literals; + + Response response; + response.data = std::make_shared("Cached value"); + response.expires = util::now() - 1h; + auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); + std::unique_ptr req; + dbfs->forward(optionalResource, response, [&] { + req = fs.request(optionalResource, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Cached value", *res.data); + ASSERT_TRUE(bool(res.expires)); + EXPECT_EQ(*response.expires, *res.expires); + EXPECT_FALSE(res.mustRevalidate); + EXPECT_FALSE(bool(res.modified)); + EXPECT_FALSE(bool(res.etag)); + loop.stop(); + }); + }); + + loop.run(); +} + +TEST(MainResourceLoader, OptionalNotFound) { + util::RunLoop loop; + MainResourceLoader fs(ResourceOptions{}); + + const Resource optionalResource{ + Resource::Unknown, "http://127.0.0.1:3000/test", {}, Resource::LoadingMethod::CacheOnly}; + + using namespace std::chrono_literals; + + std::unique_ptr req; + req = fs.request(optionalResource, [&](Response res) { + req.reset(); + ASSERT_TRUE(res.error.get()); + EXPECT_EQ(Response::Error::Reason::NotFound, res.error->reason); + EXPECT_EQ("Not found in offline database", res.error->message); + EXPECT_FALSE(res.data); + EXPECT_FALSE(bool(res.expires)); + EXPECT_FALSE(res.mustRevalidate); + EXPECT_FALSE(bool(res.modified)); + EXPECT_FALSE(bool(res.etag)); + loop.stop(); + }); + + loop.run(); +} + +// Test that a network only request doesn't attempt to load data from the cache. +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(NoCacheRefreshEtagNotModified)) { + util::RunLoop loop; + MainResourceLoader fs(ResourceOptions{}); + + Resource resource { Resource::Unknown, "http://127.0.0.1:3000/revalidate-same" }; + resource.loadingMethod = Resource::LoadingMethod::NetworkOnly; + resource.priorEtag.emplace("snowfall"); + + using namespace std::chrono_literals; + + // Put a fake value into the cache to make sure we're not retrieving anything from the cache. + Response response; + response.data = std::make_shared("Cached value"); + response.expires = util::now() + 1h; + std::unique_ptr req; + auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); + dbfs->forward(resource, response, [&] { + req = fs.request(resource, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + EXPECT_TRUE(res.notModified); + EXPECT_FALSE(res.data.get()); + ASSERT_TRUE(bool(res.expires)); + EXPECT_LT(util::now(), *res.expires); + EXPECT_TRUE(res.mustRevalidate); + EXPECT_FALSE(bool(res.modified)); + ASSERT_TRUE(bool(res.etag)); + EXPECT_EQ("snowfall", *res.etag); + loop.stop(); + }); + }); + + loop.run(); +} + +// Test that a network only request doesn't attempt to load data from the cache. +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(NoCacheRefreshEtagModified)) { + util::RunLoop loop; + MainResourceLoader fs(ResourceOptions{}); + + Resource resource { Resource::Unknown, "http://127.0.0.1:3000/revalidate-same" }; + resource.loadingMethod = Resource::LoadingMethod::NetworkOnly; + resource.priorEtag.emplace("sunshine"); + + using namespace std::chrono_literals; + + // Put a fake value into the cache to make sure we're not retrieving anything from the cache. + Response response; + response.data = std::make_shared("Cached value"); + response.expires = util::now() + 1h; + std::unique_ptr req; + auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); + dbfs->forward(resource, response, [&] { + req = fs.request(resource, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + EXPECT_FALSE(res.notModified); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Response", *res.data); + EXPECT_FALSE(bool(res.expires)); + EXPECT_TRUE(res.mustRevalidate); + EXPECT_FALSE(bool(res.modified)); + ASSERT_TRUE(bool(res.etag)); + EXPECT_EQ("snowfall", *res.etag); + loop.stop(); + }); + }); + + loop.run(); +} + +// Test that a network only request doesn't attempt to load data from the cache. +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(NoCacheFull)) { + util::RunLoop loop; + MainResourceLoader fs(ResourceOptions{}); + + Resource resource { Resource::Unknown, "http://127.0.0.1:3000/revalidate-same" }; + resource.loadingMethod = Resource::LoadingMethod::NetworkOnly; + + using namespace std::chrono_literals; + + // Put a fake value into the cache to make sure we're not retrieving anything from the cache. + Response response; + response.data = std::make_shared("Cached value"); + response.expires = util::now() + 1h; + std::unique_ptr req; + auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); + dbfs->forward(resource, response, [&] { + req = fs.request(resource, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + EXPECT_FALSE(res.notModified); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Response", *res.data); + EXPECT_FALSE(bool(res.expires)); + EXPECT_TRUE(res.mustRevalidate); + EXPECT_FALSE(bool(res.modified)); + ASSERT_TRUE(bool(res.etag)); + EXPECT_EQ("snowfall", *res.etag); + loop.stop(); + }); + }); + + loop.run(); +} + +// Test that we can make a request with a Modified field that doesn't first try to load +// from cache like a regular request +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(NoCacheRefreshModifiedNotModified)) { + util::RunLoop loop; + MainResourceLoader fs(ResourceOptions{}); + + Resource resource { Resource::Unknown, "http://127.0.0.1:3000/revalidate-modified" }; + resource.loadingMethod = Resource::LoadingMethod::NetworkOnly; + resource.priorModified.emplace(Seconds(1420070400)); // January 1, 2015 + + using namespace std::chrono_literals; + + // Put a fake value into the cache to make sure we're not retrieving anything from the cache. + Response response; + response.data = std::make_shared("Cached value"); + response.expires = util::now() + 1h; + std::unique_ptr req; + auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); + dbfs->forward(resource, response, [&] { + req = fs.request(resource, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + EXPECT_TRUE(res.notModified); + EXPECT_FALSE(res.data.get()); + ASSERT_TRUE(bool(res.expires)); + EXPECT_LT(util::now(), *res.expires); + EXPECT_TRUE(res.mustRevalidate); + ASSERT_TRUE(bool(res.modified)); + EXPECT_EQ(Timestamp{Seconds(1420070400)}, *res.modified); + EXPECT_FALSE(bool(res.etag)); + loop.stop(); + }); + }); + + loop.run(); +} + +// Test that we can make a request with a Modified field that doesn't first try to load +// from cache like a regular request +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(NoCacheRefreshModifiedModified)) { + util::RunLoop loop; + MainResourceLoader fs(ResourceOptions{}); + + Resource resource { Resource::Unknown, "http://127.0.0.1:3000/revalidate-modified" }; + resource.loadingMethod = Resource::LoadingMethod::NetworkOnly; + resource.priorModified.emplace(Seconds(1417392000)); // December 1, 2014 + + using namespace std::chrono_literals; + + // Put a fake value into the cache to make sure we're not retrieving anything from the cache. + Response response; + response.data = std::make_shared("Cached value"); + response.expires = util::now() + 1h; + std::unique_ptr req; + auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); + dbfs->forward(resource, response, [&] { + req = fs.request(resource, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + EXPECT_FALSE(res.notModified); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Response", *res.data); + EXPECT_FALSE(bool(res.expires)); + EXPECT_TRUE(res.mustRevalidate); + EXPECT_EQ(Timestamp{Seconds(1420070400)}, *res.modified); + EXPECT_FALSE(res.etag); + loop.stop(); + }); + }); + + loop.run(); +} + +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(SetResourceTransform)) { + util::RunLoop loop; + MainResourceLoader fs(ResourceOptions{}); + + auto onlinefs = std::static_pointer_cast( + FileSourceManager::get()->getFileSource(FileSourceType::Network, ResourceOptions{})); + + // Translates the URL "localhost://test to http://127.0.0.1:3000/test + Actor transform( + loop, [](Resource::Kind, const std::string& url, ResourceTransform::FinishedCallback cb) { + if (url == "localhost://test") { + cb("http://127.0.0.1:3000/test"); + } else { + cb(url); + } + }); + + onlinefs->setResourceTransform( + {[actorRef = transform.self()]( + Resource::Kind kind, const std::string& url, ResourceTransform::FinishedCallback cb) { + actorRef.invoke(&ResourceTransform::TransformCallback::operator(), kind, url, std::move(cb)); + }}); + const Resource resource1{Resource::Unknown, "localhost://test"}; + + std::unique_ptr req; + req = fs.request(resource1, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Hello World!", *res.data); + EXPECT_FALSE(bool(res.expires)); + EXPECT_FALSE(res.mustRevalidate); + EXPECT_FALSE(bool(res.modified)); + EXPECT_FALSE(bool(res.etag)); + loop.stop(); + }); + + loop.run(); + + onlinefs->setResourceTransform({}); + const Resource resource2{Resource::Unknown, "http://127.0.0.1:3000/test"}; + + req = fs.request(resource2, [&](Response res) { + req.reset(); + EXPECT_EQ(nullptr, res.error); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Hello World!", *res.data); + EXPECT_FALSE(bool(res.expires)); + EXPECT_FALSE(res.mustRevalidate); + EXPECT_FALSE(bool(res.modified)); + EXPECT_FALSE(bool(res.etag)); + loop.stop(); + }); + + loop.run(); +} + +TEST(MainResourceLoader, SetResourceCachePath) { + util::RunLoop loop; + MainResourceLoader fs(ResourceOptions{}); + auto dbfs = std::static_pointer_cast( + FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{})); + dbfs->setDatabasePath("./new_offline.db", [&loop] { loop.stop(); }); + loop.run(); +} + +// Test that a stale cache file that has must-revalidate set will trigger a response. +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(RespondToStaleMustRevalidate)) { + util::RunLoop loop; + MainResourceLoader fs(ResourceOptions{}); + + Resource resource { Resource::Unknown, "http://127.0.0.1:3000/revalidate-same" }; + resource.loadingMethod = Resource::LoadingMethod::CacheOnly; + + // using namespace std::chrono_literals; + + // Put an existing value in the cache that has expired, and has must-revalidate set. + Response response; + response.data = std::make_shared("Cached value"); + response.modified = Timestamp(Seconds(1417392000)); // December 1, 2014 + response.expires = Timestamp(Seconds(1417392000)); + response.mustRevalidate = true; + response.etag.emplace("snowfall"); + std::unique_ptr req; + auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); + dbfs->forward(resource, response, [&] { + req = fs.request(resource, [&](Response res) { + req.reset(); + ASSERT_TRUE(res.error.get()); + EXPECT_EQ(Response::Error::Reason::NotFound, res.error->reason); + EXPECT_EQ("Cached resource is unusable", res.error->message); + EXPECT_FALSE(res.notModified); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Cached value", *res.data); + ASSERT_TRUE(res.expires); + EXPECT_EQ(Timestamp{Seconds(1417392000)}, *res.expires); + EXPECT_TRUE(res.mustRevalidate); + ASSERT_TRUE(res.modified); + EXPECT_EQ(Timestamp{Seconds(1417392000)}, *res.modified); + ASSERT_TRUE(res.etag); + EXPECT_EQ("snowfall", *res.etag); + + resource.priorEtag = res.etag; + resource.priorModified = res.modified; + resource.priorExpires = res.expires; + resource.priorData = res.data; + + loop.stop(); + }); + }); + + loop.run(); + + // Now run this request again, with the data we gathered from the previous stale/unusable + // request. We're replacing the data so that we can check that the MainResourceLoader doesn't + // attempt another database access if we already have the value. + resource.loadingMethod = Resource::LoadingMethod::NetworkOnly; + resource.priorData = std::make_shared("Prior value"); + + req = fs.request(resource, [&](Response res) { + req.reset(); + ASSERT_EQ(nullptr, res.error.get()); + // Since the data was found in the cache, we're doing a revalidation request. Yet, since + // this request hasn't returned data before, we're setting notModified to false in the + // OnlineFileSource to ensure that requestors know that this is the first time they're + // seeing this data. + EXPECT_FALSE(res.notModified); + ASSERT_TRUE(res.data.get()); + // Ensure that it's the value that we manually inserted into the cache rather than the value + // the server returns, since we should be executing a revalidation request which doesn't + // return new data, only a 304 Not Modified response. + EXPECT_EQ("Prior value", *res.data); + ASSERT_TRUE(res.expires); + EXPECT_LE(util::now(), *res.expires); + EXPECT_TRUE(res.mustRevalidate); + ASSERT_TRUE(res.modified); + EXPECT_EQ(Timestamp{ Seconds(1417392000) }, *res.modified); + ASSERT_TRUE(res.etag); + EXPECT_EQ("snowfall", *res.etag); + loop.stop(); + }); + + loop.run(); +} + +// Test that requests for expired resources have lower priority than requests for new resources +TEST(MainResourceLoader, TEST_REQUIRES_SERVER(CachedResourceLowPriority)) { + util::RunLoop loop; + MainResourceLoader fs(ResourceOptions{}); + + Response response; + std::size_t online_response_counter = 0; + + using namespace std::chrono_literals; + response.expires = util::now() - 1h; + + auto dbfs = FileSourceManager::get()->getFileSource(FileSourceType::Database, ResourceOptions{}); + auto onlineFs = FileSourceManager::get()->getFileSource(FileSourceType::Network, ResourceOptions{}); + + // Put existing values into the cache. + Resource resource1{Resource::Unknown, "http://127.0.0.1:3000/load/3", {}, Resource::LoadingMethod::All}; + response.data = std::make_shared("Cached Request 3"); + dbfs->forward(resource1, response); + + Resource resource2{Resource::Unknown, "http://127.0.0.1:3000/load/4", {}, Resource::LoadingMethod::All}; + response.data = std::make_shared("Cached Request 4"); + dbfs->forward(resource2, response); + + onlineFs->setProperty("max-concurrent-requests", 1u); + fs.pause(); + NetworkStatus::Set(NetworkStatus::Status::Offline); + + // Ensure that the online requests for new resources are processed first. + Resource nonCached1{Resource::Unknown, "http://127.0.0.1:3000/load/1", {}, Resource::LoadingMethod::All}; + std::unique_ptr req1 = fs.request(nonCached1, [&](Response res) { + online_response_counter++; + req1.reset(); + EXPECT_EQ(online_response_counter, 1); // make sure this is responded first + EXPECT_EQ("Request 1", *res.data); + }); + + bool req3CachedResponseReceived = false; + std::unique_ptr req3 = fs.request(resource1, [&](Response res) { + // Offline callback is received first + if (!req3CachedResponseReceived) { + EXPECT_EQ("Cached Request 3", *res.data); + req3CachedResponseReceived = true; + } else { + online_response_counter++; + req3.reset(); + EXPECT_EQ(online_response_counter, 3); + EXPECT_EQ("Request 3", *res.data); + } + }); + + bool req4CachedResponseReceived = false; + std::unique_ptr req4 = fs.request(resource2, [&](Response res) { + // Offline callback is received first + if (!req4CachedResponseReceived) { + EXPECT_EQ("Cached Request 4", *res.data); + req4CachedResponseReceived = true; + } else { + online_response_counter++; + req4.reset(); + EXPECT_EQ(online_response_counter, 4); + EXPECT_EQ("Request 4", *res.data); + loop.stop(); + } + }); + + Resource nonCached2{Resource::Unknown, "http://127.0.0.1:3000/load/2", {}, Resource::LoadingMethod::All}; + std::unique_ptr req2 = fs.request(nonCached2, [&](Response res) { + online_response_counter++; + req2.reset(); + EXPECT_EQ(online_response_counter, 2); // make sure this is responded second + EXPECT_EQ("Request 2", *res.data); + }); + + fs.resume(); + NetworkStatus::Set(NetworkStatus::Status::Online); + + loop.run(); +} diff --git a/test/storage/offline_download.test.cpp b/test/storage/offline_download.test.cpp index c1a9bb73f4..a15c96d391 100644 --- a/test/storage/offline_download.test.cpp +++ b/test/storage/offline_download.test.cpp @@ -6,7 +6,6 @@ #include #include -#include #include #include #include @@ -64,7 +63,7 @@ public: } util::RunLoop loop; - StubOnlineFileSource fileSource; + StubFileSource fileSource; OfflineDatabase db; std::size_t size = 0; @@ -386,7 +385,7 @@ TEST(OfflineDownload, DoesNotFloodTheFileSourceWithRequests) { fileSource.respond(Resource::Kind::Style, test.response("style.json")); test.loop.runOnce(); - EXPECT_EQ(fileSource.getMaximumConcurrentRequests(), fileSource.requests.size()); + EXPECT_EQ(*fileSource.getProperty("max-concurrent-requests").getUint(), fileSource.requests.size()); } TEST(OfflineDownload, GetStatusNoResources) { @@ -822,7 +821,6 @@ TEST(OfflineDownload, AllOfflineRequestsHaveLowPriorityAndOfflineUsage) { test.loop.run(); } - #ifndef __QT__ // Qt doesn't expose the ability to register virtual file system handlers. TEST(OfflineDownload, DiskFull) { FixtureLog log; diff --git a/test/storage/online_file_source.test.cpp b/test/storage/online_file_source.test.cpp index 3e697a99ea..88dbf519f8 100644 --- a/test/storage/online_file_source.test.cpp +++ b/test/storage/online_file_source.test.cpp @@ -1,11 +1,12 @@ -#include -#include #include +#include +#include +#include #include +#include #include -#include #include -#include +#include #include @@ -52,9 +53,9 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(TemporaryError)) { OnlineFileSource fs; const auto start = Clock::now(); + int counter = 0; auto req = fs.request({ Resource::Unknown, "http://127.0.0.1:3000/temporary-error" }, [&](Response res) { - static int counter = 0; switch (counter++) { case 0: { const auto duration = std::chrono::duration(Clock::now() - start).count(); @@ -92,10 +93,10 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(ConnectionError)) { OnlineFileSource fs; const auto start = Clock::now(); + int counter = 0; + int wait = 0; std::unique_ptr req = fs.request({ Resource::Unknown, "http://127.0.0.1:3001/" }, [&](Response res) { - static int counter = 0; - static int wait = 0; const auto duration = std::chrono::duration(Clock::now() - start).count(); EXPECT_LT(wait - 0.01, duration) << "Backoff timer didn't wait 1 second"; EXPECT_GT(wait + 0.3, duration) << "Backoff timer fired too late"; @@ -157,10 +158,6 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(RetryDelayOnExpiredTile)) { EXPECT_EQ(nullptr, res.error); EXPECT_GT(util::now(), *res.expires); EXPECT_FALSE(res.mustRevalidate); - }); - - util::Timer timer; - timer.start(Milliseconds(500), Duration::zero(), [&] () { loop.stop(); }); @@ -306,12 +303,13 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(NetworkStatusChange)) { TEST(OnlineFileSource, TEST_REQUIRES_SERVER(NetworkStatusChangePreempt)) { util::RunLoop loop; OnlineFileSource fs; + fs.pause(); const auto start = Clock::now(); + int counter = 0; const Resource resource{ Resource::Unknown, "http://127.0.0.1:3001/test" }; std::unique_ptr req = fs.request(resource, [&](Response res) { - static int counter = 0; const auto duration = std::chrono::duration(Clock::now() - start).count(); if (counter == 0) { EXPECT_GT(0.2, duration) << "Response came in too late"; @@ -341,6 +339,7 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(NetworkStatusChangePreempt)) { mbgl::NetworkStatus::Reachable(); }); + fs.resume(); loop.run(); } @@ -416,14 +415,30 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(RateLimitDefault)) { loop.run(); } +TEST(OnlineFileSource, GetBaseURLAndAccessTokenWhilePaused) { + util::RunLoop loop; + OnlineFileSource fs; + + fs.pause(); + + auto baseURL = "http://url"; + auto accessToken = "access_token"; + + fs.setProperty(API_BASE_URL_KEY, baseURL); + fs.setProperty(ACCESS_TOKEN_KEY, accessToken); + + EXPECT_EQ(*fs.getProperty(API_BASE_URL_KEY).getString(), baseURL); + EXPECT_EQ(*fs.getProperty(ACCESS_TOKEN_KEY).getString(), accessToken); +} + TEST(OnlineFileSource, ChangeAPIBaseURL){ util::RunLoop loop; OnlineFileSource fs; - EXPECT_EQ(mbgl::util::API_BASE_URL, fs.getAPIBaseURL()); + EXPECT_EQ(mbgl::util::API_BASE_URL, *fs.getProperty(API_BASE_URL_KEY).getString()); const std::string customURL = "test.domain"; - fs.setAPIBaseURL(customURL); - EXPECT_EQ(customURL, fs.getAPIBaseURL()); + fs.setProperty(API_BASE_URL_KEY, customURL); + EXPECT_EQ(customURL, *fs.getProperty(API_BASE_URL_KEY).getString()); } @@ -433,34 +448,38 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(LowHighPriorityRequests)) { std::size_t response_counter = 0; const std::size_t NUM_REQUESTS = 3; - fs.setMaximumConcurrentRequests(1); - NetworkStatus::Set(NetworkStatus::Status::Offline); - - // requesting a low priority resource - Resource low_prio{ Resource::Unknown, "http://127.0.0.1:3000/load/1" }; - low_prio.setPriority(Resource::Priority::Low); - std::unique_ptr req_0 = fs.request(low_prio, [&](Response) { + fs.setProperty("max-concurrent-requests", 1u); + // After DefaultFileSource was split, OnlineFileSource lives on a separate + // thread. Pause OnlineFileSource, so that messages are queued for processing. + fs.pause(); + + // First regular request. + Resource regular1{Resource::Unknown, "http://127.0.0.1:3000/load/1"}; + std::unique_ptr req_0 = fs.request(regular1, [&](Response) { response_counter++; req_0.reset(); - EXPECT_EQ(response_counter, NUM_REQUESTS); // make sure this is responded last - loop.stop(); }); - // requesting two "regular" resources - Resource regular1{ Resource::Unknown, "http://127.0.0.1:3000/load/2" }; - std::unique_ptr req_1 = fs.request(regular1, [&](Response) { + // Low priority request that will be queued and should be requested last. + Resource low_prio{Resource::Unknown, "http://127.0.0.1:3000/load/2"}; + low_prio.setPriority(Resource::Priority::Low); + std::unique_ptr req_1 = fs.request(low_prio, [&](Response) { response_counter++; req_1.reset(); + EXPECT_EQ(response_counter, NUM_REQUESTS); // make sure this is responded last + loop.stop(); }); + + // Second regular priority request that should de-preoritize low priority request. Resource regular2{ Resource::Unknown, "http://127.0.0.1:3000/load/3" }; std::unique_ptr req_2 = fs.request(regular2, [&](Response) { response_counter++; req_2.reset(); }); + fs.resume(); NetworkStatus::Set(NetworkStatus::Status::Online); - loop.run(); } @@ -472,10 +491,9 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(LowHighPriorityRequestsMany)) { int correct_low = 0; int correct_regular = 0; - - fs.setMaximumConcurrentRequests(1); - NetworkStatus::Set(NetworkStatus::Status::Offline); + fs.setProperty("max-concurrent-requests", 1u); + fs.pause(); std::vector> collector; @@ -515,8 +533,8 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(LowHighPriorityRequestsMany)) { } } + fs.resume(); NetworkStatus::Set(NetworkStatus::Status::Online); - loop.run(); } @@ -524,11 +542,12 @@ TEST(OnlineFileSource, TEST_REQUIRES_SERVER(MaximumConcurrentRequests)) { util::RunLoop loop; OnlineFileSource fs; - ASSERT_EQ(fs.getMaximumConcurrentRequests(), 20u); + ASSERT_EQ(*fs.getProperty("max-concurrent-requests").getUint(), 20u); - fs.setMaximumConcurrentRequests(10); - ASSERT_EQ(fs.getMaximumConcurrentRequests(), 10u); + fs.setProperty("max-concurrent-requests", 10u); + ASSERT_EQ(*fs.getProperty("max-concurrent-requests").getUint(), 10u); } + TEST(OnlineFileSource, TEST_REQUIRES_SERVER(RequestSameUrlMultipleTimes)) { util::RunLoop loop; OnlineFileSource fs; diff --git a/test/storage/sync_file_source.test.cpp b/test/storage/sync_file_source.test.cpp index 4bd964199d..3cd6cd9f6e 100644 --- a/test/storage/sync_file_source.test.cpp +++ b/test/storage/sync_file_source.test.cpp @@ -1,11 +1,12 @@ -#include -#include -#include #include #include #include +#include +#include #include #include +#include +#include #include #include @@ -14,7 +15,7 @@ using namespace mbgl; class SyncFileSource : public FileSource { public: - std::unique_ptr request(const Resource& resource, FileSource::Callback callback) { + std::unique_ptr request(const Resource& resource, FileSource::Callback callback) override { Response response; auto it = assets.find(resource.url); if (it == assets.end()) { @@ -27,6 +28,8 @@ public: return nullptr; } + bool canRequest(const Resource&) const override { return true; } + void add(std::string const& key, std::string const& data) { assets.emplace(key, std::make_shared(data)); }; diff --git a/test/style/source.test.cpp b/test/style/source.test.cpp index 5eb837d92b..0286aaaec3 100644 --- a/test/style/source.test.cpp +++ b/test/style/source.test.cpp @@ -58,7 +58,7 @@ public: StubRenderSourceObserver renderSourceObserver; Transform transform; TransformState transformState; - Style style { *fileSource, 1 }; + Style style{fileSource, 1}; AnnotationManager annotationManager { style }; ImageManager imageManager; GlyphManager glyphManager; diff --git a/test/style/style.test.cpp b/test/style/style.test.cpp index b9e19d5a85..c866431ac1 100644 --- a/test/style/style.test.cpp +++ b/test/style/style.test.cpp @@ -18,7 +18,7 @@ using namespace mbgl::style; TEST(Style, Properties) { util::RunLoop loop; - StubFileSource fileSource; + auto fileSource = std::make_shared(); Style::Impl style { fileSource, 1.0 }; style.loadJSON(R"STYLE({"name": "Test"})STYLE"); @@ -60,7 +60,7 @@ TEST(Style, Properties) { TEST(Style, DuplicateSource) { util::RunLoop loop; - StubFileSource fileSource; + auto fileSource = std::make_shared(); Style::Impl style { fileSource, 1.0 }; style.loadJSON(util::read_file("test/fixtures/resources/style-unused-sources.json")); @@ -81,7 +81,7 @@ TEST(Style, RemoveSourceInUse) { auto log = new FixtureLogObserver(); Log::setObserver(std::unique_ptr(log)); - StubFileSource fileSource; + auto fileSource = std::make_shared(); Style::Impl style { fileSource, 1.0 }; style.loadJSON(util::read_file("test/fixtures/resources/style-unused-sources.json")); @@ -106,7 +106,7 @@ TEST(Style, RemoveSourceInUse) { TEST(Style, SourceImplsOrder) { util::RunLoop loop; - StubFileSource fileSource; + auto fileSource = std::make_shared(); Style::Impl style{fileSource, 1.0}; style.addSource(std::make_unique("c", "mapbox://mapbox.mapbox-terrain-v2")); diff --git a/test/style/style_layer.test.cpp b/test/style/style_layer.test.cpp index 1d60197c2d..77e936ff3b 100644 --- a/test/style/style_layer.test.cpp +++ b/test/style/style_layer.test.cpp @@ -283,7 +283,7 @@ TEST(Layer, DuplicateLayer) { util::RunLoop loop; // Setup style - StubFileSource fileSource; + auto fileSource = std::make_shared(); Style::Impl style { fileSource, 1.0 }; style.loadJSON(util::read_file("test/fixtures/resources/style-unused-sources.json")); @@ -304,7 +304,7 @@ TEST(Layer, IncompatibleLayer) { util::RunLoop loop; // Setup style - StubFileSource fileSource; + auto fileSource = std::make_shared(); Style::Impl style{fileSource, 1.0}; style.loadJSON(util::read_file("test/fixtures/resources/style-unused-sources.json")); diff --git a/test/test-files.json b/test/test-files.json index b997503877..8691583765 100644 --- a/test/test-files.json +++ b/test/test-files.json @@ -36,10 +36,10 @@ "test/src/mbgl/test/test.cpp", "test/src/mbgl/test/util.cpp", "test/storage/asset_file_source.test.cpp", - "test/storage/default_file_source.test.cpp", "test/storage/headers.test.cpp", "test/storage/http_file_source.test.cpp", "test/storage/local_file_source.test.cpp", + "test/storage/main_resource_loader.test.cpp", "test/storage/offline.test.cpp", "test/storage/offline_database.test.cpp", "test/storage/offline_download.test.cpp", diff --git a/test/tile/custom_geometry_tile.test.cpp b/test/tile/custom_geometry_tile.test.cpp index fb905ac076..f3d11ab898 100644 --- a/test/tile/custom_geometry_tile.test.cpp +++ b/test/tile/custom_geometry_tile.test.cpp @@ -25,7 +25,7 @@ public: std::shared_ptr fileSource = std::make_shared(); TransformState transformState; util::RunLoop loop; - style::Style style { *fileSource, 1 }; + style::Style style{fileSource, 1}; AnnotationManager annotationManager { style }; ImageManager imageManager; GlyphManager glyphManager; diff --git a/test/tile/geojson_tile.test.cpp b/test/tile/geojson_tile.test.cpp index d4bf1e0752..25fd268dc8 100644 --- a/test/tile/geojson_tile.test.cpp +++ b/test/tile/geojson_tile.test.cpp @@ -25,7 +25,7 @@ public: std::shared_ptr fileSource = std::make_shared(); TransformState transformState; util::RunLoop loop; - style::Style style { *fileSource, 1 }; + style::Style style{fileSource, 1}; AnnotationManager annotationManager { style }; ImageManager imageManager; GlyphManager glyphManager; diff --git a/test/tile/raster_dem_tile.test.cpp b/test/tile/raster_dem_tile.test.cpp index 42e7594720..f5f7610096 100644 --- a/test/tile/raster_dem_tile.test.cpp +++ b/test/tile/raster_dem_tile.test.cpp @@ -19,7 +19,7 @@ public: std::shared_ptr fileSource = std::make_shared(); TransformState transformState; util::RunLoop loop; - style::Style style { *fileSource, 1 }; + style::Style style{fileSource, 1}; AnnotationManager annotationManager { style }; ImageManager imageManager; GlyphManager glyphManager; diff --git a/test/tile/raster_tile.test.cpp b/test/tile/raster_tile.test.cpp index f19bd26260..a5a2875f2e 100644 --- a/test/tile/raster_tile.test.cpp +++ b/test/tile/raster_tile.test.cpp @@ -19,7 +19,7 @@ public: std::shared_ptr fileSource = std::make_shared(); TransformState transformState; util::RunLoop loop; - style::Style style { *fileSource, 1 }; + style::Style style{fileSource, 1}; AnnotationManager annotationManager { style }; ImageManager imageManager; GlyphManager glyphManager; diff --git a/test/tile/tile_cache.test.cpp b/test/tile/tile_cache.test.cpp index 7a89ece756..43b409ae87 100644 --- a/test/tile/tile_cache.test.cpp +++ b/test/tile/tile_cache.test.cpp @@ -28,7 +28,7 @@ public: std::shared_ptr fileSource = std::make_shared(); TransformState transformState; util::RunLoop loop; - style::Style style{*fileSource, 1}; + style::Style style{fileSource, 1}; AnnotationManager annotationManager{style}; ImageManager imageManager; GlyphManager glyphManager; diff --git a/test/tile/vector_tile.test.cpp b/test/tile/vector_tile.test.cpp index 940c0272db..d282c874ef 100644 --- a/test/tile/vector_tile.test.cpp +++ b/test/tile/vector_tile.test.cpp @@ -25,7 +25,7 @@ public: std::shared_ptr fileSource = std::make_shared(); TransformState transformState; util::RunLoop loop; - style::Style style { *fileSource, 1 }; + style::Style style{fileSource, 1}; AnnotationManager annotationManager { style }; ImageManager imageManager; GlyphManager glyphManager; -- cgit v1.2.1