diff options
Diffstat (limited to 'src/mbgl')
26 files changed, 685 insertions, 1042 deletions
diff --git a/src/mbgl/map/map.cpp b/src/mbgl/map/map.cpp index 3359dd317f..112499a485 100644 --- a/src/mbgl/map/map.cpp +++ b/src/mbgl/map/map.cpp @@ -24,6 +24,7 @@ #include <mbgl/platform/log.hpp> #include <mbgl/util/string.hpp> #include <mbgl/util/uv.hpp> +#include <mbgl/util/mapbox.hpp> #include <algorithm> #include <iostream> @@ -127,9 +128,7 @@ Map::~Map() { } uv::worker &Map::getWorker() { - if (!workers) { - workers = util::make_unique<uv::worker>(**loop, 4, "Tile Worker"); - } + assert(workers); return *workers; } @@ -153,8 +152,6 @@ void Map::start(bool startPaused) { workers.reset(); activeSources.clear(); - fileSource.clearLoop(); - terminating = true; // Closes all open handles on the loop. This means that the loop will automatically terminate. @@ -195,6 +192,8 @@ void Map::start(bool startPaused) { pthread_setname_np("Map"); #endif + workers = util::make_unique<uv::worker>(**loop, 4, "Tile Worker"); + run(); #ifndef NDEBUG @@ -301,7 +300,6 @@ void Map::run() { mapThread = std::thread::id(); #endif mode = Mode::None; - fileSource.clearLoop(); } view.deactivate(); @@ -386,13 +384,14 @@ void Map::setStyleJSON(std::string newStyleJSON, const std::string &base) { style = std::make_shared<Style>(); } + style->base = base; style->loadJSON((const uint8_t *)styleJSON.c_str()); style->cascadeClasses(classes); - fileSource.setBase(base); - glyphStore->setURL(style->glyph_url); - style->setDefaultTransitionDuration(defaultTransitionDuration); + const std::string glyphURL = util::mapbox::normalizeGlyphsURL(style->glyph_url, getAccessToken()); + glyphStore->setURL(glyphURL); + update(); } @@ -559,6 +558,15 @@ void Map::stopRotating() { update(); } +#pragma mark - Access Token + +void Map::setAccessToken(const std::string &token) { + accessToken = token; +} + +const std::string &Map::getAccessToken() const { + return accessToken; +} #pragma mark - Toggles @@ -674,20 +682,16 @@ void Map::updateTiles() { source->source->update(*this, getWorker(), style, *glyphAtlas, *glyphStore, *spriteAtlas, getSprite(), - *texturePool, fileSource, [this](){ update(); }); + *texturePool, fileSource, ***loop, [this](){ update(); }); } } void Map::prepare() { - if (!fileSource.hasLoop()) { - fileSource.setLoop(**loop); - } - if (!style) { style = std::make_shared<Style>(); - fileSource.request(ResourceType::JSON, styleURL)->onload([&](const Response &res) { - if (res.code == 200) { + fileSource.request({ Resource::Kind::JSON, styleURL}, **loop, [&](const Response &res) { + if (res.status == Response::Successful) { // Calculate the base const size_t pos = styleURL.rfind('/'); std::string base = ""; @@ -697,7 +701,7 @@ void Map::prepare() { setStyleJSON(res.data, base); } else { - Log::Error(Event::Setup, "loading style failed: %ld (%s)", res.code, res.message.c_str()); + Log::Error(Event::Setup, "loading style failed: %s", res.message.c_str()); } }); } diff --git a/src/mbgl/map/raster_tile_data.cpp b/src/mbgl/map/raster_tile_data.cpp index 6fac7862e7..84e9bb236a 100644 --- a/src/mbgl/map/raster_tile_data.cpp +++ b/src/mbgl/map/raster_tile_data.cpp @@ -5,8 +5,8 @@ using namespace mbgl; -RasterTileData::RasterTileData(Tile::ID const& id_, TexturePool& texturePool, const SourceInfo& source_) - : TileData(id_, source_), +RasterTileData::RasterTileData(Tile::ID const& id_, TexturePool& texturePool, const SourceInfo& source_, FileSource& fileSource_) + : TileData(id_, source_, fileSource_), bucket(texturePool, properties) { } diff --git a/src/mbgl/map/raster_tile_data.hpp b/src/mbgl/map/raster_tile_data.hpp index 42070d9c61..7f338056f5 100644 --- a/src/mbgl/map/raster_tile_data.hpp +++ b/src/mbgl/map/raster_tile_data.hpp @@ -16,7 +16,7 @@ class RasterTileData : public TileData { friend class TileParser; public: - RasterTileData(Tile::ID const& id, TexturePool&, const SourceInfo&); + RasterTileData(Tile::ID const& id, TexturePool&, const SourceInfo&, FileSource &); ~RasterTileData(); virtual void parse(); diff --git a/src/mbgl/map/source.cpp b/src/mbgl/map/source.cpp index 798cd41d1d..f23bcaa14a 100644 --- a/src/mbgl/map/source.cpp +++ b/src/mbgl/map/source.cpp @@ -15,6 +15,7 @@ #include <mbgl/geometry/glyph_atlas.hpp> #include <mbgl/style/style_layer.hpp> #include <mbgl/platform/log.hpp> +#include <mbgl/util/uv_detail.hpp> #include <mbgl/map/vector_tile_data.hpp> #include <mbgl/map/raster_tile_data.hpp> @@ -39,8 +40,9 @@ void Source::load(Map& map, FileSource& fileSource) { util::ptr<Source> source = shared_from_this(); - fileSource.request(ResourceType::JSON, info.url)->onload([source, &map](const Response &res) { - if (res.code != 200) { + const std::string url = util::mapbox::normalizeSourceURL(info.url, map.getAccessToken()); + fileSource.request({ Resource::Kind::JSON, url }, **map.loop, [source, &map](const Response &res) { + if (res.status != Response::Successful) { Log::Warning(Event::General, "failed to load source TileJSON"); return; } @@ -155,7 +157,7 @@ TileData::State Source::addTile(Map& map, uv::worker& worker, util::ptr<Style> style, GlyphAtlas& glyphAtlas, GlyphStore& glyphStore, SpriteAtlas& spriteAtlas, util::ptr<Sprite> sprite, - FileSource& fileSource, TexturePool& texturePool, + FileSource& fileSource, uv_loop_t &loop, TexturePool& texturePool, const Tile::ID& id, std::function<void ()> callback) { const TileData::State state = hasTile(id); @@ -188,14 +190,14 @@ TileData::State Source::addTile(Map& map, uv::worker& worker, new_tile.data = std::make_shared<VectorTileData>(normalized_id, map.getMaxZoom(), style, glyphAtlas, glyphStore, spriteAtlas, sprite, - texturePool, info); + texturePool, info, fileSource); } else if (info.type == SourceType::Raster) { - new_tile.data = std::make_shared<RasterTileData>(normalized_id, texturePool, info); + new_tile.data = std::make_shared<RasterTileData>(normalized_id, texturePool, info, fileSource); } else { throw std::runtime_error("source type not implemented"); } - new_tile.data->request(worker, fileSource, map.getState().getPixelRatio(), callback); + new_tile.data->request(worker, loop, map.getState().getPixelRatio(), callback); tile_data.emplace(new_tile.data->id, new_tile.data); } @@ -286,7 +288,7 @@ void Source::update(Map& map, uv::worker& worker, util::ptr<Style> style, GlyphAtlas& glyphAtlas, GlyphStore& glyphStore, SpriteAtlas& spriteAtlas, util::ptr<Sprite> sprite, - TexturePool& texturePool, FileSource& fileSource, + TexturePool& texturePool, FileSource& fileSource, uv_loop_t& loop, std::function<void ()> callback) { if (!loaded || map.getTime() <= updated) return; @@ -310,7 +312,7 @@ void Source::update(Map& map, uv::worker& worker, const TileData::State state = addTile(map, worker, style, glyphAtlas, glyphStore, spriteAtlas, sprite, - fileSource, texturePool, + fileSource, loop, texturePool, id, callback); if (state != TileData::State::parsed) { diff --git a/src/mbgl/map/source.hpp b/src/mbgl/map/source.hpp index f0023afa09..3649837a58 100644 --- a/src/mbgl/map/source.hpp +++ b/src/mbgl/map/source.hpp @@ -39,7 +39,7 @@ public: util::ptr<Style>, GlyphAtlas&, GlyphStore&, SpriteAtlas&, util::ptr<Sprite>, - TexturePool&, FileSource&, + TexturePool&, FileSource&, uv_loop_t& loop, std::function<void ()> callback); void updateMatrices(const mat4 &projMatrix, const TransformState &transform); @@ -63,7 +63,7 @@ private: util::ptr<Style>, GlyphAtlas&, GlyphStore&, SpriteAtlas&, util::ptr<Sprite>, - FileSource&, TexturePool&, + FileSource&, uv_loop_t &, TexturePool&, const Tile::ID&, std::function<void ()> callback); diff --git a/src/mbgl/map/sprite.cpp b/src/mbgl/map/sprite.cpp index c1f71e59d9..a3e264b762 100644 --- a/src/mbgl/map/sprite.cpp +++ b/src/mbgl/map/sprite.cpp @@ -52,6 +52,7 @@ Sprite::operator bool() const { // The reason this isn't part of the constructor is that calling shared_from_this() in // the constructor fails. void Sprite::load(FileSource& fileSource) { + if (!valid) { // Treat a non-existent sprite as a successfully loaded empty sprite. loadedImage = true; @@ -62,26 +63,26 @@ void Sprite::load(FileSource& fileSource) { util::ptr<Sprite> sprite = shared_from_this(); - fileSource.request(ResourceType::JSON, jsonURL)->onload([sprite](const Response &res) { - if (res.code == 200) { + fileSource.request({ Resource::Kind::JSON, jsonURL }, [sprite](const Response &res) { + if (res.status == Response::Successful) { sprite->body = res.data; sprite->parseJSON(); sprite->complete(); } else { - Log::Warning(Event::Sprite, "Failed to load sprite info: Error %d: %s", res.code, res.message.c_str()); + Log::Warning(Event::Sprite, "Failed to load sprite info: %s", res.message.c_str()); if (!sprite->future.valid()) { sprite->promise.set_exception(std::make_exception_ptr(std::runtime_error(res.message))); } } }); - fileSource.request(ResourceType::Image, spriteURL)->onload([sprite](const Response &res) { - if (res.code == 200) { + fileSource.request({ Resource::Kind::Image, spriteURL }, [sprite](const Response &res) { + if (res.status == Response::Successful) { sprite->image = res.data; sprite->parseImage(); sprite->complete(); } else { - Log::Warning(Event::Sprite, "Failed to load sprite image: Error %d: %s", res.code, res.message.c_str()); + Log::Warning(Event::Sprite, "Failed to load sprite image: Error %s", res.message.c_str()); if (!sprite->future.valid()) { sprite->promise.set_exception(std::make_exception_ptr(std::runtime_error(res.message))); } diff --git a/src/mbgl/map/tile_data.cpp b/src/mbgl/map/tile_data.cpp index f89ff15baf..9d48041239 100644 --- a/src/mbgl/map/tile_data.cpp +++ b/src/mbgl/map/tile_data.cpp @@ -6,29 +6,33 @@ #include <mbgl/util/string.hpp> #include <mbgl/storage/file_source.hpp> #include <mbgl/util/uv_detail.hpp> +#include <mbgl/platform/log.hpp> using namespace mbgl; -TileData::TileData(Tile::ID const& id_, const SourceInfo& source_) +TileData::TileData(Tile::ID const& id_, const SourceInfo& source_, FileSource& fileSource_) : id(id_), name(id), state(State::initial), source(source_), + fileSource(fileSource_), debugBucket(debugFontBuffer) { // Initialize tile debug coordinates debugFontBuffer.addText(name.c_str(), 50, 200, 5); } TileData::~TileData() { - cancel(); + if (req) { + fileSource.cancel(req); + } } const std::string TileData::toString() const { return std::string { "[tile " } + name + "]"; } -void TileData::request(uv::worker& worker, FileSource& fileSource, - float pixelRatio, std::function<void ()> callback) { +void TileData::request(uv::worker &worker, uv_loop_t &loop, + float pixelRatio, std::function<void()> callback) { if (source.tiles.empty()) return; @@ -51,8 +55,7 @@ void TileData::request(uv::worker& worker, FileSource& fileSource, // Note: Somehow this feels slower than the change to request_http() std::weak_ptr<TileData> weak_tile = shared_from_this(); - req = fileSource.request(ResourceType::Tile, url); - req->onload([weak_tile, url, callback, &worker](const Response &res) { + req = fileSource.request({ Resource::Kind::Tile, url }, &loop, [weak_tile, url, callback, &worker](const Response &res) { util::ptr<TileData> tile = weak_tile.lock(); if (!tile || tile->state == State::obsolete) { // noop. Tile is obsolete and we're now just waiting for the refcount @@ -61,9 +64,9 @@ void TileData::request(uv::worker& worker, FileSource& fileSource, } // Clear the request object. - tile->req.reset(); + tile->req = nullptr; - if (res.code == 200) { + if (res.status == Response::Successful) { tile->state = State::loaded; tile->data = res.data; @@ -71,9 +74,7 @@ void TileData::request(uv::worker& worker, FileSource& fileSource, // Schedule tile parsing in another thread tile->reparse(worker, callback); } else { -#if defined(DEBUG) - fprintf(stderr, "[%s] tile loading failed: %ld, %s\n", url.c_str(), res.code, res.message.c_str()); -#endif + Log::Error(Event::HttpRequest, "[%s] tile loading failed: %s", url.c_str(), res.message.c_str()); } }); } @@ -81,10 +82,10 @@ void TileData::request(uv::worker& worker, FileSource& fileSource, void TileData::cancel() { if (state != State::obsolete) { state = State::obsolete; - if (req) { - req->cancel(); - req.reset(); - } + } + if (req) { + fileSource.cancel(req); + req = nullptr; } } diff --git a/src/mbgl/map/tile_data.hpp b/src/mbgl/map/tile_data.hpp index 1ae215b204..a83a4648dd 100644 --- a/src/mbgl/map/tile_data.hpp +++ b/src/mbgl/map/tile_data.hpp @@ -18,6 +18,8 @@ namespace uv { class worker; } +typedef struct uv_loop_s uv_loop_t; + namespace mbgl { class Map; @@ -46,10 +48,10 @@ public: }; public: - TileData(Tile::ID const& id, const SourceInfo&); + TileData(Tile::ID const& id, const SourceInfo&, FileSource&); ~TileData(); - void request(uv::worker&, FileSource&, float pixelRatio, std::function<void ()> callback); + void request(uv::worker&, uv_loop_t&, float pixelRatio, std::function<void ()> callback); void reparse(uv::worker&, std::function<void ()> callback); void cancel(); const std::string toString() const; @@ -71,9 +73,10 @@ public: public: const SourceInfo& source; + FileSource& fileSource; protected: - std::unique_ptr<Request> req; + Request *req = nullptr; std::string data; // Contains the tile ID string for painting debug information. diff --git a/src/mbgl/map/vector_tile_data.cpp b/src/mbgl/map/vector_tile_data.cpp index 06782057f6..ca98d2695b 100644 --- a/src/mbgl/map/vector_tile_data.cpp +++ b/src/mbgl/map/vector_tile_data.cpp @@ -5,6 +5,7 @@ #include <mbgl/style/style_layer.hpp> #include <mbgl/style/style_bucket.hpp> #include <mbgl/geometry/glyph_atlas.hpp> +#include <mbgl/platform/log.hpp> using namespace mbgl; @@ -13,8 +14,8 @@ VectorTileData::VectorTileData(Tile::ID const& id_, GlyphAtlas& glyphAtlas_, GlyphStore& glyphStore_, SpriteAtlas& spriteAtlas_, util::ptr<Sprite> sprite_, TexturePool& texturePool_, - const SourceInfo& source_) - : TileData(id_, source_), + const SourceInfo& source_, FileSource &fileSource_) + : TileData(id_, source_, fileSource_), glyphAtlas(glyphAtlas_), glyphStore(glyphStore_), spriteAtlas(spriteAtlas_), @@ -44,10 +45,8 @@ void VectorTileData::parse() { texturePool); parser.parse(); } catch (const std::exception& ex) { -#if defined(DEBUG) - fprintf(stderr, "[%p] exception [%d/%d/%d]... failed: %s\n", this, id.z, id.x, id.y, ex.what()); -#endif - cancel(); + Log::Error(Event::ParseTile, "Parsing [%d/%d/%d] failed: %s", id.z, id.x, id.y, ex.what()); + state = State::obsolete; return; } diff --git a/src/mbgl/map/vector_tile_data.hpp b/src/mbgl/map/vector_tile_data.hpp index b9bf55a1b3..31318003af 100644 --- a/src/mbgl/map/vector_tile_data.hpp +++ b/src/mbgl/map/vector_tile_data.hpp @@ -36,7 +36,7 @@ public: GlyphAtlas&, GlyphStore&, SpriteAtlas&, util::ptr<Sprite>, TexturePool&, - const SourceInfo&); + const SourceInfo&, FileSource &); ~VectorTileData(); virtual void parse(); diff --git a/src/mbgl/storage/base_request.cpp b/src/mbgl/storage/base_request.cpp deleted file mode 100644 index 510bd7bf1c..0000000000 --- a/src/mbgl/storage/base_request.cpp +++ /dev/null @@ -1,87 +0,0 @@ -#include <mbgl/storage/base_request.hpp> -#include <mbgl/storage/response.hpp> -#include <mbgl/storage/request.hpp> -#include <mbgl/util/std.hpp> - -#include <uv.h> - -#include <cassert> - -namespace mbgl { - -template <typename T, typename ...Args> -void invoke(const std::forward_list<std::unique_ptr<Callback>> &list, Args&& ...args) { - for (const std::unique_ptr<Callback> &callback : list) { - assert(callback); - if (callback->is<T>()) { - callback->get<T>()(::std::forward<Args>(args)...); - } - } -} - -BaseRequest::BaseRequest(const std::string &path_) : threadId(std::this_thread::get_id()), path(path_) { -} - -// A base request can only be "canceled" by destroying the object. In that case, we'll have to -// notify all cancel callbacks. -BaseRequest::~BaseRequest() { - assert(std::this_thread::get_id() == threadId); - notify(); -} - -void BaseRequest::retryImmediately() { - // no-op. override in child class. -} - -void BaseRequest::notify() { - assert(std::this_thread::get_id() == threadId); - - // The parameter exists solely so that any calls to ->remove() - // are not going to cause deallocation of this object while this call is in progress. - util::ptr<BaseRequest> retain = self; - - // Swap the lists so that it's safe for callbacks to call ->cancel() - // on the request object, which would modify the list. - const std::forward_list<std::unique_ptr<Callback>> list = std::move(callbacks); - callbacks.clear(); - - if (response) { - invoke<CompletedCallback>(list, *response); - } else { - invoke<AbortedCallback>(list); - } - - self.reset(); -} - -Callback *BaseRequest::add(Callback &&callback, const util::ptr<BaseRequest> &request) { - assert(std::this_thread::get_id() == threadId); - assert(this == request.get()); - - if (response) { - // We already have a response. Notify right away. - if (callback.is<CompletedCallback>()) { - callback.get<CompletedCallback>()(*response); - } else { - // We already know that this request was successful. The AbortedCallback will be discarded - // here since it would never be called. - } - return nullptr; - } else { - self = request; - callbacks.push_front(util::make_unique<Callback>(std::move(callback))); - return callbacks.front().get(); - } -} - -void BaseRequest::remove(Callback *callback) { - assert(std::this_thread::get_id() == threadId); - callbacks.remove_if([=](const std::unique_ptr<Callback> &cb) { - return cb.get() == callback; - }); - if (callbacks.empty()) { - self.reset(); - } -} - -} diff --git a/src/mbgl/storage/base_request.hpp b/src/mbgl/storage/base_request.hpp deleted file mode 100644 index 5119c343e9..0000000000 --- a/src/mbgl/storage/base_request.hpp +++ /dev/null @@ -1,62 +0,0 @@ -#ifndef MBGL_STORAGE_BASE_REQUEST -#define MBGL_STORAGE_BASE_REQUEST - -#include <mbgl/storage/request_callback.hpp> -#include <mbgl/util/ptr.hpp> - -#include <string> -#include <forward_list> -#include <functional> -#include <thread> - -typedef struct uv_loop_s uv_loop_t; -typedef struct uv_async_s uv_async_t; - -namespace mbgl { - -class Response; -class Request; - -class BaseRequest { -private: - // Make noncopyable and immovable - BaseRequest(const BaseRequest &) = delete; - BaseRequest(BaseRequest &&) = delete; - BaseRequest& operator=(const BaseRequest &) = delete; - BaseRequest& operator=(BaseRequest &&) = delete; - -public: - BaseRequest(const std::string &path); - virtual ~BaseRequest(); - - Callback *add(Callback &&callback, const util::ptr<BaseRequest> &request); - void remove(Callback *callback); - - // Must be called by subclasses when a valid Response object is available. It will notify - // all listeners. - void notify(); - - // This function is called when the request ought to be stopped. Any subclass must make sure this - // is also called in its destructor. Calling this function repeatedly must be safe. - // This function must call notify(). - virtual void cancel() = 0; - - // This function is called when the request should be reattempted immediately. This is typically - // reaction to a network status change. - virtual void retryImmediately(); - -public: - const std::thread::id threadId; - const std::string path; - std::unique_ptr<Response> response; - -protected: - // This object may hold a shared_ptr to itself. It does this to prevent destruction of this object - // while a request is in progress. - util::ptr<BaseRequest> self; - std::forward_list<std::unique_ptr<Callback>> callbacks; -}; - -} - -#endif diff --git a/src/mbgl/storage/caching_http_file_source.cpp b/src/mbgl/storage/caching_http_file_source.cpp deleted file mode 100644 index 634a56f9c4..0000000000 --- a/src/mbgl/storage/caching_http_file_source.cpp +++ /dev/null @@ -1,136 +0,0 @@ -#include <mbgl/storage/caching_http_file_source.hpp> -#include <mbgl/storage/asset_request.hpp> -#include <mbgl/storage/http_request.hpp> -#include <mbgl/storage/sqlite_store.hpp> -#include <mbgl/storage/asset_request.hpp> -#include <mbgl/util/uv-messenger.h> -#include <mbgl/util/mapbox.hpp> -#include <mbgl/util/std.hpp> - -#include <uv.h> - -namespace mbgl { - -CachingHTTPFileSource::CachingHTTPFileSource(const std::string &path_) - : path(path_) {} - -CachingHTTPFileSource::~CachingHTTPFileSource() { -} - -void CachingHTTPFileSource::setLoop(uv_loop_t* loop_) { - assert(!loop); - - threadId = std::this_thread::get_id(); - store = !path.empty() ? util::ptr<SQLiteStore>(new SQLiteStore(loop_, path)) : nullptr; - loop = loop_; - queue = new uv_messenger_t; - - uv_messenger_init(loop, queue, [](void *ptr) { - std::unique_ptr<std::function<void()>> fn { reinterpret_cast<std::function<void()> *>(ptr) }; - (*fn)(); - }); - uv_unref((uv_handle_t *)&queue->async); -} - -bool CachingHTTPFileSource::hasLoop() { - return loop; -} - -void CachingHTTPFileSource::clearLoop() { - assert(std::this_thread::get_id() == threadId); - assert(loop); - - uv_messenger_stop(queue, [](uv_messenger_t *msgr) { - delete msgr; - }); - - util::ptr<BaseRequest> req; - - // Send a cancel() message to all requests that we are still holding. - for (const std::pair<std::string, std::weak_ptr<BaseRequest>> &pair : pending) { - if ((req = pair.second.lock())) { - req->cancel(); - } - } - - store.reset(); - - loop = nullptr; -} - -void CachingHTTPFileSource::setBase(std::string value) { - // TODO: Make threadsafe. - base.swap(value); -} - -void CachingHTTPFileSource::setAccessToken(std::string value) { - // TODO: Make threadsafe. - accessToken.swap(value); -} - -std::string CachingHTTPFileSource::getAccessToken() const { - return accessToken; -} - -std::unique_ptr<Request> CachingHTTPFileSource::request(ResourceType type, const std::string& url_) { - assert(std::this_thread::get_id() == threadId); - - std::string url = url_; - - // Make URL absolute. - const size_t separator = url.find("://"); - if (separator == std::string::npos) { - url = base + url; - } - - // Normalize mapbox:// URLs. - switch (type) { - case ResourceType::Glyphs: - url = util::mapbox::normalizeGlyphsURL(url, accessToken); - default: - url = util::mapbox::normalizeSourceURL(url, accessToken); - } - - util::ptr<BaseRequest> req; - - // First, try to find an existing Request object. - auto it = pending.find(url); - if (it != pending.end()) { - req = it->second.lock(); - } - - if (!req) { - if (url.substr(0, 8) == "asset://") { - req = std::make_shared<AssetRequest>(url.substr(8), loop); - } else { - req = std::make_shared<HTTPRequest>(type, url, loop, store); - } - - pending.emplace(url, req); - } - - return util::make_unique<Request>(req); -} - -void CachingHTTPFileSource::prepare(std::function<void()> fn) { - if (std::this_thread::get_id() == threadId) { - fn(); - } else { - uv_messenger_send(queue, new std::function<void()>(std::move(fn))); - } -} - -void CachingHTTPFileSource::setReachability(bool reachable) { - if (reachable && loop) { - prepare([this]() { - util::ptr<BaseRequest> req; - for (const std::pair<std::string, std::weak_ptr<BaseRequest>> &pair : pending) { - if ((req = pair.second.lock())) { - req->retryImmediately(); - } - } - }); - } -} - -} diff --git a/src/mbgl/storage/default_file_source.cpp b/src/mbgl/storage/default_file_source.cpp new file mode 100644 index 0000000000..05d87e474b --- /dev/null +++ b/src/mbgl/storage/default_file_source.cpp @@ -0,0 +1,239 @@ +#include <mbgl/storage/default/default_file_source.hpp> +#include <mbgl/storage/default/request.hpp> +#include <mbgl/storage/default/asset_request.hpp> +#include <mbgl/storage/default/http_request.hpp> + +#include <mbgl/storage/response.hpp> + +#include <mbgl/util/async_queue.hpp> +#include <mbgl/util/util.hpp> + +#include <mbgl/util/variant.hpp> + +#include <boost/algorithm/string.hpp> + +#include <thread> +#include <algorithm> +#include <cassert> + + +namespace algo = boost::algorithm; + +namespace mbgl { + +struct DefaultFileSource::ActionDispatcher { + DefaultFileSource &fileSource; + template <typename T> void operator()(T &t) { fileSource.process(t); } +}; + +struct DefaultFileSource::AddRequestAction { + Request *const request; +}; + +struct DefaultFileSource::RemoveRequestAction { + Request *const request; +}; + +struct DefaultFileSource::ResultAction { + const Resource resource; + std::unique_ptr<Response> response; +}; + +struct DefaultFileSource::StopAction { +}; + + +DefaultFileSource::DefaultFileSource(FileCache *cache_) + : loop(uv_loop_new()), + cache(cache_), + queue(new Queue(loop, [this](Action &action) { + mapbox::util::apply_visitor(ActionDispatcher{*this}, action); + })), + thread([this]() { +#ifdef __APPLE__ + pthread_setname_np("FileSource"); +#endif + uv_run(loop, UV_RUN_DEFAULT); + }) { +} + +DefaultFileSource::DefaultFileSource(FileCache *cache_, uv_loop_t *loop_) + : loop(loop_), + cache(cache_), + queue(new Queue(loop, [this](Action &action) { + mapbox::util::apply_visitor(ActionDispatcher{*this}, action); + })) { + // Make sure that the queue doesn't block the loop from exiting. + queue->unref(); +} + +DefaultFileSource::~DefaultFileSource() { + MBGL_VERIFY_THREAD(tid); + + if (thread.joinable()) { + if (queue) { + queue->send(StopAction{ }); + } + thread.join(); + uv_loop_delete(loop); + } else { + // Assume that the loop we received is running in the current thread. + StopAction action {}; + process(action); + } +} + +SharedRequestBase *DefaultFileSource::find(const Resource &resource) { + // We're using a set of pointers here instead of a map between url and SharedRequestBase because + // we need to find the requests both by pointer and by URL. Given that the number of requests + // is generally very small (typically < 10 at a time), hashing by URL incurs too much overhead + // anyway. + const auto it = pending.find(resource); + if (it != pending.end()) { + return it->second; + } + return nullptr; +} + +Request *DefaultFileSource::request(const Resource &resource, uv_loop_t *l, Callback callback) { + auto request = new Request(resource, l, std::move(callback)); + + // This function can be called from any thread. Make sure we're executing the actual call in the + // file source loop by sending it over the queue. It will be processed in processAction(). + queue->send(AddRequestAction{ request }); + return request; +} + +void DefaultFileSource::request(const Resource &resource, Callback callback) { + auto request = new Request(resource, nullptr, std::move(callback)); + + // This function can be called from any thread. Make sure we're executing the actual call in the + // file source loop by sending it over the queue. It will be processed in processAction(). + queue->send(AddRequestAction{ request }); +} + +void DefaultFileSource::cancel(Request *request) { + request->cancel(); + + // This function can be called from any thread. Make sure we're executing the actual call in the + // file source loop by sending it over the queue. It will be processed in processAction(). + queue->send(RemoveRequestAction{ request }); +} + +void DefaultFileSource::process(AddRequestAction &action) { + const Resource &resource = action.request->resource; + + // We're adding a new Request. + SharedRequestBase *sharedRequest = find(resource); + if (!sharedRequest) { + // There is no request for this URL yet. Create a new one and start it. + if (algo::starts_with(resource.url, "asset://")) { + sharedRequest = new AssetRequest(this, resource); + } else { + sharedRequest = new HTTPRequest(this, resource); + } + + // Make sure the loop stays alive when we're not running the file source in it's own thread. + if (!thread.joinable() && pending.empty()) { + queue->ref(); + } + + const bool inserted = pending.emplace(resource, sharedRequest).second; + assert(inserted); + (void (inserted)); // silence unused variable warning on Release builds. + + // But first, we're going to start querying the database if it exists. + if (!cache) { + sharedRequest->start(loop); + } else { + // Otherwise, first check the cache for existing data so that we can potentially + // revalidate the information without having to redownload everything. + cache->get(resource, [this, resource](std::unique_ptr<Response> response) { + queue->send(ResultAction { resource, std::move(response) }); + }); + } + } + sharedRequest->subscribe(action.request); +} + +void DefaultFileSource::process(RemoveRequestAction &action) { + SharedRequestBase *sharedRequest = find(action.request->resource); + if (sharedRequest) { + // If the number of dependent requests of the SharedRequestBase drops to zero, the + // unsubscribe callback triggers the removal of the SharedRequestBase pointer from the list + // of pending requests and initiates cancelation. + sharedRequest->unsubscribe(action.request); + } else { + // There is no request for this URL anymore. Likely, the request already completed + // before we got around to process the cancelation request. + } + + // Send a message back to the requesting thread and notify it that this request has been + // canceled and is now safe to be deleted. + action.request->destruct(); +} + +void DefaultFileSource::process(ResultAction &action) { + SharedRequestBase *sharedRequest = find(action.resource); + if (sharedRequest) { + if (action.response) { + // This entry was stored in the cache. Now determine if we need to revalidate. + const int64_t now = std::chrono::duration_cast<std::chrono::seconds>( + std::chrono::system_clock::now().time_since_epoch()).count(); + if (action.response->expires > now) { + // The response is fresh. We're good to notify the caller. + sharedRequest->notify(std::move(action.response), FileCache::Hint::No); + sharedRequest->cancel(); + return; + } else { + // The cached response is stale. Now run the real request. + sharedRequest->start(loop, std::move(action.response)); + } + } else { + // There is no response. Now run the real request. + sharedRequest->start(loop); + } + } else { + // There is no request for this URL anymore. Likely, the request was canceled + // before we got around to process the cache result. + } +} + +void DefaultFileSource::process(StopAction &) { + // Cancel all remaining requests. + for (auto it : pending) { + it.second->unsubscribeAll(); + } + pending.clear(); + + assert(queue); + queue->stop(); + queue = nullptr; +} + +void DefaultFileSource::notify(SharedRequestBase *sharedRequest, + const std::set<Request *> &observers, + std::shared_ptr<const Response> response, FileCache::Hint hint) { + // First, remove the request, since it might be destructed at any point now. + assert(find(sharedRequest->resource) == sharedRequest); + pending.erase(sharedRequest->resource); + + if (response) { + if (cache) { + // Store response in database + cache->put(sharedRequest->resource, response, hint); + } + + // Notify all observers. + for (auto it : observers) { + it->notify(response); + } + } + + if (!thread.joinable() && pending.empty()) { + // When there are no pending requests, we're going to allow the queue to stop. + queue->unref(); + } +} + +} diff --git a/src/mbgl/storage/http_request.cpp b/src/mbgl/storage/http_request.cpp deleted file mode 100644 index 57e6c260ef..0000000000 --- a/src/mbgl/storage/http_request.cpp +++ /dev/null @@ -1,280 +0,0 @@ -#include <mbgl/storage/http_request.hpp> -#include <mbgl/storage/sqlite_store.hpp> -#include <mbgl/storage/http_request_baton.hpp> -#include <mbgl/platform/log.hpp> - -#include <uv.h> - -#include <cassert> -#include <chrono> - -namespace mbgl { - -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wdisabled-macro-expansion" -#pragma clang diagnostic ignored "-Wexit-time-destructors" -#pragma clang diagnostic ignored "-Wglobal-constructors" - -struct CacheRequestBaton { - HTTPRequest *request = nullptr; - std::string path; - util::ptr<SQLiteStore> store; -}; - -HTTPRequest::HTTPRequest(ResourceType type_, const std::string &path_, uv_loop_t *loop_, util::ptr<SQLiteStore> store_) - : BaseRequest(path_), threadId(std::this_thread::get_id()), loop(loop_), store(store_), type(type_) { - if (store) { - startCacheRequest(); - } else { - startHTTPRequest(nullptr); - } -} - -void HTTPRequest::startCacheRequest() { - assert(std::this_thread::get_id() == threadId); - - cacheBaton = new CacheRequestBaton; - cacheBaton->request = this; - cacheBaton->path = path; - cacheBaton->store = store; - store->get(path, [](std::unique_ptr<Response> &&response_, void *ptr) { - // Wrap in a unique_ptr, so it'll always get auto-destructed. - std::unique_ptr<CacheRequestBaton> baton((CacheRequestBaton *)ptr); - if (baton->request) { - baton->request->cacheBaton = nullptr; - baton->request->handleCacheResponse(std::move(response_)); - } - }, cacheBaton); -} - -void HTTPRequest::handleCacheResponse(std::unique_ptr<Response> &&res) { - assert(std::this_thread::get_id() == threadId); - - if (res) { - // This entry was stored in the cache. Now determine if we need to revalidate. - const int64_t now = std::chrono::duration_cast<std::chrono::seconds>( - std::chrono::system_clock::now().time_since_epoch()).count(); - if (res->expires > now) { - response = std::move(res); - notify(); - // Note: after calling notify(), the request object may cease to exist. - // This HTTPRequest is completed. - return; - } else { - // TODO: notify with preliminary results. - } - } - - startHTTPRequest(std::move(res)); -} - -void HTTPRequest::startHTTPRequest(std::unique_ptr<Response> &&res) { - assert(std::this_thread::get_id() == threadId); - assert(!httpBaton); - - httpBaton = std::make_shared<HTTPRequestBaton>(path); - httpBaton->request = this; - httpBaton->async = new uv_async_t; - httpBaton->response = std::move(res); - httpBaton->async->data = new util::ptr<HTTPRequestBaton>(httpBaton); - -#if UV_VERSION_MAJOR == 0 && UV_VERSION_MINOR <= 10 - uv_async_init(loop, httpBaton->async, [](uv_async_t *async, int) { -#else - uv_async_init(loop, httpBaton->async, [](uv_async_t *async) { -#endif - util::ptr<HTTPRequestBaton> &baton = *(util::ptr<HTTPRequestBaton> *)async->data; - - if (baton->request) { - HTTPRequest *request = baton->request; - request->httpBaton.reset(); - baton->request = nullptr; - request->handleHTTPResponse(baton->type, std::move(baton->response)); - } - - delete (util::ptr<HTTPRequestBaton> *)async->data; - uv_close((uv_handle_t *)async, [](uv_handle_t *handle) { - uv_async_t *async_handle = (uv_async_t *)handle; - delete async_handle; - }); - }); - attempts++; - HTTPRequestBaton::start(httpBaton); -} - - - -void HTTPRequest::handleHTTPResponse(HTTPResponseType responseType, std::unique_ptr<Response> &&res) { - assert(std::this_thread::get_id() == threadId); - assert(!httpBaton); - assert(!response); - - switch (responseType) { - // This error was caused by a temporary error and it is likely that it will be resolved - // immediately. We are going to try again right away. This is like the TemporaryError, - // except that we will not perform exponential back-off. - case HTTPResponseType::SingularError: - if (attempts >= 4) { - // Report as error after 4 attempts. - response = std::move(res); - notify(); - } else if (attempts >= 2) { - // Switch to the back-off algorithm after the second failure. - retryHTTPRequest(std::move(res), (1 << attempts) * 1000); - return; - } else { - startHTTPRequest(std::move(res)); - } - break; - - // This error might be resolved by waiting some time (e.g. server issues). - // We are going to do an exponential back-off and will try again in a few seconds. - case HTTPResponseType::TemporaryError: - if (attempts >= 4) { - // Report error back after it failed completely. - response = std::move(res); - notify(); - } else { - retryHTTPRequest(std::move(res), (1 << attempts) * 1000); - } - break; - - // This error might be resolved once the network reachability status changes. - // We are going to watch the network status for changes and will retry as soon as the - // operating system notifies us of a network status change. - case HTTPResponseType::ConnectionError: - - if (attempts >= 4) { - // Report error back after it failed completely. - response = std::move(res); - notify(); - } else { - // By default, we will retry every 60 seconds. - retryHTTPRequest(std::move(res), 60000); - } - break; - - // The request was canceled programatically. - case HTTPResponseType::Canceled: - response.reset(); - notify(); - break; - - // This error probably won't be resolved by retrying anytime soon. We are giving up. - case HTTPResponseType::PermanentError: - response = std::move(res); - notify(); - break; - - // The request returned data successfully. We retrieved and decoded the data successfully. - case HTTPResponseType::Successful: - if (store) { - store->put(path, type, *res); - } - response = std::move(res); - notify(); - break; - - // The request confirmed that the data wasn't changed. We already have the data. - case HTTPResponseType::NotModified: - if (store) { - store->updateExpiration(path, res->expires); - } - response = std::move(res); - notify(); - break; - - default: - assert(!"Response wasn't set"); - break; - } -} - -using RetryBaton = std::pair<HTTPRequest *, std::unique_ptr<Response>>; - -void HTTPRequest::retryHTTPRequest(std::unique_ptr<Response> &&res, uint64_t timeout) { - assert(std::this_thread::get_id() == threadId); - assert(!backoffTimer); - backoffTimer = new uv_timer_t(); - uv_timer_init(loop, backoffTimer); - backoffTimer->data = new RetryBaton(this, std::move(res)); - -#if UV_VERSION_MAJOR == 0 && UV_VERSION_MINOR <= 10 - uv_timer_start(backoffTimer, [](uv_timer_t *timer, int) { -#else - uv_timer_start(backoffTimer, [](uv_timer_t *timer) { -#endif - std::unique_ptr<RetryBaton> pair { static_cast<RetryBaton *>(timer->data) }; - pair->first->startHTTPRequest(std::move(pair->second)); - pair->first->backoffTimer = nullptr; - uv_timer_stop(timer); - uv_close((uv_handle_t *)timer, [](uv_handle_t *handle) { delete (uv_timer_t *)handle; }); - }, timeout, 0); -} - -void HTTPRequest::removeHTTPBaton() { - assert(std::this_thread::get_id() == threadId); - if (httpBaton) { - httpBaton->request = nullptr; - HTTPRequestBaton::stop(httpBaton); - httpBaton.reset(); - } -} - -void HTTPRequest::removeCacheBaton() { - assert(std::this_thread::get_id() == threadId); - if (cacheBaton) { - // Make sre that this object doesn't accidentally get accessed when it is destructed before - // the callback returned. They are being run in the same thread, so just setting it to - // null is sufficient. - // Note: We don't manually delete the CacheRequestBaton since it'll be deleted by the - // callback. - cacheBaton->request = nullptr; - cacheBaton = nullptr; - } -} - -void HTTPRequest::removeBackoffTimer() { - assert(std::this_thread::get_id() == threadId); - if (backoffTimer) { - delete static_cast<RetryBaton *>(backoffTimer->data); - uv_timer_stop(backoffTimer); - uv_close((uv_handle_t *)backoffTimer, [](uv_handle_t *handle) { delete (uv_timer_t *)handle; }); - backoffTimer = nullptr; - } -} - -void HTTPRequest::retryImmediately() { - assert(std::this_thread::get_id() == threadId); - if (!cacheBaton && !httpBaton) { - if (backoffTimer) { - // Retry immediately. - uv_timer_stop(backoffTimer); - std::unique_ptr<RetryBaton> pair { static_cast<RetryBaton *>(backoffTimer->data) }; - assert(pair->first == this); - startHTTPRequest(std::move(pair->second)); - uv_close((uv_handle_t *)backoffTimer, [](uv_handle_t *handle) { delete (uv_timer_t *)handle; }); - backoffTimer = nullptr; - } else { - assert(!"We should always have a backoffTimer when there are no batons"); - } - } -} - -void HTTPRequest::cancel() { - assert(std::this_thread::get_id() == threadId); - removeCacheBaton(); - removeHTTPBaton(); - removeBackoffTimer(); - notify(); -} - - -HTTPRequest::~HTTPRequest() { - assert(std::this_thread::get_id() == threadId); - cancel(); -} - -#pragma clang diagnostic pop - -} diff --git a/src/mbgl/storage/http_request.hpp b/src/mbgl/storage/http_request.hpp deleted file mode 100644 index 7cc72101d5..0000000000 --- a/src/mbgl/storage/http_request.hpp +++ /dev/null @@ -1,58 +0,0 @@ -#ifndef MBGL_STORAGE_HTTP_REQUEST -#define MBGL_STORAGE_HTTP_REQUEST - -#include <mbgl/storage/resource_type.hpp> -#include <mbgl/storage/base_request.hpp> -#include <mbgl/storage/http_request_baton.hpp> - -#include <string> -#include <memory> -#include <cassert> -#include <thread> - -typedef struct uv_loop_s uv_loop_t; -typedef struct uv_timer_s uv_timer_t; - -namespace mbgl { - -struct CacheRequestBaton; -struct HTTPRequestBaton; -struct CacheEntry; -class SQLiteStore; - -class HTTPRequest : public BaseRequest { -public: - HTTPRequest(ResourceType type, const std::string &path, uv_loop_t *loop, util::ptr<SQLiteStore> store); - ~HTTPRequest(); - - void cancel(); - void retryImmediately(); - -private: - void startCacheRequest(); - void handleCacheResponse(std::unique_ptr<Response> &&response); - void startHTTPRequest(std::unique_ptr<Response> &&res); - void handleHTTPResponse(HTTPResponseType responseType, std::unique_ptr<Response> &&response); - - void retryHTTPRequest(std::unique_ptr<Response> &&res, uint64_t timeout); - - void removeCacheBaton(); - void removeHTTPBaton(); - void removeBackoffTimer(); - -private: - const std::thread::id threadId; - uv_loop_t *const loop; - CacheRequestBaton *cacheBaton = nullptr; - util::ptr<HTTPRequestBaton> httpBaton; - uv_timer_t *backoffTimer = nullptr; - util::ptr<SQLiteStore> store; - const ResourceType type; - uint8_t attempts = 0; - - friend struct HTTPRequestBaton; -}; - -} - -#endif diff --git a/src/mbgl/storage/http_request_baton.cpp b/src/mbgl/storage/http_request_baton.cpp deleted file mode 100644 index d781a3bdf4..0000000000 --- a/src/mbgl/storage/http_request_baton.cpp +++ /dev/null @@ -1,12 +0,0 @@ -#include <mbgl/storage/http_request_baton.hpp> -#include <uv.h> - -namespace mbgl { - -HTTPRequestBaton::HTTPRequestBaton(const std::string &path_) : threadId(std::this_thread::get_id()), path(path_) { -} - -HTTPRequestBaton::~HTTPRequestBaton() { -} - -} diff --git a/src/mbgl/storage/network_status.cpp b/src/mbgl/storage/network_status.cpp new file mode 100644 index 0000000000..04b6937d94 --- /dev/null +++ b/src/mbgl/storage/network_status.cpp @@ -0,0 +1,32 @@ +#include <mbgl/storage/network_status.hpp> + +#include <uv.h> + +// Example: Allocate a reachability object +// Reachability* reach = [Reachability reachabilityForInternetConnection]; +// reach.reachableBlock = ^(Reachability* reach) { NetworkStatus::Reachable(); }; +// [reach startNotifier]; + +namespace mbgl { + +std::mutex NetworkStatus::mtx; +std::set<uv_async_t *> NetworkStatus::observers; + +void NetworkStatus::Subscribe(uv_async_t *async) { + std::lock_guard<std::mutex> lock(NetworkStatus::mtx); + observers.insert(async); +} + +void NetworkStatus::Unsubscribe(uv_async_t *async) { + std::lock_guard<std::mutex> lock(NetworkStatus::mtx); + observers.erase(async); +} + +void NetworkStatus::Reachable() { + std::lock_guard<std::mutex> lock(NetworkStatus::mtx); + for (auto async : observers) { + uv_async_send(async); + } +} + +} diff --git a/src/mbgl/storage/request.cpp b/src/mbgl/storage/request.cpp index 39fbd36789..c771acb929 100644 --- a/src/mbgl/storage/request.cpp +++ b/src/mbgl/storage/request.cpp @@ -1,49 +1,86 @@ -#include <mbgl/storage/request.hpp> -#include <mbgl/storage/base_request.hpp> +#include <mbgl/storage/default/request.hpp> + +#include <mbgl/storage/response.hpp> + +#include <mbgl/util/util.hpp> +#include <mbgl/util/uv.hpp> #include <uv.h> #include <cassert> +#include <functional> namespace mbgl { -Request::Request(const util::ptr<BaseRequest> &base_) - : thread_id(std::this_thread::get_id()), base(base_) { -} +// Note: This requires that loop is running in the current thread (or not yet running). +Request::Request(const Resource &resource_, uv_loop_t *loop, Callback callback_) + : callback(callback_), resource(resource_) { + // When there is no loop supplied (== nullptr), the callback will be fired in an arbitrary + // thread (the thread notify() is called from) rather than kicking back to the calling thread. + if (loop) { + notify_async = new uv_async_t; + notify_async->data = this; + uv_async_init(loop, notify_async, [](uv_async_t *async, int) { + auto request = reinterpret_cast<Request *>(async->data); + uv::close(async); -Request::~Request() { - assert(thread_id == std::this_thread::get_id()); + if (!request->destruct_async) { + // We haven't created a cancel request, so we can safely delete this Request object + // since it won't be accessed in the future. + assert(request->response); + request->callback(*request->response); + delete request; + } else { + // Otherwise, we're waiting for for the destruct notification to be delivered in order + // to delete the Request object. We're doing this since we can't know whether the + // DefaultFileSource is still sending a cancel event, which means this object must still + // exist. + } + }); + } } -void Request::onload(CompletedCallback cb) { - assert(thread_id == std::this_thread::get_id()); - if (base) { - Callback *callback = base->add(std::move(cb), base); - if (callback) { - callbacks.push_front(callback); - } +Request::~Request() { + if (notify_async) { + // Request objects can be destructed in other threads when the user didn't supply a loop. + MBGL_VERIFY_THREAD(tid) } } -void Request::oncancel(AbortedCallback cb) { - assert(thread_id == std::this_thread::get_id()); - if (base) { - Callback *callback = base->add(std::move(cb), base); - if (callback) { - callbacks.push_front(callback); - } +void Request::notify(const std::shared_ptr<const Response> &response_) { + response = response_; + if (notify_async) { + uv_async_send(notify_async); + } else { + assert(response); + callback(*response); + delete this; } } void Request::cancel() { - assert(thread_id == std::this_thread::get_id()); - if (base) { - for (Callback *callback : callbacks) { - base->remove(callback); - } - base.reset(); + MBGL_VERIFY_THREAD(tid) + assert(notify_async); + assert(!destruct_async); + destruct_async = new uv_async_t; + destruct_async->data = this; + uv_async_init(notify_async->loop, destruct_async, [](uv_async_t *async, int) { + // The destruct_async will be invoked *after* the notify_async callback has already run. + auto request = reinterpret_cast<Request *>(async->data); + uv::close(async); + delete request; + }); +} + +// This gets called from the FileSource thread, and will only ever be invoked after cancel() was called +// in the original requesting thread. +void Request::destruct() { + if (notify_async) { + notify(nullptr); } - callbacks.clear(); + + assert(destruct_async); + uv_async_send(destruct_async); } } diff --git a/src/mbgl/storage/response.cpp b/src/mbgl/storage/response.cpp deleted file mode 100644 index a08a6d31ce..0000000000 --- a/src/mbgl/storage/response.cpp +++ /dev/null @@ -1,22 +0,0 @@ -#include <mbgl/storage/response.hpp> - -#include <chrono> - -namespace mbgl { - -int64_t Response::parseCacheControl(const char *value) { - if (value) { - unsigned long long seconds = 0; - // TODO: cache-control may contain other information as well: - // http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 - if (std::sscanf(value, "max-age=%llu", &seconds) == 1) { - return std::chrono::duration_cast<std::chrono::seconds>( - std::chrono::system_clock::now().time_since_epoch()).count() + - seconds; - } - } - - return -1; -} - -} diff --git a/src/mbgl/storage/sqlite_cache.cpp b/src/mbgl/storage/sqlite_cache.cpp new file mode 100644 index 0000000000..7a0fd21ce6 --- /dev/null +++ b/src/mbgl/storage/sqlite_cache.cpp @@ -0,0 +1,255 @@ +#include <mbgl/storage/default/sqlite_cache.hpp> +#include <mbgl/storage/default/request.hpp> +#include <mbgl/storage/response.hpp> + +#include <mbgl/util/util.hpp> +#include <mbgl/util/async_queue.hpp> +#include <mbgl/util/variant.hpp> + +#include "../util/sqlite3.hpp" +#include "../util/compression.hpp" + +#include <uv.h> + +#include <cassert> + +namespace mbgl { + +std::string removeAccessTokenFromURL(const std::string &url) { + const size_t token_start = url.find("access_token="); + // Ensure that token exists, isn't at the front and is preceded by either & or ?. + if (token_start == std::string::npos || token_start == 0 || !(url[token_start - 1] == '&' || url[token_start - 1] == '?')) { + return url; + } + + const size_t token_end = url.find_first_of('&', token_start); + if (token_end == std::string::npos) { + // The token is the last query argument. We slice away the "&access_token=..." part + return url.substr(0, token_start - 1); + } else { + // We slice away the "access_token=...&" part. + return url.substr(0, token_start) + url.substr(token_end + 1); + } +} + +std::string convertMapboxDomainsToProtocol(const std::string &url) { + const size_t protocol_separator = url.find("://"); + if (protocol_separator == std::string::npos) { + return url; + } + + const std::string protocol = url.substr(0, protocol_separator); + if (!(protocol == "http" || protocol == "https")) { + return url; + } + + const size_t domain_begin = protocol_separator + 3; + const size_t path_separator = url.find("/", domain_begin); + if (path_separator == std::string::npos) { + return url; + } + + const std::string domain = url.substr(domain_begin, path_separator - domain_begin); + if (domain.find(".tiles.mapbox.com") != std::string::npos) { + return "mapbox://" + url.substr(path_separator + 1); + } else { + return url; + } +} + +std::string unifyMapboxURLs(const std::string &url) { + return removeAccessTokenFromURL(convertMapboxDomainsToProtocol(url)); +} + + +using namespace mapbox::sqlite; + +struct SQLiteCache::GetAction { + const Resource resource; + const std::function<void(std::unique_ptr<Response>)> callback; +}; + +struct SQLiteCache::PutAction { + const Resource resource; + const std::shared_ptr<const Response> response; +}; + +struct SQLiteCache::RefreshAction { + const Resource resource; + const int64_t expires; +}; + +struct SQLiteCache::StopAction { +}; + +struct SQLiteCache::ActionDispatcher { + SQLiteCache &cache; + template <typename T> void operator()(T &t) { cache.process(t); } +}; + +SQLiteCache::SQLiteCache(const std::string &path_) + : path(path_), + loop(uv_loop_new()), + queue(new Queue(loop, [this](Action &action) { + mapbox::util::apply_visitor(ActionDispatcher{ *this }, action); + })), + thread([this]() { +#ifdef __APPLE__ + pthread_setname_np("SQLite Cache"); +#endif + uv_run(loop, UV_RUN_DEFAULT); + }) +{ +} + +SQLiteCache::~SQLiteCache() { + if (thread.joinable()) { + if (queue) { + queue->send(StopAction{ }); + } + thread.join(); + uv_loop_delete(loop); + } +} + + +void SQLiteCache::get(const Resource &resource, std::function<void(std::unique_ptr<Response>)> callback) { + // Can be called from any thread, but most likely from the file source thread. + // Will try to load the URL from the SQLite database and call the callback when done. + // Note that the callback is probably going to invoked from another thread, so the caller + // must make sure that it can run in that thread. + assert(queue); + queue->send(GetAction{ resource, callback }); +} + +void SQLiteCache::put(const Resource &resource, std::shared_ptr<const Response> response, Hint hint) { + // Can be called from any thread, but most likely from the file source thread. We are either + // storing a new response or updating the currently stored response, potentially setting a new + // expiry date. + assert(queue); + assert(response); + + if (hint == Hint::Full) { + queue->send(PutAction{ resource, response }); + } else if (hint == Hint::Refresh) { + queue->send(RefreshAction{ resource, response->expires }); + } +} + +void SQLiteCache::createDatabase() { + db = util::make_unique<Database>(path.c_str(), ReadWrite | Create); + + db->exec("CREATE TABLE IF NOT EXISTS `http_cache` (" + " `url` TEXT PRIMARY KEY NOT NULL," + " `status` INTEGER NOT NULL," // The response status (Successful or Error). + " `kind` INTEGER NOT NULL," // The kind of file. + " `modified` INTEGER," // Timestamp when the file was last modified. + " `etag` TEXT," + " `expires` INTEGER," // Timestamp when the server says the file expires. + " `data` BLOB," + " `compressed` INTEGER NOT NULL DEFAULT 0" // Whether the data is compressed. + ");" + "CREATE INDEX IF NOT EXISTS `http_cache_kind_idx` ON `http_cache` (`kind`);"); +} + +void SQLiteCache::process(GetAction &action) { + // This is called in the SQLite event loop. + if (!db) { + createDatabase(); + } + + if (!getStmt) { + // Initialize the statement 0 1 + getStmt = util::make_unique<Statement>(db->prepare("SELECT `status`, `modified`, " + // 2 3 4 5 1 + "`etag`, `expires`, `data`, `compressed` FROM `http_cache` WHERE `url` = ?")); + } else { + getStmt->reset(); + } + + const std::string unifiedURL = unifyMapboxURLs(action.resource.url); + getStmt->bind(1, unifiedURL.c_str()); + if (getStmt->run()) { + // There is data. + auto response = util::make_unique<Response>(); + response->status = Response::Status(getStmt->get<int>(0)); + response->modified = getStmt->get<int64_t>(1); + response->etag = getStmt->get<std::string>(2); + response->expires = getStmt->get<int64_t>(3); + response->data = getStmt->get<std::string>(4); + if (getStmt->get<int>(5)) { // == compressed + response->data = util::decompress(response->data); + } + action.callback(std::move(response)); + } else { + // There is no data. + action.callback(nullptr); + } +} + +void SQLiteCache::process(PutAction &action) { + if (!db) { + createDatabase(); + } + + if (!putStmt) { + putStmt = util::make_unique<Statement>(db->prepare("REPLACE INTO `http_cache` (" + // 1 2 3 4 5 6 7 8 + "`url`, `status`, `kind`, `modified`, `etag`, `expires`, `data`, `compressed`" + ") VALUES(?, ?, ?, ?, ?, ?, ?, ?)")); + } else { + putStmt->reset(); + } + + const std::string unifiedURL = unifyMapboxURLs(action.resource.url); + putStmt->bind(1 /* url */, unifiedURL.c_str()); + putStmt->bind(2 /* status */, int(action.response->status)); + putStmt->bind(3 /* kind */, int(action.resource.kind)); + putStmt->bind(4 /* modified */, action.response->modified); + putStmt->bind(5 /* etag */, action.response->etag.c_str()); + putStmt->bind(6 /* expires */, action.response->expires); + + std::string data; + if (action.resource.kind != Resource::Image) { + // Do not compress images, since they are typically compressed already. + data = util::compress(action.response->data); + } + + if (!data.empty() && data.size() < action.response->data.size()) { + // Store the compressed data when it is smaller than the original + // uncompressed data. + putStmt->bind(7 /* data */, data, false); // do not retain the string internally. + putStmt->bind(8 /* compressed */, true); + } else { + putStmt->bind(7 /* data */, action.response->data, false); // do not retain the string internally. + putStmt->bind(8 /* compressed */, false); + } + + putStmt->run(); +} + +void SQLiteCache::process(RefreshAction &action) { + if (!db) { + createDatabase(); + } + + if (!refreshStmt) { + refreshStmt = util::make_unique<Statement>( // 1 2 + db->prepare("UPDATE `http_cache` SET `expires` = ? WHERE `url` = ?")); + } else { + refreshStmt->reset(); + } + + const std::string unifiedURL = unifyMapboxURLs(action.resource.url); + refreshStmt->bind(1, int64_t(action.expires)); + refreshStmt->bind(2, unifiedURL.c_str()); + refreshStmt->run(); +} + +void SQLiteCache::process(StopAction &) { + assert(queue); + queue->stop(); + queue = nullptr; +} + +} diff --git a/src/mbgl/storage/sqlite_store.cpp b/src/mbgl/storage/sqlite_store.cpp deleted file mode 100644 index d382921dec..0000000000 --- a/src/mbgl/storage/sqlite_store.cpp +++ /dev/null @@ -1,228 +0,0 @@ -#include <mbgl/storage/sqlite_store.hpp> -#include <mbgl/util/compression.hpp> -#include <mbgl/util/sqlite3.hpp> -#include <mbgl/util/std.hpp> - -#include <mbgl/util/uv-worker.h> - -#include <cassert> - -using namespace mapbox::sqlite; - -std::string removeAccessTokenFromURL(const std::string &url) { - const size_t token_start = url.find("access_token="); - // Ensure that token exists, isn't at the front and is preceded by either & or ?. - if (token_start == std::string::npos || token_start == 0 || !(url[token_start - 1] == '&' || url[token_start - 1] == '?')) { - return url; - } - - const size_t token_end = url.find_first_of('&', token_start); - if (token_end == std::string::npos) { - // The token is the last query argument. We slice away the "&access_token=..." part - return url.substr(0, token_start - 1); - } else { - // We slice away the "access_token=...&" part. - return url.substr(0, token_start) + url.substr(token_end + 1); - } -} - -std::string convertMapboxDomainsToProtocol(const std::string &url) { - const size_t protocol_separator = url.find("://"); - if (protocol_separator == std::string::npos) { - return url; - } - - const std::string protocol = url.substr(0, protocol_separator); - if (!(protocol == "http" || protocol == "https")) { - return url; - } - - const size_t domain_begin = protocol_separator + 3; - const size_t path_separator = url.find("/", domain_begin); - if (path_separator == std::string::npos) { - return url; - } - - const std::string domain = url.substr(domain_begin, path_separator - domain_begin); - if (domain.find(".tiles.mapbox.com") != std::string::npos) { - return "mapbox://" + url.substr(path_separator + 1); - } else { - return url; - } -} - -std::string unifyMapboxURLs(const std::string &url) { - return removeAccessTokenFromURL(convertMapboxDomainsToProtocol(url)); -} - -namespace mbgl { - -SQLiteStore::SQLiteStore(uv_loop_t *loop, const std::string &path) - : thread_id(std::this_thread::get_id()), - db(std::make_shared<Database>(path.c_str(), ReadWrite | Create)) { - createSchema(); - worker = new uv_worker_t; - uv_worker_init(worker, loop, 1, "SQLite"); -} - -SQLiteStore::~SQLiteStore() { - // Nothing to do. This function needs to be here because we're forward-declaring - // Database, so we need the actual definition here to be able to properly destruct it. - if (worker) { - uv_worker_close(worker, [](uv_worker_t *worker_handle) { - delete worker_handle; - }); - } -} - -void SQLiteStore::createSchema() { - if (!db || !*db) { - return; - } - - db->exec("CREATE TABLE IF NOT EXISTS `http_cache` (" - " `url` TEXT PRIMARY KEY NOT NULL," - " `code` INTEGER NOT NULL," - " `type` INTEGER NOT NULL," - " `modified` INTEGER," - " `etag` TEXT," - " `expires` INTEGER," - " `data` BLOB," - " `compressed` INTEGER NOT NULL DEFAULT 0" - ");" - "CREATE INDEX IF NOT EXISTS `http_cache_type_idx` ON `http_cache` (`type`);"); -} - -struct GetBaton { - util::ptr<Database> db; - std::string path; - ResourceType type; - void *ptr = nullptr; - SQLiteStore::GetCallback callback = nullptr; - std::unique_ptr<Response> response; -}; - -void SQLiteStore::get(const std::string &path, GetCallback callback, void *ptr) { - assert(std::this_thread::get_id() == thread_id); - if (!db || !*db) { - if (callback) { - callback(nullptr, ptr); - } - return; - } - - GetBaton *get_baton = new GetBaton; - get_baton->db = db; - get_baton->path = path; - get_baton->ptr = ptr; - get_baton->callback = callback; - - uv_worker_send(worker, get_baton, [](void *data) { - GetBaton *baton = (GetBaton *)data; - const std::string url = unifyMapboxURLs(baton->path); - // 0 1 2 - Statement stmt = baton->db->prepare("SELECT `code`, `type`, `modified`, " - // 3 4 5 6 - "`etag`, `expires`, `data`, `compressed` FROM `http_cache` WHERE `url` = ?"); - - stmt.bind(1, url.c_str()); - if (stmt.run()) { - // There is data. - baton->response = util::make_unique<Response>(); - - baton->response->code = stmt.get<int>(0); - baton->type = ResourceType(stmt.get<int>(1)); - baton->response->modified = stmt.get<int64_t>(2); - baton->response->etag = stmt.get<std::string>(3); - baton->response->expires = stmt.get<int64_t>(4); - baton->response->data = stmt.get<std::string>(5); - if (stmt.get<int>(6)) { // == compressed - baton->response->data = util::decompress(baton->response->data); - } - } else { - // There is no data. - // This is a noop. - } - }, [](void *data) { - std::unique_ptr<GetBaton> baton { (GetBaton *)data }; - if (baton->callback) { - baton->callback(std::move(baton->response), baton->ptr); - } - }); -} - - -struct PutBaton { - util::ptr<Database> db; - std::string path; - ResourceType type; - Response response; -}; - -void SQLiteStore::put(const std::string &path, ResourceType type, const Response &response) { - assert(std::this_thread::get_id() == thread_id); - if (!db) return; - - PutBaton *put_baton = new PutBaton; - put_baton->db = db; - put_baton->path = path; - put_baton->type = type; - put_baton->response = response; - - uv_worker_send(worker, put_baton, [](void *data) { - PutBaton *baton = (PutBaton *)data; - const std::string url = unifyMapboxURLs(baton->path); - Statement stmt = baton->db->prepare("REPLACE INTO `http_cache` (" - // 1 2 3 4 5 6 7 8 - "`url`, `code`, `type`, `modified`, `etag`, `expires`, `data`, `compressed`" - ") VALUES(?, ?, ?, ?, ?, ?, ?, ?)"); - stmt.bind(1, url.c_str()); - stmt.bind(2, int(baton->response.code)); - stmt.bind(3, int(baton->type)); - stmt.bind(4, baton->response.modified); - stmt.bind(5, baton->response.etag.c_str()); - stmt.bind(6, baton->response.expires); - - if (baton->type == ResourceType::Image) { - stmt.bind(7, baton->response.data, false); // do not retain the string internally. - stmt.bind(8, false); - } else { - stmt.bind(7, util::compress(baton->response.data), true); // retain the string internally. - stmt.bind(8, true); - } - - stmt.run(); - }, [](void *data) { - delete (PutBaton *)data; - }); -} - -struct ExpirationBaton { - util::ptr<Database> db; - std::string path; - int64_t expires; -}; - -void SQLiteStore::updateExpiration(const std::string &path, int64_t expires) { - assert(std::this_thread::get_id() == thread_id); - if (!db || !*db) return; - - ExpirationBaton *expiration_baton = new ExpirationBaton; - expiration_baton->db = db; - expiration_baton->path = path; - expiration_baton->expires = expires; - - uv_worker_send(worker, expiration_baton, [](void *data) { - ExpirationBaton *baton = (ExpirationBaton *)data; - const std::string url = unifyMapboxURLs(baton->path); - Statement stmt = // 1 2 - baton->db->prepare("UPDATE `http_cache` SET `expires` = ? WHERE `url` = ?"); - stmt.bind<int64_t>(1, baton->expires); - stmt.bind(2, url.c_str()); - stmt.run(); - }, [](void *data) { - delete (ExpirationBaton *)data; - }); -} - -} diff --git a/src/mbgl/storage/sqlite_store.hpp b/src/mbgl/storage/sqlite_store.hpp deleted file mode 100644 index 988eca2597..0000000000 --- a/src/mbgl/storage/sqlite_store.hpp +++ /dev/null @@ -1,49 +0,0 @@ -#ifndef MBGL_STORAGE_SQLITE_STORE -#define MBGL_STORAGE_SQLITE_STORE - -#include <mbgl/storage/response.hpp> -#include <mbgl/storage/resource_type.hpp> -#include <mbgl/util/ptr.hpp> - -#include <uv.h> - -#include <string> -#include <thread> - -typedef struct uv_worker_s uv_worker_t; - -namespace mapbox { -namespace sqlite { -class Database; -} -} - -namespace mbgl { - -class SQLiteStore { -public: - SQLiteStore(uv_loop_t *loop, const std::string &path); - ~SQLiteStore(); - - typedef void (*GetCallback)(std::unique_ptr<Response> &&entry, void *ptr); - - void get(const std::string &path, GetCallback cb, void *ptr); - void put(const std::string &path, ResourceType type, const Response &entry); - void updateExpiration(const std::string &path, int64_t expires); - -private: - void createSchema(); - void closeDatabase(); - static void runGet(uv_work_t *req); - static void runPut(uv_work_t *req); - static void deliverResult(uv_work_t *req, int status); - -private: - const std::thread::id thread_id; - util::ptr<mapbox::sqlite::Database> db; - uv_worker_t *worker = nullptr; -}; - -} - -#endif diff --git a/src/mbgl/style/style.hpp b/src/mbgl/style/style.hpp index af00f9710b..5517a56a71 100644 --- a/src/mbgl/style/style.hpp +++ b/src/mbgl/style/style.hpp @@ -43,6 +43,7 @@ public: util::ptr<StyleLayerGroup> layers; std::vector<std::string> appliedClasses; std::string glyph_url; + std::string base; private: std::string sprite_url; diff --git a/src/mbgl/text/glyph_store.cpp b/src/mbgl/text/glyph_store.cpp index 2f5db180fd..0d9e70d556 100644 --- a/src/mbgl/text/glyph_store.cpp +++ b/src/mbgl/text/glyph_store.cpp @@ -148,28 +148,23 @@ GlyphPBF::GlyphPBF(const std::string &glyphURL, const std::string &fontStack, Gl }); // The prepare call jumps back to the main thread. - fileSource.prepare([&, url] { - auto request = fileSource.request(ResourceType::Glyphs, url); - request->onload([&, url](const Response &res) { - if (res.code != 200) { - // Something went wrong with loading the glyph pbf. Pass on the error to the future listeners. - const std::string msg = std::string { "[ERROR] failed to load glyphs (" } + util::toString(res.code) + "): " + res.message; - promise.set_exception(std::make_exception_ptr(std::runtime_error(msg))); - } else { - // Transfer the data to the GlyphSet and signal its availability. - // Once it is available, the caller will need to call parse() to actually - // parse the data we received. We are not doing this here since this callback is being - // called from another (unknown) thread. - data = res.data; - promise.set_value(*this); - } - }); - request->oncancel([&]() { - promise.set_exception(std::make_exception_ptr(std::runtime_error("Loading glyphs was canceled"))); - }); + fileSource.request({ Resource::Kind::Glyphs, url }, [&, url](const Response &res) { + if (res.status != Response::Successful) { + // Something went wrong with loading the glyph pbf. Pass on the error to the future listeners. + const std::string msg = std::string { "[ERROR] failed to load glyphs: " } + res.message; + promise.set_exception(std::make_exception_ptr(std::runtime_error(msg))); + } else { + // Transfer the data to the GlyphSet and signal its availability. + // Once it is available, the caller will need to call parse() to actually + // parse the data we received. We are not doing this here since this callback is being + // called from another (unknown) thread. + data = res.data; + promise.set_value(*this); + } }); } + std::shared_future<GlyphPBF &> GlyphPBF::getFuture() { return future; } diff --git a/src/mbgl/util/uv.cpp b/src/mbgl/util/uv.cpp index 7aa5bad0cf..a993e6b962 100644 --- a/src/mbgl/util/uv.cpp +++ b/src/mbgl/util/uv.cpp @@ -22,4 +22,12 @@ std::string cwd() { #endif } +const char *getFileRequestError(uv_fs_t *req) { +#if UV_VERSION_MAJOR == 0 && UV_VERSION_MINOR <= 10 + return uv_strerror(uv_last_error(req->loop)); +#else + return uv_strerror(int(req->result)); +#endif +} + } |