diff options
Diffstat (limited to 'platform/default/src/mbgl/storage')
-rw-r--r-- | platform/default/src/mbgl/storage/asset_file_source.cpp | 81 | ||||
-rw-r--r-- | platform/default/src/mbgl/storage/default_file_source.cpp | 316 | ||||
-rw-r--r-- | platform/default/src/mbgl/storage/file_source_request.cpp | 37 | ||||
-rw-r--r-- | platform/default/src/mbgl/storage/http_file_source.cpp | 495 | ||||
-rw-r--r-- | platform/default/src/mbgl/storage/local_file_source.cpp | 81 | ||||
-rw-r--r-- | platform/default/src/mbgl/storage/offline.cpp | 158 | ||||
-rw-r--r-- | platform/default/src/mbgl/storage/offline_database.cpp | 1129 | ||||
-rw-r--r-- | platform/default/src/mbgl/storage/offline_download.cpp | 453 | ||||
-rw-r--r-- | platform/default/src/mbgl/storage/online_file_source.cpp | 488 | ||||
-rw-r--r-- | platform/default/src/mbgl/storage/sqlite3.cpp | 494 |
10 files changed, 3732 insertions, 0 deletions
diff --git a/platform/default/src/mbgl/storage/asset_file_source.cpp b/platform/default/src/mbgl/storage/asset_file_source.cpp new file mode 100644 index 0000000000..7988654ae5 --- /dev/null +++ b/platform/default/src/mbgl/storage/asset_file_source.cpp @@ -0,0 +1,81 @@ +#include <mbgl/storage/asset_file_source.hpp> +#include <mbgl/storage/file_source_request.hpp> +#include <mbgl/storage/response.hpp> +#include <mbgl/util/string.hpp> +#include <mbgl/util/thread.hpp> +#include <mbgl/util/url.hpp> +#include <mbgl/util/util.hpp> +#include <mbgl/util/io.hpp> + +#include <sys/types.h> +#include <sys/stat.h> + +namespace { + +const std::string assetProtocol = "asset://"; + +} // namespace + +namespace mbgl { + +class AssetFileSource::Impl { +public: + Impl(ActorRef<Impl>, std::string root_) + : root(std::move(root_)) { + } + + void request(const std::string& url, ActorRef<FileSourceRequest> req) { + Response response; + + if (!acceptsURL(url)) { + response.error = std::make_unique<Response::Error>(Response::Error::Reason::Other, + "Invalid asset URL"); + req.invoke(&FileSourceRequest::setResponse, response); + return; + } + + // Cut off the protocol and prefix with path. + const auto path = root + "/" + mbgl::util::percentDecode(url.substr(assetProtocol.size())); + struct stat buf; + int result = stat(path.c_str(), &buf); + + if (result == 0 && (S_IFDIR & buf.st_mode)) { + response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound); + } else if (result == -1 && errno == ENOENT) { + response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound); + } else { + try { + response.data = std::make_shared<std::string>(util::read_file(path)); + } catch (...) { + response.error = std::make_unique<Response::Error>( + Response::Error::Reason::Other, + util::toString(std::current_exception())); + } + } + + req.invoke(&FileSourceRequest::setResponse, response); + } + +private: + std::string root; +}; + +AssetFileSource::AssetFileSource(const std::string& root) + : impl(std::make_unique<util::Thread<Impl>>("AssetFileSource", root)) { +} + +AssetFileSource::~AssetFileSource() = default; + +std::unique_ptr<AsyncRequest> AssetFileSource::request(const Resource& resource, Callback callback) { + auto req = std::make_unique<FileSourceRequest>(std::move(callback)); + + impl->actor().invoke(&Impl::request, resource.url, req->actor()); + + return std::move(req); +} + +bool AssetFileSource::acceptsURL(const std::string& url) { + return 0 == url.rfind(assetProtocol, 0); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/default_file_source.cpp b/platform/default/src/mbgl/storage/default_file_source.cpp new file mode 100644 index 0000000000..cad68e7de9 --- /dev/null +++ b/platform/default/src/mbgl/storage/default_file_source.cpp @@ -0,0 +1,316 @@ +#include <mbgl/storage/default_file_source.hpp> +#include <mbgl/storage/asset_file_source.hpp> +#include <mbgl/storage/file_source_request.hpp> +#include <mbgl/storage/local_file_source.hpp> +#include <mbgl/storage/online_file_source.hpp> +#include <mbgl/storage/offline_database.hpp> +#include <mbgl/storage/offline_download.hpp> +#include <mbgl/storage/resource_transform.hpp> + +#include <mbgl/util/platform.hpp> +#include <mbgl/util/url.hpp> +#include <mbgl/util/thread.hpp> +#include <mbgl/util/work_request.hpp> +#include <mbgl/util/stopwatch.hpp> + +#include <cassert> + +namespace mbgl { + +class DefaultFileSource::Impl { +public: + Impl(std::shared_ptr<FileSource> assetFileSource_, std::string cachePath, uint64_t maximumCacheSize) + : assetFileSource(assetFileSource_) + , localFileSource(std::make_unique<LocalFileSource>()) + , offlineDatabase(std::make_unique<OfflineDatabase>(cachePath, maximumCacheSize)) { + } + + void setAPIBaseURL(const std::string& url) { + onlineFileSource.setAPIBaseURL(url); + } + + std::string getAPIBaseURL() const{ + return onlineFileSource.getAPIBaseURL(); + } + + void setAccessToken(const std::string& accessToken) { + onlineFileSource.setAccessToken(accessToken); + } + + std::string getAccessToken() const { + return onlineFileSource.getAccessToken(); + } + + void setResourceTransform(optional<ActorRef<ResourceTransform>>&& transform) { + onlineFileSource.setResourceTransform(std::move(transform)); + } + + void listRegions(std::function<void (expected<OfflineRegions, std::exception_ptr>)> callback) { + callback(offlineDatabase->listRegions()); + } + + void createRegion(const OfflineRegionDefinition& definition, + const OfflineRegionMetadata& metadata, + std::function<void (expected<OfflineRegion, std::exception_ptr>)> callback) { + callback(offlineDatabase->createRegion(definition, metadata)); + } + + void mergeOfflineRegions(const std::string& sideDatabasePath, + std::function<void (expected<OfflineRegions, std::exception_ptr>)> callback) { + callback(offlineDatabase->mergeDatabase(sideDatabasePath)); + } + + void updateMetadata(const int64_t regionID, + const OfflineRegionMetadata& metadata, + std::function<void (expected<OfflineRegionMetadata, std::exception_ptr>)> callback) { + callback(offlineDatabase->updateMetadata(regionID, metadata)); + } + + void getRegionStatus(int64_t regionID, std::function<void (expected<OfflineRegionStatus, std::exception_ptr>)> callback) { + if (auto download = getDownload(regionID)) { + callback(download.value()->getStatus()); + } else { + callback(unexpected<std::exception_ptr>(download.error())); + } + } + + void deleteRegion(OfflineRegion&& region, std::function<void (std::exception_ptr)> callback) { + downloads.erase(region.getID()); + callback(offlineDatabase->deleteRegion(std::move(region))); + } + + void setRegionObserver(int64_t regionID, std::unique_ptr<OfflineRegionObserver> observer) { + if (auto download = getDownload(regionID)) { + download.value()->setObserver(std::move(observer)); + } + } + + void setRegionDownloadState(int64_t regionID, OfflineRegionDownloadState state) { + if (auto download = getDownload(regionID)) { + download.value()->setState(state); + } + } + + void request(AsyncRequest* req, Resource resource, ActorRef<FileSourceRequest> ref) { + auto callback = [ref] (const Response& res) mutable { + ref.invoke(&FileSourceRequest::setResponse, res); + }; + + if (AssetFileSource::acceptsURL(resource.url)) { + //Asset request + tasks[req] = assetFileSource->request(resource, callback); + } else if (LocalFileSource::acceptsURL(resource.url)) { + //Local file request + tasks[req] = localFileSource->request(resource, callback); + } else { + // Try the offline database + if (resource.hasLoadingMethod(Resource::LoadingMethod::Cache)) { + auto offlineResponse = offlineDatabase->get(resource); + + if (resource.loadingMethod == Resource::LoadingMethod::CacheOnly) { + if (!offlineResponse) { + // Ensure there's always a response that we can send, so the caller knows that + // there's no optional data available in the cache, when it's the only place + // we're supposed to load from. + offlineResponse.emplace(); + offlineResponse->noContent = true; + offlineResponse->error = std::make_unique<Response::Error>( + Response::Error::Reason::NotFound, "Not found in offline database"); + } else if (!offlineResponse->isUsable()) { + // Don't return resources the server requested not to show when they're stale. + // Even if we can't directly use the response, we may still use it to send a + // conditional HTTP request, which is why we're saving it above. + offlineResponse->error = std::make_unique<Response::Error>( + Response::Error::Reason::NotFound, "Cached resource is unusable"); + } + callback(*offlineResponse); + } else if (offlineResponse) { + // Copy over the fields so that we can use them when making a refresh request. + resource.priorModified = offlineResponse->modified; + resource.priorExpires = offlineResponse->expires; + resource.priorEtag = offlineResponse->etag; + resource.priorData = offlineResponse->data; + + if (offlineResponse->isUsable()) { + callback(*offlineResponse); + } + } + } + + // Get from the online file source + if (resource.hasLoadingMethod(Resource::LoadingMethod::Network)) { + MBGL_TIMING_START(watch); + tasks[req] = onlineFileSource.request(resource, [=] (Response onlineResponse) mutable { + this->offlineDatabase->put(resource, onlineResponse); + if (resource.kind == Resource::Kind::Tile) { + // onlineResponse.data will be null if data not modified + MBGL_TIMING_FINISH(watch, + " Action: " << "Requesting," << + " URL: " << resource.url.c_str() << + " Size: " << (onlineResponse.data != nullptr ? onlineResponse.data->size() : 0) << "B," << + " Time") + } + callback(onlineResponse); + }); + } + } + } + + void cancel(AsyncRequest* req) { + tasks.erase(req); + } + + void setOfflineMapboxTileCountLimit(uint64_t limit) { + offlineDatabase->setOfflineMapboxTileCountLimit(limit); + } + + void setOnlineStatus(const bool status) { + onlineFileSource.setOnlineStatus(status); + } + + void put(const Resource& resource, const Response& response) { + offlineDatabase->put(resource, response); + } + +private: + expected<OfflineDownload*, std::exception_ptr> getDownload(int64_t regionID) { + auto it = downloads.find(regionID); + if (it != downloads.end()) { + return it->second.get(); + } + auto definition = offlineDatabase->getRegionDefinition(regionID); + if (!definition) { + return unexpected<std::exception_ptr>(definition.error()); + } + auto download = std::make_unique<OfflineDownload>(regionID, std::move(definition.value()), + *offlineDatabase, onlineFileSource); + return downloads.emplace(regionID, std::move(download)).first->second.get(); + } + + // shared so that destruction is done on the creating thread + const std::shared_ptr<FileSource> assetFileSource; + const std::unique_ptr<FileSource> localFileSource; + std::unique_ptr<OfflineDatabase> offlineDatabase; + OnlineFileSource onlineFileSource; + std::unordered_map<AsyncRequest*, std::unique_ptr<AsyncRequest>> tasks; + std::unordered_map<int64_t, std::unique_ptr<OfflineDownload>> downloads; +}; + +DefaultFileSource::DefaultFileSource(const std::string& cachePath, + const std::string& assetRoot, + uint64_t maximumCacheSize) + : DefaultFileSource(cachePath, std::make_unique<AssetFileSource>(assetRoot), maximumCacheSize) { +} + +DefaultFileSource::DefaultFileSource(const std::string& cachePath, + std::unique_ptr<FileSource>&& assetFileSource_, + uint64_t maximumCacheSize) + : assetFileSource(std::move(assetFileSource_)) + , impl(std::make_unique<util::Thread<Impl>>("DefaultFileSource", assetFileSource, cachePath, maximumCacheSize)) { +} + +DefaultFileSource::~DefaultFileSource() = default; + +void DefaultFileSource::setAPIBaseURL(const std::string& baseURL) { + impl->actor().invoke(&Impl::setAPIBaseURL, baseURL); + + { + std::lock_guard<std::mutex> lock(cachedBaseURLMutex); + cachedBaseURL = baseURL; + } +} + +std::string DefaultFileSource::getAPIBaseURL() { + std::lock_guard<std::mutex> lock(cachedBaseURLMutex); + return cachedBaseURL; +} + +void DefaultFileSource::setAccessToken(const std::string& accessToken) { + impl->actor().invoke(&Impl::setAccessToken, accessToken); + + { + std::lock_guard<std::mutex> lock(cachedAccessTokenMutex); + cachedAccessToken = accessToken; + } +} + +std::string DefaultFileSource::getAccessToken() { + std::lock_guard<std::mutex> lock(cachedAccessTokenMutex); + return cachedAccessToken; +} + +void DefaultFileSource::setResourceTransform(optional<ActorRef<ResourceTransform>>&& transform) { + impl->actor().invoke(&Impl::setResourceTransform, std::move(transform)); +} + +std::unique_ptr<AsyncRequest> DefaultFileSource::request(const Resource& resource, Callback callback) { + auto req = std::make_unique<FileSourceRequest>(std::move(callback)); + + req->onCancel([fs = impl->actor(), req = req.get()] () mutable { fs.invoke(&Impl::cancel, req); }); + + impl->actor().invoke(&Impl::request, req.get(), resource, req->actor()); + + return std::move(req); +} + +void DefaultFileSource::listOfflineRegions(std::function<void (expected<OfflineRegions, std::exception_ptr>)> callback) { + impl->actor().invoke(&Impl::listRegions, callback); +} + +void DefaultFileSource::createOfflineRegion(const OfflineRegionDefinition& definition, + const OfflineRegionMetadata& metadata, + std::function<void (expected<OfflineRegion, std::exception_ptr>)> callback) { + impl->actor().invoke(&Impl::createRegion, definition, metadata, callback); +} + +void DefaultFileSource::mergeOfflineRegions(const std::string& sideDatabasePath, + std::function<void (expected<OfflineRegions, std::exception_ptr>)> callback) { + impl->actor().invoke(&Impl::mergeOfflineRegions, sideDatabasePath, callback); +} + +void DefaultFileSource::updateOfflineMetadata(const int64_t regionID, + const OfflineRegionMetadata& metadata, + std::function<void (expected<OfflineRegionMetadata, + std::exception_ptr>)> callback) { + impl->actor().invoke(&Impl::updateMetadata, regionID, metadata, callback); +} + +void DefaultFileSource::deleteOfflineRegion(OfflineRegion&& region, std::function<void (std::exception_ptr)> callback) { + impl->actor().invoke(&Impl::deleteRegion, std::move(region), callback); +} + +void DefaultFileSource::setOfflineRegionObserver(OfflineRegion& region, std::unique_ptr<OfflineRegionObserver> observer) { + impl->actor().invoke(&Impl::setRegionObserver, region.getID(), std::move(observer)); +} + +void DefaultFileSource::setOfflineRegionDownloadState(OfflineRegion& region, OfflineRegionDownloadState state) { + impl->actor().invoke(&Impl::setRegionDownloadState, region.getID(), state); +} + +void DefaultFileSource::getOfflineRegionStatus(OfflineRegion& region, std::function<void (expected<OfflineRegionStatus, std::exception_ptr>)> callback) const { + impl->actor().invoke(&Impl::getRegionStatus, region.getID(), callback); +} + +void DefaultFileSource::setOfflineMapboxTileCountLimit(uint64_t limit) const { + impl->actor().invoke(&Impl::setOfflineMapboxTileCountLimit, limit); +} + +void DefaultFileSource::pause() { + impl->pause(); +} + +void DefaultFileSource::resume() { + impl->resume(); +} + +void DefaultFileSource::put(const Resource& resource, const Response& response) { + impl->actor().invoke(&Impl::put, resource, response); +} + +// For testing only: + +void DefaultFileSource::setOnlineStatus(const bool status) { + impl->actor().invoke(&Impl::setOnlineStatus, status); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/file_source_request.cpp b/platform/default/src/mbgl/storage/file_source_request.cpp new file mode 100644 index 0000000000..09ea8cc32a --- /dev/null +++ b/platform/default/src/mbgl/storage/file_source_request.cpp @@ -0,0 +1,37 @@ +#include <mbgl/storage/file_source_request.hpp> + +#include <mbgl/actor/mailbox.hpp> +#include <mbgl/actor/scheduler.hpp> + +namespace mbgl { + +FileSourceRequest::FileSourceRequest(FileSource::Callback&& callback) + : responseCallback(callback) + , mailbox(std::make_shared<Mailbox>(*Scheduler::GetCurrent())) { +} + +FileSourceRequest::~FileSourceRequest() { + if (cancelCallback) { + cancelCallback(); + } + + mailbox->close(); +} + +void FileSourceRequest::onCancel(std::function<void()>&& callback) { + cancelCallback = std::move(callback); +} + +void FileSourceRequest::setResponse(const Response& response) { + // Copy, because calling the callback will sometimes self + // destroy this object. We cannot move because this method + // can be called more than one. + auto callback = responseCallback; + callback(response); +} + +ActorRef<FileSourceRequest> FileSourceRequest::actor() { + return ActorRef<FileSourceRequest>(*this, mailbox); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/http_file_source.cpp b/platform/default/src/mbgl/storage/http_file_source.cpp new file mode 100644 index 0000000000..213b53de98 --- /dev/null +++ b/platform/default/src/mbgl/storage/http_file_source.cpp @@ -0,0 +1,495 @@ +#include <mbgl/storage/http_file_source.hpp> +#include <mbgl/storage/resource.hpp> +#include <mbgl/storage/response.hpp> +#include <mbgl/util/logging.hpp> + +#include <mbgl/util/util.hpp> +#include <mbgl/util/optional.hpp> +#include <mbgl/util/run_loop.hpp> +#include <mbgl/util/string.hpp> +#include <mbgl/util/timer.hpp> +#include <mbgl/util/chrono.hpp> +#include <mbgl/util/http_header.hpp> + +#include <curl/curl.h> + +// Dynamically load all cURL functions. Debian-derived systems upgraded the OpenSSL version linked +// to in https://salsa.debian.org/debian/curl/commit/95c94957bb7e89e36e78b995fed468c42f64d18d +// They state: +// Rename libcurl3 to libcurl4, because libcurl exposes an SSL_CTX via +// CURLOPT_SSL_CTX_FUNCTION, and this object changes incompatibly between +// openssl 1.0 and openssl 1.1. +// Since we are not accessing the underlying OpenSSL context, we don't care whether we're linking +// against libcurl3 or libcurl4; both use the ABI version 4 which hasn't changed since 2006 +// (see https://curl.haxx.se/libcurl/abi.html). In fact, cURL's ABI compatibility is very good as +// shown on https://abi-laboratory.pro/tracker/timeline/curl/ +// Therefore, we're dynamically loading the cURL symbols we need to avoid linking against versioned +// symbols. +#include <dlfcn.h> + +namespace curl { + +#define CURL_FUNCTIONS \ + X(global_init) \ + X(getdate) \ + X(easy_strerror) \ + X(easy_init) \ + X(easy_setopt) \ + X(easy_cleanup) \ + X(easy_getinfo) \ + X(easy_reset) \ + X(multi_init) \ + X(multi_add_handle) \ + X(multi_remove_handle) \ + X(multi_cleanup) \ + X(multi_info_read) \ + X(multi_strerror) \ + X(multi_socket_action) \ + X(multi_setopt) \ + X(share_init) \ + X(share_cleanup) \ + X(slist_append) \ + X(slist_free_all) + +#define X(name) static decltype(&curl_ ## name) name = nullptr; +CURL_FUNCTIONS +#undef X + +static void* handle = nullptr; + +static void* load(const char* name) { + void* symbol = dlsym(handle, name); + if (const char* error = dlerror()) { + fprintf(stderr, "Cannot load symbol '%s': %s\n", name, error); + dlclose(handle); + handle = nullptr; + abort(); + } + return symbol; +} + +__attribute__((constructor)) +static void load() { + assert(!handle); + handle = dlopen("libcurl.so.4", RTLD_LAZY | RTLD_LOCAL); + if (!handle) { + fprintf(stderr, "Could not open shared library '%s'\n", "libcurl.so.4"); + abort(); + } + + #define X(name) name = (decltype(&curl_ ## name))load("curl_" #name); + CURL_FUNCTIONS + #undef X +} + +__attribute__((constructor)) +static void unload() { + if (handle) { + dlclose(handle); + } +} + +} // namespace curl + + +#include <queue> +#include <map> +#include <cassert> +#include <cstring> +#include <cstdio> + +static void handleError(CURLMcode code) { + if (code != CURLM_OK) { + throw std::runtime_error(std::string("CURL multi error: ") + curl::multi_strerror(code)); + } +} + +static void handleError(CURLcode code) { + if (code != CURLE_OK) { + throw std::runtime_error(std::string("CURL easy error: ") + curl::easy_strerror(code)); + } +} + +namespace mbgl { + +class HTTPFileSource::Impl { +public: + Impl(); + ~Impl(); + + static int handleSocket(CURL *handle, curl_socket_t s, int action, void *userp, void *socketp); + static int startTimeout(CURLM *multi, long timeout_ms, void *userp); + static void onTimeout(HTTPFileSource::Impl *context); + + void perform(curl_socket_t s, util::RunLoop::Event event); + CURL *getHandle(); + void returnHandle(CURL *handle); + void checkMultiInfo(); + + // Used as the CURL timer function to periodically check for socket updates. + util::Timer timeout; + + // CURL multi handle that we use to request multiple URLs at the same time, without having to + // block and spawn threads. + CURLM *multi = nullptr; + + // CURL share handles are used for sharing session state (e.g.) + CURLSH *share = nullptr; + + // A queue that we use for storing resuable CURL easy handles to avoid creating and destroying + // them all the time. + std::queue<CURL *> handles; +}; + +class HTTPRequest : public AsyncRequest { +public: + HTTPRequest(HTTPFileSource::Impl*, Resource, FileSource::Callback); + ~HTTPRequest() override; + + void handleResult(CURLcode code); + +private: + static size_t headerCallback(char *const buffer, const size_t size, const size_t nmemb, void *userp); + static size_t writeCallback(void *const contents, const size_t size, const size_t nmemb, void *userp); + + HTTPFileSource::Impl* context = nullptr; + Resource resource; + FileSource::Callback callback; + + // Will store the current response. + std::shared_ptr<std::string> data; + std::unique_ptr<Response> response; + + optional<std::string> retryAfter; + optional<std::string> xRateLimitReset; + + CURL *handle = nullptr; + curl_slist *headers = nullptr; + + char error[CURL_ERROR_SIZE] = { 0 }; +}; + +HTTPFileSource::Impl::Impl() { + if (curl::global_init(CURL_GLOBAL_ALL)) { + throw std::runtime_error("Could not init cURL"); + } + + share = curl::share_init(); + + multi = curl::multi_init(); + handleError(curl::multi_setopt(multi, CURLMOPT_SOCKETFUNCTION, handleSocket)); + handleError(curl::multi_setopt(multi, CURLMOPT_SOCKETDATA, this)); + handleError(curl::multi_setopt(multi, CURLMOPT_TIMERFUNCTION, startTimeout)); + handleError(curl::multi_setopt(multi, CURLMOPT_TIMERDATA, this)); +} + +HTTPFileSource::Impl::~Impl() { + while (!handles.empty()) { + curl::easy_cleanup(handles.front()); + handles.pop(); + } + + curl::multi_cleanup(multi); + multi = nullptr; + + curl::share_cleanup(share); + share = nullptr; + + timeout.stop(); +} + +CURL *HTTPFileSource::Impl::getHandle() { + if (!handles.empty()) { + auto handle = handles.front(); + handles.pop(); + return handle; + } else { + return curl::easy_init(); + } +} + +void HTTPFileSource::Impl::returnHandle(CURL *handle) { + curl::easy_reset(handle); + handles.push(handle); +} + +void HTTPFileSource::Impl::checkMultiInfo() { + CURLMsg *message = nullptr; + int pending = 0; + + while ((message = curl::multi_info_read(multi, &pending))) { + switch (message->msg) { + case CURLMSG_DONE: { + HTTPRequest *baton = nullptr; + curl::easy_getinfo(message->easy_handle, CURLINFO_PRIVATE, (char *)&baton); + assert(baton); + baton->handleResult(message->data.result); + } break; + + default: + // This should never happen, because there are no other message types. + throw std::runtime_error("CURLMsg returned unknown message type"); + } + } +} + +void HTTPFileSource::Impl::perform(curl_socket_t s, util::RunLoop::Event events) { + int flags = 0; + + if (events == util::RunLoop::Event::Read) { + flags |= CURL_CSELECT_IN; + } + if (events == util::RunLoop::Event::Write) { + flags |= CURL_CSELECT_OUT; + } + + + int running_handles = 0; + curl::multi_socket_action(multi, s, flags, &running_handles); + checkMultiInfo(); +} + +int HTTPFileSource::Impl::handleSocket(CURL * /* handle */, curl_socket_t s, int action, void *userp, + void * /* socketp */) { + assert(userp); + auto context = reinterpret_cast<Impl *>(userp); + + switch (action) { + case CURL_POLL_IN: { + using namespace std::placeholders; + util::RunLoop::Get()->addWatch(s, util::RunLoop::Event::Read, + std::bind(&Impl::perform, context, _1, _2)); + break; + } + case CURL_POLL_OUT: { + using namespace std::placeholders; + util::RunLoop::Get()->addWatch(s, util::RunLoop::Event::Write, + std::bind(&Impl::perform, context, _1, _2)); + break; + } + case CURL_POLL_REMOVE: + util::RunLoop::Get()->removeWatch(s); + break; + default: + throw std::runtime_error("Unhandled CURL socket action"); + } + + return 0; +} + +void HTTPFileSource::Impl::onTimeout(Impl *context) { + int running_handles; + CURLMcode error = curl::multi_socket_action(context->multi, CURL_SOCKET_TIMEOUT, 0, &running_handles); + if (error != CURLM_OK) { + throw std::runtime_error(std::string("CURL multi error: ") + curl::multi_strerror(error)); + } + context->checkMultiInfo(); +} + +int HTTPFileSource::Impl::startTimeout(CURLM * /* multi */, long timeout_ms, void *userp) { + assert(userp); + auto context = reinterpret_cast<Impl *>(userp); + + if (timeout_ms < 0) { + // A timeout of 0 ms means that the timer will invoked in the next loop iteration. + timeout_ms = 0; + } + + context->timeout.stop(); + context->timeout.start(mbgl::Milliseconds(timeout_ms), Duration::zero(), + std::bind(&Impl::onTimeout, context)); + + return 0; +} + +HTTPRequest::HTTPRequest(HTTPFileSource::Impl* context_, Resource resource_, FileSource::Callback callback_) + : context(context_), + resource(std::move(resource_)), + callback(std::move(callback_)), + handle(context->getHandle()) { + + // If there's already a response, set the correct etags/modified headers to make sure we are + // getting a 304 response if possible. This avoids redownloading unchanged data. + if (resource.priorEtag) { + const std::string header = std::string("If-None-Match: ") + *resource.priorEtag; + headers = curl::slist_append(headers, header.c_str()); + } else if (resource.priorModified) { + const std::string time = + std::string("If-Modified-Since: ") + util::rfc1123(*resource.priorModified); + headers = curl::slist_append(headers, time.c_str()); + } + + if (headers) { + curl::easy_setopt(handle, CURLOPT_HTTPHEADER, headers); + } + + handleError(curl::easy_setopt(handle, CURLOPT_PRIVATE, this)); + handleError(curl::easy_setopt(handle, CURLOPT_ERRORBUFFER, error)); + handleError(curl::easy_setopt(handle, CURLOPT_CAINFO, "ca-bundle.crt")); + handleError(curl::easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1)); + handleError(curl::easy_setopt(handle, CURLOPT_URL, resource.url.c_str())); + handleError(curl::easy_setopt(handle, CURLOPT_WRITEFUNCTION, writeCallback)); + handleError(curl::easy_setopt(handle, CURLOPT_WRITEDATA, this)); + handleError(curl::easy_setopt(handle, CURLOPT_HEADERFUNCTION, headerCallback)); + handleError(curl::easy_setopt(handle, CURLOPT_HEADERDATA, this)); +#if LIBCURL_VERSION_NUM >= ((7) << 16 | (21) << 8 | 6) // Renamed in 7.21.6 + handleError(curl::easy_setopt(handle, CURLOPT_ACCEPT_ENCODING, "gzip, deflate")); +#else + handleError(curl::easy_setopt(handle, CURLOPT_ENCODING, "gzip, deflate")); +#endif + handleError(curl::easy_setopt(handle, CURLOPT_USERAGENT, "MapboxGL/1.0")); + handleError(curl::easy_setopt(handle, CURLOPT_SHARE, context->share)); + + // Start requesting the information. + handleError(curl::multi_add_handle(context->multi, handle)); +} + +HTTPRequest::~HTTPRequest() { + handleError(curl::multi_remove_handle(context->multi, handle)); + context->returnHandle(handle); + handle = nullptr; + + if (headers) { + curl::slist_free_all(headers); + headers = nullptr; + } +} + +// This function is called when we have new data for a request. We just append it to the string +// containing the previous data. +size_t HTTPRequest::writeCallback(void *const contents, const size_t size, const size_t nmemb, void *userp) { + assert(userp); + auto impl = reinterpret_cast<HTTPRequest *>(userp); + + if (!impl->data) { + impl->data = std::make_shared<std::string>(); + } + + impl->data->append((char *)contents, size * nmemb); + return size * nmemb; +} + +// Compares the beginning of the (non-zero-terminated!) data buffer with the (zero-terminated!) +// header string. If the data buffer contains the header string at the beginning, it returns +// the length of the header string == begin of the value, otherwise it returns npos. +// The comparison of the header is ASCII-case-insensitive. +size_t headerMatches(const char *const header, const char *const buffer, const size_t length) { + const size_t headerLength = strlen(header); + if (length < headerLength) { + return std::string::npos; + } + size_t i = 0; + while (i < length && i < headerLength && std::tolower(buffer[i]) == std::tolower(header[i])) { + i++; + } + return i == headerLength ? i : std::string::npos; +} + +size_t HTTPRequest::headerCallback(char *const buffer, const size_t size, const size_t nmemb, void *userp) { + assert(userp); + auto baton = reinterpret_cast<HTTPRequest *>(userp); + + if (!baton->response) { + baton->response = std::make_unique<Response>(); + } + + const size_t length = size * nmemb; + size_t begin = std::string::npos; + if ((begin = headerMatches("last-modified: ", buffer, length)) != std::string::npos) { + // Always overwrite the modification date; We might already have a value here from the + // Date header, but this one is more accurate. + const std::string value { buffer + begin, length - begin - 2 }; // remove \r\n + baton->response->modified = Timestamp{ Seconds(curl::getdate(value.c_str(), nullptr)) }; + } else if ((begin = headerMatches("etag: ", buffer, length)) != std::string::npos) { + baton->response->etag = std::string(buffer + begin, length - begin - 2); // remove \r\n + } else if ((begin = headerMatches("cache-control: ", buffer, length)) != std::string::npos) { + const std::string value { buffer + begin, length - begin - 2 }; // remove \r\n + const auto cc = http::CacheControl::parse(value.c_str()); + baton->response->expires = cc.toTimePoint(); + baton->response->mustRevalidate = cc.mustRevalidate; + } else if ((begin = headerMatches("expires: ", buffer, length)) != std::string::npos) { + const std::string value { buffer + begin, length - begin - 2 }; // remove \r\n + baton->response->expires = Timestamp{ Seconds(curl::getdate(value.c_str(), nullptr)) }; + } else if ((begin = headerMatches("retry-after: ", buffer, length)) != std::string::npos) { + baton->retryAfter = std::string(buffer + begin, length - begin - 2); // remove \r\n + } else if ((begin = headerMatches("x-rate-limit-reset: ", buffer, length)) != std::string::npos) { + baton->xRateLimitReset = std::string(buffer + begin, length - begin - 2); // remove \r\n + } + + return length; +} + +void HTTPRequest::handleResult(CURLcode code) { + // Make sure a response object exists in case we haven't got any headers or content. + if (!response) { + response = std::make_unique<Response>(); + } + + using Error = Response::Error; + + // Add human-readable error code + if (code != CURLE_OK) { + switch (code) { + case CURLE_COULDNT_RESOLVE_PROXY: + case CURLE_COULDNT_RESOLVE_HOST: + case CURLE_COULDNT_CONNECT: + case CURLE_OPERATION_TIMEDOUT: + + response->error = std::make_unique<Error>( + Error::Reason::Connection, std::string{ curl::easy_strerror(code) } + ": " + error); + break; + + default: + response->error = std::make_unique<Error>( + Error::Reason::Other, std::string{ curl::easy_strerror(code) } + ": " + error); + break; + } + } else { + long responseCode = 0; + curl::easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &responseCode); + + if (responseCode == 200) { + if (data) { + response->data = std::move(data); + } else { + response->data = std::make_shared<std::string>(); + } + } else if (responseCode == 204 || (responseCode == 404 && resource.kind == Resource::Kind::Tile)) { + response->noContent = true; + } else if (responseCode == 304) { + response->notModified = true; + } else if (responseCode == 404) { + response->error = + std::make_unique<Error>(Error::Reason::NotFound, "HTTP status code 404"); + } else if (responseCode == 429) { + response->error = + std::make_unique<Error>(Error::Reason::RateLimit, "HTTP status code 429", + http::parseRetryHeaders(retryAfter, xRateLimitReset)); + } else if (responseCode >= 500 && responseCode < 600) { + response->error = + std::make_unique<Error>(Error::Reason::Server, std::string{ "HTTP status code " } + + util::toString(responseCode)); + } else { + response->error = + std::make_unique<Error>(Error::Reason::Other, std::string{ "HTTP status code " } + + util::toString(responseCode)); + } + } + + // Calling `callback` may result in deleting `this`. Copy data to temporaries first. + auto callback_ = callback; + auto response_ = *response; + callback_(response_); +} + +HTTPFileSource::HTTPFileSource() + : impl(std::make_unique<Impl>()) { +} + +HTTPFileSource::~HTTPFileSource() = default; + +std::unique_ptr<AsyncRequest> HTTPFileSource::request(const Resource& resource, Callback callback) { + return std::make_unique<HTTPRequest>(impl.get(), resource, callback); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/local_file_source.cpp b/platform/default/src/mbgl/storage/local_file_source.cpp new file mode 100644 index 0000000000..1b7b7b9278 --- /dev/null +++ b/platform/default/src/mbgl/storage/local_file_source.cpp @@ -0,0 +1,81 @@ +#include <mbgl/storage/local_file_source.hpp> +#include <mbgl/storage/file_source_request.hpp> +#include <mbgl/storage/response.hpp> +#include <mbgl/util/string.hpp> +#include <mbgl/util/thread.hpp> +#include <mbgl/util/url.hpp> +#include <mbgl/util/util.hpp> +#include <mbgl/util/io.hpp> + +#include <sys/types.h> +#include <sys/stat.h> + +#if defined(_WINDOWS) && !defined(S_ISDIR) +#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR) +#endif + +namespace { + +const std::string fileProtocol = "file://"; + +} // namespace + +namespace mbgl { + +class LocalFileSource::Impl { +public: + Impl(ActorRef<Impl>) {} + + void request(const std::string& url, ActorRef<FileSourceRequest> req) { + Response response; + + if (!acceptsURL(url)) { + response.error = std::make_unique<Response::Error>(Response::Error::Reason::Other, + "Invalid file URL"); + req.invoke(&FileSourceRequest::setResponse, response); + return; + } + + // Cut off the protocol and prefix with path. + const auto path = mbgl::util::percentDecode(url.substr(fileProtocol.size())); + struct stat buf; + int result = stat(path.c_str(), &buf); + + if (result == 0 && S_ISDIR(buf.st_mode)) { + response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound); + } else if (result == -1 && errno == ENOENT) { + response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound); + } else { + try { + response.data = std::make_shared<std::string>(util::read_file(path)); + } catch (...) { + response.error = std::make_unique<Response::Error>( + Response::Error::Reason::Other, + util::toString(std::current_exception())); + } + } + + req.invoke(&FileSourceRequest::setResponse, response); + } + +}; + +LocalFileSource::LocalFileSource() + : impl(std::make_unique<util::Thread<Impl>>("LocalFileSource")) { +} + +LocalFileSource::~LocalFileSource() = default; + +std::unique_ptr<AsyncRequest> LocalFileSource::request(const Resource& resource, Callback callback) { + auto req = std::make_unique<FileSourceRequest>(std::move(callback)); + + impl->actor().invoke(&Impl::request, resource.url, req->actor()); + + return std::move(req); +} + +bool LocalFileSource::acceptsURL(const std::string& url) { + return 0 == url.rfind(fileProtocol, 0); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/offline.cpp b/platform/default/src/mbgl/storage/offline.cpp new file mode 100644 index 0000000000..e1ec0acb31 --- /dev/null +++ b/platform/default/src/mbgl/storage/offline.cpp @@ -0,0 +1,158 @@ +#include <mbgl/storage/offline.hpp> +#include <mbgl/util/tileset.hpp> +#include <mbgl/util/projection.hpp> + +#include <mapbox/geojson.hpp> +#include <mapbox/geojson/rapidjson.hpp> + +#include <rapidjson/document.h> +#include <rapidjson/stringbuffer.h> +#include <rapidjson/writer.h> + +#include <cmath> + +namespace mbgl { + +// OfflineTilePyramidRegionDefinition + +OfflineTilePyramidRegionDefinition::OfflineTilePyramidRegionDefinition( + std::string styleURL_, LatLngBounds bounds_, double minZoom_, double maxZoom_, float pixelRatio_) + : styleURL(std::move(styleURL_)), + bounds(std::move(bounds_)), + minZoom(minZoom_), + maxZoom(maxZoom_), + pixelRatio(pixelRatio_) { + if (minZoom < 0 || maxZoom < 0 || maxZoom < minZoom || pixelRatio < 0 || + !std::isfinite(minZoom) || std::isnan(maxZoom) || !std::isfinite(pixelRatio)) { + throw std::invalid_argument("Invalid offline region definition"); + } +} + + +// OfflineGeometryRegionDefinition + +OfflineGeometryRegionDefinition::OfflineGeometryRegionDefinition(std::string styleURL_, Geometry<double> geometry_, double minZoom_, double maxZoom_, float pixelRatio_) + : styleURL(styleURL_) + , geometry(std::move(geometry_)) + , minZoom(minZoom_) + , maxZoom(maxZoom_) + , pixelRatio(pixelRatio_) { + if (minZoom < 0 || maxZoom < 0 || maxZoom < minZoom || pixelRatio < 0 || + !std::isfinite(minZoom) || std::isnan(maxZoom) || !std::isfinite(pixelRatio)) { + throw std::invalid_argument("Invalid offline region definition"); + } +} + +OfflineRegionDefinition decodeOfflineRegionDefinition(const std::string& region) { + rapidjson::GenericDocument<rapidjson::UTF8<>, rapidjson::CrtAllocator> doc; + doc.Parse<0>(region.c_str()); + + // validation + + auto hasValidBounds = [&] { + return doc.HasMember("bounds") && doc["bounds"].IsArray() && doc["bounds"].Size() == 4 + && doc["bounds"][0].IsDouble() && doc["bounds"][1].IsDouble() + && doc["bounds"][2].IsDouble() && doc["bounds"][3].IsDouble(); + }; + + auto hasValidGeometry = [&] { + return doc.HasMember("geometry") && doc["geometry"].IsObject(); + }; + + if (doc.HasParseError() + || !doc.HasMember("style_url") || !doc["style_url"].IsString() + || !(hasValidBounds() || hasValidGeometry()) + || !doc.HasMember("min_zoom") || !doc["min_zoom"].IsDouble() + || (doc.HasMember("max_zoom") && !doc["max_zoom"].IsDouble()) + || !doc.HasMember("pixel_ratio") || !doc["pixel_ratio"].IsDouble()) { + throw std::runtime_error("Malformed offline region definition"); + } + + // Common properties + + std::string styleURL { doc["style_url"].GetString(), doc["style_url"].GetStringLength() }; + double minZoom = doc["min_zoom"].GetDouble(); + double maxZoom = doc.HasMember("max_zoom") ? doc["max_zoom"].GetDouble() : INFINITY; + float pixelRatio = doc["pixel_ratio"].GetDouble(); + + if (doc.HasMember("bounds")) { + return OfflineTilePyramidRegionDefinition{ + styleURL, + LatLngBounds::hull( + LatLng(doc["bounds"][0].GetDouble(), doc["bounds"][1].GetDouble()), + LatLng(doc["bounds"][2].GetDouble(), doc["bounds"][3].GetDouble())), + minZoom, maxZoom, pixelRatio }; + } else { + return OfflineGeometryRegionDefinition{ + styleURL, + mapbox::geojson::convert<Geometry<double>>(doc["geometry"].GetObject()), + minZoom, maxZoom, pixelRatio }; + }; + +} + +std::string encodeOfflineRegionDefinition(const OfflineRegionDefinition& region) { + rapidjson::GenericDocument<rapidjson::UTF8<>, rapidjson::CrtAllocator> doc; + doc.SetObject(); + + // Encode common properties + region.match([&](auto& _region) { + doc.AddMember("style_url", rapidjson::StringRef(_region.styleURL.data(), _region.styleURL.length()), doc.GetAllocator()); + doc.AddMember("min_zoom", _region.minZoom, doc.GetAllocator()); + if (std::isfinite(_region.maxZoom)) { + doc.AddMember("max_zoom", _region.maxZoom, doc.GetAllocator()); + } + + doc.AddMember("pixel_ratio", _region.pixelRatio, doc.GetAllocator()); + }); + + // Encode specific properties + region.match( + [&] (const OfflineTilePyramidRegionDefinition& _region) { + rapidjson::GenericValue<rapidjson::UTF8<>, rapidjson::CrtAllocator> bounds(rapidjson::kArrayType); + bounds.PushBack(_region.bounds.south(), doc.GetAllocator()); + bounds.PushBack(_region.bounds.west(), doc.GetAllocator()); + bounds.PushBack(_region.bounds.north(), doc.GetAllocator()); + bounds.PushBack(_region.bounds.east(), doc.GetAllocator()); + doc.AddMember("bounds", bounds, doc.GetAllocator()); + + }, + [&] (const OfflineGeometryRegionDefinition& _region) { + doc.AddMember("geometry", mapbox::geojson::convert(_region.geometry, doc.GetAllocator()), doc.GetAllocator()); + + } + ); + + rapidjson::StringBuffer buffer; + rapidjson::Writer<rapidjson::StringBuffer> writer(buffer); + doc.Accept(writer); + + return buffer.GetString(); +} + + +// OfflineRegion + +OfflineRegion::OfflineRegion(int64_t id_, + OfflineRegionDefinition definition_, + OfflineRegionMetadata metadata_) + : id(id_), + definition(std::move(definition_)), + metadata(std::move(metadata_)) { +} + +OfflineRegion::OfflineRegion(OfflineRegion&&) = default; +OfflineRegion::~OfflineRegion() = default; + +const OfflineRegionDefinition& OfflineRegion::getDefinition() const { + return definition; +} + +const OfflineRegionMetadata& OfflineRegion::getMetadata() const { + return metadata; +} + +int64_t OfflineRegion::getID() const { + return id; +} +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/offline_database.cpp b/platform/default/src/mbgl/storage/offline_database.cpp new file mode 100644 index 0000000000..7732076991 --- /dev/null +++ b/platform/default/src/mbgl/storage/offline_database.cpp @@ -0,0 +1,1129 @@ +#include <mbgl/storage/offline_database.hpp> +#include <mbgl/storage/response.hpp> +#include <mbgl/storage/sqlite3.hpp> +#include <mbgl/util/compression.hpp> +#include <mbgl/util/io.hpp> +#include <mbgl/util/string.hpp> +#include <mbgl/util/chrono.hpp> +#include <mbgl/util/logging.hpp> + +#include <mbgl/storage/offline_schema.hpp> +#include <mbgl/storage/merge_sideloaded.hpp> + + +namespace mbgl { + +OfflineDatabase::OfflineDatabase(std::string path_, uint64_t maximumCacheSize_) + : path(std::move(path_)), + maximumCacheSize(maximumCacheSize_) { + try { + initialize(); + } catch (const util::IOException& ex) { + handleError(ex, "open database"); + } catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "open database"); + } + // Assume that we can't open the database right now and work with an empty database object. +} + +OfflineDatabase::~OfflineDatabase() { + // Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we + // can't throw anything. + try { + statements.clear(); + db.reset(); + } catch (const util::IOException& ex) { + handleError(ex, "close database"); + } catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "close database"); + } +} + +void OfflineDatabase::initialize() { + assert(!db); + assert(statements.empty()); + + db = std::make_unique<mapbox::sqlite::Database>( + mapbox::sqlite::Database::open(path, mapbox::sqlite::ReadWriteCreate)); + db->setBusyTimeout(Milliseconds::max()); + db->exec("PRAGMA foreign_keys = ON"); + + const auto userVersion = getPragma<int64_t>("PRAGMA user_version"); + switch (userVersion) { + case 0: + case 1: + // Newly created database, or old cache-only database; remove old table if it exists. + removeOldCacheTable(); + createSchema(); + return; + case 2: + migrateToVersion3(); + // fall through + case 3: + // Removed migration, see below. + // fall through + case 4: + migrateToVersion5(); + // fall through + case 5: + migrateToVersion6(); + // fall through + case 6: + // Happy path; we're done + return; + default: + // Downgrade: delete the database and try to reinitialize. + removeExisting(); + initialize(); + } +} + +void OfflineDatabase::handleError(const mapbox::sqlite::Exception& ex, const char* action) { + if (ex.code == mapbox::sqlite::ResultCode::NotADB || + ex.code == mapbox::sqlite::ResultCode::Corrupt || + (ex.code == mapbox::sqlite::ResultCode::ReadOnly && + ex.extendedCode == mapbox::sqlite::ExtendedResultCode::ReadOnlyDBMoved)) { + // The database was corruped, moved away, or deleted. We're going to start fresh with a + // clean slate for the next operation. + Log::Error(Event::Database, static_cast<int>(ex.code), "Can't %s: %s", action, ex.what()); + try { + removeExisting(); + } catch (const util::IOException& ioEx) { + handleError(ioEx, action); + } + } else { + // We treat the error as temporary, and pretend we have an inaccessible DB. + Log::Warning(Event::Database, static_cast<int>(ex.code), "Can't %s: %s", action, ex.what()); + } +} + +void OfflineDatabase::handleError(const util::IOException& ex, const char* action) { + // We failed to delete the database file. + Log::Error(Event::Database, ex.code, "Can't %s: %s", action, ex.what()); +} + +void OfflineDatabase::removeExisting() { + Log::Warning(Event::Database, "Removing existing incompatible offline database"); + + statements.clear(); + db.reset(); + + util::deleteFile(path); +} + +void OfflineDatabase::removeOldCacheTable() { + assert(db); + db->exec("DROP TABLE IF EXISTS http_cache"); + db->exec("VACUUM"); +} + +void OfflineDatabase::createSchema() { + assert(db); + db->exec("PRAGMA auto_vacuum = INCREMENTAL"); + db->exec("PRAGMA journal_mode = DELETE"); + db->exec("PRAGMA synchronous = FULL"); + mapbox::sqlite::Transaction transaction(*db); + db->exec(offlineDatabaseSchema); + db->exec("PRAGMA user_version = 6"); + transaction.commit(); +} + +void OfflineDatabase::migrateToVersion3() { + assert(db); + db->exec("PRAGMA auto_vacuum = INCREMENTAL"); + db->exec("VACUUM"); + db->exec("PRAGMA user_version = 3"); +} + +// Schema version 4 was WAL journal + NORMAL sync. It was reverted during pre- +// release development and the migration was removed entirely to avoid potential +// conflicts from quickly (and needlessly) switching journal and sync modes. +// +// See: https://github.com/mapbox/mapbox-gl-native/pull/6320 + +void OfflineDatabase::migrateToVersion5() { + assert(db); + db->exec("PRAGMA journal_mode = DELETE"); + db->exec("PRAGMA synchronous = FULL"); + db->exec("PRAGMA user_version = 5"); +} + +void OfflineDatabase::migrateToVersion6() { + assert(db); + mapbox::sqlite::Transaction transaction(*db); + db->exec("ALTER TABLE resources ADD COLUMN must_revalidate INTEGER NOT NULL DEFAULT 0"); + db->exec("ALTER TABLE tiles ADD COLUMN must_revalidate INTEGER NOT NULL DEFAULT 0"); + db->exec("PRAGMA user_version = 6"); + transaction.commit(); +} + +mapbox::sqlite::Statement& OfflineDatabase::getStatement(const char* sql) { + if (!db) { + initialize(); + } + auto it = statements.find(sql); + if (it == statements.end()) { + it = statements.emplace(sql, std::make_unique<mapbox::sqlite::Statement>(*db, sql)).first; + } + return *it->second; +} + +optional<Response> OfflineDatabase::get(const Resource& resource) try { + auto result = getInternal(resource); + return result ? optional<Response>{ result->first } : nullopt; +} catch (const util::IOException& ex) { + handleError(ex, "read resource"); + return nullopt; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "read resource"); + return nullopt; +} + +optional<std::pair<Response, uint64_t>> OfflineDatabase::getInternal(const Resource& resource) { + if (resource.kind == Resource::Kind::Tile) { + assert(resource.tileData); + return getTile(*resource.tileData); + } else { + return getResource(resource); + } +} + +optional<int64_t> OfflineDatabase::hasInternal(const Resource& resource) { + if (resource.kind == Resource::Kind::Tile) { + assert(resource.tileData); + return hasTile(*resource.tileData); + } else { + return hasResource(resource); + } +} + +std::pair<bool, uint64_t> OfflineDatabase::put(const Resource& resource, const Response& response) try { + if (!db) { + initialize(); + } + mapbox::sqlite::Transaction transaction(*db, mapbox::sqlite::Transaction::Immediate); + auto result = putInternal(resource, response, true); + transaction.commit(); + return result; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "write resource"); + return { false, 0 }; +} + +std::pair<bool, uint64_t> OfflineDatabase::putInternal(const Resource& resource, const Response& response, bool evict_) { + if (response.error) { + return { false, 0 }; + } + + std::string compressedData; + bool compressed = false; + uint64_t size = 0; + + if (response.data) { + compressedData = util::compress(*response.data); + compressed = compressedData.size() < response.data->size(); + size = compressed ? compressedData.size() : response.data->size(); + } + + if (evict_ && !evict(size)) { + Log::Info(Event::Database, "Unable to make space for entry"); + return { false, 0 }; + } + + bool inserted; + + if (resource.kind == Resource::Kind::Tile) { + assert(resource.tileData); + inserted = putTile(*resource.tileData, response, + compressed ? compressedData : response.data ? *response.data : "", + compressed); + } else { + inserted = putResource(resource, response, + compressed ? compressedData : response.data ? *response.data : "", + compressed); + } + + return { inserted, size }; +} + +optional<std::pair<Response, uint64_t>> OfflineDatabase::getResource(const Resource& resource) { + // Update accessed timestamp used for LRU eviction. + try { + mapbox::sqlite::Query accessedQuery{ getStatement("UPDATE resources SET accessed = ?1 WHERE url = ?2") }; + accessedQuery.bind(1, util::now()); + accessedQuery.bind(2, resource.url); + accessedQuery.run(); + } catch (const mapbox::sqlite::Exception& ex) { + if (ex.code == mapbox::sqlite::ResultCode::NotADB || + ex.code == mapbox::sqlite::ResultCode::Corrupt) { + throw; + } + + // If we don't have any indication that the database is corrupt, continue as usual. + Log::Warning(Event::Database, static_cast<int>(ex.code), "Can't update timestamp: %s", ex.what()); + } + + // clang-format off + mapbox::sqlite::Query query{ getStatement( + // 0 1 2 3 4 5 + "SELECT etag, expires, must_revalidate, modified, data, compressed " + "FROM resources " + "WHERE url = ?") }; + // clang-format on + + query.bind(1, resource.url); + + if (!query.run()) { + return nullopt; + } + + Response response; + uint64_t size = 0; + + response.etag = query.get<optional<std::string>>(0); + response.expires = query.get<optional<Timestamp>>(1); + response.mustRevalidate = query.get<bool>(2); + response.modified = query.get<optional<Timestamp>>(3); + + auto data = query.get<optional<std::string>>(4); + if (!data) { + response.noContent = true; + } else if (query.get<bool>(5)) { + response.data = std::make_shared<std::string>(util::decompress(*data)); + size = data->length(); + } else { + response.data = std::make_shared<std::string>(*data); + size = data->length(); + } + + return std::make_pair(response, size); +} + +optional<int64_t> OfflineDatabase::hasResource(const Resource& resource) { + mapbox::sqlite::Query query{ getStatement("SELECT length(data) FROM resources WHERE url = ?") }; + query.bind(1, resource.url); + if (!query.run()) { + return nullopt; + } + + return query.get<optional<int64_t>>(0); +} + +bool OfflineDatabase::putResource(const Resource& resource, + const Response& response, + const std::string& data, + bool compressed) { + if (response.notModified) { + // clang-format off + mapbox::sqlite::Query notModifiedQuery{ getStatement( + "UPDATE resources " + "SET accessed = ?1, " + " expires = ?2, " + " must_revalidate = ?3 " + "WHERE url = ?4 ") }; + // clang-format on + + notModifiedQuery.bind(1, util::now()); + notModifiedQuery.bind(2, response.expires); + notModifiedQuery.bind(3, response.mustRevalidate); + notModifiedQuery.bind(4, resource.url); + notModifiedQuery.run(); + return false; + } + + // We can't use REPLACE because it would change the id value. + // clang-format off + mapbox::sqlite::Query updateQuery{ getStatement( + "UPDATE resources " + "SET kind = ?1, " + " etag = ?2, " + " expires = ?3, " + " must_revalidate = ?4, " + " modified = ?5, " + " accessed = ?6, " + " data = ?7, " + " compressed = ?8 " + "WHERE url = ?9 ") }; + // clang-format on + + updateQuery.bind(1, int(resource.kind)); + updateQuery.bind(2, response.etag); + updateQuery.bind(3, response.expires); + updateQuery.bind(4, response.mustRevalidate); + updateQuery.bind(5, response.modified); + updateQuery.bind(6, util::now()); + updateQuery.bind(9, resource.url); + + if (response.noContent) { + updateQuery.bind(7, nullptr); + updateQuery.bind(8, false); + } else { + updateQuery.bindBlob(7, data.data(), data.size(), false); + updateQuery.bind(8, compressed); + } + + updateQuery.run(); + if (updateQuery.changes() != 0) { + return false; + } + + // clang-format off + mapbox::sqlite::Query insertQuery{ getStatement( + "INSERT INTO resources (url, kind, etag, expires, must_revalidate, modified, accessed, data, compressed) " + "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) ") }; + // clang-format on + + insertQuery.bind(1, resource.url); + insertQuery.bind(2, int(resource.kind)); + insertQuery.bind(3, response.etag); + insertQuery.bind(4, response.expires); + insertQuery.bind(5, response.mustRevalidate); + insertQuery.bind(6, response.modified); + insertQuery.bind(7, util::now()); + + if (response.noContent) { + insertQuery.bind(8, nullptr); + insertQuery.bind(9, false); + } else { + insertQuery.bindBlob(8, data.data(), data.size(), false); + insertQuery.bind(9, compressed); + } + + insertQuery.run(); + + return true; +} + +optional<std::pair<Response, uint64_t>> OfflineDatabase::getTile(const Resource::TileData& tile) { + // Update accessed timestamp used for LRU eviction. + try { + // clang-format off + mapbox::sqlite::Query accessedQuery{ getStatement( + "UPDATE tiles " + "SET accessed = ?1 " + "WHERE url_template = ?2 " + " AND pixel_ratio = ?3 " + " AND x = ?4 " + " AND y = ?5 " + " AND z = ?6 ") }; + // clang-format on + + accessedQuery.bind(1, util::now()); + accessedQuery.bind(2, tile.urlTemplate); + accessedQuery.bind(3, tile.pixelRatio); + accessedQuery.bind(4, tile.x); + accessedQuery.bind(5, tile.y); + accessedQuery.bind(6, tile.z); + accessedQuery.run(); + } catch (const mapbox::sqlite::Exception& ex) { + if (ex.code == mapbox::sqlite::ResultCode::NotADB || ex.code == mapbox::sqlite::ResultCode::Corrupt) { + throw; + } + + // If we don't have any indication that the database is corrupt, continue as usual. + Log::Warning(Event::Database, static_cast<int>(ex.code), "Can't update timestamp: %s", ex.what()); + } + + // clang-format off + mapbox::sqlite::Query query{ getStatement( + // 0 1 2, 3, 4, 5 + "SELECT etag, expires, must_revalidate, modified, data, compressed " + "FROM tiles " + "WHERE url_template = ?1 " + " AND pixel_ratio = ?2 " + " AND x = ?3 " + " AND y = ?4 " + " AND z = ?5 ") }; + // clang-format on + + query.bind(1, tile.urlTemplate); + query.bind(2, tile.pixelRatio); + query.bind(3, tile.x); + query.bind(4, tile.y); + query.bind(5, tile.z); + + if (!query.run()) { + return nullopt; + } + + Response response; + uint64_t size = 0; + + response.etag = query.get<optional<std::string>>(0); + response.expires = query.get<optional<Timestamp>>(1); + response.mustRevalidate = query.get<bool>(2); + response.modified = query.get<optional<Timestamp>>(3); + + optional<std::string> data = query.get<optional<std::string>>(4); + if (!data) { + response.noContent = true; + } else if (query.get<bool>(5)) { + response.data = std::make_shared<std::string>(util::decompress(*data)); + size = data->length(); + } else { + response.data = std::make_shared<std::string>(*data); + size = data->length(); + } + + return std::make_pair(response, size); +} + +optional<int64_t> OfflineDatabase::hasTile(const Resource::TileData& tile) { + // clang-format off + mapbox::sqlite::Query size{ getStatement( + "SELECT length(data) " + "FROM tiles " + "WHERE url_template = ?1 " + " AND pixel_ratio = ?2 " + " AND x = ?3 " + " AND y = ?4 " + " AND z = ?5 ") }; + // clang-format on + + size.bind(1, tile.urlTemplate); + size.bind(2, tile.pixelRatio); + size.bind(3, tile.x); + size.bind(4, tile.y); + size.bind(5, tile.z); + + if (!size.run()) { + return nullopt; + } + + return size.get<optional<int64_t>>(0); +} + +bool OfflineDatabase::putTile(const Resource::TileData& tile, + const Response& response, + const std::string& data, + bool compressed) { + if (response.notModified) { + // clang-format off + mapbox::sqlite::Query notModifiedQuery{ getStatement( + "UPDATE tiles " + "SET accessed = ?1, " + " expires = ?2, " + " must_revalidate = ?3 " + "WHERE url_template = ?4 " + " AND pixel_ratio = ?5 " + " AND x = ?6 " + " AND y = ?7 " + " AND z = ?8 ") }; + // clang-format on + + notModifiedQuery.bind(1, util::now()); + notModifiedQuery.bind(2, response.expires); + notModifiedQuery.bind(3, response.mustRevalidate); + notModifiedQuery.bind(4, tile.urlTemplate); + notModifiedQuery.bind(5, tile.pixelRatio); + notModifiedQuery.bind(6, tile.x); + notModifiedQuery.bind(7, tile.y); + notModifiedQuery.bind(8, tile.z); + notModifiedQuery.run(); + return false; + } + + // We can't use REPLACE because it would change the id value. + + // clang-format off + mapbox::sqlite::Query updateQuery{ getStatement( + "UPDATE tiles " + "SET modified = ?1, " + " etag = ?2, " + " expires = ?3, " + " must_revalidate = ?4, " + " accessed = ?5, " + " data = ?6, " + " compressed = ?7 " + "WHERE url_template = ?8 " + " AND pixel_ratio = ?9 " + " AND x = ?10 " + " AND y = ?11 " + " AND z = ?12 ") }; + // clang-format on + + updateQuery.bind(1, response.modified); + updateQuery.bind(2, response.etag); + updateQuery.bind(3, response.expires); + updateQuery.bind(4, response.mustRevalidate); + updateQuery.bind(5, util::now()); + updateQuery.bind(8, tile.urlTemplate); + updateQuery.bind(9, tile.pixelRatio); + updateQuery.bind(10, tile.x); + updateQuery.bind(11, tile.y); + updateQuery.bind(12, tile.z); + + if (response.noContent) { + updateQuery.bind(6, nullptr); + updateQuery.bind(7, false); + } else { + updateQuery.bindBlob(6, data.data(), data.size(), false); + updateQuery.bind(7, compressed); + } + + updateQuery.run(); + if (updateQuery.changes() != 0) { + return false; + } + + // clang-format off + mapbox::sqlite::Query insertQuery{ getStatement( + "INSERT INTO tiles (url_template, pixel_ratio, x, y, z, modified, must_revalidate, etag, expires, accessed, data, compressed) " + "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)") }; + // clang-format on + + insertQuery.bind(1, tile.urlTemplate); + insertQuery.bind(2, tile.pixelRatio); + insertQuery.bind(3, tile.x); + insertQuery.bind(4, tile.y); + insertQuery.bind(5, tile.z); + insertQuery.bind(6, response.modified); + insertQuery.bind(7, response.mustRevalidate); + insertQuery.bind(8, response.etag); + insertQuery.bind(9, response.expires); + insertQuery.bind(10, util::now()); + + if (response.noContent) { + insertQuery.bind(11, nullptr); + insertQuery.bind(12, false); + } else { + insertQuery.bindBlob(11, data.data(), data.size(), false); + insertQuery.bind(12, compressed); + } + + insertQuery.run(); + + return true; +} + +expected<OfflineRegions, std::exception_ptr> OfflineDatabase::listRegions() try { + mapbox::sqlite::Query query{ getStatement("SELECT id, definition, description FROM regions") }; + OfflineRegions result; + while (query.run()) { + const auto id = query.get<int64_t>(0); + const auto definition = query.get<std::string>(1); + const auto description = query.get<std::vector<uint8_t>>(2); + try { + // Construct, then move because this constructor is private. + OfflineRegion region(id, decodeOfflineRegionDefinition(definition), description); + result.emplace_back(std::move(region)); + } catch (const std::exception& ex) { + // Catch errors from malformed offline region definitions + // and skip them. + Log::Error(Event::General, "%s", ex.what()); + } + } + // Explicit move to avoid triggering the copy constructor. + return { std::move(result) }; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "list regions"); + return unexpected<std::exception_ptr>(std::current_exception()); +} + +expected<OfflineRegion, std::exception_ptr> +OfflineDatabase::createRegion(const OfflineRegionDefinition& definition, + const OfflineRegionMetadata& metadata) try { + // clang-format off + mapbox::sqlite::Query query{ getStatement( + "INSERT INTO regions (definition, description) " + "VALUES (?1, ?2) ") }; + // clang-format on + + query.bind(1, encodeOfflineRegionDefinition(definition)); + query.bindBlob(2, metadata); + query.run(); + return OfflineRegion(query.lastInsertRowId(), definition, metadata); +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "create region"); + return unexpected<std::exception_ptr>(std::current_exception()); +} + +expected<OfflineRegions, std::exception_ptr> +OfflineDatabase::mergeDatabase(const std::string& sideDatabasePath) { + try { + // clang-format off + mapbox::sqlite::Query query{ getStatement("ATTACH DATABASE ?1 AS side") }; + // clang-format on + + query.bind(1, sideDatabasePath); + query.run(); + } catch (const mapbox::sqlite::Exception& ex) { + Log::Error(Event::Database, static_cast<int>(ex.code), "Can't attach database (%s) for merge: %s", sideDatabasePath.c_str(), ex.what()); + return unexpected<std::exception_ptr>(std::current_exception()); + } + try { + // Support sideloaded databases at user_version = 6. Future schema version + // changes will need to implement migration paths for sideloaded databases at + // version 6. + auto sideUserVersion = static_cast<int>(getPragma<int64_t>("PRAGMA side.user_version")); + const auto mainUserVersion = getPragma<int64_t>("PRAGMA user_version"); + if (sideUserVersion < 6 || sideUserVersion != mainUserVersion) { + throw std::runtime_error("Merge database has incorrect user_version"); + } + + auto currentTileCount = getOfflineMapboxTileCount(); + // clang-format off + mapbox::sqlite::Query queryTiles{ getStatement( + "SELECT COUNT(DISTINCT st.id) " + "FROM side.tiles st " + //only consider region tiles, and not ambient tiles. + "JOIN side.region_tiles srt ON srt.tile_id = st.id " + "LEFT JOIN tiles t ON st.url_template = t.url_template AND " + "st.pixel_ratio = t.pixel_ratio AND " + "st.z = t.z AND " + "st.x = t.x AND " + "st.y = t.y " + "WHERE t.id IS NULL " + "AND st.url_template LIKE 'mapbox://%' ") }; + // clang-format on + queryTiles.run(); + auto countOfTilesToMerge = queryTiles.get<int64_t>(0); + if ((countOfTilesToMerge + currentTileCount) > offlineMapboxTileCountLimit) { + throw MapboxTileLimitExceededException(); + } + queryTiles.reset(); + + mapbox::sqlite::Transaction transaction(*db); + db->exec(mergeSideloadedDatabaseSQL); + transaction.commit(); + + // clang-format off + mapbox::sqlite::Query queryRegions{ getStatement( + "SELECT DISTINCT r.id, r.definition, r.description " + "FROM side.regions sr " + "JOIN regions r ON sr.definition = r.definition AND sr.description IS r.description") }; + // clang-format on + + OfflineRegions result; + while (queryRegions.run()) { + // Construct, then move because this constructor is private. + OfflineRegion region(queryRegions.get<int64_t>(0), + decodeOfflineRegionDefinition(queryRegions.get<std::string>(1)), + queryRegions.get<std::vector<uint8_t>>(2)); + result.emplace_back(std::move(region)); + } + db->exec("DETACH DATABASE side"); + // Explicit move to avoid triggering the copy constructor. + return { std::move(result) }; + } catch (const std::runtime_error& ex) { + db->exec("DETACH DATABASE side"); + Log::Error(Event::Database, "%s", ex.what()); + + return unexpected<std::exception_ptr>(std::current_exception()); + } + return {}; +} + +expected<OfflineRegionMetadata, std::exception_ptr> +OfflineDatabase::updateMetadata(const int64_t regionID, const OfflineRegionMetadata& metadata) try { + // clang-format off + mapbox::sqlite::Query query{ getStatement( + "UPDATE regions SET description = ?1 " + "WHERE id = ?2") }; + // clang-format on + query.bindBlob(1, metadata); + query.bind(2, regionID); + query.run(); + + return metadata; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "update region metadata"); + return unexpected<std::exception_ptr>(std::current_exception()); +} + +std::exception_ptr OfflineDatabase::deleteRegion(OfflineRegion&& region) try { + { + mapbox::sqlite::Query query{ getStatement("DELETE FROM regions WHERE id = ?") }; + query.bind(1, region.getID()); + query.run(); + } + + evict(0); + assert(db); + db->exec("PRAGMA incremental_vacuum"); + + // Ensure that the cached offlineTileCount value is recalculated. + offlineMapboxTileCount = {}; + return nullptr; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "delete region"); + return std::current_exception(); +} + +optional<std::pair<Response, uint64_t>> OfflineDatabase::getRegionResource(int64_t regionID, const Resource& resource) try { + auto response = getInternal(resource); + + if (response) { + markUsed(regionID, resource); + } + + return response; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "read region resource"); + return nullopt; +} + +optional<int64_t> OfflineDatabase::hasRegionResource(int64_t regionID, const Resource& resource) try { + auto response = hasInternal(resource); + + if (response) { + markUsed(regionID, resource); + } + + return response; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "query region resource"); + return nullopt; +} + +uint64_t OfflineDatabase::putRegionResource(int64_t regionID, + const Resource& resource, + const Response& response) try { + if (!db) { + initialize(); + } + mapbox::sqlite::Transaction transaction(*db); + auto size = putRegionResourceInternal(regionID, resource, response); + transaction.commit(); + return size; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "write region resource"); + return 0; +} + +void OfflineDatabase::putRegionResources(int64_t regionID, + const std::list<std::tuple<Resource, Response>>& resources, + OfflineRegionStatus& status) try { + if (!db) { + initialize(); + } + mapbox::sqlite::Transaction transaction(*db); + + // Accumulate all statistics locally first before adding them to the OfflineRegionStatus object + // to ensure correctness when the transaction fails. + uint64_t completedResourceCount = 0; + uint64_t completedResourceSize = 0; + uint64_t completedTileCount = 0; + uint64_t completedTileSize = 0; + + for (const auto& elem : resources) { + const auto& resource = std::get<0>(elem); + const auto& response = std::get<1>(elem); + + try { + uint64_t resourceSize = putRegionResourceInternal(regionID, resource, response); + completedResourceCount++; + completedResourceSize += resourceSize; + if (resource.kind == Resource::Kind::Tile) { + completedTileCount += 1; + completedTileSize += resourceSize; + } + } catch (const MapboxTileLimitExceededException&) { + // Commit the rest of the batch and rethrow + transaction.commit(); + throw; + } + } + + // Commit the completed batch + transaction.commit(); + + status.completedResourceCount += completedResourceCount; + status.completedResourceSize += completedResourceSize; + status.completedTileCount += completedTileCount; + status.completedTileSize += completedTileSize; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "write region resources"); +} + +uint64_t OfflineDatabase::putRegionResourceInternal(int64_t regionID, const Resource& resource, const Response& response) { + if (exceedsOfflineMapboxTileCountLimit(resource)) { + throw MapboxTileLimitExceededException(); + } + + uint64_t size = putInternal(resource, response, false).second; + bool previouslyUnused = markUsed(regionID, resource); + + if (offlineMapboxTileCount + && resource.kind == Resource::Kind::Tile + && util::mapbox::isMapboxURL(resource.url) + && previouslyUnused) { + *offlineMapboxTileCount += 1; + } + + return size; +} + +bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) { + if (resource.kind == Resource::Kind::Tile) { + // clang-format off + mapbox::sqlite::Query insertQuery{ getStatement( + "INSERT OR IGNORE INTO region_tiles (region_id, tile_id) " + "SELECT ?1, tiles.id " + "FROM tiles " + "WHERE url_template = ?2 " + " AND pixel_ratio = ?3 " + " AND x = ?4 " + " AND y = ?5 " + " AND z = ?6 ") }; + // clang-format on + + const Resource::TileData& tile = *resource.tileData; + insertQuery.bind(1, regionID); + insertQuery.bind(2, tile.urlTemplate); + insertQuery.bind(3, tile.pixelRatio); + insertQuery.bind(4, tile.x); + insertQuery.bind(5, tile.y); + insertQuery.bind(6, tile.z); + insertQuery.run(); + + if (insertQuery.changes() == 0) { + return false; + } + + // clang-format off + mapbox::sqlite::Query selectQuery{ getStatement( + "SELECT region_id " + "FROM region_tiles, tiles " + "WHERE region_id != ?1 " + " AND url_template = ?2 " + " AND pixel_ratio = ?3 " + " AND x = ?4 " + " AND y = ?5 " + " AND z = ?6 " + "LIMIT 1 ") }; + // clang-format on + + selectQuery.bind(1, regionID); + selectQuery.bind(2, tile.urlTemplate); + selectQuery.bind(3, tile.pixelRatio); + selectQuery.bind(4, tile.x); + selectQuery.bind(5, tile.y); + selectQuery.bind(6, tile.z); + return !selectQuery.run(); + } else { + // clang-format off + mapbox::sqlite::Query insertQuery{ getStatement( + "INSERT OR IGNORE INTO region_resources (region_id, resource_id) " + "SELECT ?1, resources.id " + "FROM resources " + "WHERE resources.url = ?2 ") }; + // clang-format on + + insertQuery.bind(1, regionID); + insertQuery.bind(2, resource.url); + insertQuery.run(); + + if (insertQuery.changes() == 0) { + return false; + } + + // clang-format off + mapbox::sqlite::Query selectQuery{ getStatement( + "SELECT region_id " + "FROM region_resources, resources " + "WHERE region_id != ?1 " + " AND resources.url = ?2 " + "LIMIT 1 ") }; + // clang-format on + + selectQuery.bind(1, regionID); + selectQuery.bind(2, resource.url); + return !selectQuery.run(); + } +} + +expected<OfflineRegionDefinition, std::exception_ptr> OfflineDatabase::getRegionDefinition(int64_t regionID) try { + mapbox::sqlite::Query query{ getStatement("SELECT definition FROM regions WHERE id = ?1") }; + query.bind(1, regionID); + query.run(); + + return decodeOfflineRegionDefinition(query.get<std::string>(0)); +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "load region"); + return unexpected<std::exception_ptr>(std::current_exception()); +} + +expected<OfflineRegionStatus, std::exception_ptr> OfflineDatabase::getRegionCompletedStatus(int64_t regionID) try { + OfflineRegionStatus result; + + std::tie(result.completedResourceCount, result.completedResourceSize) + = getCompletedResourceCountAndSize(regionID); + std::tie(result.completedTileCount, result.completedTileSize) + = getCompletedTileCountAndSize(regionID); + + result.completedResourceCount += result.completedTileCount; + result.completedResourceSize += result.completedTileSize; + + return result; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "get region status"); + return unexpected<std::exception_ptr>(std::current_exception()); +} + +std::pair<int64_t, int64_t> OfflineDatabase::getCompletedResourceCountAndSize(int64_t regionID) { + // clang-format off + mapbox::sqlite::Query query{ getStatement( + "SELECT COUNT(*), SUM(LENGTH(data)) " + "FROM region_resources, resources " + "WHERE region_id = ?1 " + "AND resource_id = resources.id ") }; + // clang-format on + query.bind(1, regionID); + query.run(); + return { query.get<int64_t>(0), query.get<int64_t>(1) }; +} + +std::pair<int64_t, int64_t> OfflineDatabase::getCompletedTileCountAndSize(int64_t regionID) { + // clang-format off + mapbox::sqlite::Query query{ getStatement( + "SELECT COUNT(*), SUM(LENGTH(data)) " + "FROM region_tiles, tiles " + "WHERE region_id = ?1 " + "AND tile_id = tiles.id ") }; + // clang-format on + query.bind(1, regionID); + query.run(); + return { query.get<int64_t>(0), query.get<int64_t>(1) }; +} + +template <class T> +T OfflineDatabase::getPragma(const char* sql) { + mapbox::sqlite::Query query{ getStatement(sql) }; + query.run(); + return query.get<T>(0); +} + +// Remove least-recently used resources and tiles until the used database size, +// as calculated by multiplying the number of in-use pages by the page size, is +// less than the maximum cache size. Returns false if this condition cannot be +// satisfied. +// +// SQLite database never shrinks in size unless we call VACCUM. We here +// are monitoring the soft limit (i.e. number of free pages in the file) +// and as it approaches to the hard limit (i.e. the actual file size) we +// delete an arbitrary number of old cache entries. The free pages approach saves +// us from calling VACCUM or keeping a running total, which can be costly. +bool OfflineDatabase::evict(uint64_t neededFreeSize) { + uint64_t pageSize = getPragma<int64_t>("PRAGMA page_size"); + uint64_t pageCount = getPragma<int64_t>("PRAGMA page_count"); + + auto usedSize = [&] { + return pageSize * (pageCount - getPragma<int64_t>("PRAGMA freelist_count")); + }; + + // The addition of pageSize is a fudge factor to account for non `data` column + // size, and because pages can get fragmented on the database. + while (usedSize() + neededFreeSize + pageSize > maximumCacheSize) { + // clang-format off + mapbox::sqlite::Query accessedQuery{ getStatement( + "SELECT max(accessed) " + "FROM ( " + " SELECT accessed " + " FROM resources " + " LEFT JOIN region_resources " + " ON resource_id = resources.id " + " WHERE resource_id IS NULL " + " UNION ALL " + " SELECT accessed " + " FROM tiles " + " LEFT JOIN region_tiles " + " ON tile_id = tiles.id " + " WHERE tile_id IS NULL " + " ORDER BY accessed ASC LIMIT ?1 " + ") " + ) }; + accessedQuery.bind(1, 50); + // clang-format on + if (!accessedQuery.run()) { + return false; + } + Timestamp accessed = accessedQuery.get<Timestamp>(0); + + // clang-format off + mapbox::sqlite::Query resourceQuery{ getStatement( + "DELETE FROM resources " + "WHERE id IN ( " + " SELECT id FROM resources " + " LEFT JOIN region_resources " + " ON resource_id = resources.id " + " WHERE resource_id IS NULL " + " AND accessed <= ?1 " + ") ") }; + // clang-format on + resourceQuery.bind(1, accessed); + resourceQuery.run(); + const uint64_t resourceChanges = resourceQuery.changes(); + + // clang-format off + mapbox::sqlite::Query tileQuery{ getStatement( + "DELETE FROM tiles " + "WHERE id IN ( " + " SELECT id FROM tiles " + " LEFT JOIN region_tiles " + " ON tile_id = tiles.id " + " WHERE tile_id IS NULL " + " AND accessed <= ?1 " + ") ") }; + // clang-format on + tileQuery.bind(1, accessed); + tileQuery.run(); + const uint64_t tileChanges = tileQuery.changes(); + + // The cached value of offlineTileCount does not need to be updated + // here because only non-offline tiles can be removed by eviction. + + if (resourceChanges == 0 && tileChanges == 0) { + return false; + } + } + + return true; +} + +void OfflineDatabase::setOfflineMapboxTileCountLimit(uint64_t limit) { + offlineMapboxTileCountLimit = limit; +} + +uint64_t OfflineDatabase::getOfflineMapboxTileCountLimit() { + return offlineMapboxTileCountLimit; +} + +bool OfflineDatabase::offlineMapboxTileCountLimitExceeded() { + return getOfflineMapboxTileCount() >= offlineMapboxTileCountLimit; +} + +uint64_t OfflineDatabase::getOfflineMapboxTileCount() try { + // Calculating this on every call would be much simpler than caching and + // manually updating the value, but it would make offline downloads an O(n²) + // operation, because the database query below involves an index scan of + // region_tiles. + + if (offlineMapboxTileCount) { + return *offlineMapboxTileCount; + } + + // clang-format off + mapbox::sqlite::Query query{ getStatement( + "SELECT COUNT(DISTINCT id) " + "FROM region_tiles, tiles " + "WHERE tile_id = tiles.id " + "AND url_template LIKE 'mapbox://%' ") }; + // clang-format on + + query.run(); + + offlineMapboxTileCount = query.get<int64_t>(0); + return *offlineMapboxTileCount; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "get offline Mapbox tile count"); + return std::numeric_limits<uint64_t>::max(); +} + +bool OfflineDatabase::exceedsOfflineMapboxTileCountLimit(const Resource& resource) { + return resource.kind == Resource::Kind::Tile + && util::mapbox::isMapboxURL(resource.url) + && offlineMapboxTileCountLimitExceeded(); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/offline_download.cpp b/platform/default/src/mbgl/storage/offline_download.cpp new file mode 100644 index 0000000000..c97797a5a2 --- /dev/null +++ b/platform/default/src/mbgl/storage/offline_download.cpp @@ -0,0 +1,453 @@ +#include <mbgl/storage/online_file_source.hpp> +#include <mbgl/storage/offline_database.hpp> +#include <mbgl/storage/offline_download.hpp> +#include <mbgl/storage/resource.hpp> +#include <mbgl/storage/response.hpp> +#include <mbgl/storage/http_file_source.hpp> +#include <mbgl/style/parser.hpp> +#include <mbgl/style/sources/vector_source.hpp> +#include <mbgl/style/sources/raster_source.hpp> +#include <mbgl/style/sources/raster_dem_source.hpp> +#include <mbgl/style/sources/geojson_source.hpp> +#include <mbgl/style/sources/image_source.hpp> +#include <mbgl/style/conversion/json.hpp> +#include <mbgl/style/conversion/tileset.hpp> +#include <mbgl/text/glyph.hpp> +#include <mbgl/util/mapbox.hpp> +#include <mbgl/util/run_loop.hpp> +#include <mbgl/util/tile_cover.hpp> +#include <mbgl/util/tileset.hpp> + +#include <set> + +namespace mbgl { + +using namespace style; + +// Generic functions + +template <class RegionDefinition> +Range<uint8_t> coveringZoomRange(const RegionDefinition& definition, + style::SourceType type, uint16_t tileSize, const Range<uint8_t>& zoomRange) { + double minZ = std::max<double>(util::coveringZoomLevel(definition.minZoom, type, tileSize), zoomRange.min); + double maxZ = std::min<double>(util::coveringZoomLevel(definition.maxZoom, type, tileSize), zoomRange.max); + + assert(minZ >= 0); + assert(maxZ >= 0); + assert(minZ < std::numeric_limits<uint8_t>::max()); + assert(maxZ < std::numeric_limits<uint8_t>::max()); + return { static_cast<uint8_t>(minZ), static_cast<uint8_t>(maxZ) }; +} + +template <class Geometry, class Fn> +void tileCover(const Geometry& geometry, uint8_t z, Fn&& fn) { + util::TileCover cover(geometry, z); + while (cover.hasNext()) { + fn(cover.next()->canonical); + } +} + + +template <class Fn> +void tileCover(const OfflineRegionDefinition& definition, style::SourceType type, + uint16_t tileSize, const Range<uint8_t>& zoomRange, Fn&& fn) { + const Range<uint8_t> clampedZoomRange = + definition.match([&](auto& reg) { return coveringZoomRange(reg, type, tileSize, zoomRange); }); + + for (uint8_t z = clampedZoomRange.min; z <= clampedZoomRange.max; z++) { + definition.match( + [&](const OfflineTilePyramidRegionDefinition& reg){ tileCover(reg.bounds, z, fn); }, + [&](const OfflineGeometryRegionDefinition& reg){ tileCover(reg.geometry, z, fn); } + ); + } +} + +uint64_t tileCount(const OfflineRegionDefinition& definition, style::SourceType type, + uint16_t tileSize, const Range<uint8_t>& zoomRange) { + + const Range<uint8_t> clampedZoomRange = + definition.match([&](auto& reg) { return coveringZoomRange(reg, type, tileSize, zoomRange); }); + + unsigned long result = 0;; + for (uint8_t z = clampedZoomRange.min; z <= clampedZoomRange.max; z++) { + result += definition.match( + [&](const OfflineTilePyramidRegionDefinition& reg){ return util::tileCount(reg.bounds, z); }, + [&](const OfflineGeometryRegionDefinition& reg){ return util::tileCount(reg.geometry, z); } + ); + } + + return result; +} + +// OfflineDownload + +OfflineDownload::OfflineDownload(int64_t id_, + OfflineRegionDefinition&& definition_, + OfflineDatabase& offlineDatabase_, + OnlineFileSource& onlineFileSource_) + : id(id_), + definition(definition_), + offlineDatabase(offlineDatabase_), + onlineFileSource(onlineFileSource_) { + setObserver(nullptr); +} + +OfflineDownload::~OfflineDownload() = default; + +void OfflineDownload::setObserver(std::unique_ptr<OfflineRegionObserver> observer_) { + observer = observer_ ? std::move(observer_) : std::make_unique<OfflineRegionObserver>(); +} + +void OfflineDownload::setState(OfflineRegionDownloadState state) { + if (status.downloadState == state) { + return; + } + + status.downloadState = state; + + if (status.downloadState == OfflineRegionDownloadState::Active) { + activateDownload(); + } else { + deactivateDownload(); + } + + observer->statusChanged(status); +} + +OfflineRegionStatus OfflineDownload::getStatus() const { + if (status.downloadState == OfflineRegionDownloadState::Active) { + return status; + } + + auto result = offlineDatabase.getRegionCompletedStatus(id); + if (!result) { + // We can't find this offline region because the database is unavailable, or the download + // does not exist. + return {}; + } + + result->requiredResourceCount++; + optional<Response> styleResponse = + offlineDatabase.get(Resource::style(definition.match([](auto& reg){ return reg.styleURL; }))); + if (!styleResponse) { + return *result; + } + + style::Parser parser; + parser.parse(*styleResponse->data); + + result->requiredResourceCountIsPrecise = true; + + for (const auto& source : parser.sources) { + SourceType type = source->getType(); + + auto handleTiledSource = [&] (const variant<std::string, Tileset>& urlOrTileset, const uint16_t tileSize) { + if (urlOrTileset.is<Tileset>()) { + result->requiredResourceCount += + tileCount(definition, type, tileSize, urlOrTileset.get<Tileset>().zoomRange); + } else { + result->requiredResourceCount += 1; + const auto& url = urlOrTileset.get<std::string>(); + optional<Response> sourceResponse = offlineDatabase.get(Resource::source(url)); + if (sourceResponse) { + style::conversion::Error error; + optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(*sourceResponse->data, error); + if (tileset) { + result->requiredResourceCount += + tileCount(definition, type, tileSize, (*tileset).zoomRange); + } + } else { + result->requiredResourceCountIsPrecise = false; + } + } + }; + + switch (type) { + case SourceType::Vector: { + const auto& vectorSource = *source->as<VectorSource>(); + handleTiledSource(vectorSource.getURLOrTileset(), util::tileSize); + break; + } + + case SourceType::Raster: { + const auto& rasterSource = *source->as<RasterSource>(); + handleTiledSource(rasterSource.getURLOrTileset(), rasterSource.getTileSize()); + break; + } + + case SourceType::RasterDEM: { + const auto& rasterDEMSource = *source->as<RasterDEMSource>(); + handleTiledSource(rasterDEMSource.getURLOrTileset(), rasterDEMSource.getTileSize()); + break; + } + + case SourceType::GeoJSON: { + const auto& geojsonSource = *source->as<GeoJSONSource>(); + if (geojsonSource.getURL()) { + result->requiredResourceCount += 1; + } + break; + } + + case SourceType::Image: { + const auto& imageSource = *source->as<ImageSource>(); + if (imageSource.getURL()) { + result->requiredResourceCount += 1; + } + break; + } + + case SourceType::Video: + case SourceType::Annotations: + case SourceType::CustomVector: + break; + } + } + + if (!parser.glyphURL.empty()) { + result->requiredResourceCount += parser.fontStacks().size() * GLYPH_RANGES_PER_FONT_STACK; + } + + if (!parser.spriteURL.empty()) { + result->requiredResourceCount += 4; + } + + return *result; +} + +void OfflineDownload::activateDownload() { + status = OfflineRegionStatus(); + status.downloadState = OfflineRegionDownloadState::Active; + status.requiredResourceCount++; + ensureResource(Resource::style(definition.match([](auto& reg){ return reg.styleURL; }), Resource::Priority::Low), + [&](Response styleResponse) { + status.requiredResourceCountIsPrecise = true; + + style::Parser parser; + parser.parse(*styleResponse.data); + + for (const auto& source : parser.sources) { + SourceType type = source->getType(); + + auto handleTiledSource = [&] (const variant<std::string, Tileset>& urlOrTileset, const uint16_t tileSize) { + if (urlOrTileset.is<Tileset>()) { + queueTiles(type, tileSize, urlOrTileset.get<Tileset>()); + } else { + const auto& url = urlOrTileset.get<std::string>(); + status.requiredResourceCountIsPrecise = false; + status.requiredResourceCount++; + requiredSourceURLs.insert(url); + + ensureResource(Resource::source(url, Resource::Priority::Low), [=](Response sourceResponse) { + style::conversion::Error error; + optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(*sourceResponse.data, error); + if (tileset) { + util::mapbox::canonicalizeTileset(*tileset, url, type, tileSize); + queueTiles(type, tileSize, *tileset); + + requiredSourceURLs.erase(url); + if (requiredSourceURLs.empty()) { + status.requiredResourceCountIsPrecise = true; + } + } + }); + } + }; + + switch (type) { + case SourceType::Vector: { + const auto& vectorSource = *source->as<VectorSource>(); + handleTiledSource(vectorSource.getURLOrTileset(), util::tileSize); + break; + } + + case SourceType::Raster: { + const auto& rasterSource = *source->as<RasterSource>(); + handleTiledSource(rasterSource.getURLOrTileset(), rasterSource.getTileSize()); + break; + } + + case SourceType::RasterDEM: { + const auto& rasterDEMSource = *source->as<RasterDEMSource>(); + handleTiledSource(rasterDEMSource.getURLOrTileset(), rasterDEMSource.getTileSize()); + break; + } + + case SourceType::GeoJSON: { + const auto& geojsonSource = *source->as<GeoJSONSource>(); + if (geojsonSource.getURL()) { + queueResource(Resource::source(*geojsonSource.getURL())); + } + break; + } + + case SourceType::Image: { + const auto& imageSource = *source->as<ImageSource>(); + auto imageUrl = imageSource.getURL(); + if (imageUrl && !imageUrl->empty()) { + queueResource(Resource::image(*imageUrl)); + } + break; + } + + case SourceType::Video: + case SourceType::Annotations: + case SourceType::CustomVector: + break; + } + } + + if (!parser.glyphURL.empty()) { + for (const auto& fontStack : parser.fontStacks()) { + for (char16_t i = 0; i < GLYPH_RANGES_PER_FONT_STACK; i++) { + queueResource(Resource::glyphs(parser.glyphURL, fontStack, getGlyphRange(i * GLYPHS_PER_GLYPH_RANGE))); + } + } + } + + if (!parser.spriteURL.empty()) { + // Always request 1x and @2x sprite images for portability. + queueResource(Resource::spriteImage(parser.spriteURL, 1)); + queueResource(Resource::spriteImage(parser.spriteURL, 2)); + queueResource(Resource::spriteJSON(parser.spriteURL, 1)); + queueResource(Resource::spriteJSON(parser.spriteURL, 2)); + } + + continueDownload(); + }); +} + +/* + Fill up our own request queue by requesting the next few resources. This is called + when activating the download, or when a request completes successfully. + + Note "successfully"; it's not called when a requests receives an error. A request + that errors will be retried after some delay. So in that sense it's still "active" + and consuming resources, notably the request object, its timer, and network resources + when the timer fires. + + We could try to squeeze in subsequent requests while we wait for the errored request + to retry. But that risks overloading the upstream request queue -- defeating our own + metering -- if there are a lot of errored requests that all come up for retry at the + same time. And many times, the cause of a request error will apply to many requests + of the same type. For instance if a server is unreachable, all the requests to that + host are going to error. In that case, continuing to try subsequent resources after + the first few errors is fruitless anyway. +*/ +void OfflineDownload::continueDownload() { + if (resourcesRemaining.empty() && status.complete()) { + setState(OfflineRegionDownloadState::Inactive); + return; + } + + while (!resourcesRemaining.empty() && requests.size() < onlineFileSource.getMaximumConcurrentRequests()) { + ensureResource(resourcesRemaining.front()); + resourcesRemaining.pop_front(); + } +} + +void OfflineDownload::deactivateDownload() { + requiredSourceURLs.clear(); + resourcesRemaining.clear(); + requests.clear(); +} + +void OfflineDownload::queueResource(Resource resource) { + resource.setPriority(Resource::Priority::Low); + status.requiredResourceCount++; + resourcesRemaining.push_front(std::move(resource)); +} + +void OfflineDownload::queueTiles(SourceType type, uint16_t tileSize, const Tileset& tileset) { + tileCover(definition, type, tileSize, tileset.zoomRange, [&](const auto& tile) { + status.requiredResourceCount++; + resourcesRemaining.push_back(Resource::tile( + tileset.tiles[0], definition.match([](auto& def) { return def.pixelRatio; }), tile.x, + tile.y, tile.z, tileset.scheme, Resource::Priority::Low)); + }); +} + +void OfflineDownload::ensureResource(const Resource& resource, + std::function<void(Response)> callback) { + assert(resource.priority == Resource::Priority::Low); + + auto workRequestsIt = requests.insert(requests.begin(), nullptr); + *workRequestsIt = util::RunLoop::Get()->invokeCancellable([=]() { + requests.erase(workRequestsIt); + + auto getResourceSizeInDatabase = [&] () -> optional<int64_t> { + if (!callback) { + return offlineDatabase.hasRegionResource(id, resource); + } + optional<std::pair<Response, uint64_t>> response = offlineDatabase.getRegionResource(id, resource); + if (!response) { + return {}; + } + callback(response->first); + return response->second; + }; + + optional<int64_t> offlineResponse = getResourceSizeInDatabase(); + if (offlineResponse) { + status.completedResourceCount++; + status.completedResourceSize += *offlineResponse; + if (resource.kind == Resource::Kind::Tile) { + status.completedTileCount += 1; + status.completedTileSize += *offlineResponse; + } + + observer->statusChanged(status); + continueDownload(); + return; + } + + if (offlineDatabase.exceedsOfflineMapboxTileCountLimit(resource)) { + onMapboxTileCountLimitExceeded(); + return; + } + + auto fileRequestsIt = requests.insert(requests.begin(), nullptr); + *fileRequestsIt = onlineFileSource.request(resource, [=](Response onlineResponse) { + if (onlineResponse.error) { + observer->responseError(*onlineResponse.error); + return; + } + + requests.erase(fileRequestsIt); + + if (callback) { + callback(onlineResponse); + } + + // Queue up for batched insertion + buffer.emplace_back(resource, onlineResponse); + + // Flush buffer periodically + if (buffer.size() == 64 || resourcesRemaining.size() == 0) { + try { + offlineDatabase.putRegionResources(id, buffer, status); + } catch (const MapboxTileLimitExceededException&) { + onMapboxTileCountLimitExceeded(); + return; + } + + buffer.clear(); + observer->statusChanged(status); + } + + if (offlineDatabase.exceedsOfflineMapboxTileCountLimit(resource)) { + onMapboxTileCountLimitExceeded(); + return; + } + + continueDownload(); + }); + }); +} + +void OfflineDownload::onMapboxTileCountLimitExceeded() { + observer->mapboxTileCountLimitExceeded(offlineDatabase.getOfflineMapboxTileCountLimit()); + setState(OfflineRegionDownloadState::Inactive); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/online_file_source.cpp b/platform/default/src/mbgl/storage/online_file_source.cpp new file mode 100644 index 0000000000..fce1c3e2b6 --- /dev/null +++ b/platform/default/src/mbgl/storage/online_file_source.cpp @@ -0,0 +1,488 @@ +#include <mbgl/storage/online_file_source.hpp> +#include <mbgl/storage/http_file_source.hpp> +#include <mbgl/storage/network_status.hpp> + +#include <mbgl/storage/resource_transform.hpp> +#include <mbgl/storage/response.hpp> +#include <mbgl/util/logging.hpp> + +#include <mbgl/actor/mailbox.hpp> +#include <mbgl/util/constants.hpp> +#include <mbgl/util/mapbox.hpp> +#include <mbgl/util/exception.hpp> +#include <mbgl/util/chrono.hpp> +#include <mbgl/util/async_task.hpp> +#include <mbgl/util/noncopyable.hpp> +#include <mbgl/util/run_loop.hpp> +#include <mbgl/util/timer.hpp> +#include <mbgl/util/http_timeout.hpp> + +#include <algorithm> +#include <cassert> +#include <list> +#include <unordered_set> +#include <unordered_map> + +namespace mbgl { + +static uint32_t DEFAULT_MAXIMUM_CONCURRENT_REQUESTS = 20; + +class OnlineFileRequest : public AsyncRequest { +public: + using Callback = std::function<void (Response)>; + + OnlineFileRequest(Resource, Callback, OnlineFileSource::Impl&); + ~OnlineFileRequest() override; + + void networkIsReachableAgain(); + void schedule(); + void schedule(optional<Timestamp> expires); + void completed(Response); + + void setTransformedURL(const std::string&& url); + ActorRef<OnlineFileRequest> actor(); + + OnlineFileSource::Impl& impl; + Resource resource; + std::unique_ptr<AsyncRequest> request; + util::Timer timer; + Callback callback; + + std::shared_ptr<Mailbox> mailbox; + + // Counts the number of times a response was already expired when received. We're using + // this to add a delay when making a new request so we don't keep retrying immediately + // in case of a server serving expired tiles. + uint32_t expiredRequests = 0; + + // Counts the number of subsequent failed requests. We're using this value for exponential + // backoff when retrying requests. + uint32_t failedRequests = 0; + Response::Error::Reason failedRequestReason = Response::Error::Reason::Success; + optional<Timestamp> retryAfter; +}; + +class OnlineFileSource::Impl { +public: + Impl() { + NetworkStatus::Subscribe(&reachability); + setMaximumConcurrentRequests(DEFAULT_MAXIMUM_CONCURRENT_REQUESTS); + } + + ~Impl() { + NetworkStatus::Unsubscribe(&reachability); + } + + void add(OnlineFileRequest* request) { + allRequests.insert(request); + if (resourceTransform) { + // Request the ResourceTransform actor a new url and replace the resource url with the + // transformed one before proceeding to schedule the request. + resourceTransform->invoke(&ResourceTransform::transform, request->resource.kind, + std::move(request->resource.url), [ref = request->actor()](const std::string&& url) mutable { + ref.invoke(&OnlineFileRequest::setTransformedURL, std::move(url)); + }); + } else { + request->schedule(); + } + } + + void remove(OnlineFileRequest* request) { + allRequests.erase(request); + if (activeRequests.erase(request)) { + activatePendingRequest(); + } else { + pendingRequests.remove(request); + } + } + + void activateOrQueueRequest(OnlineFileRequest* request) { + assert(allRequests.find(request) != allRequests.end()); + assert(activeRequests.find(request) == activeRequests.end()); + assert(!request->request); + + if (activeRequests.size() >= getMaximumConcurrentRequests()) { + queueRequest(request); + } else { + activateRequest(request); + } + } + + void queueRequest(OnlineFileRequest* request) { + pendingRequests.insert(request); + } + + void activateRequest(OnlineFileRequest* request) { + auto callback = [=](Response response) { + activeRequests.erase(request); + request->request.reset(); + request->completed(response); + activatePendingRequest(); + }; + + activeRequests.insert(request); + + if (online) { + request->request = httpFileSource.request(request->resource, callback); + } else { + Response response; + response.error = std::make_unique<Response::Error>(Response::Error::Reason::Connection, + "Online connectivity is disabled."); + callback(response); + } + + } + + void activatePendingRequest() { + + auto request = pendingRequests.pop(); + + if (request) { + activateRequest(*request); + } + } + + bool isPending(OnlineFileRequest* request) { + return pendingRequests.contains(request); + } + + bool isActive(OnlineFileRequest* request) { + return activeRequests.find(request) != activeRequests.end(); + } + + void setResourceTransform(optional<ActorRef<ResourceTransform>>&& transform) { + resourceTransform = std::move(transform); + } + + void setOnlineStatus(const bool status) { + online = status; + networkIsReachableAgain(); + } + + uint32_t getMaximumConcurrentRequests() const { + return maximumConcurrentRequests; + } + + void setMaximumConcurrentRequests(uint32_t maximumConcurrentRequests_) { + maximumConcurrentRequests = maximumConcurrentRequests_; + } + +private: + + void networkIsReachableAgain() { + for (auto& request : allRequests) { + request->networkIsReachableAgain(); + } + } + + // Using Pending Requests as an priority queue which processes + // file requests in a FIFO manner but prefers regular requests + // over offline requests with a low priority such that low priority + // requests do not throttle regular requests. + // + // The order of a queue is therefore: + // + // hi0 -- hi1 -- hi2 -- hi3 -- lo0 -- lo1 --lo2 + // ^ + // firstLowPriorityRequest + + struct PendingRequests { + PendingRequests() : queue(), firstLowPriorityRequest(queue.begin()) {} + + std::list<OnlineFileRequest*> queue; + std::list<OnlineFileRequest*>::iterator firstLowPriorityRequest; + + void remove(const OnlineFileRequest* request) { + auto it = std::find(queue.begin(), queue.end(), request); + if (it != queue.end()) { + if (it == firstLowPriorityRequest) { + firstLowPriorityRequest++; + } + queue.erase(it); + } + } + + void insert(OnlineFileRequest* request) { + if (request->resource.priority == Resource::Priority::Regular) { + firstLowPriorityRequest = queue.insert(firstLowPriorityRequest, request); + firstLowPriorityRequest++; + } + else { + if (firstLowPriorityRequest == queue.end()) { + firstLowPriorityRequest = queue.insert(queue.end(), request); + } + else { + queue.insert(queue.end(), request); + } + } + } + + + optional<OnlineFileRequest*> pop() { + if (queue.empty()) { + return optional<OnlineFileRequest*>(); + } + + if (queue.begin() == firstLowPriorityRequest) { + firstLowPriorityRequest++; + } + + OnlineFileRequest* next = queue.front(); + queue.pop_front(); + return optional<OnlineFileRequest*>(next); + } + + bool contains(OnlineFileRequest* request) const { + return (std::find(queue.begin(), queue.end(), request) != queue.end()); + } + + }; + + optional<ActorRef<ResourceTransform>> resourceTransform; + + /** + * The lifetime of a request is: + * + * 1. Waiting for timeout (revalidation or retry) + * 2. Pending (waiting for room in the active set) + * 3. Active (open network connection) + * 4. Back to #1 + * + * Requests in any state are in `allRequests`. Requests in the pending state are in + * `pendingRequests`. Requests in the active state are in `activeRequests`. + */ + std::unordered_set<OnlineFileRequest*> allRequests; + + PendingRequests pendingRequests; + + std::unordered_set<OnlineFileRequest*> activeRequests; + + bool online = true; + uint32_t maximumConcurrentRequests; + HTTPFileSource httpFileSource; + util::AsyncTask reachability { std::bind(&Impl::networkIsReachableAgain, this) }; +}; + +OnlineFileSource::OnlineFileSource() + : impl(std::make_unique<Impl>()) { +} + +OnlineFileSource::~OnlineFileSource() = default; + +std::unique_ptr<AsyncRequest> OnlineFileSource::request(const Resource& resource, Callback callback) { + Resource res = resource; + + switch (resource.kind) { + case Resource::Kind::Unknown: + case Resource::Kind::Image: + break; + + case Resource::Kind::Style: + res.url = mbgl::util::mapbox::normalizeStyleURL(apiBaseURL, resource.url, accessToken); + break; + + case Resource::Kind::Source: + res.url = util::mapbox::normalizeSourceURL(apiBaseURL, resource.url, accessToken); + break; + + case Resource::Kind::Glyphs: + res.url = util::mapbox::normalizeGlyphsURL(apiBaseURL, resource.url, accessToken); + break; + + case Resource::Kind::SpriteImage: + case Resource::Kind::SpriteJSON: + res.url = util::mapbox::normalizeSpriteURL(apiBaseURL, resource.url, accessToken); + break; + + case Resource::Kind::Tile: + res.url = util::mapbox::normalizeTileURL(apiBaseURL, resource.url, accessToken); + break; + } + + return std::make_unique<OnlineFileRequest>(std::move(res), std::move(callback), *impl); +} + +void OnlineFileSource::setResourceTransform(optional<ActorRef<ResourceTransform>>&& transform) { + impl->setResourceTransform(std::move(transform)); +} + +OnlineFileRequest::OnlineFileRequest(Resource resource_, Callback callback_, OnlineFileSource::Impl& impl_) + : impl(impl_), + resource(std::move(resource_)), + callback(std::move(callback_)) { + impl.add(this); +} + +void OnlineFileRequest::schedule() { + // Force an immediate first request if we don't have an expiration time. + if (resource.priorExpires) { + schedule(resource.priorExpires); + } else { + schedule(util::now()); + } +} + +OnlineFileRequest::~OnlineFileRequest() { + impl.remove(this); +} + +Timestamp interpolateExpiration(const Timestamp& current, + optional<Timestamp> prior, + bool& expired) { + auto now = util::now(); + if (current > now) { + return current; + } + + if (!bool(prior)) { + expired = true; + return current; + } + + // Expiring date is going backwards, + // fallback to exponential backoff. + if (current < *prior) { + expired = true; + return current; + } + + auto delta = current - *prior; + + // Server is serving the same expired resource + // over and over, fallback to exponential backoff. + if (delta == Duration::zero()) { + expired = true; + return current; + } + + // Assume that either the client or server clock is wrong and + // try to interpolate a valid expiration date (from the client POV) + // observing a minimum timeout. + return now + std::max<Seconds>(delta, util::CLOCK_SKEW_RETRY_TIMEOUT); +} + +void OnlineFileRequest::schedule(optional<Timestamp> expires) { + if (impl.isPending(this) || impl.isActive(this)) { + // There's already a request in progress; don't start another one. + return; + } + + // If we're not being asked for a forced refresh, calculate a timeout that depends on how many + // consecutive errors we've encountered, and on the expiration time, if present. + Duration timeout = std::min( + http::errorRetryTimeout(failedRequestReason, failedRequests, retryAfter), + http::expirationTimeout(expires, expiredRequests)); + + if (timeout == Duration::max()) { + return; + } + + // Emulate a Connection error when the Offline mode is forced with + // a really long timeout. The request will get re-triggered when + // the NetworkStatus is set back to Online. + if (NetworkStatus::Get() == NetworkStatus::Status::Offline) { + failedRequestReason = Response::Error::Reason::Connection; + failedRequests = 1; + timeout = Duration::max(); + } + + timer.start(timeout, Duration::zero(), [&] { + impl.activateOrQueueRequest(this); + }); +} + +void OnlineFileRequest::completed(Response response) { + // If we didn't get various caching headers in the response, continue using the + // previous values. Otherwise, update the previous values to the new values. + + if (!response.modified) { + response.modified = resource.priorModified; + } else { + resource.priorModified = response.modified; + } + + if (response.notModified && resource.priorData) { + // When the priorData field is set, it indicates that we had to revalidate the request and + // that the requestor hasn't gotten data yet. If we get a 304 response, this means that we + // have send the cached data to give the requestor a chance to actually obtain the data. + response.data = std::move(resource.priorData); + response.notModified = false; + } + + bool isExpired = false; + + if (response.expires) { + auto prior = resource.priorExpires; + resource.priorExpires = response.expires; + response.expires = interpolateExpiration(*response.expires, prior, isExpired); + } + + if (isExpired) { + expiredRequests++; + } else { + expiredRequests = 0; + } + + if (!response.etag) { + response.etag = resource.priorEtag; + } else { + resource.priorEtag = response.etag; + } + + if (response.error) { + failedRequests++; + failedRequestReason = response.error->reason; + retryAfter = response.error->retryAfter; + } else { + failedRequests = 0; + failedRequestReason = Response::Error::Reason::Success; + } + + schedule(response.expires); + + // Calling the callback may result in `this` being deleted. It needs to be done last, + // and needs to make a local copy of the callback to ensure that it remains valid for + // the duration of the call. + auto callback_ = callback; + callback_(response); +} + +void OnlineFileRequest::networkIsReachableAgain() { + // We need all requests to fail at least once before we are going to start retrying + // them, and we only immediately restart request that failed due to connection issues. + if (failedRequestReason == Response::Error::Reason::Connection) { + schedule(util::now()); + } +} + +void OnlineFileRequest::setTransformedURL(const std::string&& url) { + resource.url = std::move(url); + schedule(); +} + +ActorRef<OnlineFileRequest> OnlineFileRequest::actor() { + if (!mailbox) { + // Lazy constructed because this can be costly and + // the ResourceTransform is not used by many apps. + mailbox = std::make_shared<Mailbox>(*Scheduler::GetCurrent()); + } + + return ActorRef<OnlineFileRequest>(*this, mailbox); +} + +void OnlineFileSource::setMaximumConcurrentRequests(uint32_t maximumConcurrentRequests_) { + impl->setMaximumConcurrentRequests(maximumConcurrentRequests_); +} + +uint32_t OnlineFileSource::getMaximumConcurrentRequests() const { + return impl->getMaximumConcurrentRequests(); +} + + +// For testing only: + +void OnlineFileSource::setOnlineStatus(const bool status) { + impl->setOnlineStatus(status); +} + +} // namespace mbgl diff --git a/platform/default/src/mbgl/storage/sqlite3.cpp b/platform/default/src/mbgl/storage/sqlite3.cpp new file mode 100644 index 0000000000..0017dc45db --- /dev/null +++ b/platform/default/src/mbgl/storage/sqlite3.cpp @@ -0,0 +1,494 @@ +#include <mbgl/storage/sqlite3.hpp> +#include <sqlite3.h> + +#include <algorithm> +#include <cassert> +#include <cstring> +#include <cstdio> +#include <chrono> +#include <experimental/optional> + +#include <mbgl/util/traits.hpp> +#include <mbgl/util/logging.hpp> + +namespace mapbox { +namespace sqlite { + +static_assert(mbgl::underlying_type(ResultCode::OK) == SQLITE_OK, "error"); +static_assert(mbgl::underlying_type(ResultCode::Error) == SQLITE_ERROR, "error"); +static_assert(mbgl::underlying_type(ResultCode::Internal) == SQLITE_INTERNAL, "error"); +static_assert(mbgl::underlying_type(ResultCode::Perm) == SQLITE_PERM, "error"); +static_assert(mbgl::underlying_type(ResultCode::Abort) == SQLITE_ABORT, "error"); +static_assert(mbgl::underlying_type(ResultCode::Busy) == SQLITE_BUSY, "error"); +static_assert(mbgl::underlying_type(ResultCode::Locked) == SQLITE_LOCKED, "error"); +static_assert(mbgl::underlying_type(ResultCode::NoMem) == SQLITE_NOMEM, "error"); +static_assert(mbgl::underlying_type(ResultCode::ReadOnly) == SQLITE_READONLY, "error"); +static_assert(mbgl::underlying_type(ResultCode::Interrupt) == SQLITE_INTERRUPT, "error"); +static_assert(mbgl::underlying_type(ResultCode::IOErr) == SQLITE_IOERR, "error"); +static_assert(mbgl::underlying_type(ResultCode::Corrupt) == SQLITE_CORRUPT, "error"); +static_assert(mbgl::underlying_type(ResultCode::NotFound) == SQLITE_NOTFOUND, "error"); +static_assert(mbgl::underlying_type(ResultCode::Full) == SQLITE_FULL, "error"); +static_assert(mbgl::underlying_type(ResultCode::CantOpen) == SQLITE_CANTOPEN, "error"); +static_assert(mbgl::underlying_type(ResultCode::Protocol) == SQLITE_PROTOCOL, "error"); +static_assert(mbgl::underlying_type(ResultCode::Schema) == SQLITE_SCHEMA, "error"); +static_assert(mbgl::underlying_type(ResultCode::TooBig) == SQLITE_TOOBIG, "error"); +static_assert(mbgl::underlying_type(ResultCode::Constraint) == SQLITE_CONSTRAINT, "error"); +static_assert(mbgl::underlying_type(ResultCode::Mismatch) == SQLITE_MISMATCH, "error"); +static_assert(mbgl::underlying_type(ResultCode::Misuse) == SQLITE_MISUSE, "error"); +static_assert(mbgl::underlying_type(ResultCode::NoLFS) == SQLITE_NOLFS, "error"); +static_assert(mbgl::underlying_type(ResultCode::Auth) == SQLITE_AUTH, "error"); +static_assert(mbgl::underlying_type(ResultCode::Range) == SQLITE_RANGE, "error"); +static_assert(mbgl::underlying_type(ResultCode::NotADB) == SQLITE_NOTADB, "error"); + +void setTempPath(const std::string& path) { + sqlite3_temp_directory = sqlite3_mprintf("%s", path.c_str()); +} + +class DatabaseImpl { +public: + DatabaseImpl(sqlite3* db_) + : db(db_) + { + const int error = sqlite3_extended_result_codes(db, true); + if (error != SQLITE_OK) { + mbgl::Log::Warning(mbgl::Event::Database, error, "Failed to enable extended result codes: %s", sqlite3_errmsg(db)); + } + } + + ~DatabaseImpl() + { + const int error = sqlite3_close(db); + if (error != SQLITE_OK) { + mbgl::Log::Error(mbgl::Event::Database, error, "Failed to close database: %s", sqlite3_errmsg(db)); + } + } + + void setBusyTimeout(std::chrono::milliseconds timeout); + void exec(const std::string& sql); + + sqlite3* db; +}; + +class StatementImpl { +public: + StatementImpl(sqlite3* db, const char* sql) + { + const int error = sqlite3_prepare_v2(db, sql, -1, &stmt, nullptr); + if (error != SQLITE_OK) { + stmt = nullptr; + throw Exception { error, sqlite3_errmsg(db) }; + } + } + + ~StatementImpl() + { + if (!stmt) return; + + sqlite3_finalize(stmt); + } + + void check(int err) { + if (err != SQLITE_OK) { + throw Exception { err, sqlite3_errmsg(sqlite3_db_handle(stmt)) }; + } + } + + sqlite3_stmt* stmt = nullptr; + int64_t lastInsertRowId = 0; + int64_t changes = 0; +}; + +template <typename T> +using optional = std::experimental::optional<T>; + + +#ifndef NDEBUG +void logSqlMessage(void *, const int err, const char *msg) { + mbgl::Log::Record(mbgl::EventSeverity::Debug, mbgl::Event::Database, err, "%s", msg); +} +#endif + +__attribute__((constructor)) +static void initalize() { + if (sqlite3_libversion_number() / 1000000 != SQLITE_VERSION_NUMBER / 1000000) { + char message[96]; + snprintf(message, 96, + "sqlite3 libversion mismatch: headers report %d, but library reports %d", + SQLITE_VERSION_NUMBER, sqlite3_libversion_number()); + throw std::runtime_error(message); + } + +#ifndef NDEBUG + // Enable SQLite logging before initializing the database. + sqlite3_config(SQLITE_CONFIG_LOG, &logSqlMessage, nullptr); +#endif +} + +mapbox::util::variant<Database, Exception> Database::tryOpen(const std::string &filename, int flags) { + sqlite3* db = nullptr; + const int error = sqlite3_open_v2(filename.c_str(), &db, flags | SQLITE_OPEN_URI, nullptr); + if (error != SQLITE_OK) { + const auto message = sqlite3_errmsg(db); + return Exception { error, message }; + } + return Database(std::make_unique<DatabaseImpl>(db)); +} + +Database Database::open(const std::string &filename, int flags) { + auto result = tryOpen(filename, flags); + if (result.is<Exception>()) { + throw result.get<Exception>(); + } else { + return std::move(result.get<Database>()); + } +} + +Database::Database(std::unique_ptr<DatabaseImpl> impl_) + : impl(std::move(impl_)) +{} + +Database::Database(Database &&other) + : impl(std::move(other.impl)) {} + +Database &Database::operator=(Database &&other) { + std::swap(impl, other.impl); + return *this; +} + +Database::~Database() = default; + +void Database::setBusyTimeout(std::chrono::milliseconds timeout) { + assert(impl); + impl->setBusyTimeout(timeout); +} + +void DatabaseImpl::setBusyTimeout(std::chrono::milliseconds timeout) { + const int err = sqlite3_busy_timeout(db, + int(std::min<std::chrono::milliseconds::rep>(timeout.count(), std::numeric_limits<int>::max()))); + if (err != SQLITE_OK) { + throw Exception { err, sqlite3_errmsg(db) }; + } +} + +void Database::exec(const std::string &sql) { + assert(impl); + impl->exec(sql); +} + +void DatabaseImpl::exec(const std::string& sql) { + char *msg = nullptr; + const int err = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &msg); + if (msg) { + const std::string message = msg; + sqlite3_free(msg); + throw Exception { err, message }; + } else if (err != SQLITE_OK) { + throw Exception { err, sqlite3_errmsg(db) }; + } +} + +Statement::Statement(Database& db, const char* sql) + : impl(std::make_unique<StatementImpl>(db.impl->db, sql)) { +} + +Statement::~Statement() { +#ifndef NDEBUG + // Crash if we're destructing this object while we know a Query object references this. + assert(!used); +#endif +} + +Query::Query(Statement& stmt_) : stmt(stmt_) { + assert(stmt.impl); + +#ifndef NDEBUG + assert(!stmt.used); + stmt.used = true; +#endif +} + +Query::~Query() { + reset(); + clearBindings(); + +#ifndef NDEBUG + stmt.used = false; +#endif +} + +template <> void Query::bind(int offset, std::nullptr_t) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_null(stmt.impl->stmt, offset)); +} + +template <> void Query::bind(int offset, int8_t value) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value)); +} + +template <> void Query::bind(int offset, int16_t value) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value)); +} + +template <> void Query::bind(int offset, int32_t value) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value)); +} + +template <> void Query::bind(int offset, int64_t value) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value)); +} + +template <> void Query::bind(int offset, uint8_t value) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value)); +} + +template <> void Query::bind(int offset, uint16_t value) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value)); +} + +template <> void Query::bind(int offset, uint32_t value) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value)); +} + +template <> void Query::bind(int offset, float value) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_double(stmt.impl->stmt, offset, value)); +} + +template <> void Query::bind(int offset, double value) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_double(stmt.impl->stmt, offset, value)); +} + +template <> void Query::bind(int offset, bool value) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_int(stmt.impl->stmt, offset, value)); +} + +template <> void Query::bind(int offset, const char *value) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_text(stmt.impl->stmt, offset, value, -1, SQLITE_STATIC)); +} + +// We currently cannot use sqlite3_bind_blob64 / sqlite3_bind_text64 because they +// were introduced in SQLite 3.8.7, and we need to support earlier versions: +// Android 11: 3.7 +// Android 21: 3.8 +// Android 24: 3.9 +// Per https://developer.android.com/reference/android/database/sqlite/package-summary. +// The first iOS version with 3.8.7+ was 9.0, with 3.8.8. + +void Query::bind(int offset, const char * value, std::size_t length, bool retain) { + assert(stmt.impl); + if (length > std::numeric_limits<int>::max()) { + throw std::range_error("value too long for sqlite3_bind_text"); + } + stmt.impl->check(sqlite3_bind_text(stmt.impl->stmt, offset, value, int(length), + retain ? SQLITE_TRANSIENT : SQLITE_STATIC)); +} + +void Query::bind(int offset, const std::string& value, bool retain) { + bind(offset, value.data(), value.size(), retain); +} + +void Query::bindBlob(int offset, const void * value, std::size_t length, bool retain) { + assert(stmt.impl); + if (length > std::numeric_limits<int>::max()) { + throw std::range_error("value too long for sqlite3_bind_text"); + } + stmt.impl->check(sqlite3_bind_blob(stmt.impl->stmt, offset, value, int(length), + retain ? SQLITE_TRANSIENT : SQLITE_STATIC)); +} + +void Query::bindBlob(int offset, const std::vector<uint8_t>& value, bool retain) { + bindBlob(offset, value.data(), value.size(), retain); +} + +template <> +void Query::bind( + int offset, std::chrono::time_point<std::chrono::system_clock, std::chrono::seconds> value) { + assert(stmt.impl); + stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, std::chrono::system_clock::to_time_t(value))); +} + +template <> void Query::bind(int offset, optional<std::string> value) { + if (!value) { + bind(offset, nullptr); + } else { + bind(offset, *value); + } +} + +template <> +void Query::bind( + int offset, + optional<std::chrono::time_point<std::chrono::system_clock, std::chrono::seconds>> value) { + if (!value) { + bind(offset, nullptr); + } else { + bind(offset, *value); + } +} + +bool Query::run() { + assert(stmt.impl); + const int err = sqlite3_step(stmt.impl->stmt); + stmt.impl->lastInsertRowId = sqlite3_last_insert_rowid(sqlite3_db_handle(stmt.impl->stmt)); + stmt.impl->changes = sqlite3_changes(sqlite3_db_handle(stmt.impl->stmt)); + if (err == SQLITE_DONE) { + return false; + } else if (err == SQLITE_ROW) { + return true; + } else if (err != SQLITE_OK) { + throw Exception { err, sqlite3_errmsg(sqlite3_db_handle(stmt.impl->stmt)) }; + } else { + return false; + } +} + +template <> bool Query::get(int offset) { + assert(stmt.impl); + return sqlite3_column_int(stmt.impl->stmt, offset); +} + +template <> int Query::get(int offset) { + assert(stmt.impl); + return sqlite3_column_int(stmt.impl->stmt, offset); +} + +template <> int64_t Query::get(int offset) { + assert(stmt.impl); + return sqlite3_column_int64(stmt.impl->stmt, offset); +} + +template <> double Query::get(int offset) { + assert(stmt.impl); + return sqlite3_column_double(stmt.impl->stmt, offset); +} + +template <> std::string Query::get(int offset) { + assert(stmt.impl); + return { + reinterpret_cast<const char *>(sqlite3_column_blob(stmt.impl->stmt, offset)), + size_t(sqlite3_column_bytes(stmt.impl->stmt, offset)) + }; +} + +template <> std::vector<uint8_t> Query::get(int offset) { + assert(stmt.impl); + const auto* begin = reinterpret_cast<const uint8_t*>(sqlite3_column_blob(stmt.impl->stmt, offset)); + const uint8_t* end = begin + sqlite3_column_bytes(stmt.impl->stmt, offset); + return { begin, end }; +} + +template <> +std::chrono::time_point<std::chrono::system_clock, std::chrono::seconds> +Query::get(int offset) { + assert(stmt.impl); + return std::chrono::time_point_cast<std::chrono::seconds>( + std::chrono::system_clock::from_time_t(sqlite3_column_int64(stmt.impl->stmt, offset))); +} + +template <> optional<int64_t> Query::get(int offset) { + assert(stmt.impl); + if (sqlite3_column_type(stmt.impl->stmt, offset) == SQLITE_NULL) { + return optional<int64_t>(); + } else { + return get<int64_t>(offset); + } +} + +template <> optional<double> Query::get(int offset) { + assert(stmt.impl); + if (sqlite3_column_type(stmt.impl->stmt, offset) == SQLITE_NULL) { + return optional<double>(); + } else { + return get<double>(offset); + } +} + +template <> optional<std::string> Query::get(int offset) { + assert(stmt.impl); + if (sqlite3_column_type(stmt.impl->stmt, offset) == SQLITE_NULL) { + return optional<std::string>(); + } else { + return get<std::string>(offset); + } +} + +template <> +optional<std::chrono::time_point<std::chrono::system_clock, std::chrono::seconds>> +Query::get(int offset) { + assert(stmt.impl); + if (sqlite3_column_type(stmt.impl->stmt, offset) == SQLITE_NULL) { + return {}; + } else { + return get<std::chrono::time_point<std::chrono::system_clock, std::chrono::seconds>>( + offset); + } +} + +void Query::reset() { + assert(stmt.impl); + sqlite3_reset(stmt.impl->stmt); +} + +void Query::clearBindings() { + assert(stmt.impl); + sqlite3_clear_bindings(stmt.impl->stmt); +} + +int64_t Query::lastInsertRowId() const { + assert(stmt.impl); + return stmt.impl->lastInsertRowId; +} + +uint64_t Query::changes() const { + assert(stmt.impl); + auto changes_ = stmt.impl->changes; + return (changes_ < 0 ? 0 : changes_); +} + +Transaction::Transaction(Database& db_, Mode mode) + : dbImpl(*db_.impl) { + switch (mode) { + case Deferred: + dbImpl.exec("BEGIN DEFERRED TRANSACTION"); + break; + case Immediate: + dbImpl.exec("BEGIN IMMEDIATE TRANSACTION"); + break; + case Exclusive: + dbImpl.exec("BEGIN EXCLUSIVE TRANSACTION"); + break; + } +} + +Transaction::~Transaction() { + if (needRollback) { + try { + rollback(); + } catch (...) { + // Ignore failed rollbacks in destructor. + } + } +} + +void Transaction::commit() { + needRollback = false; + dbImpl.exec("COMMIT TRANSACTION"); +} + +void Transaction::rollback() { + needRollback = false; + dbImpl.exec("ROLLBACK TRANSACTION"); +} + +} // namespace sqlite +} // namespace mapbox |