summaryrefslogtreecommitdiff
path: root/platform/default/src/mbgl/storage
diff options
context:
space:
mode:
authorKonstantin Käfer <mail@kkaefer.com>2018-12-13 18:45:29 +0100
committerKonstantin Käfer <mail@kkaefer.com>2018-12-13 20:39:43 +0100
commit2049ff09c2b41a5ccff693a4a64e517d47a08e4a (patch)
tree25b8f9acdb5e1b6629293bcc6f014ea4bcb6d1a3 /platform/default/src/mbgl/storage
parentb6b1067caf6ba911efbb4a64a43425ce2d729a1a (diff)
downloadqtlocation-mapboxgl-2049ff09c2b41a5ccff693a4a64e517d47a08e4a.tar.gz
[build] rework platform/default directory and add -files.txt for vendored libsupstream/build-changes
Diffstat (limited to 'platform/default/src/mbgl/storage')
-rw-r--r--platform/default/src/mbgl/storage/asset_file_source.cpp81
-rw-r--r--platform/default/src/mbgl/storage/default_file_source.cpp316
-rw-r--r--platform/default/src/mbgl/storage/file_source_request.cpp37
-rw-r--r--platform/default/src/mbgl/storage/http_file_source.cpp495
-rw-r--r--platform/default/src/mbgl/storage/local_file_source.cpp81
-rw-r--r--platform/default/src/mbgl/storage/offline.cpp158
-rw-r--r--platform/default/src/mbgl/storage/offline_database.cpp1129
-rw-r--r--platform/default/src/mbgl/storage/offline_download.cpp453
-rw-r--r--platform/default/src/mbgl/storage/online_file_source.cpp488
-rw-r--r--platform/default/src/mbgl/storage/sqlite3.cpp494
10 files changed, 3732 insertions, 0 deletions
diff --git a/platform/default/src/mbgl/storage/asset_file_source.cpp b/platform/default/src/mbgl/storage/asset_file_source.cpp
new file mode 100644
index 0000000000..7988654ae5
--- /dev/null
+++ b/platform/default/src/mbgl/storage/asset_file_source.cpp
@@ -0,0 +1,81 @@
+#include <mbgl/storage/asset_file_source.hpp>
+#include <mbgl/storage/file_source_request.hpp>
+#include <mbgl/storage/response.hpp>
+#include <mbgl/util/string.hpp>
+#include <mbgl/util/thread.hpp>
+#include <mbgl/util/url.hpp>
+#include <mbgl/util/util.hpp>
+#include <mbgl/util/io.hpp>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+
+namespace {
+
+const std::string assetProtocol = "asset://";
+
+} // namespace
+
+namespace mbgl {
+
+class AssetFileSource::Impl {
+public:
+ Impl(ActorRef<Impl>, std::string root_)
+ : root(std::move(root_)) {
+ }
+
+ void request(const std::string& url, ActorRef<FileSourceRequest> req) {
+ Response response;
+
+ if (!acceptsURL(url)) {
+ response.error = std::make_unique<Response::Error>(Response::Error::Reason::Other,
+ "Invalid asset URL");
+ req.invoke(&FileSourceRequest::setResponse, response);
+ return;
+ }
+
+ // Cut off the protocol and prefix with path.
+ const auto path = root + "/" + mbgl::util::percentDecode(url.substr(assetProtocol.size()));
+ struct stat buf;
+ int result = stat(path.c_str(), &buf);
+
+ if (result == 0 && (S_IFDIR & buf.st_mode)) {
+ response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
+ } else if (result == -1 && errno == ENOENT) {
+ response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
+ } else {
+ try {
+ response.data = std::make_shared<std::string>(util::read_file(path));
+ } catch (...) {
+ response.error = std::make_unique<Response::Error>(
+ Response::Error::Reason::Other,
+ util::toString(std::current_exception()));
+ }
+ }
+
+ req.invoke(&FileSourceRequest::setResponse, response);
+ }
+
+private:
+ std::string root;
+};
+
+AssetFileSource::AssetFileSource(const std::string& root)
+ : impl(std::make_unique<util::Thread<Impl>>("AssetFileSource", root)) {
+}
+
+AssetFileSource::~AssetFileSource() = default;
+
+std::unique_ptr<AsyncRequest> AssetFileSource::request(const Resource& resource, Callback callback) {
+ auto req = std::make_unique<FileSourceRequest>(std::move(callback));
+
+ impl->actor().invoke(&Impl::request, resource.url, req->actor());
+
+ return std::move(req);
+}
+
+bool AssetFileSource::acceptsURL(const std::string& url) {
+ return 0 == url.rfind(assetProtocol, 0);
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/storage/default_file_source.cpp b/platform/default/src/mbgl/storage/default_file_source.cpp
new file mode 100644
index 0000000000..cad68e7de9
--- /dev/null
+++ b/platform/default/src/mbgl/storage/default_file_source.cpp
@@ -0,0 +1,316 @@
+#include <mbgl/storage/default_file_source.hpp>
+#include <mbgl/storage/asset_file_source.hpp>
+#include <mbgl/storage/file_source_request.hpp>
+#include <mbgl/storage/local_file_source.hpp>
+#include <mbgl/storage/online_file_source.hpp>
+#include <mbgl/storage/offline_database.hpp>
+#include <mbgl/storage/offline_download.hpp>
+#include <mbgl/storage/resource_transform.hpp>
+
+#include <mbgl/util/platform.hpp>
+#include <mbgl/util/url.hpp>
+#include <mbgl/util/thread.hpp>
+#include <mbgl/util/work_request.hpp>
+#include <mbgl/util/stopwatch.hpp>
+
+#include <cassert>
+
+namespace mbgl {
+
+class DefaultFileSource::Impl {
+public:
+ Impl(std::shared_ptr<FileSource> assetFileSource_, std::string cachePath, uint64_t maximumCacheSize)
+ : assetFileSource(assetFileSource_)
+ , localFileSource(std::make_unique<LocalFileSource>())
+ , offlineDatabase(std::make_unique<OfflineDatabase>(cachePath, maximumCacheSize)) {
+ }
+
+ void setAPIBaseURL(const std::string& url) {
+ onlineFileSource.setAPIBaseURL(url);
+ }
+
+ std::string getAPIBaseURL() const{
+ return onlineFileSource.getAPIBaseURL();
+ }
+
+ void setAccessToken(const std::string& accessToken) {
+ onlineFileSource.setAccessToken(accessToken);
+ }
+
+ std::string getAccessToken() const {
+ return onlineFileSource.getAccessToken();
+ }
+
+ void setResourceTransform(optional<ActorRef<ResourceTransform>>&& transform) {
+ onlineFileSource.setResourceTransform(std::move(transform));
+ }
+
+ void listRegions(std::function<void (expected<OfflineRegions, std::exception_ptr>)> callback) {
+ callback(offlineDatabase->listRegions());
+ }
+
+ void createRegion(const OfflineRegionDefinition& definition,
+ const OfflineRegionMetadata& metadata,
+ std::function<void (expected<OfflineRegion, std::exception_ptr>)> callback) {
+ callback(offlineDatabase->createRegion(definition, metadata));
+ }
+
+ void mergeOfflineRegions(const std::string& sideDatabasePath,
+ std::function<void (expected<OfflineRegions, std::exception_ptr>)> callback) {
+ callback(offlineDatabase->mergeDatabase(sideDatabasePath));
+ }
+
+ void updateMetadata(const int64_t regionID,
+ const OfflineRegionMetadata& metadata,
+ std::function<void (expected<OfflineRegionMetadata, std::exception_ptr>)> callback) {
+ callback(offlineDatabase->updateMetadata(regionID, metadata));
+ }
+
+ void getRegionStatus(int64_t regionID, std::function<void (expected<OfflineRegionStatus, std::exception_ptr>)> callback) {
+ if (auto download = getDownload(regionID)) {
+ callback(download.value()->getStatus());
+ } else {
+ callback(unexpected<std::exception_ptr>(download.error()));
+ }
+ }
+
+ void deleteRegion(OfflineRegion&& region, std::function<void (std::exception_ptr)> callback) {
+ downloads.erase(region.getID());
+ callback(offlineDatabase->deleteRegion(std::move(region)));
+ }
+
+ void setRegionObserver(int64_t regionID, std::unique_ptr<OfflineRegionObserver> observer) {
+ if (auto download = getDownload(regionID)) {
+ download.value()->setObserver(std::move(observer));
+ }
+ }
+
+ void setRegionDownloadState(int64_t regionID, OfflineRegionDownloadState state) {
+ if (auto download = getDownload(regionID)) {
+ download.value()->setState(state);
+ }
+ }
+
+ void request(AsyncRequest* req, Resource resource, ActorRef<FileSourceRequest> ref) {
+ auto callback = [ref] (const Response& res) mutable {
+ ref.invoke(&FileSourceRequest::setResponse, res);
+ };
+
+ if (AssetFileSource::acceptsURL(resource.url)) {
+ //Asset request
+ tasks[req] = assetFileSource->request(resource, callback);
+ } else if (LocalFileSource::acceptsURL(resource.url)) {
+ //Local file request
+ tasks[req] = localFileSource->request(resource, callback);
+ } else {
+ // Try the offline database
+ if (resource.hasLoadingMethod(Resource::LoadingMethod::Cache)) {
+ auto offlineResponse = offlineDatabase->get(resource);
+
+ if (resource.loadingMethod == Resource::LoadingMethod::CacheOnly) {
+ if (!offlineResponse) {
+ // Ensure there's always a response that we can send, so the caller knows that
+ // there's no optional data available in the cache, when it's the only place
+ // we're supposed to load from.
+ offlineResponse.emplace();
+ offlineResponse->noContent = true;
+ offlineResponse->error = std::make_unique<Response::Error>(
+ Response::Error::Reason::NotFound, "Not found in offline database");
+ } else if (!offlineResponse->isUsable()) {
+ // Don't return resources the server requested not to show when they're stale.
+ // Even if we can't directly use the response, we may still use it to send a
+ // conditional HTTP request, which is why we're saving it above.
+ offlineResponse->error = std::make_unique<Response::Error>(
+ Response::Error::Reason::NotFound, "Cached resource is unusable");
+ }
+ callback(*offlineResponse);
+ } else if (offlineResponse) {
+ // Copy over the fields so that we can use them when making a refresh request.
+ resource.priorModified = offlineResponse->modified;
+ resource.priorExpires = offlineResponse->expires;
+ resource.priorEtag = offlineResponse->etag;
+ resource.priorData = offlineResponse->data;
+
+ if (offlineResponse->isUsable()) {
+ callback(*offlineResponse);
+ }
+ }
+ }
+
+ // Get from the online file source
+ if (resource.hasLoadingMethod(Resource::LoadingMethod::Network)) {
+ MBGL_TIMING_START(watch);
+ tasks[req] = onlineFileSource.request(resource, [=] (Response onlineResponse) mutable {
+ this->offlineDatabase->put(resource, onlineResponse);
+ if (resource.kind == Resource::Kind::Tile) {
+ // onlineResponse.data will be null if data not modified
+ MBGL_TIMING_FINISH(watch,
+ " Action: " << "Requesting," <<
+ " URL: " << resource.url.c_str() <<
+ " Size: " << (onlineResponse.data != nullptr ? onlineResponse.data->size() : 0) << "B," <<
+ " Time")
+ }
+ callback(onlineResponse);
+ });
+ }
+ }
+ }
+
+ void cancel(AsyncRequest* req) {
+ tasks.erase(req);
+ }
+
+ void setOfflineMapboxTileCountLimit(uint64_t limit) {
+ offlineDatabase->setOfflineMapboxTileCountLimit(limit);
+ }
+
+ void setOnlineStatus(const bool status) {
+ onlineFileSource.setOnlineStatus(status);
+ }
+
+ void put(const Resource& resource, const Response& response) {
+ offlineDatabase->put(resource, response);
+ }
+
+private:
+ expected<OfflineDownload*, std::exception_ptr> getDownload(int64_t regionID) {
+ auto it = downloads.find(regionID);
+ if (it != downloads.end()) {
+ return it->second.get();
+ }
+ auto definition = offlineDatabase->getRegionDefinition(regionID);
+ if (!definition) {
+ return unexpected<std::exception_ptr>(definition.error());
+ }
+ auto download = std::make_unique<OfflineDownload>(regionID, std::move(definition.value()),
+ *offlineDatabase, onlineFileSource);
+ return downloads.emplace(regionID, std::move(download)).first->second.get();
+ }
+
+ // shared so that destruction is done on the creating thread
+ const std::shared_ptr<FileSource> assetFileSource;
+ const std::unique_ptr<FileSource> localFileSource;
+ std::unique_ptr<OfflineDatabase> offlineDatabase;
+ OnlineFileSource onlineFileSource;
+ std::unordered_map<AsyncRequest*, std::unique_ptr<AsyncRequest>> tasks;
+ std::unordered_map<int64_t, std::unique_ptr<OfflineDownload>> downloads;
+};
+
+DefaultFileSource::DefaultFileSource(const std::string& cachePath,
+ const std::string& assetRoot,
+ uint64_t maximumCacheSize)
+ : DefaultFileSource(cachePath, std::make_unique<AssetFileSource>(assetRoot), maximumCacheSize) {
+}
+
+DefaultFileSource::DefaultFileSource(const std::string& cachePath,
+ std::unique_ptr<FileSource>&& assetFileSource_,
+ uint64_t maximumCacheSize)
+ : assetFileSource(std::move(assetFileSource_))
+ , impl(std::make_unique<util::Thread<Impl>>("DefaultFileSource", assetFileSource, cachePath, maximumCacheSize)) {
+}
+
+DefaultFileSource::~DefaultFileSource() = default;
+
+void DefaultFileSource::setAPIBaseURL(const std::string& baseURL) {
+ impl->actor().invoke(&Impl::setAPIBaseURL, baseURL);
+
+ {
+ std::lock_guard<std::mutex> lock(cachedBaseURLMutex);
+ cachedBaseURL = baseURL;
+ }
+}
+
+std::string DefaultFileSource::getAPIBaseURL() {
+ std::lock_guard<std::mutex> lock(cachedBaseURLMutex);
+ return cachedBaseURL;
+}
+
+void DefaultFileSource::setAccessToken(const std::string& accessToken) {
+ impl->actor().invoke(&Impl::setAccessToken, accessToken);
+
+ {
+ std::lock_guard<std::mutex> lock(cachedAccessTokenMutex);
+ cachedAccessToken = accessToken;
+ }
+}
+
+std::string DefaultFileSource::getAccessToken() {
+ std::lock_guard<std::mutex> lock(cachedAccessTokenMutex);
+ return cachedAccessToken;
+}
+
+void DefaultFileSource::setResourceTransform(optional<ActorRef<ResourceTransform>>&& transform) {
+ impl->actor().invoke(&Impl::setResourceTransform, std::move(transform));
+}
+
+std::unique_ptr<AsyncRequest> DefaultFileSource::request(const Resource& resource, Callback callback) {
+ auto req = std::make_unique<FileSourceRequest>(std::move(callback));
+
+ req->onCancel([fs = impl->actor(), req = req.get()] () mutable { fs.invoke(&Impl::cancel, req); });
+
+ impl->actor().invoke(&Impl::request, req.get(), resource, req->actor());
+
+ return std::move(req);
+}
+
+void DefaultFileSource::listOfflineRegions(std::function<void (expected<OfflineRegions, std::exception_ptr>)> callback) {
+ impl->actor().invoke(&Impl::listRegions, callback);
+}
+
+void DefaultFileSource::createOfflineRegion(const OfflineRegionDefinition& definition,
+ const OfflineRegionMetadata& metadata,
+ std::function<void (expected<OfflineRegion, std::exception_ptr>)> callback) {
+ impl->actor().invoke(&Impl::createRegion, definition, metadata, callback);
+}
+
+void DefaultFileSource::mergeOfflineRegions(const std::string& sideDatabasePath,
+ std::function<void (expected<OfflineRegions, std::exception_ptr>)> callback) {
+ impl->actor().invoke(&Impl::mergeOfflineRegions, sideDatabasePath, callback);
+}
+
+void DefaultFileSource::updateOfflineMetadata(const int64_t regionID,
+ const OfflineRegionMetadata& metadata,
+ std::function<void (expected<OfflineRegionMetadata,
+ std::exception_ptr>)> callback) {
+ impl->actor().invoke(&Impl::updateMetadata, regionID, metadata, callback);
+}
+
+void DefaultFileSource::deleteOfflineRegion(OfflineRegion&& region, std::function<void (std::exception_ptr)> callback) {
+ impl->actor().invoke(&Impl::deleteRegion, std::move(region), callback);
+}
+
+void DefaultFileSource::setOfflineRegionObserver(OfflineRegion& region, std::unique_ptr<OfflineRegionObserver> observer) {
+ impl->actor().invoke(&Impl::setRegionObserver, region.getID(), std::move(observer));
+}
+
+void DefaultFileSource::setOfflineRegionDownloadState(OfflineRegion& region, OfflineRegionDownloadState state) {
+ impl->actor().invoke(&Impl::setRegionDownloadState, region.getID(), state);
+}
+
+void DefaultFileSource::getOfflineRegionStatus(OfflineRegion& region, std::function<void (expected<OfflineRegionStatus, std::exception_ptr>)> callback) const {
+ impl->actor().invoke(&Impl::getRegionStatus, region.getID(), callback);
+}
+
+void DefaultFileSource::setOfflineMapboxTileCountLimit(uint64_t limit) const {
+ impl->actor().invoke(&Impl::setOfflineMapboxTileCountLimit, limit);
+}
+
+void DefaultFileSource::pause() {
+ impl->pause();
+}
+
+void DefaultFileSource::resume() {
+ impl->resume();
+}
+
+void DefaultFileSource::put(const Resource& resource, const Response& response) {
+ impl->actor().invoke(&Impl::put, resource, response);
+}
+
+// For testing only:
+
+void DefaultFileSource::setOnlineStatus(const bool status) {
+ impl->actor().invoke(&Impl::setOnlineStatus, status);
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/storage/file_source_request.cpp b/platform/default/src/mbgl/storage/file_source_request.cpp
new file mode 100644
index 0000000000..09ea8cc32a
--- /dev/null
+++ b/platform/default/src/mbgl/storage/file_source_request.cpp
@@ -0,0 +1,37 @@
+#include <mbgl/storage/file_source_request.hpp>
+
+#include <mbgl/actor/mailbox.hpp>
+#include <mbgl/actor/scheduler.hpp>
+
+namespace mbgl {
+
+FileSourceRequest::FileSourceRequest(FileSource::Callback&& callback)
+ : responseCallback(callback)
+ , mailbox(std::make_shared<Mailbox>(*Scheduler::GetCurrent())) {
+}
+
+FileSourceRequest::~FileSourceRequest() {
+ if (cancelCallback) {
+ cancelCallback();
+ }
+
+ mailbox->close();
+}
+
+void FileSourceRequest::onCancel(std::function<void()>&& callback) {
+ cancelCallback = std::move(callback);
+}
+
+void FileSourceRequest::setResponse(const Response& response) {
+ // Copy, because calling the callback will sometimes self
+ // destroy this object. We cannot move because this method
+ // can be called more than one.
+ auto callback = responseCallback;
+ callback(response);
+}
+
+ActorRef<FileSourceRequest> FileSourceRequest::actor() {
+ return ActorRef<FileSourceRequest>(*this, mailbox);
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/storage/http_file_source.cpp b/platform/default/src/mbgl/storage/http_file_source.cpp
new file mode 100644
index 0000000000..213b53de98
--- /dev/null
+++ b/platform/default/src/mbgl/storage/http_file_source.cpp
@@ -0,0 +1,495 @@
+#include <mbgl/storage/http_file_source.hpp>
+#include <mbgl/storage/resource.hpp>
+#include <mbgl/storage/response.hpp>
+#include <mbgl/util/logging.hpp>
+
+#include <mbgl/util/util.hpp>
+#include <mbgl/util/optional.hpp>
+#include <mbgl/util/run_loop.hpp>
+#include <mbgl/util/string.hpp>
+#include <mbgl/util/timer.hpp>
+#include <mbgl/util/chrono.hpp>
+#include <mbgl/util/http_header.hpp>
+
+#include <curl/curl.h>
+
+// Dynamically load all cURL functions. Debian-derived systems upgraded the OpenSSL version linked
+// to in https://salsa.debian.org/debian/curl/commit/95c94957bb7e89e36e78b995fed468c42f64d18d
+// They state:
+// Rename libcurl3 to libcurl4, because libcurl exposes an SSL_CTX via
+// CURLOPT_SSL_CTX_FUNCTION, and this object changes incompatibly between
+// openssl 1.0 and openssl 1.1.
+// Since we are not accessing the underlying OpenSSL context, we don't care whether we're linking
+// against libcurl3 or libcurl4; both use the ABI version 4 which hasn't changed since 2006
+// (see https://curl.haxx.se/libcurl/abi.html). In fact, cURL's ABI compatibility is very good as
+// shown on https://abi-laboratory.pro/tracker/timeline/curl/
+// Therefore, we're dynamically loading the cURL symbols we need to avoid linking against versioned
+// symbols.
+#include <dlfcn.h>
+
+namespace curl {
+
+#define CURL_FUNCTIONS \
+ X(global_init) \
+ X(getdate) \
+ X(easy_strerror) \
+ X(easy_init) \
+ X(easy_setopt) \
+ X(easy_cleanup) \
+ X(easy_getinfo) \
+ X(easy_reset) \
+ X(multi_init) \
+ X(multi_add_handle) \
+ X(multi_remove_handle) \
+ X(multi_cleanup) \
+ X(multi_info_read) \
+ X(multi_strerror) \
+ X(multi_socket_action) \
+ X(multi_setopt) \
+ X(share_init) \
+ X(share_cleanup) \
+ X(slist_append) \
+ X(slist_free_all)
+
+#define X(name) static decltype(&curl_ ## name) name = nullptr;
+CURL_FUNCTIONS
+#undef X
+
+static void* handle = nullptr;
+
+static void* load(const char* name) {
+ void* symbol = dlsym(handle, name);
+ if (const char* error = dlerror()) {
+ fprintf(stderr, "Cannot load symbol '%s': %s\n", name, error);
+ dlclose(handle);
+ handle = nullptr;
+ abort();
+ }
+ return symbol;
+}
+
+__attribute__((constructor))
+static void load() {
+ assert(!handle);
+ handle = dlopen("libcurl.so.4", RTLD_LAZY | RTLD_LOCAL);
+ if (!handle) {
+ fprintf(stderr, "Could not open shared library '%s'\n", "libcurl.so.4");
+ abort();
+ }
+
+ #define X(name) name = (decltype(&curl_ ## name))load("curl_" #name);
+ CURL_FUNCTIONS
+ #undef X
+}
+
+__attribute__((constructor))
+static void unload() {
+ if (handle) {
+ dlclose(handle);
+ }
+}
+
+} // namespace curl
+
+
+#include <queue>
+#include <map>
+#include <cassert>
+#include <cstring>
+#include <cstdio>
+
+static void handleError(CURLMcode code) {
+ if (code != CURLM_OK) {
+ throw std::runtime_error(std::string("CURL multi error: ") + curl::multi_strerror(code));
+ }
+}
+
+static void handleError(CURLcode code) {
+ if (code != CURLE_OK) {
+ throw std::runtime_error(std::string("CURL easy error: ") + curl::easy_strerror(code));
+ }
+}
+
+namespace mbgl {
+
+class HTTPFileSource::Impl {
+public:
+ Impl();
+ ~Impl();
+
+ static int handleSocket(CURL *handle, curl_socket_t s, int action, void *userp, void *socketp);
+ static int startTimeout(CURLM *multi, long timeout_ms, void *userp);
+ static void onTimeout(HTTPFileSource::Impl *context);
+
+ void perform(curl_socket_t s, util::RunLoop::Event event);
+ CURL *getHandle();
+ void returnHandle(CURL *handle);
+ void checkMultiInfo();
+
+ // Used as the CURL timer function to periodically check for socket updates.
+ util::Timer timeout;
+
+ // CURL multi handle that we use to request multiple URLs at the same time, without having to
+ // block and spawn threads.
+ CURLM *multi = nullptr;
+
+ // CURL share handles are used for sharing session state (e.g.)
+ CURLSH *share = nullptr;
+
+ // A queue that we use for storing resuable CURL easy handles to avoid creating and destroying
+ // them all the time.
+ std::queue<CURL *> handles;
+};
+
+class HTTPRequest : public AsyncRequest {
+public:
+ HTTPRequest(HTTPFileSource::Impl*, Resource, FileSource::Callback);
+ ~HTTPRequest() override;
+
+ void handleResult(CURLcode code);
+
+private:
+ static size_t headerCallback(char *const buffer, const size_t size, const size_t nmemb, void *userp);
+ static size_t writeCallback(void *const contents, const size_t size, const size_t nmemb, void *userp);
+
+ HTTPFileSource::Impl* context = nullptr;
+ Resource resource;
+ FileSource::Callback callback;
+
+ // Will store the current response.
+ std::shared_ptr<std::string> data;
+ std::unique_ptr<Response> response;
+
+ optional<std::string> retryAfter;
+ optional<std::string> xRateLimitReset;
+
+ CURL *handle = nullptr;
+ curl_slist *headers = nullptr;
+
+ char error[CURL_ERROR_SIZE] = { 0 };
+};
+
+HTTPFileSource::Impl::Impl() {
+ if (curl::global_init(CURL_GLOBAL_ALL)) {
+ throw std::runtime_error("Could not init cURL");
+ }
+
+ share = curl::share_init();
+
+ multi = curl::multi_init();
+ handleError(curl::multi_setopt(multi, CURLMOPT_SOCKETFUNCTION, handleSocket));
+ handleError(curl::multi_setopt(multi, CURLMOPT_SOCKETDATA, this));
+ handleError(curl::multi_setopt(multi, CURLMOPT_TIMERFUNCTION, startTimeout));
+ handleError(curl::multi_setopt(multi, CURLMOPT_TIMERDATA, this));
+}
+
+HTTPFileSource::Impl::~Impl() {
+ while (!handles.empty()) {
+ curl::easy_cleanup(handles.front());
+ handles.pop();
+ }
+
+ curl::multi_cleanup(multi);
+ multi = nullptr;
+
+ curl::share_cleanup(share);
+ share = nullptr;
+
+ timeout.stop();
+}
+
+CURL *HTTPFileSource::Impl::getHandle() {
+ if (!handles.empty()) {
+ auto handle = handles.front();
+ handles.pop();
+ return handle;
+ } else {
+ return curl::easy_init();
+ }
+}
+
+void HTTPFileSource::Impl::returnHandle(CURL *handle) {
+ curl::easy_reset(handle);
+ handles.push(handle);
+}
+
+void HTTPFileSource::Impl::checkMultiInfo() {
+ CURLMsg *message = nullptr;
+ int pending = 0;
+
+ while ((message = curl::multi_info_read(multi, &pending))) {
+ switch (message->msg) {
+ case CURLMSG_DONE: {
+ HTTPRequest *baton = nullptr;
+ curl::easy_getinfo(message->easy_handle, CURLINFO_PRIVATE, (char *)&baton);
+ assert(baton);
+ baton->handleResult(message->data.result);
+ } break;
+
+ default:
+ // This should never happen, because there are no other message types.
+ throw std::runtime_error("CURLMsg returned unknown message type");
+ }
+ }
+}
+
+void HTTPFileSource::Impl::perform(curl_socket_t s, util::RunLoop::Event events) {
+ int flags = 0;
+
+ if (events == util::RunLoop::Event::Read) {
+ flags |= CURL_CSELECT_IN;
+ }
+ if (events == util::RunLoop::Event::Write) {
+ flags |= CURL_CSELECT_OUT;
+ }
+
+
+ int running_handles = 0;
+ curl::multi_socket_action(multi, s, flags, &running_handles);
+ checkMultiInfo();
+}
+
+int HTTPFileSource::Impl::handleSocket(CURL * /* handle */, curl_socket_t s, int action, void *userp,
+ void * /* socketp */) {
+ assert(userp);
+ auto context = reinterpret_cast<Impl *>(userp);
+
+ switch (action) {
+ case CURL_POLL_IN: {
+ using namespace std::placeholders;
+ util::RunLoop::Get()->addWatch(s, util::RunLoop::Event::Read,
+ std::bind(&Impl::perform, context, _1, _2));
+ break;
+ }
+ case CURL_POLL_OUT: {
+ using namespace std::placeholders;
+ util::RunLoop::Get()->addWatch(s, util::RunLoop::Event::Write,
+ std::bind(&Impl::perform, context, _1, _2));
+ break;
+ }
+ case CURL_POLL_REMOVE:
+ util::RunLoop::Get()->removeWatch(s);
+ break;
+ default:
+ throw std::runtime_error("Unhandled CURL socket action");
+ }
+
+ return 0;
+}
+
+void HTTPFileSource::Impl::onTimeout(Impl *context) {
+ int running_handles;
+ CURLMcode error = curl::multi_socket_action(context->multi, CURL_SOCKET_TIMEOUT, 0, &running_handles);
+ if (error != CURLM_OK) {
+ throw std::runtime_error(std::string("CURL multi error: ") + curl::multi_strerror(error));
+ }
+ context->checkMultiInfo();
+}
+
+int HTTPFileSource::Impl::startTimeout(CURLM * /* multi */, long timeout_ms, void *userp) {
+ assert(userp);
+ auto context = reinterpret_cast<Impl *>(userp);
+
+ if (timeout_ms < 0) {
+ // A timeout of 0 ms means that the timer will invoked in the next loop iteration.
+ timeout_ms = 0;
+ }
+
+ context->timeout.stop();
+ context->timeout.start(mbgl::Milliseconds(timeout_ms), Duration::zero(),
+ std::bind(&Impl::onTimeout, context));
+
+ return 0;
+}
+
+HTTPRequest::HTTPRequest(HTTPFileSource::Impl* context_, Resource resource_, FileSource::Callback callback_)
+ : context(context_),
+ resource(std::move(resource_)),
+ callback(std::move(callback_)),
+ handle(context->getHandle()) {
+
+ // If there's already a response, set the correct etags/modified headers to make sure we are
+ // getting a 304 response if possible. This avoids redownloading unchanged data.
+ if (resource.priorEtag) {
+ const std::string header = std::string("If-None-Match: ") + *resource.priorEtag;
+ headers = curl::slist_append(headers, header.c_str());
+ } else if (resource.priorModified) {
+ const std::string time =
+ std::string("If-Modified-Since: ") + util::rfc1123(*resource.priorModified);
+ headers = curl::slist_append(headers, time.c_str());
+ }
+
+ if (headers) {
+ curl::easy_setopt(handle, CURLOPT_HTTPHEADER, headers);
+ }
+
+ handleError(curl::easy_setopt(handle, CURLOPT_PRIVATE, this));
+ handleError(curl::easy_setopt(handle, CURLOPT_ERRORBUFFER, error));
+ handleError(curl::easy_setopt(handle, CURLOPT_CAINFO, "ca-bundle.crt"));
+ handleError(curl::easy_setopt(handle, CURLOPT_FOLLOWLOCATION, 1));
+ handleError(curl::easy_setopt(handle, CURLOPT_URL, resource.url.c_str()));
+ handleError(curl::easy_setopt(handle, CURLOPT_WRITEFUNCTION, writeCallback));
+ handleError(curl::easy_setopt(handle, CURLOPT_WRITEDATA, this));
+ handleError(curl::easy_setopt(handle, CURLOPT_HEADERFUNCTION, headerCallback));
+ handleError(curl::easy_setopt(handle, CURLOPT_HEADERDATA, this));
+#if LIBCURL_VERSION_NUM >= ((7) << 16 | (21) << 8 | 6) // Renamed in 7.21.6
+ handleError(curl::easy_setopt(handle, CURLOPT_ACCEPT_ENCODING, "gzip, deflate"));
+#else
+ handleError(curl::easy_setopt(handle, CURLOPT_ENCODING, "gzip, deflate"));
+#endif
+ handleError(curl::easy_setopt(handle, CURLOPT_USERAGENT, "MapboxGL/1.0"));
+ handleError(curl::easy_setopt(handle, CURLOPT_SHARE, context->share));
+
+ // Start requesting the information.
+ handleError(curl::multi_add_handle(context->multi, handle));
+}
+
+HTTPRequest::~HTTPRequest() {
+ handleError(curl::multi_remove_handle(context->multi, handle));
+ context->returnHandle(handle);
+ handle = nullptr;
+
+ if (headers) {
+ curl::slist_free_all(headers);
+ headers = nullptr;
+ }
+}
+
+// This function is called when we have new data for a request. We just append it to the string
+// containing the previous data.
+size_t HTTPRequest::writeCallback(void *const contents, const size_t size, const size_t nmemb, void *userp) {
+ assert(userp);
+ auto impl = reinterpret_cast<HTTPRequest *>(userp);
+
+ if (!impl->data) {
+ impl->data = std::make_shared<std::string>();
+ }
+
+ impl->data->append((char *)contents, size * nmemb);
+ return size * nmemb;
+}
+
+// Compares the beginning of the (non-zero-terminated!) data buffer with the (zero-terminated!)
+// header string. If the data buffer contains the header string at the beginning, it returns
+// the length of the header string == begin of the value, otherwise it returns npos.
+// The comparison of the header is ASCII-case-insensitive.
+size_t headerMatches(const char *const header, const char *const buffer, const size_t length) {
+ const size_t headerLength = strlen(header);
+ if (length < headerLength) {
+ return std::string::npos;
+ }
+ size_t i = 0;
+ while (i < length && i < headerLength && std::tolower(buffer[i]) == std::tolower(header[i])) {
+ i++;
+ }
+ return i == headerLength ? i : std::string::npos;
+}
+
+size_t HTTPRequest::headerCallback(char *const buffer, const size_t size, const size_t nmemb, void *userp) {
+ assert(userp);
+ auto baton = reinterpret_cast<HTTPRequest *>(userp);
+
+ if (!baton->response) {
+ baton->response = std::make_unique<Response>();
+ }
+
+ const size_t length = size * nmemb;
+ size_t begin = std::string::npos;
+ if ((begin = headerMatches("last-modified: ", buffer, length)) != std::string::npos) {
+ // Always overwrite the modification date; We might already have a value here from the
+ // Date header, but this one is more accurate.
+ const std::string value { buffer + begin, length - begin - 2 }; // remove \r\n
+ baton->response->modified = Timestamp{ Seconds(curl::getdate(value.c_str(), nullptr)) };
+ } else if ((begin = headerMatches("etag: ", buffer, length)) != std::string::npos) {
+ baton->response->etag = std::string(buffer + begin, length - begin - 2); // remove \r\n
+ } else if ((begin = headerMatches("cache-control: ", buffer, length)) != std::string::npos) {
+ const std::string value { buffer + begin, length - begin - 2 }; // remove \r\n
+ const auto cc = http::CacheControl::parse(value.c_str());
+ baton->response->expires = cc.toTimePoint();
+ baton->response->mustRevalidate = cc.mustRevalidate;
+ } else if ((begin = headerMatches("expires: ", buffer, length)) != std::string::npos) {
+ const std::string value { buffer + begin, length - begin - 2 }; // remove \r\n
+ baton->response->expires = Timestamp{ Seconds(curl::getdate(value.c_str(), nullptr)) };
+ } else if ((begin = headerMatches("retry-after: ", buffer, length)) != std::string::npos) {
+ baton->retryAfter = std::string(buffer + begin, length - begin - 2); // remove \r\n
+ } else if ((begin = headerMatches("x-rate-limit-reset: ", buffer, length)) != std::string::npos) {
+ baton->xRateLimitReset = std::string(buffer + begin, length - begin - 2); // remove \r\n
+ }
+
+ return length;
+}
+
+void HTTPRequest::handleResult(CURLcode code) {
+ // Make sure a response object exists in case we haven't got any headers or content.
+ if (!response) {
+ response = std::make_unique<Response>();
+ }
+
+ using Error = Response::Error;
+
+ // Add human-readable error code
+ if (code != CURLE_OK) {
+ switch (code) {
+ case CURLE_COULDNT_RESOLVE_PROXY:
+ case CURLE_COULDNT_RESOLVE_HOST:
+ case CURLE_COULDNT_CONNECT:
+ case CURLE_OPERATION_TIMEDOUT:
+
+ response->error = std::make_unique<Error>(
+ Error::Reason::Connection, std::string{ curl::easy_strerror(code) } + ": " + error);
+ break;
+
+ default:
+ response->error = std::make_unique<Error>(
+ Error::Reason::Other, std::string{ curl::easy_strerror(code) } + ": " + error);
+ break;
+ }
+ } else {
+ long responseCode = 0;
+ curl::easy_getinfo(handle, CURLINFO_RESPONSE_CODE, &responseCode);
+
+ if (responseCode == 200) {
+ if (data) {
+ response->data = std::move(data);
+ } else {
+ response->data = std::make_shared<std::string>();
+ }
+ } else if (responseCode == 204 || (responseCode == 404 && resource.kind == Resource::Kind::Tile)) {
+ response->noContent = true;
+ } else if (responseCode == 304) {
+ response->notModified = true;
+ } else if (responseCode == 404) {
+ response->error =
+ std::make_unique<Error>(Error::Reason::NotFound, "HTTP status code 404");
+ } else if (responseCode == 429) {
+ response->error =
+ std::make_unique<Error>(Error::Reason::RateLimit, "HTTP status code 429",
+ http::parseRetryHeaders(retryAfter, xRateLimitReset));
+ } else if (responseCode >= 500 && responseCode < 600) {
+ response->error =
+ std::make_unique<Error>(Error::Reason::Server, std::string{ "HTTP status code " } +
+ util::toString(responseCode));
+ } else {
+ response->error =
+ std::make_unique<Error>(Error::Reason::Other, std::string{ "HTTP status code " } +
+ util::toString(responseCode));
+ }
+ }
+
+ // Calling `callback` may result in deleting `this`. Copy data to temporaries first.
+ auto callback_ = callback;
+ auto response_ = *response;
+ callback_(response_);
+}
+
+HTTPFileSource::HTTPFileSource()
+ : impl(std::make_unique<Impl>()) {
+}
+
+HTTPFileSource::~HTTPFileSource() = default;
+
+std::unique_ptr<AsyncRequest> HTTPFileSource::request(const Resource& resource, Callback callback) {
+ return std::make_unique<HTTPRequest>(impl.get(), resource, callback);
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/storage/local_file_source.cpp b/platform/default/src/mbgl/storage/local_file_source.cpp
new file mode 100644
index 0000000000..1b7b7b9278
--- /dev/null
+++ b/platform/default/src/mbgl/storage/local_file_source.cpp
@@ -0,0 +1,81 @@
+#include <mbgl/storage/local_file_source.hpp>
+#include <mbgl/storage/file_source_request.hpp>
+#include <mbgl/storage/response.hpp>
+#include <mbgl/util/string.hpp>
+#include <mbgl/util/thread.hpp>
+#include <mbgl/util/url.hpp>
+#include <mbgl/util/util.hpp>
+#include <mbgl/util/io.hpp>
+
+#include <sys/types.h>
+#include <sys/stat.h>
+
+#if defined(_WINDOWS) && !defined(S_ISDIR)
+#define S_ISDIR(m) (((m) & S_IFMT) == S_IFDIR)
+#endif
+
+namespace {
+
+const std::string fileProtocol = "file://";
+
+} // namespace
+
+namespace mbgl {
+
+class LocalFileSource::Impl {
+public:
+ Impl(ActorRef<Impl>) {}
+
+ void request(const std::string& url, ActorRef<FileSourceRequest> req) {
+ Response response;
+
+ if (!acceptsURL(url)) {
+ response.error = std::make_unique<Response::Error>(Response::Error::Reason::Other,
+ "Invalid file URL");
+ req.invoke(&FileSourceRequest::setResponse, response);
+ return;
+ }
+
+ // Cut off the protocol and prefix with path.
+ const auto path = mbgl::util::percentDecode(url.substr(fileProtocol.size()));
+ struct stat buf;
+ int result = stat(path.c_str(), &buf);
+
+ if (result == 0 && S_ISDIR(buf.st_mode)) {
+ response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
+ } else if (result == -1 && errno == ENOENT) {
+ response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
+ } else {
+ try {
+ response.data = std::make_shared<std::string>(util::read_file(path));
+ } catch (...) {
+ response.error = std::make_unique<Response::Error>(
+ Response::Error::Reason::Other,
+ util::toString(std::current_exception()));
+ }
+ }
+
+ req.invoke(&FileSourceRequest::setResponse, response);
+ }
+
+};
+
+LocalFileSource::LocalFileSource()
+ : impl(std::make_unique<util::Thread<Impl>>("LocalFileSource")) {
+}
+
+LocalFileSource::~LocalFileSource() = default;
+
+std::unique_ptr<AsyncRequest> LocalFileSource::request(const Resource& resource, Callback callback) {
+ auto req = std::make_unique<FileSourceRequest>(std::move(callback));
+
+ impl->actor().invoke(&Impl::request, resource.url, req->actor());
+
+ return std::move(req);
+}
+
+bool LocalFileSource::acceptsURL(const std::string& url) {
+ return 0 == url.rfind(fileProtocol, 0);
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/storage/offline.cpp b/platform/default/src/mbgl/storage/offline.cpp
new file mode 100644
index 0000000000..e1ec0acb31
--- /dev/null
+++ b/platform/default/src/mbgl/storage/offline.cpp
@@ -0,0 +1,158 @@
+#include <mbgl/storage/offline.hpp>
+#include <mbgl/util/tileset.hpp>
+#include <mbgl/util/projection.hpp>
+
+#include <mapbox/geojson.hpp>
+#include <mapbox/geojson/rapidjson.hpp>
+
+#include <rapidjson/document.h>
+#include <rapidjson/stringbuffer.h>
+#include <rapidjson/writer.h>
+
+#include <cmath>
+
+namespace mbgl {
+
+// OfflineTilePyramidRegionDefinition
+
+OfflineTilePyramidRegionDefinition::OfflineTilePyramidRegionDefinition(
+ std::string styleURL_, LatLngBounds bounds_, double minZoom_, double maxZoom_, float pixelRatio_)
+ : styleURL(std::move(styleURL_)),
+ bounds(std::move(bounds_)),
+ minZoom(minZoom_),
+ maxZoom(maxZoom_),
+ pixelRatio(pixelRatio_) {
+ if (minZoom < 0 || maxZoom < 0 || maxZoom < minZoom || pixelRatio < 0 ||
+ !std::isfinite(minZoom) || std::isnan(maxZoom) || !std::isfinite(pixelRatio)) {
+ throw std::invalid_argument("Invalid offline region definition");
+ }
+}
+
+
+// OfflineGeometryRegionDefinition
+
+OfflineGeometryRegionDefinition::OfflineGeometryRegionDefinition(std::string styleURL_, Geometry<double> geometry_, double minZoom_, double maxZoom_, float pixelRatio_)
+ : styleURL(styleURL_)
+ , geometry(std::move(geometry_))
+ , minZoom(minZoom_)
+ , maxZoom(maxZoom_)
+ , pixelRatio(pixelRatio_) {
+ if (minZoom < 0 || maxZoom < 0 || maxZoom < minZoom || pixelRatio < 0 ||
+ !std::isfinite(minZoom) || std::isnan(maxZoom) || !std::isfinite(pixelRatio)) {
+ throw std::invalid_argument("Invalid offline region definition");
+ }
+}
+
+OfflineRegionDefinition decodeOfflineRegionDefinition(const std::string& region) {
+ rapidjson::GenericDocument<rapidjson::UTF8<>, rapidjson::CrtAllocator> doc;
+ doc.Parse<0>(region.c_str());
+
+ // validation
+
+ auto hasValidBounds = [&] {
+ return doc.HasMember("bounds") && doc["bounds"].IsArray() && doc["bounds"].Size() == 4
+ && doc["bounds"][0].IsDouble() && doc["bounds"][1].IsDouble()
+ && doc["bounds"][2].IsDouble() && doc["bounds"][3].IsDouble();
+ };
+
+ auto hasValidGeometry = [&] {
+ return doc.HasMember("geometry") && doc["geometry"].IsObject();
+ };
+
+ if (doc.HasParseError()
+ || !doc.HasMember("style_url") || !doc["style_url"].IsString()
+ || !(hasValidBounds() || hasValidGeometry())
+ || !doc.HasMember("min_zoom") || !doc["min_zoom"].IsDouble()
+ || (doc.HasMember("max_zoom") && !doc["max_zoom"].IsDouble())
+ || !doc.HasMember("pixel_ratio") || !doc["pixel_ratio"].IsDouble()) {
+ throw std::runtime_error("Malformed offline region definition");
+ }
+
+ // Common properties
+
+ std::string styleURL { doc["style_url"].GetString(), doc["style_url"].GetStringLength() };
+ double minZoom = doc["min_zoom"].GetDouble();
+ double maxZoom = doc.HasMember("max_zoom") ? doc["max_zoom"].GetDouble() : INFINITY;
+ float pixelRatio = doc["pixel_ratio"].GetDouble();
+
+ if (doc.HasMember("bounds")) {
+ return OfflineTilePyramidRegionDefinition{
+ styleURL,
+ LatLngBounds::hull(
+ LatLng(doc["bounds"][0].GetDouble(), doc["bounds"][1].GetDouble()),
+ LatLng(doc["bounds"][2].GetDouble(), doc["bounds"][3].GetDouble())),
+ minZoom, maxZoom, pixelRatio };
+ } else {
+ return OfflineGeometryRegionDefinition{
+ styleURL,
+ mapbox::geojson::convert<Geometry<double>>(doc["geometry"].GetObject()),
+ minZoom, maxZoom, pixelRatio };
+ };
+
+}
+
+std::string encodeOfflineRegionDefinition(const OfflineRegionDefinition& region) {
+ rapidjson::GenericDocument<rapidjson::UTF8<>, rapidjson::CrtAllocator> doc;
+ doc.SetObject();
+
+ // Encode common properties
+ region.match([&](auto& _region) {
+ doc.AddMember("style_url", rapidjson::StringRef(_region.styleURL.data(), _region.styleURL.length()), doc.GetAllocator());
+ doc.AddMember("min_zoom", _region.minZoom, doc.GetAllocator());
+ if (std::isfinite(_region.maxZoom)) {
+ doc.AddMember("max_zoom", _region.maxZoom, doc.GetAllocator());
+ }
+
+ doc.AddMember("pixel_ratio", _region.pixelRatio, doc.GetAllocator());
+ });
+
+ // Encode specific properties
+ region.match(
+ [&] (const OfflineTilePyramidRegionDefinition& _region) {
+ rapidjson::GenericValue<rapidjson::UTF8<>, rapidjson::CrtAllocator> bounds(rapidjson::kArrayType);
+ bounds.PushBack(_region.bounds.south(), doc.GetAllocator());
+ bounds.PushBack(_region.bounds.west(), doc.GetAllocator());
+ bounds.PushBack(_region.bounds.north(), doc.GetAllocator());
+ bounds.PushBack(_region.bounds.east(), doc.GetAllocator());
+ doc.AddMember("bounds", bounds, doc.GetAllocator());
+
+ },
+ [&] (const OfflineGeometryRegionDefinition& _region) {
+ doc.AddMember("geometry", mapbox::geojson::convert(_region.geometry, doc.GetAllocator()), doc.GetAllocator());
+
+ }
+ );
+
+ rapidjson::StringBuffer buffer;
+ rapidjson::Writer<rapidjson::StringBuffer> writer(buffer);
+ doc.Accept(writer);
+
+ return buffer.GetString();
+}
+
+
+// OfflineRegion
+
+OfflineRegion::OfflineRegion(int64_t id_,
+ OfflineRegionDefinition definition_,
+ OfflineRegionMetadata metadata_)
+ : id(id_),
+ definition(std::move(definition_)),
+ metadata(std::move(metadata_)) {
+}
+
+OfflineRegion::OfflineRegion(OfflineRegion&&) = default;
+OfflineRegion::~OfflineRegion() = default;
+
+const OfflineRegionDefinition& OfflineRegion::getDefinition() const {
+ return definition;
+}
+
+const OfflineRegionMetadata& OfflineRegion::getMetadata() const {
+ return metadata;
+}
+
+int64_t OfflineRegion::getID() const {
+ return id;
+}
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/storage/offline_database.cpp b/platform/default/src/mbgl/storage/offline_database.cpp
new file mode 100644
index 0000000000..7732076991
--- /dev/null
+++ b/platform/default/src/mbgl/storage/offline_database.cpp
@@ -0,0 +1,1129 @@
+#include <mbgl/storage/offline_database.hpp>
+#include <mbgl/storage/response.hpp>
+#include <mbgl/storage/sqlite3.hpp>
+#include <mbgl/util/compression.hpp>
+#include <mbgl/util/io.hpp>
+#include <mbgl/util/string.hpp>
+#include <mbgl/util/chrono.hpp>
+#include <mbgl/util/logging.hpp>
+
+#include <mbgl/storage/offline_schema.hpp>
+#include <mbgl/storage/merge_sideloaded.hpp>
+
+
+namespace mbgl {
+
+OfflineDatabase::OfflineDatabase(std::string path_, uint64_t maximumCacheSize_)
+ : path(std::move(path_)),
+ maximumCacheSize(maximumCacheSize_) {
+ try {
+ initialize();
+ } catch (const util::IOException& ex) {
+ handleError(ex, "open database");
+ } catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "open database");
+ }
+ // Assume that we can't open the database right now and work with an empty database object.
+}
+
+OfflineDatabase::~OfflineDatabase() {
+ // Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we
+ // can't throw anything.
+ try {
+ statements.clear();
+ db.reset();
+ } catch (const util::IOException& ex) {
+ handleError(ex, "close database");
+ } catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "close database");
+ }
+}
+
+void OfflineDatabase::initialize() {
+ assert(!db);
+ assert(statements.empty());
+
+ db = std::make_unique<mapbox::sqlite::Database>(
+ mapbox::sqlite::Database::open(path, mapbox::sqlite::ReadWriteCreate));
+ db->setBusyTimeout(Milliseconds::max());
+ db->exec("PRAGMA foreign_keys = ON");
+
+ const auto userVersion = getPragma<int64_t>("PRAGMA user_version");
+ switch (userVersion) {
+ case 0:
+ case 1:
+ // Newly created database, or old cache-only database; remove old table if it exists.
+ removeOldCacheTable();
+ createSchema();
+ return;
+ case 2:
+ migrateToVersion3();
+ // fall through
+ case 3:
+ // Removed migration, see below.
+ // fall through
+ case 4:
+ migrateToVersion5();
+ // fall through
+ case 5:
+ migrateToVersion6();
+ // fall through
+ case 6:
+ // Happy path; we're done
+ return;
+ default:
+ // Downgrade: delete the database and try to reinitialize.
+ removeExisting();
+ initialize();
+ }
+}
+
+void OfflineDatabase::handleError(const mapbox::sqlite::Exception& ex, const char* action) {
+ if (ex.code == mapbox::sqlite::ResultCode::NotADB ||
+ ex.code == mapbox::sqlite::ResultCode::Corrupt ||
+ (ex.code == mapbox::sqlite::ResultCode::ReadOnly &&
+ ex.extendedCode == mapbox::sqlite::ExtendedResultCode::ReadOnlyDBMoved)) {
+ // The database was corruped, moved away, or deleted. We're going to start fresh with a
+ // clean slate for the next operation.
+ Log::Error(Event::Database, static_cast<int>(ex.code), "Can't %s: %s", action, ex.what());
+ try {
+ removeExisting();
+ } catch (const util::IOException& ioEx) {
+ handleError(ioEx, action);
+ }
+ } else {
+ // We treat the error as temporary, and pretend we have an inaccessible DB.
+ Log::Warning(Event::Database, static_cast<int>(ex.code), "Can't %s: %s", action, ex.what());
+ }
+}
+
+void OfflineDatabase::handleError(const util::IOException& ex, const char* action) {
+ // We failed to delete the database file.
+ Log::Error(Event::Database, ex.code, "Can't %s: %s", action, ex.what());
+}
+
+void OfflineDatabase::removeExisting() {
+ Log::Warning(Event::Database, "Removing existing incompatible offline database");
+
+ statements.clear();
+ db.reset();
+
+ util::deleteFile(path);
+}
+
+void OfflineDatabase::removeOldCacheTable() {
+ assert(db);
+ db->exec("DROP TABLE IF EXISTS http_cache");
+ db->exec("VACUUM");
+}
+
+void OfflineDatabase::createSchema() {
+ assert(db);
+ db->exec("PRAGMA auto_vacuum = INCREMENTAL");
+ db->exec("PRAGMA journal_mode = DELETE");
+ db->exec("PRAGMA synchronous = FULL");
+ mapbox::sqlite::Transaction transaction(*db);
+ db->exec(offlineDatabaseSchema);
+ db->exec("PRAGMA user_version = 6");
+ transaction.commit();
+}
+
+void OfflineDatabase::migrateToVersion3() {
+ assert(db);
+ db->exec("PRAGMA auto_vacuum = INCREMENTAL");
+ db->exec("VACUUM");
+ db->exec("PRAGMA user_version = 3");
+}
+
+// Schema version 4 was WAL journal + NORMAL sync. It was reverted during pre-
+// release development and the migration was removed entirely to avoid potential
+// conflicts from quickly (and needlessly) switching journal and sync modes.
+//
+// See: https://github.com/mapbox/mapbox-gl-native/pull/6320
+
+void OfflineDatabase::migrateToVersion5() {
+ assert(db);
+ db->exec("PRAGMA journal_mode = DELETE");
+ db->exec("PRAGMA synchronous = FULL");
+ db->exec("PRAGMA user_version = 5");
+}
+
+void OfflineDatabase::migrateToVersion6() {
+ assert(db);
+ mapbox::sqlite::Transaction transaction(*db);
+ db->exec("ALTER TABLE resources ADD COLUMN must_revalidate INTEGER NOT NULL DEFAULT 0");
+ db->exec("ALTER TABLE tiles ADD COLUMN must_revalidate INTEGER NOT NULL DEFAULT 0");
+ db->exec("PRAGMA user_version = 6");
+ transaction.commit();
+}
+
+mapbox::sqlite::Statement& OfflineDatabase::getStatement(const char* sql) {
+ if (!db) {
+ initialize();
+ }
+ auto it = statements.find(sql);
+ if (it == statements.end()) {
+ it = statements.emplace(sql, std::make_unique<mapbox::sqlite::Statement>(*db, sql)).first;
+ }
+ return *it->second;
+}
+
+optional<Response> OfflineDatabase::get(const Resource& resource) try {
+ auto result = getInternal(resource);
+ return result ? optional<Response>{ result->first } : nullopt;
+} catch (const util::IOException& ex) {
+ handleError(ex, "read resource");
+ return nullopt;
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "read resource");
+ return nullopt;
+}
+
+optional<std::pair<Response, uint64_t>> OfflineDatabase::getInternal(const Resource& resource) {
+ if (resource.kind == Resource::Kind::Tile) {
+ assert(resource.tileData);
+ return getTile(*resource.tileData);
+ } else {
+ return getResource(resource);
+ }
+}
+
+optional<int64_t> OfflineDatabase::hasInternal(const Resource& resource) {
+ if (resource.kind == Resource::Kind::Tile) {
+ assert(resource.tileData);
+ return hasTile(*resource.tileData);
+ } else {
+ return hasResource(resource);
+ }
+}
+
+std::pair<bool, uint64_t> OfflineDatabase::put(const Resource& resource, const Response& response) try {
+ if (!db) {
+ initialize();
+ }
+ mapbox::sqlite::Transaction transaction(*db, mapbox::sqlite::Transaction::Immediate);
+ auto result = putInternal(resource, response, true);
+ transaction.commit();
+ return result;
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "write resource");
+ return { false, 0 };
+}
+
+std::pair<bool, uint64_t> OfflineDatabase::putInternal(const Resource& resource, const Response& response, bool evict_) {
+ if (response.error) {
+ return { false, 0 };
+ }
+
+ std::string compressedData;
+ bool compressed = false;
+ uint64_t size = 0;
+
+ if (response.data) {
+ compressedData = util::compress(*response.data);
+ compressed = compressedData.size() < response.data->size();
+ size = compressed ? compressedData.size() : response.data->size();
+ }
+
+ if (evict_ && !evict(size)) {
+ Log::Info(Event::Database, "Unable to make space for entry");
+ return { false, 0 };
+ }
+
+ bool inserted;
+
+ if (resource.kind == Resource::Kind::Tile) {
+ assert(resource.tileData);
+ inserted = putTile(*resource.tileData, response,
+ compressed ? compressedData : response.data ? *response.data : "",
+ compressed);
+ } else {
+ inserted = putResource(resource, response,
+ compressed ? compressedData : response.data ? *response.data : "",
+ compressed);
+ }
+
+ return { inserted, size };
+}
+
+optional<std::pair<Response, uint64_t>> OfflineDatabase::getResource(const Resource& resource) {
+ // Update accessed timestamp used for LRU eviction.
+ try {
+ mapbox::sqlite::Query accessedQuery{ getStatement("UPDATE resources SET accessed = ?1 WHERE url = ?2") };
+ accessedQuery.bind(1, util::now());
+ accessedQuery.bind(2, resource.url);
+ accessedQuery.run();
+ } catch (const mapbox::sqlite::Exception& ex) {
+ if (ex.code == mapbox::sqlite::ResultCode::NotADB ||
+ ex.code == mapbox::sqlite::ResultCode::Corrupt) {
+ throw;
+ }
+
+ // If we don't have any indication that the database is corrupt, continue as usual.
+ Log::Warning(Event::Database, static_cast<int>(ex.code), "Can't update timestamp: %s", ex.what());
+ }
+
+ // clang-format off
+ mapbox::sqlite::Query query{ getStatement(
+ // 0 1 2 3 4 5
+ "SELECT etag, expires, must_revalidate, modified, data, compressed "
+ "FROM resources "
+ "WHERE url = ?") };
+ // clang-format on
+
+ query.bind(1, resource.url);
+
+ if (!query.run()) {
+ return nullopt;
+ }
+
+ Response response;
+ uint64_t size = 0;
+
+ response.etag = query.get<optional<std::string>>(0);
+ response.expires = query.get<optional<Timestamp>>(1);
+ response.mustRevalidate = query.get<bool>(2);
+ response.modified = query.get<optional<Timestamp>>(3);
+
+ auto data = query.get<optional<std::string>>(4);
+ if (!data) {
+ response.noContent = true;
+ } else if (query.get<bool>(5)) {
+ response.data = std::make_shared<std::string>(util::decompress(*data));
+ size = data->length();
+ } else {
+ response.data = std::make_shared<std::string>(*data);
+ size = data->length();
+ }
+
+ return std::make_pair(response, size);
+}
+
+optional<int64_t> OfflineDatabase::hasResource(const Resource& resource) {
+ mapbox::sqlite::Query query{ getStatement("SELECT length(data) FROM resources WHERE url = ?") };
+ query.bind(1, resource.url);
+ if (!query.run()) {
+ return nullopt;
+ }
+
+ return query.get<optional<int64_t>>(0);
+}
+
+bool OfflineDatabase::putResource(const Resource& resource,
+ const Response& response,
+ const std::string& data,
+ bool compressed) {
+ if (response.notModified) {
+ // clang-format off
+ mapbox::sqlite::Query notModifiedQuery{ getStatement(
+ "UPDATE resources "
+ "SET accessed = ?1, "
+ " expires = ?2, "
+ " must_revalidate = ?3 "
+ "WHERE url = ?4 ") };
+ // clang-format on
+
+ notModifiedQuery.bind(1, util::now());
+ notModifiedQuery.bind(2, response.expires);
+ notModifiedQuery.bind(3, response.mustRevalidate);
+ notModifiedQuery.bind(4, resource.url);
+ notModifiedQuery.run();
+ return false;
+ }
+
+ // We can't use REPLACE because it would change the id value.
+ // clang-format off
+ mapbox::sqlite::Query updateQuery{ getStatement(
+ "UPDATE resources "
+ "SET kind = ?1, "
+ " etag = ?2, "
+ " expires = ?3, "
+ " must_revalidate = ?4, "
+ " modified = ?5, "
+ " accessed = ?6, "
+ " data = ?7, "
+ " compressed = ?8 "
+ "WHERE url = ?9 ") };
+ // clang-format on
+
+ updateQuery.bind(1, int(resource.kind));
+ updateQuery.bind(2, response.etag);
+ updateQuery.bind(3, response.expires);
+ updateQuery.bind(4, response.mustRevalidate);
+ updateQuery.bind(5, response.modified);
+ updateQuery.bind(6, util::now());
+ updateQuery.bind(9, resource.url);
+
+ if (response.noContent) {
+ updateQuery.bind(7, nullptr);
+ updateQuery.bind(8, false);
+ } else {
+ updateQuery.bindBlob(7, data.data(), data.size(), false);
+ updateQuery.bind(8, compressed);
+ }
+
+ updateQuery.run();
+ if (updateQuery.changes() != 0) {
+ return false;
+ }
+
+ // clang-format off
+ mapbox::sqlite::Query insertQuery{ getStatement(
+ "INSERT INTO resources (url, kind, etag, expires, must_revalidate, modified, accessed, data, compressed) "
+ "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) ") };
+ // clang-format on
+
+ insertQuery.bind(1, resource.url);
+ insertQuery.bind(2, int(resource.kind));
+ insertQuery.bind(3, response.etag);
+ insertQuery.bind(4, response.expires);
+ insertQuery.bind(5, response.mustRevalidate);
+ insertQuery.bind(6, response.modified);
+ insertQuery.bind(7, util::now());
+
+ if (response.noContent) {
+ insertQuery.bind(8, nullptr);
+ insertQuery.bind(9, false);
+ } else {
+ insertQuery.bindBlob(8, data.data(), data.size(), false);
+ insertQuery.bind(9, compressed);
+ }
+
+ insertQuery.run();
+
+ return true;
+}
+
+optional<std::pair<Response, uint64_t>> OfflineDatabase::getTile(const Resource::TileData& tile) {
+ // Update accessed timestamp used for LRU eviction.
+ try {
+ // clang-format off
+ mapbox::sqlite::Query accessedQuery{ getStatement(
+ "UPDATE tiles "
+ "SET accessed = ?1 "
+ "WHERE url_template = ?2 "
+ " AND pixel_ratio = ?3 "
+ " AND x = ?4 "
+ " AND y = ?5 "
+ " AND z = ?6 ") };
+ // clang-format on
+
+ accessedQuery.bind(1, util::now());
+ accessedQuery.bind(2, tile.urlTemplate);
+ accessedQuery.bind(3, tile.pixelRatio);
+ accessedQuery.bind(4, tile.x);
+ accessedQuery.bind(5, tile.y);
+ accessedQuery.bind(6, tile.z);
+ accessedQuery.run();
+ } catch (const mapbox::sqlite::Exception& ex) {
+ if (ex.code == mapbox::sqlite::ResultCode::NotADB || ex.code == mapbox::sqlite::ResultCode::Corrupt) {
+ throw;
+ }
+
+ // If we don't have any indication that the database is corrupt, continue as usual.
+ Log::Warning(Event::Database, static_cast<int>(ex.code), "Can't update timestamp: %s", ex.what());
+ }
+
+ // clang-format off
+ mapbox::sqlite::Query query{ getStatement(
+ // 0 1 2, 3, 4, 5
+ "SELECT etag, expires, must_revalidate, modified, data, compressed "
+ "FROM tiles "
+ "WHERE url_template = ?1 "
+ " AND pixel_ratio = ?2 "
+ " AND x = ?3 "
+ " AND y = ?4 "
+ " AND z = ?5 ") };
+ // clang-format on
+
+ query.bind(1, tile.urlTemplate);
+ query.bind(2, tile.pixelRatio);
+ query.bind(3, tile.x);
+ query.bind(4, tile.y);
+ query.bind(5, tile.z);
+
+ if (!query.run()) {
+ return nullopt;
+ }
+
+ Response response;
+ uint64_t size = 0;
+
+ response.etag = query.get<optional<std::string>>(0);
+ response.expires = query.get<optional<Timestamp>>(1);
+ response.mustRevalidate = query.get<bool>(2);
+ response.modified = query.get<optional<Timestamp>>(3);
+
+ optional<std::string> data = query.get<optional<std::string>>(4);
+ if (!data) {
+ response.noContent = true;
+ } else if (query.get<bool>(5)) {
+ response.data = std::make_shared<std::string>(util::decompress(*data));
+ size = data->length();
+ } else {
+ response.data = std::make_shared<std::string>(*data);
+ size = data->length();
+ }
+
+ return std::make_pair(response, size);
+}
+
+optional<int64_t> OfflineDatabase::hasTile(const Resource::TileData& tile) {
+ // clang-format off
+ mapbox::sqlite::Query size{ getStatement(
+ "SELECT length(data) "
+ "FROM tiles "
+ "WHERE url_template = ?1 "
+ " AND pixel_ratio = ?2 "
+ " AND x = ?3 "
+ " AND y = ?4 "
+ " AND z = ?5 ") };
+ // clang-format on
+
+ size.bind(1, tile.urlTemplate);
+ size.bind(2, tile.pixelRatio);
+ size.bind(3, tile.x);
+ size.bind(4, tile.y);
+ size.bind(5, tile.z);
+
+ if (!size.run()) {
+ return nullopt;
+ }
+
+ return size.get<optional<int64_t>>(0);
+}
+
+bool OfflineDatabase::putTile(const Resource::TileData& tile,
+ const Response& response,
+ const std::string& data,
+ bool compressed) {
+ if (response.notModified) {
+ // clang-format off
+ mapbox::sqlite::Query notModifiedQuery{ getStatement(
+ "UPDATE tiles "
+ "SET accessed = ?1, "
+ " expires = ?2, "
+ " must_revalidate = ?3 "
+ "WHERE url_template = ?4 "
+ " AND pixel_ratio = ?5 "
+ " AND x = ?6 "
+ " AND y = ?7 "
+ " AND z = ?8 ") };
+ // clang-format on
+
+ notModifiedQuery.bind(1, util::now());
+ notModifiedQuery.bind(2, response.expires);
+ notModifiedQuery.bind(3, response.mustRevalidate);
+ notModifiedQuery.bind(4, tile.urlTemplate);
+ notModifiedQuery.bind(5, tile.pixelRatio);
+ notModifiedQuery.bind(6, tile.x);
+ notModifiedQuery.bind(7, tile.y);
+ notModifiedQuery.bind(8, tile.z);
+ notModifiedQuery.run();
+ return false;
+ }
+
+ // We can't use REPLACE because it would change the id value.
+
+ // clang-format off
+ mapbox::sqlite::Query updateQuery{ getStatement(
+ "UPDATE tiles "
+ "SET modified = ?1, "
+ " etag = ?2, "
+ " expires = ?3, "
+ " must_revalidate = ?4, "
+ " accessed = ?5, "
+ " data = ?6, "
+ " compressed = ?7 "
+ "WHERE url_template = ?8 "
+ " AND pixel_ratio = ?9 "
+ " AND x = ?10 "
+ " AND y = ?11 "
+ " AND z = ?12 ") };
+ // clang-format on
+
+ updateQuery.bind(1, response.modified);
+ updateQuery.bind(2, response.etag);
+ updateQuery.bind(3, response.expires);
+ updateQuery.bind(4, response.mustRevalidate);
+ updateQuery.bind(5, util::now());
+ updateQuery.bind(8, tile.urlTemplate);
+ updateQuery.bind(9, tile.pixelRatio);
+ updateQuery.bind(10, tile.x);
+ updateQuery.bind(11, tile.y);
+ updateQuery.bind(12, tile.z);
+
+ if (response.noContent) {
+ updateQuery.bind(6, nullptr);
+ updateQuery.bind(7, false);
+ } else {
+ updateQuery.bindBlob(6, data.data(), data.size(), false);
+ updateQuery.bind(7, compressed);
+ }
+
+ updateQuery.run();
+ if (updateQuery.changes() != 0) {
+ return false;
+ }
+
+ // clang-format off
+ mapbox::sqlite::Query insertQuery{ getStatement(
+ "INSERT INTO tiles (url_template, pixel_ratio, x, y, z, modified, must_revalidate, etag, expires, accessed, data, compressed) "
+ "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)") };
+ // clang-format on
+
+ insertQuery.bind(1, tile.urlTemplate);
+ insertQuery.bind(2, tile.pixelRatio);
+ insertQuery.bind(3, tile.x);
+ insertQuery.bind(4, tile.y);
+ insertQuery.bind(5, tile.z);
+ insertQuery.bind(6, response.modified);
+ insertQuery.bind(7, response.mustRevalidate);
+ insertQuery.bind(8, response.etag);
+ insertQuery.bind(9, response.expires);
+ insertQuery.bind(10, util::now());
+
+ if (response.noContent) {
+ insertQuery.bind(11, nullptr);
+ insertQuery.bind(12, false);
+ } else {
+ insertQuery.bindBlob(11, data.data(), data.size(), false);
+ insertQuery.bind(12, compressed);
+ }
+
+ insertQuery.run();
+
+ return true;
+}
+
+expected<OfflineRegions, std::exception_ptr> OfflineDatabase::listRegions() try {
+ mapbox::sqlite::Query query{ getStatement("SELECT id, definition, description FROM regions") };
+ OfflineRegions result;
+ while (query.run()) {
+ const auto id = query.get<int64_t>(0);
+ const auto definition = query.get<std::string>(1);
+ const auto description = query.get<std::vector<uint8_t>>(2);
+ try {
+ // Construct, then move because this constructor is private.
+ OfflineRegion region(id, decodeOfflineRegionDefinition(definition), description);
+ result.emplace_back(std::move(region));
+ } catch (const std::exception& ex) {
+ // Catch errors from malformed offline region definitions
+ // and skip them.
+ Log::Error(Event::General, "%s", ex.what());
+ }
+ }
+ // Explicit move to avoid triggering the copy constructor.
+ return { std::move(result) };
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "list regions");
+ return unexpected<std::exception_ptr>(std::current_exception());
+}
+
+expected<OfflineRegion, std::exception_ptr>
+OfflineDatabase::createRegion(const OfflineRegionDefinition& definition,
+ const OfflineRegionMetadata& metadata) try {
+ // clang-format off
+ mapbox::sqlite::Query query{ getStatement(
+ "INSERT INTO regions (definition, description) "
+ "VALUES (?1, ?2) ") };
+ // clang-format on
+
+ query.bind(1, encodeOfflineRegionDefinition(definition));
+ query.bindBlob(2, metadata);
+ query.run();
+ return OfflineRegion(query.lastInsertRowId(), definition, metadata);
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "create region");
+ return unexpected<std::exception_ptr>(std::current_exception());
+}
+
+expected<OfflineRegions, std::exception_ptr>
+OfflineDatabase::mergeDatabase(const std::string& sideDatabasePath) {
+ try {
+ // clang-format off
+ mapbox::sqlite::Query query{ getStatement("ATTACH DATABASE ?1 AS side") };
+ // clang-format on
+
+ query.bind(1, sideDatabasePath);
+ query.run();
+ } catch (const mapbox::sqlite::Exception& ex) {
+ Log::Error(Event::Database, static_cast<int>(ex.code), "Can't attach database (%s) for merge: %s", sideDatabasePath.c_str(), ex.what());
+ return unexpected<std::exception_ptr>(std::current_exception());
+ }
+ try {
+ // Support sideloaded databases at user_version = 6. Future schema version
+ // changes will need to implement migration paths for sideloaded databases at
+ // version 6.
+ auto sideUserVersion = static_cast<int>(getPragma<int64_t>("PRAGMA side.user_version"));
+ const auto mainUserVersion = getPragma<int64_t>("PRAGMA user_version");
+ if (sideUserVersion < 6 || sideUserVersion != mainUserVersion) {
+ throw std::runtime_error("Merge database has incorrect user_version");
+ }
+
+ auto currentTileCount = getOfflineMapboxTileCount();
+ // clang-format off
+ mapbox::sqlite::Query queryTiles{ getStatement(
+ "SELECT COUNT(DISTINCT st.id) "
+ "FROM side.tiles st "
+ //only consider region tiles, and not ambient tiles.
+ "JOIN side.region_tiles srt ON srt.tile_id = st.id "
+ "LEFT JOIN tiles t ON st.url_template = t.url_template AND "
+ "st.pixel_ratio = t.pixel_ratio AND "
+ "st.z = t.z AND "
+ "st.x = t.x AND "
+ "st.y = t.y "
+ "WHERE t.id IS NULL "
+ "AND st.url_template LIKE 'mapbox://%' ") };
+ // clang-format on
+ queryTiles.run();
+ auto countOfTilesToMerge = queryTiles.get<int64_t>(0);
+ if ((countOfTilesToMerge + currentTileCount) > offlineMapboxTileCountLimit) {
+ throw MapboxTileLimitExceededException();
+ }
+ queryTiles.reset();
+
+ mapbox::sqlite::Transaction transaction(*db);
+ db->exec(mergeSideloadedDatabaseSQL);
+ transaction.commit();
+
+ // clang-format off
+ mapbox::sqlite::Query queryRegions{ getStatement(
+ "SELECT DISTINCT r.id, r.definition, r.description "
+ "FROM side.regions sr "
+ "JOIN regions r ON sr.definition = r.definition AND sr.description IS r.description") };
+ // clang-format on
+
+ OfflineRegions result;
+ while (queryRegions.run()) {
+ // Construct, then move because this constructor is private.
+ OfflineRegion region(queryRegions.get<int64_t>(0),
+ decodeOfflineRegionDefinition(queryRegions.get<std::string>(1)),
+ queryRegions.get<std::vector<uint8_t>>(2));
+ result.emplace_back(std::move(region));
+ }
+ db->exec("DETACH DATABASE side");
+ // Explicit move to avoid triggering the copy constructor.
+ return { std::move(result) };
+ } catch (const std::runtime_error& ex) {
+ db->exec("DETACH DATABASE side");
+ Log::Error(Event::Database, "%s", ex.what());
+
+ return unexpected<std::exception_ptr>(std::current_exception());
+ }
+ return {};
+}
+
+expected<OfflineRegionMetadata, std::exception_ptr>
+OfflineDatabase::updateMetadata(const int64_t regionID, const OfflineRegionMetadata& metadata) try {
+ // clang-format off
+ mapbox::sqlite::Query query{ getStatement(
+ "UPDATE regions SET description = ?1 "
+ "WHERE id = ?2") };
+ // clang-format on
+ query.bindBlob(1, metadata);
+ query.bind(2, regionID);
+ query.run();
+
+ return metadata;
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "update region metadata");
+ return unexpected<std::exception_ptr>(std::current_exception());
+}
+
+std::exception_ptr OfflineDatabase::deleteRegion(OfflineRegion&& region) try {
+ {
+ mapbox::sqlite::Query query{ getStatement("DELETE FROM regions WHERE id = ?") };
+ query.bind(1, region.getID());
+ query.run();
+ }
+
+ evict(0);
+ assert(db);
+ db->exec("PRAGMA incremental_vacuum");
+
+ // Ensure that the cached offlineTileCount value is recalculated.
+ offlineMapboxTileCount = {};
+ return nullptr;
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "delete region");
+ return std::current_exception();
+}
+
+optional<std::pair<Response, uint64_t>> OfflineDatabase::getRegionResource(int64_t regionID, const Resource& resource) try {
+ auto response = getInternal(resource);
+
+ if (response) {
+ markUsed(regionID, resource);
+ }
+
+ return response;
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "read region resource");
+ return nullopt;
+}
+
+optional<int64_t> OfflineDatabase::hasRegionResource(int64_t regionID, const Resource& resource) try {
+ auto response = hasInternal(resource);
+
+ if (response) {
+ markUsed(regionID, resource);
+ }
+
+ return response;
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "query region resource");
+ return nullopt;
+}
+
+uint64_t OfflineDatabase::putRegionResource(int64_t regionID,
+ const Resource& resource,
+ const Response& response) try {
+ if (!db) {
+ initialize();
+ }
+ mapbox::sqlite::Transaction transaction(*db);
+ auto size = putRegionResourceInternal(regionID, resource, response);
+ transaction.commit();
+ return size;
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "write region resource");
+ return 0;
+}
+
+void OfflineDatabase::putRegionResources(int64_t regionID,
+ const std::list<std::tuple<Resource, Response>>& resources,
+ OfflineRegionStatus& status) try {
+ if (!db) {
+ initialize();
+ }
+ mapbox::sqlite::Transaction transaction(*db);
+
+ // Accumulate all statistics locally first before adding them to the OfflineRegionStatus object
+ // to ensure correctness when the transaction fails.
+ uint64_t completedResourceCount = 0;
+ uint64_t completedResourceSize = 0;
+ uint64_t completedTileCount = 0;
+ uint64_t completedTileSize = 0;
+
+ for (const auto& elem : resources) {
+ const auto& resource = std::get<0>(elem);
+ const auto& response = std::get<1>(elem);
+
+ try {
+ uint64_t resourceSize = putRegionResourceInternal(regionID, resource, response);
+ completedResourceCount++;
+ completedResourceSize += resourceSize;
+ if (resource.kind == Resource::Kind::Tile) {
+ completedTileCount += 1;
+ completedTileSize += resourceSize;
+ }
+ } catch (const MapboxTileLimitExceededException&) {
+ // Commit the rest of the batch and rethrow
+ transaction.commit();
+ throw;
+ }
+ }
+
+ // Commit the completed batch
+ transaction.commit();
+
+ status.completedResourceCount += completedResourceCount;
+ status.completedResourceSize += completedResourceSize;
+ status.completedTileCount += completedTileCount;
+ status.completedTileSize += completedTileSize;
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "write region resources");
+}
+
+uint64_t OfflineDatabase::putRegionResourceInternal(int64_t regionID, const Resource& resource, const Response& response) {
+ if (exceedsOfflineMapboxTileCountLimit(resource)) {
+ throw MapboxTileLimitExceededException();
+ }
+
+ uint64_t size = putInternal(resource, response, false).second;
+ bool previouslyUnused = markUsed(regionID, resource);
+
+ if (offlineMapboxTileCount
+ && resource.kind == Resource::Kind::Tile
+ && util::mapbox::isMapboxURL(resource.url)
+ && previouslyUnused) {
+ *offlineMapboxTileCount += 1;
+ }
+
+ return size;
+}
+
+bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) {
+ if (resource.kind == Resource::Kind::Tile) {
+ // clang-format off
+ mapbox::sqlite::Query insertQuery{ getStatement(
+ "INSERT OR IGNORE INTO region_tiles (region_id, tile_id) "
+ "SELECT ?1, tiles.id "
+ "FROM tiles "
+ "WHERE url_template = ?2 "
+ " AND pixel_ratio = ?3 "
+ " AND x = ?4 "
+ " AND y = ?5 "
+ " AND z = ?6 ") };
+ // clang-format on
+
+ const Resource::TileData& tile = *resource.tileData;
+ insertQuery.bind(1, regionID);
+ insertQuery.bind(2, tile.urlTemplate);
+ insertQuery.bind(3, tile.pixelRatio);
+ insertQuery.bind(4, tile.x);
+ insertQuery.bind(5, tile.y);
+ insertQuery.bind(6, tile.z);
+ insertQuery.run();
+
+ if (insertQuery.changes() == 0) {
+ return false;
+ }
+
+ // clang-format off
+ mapbox::sqlite::Query selectQuery{ getStatement(
+ "SELECT region_id "
+ "FROM region_tiles, tiles "
+ "WHERE region_id != ?1 "
+ " AND url_template = ?2 "
+ " AND pixel_ratio = ?3 "
+ " AND x = ?4 "
+ " AND y = ?5 "
+ " AND z = ?6 "
+ "LIMIT 1 ") };
+ // clang-format on
+
+ selectQuery.bind(1, regionID);
+ selectQuery.bind(2, tile.urlTemplate);
+ selectQuery.bind(3, tile.pixelRatio);
+ selectQuery.bind(4, tile.x);
+ selectQuery.bind(5, tile.y);
+ selectQuery.bind(6, tile.z);
+ return !selectQuery.run();
+ } else {
+ // clang-format off
+ mapbox::sqlite::Query insertQuery{ getStatement(
+ "INSERT OR IGNORE INTO region_resources (region_id, resource_id) "
+ "SELECT ?1, resources.id "
+ "FROM resources "
+ "WHERE resources.url = ?2 ") };
+ // clang-format on
+
+ insertQuery.bind(1, regionID);
+ insertQuery.bind(2, resource.url);
+ insertQuery.run();
+
+ if (insertQuery.changes() == 0) {
+ return false;
+ }
+
+ // clang-format off
+ mapbox::sqlite::Query selectQuery{ getStatement(
+ "SELECT region_id "
+ "FROM region_resources, resources "
+ "WHERE region_id != ?1 "
+ " AND resources.url = ?2 "
+ "LIMIT 1 ") };
+ // clang-format on
+
+ selectQuery.bind(1, regionID);
+ selectQuery.bind(2, resource.url);
+ return !selectQuery.run();
+ }
+}
+
+expected<OfflineRegionDefinition, std::exception_ptr> OfflineDatabase::getRegionDefinition(int64_t regionID) try {
+ mapbox::sqlite::Query query{ getStatement("SELECT definition FROM regions WHERE id = ?1") };
+ query.bind(1, regionID);
+ query.run();
+
+ return decodeOfflineRegionDefinition(query.get<std::string>(0));
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "load region");
+ return unexpected<std::exception_ptr>(std::current_exception());
+}
+
+expected<OfflineRegionStatus, std::exception_ptr> OfflineDatabase::getRegionCompletedStatus(int64_t regionID) try {
+ OfflineRegionStatus result;
+
+ std::tie(result.completedResourceCount, result.completedResourceSize)
+ = getCompletedResourceCountAndSize(regionID);
+ std::tie(result.completedTileCount, result.completedTileSize)
+ = getCompletedTileCountAndSize(regionID);
+
+ result.completedResourceCount += result.completedTileCount;
+ result.completedResourceSize += result.completedTileSize;
+
+ return result;
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "get region status");
+ return unexpected<std::exception_ptr>(std::current_exception());
+}
+
+std::pair<int64_t, int64_t> OfflineDatabase::getCompletedResourceCountAndSize(int64_t regionID) {
+ // clang-format off
+ mapbox::sqlite::Query query{ getStatement(
+ "SELECT COUNT(*), SUM(LENGTH(data)) "
+ "FROM region_resources, resources "
+ "WHERE region_id = ?1 "
+ "AND resource_id = resources.id ") };
+ // clang-format on
+ query.bind(1, regionID);
+ query.run();
+ return { query.get<int64_t>(0), query.get<int64_t>(1) };
+}
+
+std::pair<int64_t, int64_t> OfflineDatabase::getCompletedTileCountAndSize(int64_t regionID) {
+ // clang-format off
+ mapbox::sqlite::Query query{ getStatement(
+ "SELECT COUNT(*), SUM(LENGTH(data)) "
+ "FROM region_tiles, tiles "
+ "WHERE region_id = ?1 "
+ "AND tile_id = tiles.id ") };
+ // clang-format on
+ query.bind(1, regionID);
+ query.run();
+ return { query.get<int64_t>(0), query.get<int64_t>(1) };
+}
+
+template <class T>
+T OfflineDatabase::getPragma(const char* sql) {
+ mapbox::sqlite::Query query{ getStatement(sql) };
+ query.run();
+ return query.get<T>(0);
+}
+
+// Remove least-recently used resources and tiles until the used database size,
+// as calculated by multiplying the number of in-use pages by the page size, is
+// less than the maximum cache size. Returns false if this condition cannot be
+// satisfied.
+//
+// SQLite database never shrinks in size unless we call VACCUM. We here
+// are monitoring the soft limit (i.e. number of free pages in the file)
+// and as it approaches to the hard limit (i.e. the actual file size) we
+// delete an arbitrary number of old cache entries. The free pages approach saves
+// us from calling VACCUM or keeping a running total, which can be costly.
+bool OfflineDatabase::evict(uint64_t neededFreeSize) {
+ uint64_t pageSize = getPragma<int64_t>("PRAGMA page_size");
+ uint64_t pageCount = getPragma<int64_t>("PRAGMA page_count");
+
+ auto usedSize = [&] {
+ return pageSize * (pageCount - getPragma<int64_t>("PRAGMA freelist_count"));
+ };
+
+ // The addition of pageSize is a fudge factor to account for non `data` column
+ // size, and because pages can get fragmented on the database.
+ while (usedSize() + neededFreeSize + pageSize > maximumCacheSize) {
+ // clang-format off
+ mapbox::sqlite::Query accessedQuery{ getStatement(
+ "SELECT max(accessed) "
+ "FROM ( "
+ " SELECT accessed "
+ " FROM resources "
+ " LEFT JOIN region_resources "
+ " ON resource_id = resources.id "
+ " WHERE resource_id IS NULL "
+ " UNION ALL "
+ " SELECT accessed "
+ " FROM tiles "
+ " LEFT JOIN region_tiles "
+ " ON tile_id = tiles.id "
+ " WHERE tile_id IS NULL "
+ " ORDER BY accessed ASC LIMIT ?1 "
+ ") "
+ ) };
+ accessedQuery.bind(1, 50);
+ // clang-format on
+ if (!accessedQuery.run()) {
+ return false;
+ }
+ Timestamp accessed = accessedQuery.get<Timestamp>(0);
+
+ // clang-format off
+ mapbox::sqlite::Query resourceQuery{ getStatement(
+ "DELETE FROM resources "
+ "WHERE id IN ( "
+ " SELECT id FROM resources "
+ " LEFT JOIN region_resources "
+ " ON resource_id = resources.id "
+ " WHERE resource_id IS NULL "
+ " AND accessed <= ?1 "
+ ") ") };
+ // clang-format on
+ resourceQuery.bind(1, accessed);
+ resourceQuery.run();
+ const uint64_t resourceChanges = resourceQuery.changes();
+
+ // clang-format off
+ mapbox::sqlite::Query tileQuery{ getStatement(
+ "DELETE FROM tiles "
+ "WHERE id IN ( "
+ " SELECT id FROM tiles "
+ " LEFT JOIN region_tiles "
+ " ON tile_id = tiles.id "
+ " WHERE tile_id IS NULL "
+ " AND accessed <= ?1 "
+ ") ") };
+ // clang-format on
+ tileQuery.bind(1, accessed);
+ tileQuery.run();
+ const uint64_t tileChanges = tileQuery.changes();
+
+ // The cached value of offlineTileCount does not need to be updated
+ // here because only non-offline tiles can be removed by eviction.
+
+ if (resourceChanges == 0 && tileChanges == 0) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+void OfflineDatabase::setOfflineMapboxTileCountLimit(uint64_t limit) {
+ offlineMapboxTileCountLimit = limit;
+}
+
+uint64_t OfflineDatabase::getOfflineMapboxTileCountLimit() {
+ return offlineMapboxTileCountLimit;
+}
+
+bool OfflineDatabase::offlineMapboxTileCountLimitExceeded() {
+ return getOfflineMapboxTileCount() >= offlineMapboxTileCountLimit;
+}
+
+uint64_t OfflineDatabase::getOfflineMapboxTileCount() try {
+ // Calculating this on every call would be much simpler than caching and
+ // manually updating the value, but it would make offline downloads an O(n²)
+ // operation, because the database query below involves an index scan of
+ // region_tiles.
+
+ if (offlineMapboxTileCount) {
+ return *offlineMapboxTileCount;
+ }
+
+ // clang-format off
+ mapbox::sqlite::Query query{ getStatement(
+ "SELECT COUNT(DISTINCT id) "
+ "FROM region_tiles, tiles "
+ "WHERE tile_id = tiles.id "
+ "AND url_template LIKE 'mapbox://%' ") };
+ // clang-format on
+
+ query.run();
+
+ offlineMapboxTileCount = query.get<int64_t>(0);
+ return *offlineMapboxTileCount;
+} catch (const mapbox::sqlite::Exception& ex) {
+ handleError(ex, "get offline Mapbox tile count");
+ return std::numeric_limits<uint64_t>::max();
+}
+
+bool OfflineDatabase::exceedsOfflineMapboxTileCountLimit(const Resource& resource) {
+ return resource.kind == Resource::Kind::Tile
+ && util::mapbox::isMapboxURL(resource.url)
+ && offlineMapboxTileCountLimitExceeded();
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/storage/offline_download.cpp b/platform/default/src/mbgl/storage/offline_download.cpp
new file mode 100644
index 0000000000..c97797a5a2
--- /dev/null
+++ b/platform/default/src/mbgl/storage/offline_download.cpp
@@ -0,0 +1,453 @@
+#include <mbgl/storage/online_file_source.hpp>
+#include <mbgl/storage/offline_database.hpp>
+#include <mbgl/storage/offline_download.hpp>
+#include <mbgl/storage/resource.hpp>
+#include <mbgl/storage/response.hpp>
+#include <mbgl/storage/http_file_source.hpp>
+#include <mbgl/style/parser.hpp>
+#include <mbgl/style/sources/vector_source.hpp>
+#include <mbgl/style/sources/raster_source.hpp>
+#include <mbgl/style/sources/raster_dem_source.hpp>
+#include <mbgl/style/sources/geojson_source.hpp>
+#include <mbgl/style/sources/image_source.hpp>
+#include <mbgl/style/conversion/json.hpp>
+#include <mbgl/style/conversion/tileset.hpp>
+#include <mbgl/text/glyph.hpp>
+#include <mbgl/util/mapbox.hpp>
+#include <mbgl/util/run_loop.hpp>
+#include <mbgl/util/tile_cover.hpp>
+#include <mbgl/util/tileset.hpp>
+
+#include <set>
+
+namespace mbgl {
+
+using namespace style;
+
+// Generic functions
+
+template <class RegionDefinition>
+Range<uint8_t> coveringZoomRange(const RegionDefinition& definition,
+ style::SourceType type, uint16_t tileSize, const Range<uint8_t>& zoomRange) {
+ double minZ = std::max<double>(util::coveringZoomLevel(definition.minZoom, type, tileSize), zoomRange.min);
+ double maxZ = std::min<double>(util::coveringZoomLevel(definition.maxZoom, type, tileSize), zoomRange.max);
+
+ assert(minZ >= 0);
+ assert(maxZ >= 0);
+ assert(minZ < std::numeric_limits<uint8_t>::max());
+ assert(maxZ < std::numeric_limits<uint8_t>::max());
+ return { static_cast<uint8_t>(minZ), static_cast<uint8_t>(maxZ) };
+}
+
+template <class Geometry, class Fn>
+void tileCover(const Geometry& geometry, uint8_t z, Fn&& fn) {
+ util::TileCover cover(geometry, z);
+ while (cover.hasNext()) {
+ fn(cover.next()->canonical);
+ }
+}
+
+
+template <class Fn>
+void tileCover(const OfflineRegionDefinition& definition, style::SourceType type,
+ uint16_t tileSize, const Range<uint8_t>& zoomRange, Fn&& fn) {
+ const Range<uint8_t> clampedZoomRange =
+ definition.match([&](auto& reg) { return coveringZoomRange(reg, type, tileSize, zoomRange); });
+
+ for (uint8_t z = clampedZoomRange.min; z <= clampedZoomRange.max; z++) {
+ definition.match(
+ [&](const OfflineTilePyramidRegionDefinition& reg){ tileCover(reg.bounds, z, fn); },
+ [&](const OfflineGeometryRegionDefinition& reg){ tileCover(reg.geometry, z, fn); }
+ );
+ }
+}
+
+uint64_t tileCount(const OfflineRegionDefinition& definition, style::SourceType type,
+ uint16_t tileSize, const Range<uint8_t>& zoomRange) {
+
+ const Range<uint8_t> clampedZoomRange =
+ definition.match([&](auto& reg) { return coveringZoomRange(reg, type, tileSize, zoomRange); });
+
+ unsigned long result = 0;;
+ for (uint8_t z = clampedZoomRange.min; z <= clampedZoomRange.max; z++) {
+ result += definition.match(
+ [&](const OfflineTilePyramidRegionDefinition& reg){ return util::tileCount(reg.bounds, z); },
+ [&](const OfflineGeometryRegionDefinition& reg){ return util::tileCount(reg.geometry, z); }
+ );
+ }
+
+ return result;
+}
+
+// OfflineDownload
+
+OfflineDownload::OfflineDownload(int64_t id_,
+ OfflineRegionDefinition&& definition_,
+ OfflineDatabase& offlineDatabase_,
+ OnlineFileSource& onlineFileSource_)
+ : id(id_),
+ definition(definition_),
+ offlineDatabase(offlineDatabase_),
+ onlineFileSource(onlineFileSource_) {
+ setObserver(nullptr);
+}
+
+OfflineDownload::~OfflineDownload() = default;
+
+void OfflineDownload::setObserver(std::unique_ptr<OfflineRegionObserver> observer_) {
+ observer = observer_ ? std::move(observer_) : std::make_unique<OfflineRegionObserver>();
+}
+
+void OfflineDownload::setState(OfflineRegionDownloadState state) {
+ if (status.downloadState == state) {
+ return;
+ }
+
+ status.downloadState = state;
+
+ if (status.downloadState == OfflineRegionDownloadState::Active) {
+ activateDownload();
+ } else {
+ deactivateDownload();
+ }
+
+ observer->statusChanged(status);
+}
+
+OfflineRegionStatus OfflineDownload::getStatus() const {
+ if (status.downloadState == OfflineRegionDownloadState::Active) {
+ return status;
+ }
+
+ auto result = offlineDatabase.getRegionCompletedStatus(id);
+ if (!result) {
+ // We can't find this offline region because the database is unavailable, or the download
+ // does not exist.
+ return {};
+ }
+
+ result->requiredResourceCount++;
+ optional<Response> styleResponse =
+ offlineDatabase.get(Resource::style(definition.match([](auto& reg){ return reg.styleURL; })));
+ if (!styleResponse) {
+ return *result;
+ }
+
+ style::Parser parser;
+ parser.parse(*styleResponse->data);
+
+ result->requiredResourceCountIsPrecise = true;
+
+ for (const auto& source : parser.sources) {
+ SourceType type = source->getType();
+
+ auto handleTiledSource = [&] (const variant<std::string, Tileset>& urlOrTileset, const uint16_t tileSize) {
+ if (urlOrTileset.is<Tileset>()) {
+ result->requiredResourceCount +=
+ tileCount(definition, type, tileSize, urlOrTileset.get<Tileset>().zoomRange);
+ } else {
+ result->requiredResourceCount += 1;
+ const auto& url = urlOrTileset.get<std::string>();
+ optional<Response> sourceResponse = offlineDatabase.get(Resource::source(url));
+ if (sourceResponse) {
+ style::conversion::Error error;
+ optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(*sourceResponse->data, error);
+ if (tileset) {
+ result->requiredResourceCount +=
+ tileCount(definition, type, tileSize, (*tileset).zoomRange);
+ }
+ } else {
+ result->requiredResourceCountIsPrecise = false;
+ }
+ }
+ };
+
+ switch (type) {
+ case SourceType::Vector: {
+ const auto& vectorSource = *source->as<VectorSource>();
+ handleTiledSource(vectorSource.getURLOrTileset(), util::tileSize);
+ break;
+ }
+
+ case SourceType::Raster: {
+ const auto& rasterSource = *source->as<RasterSource>();
+ handleTiledSource(rasterSource.getURLOrTileset(), rasterSource.getTileSize());
+ break;
+ }
+
+ case SourceType::RasterDEM: {
+ const auto& rasterDEMSource = *source->as<RasterDEMSource>();
+ handleTiledSource(rasterDEMSource.getURLOrTileset(), rasterDEMSource.getTileSize());
+ break;
+ }
+
+ case SourceType::GeoJSON: {
+ const auto& geojsonSource = *source->as<GeoJSONSource>();
+ if (geojsonSource.getURL()) {
+ result->requiredResourceCount += 1;
+ }
+ break;
+ }
+
+ case SourceType::Image: {
+ const auto& imageSource = *source->as<ImageSource>();
+ if (imageSource.getURL()) {
+ result->requiredResourceCount += 1;
+ }
+ break;
+ }
+
+ case SourceType::Video:
+ case SourceType::Annotations:
+ case SourceType::CustomVector:
+ break;
+ }
+ }
+
+ if (!parser.glyphURL.empty()) {
+ result->requiredResourceCount += parser.fontStacks().size() * GLYPH_RANGES_PER_FONT_STACK;
+ }
+
+ if (!parser.spriteURL.empty()) {
+ result->requiredResourceCount += 4;
+ }
+
+ return *result;
+}
+
+void OfflineDownload::activateDownload() {
+ status = OfflineRegionStatus();
+ status.downloadState = OfflineRegionDownloadState::Active;
+ status.requiredResourceCount++;
+ ensureResource(Resource::style(definition.match([](auto& reg){ return reg.styleURL; }), Resource::Priority::Low),
+ [&](Response styleResponse) {
+ status.requiredResourceCountIsPrecise = true;
+
+ style::Parser parser;
+ parser.parse(*styleResponse.data);
+
+ for (const auto& source : parser.sources) {
+ SourceType type = source->getType();
+
+ auto handleTiledSource = [&] (const variant<std::string, Tileset>& urlOrTileset, const uint16_t tileSize) {
+ if (urlOrTileset.is<Tileset>()) {
+ queueTiles(type, tileSize, urlOrTileset.get<Tileset>());
+ } else {
+ const auto& url = urlOrTileset.get<std::string>();
+ status.requiredResourceCountIsPrecise = false;
+ status.requiredResourceCount++;
+ requiredSourceURLs.insert(url);
+
+ ensureResource(Resource::source(url, Resource::Priority::Low), [=](Response sourceResponse) {
+ style::conversion::Error error;
+ optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(*sourceResponse.data, error);
+ if (tileset) {
+ util::mapbox::canonicalizeTileset(*tileset, url, type, tileSize);
+ queueTiles(type, tileSize, *tileset);
+
+ requiredSourceURLs.erase(url);
+ if (requiredSourceURLs.empty()) {
+ status.requiredResourceCountIsPrecise = true;
+ }
+ }
+ });
+ }
+ };
+
+ switch (type) {
+ case SourceType::Vector: {
+ const auto& vectorSource = *source->as<VectorSource>();
+ handleTiledSource(vectorSource.getURLOrTileset(), util::tileSize);
+ break;
+ }
+
+ case SourceType::Raster: {
+ const auto& rasterSource = *source->as<RasterSource>();
+ handleTiledSource(rasterSource.getURLOrTileset(), rasterSource.getTileSize());
+ break;
+ }
+
+ case SourceType::RasterDEM: {
+ const auto& rasterDEMSource = *source->as<RasterDEMSource>();
+ handleTiledSource(rasterDEMSource.getURLOrTileset(), rasterDEMSource.getTileSize());
+ break;
+ }
+
+ case SourceType::GeoJSON: {
+ const auto& geojsonSource = *source->as<GeoJSONSource>();
+ if (geojsonSource.getURL()) {
+ queueResource(Resource::source(*geojsonSource.getURL()));
+ }
+ break;
+ }
+
+ case SourceType::Image: {
+ const auto& imageSource = *source->as<ImageSource>();
+ auto imageUrl = imageSource.getURL();
+ if (imageUrl && !imageUrl->empty()) {
+ queueResource(Resource::image(*imageUrl));
+ }
+ break;
+ }
+
+ case SourceType::Video:
+ case SourceType::Annotations:
+ case SourceType::CustomVector:
+ break;
+ }
+ }
+
+ if (!parser.glyphURL.empty()) {
+ for (const auto& fontStack : parser.fontStacks()) {
+ for (char16_t i = 0; i < GLYPH_RANGES_PER_FONT_STACK; i++) {
+ queueResource(Resource::glyphs(parser.glyphURL, fontStack, getGlyphRange(i * GLYPHS_PER_GLYPH_RANGE)));
+ }
+ }
+ }
+
+ if (!parser.spriteURL.empty()) {
+ // Always request 1x and @2x sprite images for portability.
+ queueResource(Resource::spriteImage(parser.spriteURL, 1));
+ queueResource(Resource::spriteImage(parser.spriteURL, 2));
+ queueResource(Resource::spriteJSON(parser.spriteURL, 1));
+ queueResource(Resource::spriteJSON(parser.spriteURL, 2));
+ }
+
+ continueDownload();
+ });
+}
+
+/*
+ Fill up our own request queue by requesting the next few resources. This is called
+ when activating the download, or when a request completes successfully.
+
+ Note "successfully"; it's not called when a requests receives an error. A request
+ that errors will be retried after some delay. So in that sense it's still "active"
+ and consuming resources, notably the request object, its timer, and network resources
+ when the timer fires.
+
+ We could try to squeeze in subsequent requests while we wait for the errored request
+ to retry. But that risks overloading the upstream request queue -- defeating our own
+ metering -- if there are a lot of errored requests that all come up for retry at the
+ same time. And many times, the cause of a request error will apply to many requests
+ of the same type. For instance if a server is unreachable, all the requests to that
+ host are going to error. In that case, continuing to try subsequent resources after
+ the first few errors is fruitless anyway.
+*/
+void OfflineDownload::continueDownload() {
+ if (resourcesRemaining.empty() && status.complete()) {
+ setState(OfflineRegionDownloadState::Inactive);
+ return;
+ }
+
+ while (!resourcesRemaining.empty() && requests.size() < onlineFileSource.getMaximumConcurrentRequests()) {
+ ensureResource(resourcesRemaining.front());
+ resourcesRemaining.pop_front();
+ }
+}
+
+void OfflineDownload::deactivateDownload() {
+ requiredSourceURLs.clear();
+ resourcesRemaining.clear();
+ requests.clear();
+}
+
+void OfflineDownload::queueResource(Resource resource) {
+ resource.setPriority(Resource::Priority::Low);
+ status.requiredResourceCount++;
+ resourcesRemaining.push_front(std::move(resource));
+}
+
+void OfflineDownload::queueTiles(SourceType type, uint16_t tileSize, const Tileset& tileset) {
+ tileCover(definition, type, tileSize, tileset.zoomRange, [&](const auto& tile) {
+ status.requiredResourceCount++;
+ resourcesRemaining.push_back(Resource::tile(
+ tileset.tiles[0], definition.match([](auto& def) { return def.pixelRatio; }), tile.x,
+ tile.y, tile.z, tileset.scheme, Resource::Priority::Low));
+ });
+}
+
+void OfflineDownload::ensureResource(const Resource& resource,
+ std::function<void(Response)> callback) {
+ assert(resource.priority == Resource::Priority::Low);
+
+ auto workRequestsIt = requests.insert(requests.begin(), nullptr);
+ *workRequestsIt = util::RunLoop::Get()->invokeCancellable([=]() {
+ requests.erase(workRequestsIt);
+
+ auto getResourceSizeInDatabase = [&] () -> optional<int64_t> {
+ if (!callback) {
+ return offlineDatabase.hasRegionResource(id, resource);
+ }
+ optional<std::pair<Response, uint64_t>> response = offlineDatabase.getRegionResource(id, resource);
+ if (!response) {
+ return {};
+ }
+ callback(response->first);
+ return response->second;
+ };
+
+ optional<int64_t> offlineResponse = getResourceSizeInDatabase();
+ if (offlineResponse) {
+ status.completedResourceCount++;
+ status.completedResourceSize += *offlineResponse;
+ if (resource.kind == Resource::Kind::Tile) {
+ status.completedTileCount += 1;
+ status.completedTileSize += *offlineResponse;
+ }
+
+ observer->statusChanged(status);
+ continueDownload();
+ return;
+ }
+
+ if (offlineDatabase.exceedsOfflineMapboxTileCountLimit(resource)) {
+ onMapboxTileCountLimitExceeded();
+ return;
+ }
+
+ auto fileRequestsIt = requests.insert(requests.begin(), nullptr);
+ *fileRequestsIt = onlineFileSource.request(resource, [=](Response onlineResponse) {
+ if (onlineResponse.error) {
+ observer->responseError(*onlineResponse.error);
+ return;
+ }
+
+ requests.erase(fileRequestsIt);
+
+ if (callback) {
+ callback(onlineResponse);
+ }
+
+ // Queue up for batched insertion
+ buffer.emplace_back(resource, onlineResponse);
+
+ // Flush buffer periodically
+ if (buffer.size() == 64 || resourcesRemaining.size() == 0) {
+ try {
+ offlineDatabase.putRegionResources(id, buffer, status);
+ } catch (const MapboxTileLimitExceededException&) {
+ onMapboxTileCountLimitExceeded();
+ return;
+ }
+
+ buffer.clear();
+ observer->statusChanged(status);
+ }
+
+ if (offlineDatabase.exceedsOfflineMapboxTileCountLimit(resource)) {
+ onMapboxTileCountLimitExceeded();
+ return;
+ }
+
+ continueDownload();
+ });
+ });
+}
+
+void OfflineDownload::onMapboxTileCountLimitExceeded() {
+ observer->mapboxTileCountLimitExceeded(offlineDatabase.getOfflineMapboxTileCountLimit());
+ setState(OfflineRegionDownloadState::Inactive);
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/storage/online_file_source.cpp b/platform/default/src/mbgl/storage/online_file_source.cpp
new file mode 100644
index 0000000000..fce1c3e2b6
--- /dev/null
+++ b/platform/default/src/mbgl/storage/online_file_source.cpp
@@ -0,0 +1,488 @@
+#include <mbgl/storage/online_file_source.hpp>
+#include <mbgl/storage/http_file_source.hpp>
+#include <mbgl/storage/network_status.hpp>
+
+#include <mbgl/storage/resource_transform.hpp>
+#include <mbgl/storage/response.hpp>
+#include <mbgl/util/logging.hpp>
+
+#include <mbgl/actor/mailbox.hpp>
+#include <mbgl/util/constants.hpp>
+#include <mbgl/util/mapbox.hpp>
+#include <mbgl/util/exception.hpp>
+#include <mbgl/util/chrono.hpp>
+#include <mbgl/util/async_task.hpp>
+#include <mbgl/util/noncopyable.hpp>
+#include <mbgl/util/run_loop.hpp>
+#include <mbgl/util/timer.hpp>
+#include <mbgl/util/http_timeout.hpp>
+
+#include <algorithm>
+#include <cassert>
+#include <list>
+#include <unordered_set>
+#include <unordered_map>
+
+namespace mbgl {
+
+static uint32_t DEFAULT_MAXIMUM_CONCURRENT_REQUESTS = 20;
+
+class OnlineFileRequest : public AsyncRequest {
+public:
+ using Callback = std::function<void (Response)>;
+
+ OnlineFileRequest(Resource, Callback, OnlineFileSource::Impl&);
+ ~OnlineFileRequest() override;
+
+ void networkIsReachableAgain();
+ void schedule();
+ void schedule(optional<Timestamp> expires);
+ void completed(Response);
+
+ void setTransformedURL(const std::string&& url);
+ ActorRef<OnlineFileRequest> actor();
+
+ OnlineFileSource::Impl& impl;
+ Resource resource;
+ std::unique_ptr<AsyncRequest> request;
+ util::Timer timer;
+ Callback callback;
+
+ std::shared_ptr<Mailbox> mailbox;
+
+ // Counts the number of times a response was already expired when received. We're using
+ // this to add a delay when making a new request so we don't keep retrying immediately
+ // in case of a server serving expired tiles.
+ uint32_t expiredRequests = 0;
+
+ // Counts the number of subsequent failed requests. We're using this value for exponential
+ // backoff when retrying requests.
+ uint32_t failedRequests = 0;
+ Response::Error::Reason failedRequestReason = Response::Error::Reason::Success;
+ optional<Timestamp> retryAfter;
+};
+
+class OnlineFileSource::Impl {
+public:
+ Impl() {
+ NetworkStatus::Subscribe(&reachability);
+ setMaximumConcurrentRequests(DEFAULT_MAXIMUM_CONCURRENT_REQUESTS);
+ }
+
+ ~Impl() {
+ NetworkStatus::Unsubscribe(&reachability);
+ }
+
+ void add(OnlineFileRequest* request) {
+ allRequests.insert(request);
+ if (resourceTransform) {
+ // Request the ResourceTransform actor a new url and replace the resource url with the
+ // transformed one before proceeding to schedule the request.
+ resourceTransform->invoke(&ResourceTransform::transform, request->resource.kind,
+ std::move(request->resource.url), [ref = request->actor()](const std::string&& url) mutable {
+ ref.invoke(&OnlineFileRequest::setTransformedURL, std::move(url));
+ });
+ } else {
+ request->schedule();
+ }
+ }
+
+ void remove(OnlineFileRequest* request) {
+ allRequests.erase(request);
+ if (activeRequests.erase(request)) {
+ activatePendingRequest();
+ } else {
+ pendingRequests.remove(request);
+ }
+ }
+
+ void activateOrQueueRequest(OnlineFileRequest* request) {
+ assert(allRequests.find(request) != allRequests.end());
+ assert(activeRequests.find(request) == activeRequests.end());
+ assert(!request->request);
+
+ if (activeRequests.size() >= getMaximumConcurrentRequests()) {
+ queueRequest(request);
+ } else {
+ activateRequest(request);
+ }
+ }
+
+ void queueRequest(OnlineFileRequest* request) {
+ pendingRequests.insert(request);
+ }
+
+ void activateRequest(OnlineFileRequest* request) {
+ auto callback = [=](Response response) {
+ activeRequests.erase(request);
+ request->request.reset();
+ request->completed(response);
+ activatePendingRequest();
+ };
+
+ activeRequests.insert(request);
+
+ if (online) {
+ request->request = httpFileSource.request(request->resource, callback);
+ } else {
+ Response response;
+ response.error = std::make_unique<Response::Error>(Response::Error::Reason::Connection,
+ "Online connectivity is disabled.");
+ callback(response);
+ }
+
+ }
+
+ void activatePendingRequest() {
+
+ auto request = pendingRequests.pop();
+
+ if (request) {
+ activateRequest(*request);
+ }
+ }
+
+ bool isPending(OnlineFileRequest* request) {
+ return pendingRequests.contains(request);
+ }
+
+ bool isActive(OnlineFileRequest* request) {
+ return activeRequests.find(request) != activeRequests.end();
+ }
+
+ void setResourceTransform(optional<ActorRef<ResourceTransform>>&& transform) {
+ resourceTransform = std::move(transform);
+ }
+
+ void setOnlineStatus(const bool status) {
+ online = status;
+ networkIsReachableAgain();
+ }
+
+ uint32_t getMaximumConcurrentRequests() const {
+ return maximumConcurrentRequests;
+ }
+
+ void setMaximumConcurrentRequests(uint32_t maximumConcurrentRequests_) {
+ maximumConcurrentRequests = maximumConcurrentRequests_;
+ }
+
+private:
+
+ void networkIsReachableAgain() {
+ for (auto& request : allRequests) {
+ request->networkIsReachableAgain();
+ }
+ }
+
+ // Using Pending Requests as an priority queue which processes
+ // file requests in a FIFO manner but prefers regular requests
+ // over offline requests with a low priority such that low priority
+ // requests do not throttle regular requests.
+ //
+ // The order of a queue is therefore:
+ //
+ // hi0 -- hi1 -- hi2 -- hi3 -- lo0 -- lo1 --lo2
+ // ^
+ // firstLowPriorityRequest
+
+ struct PendingRequests {
+ PendingRequests() : queue(), firstLowPriorityRequest(queue.begin()) {}
+
+ std::list<OnlineFileRequest*> queue;
+ std::list<OnlineFileRequest*>::iterator firstLowPriorityRequest;
+
+ void remove(const OnlineFileRequest* request) {
+ auto it = std::find(queue.begin(), queue.end(), request);
+ if (it != queue.end()) {
+ if (it == firstLowPriorityRequest) {
+ firstLowPriorityRequest++;
+ }
+ queue.erase(it);
+ }
+ }
+
+ void insert(OnlineFileRequest* request) {
+ if (request->resource.priority == Resource::Priority::Regular) {
+ firstLowPriorityRequest = queue.insert(firstLowPriorityRequest, request);
+ firstLowPriorityRequest++;
+ }
+ else {
+ if (firstLowPriorityRequest == queue.end()) {
+ firstLowPriorityRequest = queue.insert(queue.end(), request);
+ }
+ else {
+ queue.insert(queue.end(), request);
+ }
+ }
+ }
+
+
+ optional<OnlineFileRequest*> pop() {
+ if (queue.empty()) {
+ return optional<OnlineFileRequest*>();
+ }
+
+ if (queue.begin() == firstLowPriorityRequest) {
+ firstLowPriorityRequest++;
+ }
+
+ OnlineFileRequest* next = queue.front();
+ queue.pop_front();
+ return optional<OnlineFileRequest*>(next);
+ }
+
+ bool contains(OnlineFileRequest* request) const {
+ return (std::find(queue.begin(), queue.end(), request) != queue.end());
+ }
+
+ };
+
+ optional<ActorRef<ResourceTransform>> resourceTransform;
+
+ /**
+ * The lifetime of a request is:
+ *
+ * 1. Waiting for timeout (revalidation or retry)
+ * 2. Pending (waiting for room in the active set)
+ * 3. Active (open network connection)
+ * 4. Back to #1
+ *
+ * Requests in any state are in `allRequests`. Requests in the pending state are in
+ * `pendingRequests`. Requests in the active state are in `activeRequests`.
+ */
+ std::unordered_set<OnlineFileRequest*> allRequests;
+
+ PendingRequests pendingRequests;
+
+ std::unordered_set<OnlineFileRequest*> activeRequests;
+
+ bool online = true;
+ uint32_t maximumConcurrentRequests;
+ HTTPFileSource httpFileSource;
+ util::AsyncTask reachability { std::bind(&Impl::networkIsReachableAgain, this) };
+};
+
+OnlineFileSource::OnlineFileSource()
+ : impl(std::make_unique<Impl>()) {
+}
+
+OnlineFileSource::~OnlineFileSource() = default;
+
+std::unique_ptr<AsyncRequest> OnlineFileSource::request(const Resource& resource, Callback callback) {
+ Resource res = resource;
+
+ switch (resource.kind) {
+ case Resource::Kind::Unknown:
+ case Resource::Kind::Image:
+ break;
+
+ case Resource::Kind::Style:
+ res.url = mbgl::util::mapbox::normalizeStyleURL(apiBaseURL, resource.url, accessToken);
+ break;
+
+ case Resource::Kind::Source:
+ res.url = util::mapbox::normalizeSourceURL(apiBaseURL, resource.url, accessToken);
+ break;
+
+ case Resource::Kind::Glyphs:
+ res.url = util::mapbox::normalizeGlyphsURL(apiBaseURL, resource.url, accessToken);
+ break;
+
+ case Resource::Kind::SpriteImage:
+ case Resource::Kind::SpriteJSON:
+ res.url = util::mapbox::normalizeSpriteURL(apiBaseURL, resource.url, accessToken);
+ break;
+
+ case Resource::Kind::Tile:
+ res.url = util::mapbox::normalizeTileURL(apiBaseURL, resource.url, accessToken);
+ break;
+ }
+
+ return std::make_unique<OnlineFileRequest>(std::move(res), std::move(callback), *impl);
+}
+
+void OnlineFileSource::setResourceTransform(optional<ActorRef<ResourceTransform>>&& transform) {
+ impl->setResourceTransform(std::move(transform));
+}
+
+OnlineFileRequest::OnlineFileRequest(Resource resource_, Callback callback_, OnlineFileSource::Impl& impl_)
+ : impl(impl_),
+ resource(std::move(resource_)),
+ callback(std::move(callback_)) {
+ impl.add(this);
+}
+
+void OnlineFileRequest::schedule() {
+ // Force an immediate first request if we don't have an expiration time.
+ if (resource.priorExpires) {
+ schedule(resource.priorExpires);
+ } else {
+ schedule(util::now());
+ }
+}
+
+OnlineFileRequest::~OnlineFileRequest() {
+ impl.remove(this);
+}
+
+Timestamp interpolateExpiration(const Timestamp& current,
+ optional<Timestamp> prior,
+ bool& expired) {
+ auto now = util::now();
+ if (current > now) {
+ return current;
+ }
+
+ if (!bool(prior)) {
+ expired = true;
+ return current;
+ }
+
+ // Expiring date is going backwards,
+ // fallback to exponential backoff.
+ if (current < *prior) {
+ expired = true;
+ return current;
+ }
+
+ auto delta = current - *prior;
+
+ // Server is serving the same expired resource
+ // over and over, fallback to exponential backoff.
+ if (delta == Duration::zero()) {
+ expired = true;
+ return current;
+ }
+
+ // Assume that either the client or server clock is wrong and
+ // try to interpolate a valid expiration date (from the client POV)
+ // observing a minimum timeout.
+ return now + std::max<Seconds>(delta, util::CLOCK_SKEW_RETRY_TIMEOUT);
+}
+
+void OnlineFileRequest::schedule(optional<Timestamp> expires) {
+ if (impl.isPending(this) || impl.isActive(this)) {
+ // There's already a request in progress; don't start another one.
+ return;
+ }
+
+ // If we're not being asked for a forced refresh, calculate a timeout that depends on how many
+ // consecutive errors we've encountered, and on the expiration time, if present.
+ Duration timeout = std::min(
+ http::errorRetryTimeout(failedRequestReason, failedRequests, retryAfter),
+ http::expirationTimeout(expires, expiredRequests));
+
+ if (timeout == Duration::max()) {
+ return;
+ }
+
+ // Emulate a Connection error when the Offline mode is forced with
+ // a really long timeout. The request will get re-triggered when
+ // the NetworkStatus is set back to Online.
+ if (NetworkStatus::Get() == NetworkStatus::Status::Offline) {
+ failedRequestReason = Response::Error::Reason::Connection;
+ failedRequests = 1;
+ timeout = Duration::max();
+ }
+
+ timer.start(timeout, Duration::zero(), [&] {
+ impl.activateOrQueueRequest(this);
+ });
+}
+
+void OnlineFileRequest::completed(Response response) {
+ // If we didn't get various caching headers in the response, continue using the
+ // previous values. Otherwise, update the previous values to the new values.
+
+ if (!response.modified) {
+ response.modified = resource.priorModified;
+ } else {
+ resource.priorModified = response.modified;
+ }
+
+ if (response.notModified && resource.priorData) {
+ // When the priorData field is set, it indicates that we had to revalidate the request and
+ // that the requestor hasn't gotten data yet. If we get a 304 response, this means that we
+ // have send the cached data to give the requestor a chance to actually obtain the data.
+ response.data = std::move(resource.priorData);
+ response.notModified = false;
+ }
+
+ bool isExpired = false;
+
+ if (response.expires) {
+ auto prior = resource.priorExpires;
+ resource.priorExpires = response.expires;
+ response.expires = interpolateExpiration(*response.expires, prior, isExpired);
+ }
+
+ if (isExpired) {
+ expiredRequests++;
+ } else {
+ expiredRequests = 0;
+ }
+
+ if (!response.etag) {
+ response.etag = resource.priorEtag;
+ } else {
+ resource.priorEtag = response.etag;
+ }
+
+ if (response.error) {
+ failedRequests++;
+ failedRequestReason = response.error->reason;
+ retryAfter = response.error->retryAfter;
+ } else {
+ failedRequests = 0;
+ failedRequestReason = Response::Error::Reason::Success;
+ }
+
+ schedule(response.expires);
+
+ // Calling the callback may result in `this` being deleted. It needs to be done last,
+ // and needs to make a local copy of the callback to ensure that it remains valid for
+ // the duration of the call.
+ auto callback_ = callback;
+ callback_(response);
+}
+
+void OnlineFileRequest::networkIsReachableAgain() {
+ // We need all requests to fail at least once before we are going to start retrying
+ // them, and we only immediately restart request that failed due to connection issues.
+ if (failedRequestReason == Response::Error::Reason::Connection) {
+ schedule(util::now());
+ }
+}
+
+void OnlineFileRequest::setTransformedURL(const std::string&& url) {
+ resource.url = std::move(url);
+ schedule();
+}
+
+ActorRef<OnlineFileRequest> OnlineFileRequest::actor() {
+ if (!mailbox) {
+ // Lazy constructed because this can be costly and
+ // the ResourceTransform is not used by many apps.
+ mailbox = std::make_shared<Mailbox>(*Scheduler::GetCurrent());
+ }
+
+ return ActorRef<OnlineFileRequest>(*this, mailbox);
+}
+
+void OnlineFileSource::setMaximumConcurrentRequests(uint32_t maximumConcurrentRequests_) {
+ impl->setMaximumConcurrentRequests(maximumConcurrentRequests_);
+}
+
+uint32_t OnlineFileSource::getMaximumConcurrentRequests() const {
+ return impl->getMaximumConcurrentRequests();
+}
+
+
+// For testing only:
+
+void OnlineFileSource::setOnlineStatus(const bool status) {
+ impl->setOnlineStatus(status);
+}
+
+} // namespace mbgl
diff --git a/platform/default/src/mbgl/storage/sqlite3.cpp b/platform/default/src/mbgl/storage/sqlite3.cpp
new file mode 100644
index 0000000000..0017dc45db
--- /dev/null
+++ b/platform/default/src/mbgl/storage/sqlite3.cpp
@@ -0,0 +1,494 @@
+#include <mbgl/storage/sqlite3.hpp>
+#include <sqlite3.h>
+
+#include <algorithm>
+#include <cassert>
+#include <cstring>
+#include <cstdio>
+#include <chrono>
+#include <experimental/optional>
+
+#include <mbgl/util/traits.hpp>
+#include <mbgl/util/logging.hpp>
+
+namespace mapbox {
+namespace sqlite {
+
+static_assert(mbgl::underlying_type(ResultCode::OK) == SQLITE_OK, "error");
+static_assert(mbgl::underlying_type(ResultCode::Error) == SQLITE_ERROR, "error");
+static_assert(mbgl::underlying_type(ResultCode::Internal) == SQLITE_INTERNAL, "error");
+static_assert(mbgl::underlying_type(ResultCode::Perm) == SQLITE_PERM, "error");
+static_assert(mbgl::underlying_type(ResultCode::Abort) == SQLITE_ABORT, "error");
+static_assert(mbgl::underlying_type(ResultCode::Busy) == SQLITE_BUSY, "error");
+static_assert(mbgl::underlying_type(ResultCode::Locked) == SQLITE_LOCKED, "error");
+static_assert(mbgl::underlying_type(ResultCode::NoMem) == SQLITE_NOMEM, "error");
+static_assert(mbgl::underlying_type(ResultCode::ReadOnly) == SQLITE_READONLY, "error");
+static_assert(mbgl::underlying_type(ResultCode::Interrupt) == SQLITE_INTERRUPT, "error");
+static_assert(mbgl::underlying_type(ResultCode::IOErr) == SQLITE_IOERR, "error");
+static_assert(mbgl::underlying_type(ResultCode::Corrupt) == SQLITE_CORRUPT, "error");
+static_assert(mbgl::underlying_type(ResultCode::NotFound) == SQLITE_NOTFOUND, "error");
+static_assert(mbgl::underlying_type(ResultCode::Full) == SQLITE_FULL, "error");
+static_assert(mbgl::underlying_type(ResultCode::CantOpen) == SQLITE_CANTOPEN, "error");
+static_assert(mbgl::underlying_type(ResultCode::Protocol) == SQLITE_PROTOCOL, "error");
+static_assert(mbgl::underlying_type(ResultCode::Schema) == SQLITE_SCHEMA, "error");
+static_assert(mbgl::underlying_type(ResultCode::TooBig) == SQLITE_TOOBIG, "error");
+static_assert(mbgl::underlying_type(ResultCode::Constraint) == SQLITE_CONSTRAINT, "error");
+static_assert(mbgl::underlying_type(ResultCode::Mismatch) == SQLITE_MISMATCH, "error");
+static_assert(mbgl::underlying_type(ResultCode::Misuse) == SQLITE_MISUSE, "error");
+static_assert(mbgl::underlying_type(ResultCode::NoLFS) == SQLITE_NOLFS, "error");
+static_assert(mbgl::underlying_type(ResultCode::Auth) == SQLITE_AUTH, "error");
+static_assert(mbgl::underlying_type(ResultCode::Range) == SQLITE_RANGE, "error");
+static_assert(mbgl::underlying_type(ResultCode::NotADB) == SQLITE_NOTADB, "error");
+
+void setTempPath(const std::string& path) {
+ sqlite3_temp_directory = sqlite3_mprintf("%s", path.c_str());
+}
+
+class DatabaseImpl {
+public:
+ DatabaseImpl(sqlite3* db_)
+ : db(db_)
+ {
+ const int error = sqlite3_extended_result_codes(db, true);
+ if (error != SQLITE_OK) {
+ mbgl::Log::Warning(mbgl::Event::Database, error, "Failed to enable extended result codes: %s", sqlite3_errmsg(db));
+ }
+ }
+
+ ~DatabaseImpl()
+ {
+ const int error = sqlite3_close(db);
+ if (error != SQLITE_OK) {
+ mbgl::Log::Error(mbgl::Event::Database, error, "Failed to close database: %s", sqlite3_errmsg(db));
+ }
+ }
+
+ void setBusyTimeout(std::chrono::milliseconds timeout);
+ void exec(const std::string& sql);
+
+ sqlite3* db;
+};
+
+class StatementImpl {
+public:
+ StatementImpl(sqlite3* db, const char* sql)
+ {
+ const int error = sqlite3_prepare_v2(db, sql, -1, &stmt, nullptr);
+ if (error != SQLITE_OK) {
+ stmt = nullptr;
+ throw Exception { error, sqlite3_errmsg(db) };
+ }
+ }
+
+ ~StatementImpl()
+ {
+ if (!stmt) return;
+
+ sqlite3_finalize(stmt);
+ }
+
+ void check(int err) {
+ if (err != SQLITE_OK) {
+ throw Exception { err, sqlite3_errmsg(sqlite3_db_handle(stmt)) };
+ }
+ }
+
+ sqlite3_stmt* stmt = nullptr;
+ int64_t lastInsertRowId = 0;
+ int64_t changes = 0;
+};
+
+template <typename T>
+using optional = std::experimental::optional<T>;
+
+
+#ifndef NDEBUG
+void logSqlMessage(void *, const int err, const char *msg) {
+ mbgl::Log::Record(mbgl::EventSeverity::Debug, mbgl::Event::Database, err, "%s", msg);
+}
+#endif
+
+__attribute__((constructor))
+static void initalize() {
+ if (sqlite3_libversion_number() / 1000000 != SQLITE_VERSION_NUMBER / 1000000) {
+ char message[96];
+ snprintf(message, 96,
+ "sqlite3 libversion mismatch: headers report %d, but library reports %d",
+ SQLITE_VERSION_NUMBER, sqlite3_libversion_number());
+ throw std::runtime_error(message);
+ }
+
+#ifndef NDEBUG
+ // Enable SQLite logging before initializing the database.
+ sqlite3_config(SQLITE_CONFIG_LOG, &logSqlMessage, nullptr);
+#endif
+}
+
+mapbox::util::variant<Database, Exception> Database::tryOpen(const std::string &filename, int flags) {
+ sqlite3* db = nullptr;
+ const int error = sqlite3_open_v2(filename.c_str(), &db, flags | SQLITE_OPEN_URI, nullptr);
+ if (error != SQLITE_OK) {
+ const auto message = sqlite3_errmsg(db);
+ return Exception { error, message };
+ }
+ return Database(std::make_unique<DatabaseImpl>(db));
+}
+
+Database Database::open(const std::string &filename, int flags) {
+ auto result = tryOpen(filename, flags);
+ if (result.is<Exception>()) {
+ throw result.get<Exception>();
+ } else {
+ return std::move(result.get<Database>());
+ }
+}
+
+Database::Database(std::unique_ptr<DatabaseImpl> impl_)
+ : impl(std::move(impl_))
+{}
+
+Database::Database(Database &&other)
+ : impl(std::move(other.impl)) {}
+
+Database &Database::operator=(Database &&other) {
+ std::swap(impl, other.impl);
+ return *this;
+}
+
+Database::~Database() = default;
+
+void Database::setBusyTimeout(std::chrono::milliseconds timeout) {
+ assert(impl);
+ impl->setBusyTimeout(timeout);
+}
+
+void DatabaseImpl::setBusyTimeout(std::chrono::milliseconds timeout) {
+ const int err = sqlite3_busy_timeout(db,
+ int(std::min<std::chrono::milliseconds::rep>(timeout.count(), std::numeric_limits<int>::max())));
+ if (err != SQLITE_OK) {
+ throw Exception { err, sqlite3_errmsg(db) };
+ }
+}
+
+void Database::exec(const std::string &sql) {
+ assert(impl);
+ impl->exec(sql);
+}
+
+void DatabaseImpl::exec(const std::string& sql) {
+ char *msg = nullptr;
+ const int err = sqlite3_exec(db, sql.c_str(), nullptr, nullptr, &msg);
+ if (msg) {
+ const std::string message = msg;
+ sqlite3_free(msg);
+ throw Exception { err, message };
+ } else if (err != SQLITE_OK) {
+ throw Exception { err, sqlite3_errmsg(db) };
+ }
+}
+
+Statement::Statement(Database& db, const char* sql)
+ : impl(std::make_unique<StatementImpl>(db.impl->db, sql)) {
+}
+
+Statement::~Statement() {
+#ifndef NDEBUG
+ // Crash if we're destructing this object while we know a Query object references this.
+ assert(!used);
+#endif
+}
+
+Query::Query(Statement& stmt_) : stmt(stmt_) {
+ assert(stmt.impl);
+
+#ifndef NDEBUG
+ assert(!stmt.used);
+ stmt.used = true;
+#endif
+}
+
+Query::~Query() {
+ reset();
+ clearBindings();
+
+#ifndef NDEBUG
+ stmt.used = false;
+#endif
+}
+
+template <> void Query::bind(int offset, std::nullptr_t) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_null(stmt.impl->stmt, offset));
+}
+
+template <> void Query::bind(int offset, int8_t value) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value));
+}
+
+template <> void Query::bind(int offset, int16_t value) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value));
+}
+
+template <> void Query::bind(int offset, int32_t value) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value));
+}
+
+template <> void Query::bind(int offset, int64_t value) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value));
+}
+
+template <> void Query::bind(int offset, uint8_t value) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value));
+}
+
+template <> void Query::bind(int offset, uint16_t value) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value));
+}
+
+template <> void Query::bind(int offset, uint32_t value) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, value));
+}
+
+template <> void Query::bind(int offset, float value) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_double(stmt.impl->stmt, offset, value));
+}
+
+template <> void Query::bind(int offset, double value) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_double(stmt.impl->stmt, offset, value));
+}
+
+template <> void Query::bind(int offset, bool value) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_int(stmt.impl->stmt, offset, value));
+}
+
+template <> void Query::bind(int offset, const char *value) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_text(stmt.impl->stmt, offset, value, -1, SQLITE_STATIC));
+}
+
+// We currently cannot use sqlite3_bind_blob64 / sqlite3_bind_text64 because they
+// were introduced in SQLite 3.8.7, and we need to support earlier versions:
+// Android 11: 3.7
+// Android 21: 3.8
+// Android 24: 3.9
+// Per https://developer.android.com/reference/android/database/sqlite/package-summary.
+// The first iOS version with 3.8.7+ was 9.0, with 3.8.8.
+
+void Query::bind(int offset, const char * value, std::size_t length, bool retain) {
+ assert(stmt.impl);
+ if (length > std::numeric_limits<int>::max()) {
+ throw std::range_error("value too long for sqlite3_bind_text");
+ }
+ stmt.impl->check(sqlite3_bind_text(stmt.impl->stmt, offset, value, int(length),
+ retain ? SQLITE_TRANSIENT : SQLITE_STATIC));
+}
+
+void Query::bind(int offset, const std::string& value, bool retain) {
+ bind(offset, value.data(), value.size(), retain);
+}
+
+void Query::bindBlob(int offset, const void * value, std::size_t length, bool retain) {
+ assert(stmt.impl);
+ if (length > std::numeric_limits<int>::max()) {
+ throw std::range_error("value too long for sqlite3_bind_text");
+ }
+ stmt.impl->check(sqlite3_bind_blob(stmt.impl->stmt, offset, value, int(length),
+ retain ? SQLITE_TRANSIENT : SQLITE_STATIC));
+}
+
+void Query::bindBlob(int offset, const std::vector<uint8_t>& value, bool retain) {
+ bindBlob(offset, value.data(), value.size(), retain);
+}
+
+template <>
+void Query::bind(
+ int offset, std::chrono::time_point<std::chrono::system_clock, std::chrono::seconds> value) {
+ assert(stmt.impl);
+ stmt.impl->check(sqlite3_bind_int64(stmt.impl->stmt, offset, std::chrono::system_clock::to_time_t(value)));
+}
+
+template <> void Query::bind(int offset, optional<std::string> value) {
+ if (!value) {
+ bind(offset, nullptr);
+ } else {
+ bind(offset, *value);
+ }
+}
+
+template <>
+void Query::bind(
+ int offset,
+ optional<std::chrono::time_point<std::chrono::system_clock, std::chrono::seconds>> value) {
+ if (!value) {
+ bind(offset, nullptr);
+ } else {
+ bind(offset, *value);
+ }
+}
+
+bool Query::run() {
+ assert(stmt.impl);
+ const int err = sqlite3_step(stmt.impl->stmt);
+ stmt.impl->lastInsertRowId = sqlite3_last_insert_rowid(sqlite3_db_handle(stmt.impl->stmt));
+ stmt.impl->changes = sqlite3_changes(sqlite3_db_handle(stmt.impl->stmt));
+ if (err == SQLITE_DONE) {
+ return false;
+ } else if (err == SQLITE_ROW) {
+ return true;
+ } else if (err != SQLITE_OK) {
+ throw Exception { err, sqlite3_errmsg(sqlite3_db_handle(stmt.impl->stmt)) };
+ } else {
+ return false;
+ }
+}
+
+template <> bool Query::get(int offset) {
+ assert(stmt.impl);
+ return sqlite3_column_int(stmt.impl->stmt, offset);
+}
+
+template <> int Query::get(int offset) {
+ assert(stmt.impl);
+ return sqlite3_column_int(stmt.impl->stmt, offset);
+}
+
+template <> int64_t Query::get(int offset) {
+ assert(stmt.impl);
+ return sqlite3_column_int64(stmt.impl->stmt, offset);
+}
+
+template <> double Query::get(int offset) {
+ assert(stmt.impl);
+ return sqlite3_column_double(stmt.impl->stmt, offset);
+}
+
+template <> std::string Query::get(int offset) {
+ assert(stmt.impl);
+ return {
+ reinterpret_cast<const char *>(sqlite3_column_blob(stmt.impl->stmt, offset)),
+ size_t(sqlite3_column_bytes(stmt.impl->stmt, offset))
+ };
+}
+
+template <> std::vector<uint8_t> Query::get(int offset) {
+ assert(stmt.impl);
+ const auto* begin = reinterpret_cast<const uint8_t*>(sqlite3_column_blob(stmt.impl->stmt, offset));
+ const uint8_t* end = begin + sqlite3_column_bytes(stmt.impl->stmt, offset);
+ return { begin, end };
+}
+
+template <>
+std::chrono::time_point<std::chrono::system_clock, std::chrono::seconds>
+Query::get(int offset) {
+ assert(stmt.impl);
+ return std::chrono::time_point_cast<std::chrono::seconds>(
+ std::chrono::system_clock::from_time_t(sqlite3_column_int64(stmt.impl->stmt, offset)));
+}
+
+template <> optional<int64_t> Query::get(int offset) {
+ assert(stmt.impl);
+ if (sqlite3_column_type(stmt.impl->stmt, offset) == SQLITE_NULL) {
+ return optional<int64_t>();
+ } else {
+ return get<int64_t>(offset);
+ }
+}
+
+template <> optional<double> Query::get(int offset) {
+ assert(stmt.impl);
+ if (sqlite3_column_type(stmt.impl->stmt, offset) == SQLITE_NULL) {
+ return optional<double>();
+ } else {
+ return get<double>(offset);
+ }
+}
+
+template <> optional<std::string> Query::get(int offset) {
+ assert(stmt.impl);
+ if (sqlite3_column_type(stmt.impl->stmt, offset) == SQLITE_NULL) {
+ return optional<std::string>();
+ } else {
+ return get<std::string>(offset);
+ }
+}
+
+template <>
+optional<std::chrono::time_point<std::chrono::system_clock, std::chrono::seconds>>
+Query::get(int offset) {
+ assert(stmt.impl);
+ if (sqlite3_column_type(stmt.impl->stmt, offset) == SQLITE_NULL) {
+ return {};
+ } else {
+ return get<std::chrono::time_point<std::chrono::system_clock, std::chrono::seconds>>(
+ offset);
+ }
+}
+
+void Query::reset() {
+ assert(stmt.impl);
+ sqlite3_reset(stmt.impl->stmt);
+}
+
+void Query::clearBindings() {
+ assert(stmt.impl);
+ sqlite3_clear_bindings(stmt.impl->stmt);
+}
+
+int64_t Query::lastInsertRowId() const {
+ assert(stmt.impl);
+ return stmt.impl->lastInsertRowId;
+}
+
+uint64_t Query::changes() const {
+ assert(stmt.impl);
+ auto changes_ = stmt.impl->changes;
+ return (changes_ < 0 ? 0 : changes_);
+}
+
+Transaction::Transaction(Database& db_, Mode mode)
+ : dbImpl(*db_.impl) {
+ switch (mode) {
+ case Deferred:
+ dbImpl.exec("BEGIN DEFERRED TRANSACTION");
+ break;
+ case Immediate:
+ dbImpl.exec("BEGIN IMMEDIATE TRANSACTION");
+ break;
+ case Exclusive:
+ dbImpl.exec("BEGIN EXCLUSIVE TRANSACTION");
+ break;
+ }
+}
+
+Transaction::~Transaction() {
+ if (needRollback) {
+ try {
+ rollback();
+ } catch (...) {
+ // Ignore failed rollbacks in destructor.
+ }
+ }
+}
+
+void Transaction::commit() {
+ needRollback = false;
+ dbImpl.exec("COMMIT TRANSACTION");
+}
+
+void Transaction::rollback() {
+ needRollback = false;
+ dbImpl.exec("ROLLBACK TRANSACTION");
+}
+
+} // namespace sqlite
+} // namespace mapbox