#include "sqlite_cache_impl.hpp" #include #include #include #include #include #include #include "sqlite3.hpp" #include namespace mbgl { std::string removeAccessTokenFromURL(const std::string &url) { const size_t token_start = url.find("access_token="); // Ensure that token exists, isn't at the front and is preceded by either & or ?. if (token_start == std::string::npos || token_start == 0 || !(url[token_start - 1] == '&' || url[token_start - 1] == '?')) { return url; } const size_t token_end = url.find_first_of('&', token_start); if (token_end == std::string::npos) { // The token is the last query argument. We slice away the "&access_token=..." part return url.substr(0, token_start - 1); } else { // We slice away the "access_token=...&" part. return url.substr(0, token_start) + url.substr(token_end + 1); } } std::string convertMapboxDomainsToProtocol(const std::string &url) { const size_t protocol_separator = url.find("://"); if (protocol_separator == std::string::npos) { return url; } const std::string protocol = url.substr(0, protocol_separator); if (!(protocol == "http" || protocol == "https")) { return url; } const size_t domain_begin = protocol_separator + 3; const size_t path_separator = url.find("/", domain_begin); if (path_separator == std::string::npos) { return url; } const std::string domain = url.substr(domain_begin, path_separator - domain_begin); if (domain.find(".tiles.mapbox.com") != std::string::npos) { return "mapbox://" + url.substr(path_separator + 1); } else { return url; } } std::string unifyMapboxURLs(const std::string &url) { return removeAccessTokenFromURL(convertMapboxDomainsToProtocol(url)); } using namespace mapbox::sqlite; SQLiteCache::SQLiteCache(const std::string& path_) : thread(std::make_unique>(util::ThreadContext{"SQLite Cache", util::ThreadType::Unknown, util::ThreadPriority::Low}, path_)) { } SQLiteCache::~SQLiteCache() = default; SQLiteCache::Impl::Impl(uv_loop_t*, const std::string& path_) : path(path_) { } SQLiteCache::Impl::~Impl() { // Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we // can't throw anything. try { getStmt.reset(); putStmt.reset(); refreshStmt.reset(); db.reset(); } catch (mapbox::sqlite::Exception& ex) { Log::Error(Event::Database, ex.code, ex.what()); } } void SQLiteCache::Impl::createDatabase() { db = std::make_unique(path.c_str(), ReadWrite | Create); } void SQLiteCache::Impl::createSchema() { constexpr const char *const sql = "" "CREATE TABLE IF NOT EXISTS `http_cache` (" " `url` TEXT PRIMARY KEY NOT NULL," " `status` INTEGER NOT NULL," // The response status (Successful or Error). " `kind` INTEGER NOT NULL," // The kind of file. " `modified` INTEGER," // Timestamp when the file was last modified. " `etag` TEXT," " `expires` INTEGER," // Timestamp when the server says the file expires. " `data` BLOB," " `compressed` INTEGER NOT NULL DEFAULT 0" // Whether the data is compressed. ");" "CREATE INDEX IF NOT EXISTS `http_cache_kind_idx` ON `http_cache` (`kind`);"; try { db->exec(sql); schema = true; } catch (mapbox::sqlite::Exception &ex) { if (ex.code == SQLITE_NOTADB) { Log::Warning(Event::Database, "Trashing invalid database"); db.reset(); try { util::deleteFile(path); } catch (util::IOException& ioEx) { Log::Error(Event::Database, ex.code, ex.what()); } db = std::make_unique(path.c_str(), ReadWrite | Create); } else { Log::Error(Event::Database, ex.code, ex.what()); } // Creating the database table + index failed. That means there may already be one, likely // with different columsn. Drop it and try to create a new one. db->exec("DROP TABLE IF EXISTS `http_cache`"); db->exec(sql); } } void SQLiteCache::get(const Resource &resource, Callback callback) { // Can be called from any thread, but most likely from the file source thread. // Will try to load the URL from the SQLite database and call the callback when done. // Note that the callback is probably going to invoked from another thread, so the caller // must make sure that it can run in that thread. thread->invokeWithResult(&Impl::get, std::move(callback), resource); } std::unique_ptr SQLiteCache::Impl::get(const Resource &resource) { try { // This is called in the SQLite event loop. if (!db) { createDatabase(); } if (!schema) { createSchema(); } if (!getStmt) { // Initialize the statement 0 1 getStmt = std::make_unique(db->prepare("SELECT `status`, `modified`, " // 2 3 4 5 1 "`etag`, `expires`, `data`, `compressed` FROM `http_cache` WHERE `url` = ?")); } else { getStmt->reset(); } const std::string unifiedURL = unifyMapboxURLs(resource.url); getStmt->bind(1, unifiedURL.c_str()); if (getStmt->run()) { // There is data. auto response = std::make_unique(); response->status = Response::Status(getStmt->get(0)); response->modified = getStmt->get(1); response->etag = getStmt->get(2); response->expires = getStmt->get(3); response->data = getStmt->get(4); if (getStmt->get(5)) { // == compressed response->data = util::decompress(response->data); } return std::move(response); } else { // There is no data. return nullptr; } } catch (mapbox::sqlite::Exception& ex) { Log::Error(Event::Database, ex.code, ex.what()); return nullptr; } } void SQLiteCache::put(const Resource &resource, std::shared_ptr response, Hint hint) { // Can be called from any thread, but most likely from the file source thread. We are either // storing a new response or updating the currently stored response, potentially setting a new // expiry date. if (hint == Hint::Full) { thread->invoke(&Impl::put, resource, response); } else if (hint == Hint::Refresh) { thread->invoke(&Impl::refresh, resource, response->expires); } } void SQLiteCache::Impl::put(const Resource& resource, std::shared_ptr response) { try { if (!db) { createDatabase(); } if (!schema) { createSchema(); } if (!putStmt) { putStmt = std::make_unique(db->prepare("REPLACE INTO `http_cache` (" // 1 2 3 4 5 6 7 8 "`url`, `status`, `kind`, `modified`, `etag`, `expires`, `data`, `compressed`" ") VALUES(?, ?, ?, ?, ?, ?, ?, ?)")); } else { putStmt->reset(); } const std::string unifiedURL = unifyMapboxURLs(resource.url); putStmt->bind(1 /* url */, unifiedURL.c_str()); putStmt->bind(2 /* status */, int(response->status)); putStmt->bind(3 /* kind */, int(resource.kind)); putStmt->bind(4 /* modified */, response->modified); putStmt->bind(5 /* etag */, response->etag.c_str()); putStmt->bind(6 /* expires */, response->expires); std::string data; if (resource.kind != Resource::Image) { // Do not compress images, since they are typically compressed already. data = util::compress(response->data); } if (!data.empty() && data.size() < response->data.size()) { // Store the compressed data when it is smaller than the original // uncompressed data. putStmt->bind(7 /* data */, data, false); // do not retain the string internally. putStmt->bind(8 /* compressed */, true); } else { putStmt->bind(7 /* data */, response->data, false); // do not retain the string internally. putStmt->bind(8 /* compressed */, false); } putStmt->run(); } catch (mapbox::sqlite::Exception& ex) { Log::Error(Event::Database, ex.code, ex.what()); } } void SQLiteCache::Impl::refresh(const Resource& resource, int64_t expires) { try { if (!db) { createDatabase(); } if (!schema) { createSchema(); } if (!refreshStmt) { refreshStmt = std::make_unique( // 1 2 db->prepare("UPDATE `http_cache` SET `expires` = ? WHERE `url` = ?")); } else { refreshStmt->reset(); } const std::string unifiedURL = unifyMapboxURLs(resource.url); refreshStmt->bind(1, int64_t(expires)); refreshStmt->bind(2, unifiedURL.c_str()); refreshStmt->run(); } catch (mapbox::sqlite::Exception& ex) { Log::Error(Event::Database, ex.code, ex.what()); } } }