From 34fe7dc627f08d57ac259c383e248ddf2f6fb0d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Fri, 22 Aug 2014 18:50:09 +0200 Subject: add basic request caching with sqlite --- src/util/filesource.cpp | 143 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 141 insertions(+), 2 deletions(-) (limited to 'src/util/filesource.cpp') diff --git a/src/util/filesource.cpp b/src/util/filesource.cpp index 503d4dfb1d..3f52a09119 100644 --- a/src/util/filesource.cpp +++ b/src/util/filesource.cpp @@ -1,13 +1,46 @@ #include #include +#include + +#include #include #include namespace mbgl { -FileSource::FileSource() {} +FileSource::FileSource() { + const int err = sqlite3_open_v2("cache.db", &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nullptr); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + db = nullptr; + } + + createSchema(); +} + +FileSource::~FileSource() { + closeDatabase(); +} + +void FileSource::createSchema() { + if (db) { + // Create schema + const int err = sqlite3_exec(db, "CREATE TABLE IF NOT EXISTS `http_cache` (`url` TEXT PRIMARY KEY, `expires` INTEGER, `data` BLOB)", nullptr, nullptr, nullptr); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + closeDatabase(); + } + } +} +void FileSource::closeDatabase() { + if (db) { + const int err = sqlite3_close_v2(db); + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + db = nullptr; + } +} void FileSource::setBase(const std::string &value) { base = value; @@ -17,6 +50,104 @@ const std::string &FileSource::getBase() const { return base; } +bool FileSource::loadFile(const std::string &url, std::function callback) { + if (!db) { + return false; + } + + sqlite3_stmt *stmt = nullptr; + int err; + + err = sqlite3_prepare_v2(db, "SELECT `data` FROM `http_cache` WHERE `url` = ?", -1, &stmt, nullptr); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + return false; + } + + err = sqlite3_bind_text(stmt, 1, url.data(), url.size(), nullptr); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + return false; + } + + bool status = false; + + err = sqlite3_step(stmt); + if (err == SQLITE_ROW) { + // There is data. + platform::Response res(callback); + res.code = 200; + const char *data = reinterpret_cast(sqlite3_column_blob(stmt, 0)); + const size_t size = sqlite3_column_bytes(stmt, 0); + res.body = { data, size }; + callback(&res); + status = true; + } else if (err == SQLITE_DONE) { + // There is no data. + // This is a noop. + } else { + // Another error occured. + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + return status; +} + +void FileSource::saveFile(const std::string &url, platform::Response *const res) { + if (!db) { + return; + } + + sqlite3_stmt *stmt = nullptr; + int err; + + err = sqlite3_prepare_v2(db, "REPLACE INTO `http_cache` (`url`, `data`) VALUES(?, ?)", -1, &stmt, nullptr); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + return; + } + + err = sqlite3_bind_text(stmt, 1, url.data(), url.size(), nullptr); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + return; + } + + err = sqlite3_bind_blob(stmt, 2, res->body.data(), res->body.size(), nullptr); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + return; + } + + err = sqlite3_step(stmt); + if (err != SQLITE_DONE) { + // Another error occured. + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } +} + + void FileSource::load(ResourceType /*type*/, const std::string &url, std::function callback, const std::shared_ptr loop) { // convert relative URLs to absolute URLs @@ -51,7 +182,15 @@ void FileSource::load(ResourceType /*type*/, const std::string &url, std::functi callback(&response); } else { // load from the internet - platform::request_http(absoluteURL, callback, loop); + if (!loadFile(absoluteURL, callback)) { + const std::shared_ptr source = shared_from_this(); + platform::request_http(absoluteURL, [=](platform::Response *res) { + if (res->code == 200) { + source->saveFile(absoluteURL, res); + } + callback(res); + }, loop); + } } } -- cgit v1.2.1 From a9a2b3ad330073894fcf7c65d4aacb205b723ef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Mon, 25 Aug 2014 12:42:34 +0200 Subject: remove access token from URL --- src/util/filesource.cpp | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) (limited to 'src/util/filesource.cpp') diff --git a/src/util/filesource.cpp b/src/util/filesource.cpp index 3f52a09119..bca2cfbc41 100644 --- a/src/util/filesource.cpp +++ b/src/util/filesource.cpp @@ -50,6 +50,23 @@ const std::string &FileSource::getBase() const { return base; } +std::string removeAccessTokenFromURL(const std::string &url) { + const size_t token_start = url.find("access_token="); + // Ensure that token exists, isn't at the front and is preceded by either & or ?. + if (token_start == std::string::npos || token_start == 0 || !(url[token_start - 1] == '&' || url[token_start - 1] == '?')) { + return url; + } + + const size_t token_end = url.find_first_of('&', token_start); + if (token_end == std::string::npos) { + // The token is the last query argument. We slice away the "&access_token=..." part + return url.substr(0, token_start - 1); + } else { + // We slice away the "access_token=...&" part. + return url.substr(0, token_start) + url.substr(token_end + 1); + } +} + bool FileSource::loadFile(const std::string &url, std::function callback) { if (!db) { return false; @@ -181,12 +198,15 @@ void FileSource::load(ResourceType /*type*/, const std::string &url, std::functi callback(&response); } else { + // Don't use the access token when storing/retrieving URLs from the database + const std::string cleanURL = removeAccessTokenFromURL(url); + // load from the internet - if (!loadFile(absoluteURL, callback)) { + if (!loadFile(cleanURL, callback)) { const std::shared_ptr source = shared_from_this(); platform::request_http(absoluteURL, [=](platform::Response *res) { if (res->code == 200) { - source->saveFile(absoluteURL, res); + source->saveFile(cleanURL, res); } callback(res); }, loop); -- cgit v1.2.1 From 83fe11e0fc03dc7851f1e74974bbeb0ee3e37e21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Mon, 25 Aug 2014 12:50:01 +0200 Subject: also store other response codes in the cache --- src/util/filesource.cpp | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) (limited to 'src/util/filesource.cpp') diff --git a/src/util/filesource.cpp b/src/util/filesource.cpp index bca2cfbc41..0184659905 100644 --- a/src/util/filesource.cpp +++ b/src/util/filesource.cpp @@ -26,7 +26,13 @@ FileSource::~FileSource() { void FileSource::createSchema() { if (db) { // Create schema - const int err = sqlite3_exec(db, "CREATE TABLE IF NOT EXISTS `http_cache` (`url` TEXT PRIMARY KEY, `expires` INTEGER, `data` BLOB)", nullptr, nullptr, nullptr); + const int err = sqlite3_exec(db, + "CREATE TABLE IF NOT EXISTS `http_cache` (" + "`url` TEXT PRIMARY KEY NOT NULL," + "`code` INTEGER NOT NULL," + "`expires` INTEGER," + "`data` BLOB" + ")", nullptr, nullptr, nullptr); if (err != SQLITE_OK) { Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); closeDatabase(); @@ -75,7 +81,7 @@ bool FileSource::loadFile(const std::string &url, std::function(sqlite3_column_blob(stmt, 0)); - const size_t size = sqlite3_column_bytes(stmt, 0); + res.code = sqlite3_column_int(stmt, 0); + const char *data = reinterpret_cast(sqlite3_column_blob(stmt, 1)); + const size_t size = sqlite3_column_bytes(stmt, 1); res.body = { data, size }; callback(&res); status = true; @@ -126,7 +132,7 @@ void FileSource::saveFile(const std::string &url, platform::Response *const res) sqlite3_stmt *stmt = nullptr; int err; - err = sqlite3_prepare_v2(db, "REPLACE INTO `http_cache` (`url`, `data`) VALUES(?, ?)", -1, &stmt, nullptr); + err = sqlite3_prepare_v2(db, "REPLACE INTO `http_cache` (`url`, `code`, `data`) VALUES(?, ?, ?)", -1, &stmt, nullptr); if (err != SQLITE_OK) { Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); return; @@ -142,7 +148,17 @@ void FileSource::saveFile(const std::string &url, platform::Response *const res) return; } - err = sqlite3_bind_blob(stmt, 2, res->body.data(), res->body.size(), nullptr); + err = sqlite3_bind_int(stmt, 2, res->code); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + return; + } + + err = sqlite3_bind_blob(stmt, 3, res->body.data(), res->body.size(), nullptr); if (err != SQLITE_OK) { Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); err = sqlite3_finalize(stmt); @@ -205,9 +221,7 @@ void FileSource::load(ResourceType /*type*/, const std::string &url, std::functi if (!loadFile(cleanURL, callback)) { const std::shared_ptr source = shared_from_this(); platform::request_http(absoluteURL, [=](platform::Response *res) { - if (res->code == 200) { - source->saveFile(cleanURL, res); - } + source->saveFile(cleanURL, res); callback(res); }, loop); } -- cgit v1.2.1 From 28446bed24689b2bac7c8bfbd37741a7fa4fa6be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Mon, 25 Aug 2014 14:02:40 +0200 Subject: compress data and store type --- src/util/filesource.cpp | 63 ++++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 52 insertions(+), 11 deletions(-) (limited to 'src/util/filesource.cpp') diff --git a/src/util/filesource.cpp b/src/util/filesource.cpp index 0184659905..d626092bce 100644 --- a/src/util/filesource.cpp +++ b/src/util/filesource.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include @@ -30,9 +31,12 @@ void FileSource::createSchema() { "CREATE TABLE IF NOT EXISTS `http_cache` (" "`url` TEXT PRIMARY KEY NOT NULL," "`code` INTEGER NOT NULL," + "`type` INTEGER NOT NULL," "`expires` INTEGER," - "`data` BLOB" - ")", nullptr, nullptr, nullptr); + "`data` BLOB," + "`compressed` INTEGER NOT NULL DEFAULT 0" + ");" + "CREATE INDEX IF NOT EXISTS `http_cache_type_idx` ON `http_cache` (`type`);", nullptr, nullptr, nullptr); if (err != SQLITE_OK) { Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); closeDatabase(); @@ -73,7 +77,7 @@ std::string removeAccessTokenFromURL(const std::string &url) { } } -bool FileSource::loadFile(const std::string &url, std::function callback) { +bool FileSource::loadFile(ResourceType /*type*/, const std::string &url, std::function callback) { if (!db) { return false; } @@ -81,7 +85,7 @@ bool FileSource::loadFile(const std::string &url, std::function(sqlite3_column_blob(stmt, 1)); const size_t size = sqlite3_column_bytes(stmt, 1); - res.body = { data, size }; + const bool compressed = sqlite3_column_int(stmt, 2); + + if (compressed) { + res.body = util::decompress({ data, size }); + } else { + res.body = { data, size }; + } + callback(&res); status = true; } else if (err == SQLITE_DONE) { @@ -124,7 +135,7 @@ bool FileSource::loadFile(const std::string &url, std::functionbody.data(), res->body.size(), nullptr); + err = sqlite3_bind_int(stmt, 3, int(type)); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + return; + } + + bool compressed = false; + switch (type) { + case ResourceType::Image: + err = sqlite3_bind_blob(stmt, 4, res->body.data(), res->body.size(), SQLITE_STATIC); + break; + default: + const std::string *data = new std::string(std::move(util::compress(res->body))); + compressed = true; + err = sqlite3_bind_blob(stmt, 4, data->data(), data->size(), [](void *p) { delete (std::string *)p; }); + break; + } + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + return; + } + + err = sqlite3_bind_int(stmt, 5, compressed); if (err != SQLITE_OK) { Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); err = sqlite3_finalize(stmt); @@ -181,7 +222,7 @@ void FileSource::saveFile(const std::string &url, platform::Response *const res) } -void FileSource::load(ResourceType /*type*/, const std::string &url, std::function callback, const std::shared_ptr loop) { +void FileSource::load(ResourceType type, const std::string &url, std::function callback, const std::shared_ptr loop) { // convert relative URLs to absolute URLs const std::string absoluteURL = [&]() -> std::string { @@ -218,10 +259,10 @@ void FileSource::load(ResourceType /*type*/, const std::string &url, std::functi const std::string cleanURL = removeAccessTokenFromURL(url); // load from the internet - if (!loadFile(cleanURL, callback)) { + if (!loadFile(type, cleanURL, callback)) { const std::shared_ptr source = shared_from_this(); platform::request_http(absoluteURL, [=](platform::Response *res) { - source->saveFile(cleanURL, res); + source->saveFile(type, cleanURL, res); callback(res); }, loop); } -- cgit v1.2.1 From b94635f70430c3659cd57596e49649d376284473 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Mon, 25 Aug 2014 16:56:56 +0200 Subject: parse cache-control and last-modified headers --- src/util/filesource.cpp | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) (limited to 'src/util/filesource.cpp') diff --git a/src/util/filesource.cpp b/src/util/filesource.cpp index d626092bce..8c14f6e4e0 100644 --- a/src/util/filesource.cpp +++ b/src/util/filesource.cpp @@ -32,6 +32,7 @@ void FileSource::createSchema() { "`url` TEXT PRIMARY KEY NOT NULL," "`code` INTEGER NOT NULL," "`type` INTEGER NOT NULL," + "`modified` INTEGER," "`expires` INTEGER," "`data` BLOB," "`compressed` INTEGER NOT NULL DEFAULT 0" @@ -143,7 +144,7 @@ void FileSource::saveFile(ResourceType type, const std::string &url, platform::R sqlite3_stmt *stmt = nullptr; int err; - err = sqlite3_prepare_v2(db, "REPLACE INTO `http_cache` (`url`, `code`, `type`, `data`, `compressed`) VALUES(?, ?, ?, ?, ?)", -1, &stmt, nullptr); + err = sqlite3_prepare_v2(db, "REPLACE INTO `http_cache` (`url`, `code`, `type`, `modified`, `expires`, `data`, `compressed`) VALUES(?, ?, ?, ?, ?, ?, ?)", -1, &stmt, nullptr); if (err != SQLITE_OK) { Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); return; @@ -179,15 +180,35 @@ void FileSource::saveFile(ResourceType type, const std::string &url, platform::R return; } + err = sqlite3_bind_int(stmt, 4, res->modified); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + return; + } + + err = sqlite3_bind_int(stmt, 5, res->expires); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + return; + } + bool compressed = false; switch (type) { case ResourceType::Image: - err = sqlite3_bind_blob(stmt, 4, res->body.data(), res->body.size(), SQLITE_STATIC); + err = sqlite3_bind_blob(stmt, 6, res->body.data(), res->body.size(), SQLITE_STATIC); break; default: const std::string *data = new std::string(std::move(util::compress(res->body))); compressed = true; - err = sqlite3_bind_blob(stmt, 4, data->data(), data->size(), [](void *p) { delete (std::string *)p; }); + err = sqlite3_bind_blob(stmt, 6, data->data(), data->size(), [](void *p) { delete (std::string *)p; }); break; } if (err != SQLITE_OK) { @@ -199,7 +220,7 @@ void FileSource::saveFile(ResourceType type, const std::string &url, platform::R return; } - err = sqlite3_bind_int(stmt, 5, compressed); + err = sqlite3_bind_int(stmt, 7, compressed); if (err != SQLITE_OK) { Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); err = sqlite3_finalize(stmt); -- cgit v1.2.1 From a6fc961e2f8498afac3c0c94fe2adeeef8156210 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Mon, 25 Aug 2014 17:12:24 +0200 Subject: don't report SQLITE_OK as an error --- src/util/filesource.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'src/util/filesource.cpp') diff --git a/src/util/filesource.cpp b/src/util/filesource.cpp index 8c14f6e4e0..6f7ad66b24 100644 --- a/src/util/filesource.cpp +++ b/src/util/filesource.cpp @@ -48,7 +48,9 @@ void FileSource::createSchema() { void FileSource::closeDatabase() { if (db) { const int err = sqlite3_close_v2(db); - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } db = nullptr; } } -- cgit v1.2.1 From b757805ffb2d38162e53359f6321e87fcd1ca1de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Tue, 26 Aug 2014 02:49:06 -0700 Subject: fix issue that deleted the data part of a std::string as a std::string object --- src/util/filesource.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'src/util/filesource.cpp') diff --git a/src/util/filesource.cpp b/src/util/filesource.cpp index 6f7ad66b24..ec46247b61 100644 --- a/src/util/filesource.cpp +++ b/src/util/filesource.cpp @@ -202,15 +202,16 @@ void FileSource::saveFile(ResourceType type, const std::string &url, platform::R return; } + std::string data; bool compressed = false; switch (type) { case ResourceType::Image: err = sqlite3_bind_blob(stmt, 6, res->body.data(), res->body.size(), SQLITE_STATIC); break; default: - const std::string *data = new std::string(std::move(util::compress(res->body))); + data = std::move(util::compress(res->body)); compressed = true; - err = sqlite3_bind_blob(stmt, 6, data->data(), data->size(), [](void *p) { delete (std::string *)p; }); + err = sqlite3_bind_blob(stmt, 6, data.data(), data.size(), SQLITE_STATIC); break; } if (err != SQLITE_OK) { -- cgit v1.2.1 From 062e911c6d570a794431023f9f0cb0b02cd85667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Tue, 26 Aug 2014 16:55:17 +0200 Subject: remove shared_from_this() usage --- src/util/filesource.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'src/util/filesource.cpp') diff --git a/src/util/filesource.cpp b/src/util/filesource.cpp index ec46247b61..339b5b1e1d 100644 --- a/src/util/filesource.cpp +++ b/src/util/filesource.cpp @@ -284,7 +284,7 @@ void FileSource::load(ResourceType type, const std::string &url, std::function source = shared_from_this(); + FileSource *source = this; platform::request_http(absoluteURL, [=](platform::Response *res) { source->saveFile(type, cleanURL, res); callback(res); -- cgit v1.2.1 From d9fc7708a2dfb6e2506a5d10d896a813557c056d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Mon, 15 Sep 2014 17:26:44 +0200 Subject: do 304 requests and cache them in sqlite --- src/util/filesource.cpp | 296 ------------------------------------------------ 1 file changed, 296 deletions(-) delete mode 100644 src/util/filesource.cpp (limited to 'src/util/filesource.cpp') diff --git a/src/util/filesource.cpp b/src/util/filesource.cpp deleted file mode 100644 index 339b5b1e1d..0000000000 --- a/src/util/filesource.cpp +++ /dev/null @@ -1,296 +0,0 @@ -#include -#include -#include -#include - -#include - -#include -#include - -namespace mbgl { - -FileSource::FileSource() { - const int err = sqlite3_open_v2("cache.db", &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nullptr); - if (err != SQLITE_OK) { - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - db = nullptr; - } - - createSchema(); -} - -FileSource::~FileSource() { - closeDatabase(); -} - -void FileSource::createSchema() { - if (db) { - // Create schema - const int err = sqlite3_exec(db, - "CREATE TABLE IF NOT EXISTS `http_cache` (" - "`url` TEXT PRIMARY KEY NOT NULL," - "`code` INTEGER NOT NULL," - "`type` INTEGER NOT NULL," - "`modified` INTEGER," - "`expires` INTEGER," - "`data` BLOB," - "`compressed` INTEGER NOT NULL DEFAULT 0" - ");" - "CREATE INDEX IF NOT EXISTS `http_cache_type_idx` ON `http_cache` (`type`);", nullptr, nullptr, nullptr); - if (err != SQLITE_OK) { - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - closeDatabase(); - } - } -} - -void FileSource::closeDatabase() { - if (db) { - const int err = sqlite3_close_v2(db); - if (err != SQLITE_OK) { - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - } - db = nullptr; - } -} - -void FileSource::setBase(const std::string &value) { - base = value; -} - -const std::string &FileSource::getBase() const { - return base; -} - -std::string removeAccessTokenFromURL(const std::string &url) { - const size_t token_start = url.find("access_token="); - // Ensure that token exists, isn't at the front and is preceded by either & or ?. - if (token_start == std::string::npos || token_start == 0 || !(url[token_start - 1] == '&' || url[token_start - 1] == '?')) { - return url; - } - - const size_t token_end = url.find_first_of('&', token_start); - if (token_end == std::string::npos) { - // The token is the last query argument. We slice away the "&access_token=..." part - return url.substr(0, token_start - 1); - } else { - // We slice away the "access_token=...&" part. - return url.substr(0, token_start) + url.substr(token_end + 1); - } -} - -bool FileSource::loadFile(ResourceType /*type*/, const std::string &url, std::function callback) { - if (!db) { - return false; - } - - sqlite3_stmt *stmt = nullptr; - int err; - - err = sqlite3_prepare_v2(db, "SELECT `code`, `data`, `compressed` FROM `http_cache` WHERE `url` = ?", -1, &stmt, nullptr); - if (err != SQLITE_OK) { - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - return false; - } - - err = sqlite3_bind_text(stmt, 1, url.data(), url.size(), nullptr); - if (err != SQLITE_OK) { - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - err = sqlite3_finalize(stmt); - if (err != SQLITE_OK) { - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - } - return false; - } - - bool status = false; - - err = sqlite3_step(stmt); - if (err == SQLITE_ROW) { - // There is data. - platform::Response res(callback); - res.code = sqlite3_column_int(stmt, 0); - const char *data = reinterpret_cast(sqlite3_column_blob(stmt, 1)); - const size_t size = sqlite3_column_bytes(stmt, 1); - const bool compressed = sqlite3_column_int(stmt, 2); - - if (compressed) { - res.body = util::decompress({ data, size }); - } else { - res.body = { data, size }; - } - - callback(&res); - status = true; - } else if (err == SQLITE_DONE) { - // There is no data. - // This is a noop. - } else { - // Another error occured. - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - } - - err = sqlite3_finalize(stmt); - if (err != SQLITE_OK) { - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - } - return status; -} - -void FileSource::saveFile(ResourceType type, const std::string &url, platform::Response *const res) { - if (!db) { - return; - } - - sqlite3_stmt *stmt = nullptr; - int err; - - err = sqlite3_prepare_v2(db, "REPLACE INTO `http_cache` (`url`, `code`, `type`, `modified`, `expires`, `data`, `compressed`) VALUES(?, ?, ?, ?, ?, ?, ?)", -1, &stmt, nullptr); - if (err != SQLITE_OK) { - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - return; - } - - err = sqlite3_bind_text(stmt, 1, url.data(), url.size(), nullptr); - if (err != SQLITE_OK) { - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - err = sqlite3_finalize(stmt); - if (err != SQLITE_OK) { - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - } - return; - } - - err = sqlite3_bind_int(stmt, 2, res->code); - if (err != SQLITE_OK) { - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - err = sqlite3_finalize(stmt); - if (err != SQLITE_OK) { - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - } - return; - } - - err = sqlite3_bind_int(stmt, 3, int(type)); - if (err != SQLITE_OK) { - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - err = sqlite3_finalize(stmt); - if (err != SQLITE_OK) { - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - } - return; - } - - err = sqlite3_bind_int(stmt, 4, res->modified); - if (err != SQLITE_OK) { - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - err = sqlite3_finalize(stmt); - if (err != SQLITE_OK) { - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - } - return; - } - - err = sqlite3_bind_int(stmt, 5, res->expires); - if (err != SQLITE_OK) { - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - err = sqlite3_finalize(stmt); - if (err != SQLITE_OK) { - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - } - return; - } - - std::string data; - bool compressed = false; - switch (type) { - case ResourceType::Image: - err = sqlite3_bind_blob(stmt, 6, res->body.data(), res->body.size(), SQLITE_STATIC); - break; - default: - data = std::move(util::compress(res->body)); - compressed = true; - err = sqlite3_bind_blob(stmt, 6, data.data(), data.size(), SQLITE_STATIC); - break; - } - if (err != SQLITE_OK) { - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - err = sqlite3_finalize(stmt); - if (err != SQLITE_OK) { - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - } - return; - } - - err = sqlite3_bind_int(stmt, 7, compressed); - if (err != SQLITE_OK) { - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - err = sqlite3_finalize(stmt); - if (err != SQLITE_OK) { - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - } - return; - } - - err = sqlite3_step(stmt); - if (err != SQLITE_DONE) { - // Another error occured. - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - } - - err = sqlite3_finalize(stmt); - if (err != SQLITE_OK) { - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); - } -} - - -void FileSource::load(ResourceType type, const std::string &url, std::function callback, const std::shared_ptr loop) { - // convert relative URLs to absolute URLs - - const std::string absoluteURL = [&]() -> std::string { - const size_t separator = url.find("://"); - if (separator == std::string::npos) { - // Relative URL. - return base + url; - } else { - return url; - } - }(); - - const size_t separator = absoluteURL.find("://"); - const std::string protocol = separator != std::string::npos ? absoluteURL.substr(0, separator) : ""; - - if (protocol == "file") { - // load from disk - const std::string path = absoluteURL.substr(separator + 3); - std::ifstream file(path); - - platform::Response response(callback); - if (!file.good()) { - response.error_message = "file not found (" + path + ")"; - } else { - std::stringstream data; - data << file.rdbuf(); - response.code = 200; - response.body = data.str(); - } - - callback(&response); - } else { - // Don't use the access token when storing/retrieving URLs from the database - const std::string cleanURL = removeAccessTokenFromURL(url); - - // load from the internet - if (!loadFile(type, cleanURL, callback)) { - FileSource *source = this; - platform::request_http(absoluteURL, [=](platform::Response *res) { - source->saveFile(type, cleanURL, res); - callback(res); - }, loop); - } - } -} - -} \ No newline at end of file -- cgit v1.2.1