From 34fe7dc627f08d57ac259c383e248ddf2f6fb0d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Fri, 22 Aug 2014 18:50:09 +0200 Subject: add basic request caching with sqlite --- configure | 9 +++ include/mbgl/platform/event.hpp | 2 + include/mbgl/util/filesource.hpp | 15 +++- mapboxgl.gyp | 53 ++++++++++----- setup-libraries.sh | 7 ++ src/util/filesource.cpp | 143 ++++++++++++++++++++++++++++++++++++++- 6 files changed, 210 insertions(+), 19 deletions(-) diff --git a/configure b/configure index e8f880b543..b92f97758b 100755 --- a/configure +++ b/configure @@ -107,6 +107,14 @@ def configure_curl(o): o['variables']['curl_libraries'] = ret[0].split() o['variables']['curl_cflags'] = ret[1].split() +def configure_sqlite3(o): + ret = pkg_config('sqlite3', options.pkgconfig_root) + if not ret: + sys.stderr.write('could not find sqlite3 with pkg-config') + sys.exit(-1) + o['variables']['sqlite3_libraries'] = ret[0].split() + o['variables']['sqlite3_cflags'] = ret[1].split() + def write(filename, data): filename = os.path.join(root_dir, filename) print "creating ", filename @@ -129,6 +137,7 @@ if __name__ == '__main__': configure_uv(output) configure_png(output) configure_curl(output) + configure_sqlite3(output) pprint.pprint(output, indent=2) write('config.gypi', "# Do not edit. Generated by the configure script.\n" + diff --git a/include/mbgl/platform/event.hpp b/include/mbgl/platform/event.hpp index 1676f40d2c..d2a06618a3 100644 --- a/include/mbgl/platform/event.hpp +++ b/include/mbgl/platform/event.hpp @@ -29,6 +29,7 @@ enum class Event : uint8_t { ParseStyle, ParseTile, Render, + Database, HttpRequest, Sprite, }; @@ -40,6 +41,7 @@ MBGL_DEFINE_ENUM_CLASS(EventClass, Event, { { Event::ParseStyle, "ParseStyle" }, { Event::ParseTile, "ParseTile" }, { Event::Render, "Render" }, + { Event::Database, "Database" }, { Event::HttpRequest, "HttpRequest" }, { Event::Sprite, "Sprite" }, { Event(-1), "Unknown" }, diff --git a/include/mbgl/util/filesource.hpp b/include/mbgl/util/filesource.hpp index 18c63ddfeb..bc0a2927de 100644 --- a/include/mbgl/util/filesource.hpp +++ b/include/mbgl/util/filesource.hpp @@ -2,11 +2,15 @@ #define MBGL_UTIL_FILESOURCE #include +#include #include #include #include +typedef struct sqlite3 sqlite3; +typedef struct sqlite3_stmt sqlite3_stmt; + namespace mbgl { namespace platform { @@ -21,21 +25,30 @@ enum class ResourceType : uint8_t { JSON }; -class FileSource { +class FileSource : private util::noncopyable, public std::enable_shared_from_this { public: FileSource(); + ~FileSource(); void setBase(const std::string &value); const std::string &getBase() const; void load(ResourceType type, const std::string &url, std::function callback, const std::shared_ptr loop = nullptr); +private: + void closeDatabase(); + void createSchema(); + bool loadFile(const std::string &url, std::function callback); + void saveFile(const std::string &url, platform::Response *res); + private: // Stores a URL that is used as a base for loading resources with relative path. std::string base; // Stores the absolute path to the cache directory. const std::string cache; + + sqlite3 *db = nullptr; }; } diff --git a/mapboxgl.gyp b/mapboxgl.gyp index dcd89f3a76..49c4150aa1 100644 --- a/mapboxgl.gyp +++ b/mapboxgl.gyp @@ -126,23 +126,36 @@ ' #include +#include + +#include #include #include namespace mbgl { -FileSource::FileSource() {} +FileSource::FileSource() { + const int err = sqlite3_open_v2("cache.db", &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nullptr); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + db = nullptr; + } + + createSchema(); +} + +FileSource::~FileSource() { + closeDatabase(); +} + +void FileSource::createSchema() { + if (db) { + // Create schema + const int err = sqlite3_exec(db, "CREATE TABLE IF NOT EXISTS `http_cache` (`url` TEXT PRIMARY KEY, `expires` INTEGER, `data` BLOB)", nullptr, nullptr, nullptr); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + closeDatabase(); + } + } +} +void FileSource::closeDatabase() { + if (db) { + const int err = sqlite3_close_v2(db); + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + db = nullptr; + } +} void FileSource::setBase(const std::string &value) { base = value; @@ -17,6 +50,104 @@ const std::string &FileSource::getBase() const { return base; } +bool FileSource::loadFile(const std::string &url, std::function callback) { + if (!db) { + return false; + } + + sqlite3_stmt *stmt = nullptr; + int err; + + err = sqlite3_prepare_v2(db, "SELECT `data` FROM `http_cache` WHERE `url` = ?", -1, &stmt, nullptr); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + return false; + } + + err = sqlite3_bind_text(stmt, 1, url.data(), url.size(), nullptr); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + return false; + } + + bool status = false; + + err = sqlite3_step(stmt); + if (err == SQLITE_ROW) { + // There is data. + platform::Response res(callback); + res.code = 200; + const char *data = reinterpret_cast(sqlite3_column_blob(stmt, 0)); + const size_t size = sqlite3_column_bytes(stmt, 0); + res.body = { data, size }; + callback(&res); + status = true; + } else if (err == SQLITE_DONE) { + // There is no data. + // This is a noop. + } else { + // Another error occured. + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + return status; +} + +void FileSource::saveFile(const std::string &url, platform::Response *const res) { + if (!db) { + return; + } + + sqlite3_stmt *stmt = nullptr; + int err; + + err = sqlite3_prepare_v2(db, "REPLACE INTO `http_cache` (`url`, `data`) VALUES(?, ?)", -1, &stmt, nullptr); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + return; + } + + err = sqlite3_bind_text(stmt, 1, url.data(), url.size(), nullptr); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + return; + } + + err = sqlite3_bind_blob(stmt, 2, res->body.data(), res->body.size(), nullptr); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + return; + } + + err = sqlite3_step(stmt); + if (err != SQLITE_DONE) { + // Another error occured. + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } +} + + void FileSource::load(ResourceType /*type*/, const std::string &url, std::function callback, const std::shared_ptr loop) { // convert relative URLs to absolute URLs @@ -51,7 +182,15 @@ void FileSource::load(ResourceType /*type*/, const std::string &url, std::functi callback(&response); } else { // load from the internet - platform::request_http(absoluteURL, callback, loop); + if (!loadFile(absoluteURL, callback)) { + const std::shared_ptr source = shared_from_this(); + platform::request_http(absoluteURL, [=](platform::Response *res) { + if (res->code == 200) { + source->saveFile(absoluteURL, res); + } + callback(res); + }, loop); + } } } -- cgit v1.2.1 From a9a2b3ad330073894fcf7c65d4aacb205b723ef0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Mon, 25 Aug 2014 12:42:34 +0200 Subject: remove access token from URL --- src/util/filesource.cpp | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/src/util/filesource.cpp b/src/util/filesource.cpp index 3f52a09119..bca2cfbc41 100644 --- a/src/util/filesource.cpp +++ b/src/util/filesource.cpp @@ -50,6 +50,23 @@ const std::string &FileSource::getBase() const { return base; } +std::string removeAccessTokenFromURL(const std::string &url) { + const size_t token_start = url.find("access_token="); + // Ensure that token exists, isn't at the front and is preceded by either & or ?. + if (token_start == std::string::npos || token_start == 0 || !(url[token_start - 1] == '&' || url[token_start - 1] == '?')) { + return url; + } + + const size_t token_end = url.find_first_of('&', token_start); + if (token_end == std::string::npos) { + // The token is the last query argument. We slice away the "&access_token=..." part + return url.substr(0, token_start - 1); + } else { + // We slice away the "access_token=...&" part. + return url.substr(0, token_start) + url.substr(token_end + 1); + } +} + bool FileSource::loadFile(const std::string &url, std::function callback) { if (!db) { return false; @@ -181,12 +198,15 @@ void FileSource::load(ResourceType /*type*/, const std::string &url, std::functi callback(&response); } else { + // Don't use the access token when storing/retrieving URLs from the database + const std::string cleanURL = removeAccessTokenFromURL(url); + // load from the internet - if (!loadFile(absoluteURL, callback)) { + if (!loadFile(cleanURL, callback)) { const std::shared_ptr source = shared_from_this(); platform::request_http(absoluteURL, [=](platform::Response *res) { if (res->code == 200) { - source->saveFile(absoluteURL, res); + source->saveFile(cleanURL, res); } callback(res); }, loop); -- cgit v1.2.1 From 83fe11e0fc03dc7851f1e74974bbeb0ee3e37e21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Mon, 25 Aug 2014 12:50:01 +0200 Subject: also store other response codes in the cache --- src/util/filesource.cpp | 34 ++++++++++++++++++++++++---------- 1 file changed, 24 insertions(+), 10 deletions(-) diff --git a/src/util/filesource.cpp b/src/util/filesource.cpp index bca2cfbc41..0184659905 100644 --- a/src/util/filesource.cpp +++ b/src/util/filesource.cpp @@ -26,7 +26,13 @@ FileSource::~FileSource() { void FileSource::createSchema() { if (db) { // Create schema - const int err = sqlite3_exec(db, "CREATE TABLE IF NOT EXISTS `http_cache` (`url` TEXT PRIMARY KEY, `expires` INTEGER, `data` BLOB)", nullptr, nullptr, nullptr); + const int err = sqlite3_exec(db, + "CREATE TABLE IF NOT EXISTS `http_cache` (" + "`url` TEXT PRIMARY KEY NOT NULL," + "`code` INTEGER NOT NULL," + "`expires` INTEGER," + "`data` BLOB" + ")", nullptr, nullptr, nullptr); if (err != SQLITE_OK) { Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); closeDatabase(); @@ -75,7 +81,7 @@ bool FileSource::loadFile(const std::string &url, std::function(sqlite3_column_blob(stmt, 0)); - const size_t size = sqlite3_column_bytes(stmt, 0); + res.code = sqlite3_column_int(stmt, 0); + const char *data = reinterpret_cast(sqlite3_column_blob(stmt, 1)); + const size_t size = sqlite3_column_bytes(stmt, 1); res.body = { data, size }; callback(&res); status = true; @@ -126,7 +132,7 @@ void FileSource::saveFile(const std::string &url, platform::Response *const res) sqlite3_stmt *stmt = nullptr; int err; - err = sqlite3_prepare_v2(db, "REPLACE INTO `http_cache` (`url`, `data`) VALUES(?, ?)", -1, &stmt, nullptr); + err = sqlite3_prepare_v2(db, "REPLACE INTO `http_cache` (`url`, `code`, `data`) VALUES(?, ?, ?)", -1, &stmt, nullptr); if (err != SQLITE_OK) { Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); return; @@ -142,7 +148,17 @@ void FileSource::saveFile(const std::string &url, platform::Response *const res) return; } - err = sqlite3_bind_blob(stmt, 2, res->body.data(), res->body.size(), nullptr); + err = sqlite3_bind_int(stmt, 2, res->code); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + return; + } + + err = sqlite3_bind_blob(stmt, 3, res->body.data(), res->body.size(), nullptr); if (err != SQLITE_OK) { Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); err = sqlite3_finalize(stmt); @@ -205,9 +221,7 @@ void FileSource::load(ResourceType /*type*/, const std::string &url, std::functi if (!loadFile(cleanURL, callback)) { const std::shared_ptr source = shared_from_this(); platform::request_http(absoluteURL, [=](platform::Response *res) { - if (res->code == 200) { - source->saveFile(cleanURL, res); - } + source->saveFile(cleanURL, res); callback(res); }, loop); } -- cgit v1.2.1 From 28446bed24689b2bac7c8bfbd37741a7fa4fa6be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Mon, 25 Aug 2014 14:02:40 +0200 Subject: compress data and store type --- include/mbgl/util/compression.hpp | 15 ++++++++ include/mbgl/util/filesource.hpp | 14 +++---- src/util/compression.cpp | 78 +++++++++++++++++++++++++++++++++++++++ src/util/filesource.cpp | 63 +++++++++++++++++++++++++------ 4 files changed, 152 insertions(+), 18 deletions(-) create mode 100644 include/mbgl/util/compression.hpp create mode 100644 src/util/compression.cpp diff --git a/include/mbgl/util/compression.hpp b/include/mbgl/util/compression.hpp new file mode 100644 index 0000000000..a33b2476a7 --- /dev/null +++ b/include/mbgl/util/compression.hpp @@ -0,0 +1,15 @@ +#ifndef MBGL_UTIL_COMPRESSION +#define MBGL_UTIL_COMPRESSION + +#include + +namespace mbgl { +namespace util { + +std::string compress(const std::string &raw); +std::string decompress(const std::string &raw); + +} +} + +#endif diff --git a/include/mbgl/util/filesource.hpp b/include/mbgl/util/filesource.hpp index bc0a2927de..ccff4b5122 100644 --- a/include/mbgl/util/filesource.hpp +++ b/include/mbgl/util/filesource.hpp @@ -18,11 +18,11 @@ struct Response; } enum class ResourceType : uint8_t { - Unknown, - Tile, - Glyphs, - Image, - JSON + Unknown = 0, + Tile = 1, + Glyphs = 2, + Image = 3, + JSON = 4 }; class FileSource : private util::noncopyable, public std::enable_shared_from_this { @@ -38,8 +38,8 @@ public: private: void closeDatabase(); void createSchema(); - bool loadFile(const std::string &url, std::function callback); - void saveFile(const std::string &url, platform::Response *res); + bool loadFile(ResourceType type, const std::string &url, std::function callback); + void saveFile(ResourceType type, const std::string &url, platform::Response *res); private: // Stores a URL that is used as a base for loading resources with relative path. diff --git a/src/util/compression.cpp b/src/util/compression.cpp new file mode 100644 index 0000000000..3922501868 --- /dev/null +++ b/src/util/compression.cpp @@ -0,0 +1,78 @@ +#include + +#include + +namespace mbgl { +namespace util { + +std::string compress(const std::string &raw) { + z_stream deflate_stream; + memset(&deflate_stream, 0, sizeof(deflate_stream)); + + // TODO: reuse z_streams + if (deflateInit(&deflate_stream, Z_DEFAULT_COMPRESSION) != Z_OK) { + throw std::runtime_error("failed to initialize deflate"); + } + + deflate_stream.next_in = (Bytef *)raw.data(); + deflate_stream.avail_in = raw.size(); + + std::string result; + char out[16384]; + + int code; + do { + deflate_stream.next_out = reinterpret_cast(out); + deflate_stream.avail_out = sizeof(out); + code = deflate(&deflate_stream, Z_FINISH); + if (result.size() < deflate_stream.total_out) { + // append the block to the output string + result.append(out, deflate_stream.total_out - result.size()); + } + } while (code == Z_OK); + + deflateEnd(&deflate_stream); + + if (code != Z_STREAM_END) { + throw std::runtime_error(deflate_stream.msg); + } + + return result; +} + +std::string decompress(const std::string &raw) { + z_stream inflate_stream; + memset(&inflate_stream, 0, sizeof(inflate_stream)); + + // TODO: reuse z_streams + if (inflateInit(&inflate_stream) != Z_OK) { + throw std::runtime_error("failed to initialize inflate"); + } + + inflate_stream.next_in = (Bytef *)raw.data(); + inflate_stream.avail_in = raw.size(); + + std::string result; + char out[15384]; + + int code; + do { + inflate_stream.next_out = reinterpret_cast(out); + inflate_stream.avail_out = sizeof(out); + code = inflate(&inflate_stream, 0); + // result.append(out, sizeof(out) - inflate_stream.avail_out); + if (result.size() < inflate_stream.total_out) { + result.append(out, inflate_stream.total_out - result.size()); + } + } while (code == Z_OK); + + inflateEnd(&inflate_stream); + + if (code != Z_STREAM_END) { + throw std::runtime_error(inflate_stream.msg); + } + + return result; +} +} +} diff --git a/src/util/filesource.cpp b/src/util/filesource.cpp index 0184659905..d626092bce 100644 --- a/src/util/filesource.cpp +++ b/src/util/filesource.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include @@ -30,9 +31,12 @@ void FileSource::createSchema() { "CREATE TABLE IF NOT EXISTS `http_cache` (" "`url` TEXT PRIMARY KEY NOT NULL," "`code` INTEGER NOT NULL," + "`type` INTEGER NOT NULL," "`expires` INTEGER," - "`data` BLOB" - ")", nullptr, nullptr, nullptr); + "`data` BLOB," + "`compressed` INTEGER NOT NULL DEFAULT 0" + ");" + "CREATE INDEX IF NOT EXISTS `http_cache_type_idx` ON `http_cache` (`type`);", nullptr, nullptr, nullptr); if (err != SQLITE_OK) { Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); closeDatabase(); @@ -73,7 +77,7 @@ std::string removeAccessTokenFromURL(const std::string &url) { } } -bool FileSource::loadFile(const std::string &url, std::function callback) { +bool FileSource::loadFile(ResourceType /*type*/, const std::string &url, std::function callback) { if (!db) { return false; } @@ -81,7 +85,7 @@ bool FileSource::loadFile(const std::string &url, std::function(sqlite3_column_blob(stmt, 1)); const size_t size = sqlite3_column_bytes(stmt, 1); - res.body = { data, size }; + const bool compressed = sqlite3_column_int(stmt, 2); + + if (compressed) { + res.body = util::decompress({ data, size }); + } else { + res.body = { data, size }; + } + callback(&res); status = true; } else if (err == SQLITE_DONE) { @@ -124,7 +135,7 @@ bool FileSource::loadFile(const std::string &url, std::functionbody.data(), res->body.size(), nullptr); + err = sqlite3_bind_int(stmt, 3, int(type)); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + return; + } + + bool compressed = false; + switch (type) { + case ResourceType::Image: + err = sqlite3_bind_blob(stmt, 4, res->body.data(), res->body.size(), SQLITE_STATIC); + break; + default: + const std::string *data = new std::string(std::move(util::compress(res->body))); + compressed = true; + err = sqlite3_bind_blob(stmt, 4, data->data(), data->size(), [](void *p) { delete (std::string *)p; }); + break; + } + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + return; + } + + err = sqlite3_bind_int(stmt, 5, compressed); if (err != SQLITE_OK) { Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); err = sqlite3_finalize(stmt); @@ -181,7 +222,7 @@ void FileSource::saveFile(const std::string &url, platform::Response *const res) } -void FileSource::load(ResourceType /*type*/, const std::string &url, std::function callback, const std::shared_ptr loop) { +void FileSource::load(ResourceType type, const std::string &url, std::function callback, const std::shared_ptr loop) { // convert relative URLs to absolute URLs const std::string absoluteURL = [&]() -> std::string { @@ -218,10 +259,10 @@ void FileSource::load(ResourceType /*type*/, const std::string &url, std::functi const std::string cleanURL = removeAccessTokenFromURL(url); // load from the internet - if (!loadFile(cleanURL, callback)) { + if (!loadFile(type, cleanURL, callback)) { const std::shared_ptr source = shared_from_this(); platform::request_http(absoluteURL, [=](platform::Response *res) { - source->saveFile(cleanURL, res); + source->saveFile(type, cleanURL, res); callback(res); }, loop); } -- cgit v1.2.1 From b94635f70430c3659cd57596e49649d376284473 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Mon, 25 Aug 2014 16:56:56 +0200 Subject: parse cache-control and last-modified headers --- common/curl_request.cpp | 31 +++++++++++++++++++++++++++++++ common/foundation_request.mm | 4 ++++ include/mbgl/platform/platform.hpp | 11 ++--------- include/mbgl/platform/response.hpp | 29 +++++++++++++++++++++++++++++ linux/mapboxgl-app.gyp | 2 -- mapboxgl.gyp | 4 ++++ src/platform/response.cpp | 26 ++++++++++++++++++++++++++ src/util/filesource.cpp | 29 +++++++++++++++++++++++++---- 8 files changed, 121 insertions(+), 15 deletions(-) create mode 100644 include/mbgl/platform/response.hpp create mode 100644 src/platform/response.cpp diff --git a/common/curl_request.cpp b/common/curl_request.cpp index 416ed90cd1..0957481226 100644 --- a/common/curl_request.cpp +++ b/common/curl_request.cpp @@ -286,6 +286,35 @@ size_t curl_write_cb(void *contents, size_t size, size_t nmemb, void *userp) { return size * nmemb; } +// Compares the beginning of the (non-zero-terminated!) data buffer with the (zero-terminated!) +// header string. If the data buffer contains the header string at the beginning, it returns +// the length of the header string == begin of the value, otherwise it returns npos. +// The comparison of the header is ASCII-case-insensitive. +size_t header_matches(const char *header, const char *buffer, size_t length) { + const size_t header_length = strlen(header); + if (length < header_length) return std::string::npos; + size_t i = 0; + while (i < length && i < header_length && std::tolower(buffer[i]) == header[i]) { + i++; + } + return i == header_length ? i : std::string::npos; +} + +size_t curl_header_cb(char *buffer, size_t size, size_t nmemb, void *userp) { + const size_t length = size * nmemb; + + size_t begin = std::string::npos; + if ((begin = header_matches("last-modified: ", buffer, length)) != std::string::npos) { + const std::string value { buffer + begin, length - begin - 2 /* remove \r\n */ }; + static_cast(userp)->setLastModified(value.c_str()); + } else if ((begin = header_matches("cache-control: ", buffer, length)) != std::string::npos) { + const std::string value { buffer + begin, length - begin - 2 /* remove \r\n */ }; + static_cast(userp)->setCacheControl(value.c_str()); + } + + return length; +} + // This callback is called in the request event loop (on the request thread). // It initializes newly queued up download requests and adds them to the CURL // multi handle. @@ -315,6 +344,8 @@ void async_add_cb(uv_async_t * /*async*/) { curl_easy_setopt(handle, CURLOPT_URL, (*req)->url.c_str()); curl_easy_setopt(handle, CURLOPT_WRITEFUNCTION, curl_write_cb); curl_easy_setopt(handle, CURLOPT_WRITEDATA, &(*req)->res->body); + curl_easy_setopt(handle, CURLOPT_HEADERFUNCTION, curl_header_cb); + curl_easy_setopt(handle, CURLOPT_HEADERDATA, (*req)->res.get()); curl_easy_setopt(handle, CURLOPT_ACCEPT_ENCODING, "gzip, deflate"); curl_easy_setopt(handle, CURLOPT_SHARE, curl_share); curl_multi_add_handle(curl_multi, handle); diff --git a/common/foundation_request.mm b/common/foundation_request.mm index b7eafdb96c..478dadf4d9 100644 --- a/common/foundation_request.mm +++ b/common/foundation_request.mm @@ -91,8 +91,12 @@ mbgl::platform::request_http(const std::string &url, } if (!error && [response isKindOfClass:[NSHTTPURLResponse class]]) { + NSDictionary *headers = [(NSHTTPURLResponse *)response allHeaderFields]; (*req_ptr)->res->code = [(NSHTTPURLResponse *)response statusCode]; (*req_ptr)->res->body = {(const char *)[data bytes], [data length]}; + (*req_ptr)->res->setCacheControl([[headers objectForKey:@"Cache-Control"] UTF8String]); + (*req_ptr)->res->setLastModified([[headers objectForKey:@"Last-Modified"] UTF8String]); + } else { (*req_ptr)->res->error_message = [[error localizedDescription] UTF8String]; } diff --git a/include/mbgl/platform/platform.hpp b/include/mbgl/platform/platform.hpp index 22405a4cfd..e08cac5312 100644 --- a/include/mbgl/platform/platform.hpp +++ b/include/mbgl/platform/platform.hpp @@ -1,10 +1,11 @@ #ifndef MBGL_PLATFORM_PLATFORM #define MBGL_PLATFORM_PLATFORM +#include + #include #include -#include #include namespace mbgl { @@ -12,14 +13,6 @@ namespace platform { class Request; -struct Response { - Response(std::function callback) : callback(callback) {} - int16_t code = -1; - std::string body; - std::string error_message; - std::function callback; -}; - // Makes an HTTP request of a URL, preferrably on a background thread, and calls a function with the // results in the original thread (which runs the libuv loop). // If the loop pointer is NULL, the callback function will be called on an arbitrary thread. diff --git a/include/mbgl/platform/response.hpp b/include/mbgl/platform/response.hpp new file mode 100644 index 0000000000..345a2ee3df --- /dev/null +++ b/include/mbgl/platform/response.hpp @@ -0,0 +1,29 @@ +#ifndef MBGL_PLATFORM_RESPONSE +#define MBGL_PLATFORM_RESPONSE + +#include +#include +#include + +#include + +namespace mbgl { +namespace platform { + +struct Response : private util::noncopyable { + Response(std::function callback) : callback(callback) {} + int16_t code = -1; + std::string body; + std::string error_message; + std::time_t modified; + std::time_t expires; + std::function callback; + + void setCacheControl(const char *value); + void setLastModified(const char *value); +}; + +} +} + +#endif diff --git a/linux/mapboxgl-app.gyp b/linux/mapboxgl-app.gyp index e19768caf9..2a76fea1d6 100644 --- a/linux/mapboxgl-app.gyp +++ b/linux/mapboxgl-app.gyp @@ -32,7 +32,6 @@ ], 'OTHER_LDFLAGS': [ '<@(glfw3_libraries)', - '<@(curl_libraries)', ], } }, @@ -46,7 +45,6 @@ 'link_settings': { 'libraries': [ '<@(glfw3_libraries)', - '<@(curl_libraries)', '-lboost_regex' ], }, diff --git a/mapboxgl.gyp b/mapboxgl.gyp index 49c4150aa1..6d8bd3f265 100644 --- a/mapboxgl.gyp +++ b/mapboxgl.gyp @@ -139,6 +139,7 @@ 'OTHER_CPLUSPLUSFLAGS':[ '<@(png_cflags)', '<@(uv_cflags)', + '<@(curl_cflags)', '<@(sqlite3_cflags)', '-I<(boost_root)/include', ], @@ -149,6 +150,7 @@ }, { 'cflags': [ '<@(png_cflags)', + '<@(curl_cflags)', '<@(sqlite3_cflags)', '-I<(boost_root)/include', ], @@ -177,12 +179,14 @@ 'OTHER_LDFLAGS': [ '<@(png_libraries)', '<@(uv_libraries)', + '<@(curl_libraries)', ] } }, { 'libraries': [ '<@(png_libraries)', '<@(uv_libraries)', + '<@(curl_libraries)', ] }] ] diff --git a/src/platform/response.cpp b/src/platform/response.cpp new file mode 100644 index 0000000000..f2117c8894 --- /dev/null +++ b/src/platform/response.cpp @@ -0,0 +1,26 @@ +#include +#include + +#include + +namespace mbgl { +namespace platform { + + +void Response::setCacheControl(const char *value) { + int seconds = 0; + // TODO: cache-control may contain other information as well: + // http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.9 + if (std::sscanf(value, "max-age=%u", &seconds) == 1) { + if (std::time(&expires) != -1) { + expires += seconds; + } + } +} + +void Response::setLastModified(const char *value) { + modified = curl_getdate(value, nullptr); +} + +} +} diff --git a/src/util/filesource.cpp b/src/util/filesource.cpp index d626092bce..8c14f6e4e0 100644 --- a/src/util/filesource.cpp +++ b/src/util/filesource.cpp @@ -32,6 +32,7 @@ void FileSource::createSchema() { "`url` TEXT PRIMARY KEY NOT NULL," "`code` INTEGER NOT NULL," "`type` INTEGER NOT NULL," + "`modified` INTEGER," "`expires` INTEGER," "`data` BLOB," "`compressed` INTEGER NOT NULL DEFAULT 0" @@ -143,7 +144,7 @@ void FileSource::saveFile(ResourceType type, const std::string &url, platform::R sqlite3_stmt *stmt = nullptr; int err; - err = sqlite3_prepare_v2(db, "REPLACE INTO `http_cache` (`url`, `code`, `type`, `data`, `compressed`) VALUES(?, ?, ?, ?, ?)", -1, &stmt, nullptr); + err = sqlite3_prepare_v2(db, "REPLACE INTO `http_cache` (`url`, `code`, `type`, `modified`, `expires`, `data`, `compressed`) VALUES(?, ?, ?, ?, ?, ?, ?)", -1, &stmt, nullptr); if (err != SQLITE_OK) { Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); return; @@ -179,15 +180,35 @@ void FileSource::saveFile(ResourceType type, const std::string &url, platform::R return; } + err = sqlite3_bind_int(stmt, 4, res->modified); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + return; + } + + err = sqlite3_bind_int(stmt, 5, res->expires); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + err = sqlite3_finalize(stmt); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } + return; + } + bool compressed = false; switch (type) { case ResourceType::Image: - err = sqlite3_bind_blob(stmt, 4, res->body.data(), res->body.size(), SQLITE_STATIC); + err = sqlite3_bind_blob(stmt, 6, res->body.data(), res->body.size(), SQLITE_STATIC); break; default: const std::string *data = new std::string(std::move(util::compress(res->body))); compressed = true; - err = sqlite3_bind_blob(stmt, 4, data->data(), data->size(), [](void *p) { delete (std::string *)p; }); + err = sqlite3_bind_blob(stmt, 6, data->data(), data->size(), [](void *p) { delete (std::string *)p; }); break; } if (err != SQLITE_OK) { @@ -199,7 +220,7 @@ void FileSource::saveFile(ResourceType type, const std::string &url, platform::R return; } - err = sqlite3_bind_int(stmt, 5, compressed); + err = sqlite3_bind_int(stmt, 7, compressed); if (err != SQLITE_OK) { Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); err = sqlite3_finalize(stmt); -- cgit v1.2.1 From a6fc961e2f8498afac3c0c94fe2adeeef8156210 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Mon, 25 Aug 2014 17:12:24 +0200 Subject: don't report SQLITE_OK as an error --- src/util/filesource.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/util/filesource.cpp b/src/util/filesource.cpp index 8c14f6e4e0..6f7ad66b24 100644 --- a/src/util/filesource.cpp +++ b/src/util/filesource.cpp @@ -48,7 +48,9 @@ void FileSource::createSchema() { void FileSource::closeDatabase() { if (db) { const int err = sqlite3_close_v2(db); - Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + if (err != SQLITE_OK) { + Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db)); + } db = nullptr; } } -- cgit v1.2.1 From c2eb6502c802ccf4e7da915560a26de3cb9852ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Mon, 25 Aug 2014 17:14:47 +0200 Subject: add missing headers --- src/util/compression.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/util/compression.cpp b/src/util/compression.cpp index 3922501868..de4e09764c 100644 --- a/src/util/compression.cpp +++ b/src/util/compression.cpp @@ -2,6 +2,9 @@ #include +#include +#include + namespace mbgl { namespace util { -- cgit v1.2.1 From 934cb7cd0a30c978405fd54ab6923ef0a470bf3c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Mon, 25 Aug 2014 17:57:48 +0200 Subject: link dependencies with sqlite3 --- mapboxgl.gyp | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/mapboxgl.gyp b/mapboxgl.gyp index 6d8bd3f265..3b5fe13d2b 100644 --- a/mapboxgl.gyp +++ b/mapboxgl.gyp @@ -143,9 +143,6 @@ '<@(sqlite3_cflags)', '-I<(boost_root)/include', ], - 'OTHER_LDFLAGS': [ - '<(sqlite3_libraries)', - ], }, }, { 'cflags': [ @@ -154,9 +151,6 @@ '<@(sqlite3_cflags)', '-I<(boost_root)/include', ], - 'libraries': [ - '<@(sqlite3_libraries)', - ], }] ], 'direct_dependent_settings': { @@ -180,6 +174,7 @@ '<@(png_libraries)', '<@(uv_libraries)', '<@(curl_libraries)', + '<@(sqlite3_libraries)', ] } }, { @@ -187,6 +182,7 @@ '<@(png_libraries)', '<@(uv_libraries)', '<@(curl_libraries)', + '<@(sqlite3_libraries)', ] }] ] -- cgit v1.2.1 From b757805ffb2d38162e53359f6321e87fcd1ca1de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Tue, 26 Aug 2014 02:49:06 -0700 Subject: fix issue that deleted the data part of a std::string as a std::string object --- src/util/filesource.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/util/filesource.cpp b/src/util/filesource.cpp index 6f7ad66b24..ec46247b61 100644 --- a/src/util/filesource.cpp +++ b/src/util/filesource.cpp @@ -202,15 +202,16 @@ void FileSource::saveFile(ResourceType type, const std::string &url, platform::R return; } + std::string data; bool compressed = false; switch (type) { case ResourceType::Image: err = sqlite3_bind_blob(stmt, 6, res->body.data(), res->body.size(), SQLITE_STATIC); break; default: - const std::string *data = new std::string(std::move(util::compress(res->body))); + data = std::move(util::compress(res->body)); compressed = true; - err = sqlite3_bind_blob(stmt, 6, data->data(), data->size(), [](void *p) { delete (std::string *)p; }); + err = sqlite3_bind_blob(stmt, 6, data.data(), data.size(), SQLITE_STATIC); break; } if (err != SQLITE_OK) { -- cgit v1.2.1 From 2f088b5938344f75d198c1ec6421a96e562bc405 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Tue, 26 Aug 2014 12:51:42 +0200 Subject: add libcurl build --- setup-libraries.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/setup-libraries.sh b/setup-libraries.sh index 34baf167b5..35f8521797 100755 --- a/setup-libraries.sh +++ b/setup-libraries.sh @@ -82,30 +82,35 @@ if test -z "${TRAVIS:-}" || ! test -d out/build-cpp11-libcpp-universal; then source iPhoneOS.sh if [ ! -f out/build-cpp11-libcpp-armv7-iphoneos/lib/libpng.a ] ; then ./scripts/build_png.sh ; fi if [ ! -f out/build-cpp11-libcpp-armv7-iphoneos/lib/libuv.a ] ; then ./scripts/build_libuv.sh ; fi + if [ ! -f out/build-cpp11-libcpp-armv7-iphoneos/lib/libcurl.a ] ; then ./scripts/build_curl.sh ; fi if [ ! -f out/build-cpp11-libcpp-armv7-iphoneos/lib/libsqlite3.a ] ; then ./scripts/build_sqlite.sh ; fi echo ' ...done' source iPhoneOSs.sh if [ ! -f out/build-cpp11-libcpp-armv7s-iphoneoss/lib/libpng.a ] ; then ./scripts/build_png.sh ; fi if [ ! -f out/build-cpp11-libcpp-armv7s-iphoneoss/lib/libuv.a ] ; then ./scripts/build_libuv.sh ; fi + if [ ! -f out/build-cpp11-libcpp-armv7s-iphoneoss/lib/libcurl.a ] ; then ./scripts/build_curl.sh ; fi if [ ! -f out/build-cpp11-libcpp-armv7s-iphoneoss/lib/libsqlite3.a ] ; then ./scripts/build_libsqlite.sh ; fi echo ' ...done' source iPhoneOS64.sh if [ ! -f out/build-cpp11-libcpp-arm64-iphoneos64/lib/libpng.a ] ; then ./scripts/build_png.sh ; fi if [ ! -f out/build-cpp11-libcpp-arm64-iphoneos64/lib/libuv.a ] ; then ./scripts/build_libuv.sh ; fi + if [ ! -f out/build-cpp11-libcpp-arm64-iphoneos64/lib/libcurl.a ] ; then ./scripts/build_curl.sh ; fi if [ ! -f out/build-cpp11-libcpp-arm64-iphoneos64/lib/libsqlite3.a ] ; then ./scripts/build_libsqlite.sh ; fi echo ' ...done' source iPhoneSimulator.sh if [ ! -f out/build-cpp11-libcpp-i386-iphonesimulator/lib/libpng.a ] ; then ./scripts/build_png.sh ; fi if [ ! -f out/build-cpp11-libcpp-i386-iphonesimulator/lib/libuv.a ] ; then ./scripts/build_libuv.sh ; fi + if [ ! -f out/build-cpp11-libcpp-i386-iphonesimulator/lib/libcurl.a ] ; then ./scripts/build_curl.sh ; fi if [ ! -f out/build-cpp11-libcpp-i386-iphonesimulator/lib/libsqlite3.a ] ; then ./scripts/build_sqlite.sh ; fi echo ' ...done' source iPhoneSimulator64.sh if [ ! -f out/build-cpp11-libcpp-x86_64-iphonesimulator/lib/libpng.a ] ; then ./scripts/build_png.sh ; fi if [ ! -f out/build-cpp11-libcpp-x86_64-iphonesimulator/lib/libuv.a ] ; then ./scripts/build_libuv.sh ; fi + if [ ! -f out/build-cpp11-libcpp-x86_64-iphonesimulator/lib/libcurl.a ] ; then ./scripts/build_curl.sh ; fi if [ ! -f out/build-cpp11-libcpp-x86_64-iphonesimulator/lib/libsqlite3.a ] ; then ./scripts/build_sqlite.sh ; fi echo ' ...done' -- cgit v1.2.1 From 91f83189816e9002dbb22d122683d409986f5fe9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Tue, 26 Aug 2014 13:34:46 +0200 Subject: fix build issues --- mapboxgl.gyp | 117 ++++++++++++++++------------------------------ setup-libraries.sh | 7 ++- src/platform/response.cpp | 5 ++ 3 files changed, 50 insertions(+), 79 deletions(-) diff --git a/mapboxgl.gyp b/mapboxgl.gyp index 3b5fe13d2b..aa40402ff0 100644 --- a/mapboxgl.gyp +++ b/mapboxgl.gyp @@ -31,7 +31,7 @@ '<(SHARED_INTERMEDIATE_DIR)/src/shader/shaders_gl.cpp', '<(SHARED_INTERMEDIATE_DIR)/src/shader/shaders_gles2.cpp' ], - 'include_dirs':[ + 'include_dirs': [ '<(SHARED_INTERMEDIATE_DIR)/include/', ] } @@ -126,17 +126,17 @@ ' Date: Tue, 26 Aug 2014 14:33:09 +0200 Subject: pass libuv, libcurl and boost headers down to dependents --- Makefile | 2 +- mapboxgl.gyp | 10 ++++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b7aa397e1a..584fe2a2e4 100644 --- a/Makefile +++ b/Makefile @@ -108,4 +108,4 @@ distclean: clean -rm -rf ./mapnik-packaging/osx/out/universal -find ./mapnik-packaging/osx/out/packages -type d ! -name 'packages' -maxdepth 1 -exec rm -rf {} \; -.PHONY: mbgl test linux +.PHONY: mbgl test linux build/test/Makefile diff --git a/mapboxgl.gyp b/mapboxgl.gyp index aa40402ff0..34908aec59 100644 --- a/mapboxgl.gyp +++ b/mapboxgl.gyp @@ -161,6 +161,11 @@ 'conditions': [ ['OS == "mac"', { 'xcode_settings': { + 'OTHER_CPLUSPLUSFLAGS': [ + '<@(uv_cflags)', + '<@(curl_cflags)', + '-I<(boost_root)/include', + ], 'OTHER_LDFLAGS': [ '<@(png_libraries)', '<@(uv_libraries)', @@ -169,6 +174,11 @@ ] } }, { + 'cflags': [ + '<@(uv_cflags)', + '<@(curl_cflags)', + '-I<(boost_root)/include', + ], 'libraries': [ '<@(png_libraries)', '<@(uv_libraries)', -- cgit v1.2.1 From cc4799510b3dbe248bfed0ce99b9747e742ddbcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Tue, 26 Aug 2014 15:53:23 +0200 Subject: add -g --- scripts/flags.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/flags.sh b/scripts/flags.sh index 0c0981cdc1..58aee18a04 100755 --- a/scripts/flags.sh +++ b/scripts/flags.sh @@ -16,12 +16,12 @@ if [[ ${TRAVIS_OS_NAME} == "linux" ]]; then # some limited coverage if [[ ${BUILDTYPE} == "Debug" ]]; then if [[ ${CXX} == "g++" ]]; then - export CXXFLAGS="-fsanitize=address ${CXXFLAGS}" - export CFLAGS="-fsanitize=address ${CFLAGS}" + export CXXFLAGS="-fsanitize=address -g ${CXXFLAGS}" + export CFLAGS="-fsanitize=address -g ${CFLAGS}" export LDFLAGS="-fsanitize=address ${LDFLAGS}" elif [[ ${CXX} == "clang++" ]]; then - export CXXFLAGS="-fsanitize=thread -fPIC ${CXXFLAGS}" - export CFLAGS="-fsanitize=thread ${CFLAGS}" + export CXXFLAGS="-fsanitize=thread -g -fPIC ${CXXFLAGS}" + export CFLAGS="-fsanitize=thread -g ${CFLAGS}" export LDFLAGS="-fsanitize=thread -pie ${LDFLAGS}" fi fi -- cgit v1.2.1 From 062e911c6d570a794431023f9f0cb0b02cd85667 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Tue, 26 Aug 2014 16:55:17 +0200 Subject: remove shared_from_this() usage --- src/util/filesource.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/util/filesource.cpp b/src/util/filesource.cpp index ec46247b61..339b5b1e1d 100644 --- a/src/util/filesource.cpp +++ b/src/util/filesource.cpp @@ -284,7 +284,7 @@ void FileSource::load(ResourceType type, const std::string &url, std::function source = shared_from_this(); + FileSource *source = this; platform::request_http(absoluteURL, [=](platform::Response *res) { source->saveFile(type, cleanURL, res); callback(res); -- cgit v1.2.1 From d9fc7708a2dfb6e2506a5d10d896a813557c056d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20Ka=CC=88fer?= Date: Mon, 15 Sep 2014 17:26:44 +0200 Subject: do 304 requests and cache them in sqlite --- common/foundation_request.h | 1 - common/foundation_request.mm | 119 ----- include/mbgl/map/map.hpp | 8 +- include/mbgl/map/tile_data.hpp | 5 +- include/mbgl/platform/platform.hpp | 13 - include/mbgl/platform/request.hpp | 41 -- include/mbgl/platform/response.hpp | 29 -- include/mbgl/storage/base_request.hpp | 46 ++ include/mbgl/storage/file_request.hpp | 32 ++ include/mbgl/storage/file_source.hpp | 54 +++ include/mbgl/storage/http_request.hpp | 38 ++ include/mbgl/storage/http_request_baton.hpp | 28 ++ include/mbgl/storage/request.hpp | 39 ++ include/mbgl/storage/resource_type.hpp | 18 + include/mbgl/storage/response.hpp | 23 + include/mbgl/storage/sqlite_store.hpp | 47 ++ include/mbgl/text/glyph_store.hpp | 7 + include/mbgl/util/filesource.hpp | 56 --- include/mbgl/util/interpolate.hpp | 2 + include/mbgl/util/parsedate.h | 38 ++ include/mbgl/util/queue.h | 92 ++++ include/mbgl/util/sqlite3.hpp | 74 +++ include/mbgl/util/uv-channel.h | 29 ++ include/mbgl/util/uv-messenger.h | 31 ++ include/mbgl/util/uv-worker.h | 36 ++ macosx/main.mm | 3 + macosx/mapboxgl-app.gyp | 3 +- mapboxgl.gyp | 8 +- src/map/map.cpp | 58 ++- src/map/source.cpp | 9 +- src/map/sprite.cpp | 22 +- src/map/tile_data.cpp | 19 +- src/map/vector_tile_data.cpp | 16 +- src/platform/request.cpp | 54 --- src/platform/response.cpp | 31 -- src/storage/base_request.cpp | 66 +++ src/storage/file_request.cpp | 179 ++++++++ src/storage/file_source.cpp | 80 ++++ src/storage/http_request.cpp | 112 +++++ src/storage/http_request_baton_darwin.mm | 103 +++++ src/storage/request.cpp | 39 ++ src/storage/response.cpp | 22 + src/storage/sqlite_store.cpp | 194 ++++++++ src/text/glyph_store.cpp | 35 +- src/util/filesource.cpp | 296 ------------ src/util/parsedate.c | 689 ++++++++++++++++++++++++++++ src/util/sqlite3.cpp | 166 +++++++ src/util/uv-channel.c | 55 +++ src/util/uv-messenger.c | 70 +++ src/util/uv-worker.c | 62 +++ 50 files changed, 2581 insertions(+), 716 deletions(-) delete mode 100644 common/foundation_request.h delete mode 100644 common/foundation_request.mm delete mode 100644 include/mbgl/platform/request.hpp delete mode 100644 include/mbgl/platform/response.hpp create mode 100644 include/mbgl/storage/base_request.hpp create mode 100644 include/mbgl/storage/file_request.hpp create mode 100644 include/mbgl/storage/file_source.hpp create mode 100644 include/mbgl/storage/http_request.hpp create mode 100644 include/mbgl/storage/http_request_baton.hpp create mode 100644 include/mbgl/storage/request.hpp create mode 100644 include/mbgl/storage/resource_type.hpp create mode 100644 include/mbgl/storage/response.hpp create mode 100644 include/mbgl/storage/sqlite_store.hpp delete mode 100644 include/mbgl/util/filesource.hpp create mode 100644 include/mbgl/util/parsedate.h create mode 100644 include/mbgl/util/queue.h create mode 100644 include/mbgl/util/sqlite3.hpp create mode 100644 include/mbgl/util/uv-channel.h create mode 100644 include/mbgl/util/uv-messenger.h create mode 100644 include/mbgl/util/uv-worker.h delete mode 100644 src/platform/request.cpp delete mode 100644 src/platform/response.cpp create mode 100644 src/storage/base_request.cpp create mode 100644 src/storage/file_request.cpp create mode 100644 src/storage/file_source.cpp create mode 100644 src/storage/http_request.cpp create mode 100644 src/storage/http_request_baton_darwin.mm create mode 100644 src/storage/request.cpp create mode 100644 src/storage/response.cpp create mode 100644 src/storage/sqlite_store.cpp delete mode 100644 src/util/filesource.cpp create mode 100644 src/util/parsedate.c create mode 100644 src/util/sqlite3.cpp create mode 100644 src/util/uv-channel.c create mode 100644 src/util/uv-messenger.c create mode 100644 src/util/uv-worker.c diff --git a/common/foundation_request.h b/common/foundation_request.h deleted file mode 100644 index 0b8c4b8fb0..0000000000 --- a/common/foundation_request.h +++ /dev/null @@ -1 +0,0 @@ -#import diff --git a/common/foundation_request.mm b/common/foundation_request.mm deleted file mode 100644 index 478dadf4d9..0000000000 --- a/common/foundation_request.mm +++ /dev/null @@ -1,119 +0,0 @@ -#import "foundation_request.h" - -#include "TargetConditionals.h" -#if TARGET_OS_IPHONE -#import -#include -#endif - -#include -#include -#include -#include -#include -#include -#include - -dispatch_once_t request_initialize = 0; -NSURLSession *session = nullptr; - -#if TARGET_OS_IPHONE -std::atomic active_tasks; -#endif - - -// We're using a child class to make sure ARC is working correctly, as well as to add activity -// indicators on iOS. -class FoundationRequest : public mbgl::platform::Request { -public: - FoundationRequest(const std::string &url, - std::function callback, - std::shared_ptr loop) - : Request(url, callback, loop) { -#if TARGET_OS_IPHONE - active_tasks++; - dispatch_async(dispatch_get_main_queue(), ^(void) { - [[UIApplication sharedApplication] - setNetworkActivityIndicatorVisible:(active_tasks > 0)]; - }); -#endif - } - - ~FoundationRequest() { -#if TARGET_OS_IPHONE - active_tasks--; - dispatch_async(dispatch_get_main_queue(), ^(void) { - [[UIApplication sharedApplication] - setNetworkActivityIndicatorVisible:(active_tasks > 0)]; - }); -#endif - } - - NSURLSessionDataTask *task = nullptr; -}; - -std::shared_ptr -mbgl::platform::request_http(const std::string &url, - std::function callback, - std::shared_ptr loop) { - dispatch_once(&request_initialize, ^{ - NSURLSessionConfiguration *sessionConfig = - [NSURLSessionConfiguration defaultSessionConfiguration]; - sessionConfig.timeoutIntervalForResource = 30; - sessionConfig.HTTPMaximumConnectionsPerHost = 8; - sessionConfig.requestCachePolicy = NSURLRequestUseProtocolCachePolicy; - - session = [NSURLSession sessionWithConfiguration:sessionConfig]; - -#if TARGET_OS_IPHONE - active_tasks = 0; -#endif - }); - - std::shared_ptr req = - std::make_shared(url, callback, loop); - - // Note that we are creating a new shared_ptr pointer(!) to make sure there is at least one - // shared_ptr in existence while the NSURLSession is loading our data. We are making sure in the - // callback that this pointer gets destroyed again. - std::shared_ptr *req_ptr = new std::shared_ptr(req); - - NSURLSessionDataTask *task = [session - dataTaskWithURL:[NSURL URLWithString:@(url.c_str())] - completionHandler:^(NSData *data, NSURLResponse *response, NSError *error) { - if ([error code] == NSURLErrorCancelled) { - // We intentionally cancelled this request. Make sure we clear the shared_ptr to resolve - // the circular reference. We're referencing this shared_ptr by value so that the object - // stays around until this completion handler is invoked. - delete req_ptr; - - return; - } - - if (!error && [response isKindOfClass:[NSHTTPURLResponse class]]) { - NSDictionary *headers = [(NSHTTPURLResponse *)response allHeaderFields]; - (*req_ptr)->res->code = [(NSHTTPURLResponse *)response statusCode]; - (*req_ptr)->res->body = {(const char *)[data bytes], [data length]}; - (*req_ptr)->res->setCacheControl([[headers objectForKey:@"Cache-Control"] UTF8String]); - (*req_ptr)->res->setLastModified([[headers objectForKey:@"Last-Modified"] UTF8String]); - - } else { - (*req_ptr)->res->error_message = [[error localizedDescription] UTF8String]; - } - - (*req_ptr)->complete(); - - delete req_ptr; - }]; - - req->task = task; - - [task resume]; - return req; -} - -void mbgl::platform::cancel_request_http(const std::shared_ptr &req) { - if (req) { - [((FoundationRequest *)(req.get()))->task cancel]; - } -} diff --git a/include/mbgl/map/map.hpp b/include/mbgl/map/map.hpp index fba38879ed..98951a375c 100644 --- a/include/mbgl/map/map.hpp +++ b/include/mbgl/map/map.hpp @@ -35,11 +35,14 @@ public: explicit Map(View &view); ~Map(); - // Start/stop the map render thread + // Start/stop the map render thread. the start() call is asynchronous, while the stop() call + // will block until the map rendering thread stopped. void start(); void stop(); - // Runs the map event loop. + // Runs the map event loop. ONLY run this function when you want to get render a single frame + // with this map object. It will *not* spawn a separate thread and instead block until the + // frame is completely rendered. void run(); // Triggers a lazy rerender: only performs a render when the map is not clean. @@ -192,6 +195,7 @@ private: Painter painter; + std::string styleURL; std::string styleJSON = ""; std::string accessToken = ""; diff --git a/include/mbgl/map/tile_data.hpp b/include/mbgl/map/tile_data.hpp index 9aaef84e04..900224be2d 100644 --- a/include/mbgl/map/tile_data.hpp +++ b/include/mbgl/map/tile_data.hpp @@ -19,8 +19,7 @@ class Map; class Painter; class SourceInfo; class StyleLayer; - -namespace platform { class Request; } +class Request; class TileData : public std::enable_shared_from_this, private util::noncopyable { @@ -72,7 +71,7 @@ public: const SourceInfo &source; protected: - std::weak_ptr req; + std::unique_ptr req; std::string data; // Contains the tile ID string for painting debug information. diff --git a/include/mbgl/platform/platform.hpp b/include/mbgl/platform/platform.hpp index e08cac5312..6356d81bb1 100644 --- a/include/mbgl/platform/platform.hpp +++ b/include/mbgl/platform/platform.hpp @@ -1,8 +1,6 @@ #ifndef MBGL_PLATFORM_PLATFORM #define MBGL_PLATFORM_PLATFORM -#include - #include #include @@ -13,23 +11,12 @@ namespace platform { class Request; -// Makes an HTTP request of a URL, preferrably on a background thread, and calls a function with the -// results in the original thread (which runs the libuv loop). -// If the loop pointer is NULL, the callback function will be called on an arbitrary thread. -// Returns a cancellable request. -std::shared_ptr request_http(const std::string &url, - std::function callback, - std::shared_ptr loop = nullptr); - // Uppercase a string, potentially using platform-specific routines. std::string uppercase(const std::string &string); // Lowercase a string, potentially using platform-specific routines. std::string lowercase(const std::string &string); -// Cancels an HTTP request. -void cancel_request_http(const std::shared_ptr &req); - // Shows an alpha image with the specified dimensions in a named window. void show_debug_image(std::string name, const char *data, size_t width, size_t height); diff --git a/include/mbgl/platform/request.hpp b/include/mbgl/platform/request.hpp deleted file mode 100644 index 0cbacf645d..0000000000 --- a/include/mbgl/platform/request.hpp +++ /dev/null @@ -1,41 +0,0 @@ -#ifndef MBGL_PLATFORM_REQUEST -#define MBGL_PLATFORM_REQUEST - -#include -#include -#include -#include - -#include -#include - -namespace mbgl { -namespace platform { - -struct Response; - -class Request : public std::enable_shared_from_this, private util::noncopyable { -public: - Request(const std::string &url, - std::function callback, - std::shared_ptr loop); - ~Request(); - - void complete(); - -private: - static void complete(uv_async_t *async); - -public: - const std::string url; - std::unique_ptr res; - std::atomic cancelled; - -public: - uv_async_t *async = nullptr; - std::shared_ptr loop; -}; -} -} - -#endif diff --git a/include/mbgl/platform/response.hpp b/include/mbgl/platform/response.hpp deleted file mode 100644 index 345a2ee3df..0000000000 --- a/include/mbgl/platform/response.hpp +++ /dev/null @@ -1,29 +0,0 @@ -#ifndef MBGL_PLATFORM_RESPONSE -#define MBGL_PLATFORM_RESPONSE - -#include -#include -#include - -#include - -namespace mbgl { -namespace platform { - -struct Response : private util::noncopyable { - Response(std::function callback) : callback(callback) {} - int16_t code = -1; - std::string body; - std::string error_message; - std::time_t modified; - std::time_t expires; - std::function callback; - - void setCacheControl(const char *value); - void setLastModified(const char *value); -}; - -} -} - -#endif diff --git a/include/mbgl/storage/base_request.hpp b/include/mbgl/storage/base_request.hpp new file mode 100644 index 0000000000..b8ebd368e4 --- /dev/null +++ b/include/mbgl/storage/base_request.hpp @@ -0,0 +1,46 @@ +#ifndef MBGL_STORAGE_BASE_REQUEST +#define MBGL_STORAGE_BASE_REQUEST + +#include +#include +#include +#include + +typedef struct uv_loop_s uv_loop_t; +typedef struct uv_async_s uv_async_t; + +namespace mbgl { + +class Response; +class Request; +using Callback = std::function; + + +class BaseRequest { +private: + // Make noncopyable and immovable + BaseRequest(const BaseRequest &) = delete; + BaseRequest(BaseRequest &&) = delete; + BaseRequest& operator=(const BaseRequest &) = delete; + BaseRequest& operator=(BaseRequest &&) = delete; + +public: + BaseRequest(); + virtual ~BaseRequest(); + + Callback *add(Callback &&callback, const std::shared_ptr &request); + void remove(Callback *callback); + void notify(); + +public: + const unsigned long thread_id; + std::unique_ptr response; + +private: + std::shared_ptr self; + std::forward_list> callbacks; +}; + +} + +#endif diff --git a/include/mbgl/storage/file_request.hpp b/include/mbgl/storage/file_request.hpp new file mode 100644 index 0000000000..156fd6dfe7 --- /dev/null +++ b/include/mbgl/storage/file_request.hpp @@ -0,0 +1,32 @@ +#ifndef MBGL_STORAGE_FILE_REQUEST +#define MBGL_STORAGE_FILE_REQUEST + + +#include +#include +#include + +#include + +namespace mbgl { + +typedef struct uv_loop_s uv_loop_t; + +struct FileRequestBaton; + +class FileRequest : public BaseRequest { +public: + FileRequest(const std::string &path, uv_loop_t *loop); + ~FileRequest(); + +private: + const std::string path; + const unsigned long thread_id; + FileRequestBaton *ptr = nullptr; + + friend struct FileRequestBaton; +}; + +} + +#endif \ No newline at end of file diff --git a/include/mbgl/storage/file_source.hpp b/include/mbgl/storage/file_source.hpp new file mode 100644 index 0000000000..4cc95ae24e --- /dev/null +++ b/include/mbgl/storage/file_source.hpp @@ -0,0 +1,54 @@ +#ifndef MBGL_STORAGE_FILE_SOURCE +#define MBGL_STORAGE_FILE_SOURCE + +#include +#include + +#include +#include +#include +#include + +typedef struct uv_loop_s uv_loop_t; +typedef struct uv_messenger_s uv_messenger_t; + +namespace mbgl { + +class BaseRequest; +class SQLiteStore; + +class FileSource { +private: + FileSource(const FileSource &) = delete; + FileSource(FileSource &&) = delete; + FileSource& operator=(const FileSource &) = delete; + FileSource& operator=(FileSource &&) = delete; + +public: + FileSource(uv_loop_t *loop); + ~FileSource(); + +public: + // Stores and retrieves the base path/URL for relative requests + void setBase(const std::string &value); + const std::string &getBase() const; + + std::unique_ptr request(ResourceType type, const std::string &url); + + void prepare(std::function fn); + +private: + const unsigned long thread_id; + + // Stores a URL that is used as a base for loading resources with relative path. + std::string base; + + std::unordered_map> pending; + std::shared_ptr store; + uv_loop_t *loop = nullptr; + uv_messenger_t *queue = nullptr; +}; + +} + +#endif diff --git a/include/mbgl/storage/http_request.hpp b/include/mbgl/storage/http_request.hpp new file mode 100644 index 0000000000..30f7b3aa6d --- /dev/null +++ b/include/mbgl/storage/http_request.hpp @@ -0,0 +1,38 @@ +#ifndef MBGL_STORAGE_HTTP_REQUEST +#define MBGL_STORAGE_HTTP_REQUEST + +#include +#include + +#include +#include +#include + +typedef struct uv_loop_s uv_loop_t; + +namespace mbgl { + +struct CacheRequestBaton; +struct HTTPRequestBaton; +struct CacheEntry; +class SQLiteStore; + +class HTTPRequest : public BaseRequest { +public: + HTTPRequest(ResourceType type, const std::string &path, uv_loop_t *loop, std::shared_ptr store); + ~HTTPRequest(); + +private: + void loadedCacheEntry(std::unique_ptr &&response); + +private: + const unsigned long thread_id; + CacheRequestBaton *cache_baton = nullptr; + HTTPRequestBaton *http_baton = nullptr; + std::shared_ptr store; + const ResourceType type; +}; + +} + +#endif \ No newline at end of file diff --git a/include/mbgl/storage/http_request_baton.hpp b/include/mbgl/storage/http_request_baton.hpp new file mode 100644 index 0000000000..5f06a68cd1 --- /dev/null +++ b/include/mbgl/storage/http_request_baton.hpp @@ -0,0 +1,28 @@ +#ifndef MBGL_STORAGE_HTTP_REQUEST_BATON +#define MBGL_STORAGE_HTTP_REQUEST_BATON + +#include + +#include + +typedef struct uv_async_s uv_async_t; + +namespace mbgl { + +class HTTPRequest; + +struct HTTPRequestBaton { + HTTPRequest *request = nullptr; + std::string path; + uv_async_t *async = nullptr; + std::unique_ptr response; + void *ptr = nullptr; + bool not_modified = false; + + void start(); + void cancel(); +}; + +} + +#endif diff --git a/include/mbgl/storage/request.hpp b/include/mbgl/storage/request.hpp new file mode 100644 index 0000000000..fa27fbc781 --- /dev/null +++ b/include/mbgl/storage/request.hpp @@ -0,0 +1,39 @@ +#ifndef MBGL_STORAGE_REQUEST +#define MBGL_STORAGE_REQUEST + +#include + +#include +#include +#include + +typedef struct uv_loop_s uv_loop_t; + +namespace mbgl { + +class BaseRequest; +using Callback = std::function; + +class Request { +private: + Request(const Request &) = delete; + Request(Request &&) = delete; + Request& operator=(const Request &) = delete; + Request& operator=(Request &&) = delete; + +public: + Request(const std::shared_ptr &base); + ~Request(); + + void onload(Callback cb); + void cancel(); + +private: + const unsigned long thread_id; + std::shared_ptr base; + std::forward_list callbacks; +}; + +} + +#endif \ No newline at end of file diff --git a/include/mbgl/storage/resource_type.hpp b/include/mbgl/storage/resource_type.hpp new file mode 100644 index 0000000000..b7204a9fa1 --- /dev/null +++ b/include/mbgl/storage/resource_type.hpp @@ -0,0 +1,18 @@ +#ifndef MBGL_STORAGE_RESOURCE_TYPE +#define MBGL_STORAGE_RESOURCE_TYPE + +#include + +namespace mbgl { + +enum class ResourceType : uint8_t { + Unknown = 0, + Tile = 1, + Glyphs = 2, + Image = 3, + JSON = 4 +}; + +} + +#endif diff --git a/include/mbgl/storage/response.hpp b/include/mbgl/storage/response.hpp new file mode 100644 index 0000000000..4960173f9e --- /dev/null +++ b/include/mbgl/storage/response.hpp @@ -0,0 +1,23 @@ +#ifndef MBGL_STORAGE_RESPONSE +#define MBGL_STORAGE_RESPONSE + +#include +#include + +namespace mbgl { + +class Response { +public: + long code = 0; + int64_t modified = 0; + int64_t expires = 0; + std::string data; + + std::string message; + + static int64_t parseCacheControl(const char *value); +}; + +} + +#endif \ No newline at end of file diff --git a/include/mbgl/storage/sqlite_store.hpp b/include/mbgl/storage/sqlite_store.hpp new file mode 100644 index 0000000000..e03e6cf2bc --- /dev/null +++ b/include/mbgl/storage/sqlite_store.hpp @@ -0,0 +1,47 @@ +#ifndef MBGL_STORAGE_SQLITE_STORE +#define MBGL_STORAGE_SQLITE_STORE + +#include +#include + +#include + +#include + +typedef struct uv_worker_s uv_worker_t; + +namespace mapbox { +namespace sqlite { +class Database; +} +} + +namespace mbgl { + +class SQLiteStore { +public: + SQLiteStore(uv_loop_t *loop, const std::string &path); + ~SQLiteStore(); + + typedef void (*GetCallback)(std::unique_ptr &&entry, void *ptr); + + void get(const std::string &path, GetCallback cb, void *ptr); + void put(const std::string &path, ResourceType type, const Response &entry); + void updateExpiration(const std::string &path, int64_t expires); + +private: + void createSchema(); + void closeDatabase(); + static void runGet(uv_work_t *req); + static void runPut(uv_work_t *req); + static void deliverResult(uv_work_t *req, int status); + +private: + const unsigned long thread_id; + std::shared_ptr db; + uv_worker_t *worker = nullptr; +}; + +} + +#endif diff --git a/include/mbgl/text/glyph_store.hpp b/include/mbgl/text/glyph_store.hpp index e0c0391c73..9730967053 100644 --- a/include/mbgl/text/glyph_store.hpp +++ b/include/mbgl/text/glyph_store.hpp @@ -49,6 +49,13 @@ class GlyphPBF { public: GlyphPBF(const std::string &glyphURL, const std::string &fontStack, GlyphRange glyphRange, const std::shared_ptr &fileSource); +private: + GlyphPBF(const GlyphPBF &) = delete; + GlyphPBF(GlyphPBF &&) = delete; + GlyphPBF &operator=(const GlyphPBF &) = delete; + GlyphPBF &operator=(GlyphPBF &&) = delete; + +public: void parse(FontStack &stack); std::shared_future getFuture(); diff --git a/include/mbgl/util/filesource.hpp b/include/mbgl/util/filesource.hpp deleted file mode 100644 index ccff4b5122..0000000000 --- a/include/mbgl/util/filesource.hpp +++ /dev/null @@ -1,56 +0,0 @@ -#ifndef MBGL_UTIL_FILESOURCE -#define MBGL_UTIL_FILESOURCE - -#include -#include - -#include -#include -#include - -typedef struct sqlite3 sqlite3; -typedef struct sqlite3_stmt sqlite3_stmt; - -namespace mbgl { - -namespace platform { -struct Response; -} - -enum class ResourceType : uint8_t { - Unknown = 0, - Tile = 1, - Glyphs = 2, - Image = 3, - JSON = 4 -}; - -class FileSource : private util::noncopyable, public std::enable_shared_from_this { -public: - FileSource(); - ~FileSource(); - - void setBase(const std::string &value); - const std::string &getBase() const; - - void load(ResourceType type, const std::string &url, std::function callback, const std::shared_ptr loop = nullptr); - -private: - void closeDatabase(); - void createSchema(); - bool loadFile(ResourceType type, const std::string &url, std::function callback); - void saveFile(ResourceType type, const std::string &url, platform::Response *res); - -private: - // Stores a URL that is used as a base for loading resources with relative path. - std::string base; - - // Stores the absolute path to the cache directory. - const std::string cache; - - sqlite3 *db = nullptr; -}; - -} - -#endif diff --git a/include/mbgl/util/interpolate.hpp b/include/mbgl/util/interpolate.hpp index e8c3389350..c9232db4eb 100644 --- a/include/mbgl/util/interpolate.hpp +++ b/include/mbgl/util/interpolate.hpp @@ -1,6 +1,8 @@ #ifndef MBGL_UTIL_INTERPOLATE #define MBGL_UTIL_INTERPOLATE +#include + namespace mbgl { namespace util { diff --git a/include/mbgl/util/parsedate.h b/include/mbgl/util/parsedate.h new file mode 100644 index 0000000000..6905e361d4 --- /dev/null +++ b/include/mbgl/util/parsedate.h @@ -0,0 +1,38 @@ +#ifndef HEADER_PARSEDATE_H +#define HEADER_PARSEDATE_H +/*************************************************************************** + * _ _ ____ _ + * Project ___| | | | _ \| | + * / __| | | | |_) | | + * | (__| |_| | _ <| |___ + * \___|\___/|_| \_\_____| + * + * Copyright (C) 1998 - 2011, Daniel Stenberg, , et al. + * + * This software is licensed as described in the file COPYING, which + * you should have received as part of this distribution. The terms + * are also available at http://curl.haxx.se/docs/copyright.html. + * + * You may opt to use, copy, modify, merge, publish, distribute and/or sell + * copies of the Software, and permit persons to whom the Software is + * furnished to do so, under the terms of the COPYING file. + * + * This software is distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY + * KIND, either express or implied. + * + ***************************************************************************/ + + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +time_t parse_date(const char *p); + +#ifdef __cplusplus +} +#endif + +#endif /* HEADER_PARSEDATE_H */ diff --git a/include/mbgl/util/queue.h b/include/mbgl/util/queue.h new file mode 100644 index 0000000000..fe02b454ea --- /dev/null +++ b/include/mbgl/util/queue.h @@ -0,0 +1,92 @@ +/* Copyright (c) 2013, Ben Noordhuis + * + * Permission to use, copy, modify, and/or distribute this software for any + * purpose with or without fee is hereby granted, provided that the above + * copyright notice and this permission notice appear in all copies. + * + * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES + * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF + * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR + * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES + * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN + * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF + * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + */ + +#ifndef QUEUE_H_ +#define QUEUE_H_ + +typedef void *QUEUE[2]; + +/* Private macros. */ +#define QUEUE_NEXT(q) (*(QUEUE **) &((*(q))[0])) +#define QUEUE_PREV(q) (*(QUEUE **) &((*(q))[1])) +#define QUEUE_PREV_NEXT(q) (QUEUE_NEXT(QUEUE_PREV(q))) +#define QUEUE_NEXT_PREV(q) (QUEUE_PREV(QUEUE_NEXT(q))) + +/* Public macros. */ +#define QUEUE_DATA(ptr, type, field) \ + ((type *) ((char *) (ptr) - ((char *) &((type *) 0)->field))) + +#define QUEUE_FOREACH(q, h) \ + for ((q) = QUEUE_NEXT(h); (q) != (h); (q) = QUEUE_NEXT(q)) + +#define QUEUE_EMPTY(q) \ + ((const QUEUE *) (q) == (const QUEUE *) QUEUE_NEXT(q)) + +#define QUEUE_HEAD(q) \ + (QUEUE_NEXT(q)) + +#define QUEUE_INIT(q) \ + do { \ + QUEUE_NEXT(q) = (q); \ + QUEUE_PREV(q) = (q); \ + } \ + while (0) + +#define QUEUE_ADD(h, n) \ + do { \ + QUEUE_PREV_NEXT(h) = QUEUE_NEXT(n); \ + QUEUE_NEXT_PREV(n) = QUEUE_PREV(h); \ + QUEUE_PREV(h) = QUEUE_PREV(n); \ + QUEUE_PREV_NEXT(h) = (h); \ + } \ + while (0) + +#define QUEUE_SPLIT(h, q, n) \ + do { \ + QUEUE_PREV(n) = QUEUE_PREV(h); \ + QUEUE_PREV_NEXT(n) = (n); \ + QUEUE_NEXT(n) = (q); \ + QUEUE_PREV(h) = QUEUE_PREV(q); \ + QUEUE_PREV_NEXT(h) = (h); \ + QUEUE_PREV(q) = (n); \ + } \ + while (0) + +#define QUEUE_INSERT_HEAD(h, q) \ + do { \ + QUEUE_NEXT(q) = QUEUE_NEXT(h); \ + QUEUE_PREV(q) = (h); \ + QUEUE_NEXT_PREV(q) = (q); \ + QUEUE_NEXT(h) = (q); \ + } \ + while (0) + +#define QUEUE_INSERT_TAIL(h, q) \ + do { \ + QUEUE_NEXT(q) = (h); \ + QUEUE_PREV(q) = QUEUE_PREV(h); \ + QUEUE_PREV_NEXT(q) = (q); \ + QUEUE_PREV(h) = (q); \ + } \ + while (0) + +#define QUEUE_REMOVE(q) \ + do { \ + QUEUE_PREV_NEXT(q) = QUEUE_NEXT(q); \ + QUEUE_NEXT_PREV(q) = QUEUE_PREV(q); \ + } \ + while (0) + +#endif /* QUEUE_H_ */ diff --git a/include/mbgl/util/sqlite3.hpp b/include/mbgl/util/sqlite3.hpp new file mode 100644 index 0000000000..3e324f7ce1 --- /dev/null +++ b/include/mbgl/util/sqlite3.hpp @@ -0,0 +1,74 @@ +#pragma once + +#include +#include + +typedef struct sqlite3 sqlite3; +typedef struct sqlite3_stmt sqlite3_stmt; + +namespace mapbox { +namespace sqlite { + +enum OpenFlag : int { + ReadOnly = 0x00000001, + ReadWrite = 0x00000002, + Create = 0x00000004, + NoMutex = 0x00008000, + FullMutex = 0x00010000, + SharedCache = 0x00020000, + PrivateCache = 0x00040000, +}; + +struct Exception : std::runtime_error { + inline Exception(int err, const char *msg) : std::runtime_error(msg), code(err) {} + const int code = 0; +}; + +class Statement; + +class Database { +private: + Database(const Database &) = delete; + Database &operator=(const Database &) = delete; + +public: + Database(const std::string &filename, int flags = 0); + Database(Database &&); + ~Database(); + Database &operator=(Database &&); + + operator bool() const; + + void exec(const std::string &sql); + Statement prepare(const char *query); + +private: + sqlite3 *db = nullptr; +}; + +class Statement { +private: + Statement(const Statement &) = delete; + Statement &operator=(const Statement &) = delete; + +public: + Statement(sqlite3 *db, const char *sql); + Statement(Statement &&); + ~Statement(); + Statement &operator=(Statement &&); + + operator bool() const; + + template void bind(int offset, T value); + void bind(int offset, const std::string &value, bool retain = true); + template T get(int offset); + + bool run(); + void reset(); + +private: + sqlite3_stmt *stmt = nullptr; +}; + +} +} diff --git a/include/mbgl/util/uv-channel.h b/include/mbgl/util/uv-channel.h new file mode 100644 index 0000000000..ea5c279f65 --- /dev/null +++ b/include/mbgl/util/uv-channel.h @@ -0,0 +1,29 @@ +#ifndef MBGL_UTIL_UV_CHANNEL +#define MBGL_UTIL_UV_CHANNEL + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +// Taken from http://navaneeth.github.io/blog/2013/08/02/channels-in-libuv/ + +typedef struct uv_chan_s uv_chan_t; + +struct uv_chan_s { + uv_mutex_t mutex; + uv_cond_t cond; + void *q[2]; +}; + +int uv_chan_init(uv_chan_t *chan); +void uv_chan_send(uv_chan_t *chan, void *data); +void *uv_chan_receive(uv_chan_t *chan); +void uv_chan_destroy(uv_chan_t *chan); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/mbgl/util/uv-messenger.h b/include/mbgl/util/uv-messenger.h new file mode 100644 index 0000000000..b082466b60 --- /dev/null +++ b/include/mbgl/util/uv-messenger.h @@ -0,0 +1,31 @@ +#ifndef MBGL_UTIL_UV_MESSENGER +#define MBGL_UTIL_UV_MESSENGER + +#ifdef __cplusplus +extern "C" { +#endif + +#include + +typedef struct uv_messenger_s uv_messenger_t; +typedef void (*uv_messenger_cb)(void *arg); +typedef void (*uv_messenger_stop_cb)(uv_messenger_t *msgr); + +struct uv_messenger_s { + uv_mutex_t mutex; + uv_async_t async; + uv_messenger_cb callback; + void *data; + void *queue[2]; +}; + +int uv_messenger_init(uv_loop_t *loop, uv_messenger_t *msgr, uv_messenger_cb callback); +void uv_messenger_send(uv_messenger_t *msgr, void *arg); +void uv_messenger_stop(uv_messenger_t *msgr); +void uv_messenger_unref(uv_messenger_t *msgr); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/include/mbgl/util/uv-worker.h b/include/mbgl/util/uv-worker.h new file mode 100644 index 0000000000..b9eb10fb70 --- /dev/null +++ b/include/mbgl/util/uv-worker.h @@ -0,0 +1,36 @@ +#ifndef MBGL_UTIL_UV_WORKER +#define MBGL_UTIL_UV_WORKER + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct uv_messenger_s uv_messenger_t; + +#include + +#include + +typedef struct uv_worker_s uv_worker_t; + +struct uv_worker_s { + uv_thread_t thread; + uv_messenger_t *msgr; + uv_chan_t chan; + const char *name; +}; + +typedef void (*uv_worker_cb)(void *data); +typedef void (*uv_worker_after_cb)(void *data); + +int uv_worker_init(uv_worker_t *worker, uv_loop_t *loop); +int uv_worker_init_named(uv_worker_t *worker, uv_loop_t *loop, const char *name); +void uv_worker_send(uv_worker_t *worker, void *data, uv_worker_cb work_cb, + uv_worker_after_cb after_work_cb); +void uv_worker_close(uv_worker_t *worker); + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/macosx/main.mm b/macosx/main.mm index 9ecc307c05..7f22a32cc2 100644 --- a/macosx/main.mm +++ b/macosx/main.mm @@ -4,6 +4,8 @@ #import +#include + @interface URLHandler : NSObject @property (nonatomic) mbgl::Map *map; @@ -68,6 +70,7 @@ @end int main() { + fprintf(stderr, "main thread: 0x%lx\n", uv_thread_self()); mbgl::Log::Set(); GLFWView view; diff --git a/macosx/mapboxgl-app.gyp b/macosx/mapboxgl-app.gyp index da5c68ea35..f673d68f0f 100644 --- a/macosx/mapboxgl-app.gyp +++ b/macosx/mapboxgl-app.gyp @@ -15,8 +15,7 @@ '../common/platform_nsstring.mm', '../common/glfw_view.hpp', '../common/glfw_view.cpp', - '../common/foundation_request.h', - '../common/foundation_request.mm', + '../src/storage/http_request_baton_darwin.mm', '../common/nslog_log.hpp', '../common/nslog_log.mm', ], diff --git a/mapboxgl.gyp b/mapboxgl.gyp index 34908aec59..f77e6d770f 100644 --- a/mapboxgl.gyp +++ b/mapboxgl.gyp @@ -143,6 +143,9 @@ '<@(sqlite3_cflags)', '-I<(boost_root)/include', ], + 'OTHER_CFLAGS': [ + '<@(uv_cflags)', + ], }, }, { 'cflags': [ @@ -237,7 +240,10 @@ '<@(uv_libraries)', '<@(curl_libraries)', '<@(sqlite3_libraries)', - ] + ], + 'OTHER_CFLAGS': [ + '<@(uv_cflags)', + ], } } } diff --git a/src/map/map.cpp b/src/map/map.cpp index 9984f1b181..1f9c1428f1 100644 --- a/src/map/map.cpp +++ b/src/map/map.cpp @@ -19,7 +19,7 @@ #include #include #include -#include +#include #include #include @@ -36,10 +36,7 @@ Map::Map(View& view) thread(std::make_unique()), view(view), transform(view), - fileSource(std::make_shared()), - style(std::make_shared