summaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorKonstantin Käfer <mail@kkaefer.com>2014-08-25 14:02:40 +0200
committerKonstantin Käfer <mail@kkaefer.com>2014-09-24 16:12:02 +0200
commit28446bed24689b2bac7c8bfbd37741a7fa4fa6be (patch)
tree96f8fd7a68025ec79516d83c5ef6e92209d9473f /src
parent83fe11e0fc03dc7851f1e74974bbeb0ee3e37e21 (diff)
downloadqtlocation-mapboxgl-28446bed24689b2bac7c8bfbd37741a7fa4fa6be.tar.gz
compress data and store type
Diffstat (limited to 'src')
-rw-r--r--src/util/compression.cpp78
-rw-r--r--src/util/filesource.cpp63
2 files changed, 130 insertions, 11 deletions
diff --git a/src/util/compression.cpp b/src/util/compression.cpp
new file mode 100644
index 0000000000..3922501868
--- /dev/null
+++ b/src/util/compression.cpp
@@ -0,0 +1,78 @@
+#include <mbgl/util/compression.hpp>
+
+#include <zlib.h>
+
+namespace mbgl {
+namespace util {
+
+std::string compress(const std::string &raw) {
+ z_stream deflate_stream;
+ memset(&deflate_stream, 0, sizeof(deflate_stream));
+
+ // TODO: reuse z_streams
+ if (deflateInit(&deflate_stream, Z_DEFAULT_COMPRESSION) != Z_OK) {
+ throw std::runtime_error("failed to initialize deflate");
+ }
+
+ deflate_stream.next_in = (Bytef *)raw.data();
+ deflate_stream.avail_in = raw.size();
+
+ std::string result;
+ char out[16384];
+
+ int code;
+ do {
+ deflate_stream.next_out = reinterpret_cast<Bytef *>(out);
+ deflate_stream.avail_out = sizeof(out);
+ code = deflate(&deflate_stream, Z_FINISH);
+ if (result.size() < deflate_stream.total_out) {
+ // append the block to the output string
+ result.append(out, deflate_stream.total_out - result.size());
+ }
+ } while (code == Z_OK);
+
+ deflateEnd(&deflate_stream);
+
+ if (code != Z_STREAM_END) {
+ throw std::runtime_error(deflate_stream.msg);
+ }
+
+ return result;
+}
+
+std::string decompress(const std::string &raw) {
+ z_stream inflate_stream;
+ memset(&inflate_stream, 0, sizeof(inflate_stream));
+
+ // TODO: reuse z_streams
+ if (inflateInit(&inflate_stream) != Z_OK) {
+ throw std::runtime_error("failed to initialize inflate");
+ }
+
+ inflate_stream.next_in = (Bytef *)raw.data();
+ inflate_stream.avail_in = raw.size();
+
+ std::string result;
+ char out[15384];
+
+ int code;
+ do {
+ inflate_stream.next_out = reinterpret_cast<Bytef *>(out);
+ inflate_stream.avail_out = sizeof(out);
+ code = inflate(&inflate_stream, 0);
+ // result.append(out, sizeof(out) - inflate_stream.avail_out);
+ if (result.size() < inflate_stream.total_out) {
+ result.append(out, inflate_stream.total_out - result.size());
+ }
+ } while (code == Z_OK);
+
+ inflateEnd(&inflate_stream);
+
+ if (code != Z_STREAM_END) {
+ throw std::runtime_error(inflate_stream.msg);
+ }
+
+ return result;
+}
+}
+}
diff --git a/src/util/filesource.cpp b/src/util/filesource.cpp
index 0184659905..d626092bce 100644
--- a/src/util/filesource.cpp
+++ b/src/util/filesource.cpp
@@ -1,6 +1,7 @@
#include <mbgl/util/filesource.hpp>
#include <mbgl/platform/platform.hpp>
#include <mbgl/platform/log.hpp>
+#include <mbgl/util/compression.hpp>
#include <sqlite3.h>
@@ -30,9 +31,12 @@ void FileSource::createSchema() {
"CREATE TABLE IF NOT EXISTS `http_cache` ("
"`url` TEXT PRIMARY KEY NOT NULL,"
"`code` INTEGER NOT NULL,"
+ "`type` INTEGER NOT NULL,"
"`expires` INTEGER,"
- "`data` BLOB"
- ")", nullptr, nullptr, nullptr);
+ "`data` BLOB,"
+ "`compressed` INTEGER NOT NULL DEFAULT 0"
+ ");"
+ "CREATE INDEX IF NOT EXISTS `http_cache_type_idx` ON `http_cache` (`type`);", nullptr, nullptr, nullptr);
if (err != SQLITE_OK) {
Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db));
closeDatabase();
@@ -73,7 +77,7 @@ std::string removeAccessTokenFromURL(const std::string &url) {
}
}
-bool FileSource::loadFile(const std::string &url, std::function<void(platform::Response *)> callback) {
+bool FileSource::loadFile(ResourceType /*type*/, const std::string &url, std::function<void(platform::Response *)> callback) {
if (!db) {
return false;
}
@@ -81,7 +85,7 @@ bool FileSource::loadFile(const std::string &url, std::function<void(platform::R
sqlite3_stmt *stmt = nullptr;
int err;
- err = sqlite3_prepare_v2(db, "SELECT `code`, `data` FROM `http_cache` WHERE `url` = ?", -1, &stmt, nullptr);
+ err = sqlite3_prepare_v2(db, "SELECT `code`, `data`, `compressed` FROM `http_cache` WHERE `url` = ?", -1, &stmt, nullptr);
if (err != SQLITE_OK) {
Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db));
return false;
@@ -106,7 +110,14 @@ bool FileSource::loadFile(const std::string &url, std::function<void(platform::R
res.code = sqlite3_column_int(stmt, 0);
const char *data = reinterpret_cast<const char *>(sqlite3_column_blob(stmt, 1));
const size_t size = sqlite3_column_bytes(stmt, 1);
- res.body = { data, size };
+ const bool compressed = sqlite3_column_int(stmt, 2);
+
+ if (compressed) {
+ res.body = util::decompress({ data, size });
+ } else {
+ res.body = { data, size };
+ }
+
callback(&res);
status = true;
} else if (err == SQLITE_DONE) {
@@ -124,7 +135,7 @@ bool FileSource::loadFile(const std::string &url, std::function<void(platform::R
return status;
}
-void FileSource::saveFile(const std::string &url, platform::Response *const res) {
+void FileSource::saveFile(ResourceType type, const std::string &url, platform::Response *const res) {
if (!db) {
return;
}
@@ -132,7 +143,7 @@ void FileSource::saveFile(const std::string &url, platform::Response *const res)
sqlite3_stmt *stmt = nullptr;
int err;
- err = sqlite3_prepare_v2(db, "REPLACE INTO `http_cache` (`url`, `code`, `data`) VALUES(?, ?, ?)", -1, &stmt, nullptr);
+ err = sqlite3_prepare_v2(db, "REPLACE INTO `http_cache` (`url`, `code`, `type`, `data`, `compressed`) VALUES(?, ?, ?, ?, ?)", -1, &stmt, nullptr);
if (err != SQLITE_OK) {
Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db));
return;
@@ -158,7 +169,37 @@ void FileSource::saveFile(const std::string &url, platform::Response *const res)
return;
}
- err = sqlite3_bind_blob(stmt, 3, res->body.data(), res->body.size(), nullptr);
+ err = sqlite3_bind_int(stmt, 3, int(type));
+ if (err != SQLITE_OK) {
+ Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db));
+ err = sqlite3_finalize(stmt);
+ if (err != SQLITE_OK) {
+ Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db));
+ }
+ return;
+ }
+
+ bool compressed = false;
+ switch (type) {
+ case ResourceType::Image:
+ err = sqlite3_bind_blob(stmt, 4, res->body.data(), res->body.size(), SQLITE_STATIC);
+ break;
+ default:
+ const std::string *data = new std::string(std::move(util::compress(res->body)));
+ compressed = true;
+ err = sqlite3_bind_blob(stmt, 4, data->data(), data->size(), [](void *p) { delete (std::string *)p; });
+ break;
+ }
+ if (err != SQLITE_OK) {
+ Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db));
+ err = sqlite3_finalize(stmt);
+ if (err != SQLITE_OK) {
+ Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db));
+ }
+ return;
+ }
+
+ err = sqlite3_bind_int(stmt, 5, compressed);
if (err != SQLITE_OK) {
Log::Warning(Event::Database, "%s: %s", sqlite3_errstr(err), sqlite3_errmsg(db));
err = sqlite3_finalize(stmt);
@@ -181,7 +222,7 @@ void FileSource::saveFile(const std::string &url, platform::Response *const res)
}
-void FileSource::load(ResourceType /*type*/, const std::string &url, std::function<void(platform::Response *)> callback, const std::shared_ptr<uv::loop> loop) {
+void FileSource::load(ResourceType type, const std::string &url, std::function<void(platform::Response *)> callback, const std::shared_ptr<uv::loop> loop) {
// convert relative URLs to absolute URLs
const std::string absoluteURL = [&]() -> std::string {
@@ -218,10 +259,10 @@ void FileSource::load(ResourceType /*type*/, const std::string &url, std::functi
const std::string cleanURL = removeAccessTokenFromURL(url);
// load from the internet
- if (!loadFile(cleanURL, callback)) {
+ if (!loadFile(type, cleanURL, callback)) {
const std::shared_ptr<FileSource> source = shared_from_this();
platform::request_http(absoluteURL, [=](platform::Response *res) {
- source->saveFile(cleanURL, res);
+ source->saveFile(type, cleanURL, res);
callback(res);
}, loop);
}