diff options
Diffstat (limited to 'platform/default')
-rw-r--r-- | platform/default/default_file_source.cpp | 136 | ||||
-rw-r--r-- | platform/default/mbgl/storage/offline_database.cpp | 297 | ||||
-rw-r--r-- | platform/default/mbgl/storage/offline_database.hpp | 50 | ||||
-rw-r--r-- | platform/default/mbgl/storage/offline_schema.cpp.include | 51 | ||||
-rw-r--r-- | platform/default/mbgl/storage/offline_schema.js | 24 | ||||
-rw-r--r-- | platform/default/mbgl/storage/offline_schema.sql | 62 | ||||
-rw-r--r-- | platform/default/sqlite3.cpp | 34 | ||||
-rw-r--r-- | platform/default/sqlite_cache.cpp | 502 | ||||
-rw-r--r-- | platform/default/sqlite_cache_impl.hpp | 68 |
9 files changed, 605 insertions, 619 deletions
diff --git a/platform/default/default_file_source.cpp b/platform/default/default_file_source.cpp index 56b47539b1..efe893d49b 100644 --- a/platform/default/default_file_source.cpp +++ b/platform/default/default_file_source.cpp @@ -1,10 +1,11 @@ #include <mbgl/storage/default_file_source.hpp> #include <mbgl/storage/asset_file_source.hpp> #include <mbgl/storage/online_file_source.hpp> -#include <mbgl/storage/sqlite_cache.hpp> +#include <mbgl/storage/offline_database.hpp> #include <mbgl/platform/platform.hpp> #include <mbgl/util/url.hpp> +#include <mbgl/util/thread.hpp> #include <mbgl/util/work_request.hpp> #include <cassert> @@ -23,76 +24,117 @@ namespace mbgl { class DefaultFileSource::Impl { public: - Impl(const std::string& cachePath, const std::string& assetRoot) - : assetFileSource(assetRoot), - cache(SQLiteCache::getShared(cachePath)) { + class Task { + public: + Task(Resource resource, FileSource::Callback callback, DefaultFileSource::Impl* impl) { + auto offlineResponse = impl->offlineDatabase.get(resource); + + Resource revalidation = resource; + + if (offlineResponse) { + revalidation.priorModified = offlineResponse->modified; + revalidation.priorExpires = offlineResponse->expires; + revalidation.priorEtag = offlineResponse->etag; + callback(*offlineResponse); + } + + if (!impl->offline) { + onlineRequest = impl->onlineFileSource.request(revalidation, [=] (Response onlineResponse) { + impl->offlineDatabase.put(revalidation, onlineResponse); + callback(onlineResponse); + }); + } + } + + std::unique_ptr<FileRequest> onlineRequest; + }; + + Impl(const std::string& cachePath) + : offlineDatabase(cachePath) { + } + + void setAccessToken(const std::string& accessToken) { + onlineFileSource.setAccessToken(accessToken); + } + + std::string getAccessToken() const { + return onlineFileSource.getAccessToken(); + } + + void add(FileRequest* req, Resource resource, Callback callback) { + tasks[req] = std::make_unique<Task>(resource, callback, this); } - AssetFileSource assetFileSource; - std::shared_ptr<SQLiteCache> cache; + void cancel(FileRequest* req) { + tasks.erase(req); + } + + void put(const Resource& resource, const Response& response) { + offlineDatabase.put(resource, response); + } + + void goOffline() { + offline = true; + } + + OfflineDatabase offlineDatabase; OnlineFileSource onlineFileSource; + std::unordered_map<FileRequest*, std::unique_ptr<Task>> tasks; + bool offline = false; +}; + +class DefaultFileRequest : public FileRequest { +public: + DefaultFileRequest(Resource resource, FileSource::Callback callback, util::Thread<DefaultFileSource::Impl>& thread_) + : thread(thread_), + workRequest(thread.invokeWithCallback(&DefaultFileSource::Impl::add, callback, this, resource)) { + } + + ~DefaultFileRequest() { + thread.invoke(&DefaultFileSource::Impl::cancel, this); + } + + util::Thread<DefaultFileSource::Impl>& thread; + std::unique_ptr<WorkRequest> workRequest; }; DefaultFileSource::DefaultFileSource(const std::string& cachePath, const std::string& assetRoot) - : impl(std::make_unique<DefaultFileSource::Impl>(cachePath, assetRoot)) { + : thread(std::make_unique<util::Thread<DefaultFileSource::Impl>>(util::ThreadContext{"DefaultFileSource", util::ThreadType::Unknown, util::ThreadPriority::Low}, cachePath)), + assetFileSource(std::make_unique<AssetFileSource>(assetRoot)) { } DefaultFileSource::~DefaultFileSource() = default; void DefaultFileSource::setAccessToken(const std::string& accessToken) { - impl->onlineFileSource.setAccessToken(accessToken); + thread->invokeSync(&Impl::setAccessToken, accessToken); } std::string DefaultFileSource::getAccessToken() const { - return impl->onlineFileSource.getAccessToken(); -} - -void DefaultFileSource::setMaximumCacheSize(uint64_t size) { - impl->cache->setMaximumCacheSize(size); + return thread->invokeSync<std::string>(&Impl::getAccessToken); } -void DefaultFileSource::setMaximumCacheEntrySize(uint64_t size) { - impl->cache->setMaximumCacheEntrySize(size); +void DefaultFileSource::setMaximumCacheSize(uint64_t) { + // TODO } -SQLiteCache& DefaultFileSource::getCache() { - return *impl->cache; +void DefaultFileSource::setMaximumCacheEntrySize(uint64_t) { + // TODO } -class DefaultFileRequest : public FileRequest { -public: - DefaultFileRequest(Resource resource, FileSource::Callback callback, DefaultFileSource::Impl* impl) { - cacheRequest = impl->cache->get(resource, [=](std::shared_ptr<Response> cacheResponse) mutable { - cacheRequest.reset(); - - if (cacheResponse) { - resource.priorModified = cacheResponse->modified; - resource.priorExpires = cacheResponse->expires; - resource.priorEtag = cacheResponse->etag; - } - - onlineRequest = impl->onlineFileSource.request(resource, [=] (Response onlineResponse) { - impl->cache->put(resource, onlineResponse); - callback(onlineResponse); - }); - - // Do this last because it may result in deleting this DefaultFileRequest. - if (cacheResponse) { - callback(*cacheResponse); - } - }); - } - - std::unique_ptr<WorkRequest> cacheRequest; - std::unique_ptr<FileRequest> onlineRequest; -}; - std::unique_ptr<FileRequest> DefaultFileSource::request(const Resource& resource, Callback callback) { if (isAssetURL(resource.url)) { - return impl->assetFileSource.request(resource, callback); + return assetFileSource->request(resource, callback); } else { - return std::make_unique<DefaultFileRequest>(resource, callback, impl.get()); + return std::make_unique<DefaultFileRequest>(resource, callback, *thread); } } +void DefaultFileSource::put(const Resource& resource, const Response& response) { + thread->invokeSync(&Impl::put, resource, response); +} + +void DefaultFileSource::goOffline() { + thread->invokeSync(&Impl::goOffline); +} + } // namespace mbgl diff --git a/platform/default/mbgl/storage/offline_database.cpp b/platform/default/mbgl/storage/offline_database.cpp new file mode 100644 index 0000000000..cd0f88b7fe --- /dev/null +++ b/platform/default/mbgl/storage/offline_database.cpp @@ -0,0 +1,297 @@ +#include <mbgl/storage/offline_database.hpp> +#include <mbgl/storage/response.hpp> +#include <mbgl/util/compression.hpp> +#include <mbgl/util/io.hpp> +#include <mbgl/util/string.hpp> +#include <mbgl/util/chrono.hpp> +#include <mbgl/map/tile_id.hpp> +#include <mbgl/platform/log.hpp> + +#include "sqlite3.hpp" +#include <sqlite3.h> + +namespace mbgl { + +using namespace mapbox::sqlite; + +// If you change the schema you must write a migration from the previous version. +static const uint32_t schemaVersion = 2; + +OfflineDatabase::OfflineDatabase(const std::string& path_) + : path(path_) { + ensureSchema(); +} + +OfflineDatabase::~OfflineDatabase() { + // Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we + // can't throw anything. + try { + statements.clear(); + db.reset(); + } catch (mapbox::sqlite::Exception& ex) { + Log::Error(Event::Database, ex.code, ex.what()); + } +} + +void OfflineDatabase::ensureSchema() { + if (path != ":memory:") { + try { + db = std::make_unique<Database>(path.c_str(), ReadWrite); + + { + Statement userVersionStmt(db->prepare("PRAGMA user_version")); + userVersionStmt.run(); + switch (userVersionStmt.get<int>(0)) { + case 0: break; // cache-only database; ok to delete + case 1: break; // cache-only database; ok to delete + case 2: return; + default: throw std::runtime_error("unknown schema version"); + } + } + + removeExisting(); + db = std::make_unique<Database>(path.c_str(), ReadWrite | Create); + } catch (mapbox::sqlite::Exception& ex) { + if (ex.code == SQLITE_CANTOPEN) { + db = std::make_unique<Database>(path.c_str(), ReadWrite | Create); + } else if (ex.code == SQLITE_NOTADB) { + removeExisting(); + db = std::make_unique<Database>(path.c_str(), ReadWrite | Create); + } + } + } + + #include "offline_schema.cpp.include" + + db = std::make_unique<Database>(path.c_str(), ReadWrite | Create); + db->exec(schema); + db->exec("PRAGMA user_version = " + util::toString(schemaVersion)); +} + +void OfflineDatabase::removeExisting() { + Log::Warning(Event::Database, "Removing existing incompatible offline database"); + + db.reset(); + + try { + util::deleteFile(path); + } catch (util::IOException& ex) { + Log::Error(Event::Database, ex.code, ex.what()); + } +} + +mapbox::sqlite::Statement& OfflineDatabase::getStatement(const char * sql) { + auto it = statements.find(sql); + + if (it != statements.end()) { + it->second->reset(); + return *it->second; + } + + return *(statements[sql] = std::make_unique<Statement>(db->prepare(sql))); +} + +optional<Response> OfflineDatabase::get(const Resource& resource) { + if (resource.kind == Resource::Kind::Tile) { + assert(resource.tileData); + return getTile(*resource.tileData); + } else { + return getResource(resource); + } +} + +void OfflineDatabase::put(const Resource& resource, const Response& response) { + // Except for 404s, don't store errors in the cache. + if (response.error && response.error->reason != Response::Error::Reason::NotFound) { + return; + } + + if (resource.kind == Resource::Kind::Tile) { + assert(resource.tileData); + putTile(*resource.tileData, response); + } else { + putResource(resource, response); + } +} + +optional<Response> OfflineDatabase::getResource(const Resource& resource) { + mapbox::sqlite::Statement& stmt = getStatement( + // 0 1 2 3 4 + "SELECT etag, expires, modified, data, compressed " + "FROM resources " + "WHERE url = ?"); + + stmt.bind(1, resource.url); + + if (!stmt.run()) { + return {}; + } + + Response response; + + response.etag = stmt.get<optional<std::string>>(0); + response.expires = stmt.get<optional<SystemTimePoint>>(1); + response.modified = stmt.get<optional<SystemTimePoint>>(2); + + optional<std::string> data = stmt.get<optional<std::string>>(3); + if (!data) { + response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound); + } else if (stmt.get<int>(4)) { + response.data = std::make_shared<std::string>(util::decompress(*data)); + } else { + response.data = std::make_shared<std::string>(*data); + } + + return response; +} + +void OfflineDatabase::putResource(const Resource& resource, const Response& response) { + if (response.notModified) { + mapbox::sqlite::Statement& stmt = getStatement( + // 1 2 3 + "UPDATE resources SET accessed = ?, expires = ? WHERE url = ?"); + + stmt.bind(1, SystemClock::now()); + stmt.bind(2, response.expires); + stmt.bind(3, resource.url); + stmt.run(); + } else { + mapbox::sqlite::Statement& stmt = getStatement( + // 1 2 3 4 5 6 7 8 + "REPLACE INTO resources (url, kind, etag, expires, modified, accessed, data, compressed) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?)"); + + stmt.bind(1 /* url */, resource.url); + stmt.bind(2 /* kind */, int(resource.kind)); + stmt.bind(3 /* etag */, response.etag); + stmt.bind(4 /* expires */, response.expires); + stmt.bind(5 /* modified */, response.modified); + stmt.bind(6 /* accessed */, SystemClock::now()); + + std::string data; + + if (response.error) { // Can only be NotFound + stmt.bind(7 /* data */, nullptr); + stmt.bind(8 /* compressed */, false); + } else { + data = util::compress(*response.data); + if (data.size() < response.data->size()) { + stmt.bind(7 /* data */, data, false); // do not retain the string internally. + stmt.bind(8 /* compressed */, true); + } else { + stmt.bind(7 /* data */, *response.data, false); // do not retain the string internally. + stmt.bind(8 /* compressed */, false); + } + } + + stmt.run(); + } +} + +optional<Response> OfflineDatabase::getTile(const Resource::TileData& tile) { + mapbox::sqlite::Statement& stmt = getStatement( + // 0 1 2 3 4 + "SELECT etag, expires, modified, data, compressed " + "FROM tilesets, tiles " + "WHERE tilesets.url_template = ? " // 1 + "AND tilesets.pixel_ratio = ? " // 2 + "AND tiles.x = ? " // 3 + "AND tiles.y = ? " // 4 + "AND tiles.z = ? " // 5 + "AND tilesets.id = tiles.tileset_id "); + + stmt.bind(1, tile.urlTemplate); + stmt.bind(2, tile.pixelRatio); + stmt.bind(3, tile.x); + stmt.bind(4, tile.y); + stmt.bind(5, tile.z); + + if (!stmt.run()) { + return {}; + } + + Response response; + + response.etag = stmt.get<optional<std::string>>(0); + response.expires = stmt.get<optional<SystemTimePoint>>(1); + response.modified = stmt.get<optional<SystemTimePoint>>(2); + + optional<std::string> data = stmt.get<optional<std::string>>(3); + if (!data) { + response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound); + } else if (stmt.get<int>(4)) { + response.data = std::make_shared<std::string>(util::decompress(*data)); + } else { + response.data = std::make_shared<std::string>(*data); + } + + return response; +} + +void OfflineDatabase::putTile(const Resource::TileData& tile, const Response& response) { + if (response.notModified) { + mapbox::sqlite::Statement& stmt = getStatement( + "UPDATE tiles SET accessed = ?1, expires = ?2 " + "WHERE tileset_id = ( " + " SELECT id FROM tilesets " + " WHERE url_template = ?3 " + " AND pixel_ratio = ?4) " + "AND tiles.x = ?5 " + "AND tiles.y = ?6 " + "AND tiles.z = ?7 "); + + stmt.bind(1, SystemClock::now()); + stmt.bind(2, response.expires); + stmt.bind(3, tile.urlTemplate); + stmt.bind(4, tile.pixelRatio); + stmt.bind(5, tile.x); + stmt.bind(6, tile.y); + stmt.bind(7, tile.z); + stmt.run(); + } else { + mapbox::sqlite::Statement& stmt1 = getStatement( + "REPLACE INTO tilesets (url_template, pixel_ratio) " + "VALUES (?1, ?2) "); + + stmt1.bind(1 /* url_template */, tile.urlTemplate); + stmt1.bind(2 /* pixel_ratio */, tile.pixelRatio); + stmt1.run(); + + mapbox::sqlite::Statement& stmt2 = getStatement( + "REPLACE INTO tiles (tileset_id, x, y, z, modified, etag, expires, accessed, data, compressed) " + "SELECT tilesets.id, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11 " + "FROM tilesets " + "WHERE url_template = ?1 " + "AND pixel_ratio = ?2 "); + + stmt2.bind(1 /* url_template */, tile.urlTemplate); + stmt2.bind(2 /* pixel_ratio */, tile.pixelRatio); + stmt2.bind(3 /* x */, tile.x); + stmt2.bind(4 /* y */, tile.y); + stmt2.bind(5 /* z */, tile.z); + stmt2.bind(6 /* modified */, response.modified); + stmt2.bind(7 /* etag */, response.etag); + stmt2.bind(8 /* expires */, response.expires); + stmt2.bind(9 /* accessed */, SystemClock::now()); + + std::string data; + + if (response.error) { // Can only be NotFound + stmt2.bind(10 /* data */, nullptr); + stmt2.bind(11 /* compressed */, false); + } else { + data = util::compress(*response.data); + if (data.size() < response.data->size()) { + stmt2.bind(10 /* data */, data, false); // do not retain the string internally. + stmt2.bind(11 /* compressed */, true); + } else { + stmt2.bind(10 /* data */, *response.data, false); // do not retain the string internally. + stmt2.bind(11 /* compressed */, false); + } + } + + stmt2.run(); + } +} + +} // namespace mbgl diff --git a/platform/default/mbgl/storage/offline_database.hpp b/platform/default/mbgl/storage/offline_database.hpp new file mode 100644 index 0000000000..bc6f784d50 --- /dev/null +++ b/platform/default/mbgl/storage/offline_database.hpp @@ -0,0 +1,50 @@ +#ifndef MBGL_OFFLINE_DATABASE +#define MBGL_OFFLINE_DATABASE + +#include <mbgl/storage/resource.hpp> +#include <mbgl/util/noncopyable.hpp> +#include <mbgl/util/optional.hpp> + +#include <unordered_map> +#include <memory> +#include <string> + +namespace mapbox { +namespace sqlite { +class Database; +class Statement; +} +} + +namespace mbgl { + +class Response; +class TileID; + +class OfflineDatabase : private util::noncopyable { +public: + OfflineDatabase(const std::string& path); + ~OfflineDatabase(); + + optional<Response> get(const Resource&); + void put(const Resource&, const Response&); + +private: + void ensureSchema(); + void removeExisting(); + mapbox::sqlite::Statement& getStatement(const char *); + + optional<Response> getTile(const Resource::TileData&); + void putTile(const Resource::TileData&, const Response&); + + optional<Response> getResource(const Resource&); + void putResource(const Resource&, const Response&); + + const std::string path; + std::unique_ptr<::mapbox::sqlite::Database> db; + std::unordered_map<const char *, std::unique_ptr<::mapbox::sqlite::Statement>> statements; +}; + +} // namespace mbgl + +#endif diff --git a/platform/default/mbgl/storage/offline_schema.cpp.include b/platform/default/mbgl/storage/offline_schema.cpp.include new file mode 100644 index 0000000000..31b11d5f12 --- /dev/null +++ b/platform/default/mbgl/storage/offline_schema.cpp.include @@ -0,0 +1,51 @@ +/* THIS IS A GENERATED FILE; EDIT offline_schema.sql INSTEAD */ +static const char * schema = +"CREATE TABLE resources (\n" +" url TEXT NOT NULL PRIMARY KEY,\n" +" kind INTEGER NOT NULL,\n" +" expires INTEGER,\n" +" modified INTEGER,\n" +" accessed INTEGER,\n" +" etag TEXT,\n" +" data BLOB,\n" +" compressed INTEGER NOT NULL DEFAULT 0\n" +");\n" +"CREATE TABLE tilesets (\n" +" id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,\n" +" url_template TEXT NOT NULL,\n" +" pixel_ratio INTEGER,\n" +" UNIQUE (url_template, pixel_ratio)\n" +");\n" +"CREATE TABLE tiles (\n" +" tileset_id INTEGER NOT NULL REFERENCES tilesets(id),\n" +" z INTEGER NOT NULL,\n" +" x INTEGER NOT NULL,\n" +" y INTEGER NOT NULL,\n" +" expires INTEGER,\n" +" modified INTEGER,\n" +" accessed INTEGER,\n" +" etag TEXT,\n" +" data BLOB,\n" +" compressed INTEGER NOT NULL DEFAULT 0,\n" +" PRIMARY KEY (tileset_id, z, x, y)\n" +");\n" +"CREATE TABLE regions (\n" +" id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,\n" +" definition TEXT NOT NULL,\n" +" description BLOB\n" +");\n" +"CREATE TABLE region_resources (\n" +" region_id INTEGER NOT NULL REFERENCES regions(id),\n" +" resource_url TEXT NOT NULL REFERENCES resources(url),\n" +" PRIMARY KEY (region_id, resource_url)\n" +");\n" +"CREATE TABLE region_tiles (\n" +" region_id INTEGER NOT NULL REFERENCES regions(id),\n" +" tileset_id INTEGER NOT NULL REFERENCES tilesets(id),\n" +" z INTEGER NOT NULL,\n" +" x INTEGER NOT NULL,\n" +" y INTEGER NOT NULL,\n" +" PRIMARY KEY (region_id, tileset_id, z, x, y),\n" +" FOREIGN KEY (tileset_id, z, x, y) REFERENCES tiles (tileset_id, z, x, y)\n" +");\n" +; diff --git a/platform/default/mbgl/storage/offline_schema.js b/platform/default/mbgl/storage/offline_schema.js new file mode 100644 index 0000000000..153ba34e38 --- /dev/null +++ b/platform/default/mbgl/storage/offline_schema.js @@ -0,0 +1,24 @@ +// To regenerate: +// (cd platform/default/mbgl/storage && node offline_schema.js) + +var fs = require('fs'); +var readline = require('readline'); + +var lineReader = readline.createInterface({ + input: fs.createReadStream('offline_schema.sql') +}); + +var lines = [ + "/* THIS IS A GENERATED FILE; EDIT offline_schema.sql INSTEAD */", + "static const char * schema = ", +]; + +lineReader + .on('line', function (line) { + line = line.replace(/ *--.*/, ''); + if (line) lines.push('"' + line + '\\n"'); + }) + .on('close', function () { + lines.push(';\n'); + fs.writeFileSync('offline_schema.cpp.include', lines.join('\n')); + }); diff --git a/platform/default/mbgl/storage/offline_schema.sql b/platform/default/mbgl/storage/offline_schema.sql new file mode 100644 index 0000000000..e55517c489 --- /dev/null +++ b/platform/default/mbgl/storage/offline_schema.sql @@ -0,0 +1,62 @@ +CREATE TABLE resources ( -- Generic table for style, source, sprite, and glyph resources. + url TEXT NOT NULL PRIMARY KEY, -- Same schema as http_cache table. + kind INTEGER NOT NULL, + expires INTEGER, + modified INTEGER, + accessed INTEGER, + etag TEXT, + data BLOB, -- NULL if the response was a 404 + compressed INTEGER NOT NULL DEFAULT 0 +); + +CREATE TABLE tilesets ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + url_template TEXT NOT NULL, -- As it would appear in TileJSON (but no support for host sharding). + pixel_ratio INTEGER, -- If NULL, 1 is assumed for raster sources. + UNIQUE (url_template, pixel_ratio) -- Capable of caching the same tileset at multiple resolutions. +); + +CREATE TABLE tiles ( + tileset_id INTEGER NOT NULL REFERENCES tilesets(id), + z INTEGER NOT NULL, -- Fully abandon TMS coordinates in favor of ZXY. + x INTEGER NOT NULL, + y INTEGER NOT NULL, + expires INTEGER, + modified INTEGER, + accessed INTEGER, + etag TEXT, + data BLOB, -- NULL if the response was a 404 + compressed INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (tileset_id, z, x, y) +); + +CREATE TABLE regions ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + definition TEXT NOT NULL, -- JSON formatted definition of region. Regions may be of variant types: + -- e.g. bbox and zoom range, route path, flyTo parameters, etc. Note that + -- the set of tiles required for a region may span multiple sources. + description BLOB -- User provided data in user-defined format +); + +CREATE TABLE region_resources ( + region_id INTEGER NOT NULL REFERENCES regions(id), + resource_url TEXT NOT NULL REFERENCES resources(url), + PRIMARY KEY (region_id, resource_url) +); + +CREATE TABLE region_tiles ( + region_id INTEGER NOT NULL REFERENCES regions(id), + tileset_id INTEGER NOT NULL REFERENCES tilesets(id), + z INTEGER NOT NULL, + x INTEGER NOT NULL, + y INTEGER NOT NULL, + PRIMARY KEY (region_id, tileset_id, z, x, y), + FOREIGN KEY (tileset_id, z, x, y) REFERENCES tiles (tileset_id, z, x, y) +); + +-- `region_resources` and `region_tiles` are used for deduplication and deletion logic. +-- A row in `tiles` exists IFF one or more corresponding rows exist in `region_tiles`. If +-- more than one corresponding row exists, it indicates multiple regions contain the tile, and +-- storage for the tile is being deduplicated. When a region is deleted, corresponding rows in +-- `region_tiles` must also be deleted, and then rows in `tiles` and `tilesets` without a +-- corresponding `region_tiles` row must be deleted. Similarly for `resources` / `region_resources`. diff --git a/platform/default/sqlite3.cpp b/platform/default/sqlite3.cpp index 5122e01015..09301bc4d9 100644 --- a/platform/default/sqlite3.cpp +++ b/platform/default/sqlite3.cpp @@ -115,9 +115,19 @@ template <> void Statement::bind(int offset, std::nullptr_t) { check(sqlite3_bind_null(stmt, offset)); } -template <> void Statement::bind(int offset, int value) { +template <> void Statement::bind(int offset, int8_t value) { assert(stmt); - check(sqlite3_bind_int(stmt, offset, value)); + check(sqlite3_bind_int64(stmt, offset, value)); +} + +template <> void Statement::bind(int offset, int16_t value) { + assert(stmt); + check(sqlite3_bind_int64(stmt, offset, value)); +} + +template <> void Statement::bind(int offset, int32_t value) { + assert(stmt); + check(sqlite3_bind_int64(stmt, offset, value)); } template <> void Statement::bind(int offset, int64_t value) { @@ -125,6 +135,26 @@ template <> void Statement::bind(int offset, int64_t value) { check(sqlite3_bind_int64(stmt, offset, value)); } +template <> void Statement::bind(int offset, uint8_t value) { + assert(stmt); + check(sqlite3_bind_int64(stmt, offset, value)); +} + +template <> void Statement::bind(int offset, uint16_t value) { + assert(stmt); + check(sqlite3_bind_int64(stmt, offset, value)); +} + +template <> void Statement::bind(int offset, uint32_t value) { + assert(stmt); + check(sqlite3_bind_int64(stmt, offset, value)); +} + +template <> void Statement::bind(int offset, float value) { + assert(stmt); + check(sqlite3_bind_double(stmt, offset, value)); +} + template <> void Statement::bind(int offset, double value) { assert(stmt); check(sqlite3_bind_double(stmt, offset, value)); diff --git a/platform/default/sqlite_cache.cpp b/platform/default/sqlite_cache.cpp deleted file mode 100644 index b3f528ac92..0000000000 --- a/platform/default/sqlite_cache.cpp +++ /dev/null @@ -1,502 +0,0 @@ -#include "sqlite_cache_impl.hpp" -#include <mbgl/storage/resource.hpp> -#include <mbgl/storage/response.hpp> - -#include <mbgl/util/compression.hpp> -#include <mbgl/util/io.hpp> -#include <mbgl/util/string.hpp> -#include <mbgl/util/thread.hpp> -#include <mbgl/util/mapbox.hpp> -#include <mbgl/platform/log.hpp> - -#include "sqlite3.hpp" -#include <sqlite3.h> - -#include <unordered_map> -#include <mutex> - -namespace { - -// The cache won't accept entries larger than this arbitrary size -// and will silently discard request for adding them to the cache. -// Large entries can cause the database to grow in disk size and -// never shrink again. -const uint64_t kMaximumCacheEntrySize = 5 * 1024 * 1024; // 5 MB - -// Number of records we delete when we are close to the maximum -// database size, if set. The current criteria is to prune -// the least used entries based on `accessed` time. -const int kPrunedEntriesLimit = 100; - -} // namespace - -namespace mbgl { - -using namespace mapbox::sqlite; - -SQLiteCache::SQLiteCache(const std::string& path_) - : thread(std::make_unique<util::Thread<Impl>>(util::ThreadContext{"SQLiteCache", util::ThreadType::Unknown, util::ThreadPriority::Low}, path_)) { -} - -SQLiteCache::~SQLiteCache() = default; - -SQLiteCache::Impl::Impl(const std::string& path_) - : maximumCacheSize(0), // Unlimited - maximumCacheEntrySize(kMaximumCacheEntrySize), - path(path_) { -} - -SQLiteCache::Impl::~Impl() { - // Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we - // can't throw anything. - try { - getStmt.reset(); - putStmt.reset(); - refreshStmt.reset(); - countStmt.reset(); - freeStmt.reset(); - pruneStmt.reset(); - accessedStmt.reset(); - db.reset(); - } catch (mapbox::sqlite::Exception& ex) { - Log::Error(Event::Database, ex.code, ex.what()); - } -} - -void SQLiteCache::Impl::createDatabase() { - db = std::make_unique<Database>(path.c_str(), ReadWrite | Create); -} - -int SQLiteCache::Impl::schemaVersion() const { - // WARNING: Bump the version when changing the cache - // scheme to force the table to be recreated. - return 1; -} - -void SQLiteCache::Impl::createSchema() { - constexpr const char *const sql = "" - "CREATE TABLE IF NOT EXISTS `http_cache` (" - " `url` TEXT PRIMARY KEY NOT NULL," - " `status` INTEGER NOT NULL," // The response status (Successful or Error). - " `kind` INTEGER NOT NULL," // The kind of file. - " `modified` INTEGER," // Timestamp when the file was last modified. - " `etag` TEXT," - " `expires` INTEGER," // Timestamp when the server says the file expires. - " `accessed` INTEGER," // Timestamp when the database record was last accessed. - " `data` BLOB," - " `compressed` INTEGER NOT NULL DEFAULT 0" // Whether the data is compressed. - ");" - "CREATE INDEX IF NOT EXISTS `http_cache_kind_idx` ON `http_cache` (`kind`);" - "CREATE INDEX IF NOT EXISTS `http_cache_accessed_idx` ON `http_cache` (`accessed`);"; - - ensureSchemaVersion(); - - try { - db->exec(sql); - db->exec("PRAGMA user_version = " + util::toString(schemaVersion())); - schema = true; - } catch (mapbox::sqlite::Exception &ex) { - if (ex.code == SQLITE_NOTADB) { - Log::Warning(Event::Database, "Trashing invalid database"); - db.reset(); - try { - util::deleteFile(path); - } catch (util::IOException& ioEx) { - Log::Error(Event::Database, ex.code, ex.what()); - } - db = std::make_unique<Database>(path.c_str(), ReadWrite | Create); - } else { - Log::Error(Event::Database, ex.code, ex.what()); - } - - // Creating the database table + index failed. That means there may already be one, likely - // with different columns. Drop it and try to create a new one. - db->exec("DROP TABLE IF EXISTS `http_cache`"); - db->exec(sql); - db->exec("PRAGMA user_version = " + util::toString(schemaVersion())); - } -} - -void SQLiteCache::Impl::ensureSchemaVersion() { - try { - Statement userVersionStmt(db->prepare("PRAGMA user_version")); - if (userVersionStmt.run() && userVersionStmt.get<int>(0) == schemaVersion()) { - return; - } - } catch (mapbox::sqlite::Exception& ex) { - if (ex.code == SQLITE_NOTADB) { - return; - } - - Log::Error(Event::Database, ex.code, ex.what()); - } - - // Version mismatch, drop the table so it will - // get recreated by `createSchema()`. - try { - db->exec("DROP TABLE IF EXISTS `http_cache`"); - } catch (mapbox::sqlite::Exception& ex) { - Log::Error(Event::Database, ex.code, ex.what()); - } -} - -void SQLiteCache::setMaximumCacheSize(uint64_t size) { - thread->invoke(&Impl::setMaximumCacheSize, size); -} - -void SQLiteCache::Impl::setMaximumCacheSize(uint64_t size) { - maximumCacheSize = size; - - // Unlimited. - if (size == 0) { - return; - } - - uint64_t lastSoftSize = cacheSoftSize(); - - // Keep pruning until we fit in the new - // size limit. - while (lastSoftSize > maximumCacheSize) { - pruneEntries(); - - if (lastSoftSize != cacheSoftSize()) { - lastSoftSize = cacheSoftSize(); - } else { - break; - } - } - - if (cacheHardSize() > size) { - Log::Warning(mbgl::Event::Database, - "Current cache hard size is bigger than the defined " - "maximum size. Database won't get truncated."); - } -} - -void SQLiteCache::setMaximumCacheEntrySize(uint64_t size) { - thread->invoke(&Impl::setMaximumCacheEntrySize, size); -} - -void SQLiteCache::Impl::setMaximumCacheEntrySize(uint64_t size) { - maximumCacheEntrySize = size; -} - -void SQLiteCache::Impl::initializeDatabase() { - if (!db) { - createDatabase(); - } - - if (!schema) { - createSchema(); - } -} - -int SQLiteCache::Impl::cachePageSize() { - try { - if (!pageSize) { - Statement pageSizeStmt(db->prepare("PRAGMA page_size")); - if (pageSizeStmt.run()) { - pageSize = pageSizeStmt.get<int>(0); - } - } - } catch (mapbox::sqlite::Exception& ex) { - Log::Error(Event::Database, ex.code, ex.what()); - } - - return pageSize; -} - -uint64_t SQLiteCache::Impl::cacheHardSize() { - try { - initializeDatabase(); - - if (!countStmt) { - countStmt = std::make_unique<Statement>(db->prepare("PRAGMA page_count")); - } else { - countStmt->reset(); - } - - if (countStmt->run()) { - return cachePageSize() * countStmt->get<int>(0); - } - } catch (mapbox::sqlite::Exception& ex) { - Log::Error(Event::Database, ex.code, ex.what()); - } - - return 0; -} - -uint64_t SQLiteCache::Impl::cacheSoftSize() { - if (!softSizeDirty) { - return softSize; - } - - try { - initializeDatabase(); - - if (!freeStmt) { - freeStmt = std::make_unique<Statement>(db->prepare("PRAGMA freelist_count")); - } else { - freeStmt->reset(); - } - - uint64_t hardSize = cacheHardSize(); - if (!hardSize) { - return 0; - } - - if (freeStmt->run()) { - return hardSize - cachePageSize() * freeStmt->get<int>(0); - } - - softSizeDirty = false; - } catch (mapbox::sqlite::Exception& ex) { - Log::Error(Event::Database, ex.code, ex.what()); - } - - return 0; -} - -bool SQLiteCache::Impl::needsPruning() { - // SQLite database never shrinks in size unless we call VACCUM. We here - // are monitoring the soft limit (i.e. number of free pages in the file) - // and as it approaches to the hard limit (i.e. the actual file size) we - // delete an arbitrary number of old cache entries. - // - // The free pages approach saves us from calling VACCUM or keeping a - // running total, which can be costly. We need a buffer because pages can - // get fragmented on the database. - if (cacheSoftSize() + maximumCacheEntrySize * 2 < maximumCacheSize) { - return false; - } else { - return true; - } -} - -void SQLiteCache::Impl::pruneEntries() { - if (!maximumCacheSize) { - return; - } - - if (!needsPruning()) { - return; - } - - try { - if (!pruneStmt) { - pruneStmt = std::make_unique<Statement>(db->prepare( - "DELETE FROM `http_cache` WHERE `rowid` IN (SELECT `rowid` FROM " - // 1 - "`http_cache` ORDER BY `accessed` ASC LIMIT ?)")); - } else { - pruneStmt->reset(); - } - - pruneStmt->bind(1, kPrunedEntriesLimit); - - pruneStmt->run(); - softSizeDirty = true; - } catch (mapbox::sqlite::Exception& ex) { - Log::Error(Event::Database, ex.code, ex.what()); - } -} - -std::unique_ptr<WorkRequest> SQLiteCache::get(const Resource &resource, Callback callback) { - // Can be called from any thread, but most likely from the file source thread. - // Will try to load the URL from the SQLite database and call the callback when done. - // Note that the callback is probably going to invoked from another thread, so the caller - // must make sure that it can run in that thread. - return thread->invokeWithCallback(&Impl::get, callback, resource); -} - -void SQLiteCache::Impl::get(const Resource &resource, Callback callback) { - try { - initializeDatabase(); - - if (!getStmt) { - // Initialize the statement 0 1 - getStmt = std::make_unique<Statement>(db->prepare("SELECT `status`, `modified`, " - // 2 3 4 5 1 - "`etag`, `expires`, `data`, `compressed` FROM `http_cache` WHERE `url` = ?")); - } else { - getStmt->reset(); - } - - getStmt->bind(1, resource.url); - if (getStmt->run()) { - // There is data. - auto response = std::make_unique<Response>(); - const auto status = getStmt->get<int>(0); - if (status > 1) { - // Status codes > 1 indicate an error - response->error = std::make_unique<Response::Error>(Response::Error::Reason(status)); - } - response->modified = getStmt->get<optional<SystemTimePoint>>(1); - response->etag = getStmt->get<optional<std::string>>(2); - response->expires = getStmt->get<optional<SystemTimePoint>>(3); - response->data = std::make_shared<std::string>(getStmt->get<std::string>(4)); - if (getStmt->get<int>(5)) { // == compressed - response->data = std::make_shared<std::string>(util::decompress(*response->data)); - } - callback(std::move(response)); - } else { - // There is no data. - callback(nullptr); - } - - // We do an extra query for refreshing the last time - // the record was accessed that can be costly and is only - // worth doing if we are monitoring the database size. - if (maximumCacheSize) { - if (!accessedStmt) { - accessedStmt = std::make_unique<Statement>( - // 1 2 - db->prepare("UPDATE `http_cache` SET `accessed` = ? WHERE `url` = ?")); - } else { - accessedStmt->reset(); - } - - accessedStmt->bind(1, SystemClock::now()); - accessedStmt->bind(2, resource.url); - accessedStmt->run(); - } - } catch (mapbox::sqlite::Exception& ex) { - Log::Error(Event::Database, ex.code, ex.what()); - callback(nullptr); - } catch (std::runtime_error& ex) { - Log::Error(Event::Database, "%s", ex.what()); - callback(nullptr); - } -} - -void SQLiteCache::put(const Resource& resource, const Response& response) { - // Except for 404s, don't store errors in the cache. - if (response.error && response.error->reason != Response::Error::Reason::NotFound) { - return; - } - - if (response.notModified) { - thread->invoke(&Impl::refresh, resource, response.expires); - } else { - thread->invoke(&Impl::put, resource, response); - } -} - -void SQLiteCache::Impl::put(const Resource& resource, const Response& response) { - try { - initializeDatabase(); - pruneEntries(); - - if (response.data) { - auto entrySize = response.data->size(); - - if (entrySize > maximumCacheEntrySize) { - Log::Warning(Event::Database, "Entry too big for caching."); - return; - } - - if (maximumCacheSize && entrySize + cacheSoftSize() > maximumCacheSize) { - Log::Warning(Event::Database, "Unable to make space for new entries."); - return; - } - } - - if (!putStmt) { - putStmt = std::make_unique<Statement>(db->prepare("REPLACE INTO `http_cache` (" - // 1 2 3 4 5 6 7 8 9 - "`url`, `status`, `kind`, `modified`, `etag`, `expires`, `accessed`, `data`, `compressed`" - ") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)")); - } else { - putStmt->reset(); - } - - putStmt->bind(1 /* url */, resource.url); - if (response.error) { - putStmt->bind(2 /* status */, int(response.error->reason)); - } else { - putStmt->bind(2 /* status */, 1 /* success */); - } - putStmt->bind(3 /* kind */, int(resource.kind)); - putStmt->bind(4 /* modified */, response.modified); - putStmt->bind(5 /* etag */, response.etag); - putStmt->bind(6 /* expires */, response.expires); - putStmt->bind(7 /* accessed */, SystemClock::now()); - - std::string data; - if (resource.kind != Resource::SpriteImage && response.data) { - // Do not compress images, since they are typically compressed already. - data = util::compress(*response.data); - } - - if (!data.empty() && data.size() < response.data->size()) { - // Store the compressed data when it is smaller than the original - // uncompressed data. - putStmt->bind(8 /* data */, data, false); // do not retain the string internally. - putStmt->bind(9 /* compressed */, true); - } else if (response.data) { - putStmt->bind(8 /* data */, *response.data, false); // do not retain the string internally. - putStmt->bind(9 /* compressed */, false); - } else { - putStmt->bind(8 /* data */, "", false); - putStmt->bind(9 /* compressed */, false); - } - - putStmt->run(); - softSizeDirty = true; - } catch (mapbox::sqlite::Exception& ex) { - Log::Error(Event::Database, ex.code, ex.what()); - } catch (std::runtime_error& ex) { - Log::Error(Event::Database, "%s", ex.what()); - } -} - -void SQLiteCache::Impl::refresh(const Resource& resource, optional<SystemTimePoint> expires) { - try { - initializeDatabase(); - - if (!refreshStmt) { - refreshStmt = std::make_unique<Statement>( - db->prepare("UPDATE `http_cache` SET " - // 1 2 3 - "`accessed` = ?, `expires` = ? WHERE `url` = ?")); - } else { - refreshStmt->reset(); - } - - refreshStmt->bind(1, SystemClock::now()); - refreshStmt->bind(2, expires); - refreshStmt->bind(3, resource.url); - refreshStmt->run(); - } catch (mapbox::sqlite::Exception& ex) { - Log::Error(Event::Database, ex.code, ex.what()); - } -} - -namespace { - -static std::mutex sharedMutex; -static std::unordered_map<std::string, std::weak_ptr<SQLiteCache>> shared; - -} // namespace - -std::shared_ptr<SQLiteCache> SQLiteCache::getShared(const std::string &path) { - std::lock_guard<std::mutex> lock(sharedMutex); - - std::shared_ptr<SQLiteCache> cache; - - auto it = shared.find(path); - if (it != shared.end()) { - cache = it->second.lock(); - if (!cache) { - cache = std::make_shared<SQLiteCache>(path); - it->second = cache; - } - } else { - cache = std::make_shared<SQLiteCache>(path); - shared.emplace(path, cache); - } - - return cache; -} - -} // namespace mbgl diff --git a/platform/default/sqlite_cache_impl.hpp b/platform/default/sqlite_cache_impl.hpp deleted file mode 100644 index e156532402..0000000000 --- a/platform/default/sqlite_cache_impl.hpp +++ /dev/null @@ -1,68 +0,0 @@ -#ifndef MBGL_STORAGE_DEFAULT_SQLITE_CACHE_IMPL -#define MBGL_STORAGE_DEFAULT_SQLITE_CACHE_IMPL - -#include <mbgl/storage/sqlite_cache.hpp> -#include <mbgl/util/chrono.hpp> -#include <mbgl/util/optional.hpp> - -namespace mapbox { -namespace sqlite { -class Database; -class Statement; -} -} - -namespace mbgl { - -class SQLiteCache::Impl { -public: - explicit Impl(const std::string &path = ":memory:"); - ~Impl(); - - void setMaximumCacheSize(uint64_t size); - void setMaximumCacheEntrySize(uint64_t size); - - void get(const Resource&, Callback); - void put(const Resource&, const Response&); - void refresh(const Resource&, optional<SystemTimePoint> expires); - -private: - void initializeDatabase(); - - int cachePageSize(); - - uint64_t cacheHardSize(); - uint64_t cacheSoftSize(); - - uint64_t softSize = 0; - bool softSizeDirty = true; - - bool needsPruning(); - void pruneEntries(); - - void createDatabase(); - void createSchema(); - - int schemaVersion() const; - void ensureSchemaVersion(); - - int pageSize = 0; - - uint64_t maximumCacheSize; - uint64_t maximumCacheEntrySize; - - const std::string path; - std::unique_ptr<::mapbox::sqlite::Database> db; - std::unique_ptr<::mapbox::sqlite::Statement> getStmt; - std::unique_ptr<::mapbox::sqlite::Statement> putStmt; - std::unique_ptr<::mapbox::sqlite::Statement> refreshStmt; - std::unique_ptr<::mapbox::sqlite::Statement> countStmt; - std::unique_ptr<::mapbox::sqlite::Statement> freeStmt; - std::unique_ptr<::mapbox::sqlite::Statement> pruneStmt; - std::unique_ptr<::mapbox::sqlite::Statement> accessedStmt; - bool schema = false; -}; - -} // namespace mbgl - -#endif |