diff options
author | John Firebaugh <john.firebaugh@gmail.com> | 2016-01-26 12:19:04 -0800 |
---|---|---|
committer | John Firebaugh <john.firebaugh@gmail.com> | 2016-02-10 15:40:20 -0800 |
commit | cdedb66387065680efd318dacb61572c920b81b1 (patch) | |
tree | 3cc2676e15fac4c7b024cd17603c46e341f5a589 /platform/default/mbgl | |
parent | 025375ad0b365a06e0742b92fecc9bc538b5a6e0 (diff) | |
download | qtlocation-mapboxgl-cdedb66387065680efd318dacb61572c920b81b1.tar.gz |
[core] Reimplement existing caching within an offline-capable database schema
Diffstat (limited to 'platform/default/mbgl')
-rw-r--r-- | platform/default/mbgl/storage/offline_database.cpp | 297 | ||||
-rw-r--r-- | platform/default/mbgl/storage/offline_database.hpp | 50 | ||||
-rw-r--r-- | platform/default/mbgl/storage/offline_schema.cpp.include | 51 | ||||
-rw-r--r-- | platform/default/mbgl/storage/offline_schema.js | 24 | ||||
-rw-r--r-- | platform/default/mbgl/storage/offline_schema.sql | 62 |
5 files changed, 484 insertions, 0 deletions
diff --git a/platform/default/mbgl/storage/offline_database.cpp b/platform/default/mbgl/storage/offline_database.cpp new file mode 100644 index 0000000000..cd0f88b7fe --- /dev/null +++ b/platform/default/mbgl/storage/offline_database.cpp @@ -0,0 +1,297 @@ +#include <mbgl/storage/offline_database.hpp> +#include <mbgl/storage/response.hpp> +#include <mbgl/util/compression.hpp> +#include <mbgl/util/io.hpp> +#include <mbgl/util/string.hpp> +#include <mbgl/util/chrono.hpp> +#include <mbgl/map/tile_id.hpp> +#include <mbgl/platform/log.hpp> + +#include "sqlite3.hpp" +#include <sqlite3.h> + +namespace mbgl { + +using namespace mapbox::sqlite; + +// If you change the schema you must write a migration from the previous version. +static const uint32_t schemaVersion = 2; + +OfflineDatabase::OfflineDatabase(const std::string& path_) + : path(path_) { + ensureSchema(); +} + +OfflineDatabase::~OfflineDatabase() { + // Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we + // can't throw anything. + try { + statements.clear(); + db.reset(); + } catch (mapbox::sqlite::Exception& ex) { + Log::Error(Event::Database, ex.code, ex.what()); + } +} + +void OfflineDatabase::ensureSchema() { + if (path != ":memory:") { + try { + db = std::make_unique<Database>(path.c_str(), ReadWrite); + + { + Statement userVersionStmt(db->prepare("PRAGMA user_version")); + userVersionStmt.run(); + switch (userVersionStmt.get<int>(0)) { + case 0: break; // cache-only database; ok to delete + case 1: break; // cache-only database; ok to delete + case 2: return; + default: throw std::runtime_error("unknown schema version"); + } + } + + removeExisting(); + db = std::make_unique<Database>(path.c_str(), ReadWrite | Create); + } catch (mapbox::sqlite::Exception& ex) { + if (ex.code == SQLITE_CANTOPEN) { + db = std::make_unique<Database>(path.c_str(), ReadWrite | Create); + } else if (ex.code == SQLITE_NOTADB) { + removeExisting(); + db = std::make_unique<Database>(path.c_str(), ReadWrite | Create); + } + } + } + + #include "offline_schema.cpp.include" + + db = std::make_unique<Database>(path.c_str(), ReadWrite | Create); + db->exec(schema); + db->exec("PRAGMA user_version = " + util::toString(schemaVersion)); +} + +void OfflineDatabase::removeExisting() { + Log::Warning(Event::Database, "Removing existing incompatible offline database"); + + db.reset(); + + try { + util::deleteFile(path); + } catch (util::IOException& ex) { + Log::Error(Event::Database, ex.code, ex.what()); + } +} + +mapbox::sqlite::Statement& OfflineDatabase::getStatement(const char * sql) { + auto it = statements.find(sql); + + if (it != statements.end()) { + it->second->reset(); + return *it->second; + } + + return *(statements[sql] = std::make_unique<Statement>(db->prepare(sql))); +} + +optional<Response> OfflineDatabase::get(const Resource& resource) { + if (resource.kind == Resource::Kind::Tile) { + assert(resource.tileData); + return getTile(*resource.tileData); + } else { + return getResource(resource); + } +} + +void OfflineDatabase::put(const Resource& resource, const Response& response) { + // Except for 404s, don't store errors in the cache. + if (response.error && response.error->reason != Response::Error::Reason::NotFound) { + return; + } + + if (resource.kind == Resource::Kind::Tile) { + assert(resource.tileData); + putTile(*resource.tileData, response); + } else { + putResource(resource, response); + } +} + +optional<Response> OfflineDatabase::getResource(const Resource& resource) { + mapbox::sqlite::Statement& stmt = getStatement( + // 0 1 2 3 4 + "SELECT etag, expires, modified, data, compressed " + "FROM resources " + "WHERE url = ?"); + + stmt.bind(1, resource.url); + + if (!stmt.run()) { + return {}; + } + + Response response; + + response.etag = stmt.get<optional<std::string>>(0); + response.expires = stmt.get<optional<SystemTimePoint>>(1); + response.modified = stmt.get<optional<SystemTimePoint>>(2); + + optional<std::string> data = stmt.get<optional<std::string>>(3); + if (!data) { + response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound); + } else if (stmt.get<int>(4)) { + response.data = std::make_shared<std::string>(util::decompress(*data)); + } else { + response.data = std::make_shared<std::string>(*data); + } + + return response; +} + +void OfflineDatabase::putResource(const Resource& resource, const Response& response) { + if (response.notModified) { + mapbox::sqlite::Statement& stmt = getStatement( + // 1 2 3 + "UPDATE resources SET accessed = ?, expires = ? WHERE url = ?"); + + stmt.bind(1, SystemClock::now()); + stmt.bind(2, response.expires); + stmt.bind(3, resource.url); + stmt.run(); + } else { + mapbox::sqlite::Statement& stmt = getStatement( + // 1 2 3 4 5 6 7 8 + "REPLACE INTO resources (url, kind, etag, expires, modified, accessed, data, compressed) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?)"); + + stmt.bind(1 /* url */, resource.url); + stmt.bind(2 /* kind */, int(resource.kind)); + stmt.bind(3 /* etag */, response.etag); + stmt.bind(4 /* expires */, response.expires); + stmt.bind(5 /* modified */, response.modified); + stmt.bind(6 /* accessed */, SystemClock::now()); + + std::string data; + + if (response.error) { // Can only be NotFound + stmt.bind(7 /* data */, nullptr); + stmt.bind(8 /* compressed */, false); + } else { + data = util::compress(*response.data); + if (data.size() < response.data->size()) { + stmt.bind(7 /* data */, data, false); // do not retain the string internally. + stmt.bind(8 /* compressed */, true); + } else { + stmt.bind(7 /* data */, *response.data, false); // do not retain the string internally. + stmt.bind(8 /* compressed */, false); + } + } + + stmt.run(); + } +} + +optional<Response> OfflineDatabase::getTile(const Resource::TileData& tile) { + mapbox::sqlite::Statement& stmt = getStatement( + // 0 1 2 3 4 + "SELECT etag, expires, modified, data, compressed " + "FROM tilesets, tiles " + "WHERE tilesets.url_template = ? " // 1 + "AND tilesets.pixel_ratio = ? " // 2 + "AND tiles.x = ? " // 3 + "AND tiles.y = ? " // 4 + "AND tiles.z = ? " // 5 + "AND tilesets.id = tiles.tileset_id "); + + stmt.bind(1, tile.urlTemplate); + stmt.bind(2, tile.pixelRatio); + stmt.bind(3, tile.x); + stmt.bind(4, tile.y); + stmt.bind(5, tile.z); + + if (!stmt.run()) { + return {}; + } + + Response response; + + response.etag = stmt.get<optional<std::string>>(0); + response.expires = stmt.get<optional<SystemTimePoint>>(1); + response.modified = stmt.get<optional<SystemTimePoint>>(2); + + optional<std::string> data = stmt.get<optional<std::string>>(3); + if (!data) { + response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound); + } else if (stmt.get<int>(4)) { + response.data = std::make_shared<std::string>(util::decompress(*data)); + } else { + response.data = std::make_shared<std::string>(*data); + } + + return response; +} + +void OfflineDatabase::putTile(const Resource::TileData& tile, const Response& response) { + if (response.notModified) { + mapbox::sqlite::Statement& stmt = getStatement( + "UPDATE tiles SET accessed = ?1, expires = ?2 " + "WHERE tileset_id = ( " + " SELECT id FROM tilesets " + " WHERE url_template = ?3 " + " AND pixel_ratio = ?4) " + "AND tiles.x = ?5 " + "AND tiles.y = ?6 " + "AND tiles.z = ?7 "); + + stmt.bind(1, SystemClock::now()); + stmt.bind(2, response.expires); + stmt.bind(3, tile.urlTemplate); + stmt.bind(4, tile.pixelRatio); + stmt.bind(5, tile.x); + stmt.bind(6, tile.y); + stmt.bind(7, tile.z); + stmt.run(); + } else { + mapbox::sqlite::Statement& stmt1 = getStatement( + "REPLACE INTO tilesets (url_template, pixel_ratio) " + "VALUES (?1, ?2) "); + + stmt1.bind(1 /* url_template */, tile.urlTemplate); + stmt1.bind(2 /* pixel_ratio */, tile.pixelRatio); + stmt1.run(); + + mapbox::sqlite::Statement& stmt2 = getStatement( + "REPLACE INTO tiles (tileset_id, x, y, z, modified, etag, expires, accessed, data, compressed) " + "SELECT tilesets.id, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11 " + "FROM tilesets " + "WHERE url_template = ?1 " + "AND pixel_ratio = ?2 "); + + stmt2.bind(1 /* url_template */, tile.urlTemplate); + stmt2.bind(2 /* pixel_ratio */, tile.pixelRatio); + stmt2.bind(3 /* x */, tile.x); + stmt2.bind(4 /* y */, tile.y); + stmt2.bind(5 /* z */, tile.z); + stmt2.bind(6 /* modified */, response.modified); + stmt2.bind(7 /* etag */, response.etag); + stmt2.bind(8 /* expires */, response.expires); + stmt2.bind(9 /* accessed */, SystemClock::now()); + + std::string data; + + if (response.error) { // Can only be NotFound + stmt2.bind(10 /* data */, nullptr); + stmt2.bind(11 /* compressed */, false); + } else { + data = util::compress(*response.data); + if (data.size() < response.data->size()) { + stmt2.bind(10 /* data */, data, false); // do not retain the string internally. + stmt2.bind(11 /* compressed */, true); + } else { + stmt2.bind(10 /* data */, *response.data, false); // do not retain the string internally. + stmt2.bind(11 /* compressed */, false); + } + } + + stmt2.run(); + } +} + +} // namespace mbgl diff --git a/platform/default/mbgl/storage/offline_database.hpp b/platform/default/mbgl/storage/offline_database.hpp new file mode 100644 index 0000000000..bc6f784d50 --- /dev/null +++ b/platform/default/mbgl/storage/offline_database.hpp @@ -0,0 +1,50 @@ +#ifndef MBGL_OFFLINE_DATABASE +#define MBGL_OFFLINE_DATABASE + +#include <mbgl/storage/resource.hpp> +#include <mbgl/util/noncopyable.hpp> +#include <mbgl/util/optional.hpp> + +#include <unordered_map> +#include <memory> +#include <string> + +namespace mapbox { +namespace sqlite { +class Database; +class Statement; +} +} + +namespace mbgl { + +class Response; +class TileID; + +class OfflineDatabase : private util::noncopyable { +public: + OfflineDatabase(const std::string& path); + ~OfflineDatabase(); + + optional<Response> get(const Resource&); + void put(const Resource&, const Response&); + +private: + void ensureSchema(); + void removeExisting(); + mapbox::sqlite::Statement& getStatement(const char *); + + optional<Response> getTile(const Resource::TileData&); + void putTile(const Resource::TileData&, const Response&); + + optional<Response> getResource(const Resource&); + void putResource(const Resource&, const Response&); + + const std::string path; + std::unique_ptr<::mapbox::sqlite::Database> db; + std::unordered_map<const char *, std::unique_ptr<::mapbox::sqlite::Statement>> statements; +}; + +} // namespace mbgl + +#endif diff --git a/platform/default/mbgl/storage/offline_schema.cpp.include b/platform/default/mbgl/storage/offline_schema.cpp.include new file mode 100644 index 0000000000..31b11d5f12 --- /dev/null +++ b/platform/default/mbgl/storage/offline_schema.cpp.include @@ -0,0 +1,51 @@ +/* THIS IS A GENERATED FILE; EDIT offline_schema.sql INSTEAD */ +static const char * schema = +"CREATE TABLE resources (\n" +" url TEXT NOT NULL PRIMARY KEY,\n" +" kind INTEGER NOT NULL,\n" +" expires INTEGER,\n" +" modified INTEGER,\n" +" accessed INTEGER,\n" +" etag TEXT,\n" +" data BLOB,\n" +" compressed INTEGER NOT NULL DEFAULT 0\n" +");\n" +"CREATE TABLE tilesets (\n" +" id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,\n" +" url_template TEXT NOT NULL,\n" +" pixel_ratio INTEGER,\n" +" UNIQUE (url_template, pixel_ratio)\n" +");\n" +"CREATE TABLE tiles (\n" +" tileset_id INTEGER NOT NULL REFERENCES tilesets(id),\n" +" z INTEGER NOT NULL,\n" +" x INTEGER NOT NULL,\n" +" y INTEGER NOT NULL,\n" +" expires INTEGER,\n" +" modified INTEGER,\n" +" accessed INTEGER,\n" +" etag TEXT,\n" +" data BLOB,\n" +" compressed INTEGER NOT NULL DEFAULT 0,\n" +" PRIMARY KEY (tileset_id, z, x, y)\n" +");\n" +"CREATE TABLE regions (\n" +" id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,\n" +" definition TEXT NOT NULL,\n" +" description BLOB\n" +");\n" +"CREATE TABLE region_resources (\n" +" region_id INTEGER NOT NULL REFERENCES regions(id),\n" +" resource_url TEXT NOT NULL REFERENCES resources(url),\n" +" PRIMARY KEY (region_id, resource_url)\n" +");\n" +"CREATE TABLE region_tiles (\n" +" region_id INTEGER NOT NULL REFERENCES regions(id),\n" +" tileset_id INTEGER NOT NULL REFERENCES tilesets(id),\n" +" z INTEGER NOT NULL,\n" +" x INTEGER NOT NULL,\n" +" y INTEGER NOT NULL,\n" +" PRIMARY KEY (region_id, tileset_id, z, x, y),\n" +" FOREIGN KEY (tileset_id, z, x, y) REFERENCES tiles (tileset_id, z, x, y)\n" +");\n" +; diff --git a/platform/default/mbgl/storage/offline_schema.js b/platform/default/mbgl/storage/offline_schema.js new file mode 100644 index 0000000000..153ba34e38 --- /dev/null +++ b/platform/default/mbgl/storage/offline_schema.js @@ -0,0 +1,24 @@ +// To regenerate: +// (cd platform/default/mbgl/storage && node offline_schema.js) + +var fs = require('fs'); +var readline = require('readline'); + +var lineReader = readline.createInterface({ + input: fs.createReadStream('offline_schema.sql') +}); + +var lines = [ + "/* THIS IS A GENERATED FILE; EDIT offline_schema.sql INSTEAD */", + "static const char * schema = ", +]; + +lineReader + .on('line', function (line) { + line = line.replace(/ *--.*/, ''); + if (line) lines.push('"' + line + '\\n"'); + }) + .on('close', function () { + lines.push(';\n'); + fs.writeFileSync('offline_schema.cpp.include', lines.join('\n')); + }); diff --git a/platform/default/mbgl/storage/offline_schema.sql b/platform/default/mbgl/storage/offline_schema.sql new file mode 100644 index 0000000000..e55517c489 --- /dev/null +++ b/platform/default/mbgl/storage/offline_schema.sql @@ -0,0 +1,62 @@ +CREATE TABLE resources ( -- Generic table for style, source, sprite, and glyph resources. + url TEXT NOT NULL PRIMARY KEY, -- Same schema as http_cache table. + kind INTEGER NOT NULL, + expires INTEGER, + modified INTEGER, + accessed INTEGER, + etag TEXT, + data BLOB, -- NULL if the response was a 404 + compressed INTEGER NOT NULL DEFAULT 0 +); + +CREATE TABLE tilesets ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + url_template TEXT NOT NULL, -- As it would appear in TileJSON (but no support for host sharding). + pixel_ratio INTEGER, -- If NULL, 1 is assumed for raster sources. + UNIQUE (url_template, pixel_ratio) -- Capable of caching the same tileset at multiple resolutions. +); + +CREATE TABLE tiles ( + tileset_id INTEGER NOT NULL REFERENCES tilesets(id), + z INTEGER NOT NULL, -- Fully abandon TMS coordinates in favor of ZXY. + x INTEGER NOT NULL, + y INTEGER NOT NULL, + expires INTEGER, + modified INTEGER, + accessed INTEGER, + etag TEXT, + data BLOB, -- NULL if the response was a 404 + compressed INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (tileset_id, z, x, y) +); + +CREATE TABLE regions ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + definition TEXT NOT NULL, -- JSON formatted definition of region. Regions may be of variant types: + -- e.g. bbox and zoom range, route path, flyTo parameters, etc. Note that + -- the set of tiles required for a region may span multiple sources. + description BLOB -- User provided data in user-defined format +); + +CREATE TABLE region_resources ( + region_id INTEGER NOT NULL REFERENCES regions(id), + resource_url TEXT NOT NULL REFERENCES resources(url), + PRIMARY KEY (region_id, resource_url) +); + +CREATE TABLE region_tiles ( + region_id INTEGER NOT NULL REFERENCES regions(id), + tileset_id INTEGER NOT NULL REFERENCES tilesets(id), + z INTEGER NOT NULL, + x INTEGER NOT NULL, + y INTEGER NOT NULL, + PRIMARY KEY (region_id, tileset_id, z, x, y), + FOREIGN KEY (tileset_id, z, x, y) REFERENCES tiles (tileset_id, z, x, y) +); + +-- `region_resources` and `region_tiles` are used for deduplication and deletion logic. +-- A row in `tiles` exists IFF one or more corresponding rows exist in `region_tiles`. If +-- more than one corresponding row exists, it indicates multiple regions contain the tile, and +-- storage for the tile is being deduplicated. When a region is deleted, corresponding rows in +-- `region_tiles` must also be deleted, and then rows in `tiles` and `tilesets` without a +-- corresponding `region_tiles` row must be deleted. Similarly for `resources` / `region_resources`. |