diff options
Diffstat (limited to 'platform/default/mbgl')
| -rw-r--r-- | platform/default/mbgl/storage/offline.cpp | 122 | ||||
| -rw-r--r-- | platform/default/mbgl/storage/offline_database.cpp | 523 | ||||
| -rw-r--r-- | platform/default/mbgl/storage/offline_database.hpp | 94 | ||||
| -rw-r--r-- | platform/default/mbgl/storage/offline_download.cpp | 241 | ||||
| -rw-r--r-- | platform/default/mbgl/storage/offline_download.hpp | 62 | ||||
| -rw-r--r-- | platform/default/mbgl/storage/offline_schema.cpp.include | 53 | ||||
| -rw-r--r-- | platform/default/mbgl/storage/offline_schema.js | 24 | ||||
| -rw-r--r-- | platform/default/mbgl/storage/offline_schema.sql | 62 |
8 files changed, 1181 insertions, 0 deletions
diff --git a/platform/default/mbgl/storage/offline.cpp b/platform/default/mbgl/storage/offline.cpp new file mode 100644 index 0000000000..931e079771 --- /dev/null +++ b/platform/default/mbgl/storage/offline.cpp @@ -0,0 +1,122 @@ +#include <mbgl/storage/offline.hpp> +#include <mbgl/util/tile_cover.hpp> +#include <mbgl/source/source_info.hpp> + +#include <rapidjson/document.h> +#include <rapidjson/stringbuffer.h> +#include <rapidjson/writer.h> + +#include <cmath> + +namespace mbgl { + +OfflineTilePyramidRegionDefinition::OfflineTilePyramidRegionDefinition( + const std::string& styleURL_, const LatLngBounds& bounds_, double minZoom_, double maxZoom_, float pixelRatio_) + : styleURL(styleURL_), + bounds(bounds_), + minZoom(minZoom_), + maxZoom(maxZoom_), + pixelRatio(pixelRatio_) { + if (minZoom < 0 || maxZoom < 0 || maxZoom < minZoom || pixelRatio < 0 || + !std::isfinite(minZoom) || std::isnan(maxZoom) || !std::isfinite(pixelRatio)) { + throw std::invalid_argument("Invalid offline region definition"); + } +} + +std::vector<TileID> OfflineTilePyramidRegionDefinition::tileCover(SourceType type, uint16_t tileSize, const SourceInfo& info) const { + double minZ = std::max<double>(coveringZoomLevel(minZoom, type, tileSize), info.minZoom); + double maxZ = std::min<double>(coveringZoomLevel(maxZoom, type, tileSize), info.maxZoom); + + assert(minZ >= 0); + assert(maxZ >= 0); + assert(minZ < std::numeric_limits<uint8_t>::max()); + assert(maxZ < std::numeric_limits<uint8_t>::max()); + + std::vector<TileID> result; + + for (uint8_t z = minZ; z <= maxZ; z++) { + for (const auto& tile : mbgl::tileCover(bounds, z, z)) { + result.push_back(tile.normalized()); + } + } + + return result; +} + +OfflineRegionDefinition decodeOfflineRegionDefinition(const std::string& region) { + rapidjson::GenericDocument<rapidjson::UTF8<>, rapidjson::CrtAllocator> doc; + doc.Parse<0>(region.c_str()); + + if (doc.HasParseError() || + !doc.HasMember("style_url") || !doc["style_url"].IsString() || + !doc.HasMember("bounds") || !doc["bounds"].IsArray() || doc["bounds"].Size() != 4 || + !doc["bounds"][0].IsDouble() || !doc["bounds"][1].IsDouble() || + !doc["bounds"][2].IsDouble() || !doc["bounds"][3].IsDouble() || + !doc.HasMember("min_zoom") || !doc["min_zoom"].IsDouble() || + (doc.HasMember("max_zoom") && !doc["max_zoom"].IsDouble()) || + !doc.HasMember("pixel_ratio") || !doc["pixel_ratio"].IsDouble()) { + throw std::runtime_error("Malformed offline region definition"); + } + + std::string styleURL { doc["style_url"].GetString(), doc["style_url"].GetStringLength() }; + LatLngBounds bounds = LatLngBounds::hull( + LatLng(doc["bounds"][0].GetDouble(), doc["bounds"][1].GetDouble()), + LatLng(doc["bounds"][2].GetDouble(), doc["bounds"][3].GetDouble())); + double minZoom = doc["min_zoom"].GetDouble(); + double maxZoom = doc.HasMember("max_zoom") ? doc["max_zoom"].GetDouble() : INFINITY; + float pixelRatio = doc["pixel_ratio"].GetDouble(); + + return { styleURL, bounds, minZoom, maxZoom, pixelRatio }; +} + +std::string encodeOfflineRegionDefinition(const OfflineRegionDefinition& region) { + rapidjson::GenericDocument<rapidjson::UTF8<>, rapidjson::CrtAllocator> doc; + doc.SetObject(); + + doc.AddMember("style_url", rapidjson::StringRef(region.styleURL.data(), region.styleURL.length()), doc.GetAllocator()); + + rapidjson::GenericValue<rapidjson::UTF8<>, rapidjson::CrtAllocator> bounds(rapidjson::kArrayType); + bounds.PushBack(region.bounds.south(), doc.GetAllocator()); + bounds.PushBack(region.bounds.west(), doc.GetAllocator()); + bounds.PushBack(region.bounds.north(), doc.GetAllocator()); + bounds.PushBack(region.bounds.east(), doc.GetAllocator()); + doc.AddMember("bounds", bounds, doc.GetAllocator()); + + doc.AddMember("min_zoom", region.minZoom, doc.GetAllocator()); + if (std::isfinite(region.maxZoom)) { + doc.AddMember("max_zoom", region.maxZoom, doc.GetAllocator()); + } + + doc.AddMember("pixel_ratio", region.pixelRatio, doc.GetAllocator()); + + rapidjson::StringBuffer buffer; + rapidjson::Writer<rapidjson::StringBuffer> writer(buffer); + doc.Accept(writer); + + return buffer.GetString(); +} + +OfflineRegion::OfflineRegion(int64_t id_, + const OfflineRegionDefinition& definition_, + const OfflineRegionMetadata& metadata_) + : id(id_), + definition(definition_), + metadata(metadata_) { +} + +OfflineRegion::OfflineRegion(OfflineRegion&&) = default; +OfflineRegion::~OfflineRegion() = default; + +const OfflineRegionDefinition& OfflineRegion::getDefinition() const { + return definition; +} + +const OfflineRegionMetadata& OfflineRegion::getMetadata() const { + return metadata; +} + +int64_t OfflineRegion::getID() const { + return id; +} + +} // namespace mbgl diff --git a/platform/default/mbgl/storage/offline_database.cpp b/platform/default/mbgl/storage/offline_database.cpp new file mode 100644 index 0000000000..78f0272fef --- /dev/null +++ b/platform/default/mbgl/storage/offline_database.cpp @@ -0,0 +1,523 @@ +#include <mbgl/storage/offline_database.hpp> +#include <mbgl/storage/response.hpp> +#include <mbgl/util/compression.hpp> +#include <mbgl/util/io.hpp> +#include <mbgl/util/string.hpp> +#include <mbgl/util/chrono.hpp> +#include <mbgl/map/tile_id.hpp> +#include <mbgl/platform/log.hpp> + +#include "sqlite3.hpp" +#include <sqlite3.h> + +namespace mbgl { + +using namespace mapbox::sqlite; + +// If you change the schema you must write a migration from the previous version. +static const uint32_t schemaVersion = 2; + +OfflineDatabase::Statement::~Statement() { + stmt.reset(); + stmt.clearBindings(); +} + +OfflineDatabase::OfflineDatabase(const std::string& path_, uint64_t maximumCacheSize_) + : path(path_), + maximumCacheSize(maximumCacheSize_) { + ensureSchema(); +} + +OfflineDatabase::~OfflineDatabase() { + // Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we + // can't throw anything. + try { + statements.clear(); + db.reset(); + } catch (mapbox::sqlite::Exception& ex) { + Log::Error(Event::Database, ex.code, ex.what()); + } +} + +void OfflineDatabase::ensureSchema() { + if (path != ":memory:") { + try { + db = std::make_unique<Database>(path.c_str(), ReadWrite); + db->setBusyTimeout(Milliseconds::max()); + + { + auto userVersionStmt = db->prepare("PRAGMA user_version"); + userVersionStmt.run(); + switch (userVersionStmt.get<int>(0)) { + case 0: break; // cache-only database; ok to delete + case 1: break; // cache-only database; ok to delete + case 2: return; + default: throw std::runtime_error("unknown schema version"); + } + } + + removeExisting(); + db = std::make_unique<Database>(path.c_str(), ReadWrite | Create); + db->setBusyTimeout(Milliseconds::max()); + } catch (mapbox::sqlite::Exception& ex) { + if (ex.code == SQLITE_CANTOPEN) { + db = std::make_unique<Database>(path.c_str(), ReadWrite | Create); + db->setBusyTimeout(Milliseconds::max()); + } else if (ex.code == SQLITE_NOTADB) { + removeExisting(); + db = std::make_unique<Database>(path.c_str(), ReadWrite | Create); + db->setBusyTimeout(Milliseconds::max()); + } + } + } + + #include "offline_schema.cpp.include" + + db = std::make_unique<Database>(path.c_str(), ReadWrite | Create); + db->setBusyTimeout(Milliseconds::max()); + db->exec(schema); + db->exec("PRAGMA user_version = " + util::toString(schemaVersion)); +} + +void OfflineDatabase::removeExisting() { + Log::Warning(Event::Database, "Removing existing incompatible offline database"); + + db.reset(); + + try { + util::deleteFile(path); + } catch (util::IOException& ex) { + Log::Error(Event::Database, ex.code, ex.what()); + } +} + +OfflineDatabase::Statement OfflineDatabase::getStatement(const char * sql) { + auto it = statements.find(sql); + + if (it != statements.end()) { + return Statement(*it->second); + } + + return Statement(*statements.emplace(sql, std::make_unique<mapbox::sqlite::Statement>(db->prepare(sql))).first->second); +} + +optional<Response> OfflineDatabase::get(const Resource& resource) { + if (resource.kind == Resource::Kind::Tile) { + assert(resource.tileData); + return getTile(*resource.tileData); + } else { + return getResource(resource); + } +} + +uint64_t OfflineDatabase::put(const Resource& resource, const Response& response) { + return putInternal(resource, response, true); +} + +uint64_t OfflineDatabase::putInternal(const Resource& resource, const Response& response, bool evict_) { + if (response.error) { + return 0; + } + + std::string compressedData; + bool compressed = false; + uint64_t size = 0; + + if (response.data) { + compressedData = util::compress(*response.data); + compressed = compressedData.size() < response.data->size(); + size = compressed ? compressedData.size() : response.data->size(); + } + + if (evict_ && !evict(size)) { + Log::Warning(Event::Database, "Unable to make space for entry"); + return 0; + } + + if (resource.kind == Resource::Kind::Tile) { + assert(resource.tileData); + putTile(*resource.tileData, response, + compressed ? compressedData : *response.data, + compressed); + } else { + putResource(resource, response, + compressed ? compressedData : *response.data, + compressed); + } + + return size; +} + +optional<Response> OfflineDatabase::getResource(const Resource& resource) { + Statement accessedStmt = getStatement( + "UPDATE resources SET accessed = ?1 WHERE url = ?2"); + + accessedStmt->bind(1, SystemClock::now()); + accessedStmt->bind(2, resource.url); + accessedStmt->run(); + + Statement stmt = getStatement( + // 0 1 2 3 4 + "SELECT etag, expires, modified, data, compressed " + "FROM resources " + "WHERE url = ?"); + + stmt->bind(1, resource.url); + + if (!stmt->run()) { + return {}; + } + + Response response; + + response.etag = stmt->get<optional<std::string>>(0); + response.expires = stmt->get<optional<SystemTimePoint>>(1); + response.modified = stmt->get<optional<SystemTimePoint>>(2); + + optional<std::string> data = stmt->get<optional<std::string>>(3); + if (!data) { + response.noContent = true; + } else if (stmt->get<int>(4)) { + response.data = std::make_shared<std::string>(util::decompress(*data)); + } else { + response.data = std::make_shared<std::string>(*data); + } + + return response; +} + +void OfflineDatabase::putResource(const Resource& resource, + const Response& response, + const std::string& data, + bool compressed) { + if (response.notModified) { + Statement stmt = getStatement( + "UPDATE resources " + "SET accessed = ?1, " + " expires = ?2 " + "WHERE url = ?3 "); + + stmt->bind(1, SystemClock::now()); + stmt->bind(2, response.expires); + stmt->bind(3, resource.url); + stmt->run(); + } else { + Statement stmt = getStatement( + "REPLACE INTO resources (url, kind, etag, expires, modified, accessed, data, compressed) " + "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) "); + + stmt->bind(1, resource.url); + stmt->bind(2, int(resource.kind)); + stmt->bind(3, response.etag); + stmt->bind(4, response.expires); + stmt->bind(5, response.modified); + stmt->bind(6, SystemClock::now()); + + if (response.noContent) { + stmt->bind(7, nullptr); + stmt->bind(8, false); + } else { + stmt->bindBlob(7, data.data(), data.size(), false); + stmt->bind(8, compressed); + } + + stmt->run(); + } +} + +optional<Response> OfflineDatabase::getTile(const Resource::TileData& tile) { + Statement accessedStmt = getStatement( + "UPDATE tiles " + "SET accessed = ?1 " + "WHERE url_template = ?2 " + " AND pixel_ratio = ?3 " + " AND x = ?4 " + " AND y = ?5 " + " AND z = ?6 "); + + accessedStmt->bind(1, SystemClock::now()); + accessedStmt->bind(2, tile.urlTemplate); + accessedStmt->bind(3, tile.pixelRatio); + accessedStmt->bind(4, tile.x); + accessedStmt->bind(5, tile.y); + accessedStmt->bind(6, tile.z); + accessedStmt->run(); + + Statement stmt = getStatement( + // 0 1 2 3 4 + "SELECT etag, expires, modified, data, compressed " + "FROM tiles " + "WHERE url_template = ?1 " + " AND pixel_ratio = ?2 " + " AND x = ?3 " + " AND y = ?4 " + " AND z = ?5 "); + + stmt->bind(1, tile.urlTemplate); + stmt->bind(2, tile.pixelRatio); + stmt->bind(3, tile.x); + stmt->bind(4, tile.y); + stmt->bind(5, tile.z); + + if (!stmt->run()) { + return {}; + } + + Response response; + + response.etag = stmt->get<optional<std::string>>(0); + response.expires = stmt->get<optional<SystemTimePoint>>(1); + response.modified = stmt->get<optional<SystemTimePoint>>(2); + + optional<std::string> data = stmt->get<optional<std::string>>(3); + if (!data) { + response.noContent = true; + } else if (stmt->get<int>(4)) { + response.data = std::make_shared<std::string>(util::decompress(*data)); + } else { + response.data = std::make_shared<std::string>(*data); + } + + return response; +} + +void OfflineDatabase::putTile(const Resource::TileData& tile, + const Response& response, + const std::string& data, + bool compressed) { + if (response.notModified) { + Statement stmt = getStatement( + "UPDATE tiles " + "SET accessed = ?1, " + " expires = ?2 " + "WHERE url_template = ?3 " + " AND pixel_ratio = ?4 " + " AND x = ?5 " + " AND y = ?6 " + " AND z = ?7 "); + + stmt->bind(1, SystemClock::now()); + stmt->bind(2, response.expires); + stmt->bind(3, tile.urlTemplate); + stmt->bind(4, tile.pixelRatio); + stmt->bind(5, tile.x); + stmt->bind(6, tile.y); + stmt->bind(7, tile.z); + stmt->run(); + } else { + Statement stmt2 = getStatement( + "REPLACE INTO tiles (url_template, pixel_ratio, x, y, z, modified, etag, expires, accessed, data, compressed) " + "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11) "); + + stmt2->bind(1, tile.urlTemplate); + stmt2->bind(2, tile.pixelRatio); + stmt2->bind(3, tile.x); + stmt2->bind(4, tile.y); + stmt2->bind(5, tile.z); + stmt2->bind(6, response.modified); + stmt2->bind(7, response.etag); + stmt2->bind(8, response.expires); + stmt2->bind(9, SystemClock::now()); + + if (response.noContent) { + stmt2->bind(10, nullptr); + stmt2->bind(11, false); + } else { + stmt2->bindBlob(10, data.data(), data.size(), false); + stmt2->bind(11, compressed); + } + + stmt2->run(); + } +} + +std::vector<OfflineRegion> OfflineDatabase::listRegions() { + Statement stmt = getStatement( + "SELECT id, definition, description FROM regions"); + + std::vector<OfflineRegion> result; + + while (stmt->run()) { + result.push_back(OfflineRegion( + stmt->get<int64_t>(0), + decodeOfflineRegionDefinition(stmt->get<std::string>(1)), + stmt->get<std::vector<uint8_t>>(2))); + } + + return result; +} + +OfflineRegion OfflineDatabase::createRegion(const OfflineRegionDefinition& definition, + const OfflineRegionMetadata& metadata) { + Statement stmt = getStatement( + "INSERT INTO regions (definition, description) " + "VALUES (?1, ?2) "); + + stmt->bind(1, encodeOfflineRegionDefinition(definition)); + stmt->bindBlob(2, metadata); + stmt->run(); + + return OfflineRegion(db->lastInsertRowid(), definition, metadata); +} + +void OfflineDatabase::deleteRegion(OfflineRegion&& region) { + Statement stmt = getStatement( + "DELETE FROM regions WHERE id = ?"); + + stmt->bind(1, region.getID()); + stmt->run(); + + evict(0); +} + +optional<Response> OfflineDatabase::getRegionResource(int64_t regionID, const Resource& resource) { + auto response = get(resource); + + if (response) { + markUsed(regionID, resource); + } + + return response; +} + +uint64_t OfflineDatabase::putRegionResource(int64_t regionID, const Resource& resource, const Response& response) { + uint64_t result = putInternal(resource, response, false); + markUsed(regionID, resource); + return result; +} + +void OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) { + if (resource.kind == Resource::Kind::Tile) { + Statement stmt = getStatement( + "REPLACE INTO region_tiles (region_id, tile_id) " + "SELECT ?1, tiles.id " + "FROM tiles " + "WHERE url_template = ?2 " + " AND pixel_ratio = ?3 " + " AND x = ?4 " + " AND y = ?5 " + " AND z = ?6 "); + + const Resource::TileData& tile = *resource.tileData; + stmt->bind(1, regionID); + stmt->bind(2, tile.urlTemplate); + stmt->bind(3, tile.pixelRatio); + stmt->bind(4, tile.x); + stmt->bind(5, tile.y); + stmt->bind(6, tile.z); + stmt->run(); + } else { + Statement stmt = getStatement( + "REPLACE INTO region_resources (region_id, resource_id) " + "SELECT ?1, resources.id " + "FROM resources " + "WHERE resources.url = ?2 "); + + stmt->bind(1, regionID); + stmt->bind(2, resource.url); + stmt->run(); + } +} + +OfflineRegionDefinition OfflineDatabase::getRegionDefinition(int64_t regionID) { + Statement stmt = getStatement( + "SELECT definition FROM regions WHERE id = ?1"); + + stmt->bind(1, regionID); + stmt->run(); + + return decodeOfflineRegionDefinition(stmt->get<std::string>(0)); +} + +OfflineRegionStatus OfflineDatabase::getRegionCompletedStatus(int64_t regionID) { + OfflineRegionStatus result; + + Statement stmt = getStatement( + "SELECT COUNT(*), SUM(size) FROM ( " + " SELECT LENGTH(data) as size " + " FROM region_resources, resources " + " WHERE region_id = ?1 " + " AND resource_id = resources.id " + " UNION ALL " + " SELECT LENGTH(data) as size " + " FROM region_tiles, tiles " + " WHERE region_id = ?1 " + " AND tile_id = tiles.id " + ") "); + + stmt->bind(1, regionID); + stmt->run(); + + result.completedResourceCount = stmt->get<int64_t>(0); + result.completedResourceSize = stmt->get<int64_t>(1); + + return result; +} + +template <class T> +T OfflineDatabase::getPragma(const char * sql) { + Statement stmt = getStatement(sql); + stmt->run(); + return stmt->get<T>(0); +} + +// Remove least-recently used resources and tiles until the used database size, +// as calculated by multiplying the number of in-use pages by the page size, is +// less than the maximum cache size. Returns false if this condition cannot be +// satisfied. +// +// SQLite database never shrinks in size unless we call VACCUM. We here +// are monitoring the soft limit (i.e. number of free pages in the file) +// and as it approaches to the hard limit (i.e. the actual file size) we +// delete an arbitrary number of old cache entries. The free pages approach saves +// us from calling VACCUM or keeping a running total, which can be costly. +bool OfflineDatabase::evict(uint64_t neededFreeSize) { + uint64_t pageSize = getPragma<int64_t>("PRAGMA page_size"); + uint64_t pageCount = getPragma<int64_t>("PRAGMA page_count"); + + if (pageSize * pageCount > maximumCacheSize) { + Log::Warning(mbgl::Event::Database, "Current size is larger than the maximum size. Database won't get truncated."); + } + + auto usedSize = [&] { + return pageSize * (pageCount - getPragma<int64_t>("PRAGMA freelist_count")); + }; + + // The addition of pageSize is a fudge factor to account for non `data` column + // size, and because pages can get fragmented on the database. + while (usedSize() + neededFreeSize + pageSize > maximumCacheSize) { + Statement stmt1 = getStatement( + "DELETE FROM resources " + "WHERE id IN ( " + " SELECT id FROM resources " + " LEFT JOIN region_resources " + " ON resource_id = resources.id " + " WHERE resource_id IS NULL " + " ORDER BY accessed ASC LIMIT ?1 " + ") "); + stmt1->bind(1, 50); + stmt1->run(); + uint64_t changes1 = db->changes(); + + Statement stmt2 = getStatement( + "DELETE FROM tiles " + "WHERE id IN ( " + " SELECT id FROM tiles " + " LEFT JOIN region_tiles " + " ON tile_id = tiles.id " + " WHERE tile_id IS NULL " + " ORDER BY accessed ASC LIMIT ?1 " + ") "); + stmt2->bind(1, 50); + stmt2->run(); + uint64_t changes2 = db->changes(); + + if (changes1 == 0 && changes2 == 0) { + return false; + } + } + + return true; +} + +} // namespace mbgl diff --git a/platform/default/mbgl/storage/offline_database.hpp b/platform/default/mbgl/storage/offline_database.hpp new file mode 100644 index 0000000000..854ebdb47d --- /dev/null +++ b/platform/default/mbgl/storage/offline_database.hpp @@ -0,0 +1,94 @@ +#ifndef MBGL_OFFLINE_DATABASE +#define MBGL_OFFLINE_DATABASE + +#include <mbgl/storage/resource.hpp> +#include <mbgl/storage/offline.hpp> +#include <mbgl/util/noncopyable.hpp> +#include <mbgl/util/optional.hpp> +#include <mbgl/util/constants.hpp> + +#include <unordered_map> +#include <memory> +#include <string> + +namespace mapbox { +namespace sqlite { +class Database; +class Statement; +} +} + +namespace mbgl { + +class Response; +class TileID; + +class OfflineDatabase : private util::noncopyable { +public: + // Limits affect ambient caching (put) only; resources required by offline + // regions are exempt. + OfflineDatabase(const std::string& path, + uint64_t maximumCacheSize = util::DEFAULT_MAX_CACHE_SIZE); + ~OfflineDatabase(); + + optional<Response> get(const Resource&); + uint64_t put(const Resource&, const Response&); + + std::vector<OfflineRegion> listRegions(); + + OfflineRegion createRegion(const OfflineRegionDefinition&, + const OfflineRegionMetadata&); + + void deleteRegion(OfflineRegion&&); + + optional<Response> getRegionResource(int64_t regionID, const Resource&); + uint64_t putRegionResource(int64_t regionID, const Resource&, const Response&); + + OfflineRegionDefinition getRegionDefinition(int64_t regionID); + OfflineRegionStatus getRegionCompletedStatus(int64_t regionID); + +private: + void ensureSchema(); + void removeExisting(); + + class Statement { + public: + explicit Statement(mapbox::sqlite::Statement& stmt_) : stmt(stmt_) {} + Statement(Statement&&) = default; + Statement(const Statement&) = delete; + ~Statement(); + + mapbox::sqlite::Statement* operator->() { return &stmt; }; + + private: + mapbox::sqlite::Statement& stmt; + }; + + Statement getStatement(const char *); + + optional<Response> getTile(const Resource::TileData&); + void putTile(const Resource::TileData&, const Response&, + const std::string&, bool compressed); + + optional<Response> getResource(const Resource&); + void putResource(const Resource&, const Response&, + const std::string&, bool compressed); + + uint64_t putInternal(const Resource&, const Response&, bool evict); + void markUsed(int64_t regionID, const Resource&); + + const std::string path; + std::unique_ptr<::mapbox::sqlite::Database> db; + std::unordered_map<const char *, std::unique_ptr<::mapbox::sqlite::Statement>> statements; + + template <class T> + T getPragma(const char *); + + uint64_t maximumCacheSize; + + bool evict(uint64_t neededFreeSize); +}; + +} // namespace mbgl + +#endif diff --git a/platform/default/mbgl/storage/offline_download.cpp b/platform/default/mbgl/storage/offline_download.cpp new file mode 100644 index 0000000000..27bf3b8c5b --- /dev/null +++ b/platform/default/mbgl/storage/offline_download.cpp @@ -0,0 +1,241 @@ +#include <mbgl/storage/offline_download.hpp> +#include <mbgl/storage/offline_database.hpp> +#include <mbgl/storage/file_source.hpp> +#include <mbgl/storage/resource.hpp> +#include <mbgl/storage/response.hpp> +#include <mbgl/style/style_parser.hpp> +#include <mbgl/layer/symbol_layer.hpp> +#include <mbgl/text/glyph.hpp> +#include <mbgl/util/tile_cover.hpp> + +#include <set> + +namespace mbgl { + +OfflineDownload::OfflineDownload(int64_t id_, + OfflineRegionDefinition&& definition_, + OfflineDatabase& offlineDatabase_, + FileSource& onlineFileSource_) + : id(id_), + definition(definition_), + offlineDatabase(offlineDatabase_), + onlineFileSource(onlineFileSource_) { + setObserver(nullptr); +} + +OfflineDownload::~OfflineDownload() = default; + +void OfflineDownload::setObserver(std::unique_ptr<OfflineRegionObserver> observer_) { + observer = observer_ ? std::move(observer_) : std::make_unique<OfflineRegionObserver>(); +} + +void OfflineDownload::setState(OfflineRegionDownloadState state) { + if (status.downloadState == state) { + return; + } + + status.downloadState = state; + + if (status.downloadState == OfflineRegionDownloadState::Active) { + activateDownload(); + } else { + deactivateDownload(); + } +} + +std::vector<Resource> OfflineDownload::spriteResources(const StyleParser& parser) const { + std::vector<Resource> result; + + if (!parser.spriteURL.empty()) { + result.push_back(Resource::spriteImage(parser.spriteURL, definition.pixelRatio)); + result.push_back(Resource::spriteJSON(parser.spriteURL, definition.pixelRatio)); + } + + return result; +} + +std::vector<Resource> OfflineDownload::glyphResources(const StyleParser& parser) const { + std::vector<Resource> result; + + if (!parser.glyphURL.empty()) { + for (const auto& fontStack : parser.fontStacks()) { + for (uint32_t i = 0; i < 256; i++) { + result.push_back(Resource::glyphs(parser.glyphURL, fontStack, getGlyphRange(i * 256))); + } + } + } + + return result; +} + +std::vector<Resource> OfflineDownload::tileResources(SourceType type, uint16_t tileSize, const SourceInfo& info) const { + std::vector<Resource> result; + + for (const auto& tile : definition.tileCover(type, tileSize, info)) { + result.push_back(Resource::tile(info.tiles[0], definition.pixelRatio, tile.x, tile.y, tile.z)); + } + + return result; +} + +OfflineRegionStatus OfflineDownload::getStatus() const { + if (status.downloadState == OfflineRegionDownloadState::Active) { + return status; + } + + OfflineRegionStatus result = offlineDatabase.getRegionCompletedStatus(id); + + result.requiredResourceCount++; + optional<Response> styleResponse = offlineDatabase.get(Resource::style(definition.styleURL)); + if (!styleResponse) { + return result; + } + + StyleParser parser; + parser.parse(*styleResponse->data); + + result.requiredResourceCountIsIndeterminate = false; + + for (const auto& source : parser.sources) { + switch (source->type) { + case SourceType::Vector: + case SourceType::Raster: + if (source->getInfo()) { + result.requiredResourceCount += tileResources(source->type, source->tileSize, *source->getInfo()).size(); + } else { + result.requiredResourceCount += 1; + optional<Response> sourceResponse = offlineDatabase.get(Resource::source(source->url)); + if (sourceResponse) { + result.requiredResourceCount += tileResources(source->type, source->tileSize, + *StyleParser::parseTileJSON(*sourceResponse->data, source->url, source->type, source->tileSize)).size(); + } else { + result.requiredResourceCountIsIndeterminate = true; + } + } + break; + + case SourceType::GeoJSON: + if (!source->url.empty()) { + result.requiredResourceCount += 1; + } + break; + + case SourceType::Video: + case SourceType::Annotations: + break; + } + } + + result.requiredResourceCount += spriteResources(parser).size(); + result.requiredResourceCount += glyphResources(parser).size(); + + return result; +} + +void OfflineDownload::activateDownload() { + status = offlineDatabase.getRegionCompletedStatus(id); + requiredSourceURLs.clear(); + + ensureResource(Resource::style(definition.styleURL), [&] (Response styleResponse) { + status.requiredResourceCountIsIndeterminate = false; + + StyleParser parser; + parser.parse(*styleResponse.data); + + for (const auto& source : parser.sources) { + SourceType type = source->type; + uint16_t tileSize = source->tileSize; + std::string url = source->url; + + switch (type) { + case SourceType::Vector: + case SourceType::Raster: + if (source->getInfo()) { + ensureTiles(type, tileSize, *source->getInfo()); + } else { + status.requiredResourceCountIsIndeterminate = true; + requiredSourceURLs.insert(url); + + ensureResource(Resource::source(url), [=] (Response sourceResponse) { + ensureTiles(type, tileSize, *StyleParser::parseTileJSON(*sourceResponse.data, url, type, tileSize)); + + requiredSourceURLs.erase(url); + if (requiredSourceURLs.empty()) { + status.requiredResourceCountIsIndeterminate = false; + } + }); + } + break; + + case SourceType::GeoJSON: + if (!source->url.empty()) { + ensureResource(Resource::source(source->url)); + } + break; + + case SourceType::Video: + case SourceType::Annotations: + break; + } + } + + for (const auto& resource : spriteResources(parser)) { + ensureResource(resource); + } + + for (const auto& resource : glyphResources(parser)) { + ensureResource(resource); + } + }); + + // This will be the initial notification, after we've incremented requiredResourceCount + // to the reflect the extent to which required resources are already in the database. + observer->statusChanged(status); +} + +void OfflineDownload::deactivateDownload() { + requests.clear(); +} + +void OfflineDownload::ensureTiles(SourceType type, uint16_t tileSize, const SourceInfo& info) { + for (const auto& resource : tileResources(type, tileSize, info)) { + ensureResource(resource); + } +} + +void OfflineDownload::ensureResource(const Resource& resource, std::function<void (Response)> callback) { + status.requiredResourceCount++; + + optional<Response> offlineResponse = offlineDatabase.getRegionResource(id, resource); + if (offlineResponse) { + if (callback) { + callback(*offlineResponse); + } + + // Not incrementing status.completedResource{Size,Count} here because previously-existing + // resources are already accounted for by offlineDatabase.getRegionCompletedStatus(); + + return; + } + + auto it = requests.insert(requests.begin(), nullptr); + *it = onlineFileSource.request(resource, [=] (Response onlineResponse) { + if (onlineResponse.error) { + observer->responseError(*onlineResponse.error); + return; + } + + requests.erase(it); + + if (callback) { + callback(onlineResponse); + } + + status.completedResourceCount++; + status.completedResourceSize += offlineDatabase.putRegionResource(id, resource, onlineResponse); + + observer->statusChanged(status); + }); +} + +} // namespace mbgl diff --git a/platform/default/mbgl/storage/offline_download.hpp b/platform/default/mbgl/storage/offline_download.hpp new file mode 100644 index 0000000000..4200020487 --- /dev/null +++ b/platform/default/mbgl/storage/offline_download.hpp @@ -0,0 +1,62 @@ + #pragma once + +#include <mbgl/storage/offline.hpp> +#include <mbgl/style/types.hpp> + +#include <list> +#include <set> +#include <memory> + +namespace mbgl { + +class OfflineDatabase; +class FileSource; +class FileRequest; +class Resource; +class Response; +class SourceInfo; +class StyleParser; +class Source; + +/** + * Coordinates the request and storage of all resources for an offline region. + + * @private + */ +class OfflineDownload { +public: + OfflineDownload(int64_t id, OfflineRegionDefinition&&, OfflineDatabase& offline, FileSource& online); + ~OfflineDownload(); + + void setObserver(std::unique_ptr<OfflineRegionObserver>); + void setState(OfflineRegionDownloadState); + + OfflineRegionStatus getStatus() const; + +private: + void activateDownload(); + void deactivateDownload(); + + std::vector<Resource> spriteResources(const StyleParser&) const; + std::vector<Resource> glyphResources(const StyleParser&) const; + std::vector<Resource> tileResources(SourceType, uint16_t, const SourceInfo&) const; + + /* + * Ensure that the resource is stored in the database, requesting it if necessary. + * While the request is in progress, it is recorded in `requests`. If the download + * is deactivated, all in progress requests are cancelled. + */ + void ensureResource(const Resource&, std::function<void (Response)> = {}); + void ensureTiles(SourceType, uint16_t, const SourceInfo&); + + int64_t id; + OfflineRegionDefinition definition; + OfflineDatabase& offlineDatabase; + FileSource& onlineFileSource; + OfflineRegionStatus status; + std::unique_ptr<OfflineRegionObserver> observer; + std::list<std::unique_ptr<FileRequest>> requests; + std::set<std::string> requiredSourceURLs; +}; + +} // namespace mbgl diff --git a/platform/default/mbgl/storage/offline_schema.cpp.include b/platform/default/mbgl/storage/offline_schema.cpp.include new file mode 100644 index 0000000000..3068dadfe0 --- /dev/null +++ b/platform/default/mbgl/storage/offline_schema.cpp.include @@ -0,0 +1,53 @@ +/* THIS IS A GENERATED FILE; EDIT offline_schema.sql INSTEAD */ +static const char * schema = +"CREATE TABLE resources (\n" +" id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,\n" +" url TEXT NOT NULL,\n" +" kind INTEGER NOT NULL,\n" +" expires INTEGER,\n" +" modified INTEGER,\n" +" etag TEXT,\n" +" data BLOB,\n" +" compressed INTEGER NOT NULL DEFAULT 0,\n" +" accessed INTEGER NOT NULL,\n" +" UNIQUE (url)\n" +");\n" +"CREATE TABLE tiles (\n" +" id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,\n" +" url_template TEXT NOT NULL,\n" +" pixel_ratio INTEGER NOT NULL,\n" +" z INTEGER NOT NULL,\n" +" x INTEGER NOT NULL,\n" +" y INTEGER NOT NULL,\n" +" expires INTEGER,\n" +" modified INTEGER,\n" +" etag TEXT,\n" +" data BLOB,\n" +" compressed INTEGER NOT NULL DEFAULT 0,\n" +" accessed INTEGER NOT NULL,\n" +" UNIQUE (url_template, pixel_ratio, z, x, y)\n" +");\n" +"CREATE TABLE regions (\n" +" id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,\n" +" definition TEXT NOT NULL,\n" +" description BLOB\n" +");\n" +"CREATE TABLE region_resources (\n" +" region_id INTEGER NOT NULL REFERENCES regions(id),\n" +" resource_id INTEGER NOT NULL REFERENCES resources(id),\n" +" UNIQUE (region_id, resource_id)\n" +");\n" +"CREATE TABLE region_tiles (\n" +" region_id INTEGER NOT NULL REFERENCES regions(id),\n" +" tile_id INTEGER NOT NULL REFERENCES tiles(id),\n" +" UNIQUE (region_id, tile_id)\n" +");\n" +"CREATE INDEX resources_accessed\n" +"ON resources (accessed);\n" +"CREATE INDEX tiles_accessed\n" +"ON tiles (accessed);\n" +"CREATE INDEX region_resources_resource_id\n" +"ON region_resources (resource_id);\n" +"CREATE INDEX region_tiles_tile_id\n" +"ON region_tiles (tile_id);\n" +; diff --git a/platform/default/mbgl/storage/offline_schema.js b/platform/default/mbgl/storage/offline_schema.js new file mode 100644 index 0000000000..153ba34e38 --- /dev/null +++ b/platform/default/mbgl/storage/offline_schema.js @@ -0,0 +1,24 @@ +// To regenerate: +// (cd platform/default/mbgl/storage && node offline_schema.js) + +var fs = require('fs'); +var readline = require('readline'); + +var lineReader = readline.createInterface({ + input: fs.createReadStream('offline_schema.sql') +}); + +var lines = [ + "/* THIS IS A GENERATED FILE; EDIT offline_schema.sql INSTEAD */", + "static const char * schema = ", +]; + +lineReader + .on('line', function (line) { + line = line.replace(/ *--.*/, ''); + if (line) lines.push('"' + line + '\\n"'); + }) + .on('close', function () { + lines.push(';\n'); + fs.writeFileSync('offline_schema.cpp.include', lines.join('\n')); + }); diff --git a/platform/default/mbgl/storage/offline_schema.sql b/platform/default/mbgl/storage/offline_schema.sql new file mode 100644 index 0000000000..cba922f3f7 --- /dev/null +++ b/platform/default/mbgl/storage/offline_schema.sql @@ -0,0 +1,62 @@ +CREATE TABLE resources ( -- Generic table for style, source, sprite, and glyph resources. + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + url TEXT NOT NULL, + kind INTEGER NOT NULL, + expires INTEGER, + modified INTEGER, + etag TEXT, + data BLOB, + compressed INTEGER NOT NULL DEFAULT 0, + accessed INTEGER NOT NULL, + UNIQUE (url) +); + +CREATE TABLE tiles ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + url_template TEXT NOT NULL, + pixel_ratio INTEGER NOT NULL, + z INTEGER NOT NULL, + x INTEGER NOT NULL, + y INTEGER NOT NULL, + expires INTEGER, + modified INTEGER, + etag TEXT, + data BLOB, + compressed INTEGER NOT NULL DEFAULT 0, + accessed INTEGER NOT NULL, + UNIQUE (url_template, pixel_ratio, z, x, y) +); + +CREATE TABLE regions ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + definition TEXT NOT NULL, -- JSON formatted definition of region. Regions may be of variant types: + -- e.g. bbox and zoom range, route path, flyTo parameters, etc. Note that + -- the set of tiles required for a region may span multiple sources. + description BLOB -- User provided data in user-defined format +); + +CREATE TABLE region_resources ( + region_id INTEGER NOT NULL REFERENCES regions(id), + resource_id INTEGER NOT NULL REFERENCES resources(id), + UNIQUE (region_id, resource_id) +); + +CREATE TABLE region_tiles ( + region_id INTEGER NOT NULL REFERENCES regions(id), + tile_id INTEGER NOT NULL REFERENCES tiles(id), + UNIQUE (region_id, tile_id) +); + +-- Indexes for efficient eviction queries + +CREATE INDEX resources_accessed +ON resources (accessed); + +CREATE INDEX tiles_accessed +ON tiles (accessed); + +CREATE INDEX region_resources_resource_id +ON region_resources (resource_id); + +CREATE INDEX region_tiles_tile_id +ON region_tiles (tile_id); |