diff options
Diffstat (limited to 'platform/default/mbgl')
| -rw-r--r-- | platform/default/mbgl/storage/offline.cpp | 133 | ||||
| -rw-r--r-- | platform/default/mbgl/storage/offline_database.cpp | 325 | ||||
| -rw-r--r-- | platform/default/mbgl/storage/offline_database.hpp | 30 | ||||
| -rw-r--r-- | platform/default/mbgl/storage/offline_download.cpp | 116 |
4 files changed, 399 insertions, 205 deletions
diff --git a/platform/default/mbgl/storage/offline.cpp b/platform/default/mbgl/storage/offline.cpp index 598a0b182b..e1ec0acb31 100644 --- a/platform/default/mbgl/storage/offline.cpp +++ b/platform/default/mbgl/storage/offline.cpp @@ -1,8 +1,10 @@ #include <mbgl/storage/offline.hpp> -#include <mbgl/util/tile_cover.hpp> #include <mbgl/util/tileset.hpp> #include <mbgl/util/projection.hpp> +#include <mapbox/geojson.hpp> +#include <mapbox/geojson/rapidjson.hpp> + #include <rapidjson/document.h> #include <rapidjson/stringbuffer.h> #include <rapidjson/writer.h> @@ -11,6 +13,8 @@ namespace mbgl { +// OfflineTilePyramidRegionDefinition + OfflineTilePyramidRegionDefinition::OfflineTilePyramidRegionDefinition( std::string styleURL_, LatLngBounds bounds_, double minZoom_, double maxZoom_, float pixelRatio_) : styleURL(std::move(styleURL_)), @@ -24,87 +28,100 @@ OfflineTilePyramidRegionDefinition::OfflineTilePyramidRegionDefinition( } } -std::vector<CanonicalTileID> OfflineTilePyramidRegionDefinition::tileCover(style::SourceType type, uint16_t tileSize, const Range<uint8_t>& zoomRange) const { - const Range<uint8_t> clampedZoomRange = coveringZoomRange(type, tileSize, zoomRange); - std::vector<CanonicalTileID> result; - - for (uint8_t z = clampedZoomRange.min; z <= clampedZoomRange.max; z++) { - for (const auto& tile : util::tileCover(bounds, z)) { - result.emplace_back(tile.canonical); - } - } - - return result; -} +// OfflineGeometryRegionDefinition -uint64_t OfflineTilePyramidRegionDefinition::tileCount(style::SourceType type, uint16_t tileSize, const Range<uint8_t>& zoomRange) const { - - const Range<uint8_t> clampedZoomRange = coveringZoomRange(type, tileSize, zoomRange); - unsigned long result = 0;; - for (uint8_t z = clampedZoomRange.min; z <= clampedZoomRange.max; z++) { - result += util::tileCount(bounds, z); +OfflineGeometryRegionDefinition::OfflineGeometryRegionDefinition(std::string styleURL_, Geometry<double> geometry_, double minZoom_, double maxZoom_, float pixelRatio_) + : styleURL(styleURL_) + , geometry(std::move(geometry_)) + , minZoom(minZoom_) + , maxZoom(maxZoom_) + , pixelRatio(pixelRatio_) { + if (minZoom < 0 || maxZoom < 0 || maxZoom < minZoom || pixelRatio < 0 || + !std::isfinite(minZoom) || std::isnan(maxZoom) || !std::isfinite(pixelRatio)) { + throw std::invalid_argument("Invalid offline region definition"); } - - return result; -} - -Range<uint8_t> OfflineTilePyramidRegionDefinition::coveringZoomRange(style::SourceType type, uint16_t tileSize, const Range<uint8_t>& zoomRange) const { - double minZ = std::max<double>(util::coveringZoomLevel(minZoom, type, tileSize), zoomRange.min); - double maxZ = std::min<double>(util::coveringZoomLevel(maxZoom, type, tileSize), zoomRange.max); - - assert(minZ >= 0); - assert(maxZ >= 0); - assert(minZ < std::numeric_limits<uint8_t>::max()); - assert(maxZ < std::numeric_limits<uint8_t>::max()); - return { static_cast<uint8_t>(minZ), static_cast<uint8_t>(maxZ) }; } OfflineRegionDefinition decodeOfflineRegionDefinition(const std::string& region) { rapidjson::GenericDocument<rapidjson::UTF8<>, rapidjson::CrtAllocator> doc; doc.Parse<0>(region.c_str()); - if (doc.HasParseError() || - !doc.HasMember("style_url") || !doc["style_url"].IsString() || - !doc.HasMember("bounds") || !doc["bounds"].IsArray() || doc["bounds"].Size() != 4 || - !doc["bounds"][0].IsDouble() || !doc["bounds"][1].IsDouble() || - !doc["bounds"][2].IsDouble() || !doc["bounds"][3].IsDouble() || - !doc.HasMember("min_zoom") || !doc["min_zoom"].IsDouble() || - (doc.HasMember("max_zoom") && !doc["max_zoom"].IsDouble()) || - !doc.HasMember("pixel_ratio") || !doc["pixel_ratio"].IsDouble()) { + // validation + + auto hasValidBounds = [&] { + return doc.HasMember("bounds") && doc["bounds"].IsArray() && doc["bounds"].Size() == 4 + && doc["bounds"][0].IsDouble() && doc["bounds"][1].IsDouble() + && doc["bounds"][2].IsDouble() && doc["bounds"][3].IsDouble(); + }; + + auto hasValidGeometry = [&] { + return doc.HasMember("geometry") && doc["geometry"].IsObject(); + }; + + if (doc.HasParseError() + || !doc.HasMember("style_url") || !doc["style_url"].IsString() + || !(hasValidBounds() || hasValidGeometry()) + || !doc.HasMember("min_zoom") || !doc["min_zoom"].IsDouble() + || (doc.HasMember("max_zoom") && !doc["max_zoom"].IsDouble()) + || !doc.HasMember("pixel_ratio") || !doc["pixel_ratio"].IsDouble()) { throw std::runtime_error("Malformed offline region definition"); } + // Common properties + std::string styleURL { doc["style_url"].GetString(), doc["style_url"].GetStringLength() }; - LatLngBounds bounds = LatLngBounds::hull( - LatLng(doc["bounds"][0].GetDouble(), doc["bounds"][1].GetDouble()), - LatLng(doc["bounds"][2].GetDouble(), doc["bounds"][3].GetDouble())); double minZoom = doc["min_zoom"].GetDouble(); double maxZoom = doc.HasMember("max_zoom") ? doc["max_zoom"].GetDouble() : INFINITY; float pixelRatio = doc["pixel_ratio"].GetDouble(); + + if (doc.HasMember("bounds")) { + return OfflineTilePyramidRegionDefinition{ + styleURL, + LatLngBounds::hull( + LatLng(doc["bounds"][0].GetDouble(), doc["bounds"][1].GetDouble()), + LatLng(doc["bounds"][2].GetDouble(), doc["bounds"][3].GetDouble())), + minZoom, maxZoom, pixelRatio }; + } else { + return OfflineGeometryRegionDefinition{ + styleURL, + mapbox::geojson::convert<Geometry<double>>(doc["geometry"].GetObject()), + minZoom, maxZoom, pixelRatio }; + }; - return { styleURL, bounds, minZoom, maxZoom, pixelRatio }; } std::string encodeOfflineRegionDefinition(const OfflineRegionDefinition& region) { rapidjson::GenericDocument<rapidjson::UTF8<>, rapidjson::CrtAllocator> doc; doc.SetObject(); - doc.AddMember("style_url", rapidjson::StringRef(region.styleURL.data(), region.styleURL.length()), doc.GetAllocator()); + // Encode common properties + region.match([&](auto& _region) { + doc.AddMember("style_url", rapidjson::StringRef(_region.styleURL.data(), _region.styleURL.length()), doc.GetAllocator()); + doc.AddMember("min_zoom", _region.minZoom, doc.GetAllocator()); + if (std::isfinite(_region.maxZoom)) { + doc.AddMember("max_zoom", _region.maxZoom, doc.GetAllocator()); + } - rapidjson::GenericValue<rapidjson::UTF8<>, rapidjson::CrtAllocator> bounds(rapidjson::kArrayType); - bounds.PushBack(region.bounds.south(), doc.GetAllocator()); - bounds.PushBack(region.bounds.west(), doc.GetAllocator()); - bounds.PushBack(region.bounds.north(), doc.GetAllocator()); - bounds.PushBack(region.bounds.east(), doc.GetAllocator()); - doc.AddMember("bounds", bounds, doc.GetAllocator()); + doc.AddMember("pixel_ratio", _region.pixelRatio, doc.GetAllocator()); + }); - doc.AddMember("min_zoom", region.minZoom, doc.GetAllocator()); - if (std::isfinite(region.maxZoom)) { - doc.AddMember("max_zoom", region.maxZoom, doc.GetAllocator()); - } + // Encode specific properties + region.match( + [&] (const OfflineTilePyramidRegionDefinition& _region) { + rapidjson::GenericValue<rapidjson::UTF8<>, rapidjson::CrtAllocator> bounds(rapidjson::kArrayType); + bounds.PushBack(_region.bounds.south(), doc.GetAllocator()); + bounds.PushBack(_region.bounds.west(), doc.GetAllocator()); + bounds.PushBack(_region.bounds.north(), doc.GetAllocator()); + bounds.PushBack(_region.bounds.east(), doc.GetAllocator()); + doc.AddMember("bounds", bounds, doc.GetAllocator()); + + }, + [&] (const OfflineGeometryRegionDefinition& _region) { + doc.AddMember("geometry", mapbox::geojson::convert(_region.geometry, doc.GetAllocator()), doc.GetAllocator()); - doc.AddMember("pixel_ratio", region.pixelRatio, doc.GetAllocator()); + } + ); rapidjson::StringBuffer buffer; rapidjson::Writer<rapidjson::StringBuffer> writer(buffer); @@ -113,6 +130,9 @@ std::string encodeOfflineRegionDefinition(const OfflineRegionDefinition& region) return buffer.GetString(); } + +// OfflineRegion + OfflineRegion::OfflineRegion(int64_t id_, OfflineRegionDefinition definition_, OfflineRegionMetadata metadata_) @@ -135,5 +155,4 @@ const OfflineRegionMetadata& OfflineRegion::getMetadata() const { int64_t OfflineRegion::getID() const { return id; } - } // namespace mbgl diff --git a/platform/default/mbgl/storage/offline_database.cpp b/platform/default/mbgl/storage/offline_database.cpp index 8f7f0965f4..79bc3c8f27 100644 --- a/platform/default/mbgl/storage/offline_database.cpp +++ b/platform/default/mbgl/storage/offline_database.cpp @@ -15,7 +15,14 @@ namespace mbgl { OfflineDatabase::OfflineDatabase(std::string path_, uint64_t maximumCacheSize_) : path(std::move(path_)), maximumCacheSize(maximumCacheSize_) { - ensureSchema(); + try { + initialize(); + } catch (const util::IOException& ex) { + handleError(ex, "open database"); + } catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "open database"); + } + // Assume that we can't open the database right now and work with an empty database object. } OfflineDatabase::~OfflineDatabase() { @@ -24,87 +31,74 @@ OfflineDatabase::~OfflineDatabase() { try { statements.clear(); db.reset(); - } catch (mapbox::sqlite::Exception& ex) { - Log::Error(Event::Database, (int)ex.code, ex.what()); + } catch (const util::IOException& ex) { + handleError(ex, "close database"); + } catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "close database"); } } -void OfflineDatabase::ensureSchema() { - auto result = mapbox::sqlite::Database::tryOpen(path, mapbox::sqlite::ReadWriteCreate); - if (result.is<mapbox::sqlite::Exception>()) { - const auto& ex = result.get<mapbox::sqlite::Exception>(); - if (ex.code == mapbox::sqlite::ResultCode::NotADB) { - // Corrupted; blow it away. - removeExisting(); - result = mapbox::sqlite::Database::open(path, mapbox::sqlite::ReadWriteCreate); - } else { - Log::Error(Event::Database, "Unexpected error connecting to database: %s", ex.what()); - throw ex; - } +void OfflineDatabase::initialize() { + assert(!db); + assert(statements.empty()); + + db = std::make_unique<mapbox::sqlite::Database>( + mapbox::sqlite::Database::open(path, mapbox::sqlite::ReadWriteCreate)); + db->setBusyTimeout(Milliseconds::max()); + db->exec("PRAGMA foreign_keys = ON"); + + const auto userVersion = getPragma<int64_t>("PRAGMA user_version"); + switch (userVersion) { + case 0: + case 1: + // Newly created database, or old cache-only database; remove old table if it exists. + removeOldCacheTable(); + createSchema(); + return; + case 2: + migrateToVersion3(); + // fall through + case 3: + // Removed migration, see below. + // fall through + case 4: + migrateToVersion5(); + // fall through + case 5: + migrateToVersion6(); + // fall through + case 6: + // Happy path; we're done + return; + default: + // Downgrade: delete the database and try to reinitialize. + removeExisting(); + initialize(); } +} - try { - assert(result.is<mapbox::sqlite::Database>()); - db = std::make_unique<mapbox::sqlite::Database>(std::move(result.get<mapbox::sqlite::Database>())); - db->setBusyTimeout(Milliseconds::max()); - db->exec("PRAGMA foreign_keys = ON"); - - switch (userVersion()) { - case 0: - case 1: - // Newly created database, or old cache-only database; remove old table if it exists. - removeOldCacheTable(); - break; - case 2: - migrateToVersion3(); - // fall through - case 3: - case 4: - migrateToVersion5(); - // fall through - case 5: - migrateToVersion6(); - // fall through - case 6: - // happy path; we're done - return; - default: - // downgrade, delete the database - removeExisting(); - break; - } - } catch (const mapbox::sqlite::Exception& ex) { - // Unfortunately, SQLITE_NOTADB is not always reported upon opening the database. - // Apparently sometimes it is delayed until the first read operation. - if (ex.code == mapbox::sqlite::ResultCode::NotADB) { +void OfflineDatabase::handleError(const mapbox::sqlite::Exception& ex, const char* action) { + if (ex.code == mapbox::sqlite::ResultCode::NotADB || + ex.code == mapbox::sqlite::ResultCode::Corrupt || + (ex.code == mapbox::sqlite::ResultCode::ReadOnly && + ex.extendedCode == mapbox::sqlite::ExtendedResultCode::ReadOnlyDBMoved)) { + // The database was corruped, moved away, or deleted. We're going to start fresh with a + // clean slate for the next operation. + Log::Error(Event::Database, static_cast<int>(ex.code), "Can't %s: %s", action, ex.what()); + try { removeExisting(); - } else { - throw; - } - } - - try { - // When downgrading the database, or when the database is corrupt, we've deleted the old database handle, - // so we need to reopen it. - if (!db) { - db = std::make_unique<mapbox::sqlite::Database>(mapbox::sqlite::Database::open(path, mapbox::sqlite::ReadWriteCreate)); - db->setBusyTimeout(Milliseconds::max()); - db->exec("PRAGMA foreign_keys = ON"); + } catch (const util::IOException& ioEx) { + handleError(ioEx, action); } - - db->exec("PRAGMA auto_vacuum = INCREMENTAL"); - db->exec("PRAGMA journal_mode = DELETE"); - db->exec("PRAGMA synchronous = FULL"); - db->exec(offlineDatabaseSchema); - db->exec("PRAGMA user_version = 6"); - } catch (...) { - Log::Error(Event::Database, "Unexpected error creating database schema: %s", util::toString(std::current_exception()).c_str()); - throw; + } else { + // We treat the error as temporary, and pretend we have an inaccessible DB. + Log::Warning(Event::Database, static_cast<int>(ex.code), "Can't %s: %s", action, ex.what()); } } -int OfflineDatabase::userVersion() { - return static_cast<int>(getPragma<int64_t>("PRAGMA user_version")); +void OfflineDatabase::handleError(const util::IOException& ex, const char* action) { + // We failed to delete the database file. + Log::Error(Event::Database, ex.code, "Can't %s: %s", action, ex.what()); } void OfflineDatabase::removeExisting() { @@ -113,19 +107,28 @@ void OfflineDatabase::removeExisting() { statements.clear(); db.reset(); - try { - util::deleteFile(path); - } catch (util::IOException& ex) { - Log::Error(Event::Database, ex.code, ex.what()); - } + util::deleteFile(path); } void OfflineDatabase::removeOldCacheTable() { + assert(db); db->exec("DROP TABLE IF EXISTS http_cache"); db->exec("VACUUM"); } +void OfflineDatabase::createSchema() { + assert(db); + db->exec("PRAGMA auto_vacuum = INCREMENTAL"); + db->exec("PRAGMA journal_mode = DELETE"); + db->exec("PRAGMA synchronous = FULL"); + mapbox::sqlite::Transaction transaction(*db); + db->exec(offlineDatabaseSchema); + db->exec("PRAGMA user_version = 6"); + transaction.commit(); +} + void OfflineDatabase::migrateToVersion3() { + assert(db); db->exec("PRAGMA auto_vacuum = INCREMENTAL"); db->exec("VACUUM"); db->exec("PRAGMA user_version = 3"); @@ -138,12 +141,14 @@ void OfflineDatabase::migrateToVersion3() { // See: https://github.com/mapbox/mapbox-gl-native/pull/6320 void OfflineDatabase::migrateToVersion5() { + assert(db); db->exec("PRAGMA journal_mode = DELETE"); db->exec("PRAGMA synchronous = FULL"); db->exec("PRAGMA user_version = 5"); } void OfflineDatabase::migrateToVersion6() { + assert(db); mapbox::sqlite::Transaction transaction(*db); db->exec("ALTER TABLE resources ADD COLUMN must_revalidate INTEGER NOT NULL DEFAULT 0"); db->exec("ALTER TABLE tiles ADD COLUMN must_revalidate INTEGER NOT NULL DEFAULT 0"); @@ -152,6 +157,9 @@ void OfflineDatabase::migrateToVersion6() { } mapbox::sqlite::Statement& OfflineDatabase::getStatement(const char* sql) { + if (!db) { + initialize(); + } auto it = statements.find(sql); if (it == statements.end()) { it = statements.emplace(sql, std::make_unique<mapbox::sqlite::Statement>(*db, sql)).first; @@ -159,9 +167,15 @@ mapbox::sqlite::Statement& OfflineDatabase::getStatement(const char* sql) { return *it->second; } -optional<Response> OfflineDatabase::get(const Resource& resource) { +optional<Response> OfflineDatabase::get(const Resource& resource) try { auto result = getInternal(resource); - return result ? result->first : optional<Response>(); + return result ? optional<Response>{ result->first } : nullopt; +} catch (const util::IOException& ex) { + handleError(ex, "read resource"); + return nullopt; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "read resource"); + return nullopt; } optional<std::pair<Response, uint64_t>> OfflineDatabase::getInternal(const Resource& resource) { @@ -182,11 +196,17 @@ optional<int64_t> OfflineDatabase::hasInternal(const Resource& resource) { } } -std::pair<bool, uint64_t> OfflineDatabase::put(const Resource& resource, const Response& response) { +std::pair<bool, uint64_t> OfflineDatabase::put(const Resource& resource, const Response& response) try { + if (!db) { + initialize(); + } mapbox::sqlite::Transaction transaction(*db, mapbox::sqlite::Transaction::Immediate); auto result = putInternal(resource, response, true); transaction.commit(); return result; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "write resource"); + return { false, 0 }; } std::pair<bool, uint64_t> OfflineDatabase::putInternal(const Resource& resource, const Response& response, bool evict_) { @@ -227,11 +247,19 @@ std::pair<bool, uint64_t> OfflineDatabase::putInternal(const Resource& resource, optional<std::pair<Response, uint64_t>> OfflineDatabase::getResource(const Resource& resource) { // Update accessed timestamp used for LRU eviction. - { + try { mapbox::sqlite::Query accessedQuery{ getStatement("UPDATE resources SET accessed = ?1 WHERE url = ?2") }; accessedQuery.bind(1, util::now()); accessedQuery.bind(2, resource.url); accessedQuery.run(); + } catch (const mapbox::sqlite::Exception& ex) { + if (ex.code == mapbox::sqlite::ResultCode::NotADB || + ex.code == mapbox::sqlite::ResultCode::Corrupt) { + throw; + } + + // If we don't have any indication that the database is corrupt, continue as usual. + Log::Warning(Event::Database, static_cast<int>(ex.code), "Can't update timestamp: %s", ex.what()); } // clang-format off @@ -245,7 +273,7 @@ optional<std::pair<Response, uint64_t>> OfflineDatabase::getResource(const Resou query.bind(1, resource.url); if (!query.run()) { - return {}; + return nullopt; } Response response; @@ -274,7 +302,7 @@ optional<int64_t> OfflineDatabase::hasResource(const Resource& resource) { mapbox::sqlite::Query query{ getStatement("SELECT length(data) FROM resources WHERE url = ?") }; query.bind(1, resource.url); if (!query.run()) { - return {}; + return nullopt; } return query.get<optional<int64_t>>(0); @@ -366,7 +394,8 @@ bool OfflineDatabase::putResource(const Resource& resource, } optional<std::pair<Response, uint64_t>> OfflineDatabase::getTile(const Resource::TileData& tile) { - { + // Update accessed timestamp used for LRU eviction. + try { // clang-format off mapbox::sqlite::Query accessedQuery{ getStatement( "UPDATE tiles " @@ -385,6 +414,13 @@ optional<std::pair<Response, uint64_t>> OfflineDatabase::getTile(const Resource: accessedQuery.bind(5, tile.y); accessedQuery.bind(6, tile.z); accessedQuery.run(); + } catch (const mapbox::sqlite::Exception& ex) { + if (ex.code == mapbox::sqlite::ResultCode::NotADB || ex.code == mapbox::sqlite::ResultCode::Corrupt) { + throw; + } + + // If we don't have any indication that the database is corrupt, continue as usual. + Log::Warning(Event::Database, static_cast<int>(ex.code), "Can't update timestamp: %s", ex.what()); } // clang-format off @@ -406,7 +442,7 @@ optional<std::pair<Response, uint64_t>> OfflineDatabase::getTile(const Resource: query.bind(5, tile.z); if (!query.run()) { - return {}; + return nullopt; } Response response; @@ -450,7 +486,7 @@ optional<int64_t> OfflineDatabase::hasTile(const Resource::TileData& tile) { size.bind(5, tile.z); if (!size.run()) { - return {}; + return nullopt; } return size.get<optional<int64_t>>(0); @@ -559,23 +595,33 @@ bool OfflineDatabase::putTile(const Resource::TileData& tile, return true; } -std::vector<OfflineRegion> OfflineDatabase::listRegions() { +expected<OfflineRegions, std::exception_ptr> OfflineDatabase::listRegions() try { mapbox::sqlite::Query query{ getStatement("SELECT id, definition, description FROM regions") }; - - std::vector<OfflineRegion> result; - + OfflineRegions result; while (query.run()) { - result.push_back(OfflineRegion( - query.get<int64_t>(0), - decodeOfflineRegionDefinition(query.get<std::string>(1)), - query.get<std::vector<uint8_t>>(2))); + const auto id = query.get<int64_t>(0); + const auto definition = query.get<std::string>(1); + const auto description = query.get<std::vector<uint8_t>>(2); + try { + // Construct, then move because this constructor is private. + OfflineRegion region(id, decodeOfflineRegionDefinition(definition), description); + result.emplace_back(std::move(region)); + } catch (const std::exception& ex) { + // Catch errors from malformed offline region definitions + // and skip them. + Log::Error(Event::General, "%s", ex.what()); + } } - - return result; + // Explicit move to avoid triggering the copy constructor. + return { std::move(result) }; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "list regions"); + return unexpected<std::exception_ptr>(std::current_exception()); } -OfflineRegion OfflineDatabase::createRegion(const OfflineRegionDefinition& definition, - const OfflineRegionMetadata& metadata) { +expected<OfflineRegion, std::exception_ptr> +OfflineDatabase::createRegion(const OfflineRegionDefinition& definition, + const OfflineRegionMetadata& metadata) try { // clang-format off mapbox::sqlite::Query query{ getStatement( "INSERT INTO regions (definition, description) " @@ -585,11 +631,14 @@ OfflineRegion OfflineDatabase::createRegion(const OfflineRegionDefinition& defin query.bind(1, encodeOfflineRegionDefinition(definition)); query.bindBlob(2, metadata); query.run(); - return OfflineRegion(query.lastInsertRowId(), definition, metadata); +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "create region"); + return unexpected<std::exception_ptr>(std::current_exception()); } -OfflineRegionMetadata OfflineDatabase::updateMetadata(const int64_t regionID, const OfflineRegionMetadata& metadata) { +expected<OfflineRegionMetadata, std::exception_ptr> +OfflineDatabase::updateMetadata(const int64_t regionID, const OfflineRegionMetadata& metadata) try { // clang-format off mapbox::sqlite::Query query{ getStatement( "UPDATE regions SET description = ?1 " @@ -600,9 +649,12 @@ OfflineRegionMetadata OfflineDatabase::updateMetadata(const int64_t regionID, co query.run(); return metadata; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "update region metadata"); + return unexpected<std::exception_ptr>(std::current_exception()); } -void OfflineDatabase::deleteRegion(OfflineRegion&& region) { +std::exception_ptr OfflineDatabase::deleteRegion(OfflineRegion&& region) try { { mapbox::sqlite::Query query{ getStatement("DELETE FROM regions WHERE id = ?") }; query.bind(1, region.getID()); @@ -610,13 +662,18 @@ void OfflineDatabase::deleteRegion(OfflineRegion&& region) { } evict(0); + assert(db); db->exec("PRAGMA incremental_vacuum"); // Ensure that the cached offlineTileCount value is recalculated. offlineMapboxTileCount = {}; + return nullptr; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "delete region"); + return std::current_exception(); } -optional<std::pair<Response, uint64_t>> OfflineDatabase::getRegionResource(int64_t regionID, const Resource& resource) { +optional<std::pair<Response, uint64_t>> OfflineDatabase::getRegionResource(int64_t regionID, const Resource& resource) try { auto response = getInternal(resource); if (response) { @@ -624,9 +681,12 @@ optional<std::pair<Response, uint64_t>> OfflineDatabase::getRegionResource(int64 } return response; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "read region resource"); + return nullopt; } -optional<int64_t> OfflineDatabase::hasRegionResource(int64_t regionID, const Resource& resource) { +optional<int64_t> OfflineDatabase::hasRegionResource(int64_t regionID, const Resource& resource) try { auto response = hasInternal(resource); if (response) { @@ -634,29 +694,52 @@ optional<int64_t> OfflineDatabase::hasRegionResource(int64_t regionID, const Res } return response; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "query region resource"); + return nullopt; } -uint64_t OfflineDatabase::putRegionResource(int64_t regionID, const Resource& resource, const Response& response) { +uint64_t OfflineDatabase::putRegionResource(int64_t regionID, + const Resource& resource, + const Response& response) try { + if (!db) { + initialize(); + } mapbox::sqlite::Transaction transaction(*db); auto size = putRegionResourceInternal(regionID, resource, response); transaction.commit(); return size; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "write region resource"); + return 0; } -void OfflineDatabase::putRegionResources(int64_t regionID, const std::list<std::tuple<Resource, Response>>& resources, OfflineRegionStatus& status) { +void OfflineDatabase::putRegionResources(int64_t regionID, + const std::list<std::tuple<Resource, Response>>& resources, + OfflineRegionStatus& status) try { + if (!db) { + initialize(); + } mapbox::sqlite::Transaction transaction(*db); + // Accumulate all statistics locally first before adding them to the OfflineRegionStatus object + // to ensure correctness when the transaction fails. + uint64_t completedResourceCount = 0; + uint64_t completedResourceSize = 0; + uint64_t completedTileCount = 0; + uint64_t completedTileSize = 0; + for (const auto& elem : resources) { const auto& resource = std::get<0>(elem); const auto& response = std::get<1>(elem); try { uint64_t resourceSize = putRegionResourceInternal(regionID, resource, response); - status.completedResourceCount++; - status.completedResourceSize += resourceSize; + completedResourceCount++; + completedResourceSize += resourceSize; if (resource.kind == Resource::Kind::Tile) { - status.completedTileCount += 1; - status.completedTileSize += resourceSize; + completedTileCount += 1; + completedTileSize += resourceSize; } } catch (const MapboxTileLimitExceededException&) { // Commit the rest of the batch and retrow @@ -667,6 +750,13 @@ void OfflineDatabase::putRegionResources(int64_t regionID, const std::list<std:: // Commit the completed batch transaction.commit(); + + status.completedResourceCount += completedResourceCount; + status.completedResourceSize += completedResourceSize; + status.completedTileCount += completedTileCount; + status.completedTileSize += completedTileSize; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "write region resources"); } uint64_t OfflineDatabase::putRegionResourceInternal(int64_t regionID, const Resource& resource, const Response& response) { @@ -755,7 +845,7 @@ bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) { mapbox::sqlite::Query selectQuery{ getStatement( "SELECT region_id " "FROM region_resources, resources " - "WHERE region_id != ?1 " + "WHERE region_id != ?1 " " AND resources.url = ?2 " "LIMIT 1 ") }; // clang-format on @@ -766,15 +856,18 @@ bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) { } } -OfflineRegionDefinition OfflineDatabase::getRegionDefinition(int64_t regionID) { +expected<OfflineRegionDefinition, std::exception_ptr> OfflineDatabase::getRegionDefinition(int64_t regionID) try { mapbox::sqlite::Query query{ getStatement("SELECT definition FROM regions WHERE id = ?1") }; query.bind(1, regionID); query.run(); return decodeOfflineRegionDefinition(query.get<std::string>(0)); +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "load region"); + return unexpected<std::exception_ptr>(std::current_exception()); } -OfflineRegionStatus OfflineDatabase::getRegionCompletedStatus(int64_t regionID) { +expected<OfflineRegionStatus, std::exception_ptr> OfflineDatabase::getRegionCompletedStatus(int64_t regionID) try { OfflineRegionStatus result; std::tie(result.completedResourceCount, result.completedResourceSize) @@ -786,6 +879,9 @@ OfflineRegionStatus OfflineDatabase::getRegionCompletedStatus(int64_t regionID) result.completedResourceSize += result.completedTileSize; return result; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "get region status"); + return unexpected<std::exception_ptr>(std::current_exception()); } std::pair<int64_t, int64_t> OfflineDatabase::getCompletedResourceCountAndSize(int64_t regionID) { @@ -920,7 +1016,7 @@ bool OfflineDatabase::offlineMapboxTileCountLimitExceeded() { return getOfflineMapboxTileCount() >= offlineMapboxTileCountLimit; } -uint64_t OfflineDatabase::getOfflineMapboxTileCount() { +uint64_t OfflineDatabase::getOfflineMapboxTileCount() try { // Calculating this on every call would be much simpler than caching and // manually updating the value, but it would make offline downloads an O(n²) // operation, because the database query below involves an index scan of @@ -942,6 +1038,9 @@ uint64_t OfflineDatabase::getOfflineMapboxTileCount() { offlineMapboxTileCount = query.get<int64_t>(0); return *offlineMapboxTileCount; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "get offline Mapbox tile count"); + return std::numeric_limits<uint64_t>::max(); } bool OfflineDatabase::exceedsOfflineMapboxTileCountLimit(const Resource& resource) { diff --git a/platform/default/mbgl/storage/offline_database.hpp b/platform/default/mbgl/storage/offline_database.hpp index 38eb3783ba..fbe6c707e1 100644 --- a/platform/default/mbgl/storage/offline_database.hpp +++ b/platform/default/mbgl/storage/offline_database.hpp @@ -7,6 +7,7 @@ #include <mbgl/util/optional.hpp> #include <mbgl/util/constants.hpp> #include <mbgl/util/mapbox.hpp> +#include <mbgl/util/expected.hpp> #include <unordered_map> #include <memory> @@ -18,6 +19,7 @@ namespace sqlite { class Database; class Statement; class Query; +class Exception; } // namespace sqlite } // namespace mapbox @@ -26,6 +28,10 @@ namespace mbgl { class Response; class TileID; +namespace util { +struct IOException; +} // namespace util + struct MapboxTileLimitExceededException : util::Exception { MapboxTileLimitExceededException() : util::Exception("Mapbox tile limit exceeded") {} }; @@ -42,14 +48,15 @@ public: // Return value is (inserted, stored size) std::pair<bool, uint64_t> put(const Resource&, const Response&); - std::vector<OfflineRegion> listRegions(); + expected<OfflineRegions, std::exception_ptr> listRegions(); - OfflineRegion createRegion(const OfflineRegionDefinition&, - const OfflineRegionMetadata&); + expected<OfflineRegion, std::exception_ptr> createRegion(const OfflineRegionDefinition&, + const OfflineRegionMetadata&); - OfflineRegionMetadata updateMetadata(const int64_t regionID, const OfflineRegionMetadata&); + expected<OfflineRegionMetadata, std::exception_ptr> + updateMetadata(const int64_t regionID, const OfflineRegionMetadata&); - void deleteRegion(OfflineRegion&&); + std::exception_ptr deleteRegion(OfflineRegion&&); // Return value is (response, stored size) optional<std::pair<Response, uint64_t>> getRegionResource(int64_t regionID, const Resource&); @@ -57,8 +64,8 @@ public: uint64_t putRegionResource(int64_t regionID, const Resource&, const Response&); void putRegionResources(int64_t regionID, const std::list<std::tuple<Resource, Response>>&, OfflineRegionStatus&); - OfflineRegionDefinition getRegionDefinition(int64_t regionID); - OfflineRegionStatus getRegionCompletedStatus(int64_t regionID); + expected<OfflineRegionDefinition, std::exception_ptr> getRegionDefinition(int64_t regionID); + expected<OfflineRegionStatus, std::exception_ptr> getRegionCompletedStatus(int64_t regionID); void setOfflineMapboxTileCountLimit(uint64_t); uint64_t getOfflineMapboxTileCountLimit(); @@ -67,12 +74,15 @@ public: bool exceedsOfflineMapboxTileCountLimit(const Resource&); private: - int userVersion(); - void ensureSchema(); + void initialize(); + void handleError(const mapbox::sqlite::Exception&, const char* action); + void handleError(const util::IOException&, const char* action); + void removeExisting(); void removeOldCacheTable(); - void migrateToVersion3(); + void createSchema(); void migrateToVersion5(); + void migrateToVersion3(); void migrateToVersion6(); mapbox::sqlite::Statement& getStatement(const char *); diff --git a/platform/default/mbgl/storage/offline_download.cpp b/platform/default/mbgl/storage/offline_download.cpp index 4da51db655..118f3aad88 100644 --- a/platform/default/mbgl/storage/offline_download.cpp +++ b/platform/default/mbgl/storage/offline_download.cpp @@ -24,6 +24,63 @@ namespace mbgl { using namespace style; +// Generic functions + +template <class RegionDefinition> +Range<uint8_t> coveringZoomRange(const RegionDefinition& definition, + style::SourceType type, uint16_t tileSize, const Range<uint8_t>& zoomRange) { + double minZ = std::max<double>(util::coveringZoomLevel(definition.minZoom, type, tileSize), zoomRange.min); + double maxZ = std::min<double>(util::coveringZoomLevel(definition.maxZoom, type, tileSize), zoomRange.max); + + assert(minZ >= 0); + assert(maxZ >= 0); + assert(minZ < std::numeric_limits<uint8_t>::max()); + assert(maxZ < std::numeric_limits<uint8_t>::max()); + return { static_cast<uint8_t>(minZ), static_cast<uint8_t>(maxZ) }; +} + +template <class Geometry, class Fn> +void tileCover(const Geometry& geometry, uint8_t z, Fn&& fn) { + util::TileCover cover(geometry, z); + while (cover.hasNext()) { + fn(cover.next()->canonical); + } +} + + +template <class Fn> +void tileCover(const OfflineRegionDefinition& definition, style::SourceType type, + uint16_t tileSize, const Range<uint8_t>& zoomRange, Fn&& fn) { + const Range<uint8_t> clampedZoomRange = + definition.match([&](auto& reg) { return coveringZoomRange(reg, type, tileSize, zoomRange); }); + + for (uint8_t z = clampedZoomRange.min; z <= clampedZoomRange.max; z++) { + definition.match( + [&](const OfflineTilePyramidRegionDefinition& reg){ tileCover(reg.bounds, z, fn); }, + [&](const OfflineGeometryRegionDefinition& reg){ tileCover(reg.geometry, z, fn); } + ); + } +} + +uint64_t tileCount(const OfflineRegionDefinition& definition, style::SourceType type, + uint16_t tileSize, const Range<uint8_t>& zoomRange) { + + const Range<uint8_t> clampedZoomRange = + definition.match([&](auto& reg) { return coveringZoomRange(reg, type, tileSize, zoomRange); }); + + unsigned long result = 0;; + for (uint8_t z = clampedZoomRange.min; z <= clampedZoomRange.max; z++) { + result += definition.match( + [&](const OfflineTilePyramidRegionDefinition& reg){ return util::tileCount(reg.bounds, z); }, + [&](const OfflineGeometryRegionDefinition& reg){ return util::tileCount(reg.geometry, z); } + ); + } + + return result; +} + +// OfflineDownload + OfflineDownload::OfflineDownload(int64_t id_, OfflineRegionDefinition&& definition_, OfflineDatabase& offlineDatabase_, @@ -62,39 +119,45 @@ OfflineRegionStatus OfflineDownload::getStatus() const { return status; } - OfflineRegionStatus result = offlineDatabase.getRegionCompletedStatus(id); + auto result = offlineDatabase.getRegionCompletedStatus(id); + if (!result) { + // We can't find this offline region because the database is unavailable, or the download + // does not exist. + return {}; + } - result.requiredResourceCount++; - optional<Response> styleResponse = offlineDatabase.get(Resource::style(definition.styleURL)); + result->requiredResourceCount++; + optional<Response> styleResponse = + offlineDatabase.get(Resource::style(definition.match([](auto& reg){ return reg.styleURL; }))); if (!styleResponse) { - return result; + return *result; } style::Parser parser; parser.parse(*styleResponse->data); - result.requiredResourceCountIsPrecise = true; + result->requiredResourceCountIsPrecise = true; for (const auto& source : parser.sources) { SourceType type = source->getType(); auto handleTiledSource = [&] (const variant<std::string, Tileset>& urlOrTileset, const uint16_t tileSize) { if (urlOrTileset.is<Tileset>()) { - result.requiredResourceCount += - definition.tileCount(type, tileSize, urlOrTileset.get<Tileset>().zoomRange); + result->requiredResourceCount += + tileCount(definition, type, tileSize, urlOrTileset.get<Tileset>().zoomRange); } else { - result.requiredResourceCount += 1; + result->requiredResourceCount += 1; const auto& url = urlOrTileset.get<std::string>(); optional<Response> sourceResponse = offlineDatabase.get(Resource::source(url)); if (sourceResponse) { style::conversion::Error error; optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(*sourceResponse->data, error); if (tileset) { - result.requiredResourceCount += - definition.tileCount(type, tileSize, (*tileset).zoomRange); + result->requiredResourceCount += + tileCount(definition, type, tileSize, (*tileset).zoomRange); } } else { - result.requiredResourceCountIsPrecise = false; + result->requiredResourceCountIsPrecise = false; } } }; @@ -111,7 +174,7 @@ OfflineRegionStatus OfflineDownload::getStatus() const { handleTiledSource(rasterSource.getURLOrTileset(), rasterSource.getTileSize()); break; } - + case SourceType::RasterDEM: { const auto& rasterDEMSource = *source->as<RasterDEMSource>(); handleTiledSource(rasterDEMSource.getURLOrTileset(), rasterDEMSource.getTileSize()); @@ -121,7 +184,7 @@ OfflineRegionStatus OfflineDownload::getStatus() const { case SourceType::GeoJSON: { const auto& geojsonSource = *source->as<GeoJSONSource>(); if (geojsonSource.getURL()) { - result.requiredResourceCount += 1; + result->requiredResourceCount += 1; } break; } @@ -129,7 +192,7 @@ OfflineRegionStatus OfflineDownload::getStatus() const { case SourceType::Image: { const auto& imageSource = *source->as<ImageSource>(); if (imageSource.getURL()) { - result.requiredResourceCount += 1; + result->requiredResourceCount += 1; } break; } @@ -142,21 +205,22 @@ OfflineRegionStatus OfflineDownload::getStatus() const { } if (!parser.glyphURL.empty()) { - result.requiredResourceCount += parser.fontStacks().size() * GLYPH_RANGES_PER_FONT_STACK; + result->requiredResourceCount += parser.fontStacks().size() * GLYPH_RANGES_PER_FONT_STACK; } if (!parser.spriteURL.empty()) { - result.requiredResourceCount += 2; + result->requiredResourceCount += 2; } - return result; + return *result; } void OfflineDownload::activateDownload() { status = OfflineRegionStatus(); status.downloadState = OfflineRegionDownloadState::Active; status.requiredResourceCount++; - ensureResource(Resource::style(definition.styleURL), [&](Response styleResponse) { + ensureResource(Resource::style(definition.match([](auto& reg){ return reg.styleURL; })), + [&](Response styleResponse) { status.requiredResourceCountIsPrecise = true; style::Parser parser; @@ -202,7 +266,7 @@ void OfflineDownload::activateDownload() { handleTiledSource(rasterSource.getURLOrTileset(), rasterSource.getTileSize()); break; } - + case SourceType::RasterDEM: { const auto& rasterDEMSource = *source->as<RasterDEMSource>(); handleTiledSource(rasterDEMSource.getURLOrTileset(), rasterDEMSource.getTileSize()); @@ -242,8 +306,9 @@ void OfflineDownload::activateDownload() { } if (!parser.spriteURL.empty()) { - queueResource(Resource::spriteImage(parser.spriteURL, definition.pixelRatio)); - queueResource(Resource::spriteJSON(parser.spriteURL, definition.pixelRatio)); + auto pixelRatio = definition.match([](auto& reg){ return reg.pixelRatio; }); + queueResource(Resource::spriteImage(parser.spriteURL, pixelRatio)); + queueResource(Resource::spriteJSON(parser.spriteURL, pixelRatio)); } continueDownload(); @@ -291,11 +356,12 @@ void OfflineDownload::queueResource(Resource resource) { } void OfflineDownload::queueTiles(SourceType type, uint16_t tileSize, const Tileset& tileset) { - for (const auto& tile : definition.tileCover(type, tileSize, tileset.zoomRange)) { + tileCover(definition, type, tileSize, tileset.zoomRange, [&](const auto& tile) { status.requiredResourceCount++; - resourcesRemaining.push_back( - Resource::tile(tileset.tiles[0], definition.pixelRatio, tile.x, tile.y, tile.z, tileset.scheme)); - } + resourcesRemaining.push_back(Resource::tile( + tileset.tiles[0], definition.match([](auto& def) { return def.pixelRatio; }), tile.x, + tile.y, tile.z, tileset.scheme)); + }); } void OfflineDownload::ensureResource(const Resource& resource, |