diff options
Diffstat (limited to 'platform/default/mbgl/storage/offline_database.cpp')
| -rw-r--r-- | platform/default/mbgl/storage/offline_database.cpp | 295 |
1 files changed, 184 insertions, 111 deletions
diff --git a/platform/default/mbgl/storage/offline_database.cpp b/platform/default/mbgl/storage/offline_database.cpp index 92e534634d..d9a06fbc92 100644 --- a/platform/default/mbgl/storage/offline_database.cpp +++ b/platform/default/mbgl/storage/offline_database.cpp @@ -13,7 +13,12 @@ namespace mbgl { OfflineDatabase::OfflineDatabase(std::string path_, uint64_t maximumCacheSize_) : path(std::move(path_)), maximumCacheSize(maximumCacheSize_) { - ensureSchema(); + try { + initialize(); + } catch (const std::exception& ex) { + handleError(ex, "open database"); + // Assume that we can't open the database right now and work with an empty database object. + } } OfflineDatabase::~OfflineDatabase() { @@ -22,110 +27,102 @@ OfflineDatabase::~OfflineDatabase() { try { statements.clear(); db.reset(); - } catch (mapbox::sqlite::Exception& ex) { - Log::Error(Event::Database, (int)ex.code, ex.what()); + } catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "close database"); } } -void OfflineDatabase::ensureSchema() { - auto result = mapbox::sqlite::Database::tryOpen(path, mapbox::sqlite::ReadWriteCreate); - if (result.is<mapbox::sqlite::Exception>()) { - const auto& ex = result.get<mapbox::sqlite::Exception>(); - if (ex.code == mapbox::sqlite::ResultCode::NotADB) { - // Corrupted; blow it away. - removeExisting(); - result = mapbox::sqlite::Database::open(path, mapbox::sqlite::ReadWriteCreate); - } else { - Log::Error(Event::Database, "Unexpected error connecting to database: %s", ex.what()); - throw ex; - } +void OfflineDatabase::initialize() { + assert(!db); + assert(statements.empty()); + + db = std::make_unique<mapbox::sqlite::Database>( + mapbox::sqlite::Database::open(path, mapbox::sqlite::ReadWriteCreate)); + db->setBusyTimeout(Milliseconds::max()); + db->exec("PRAGMA foreign_keys = ON"); + + const auto userVersion = getPragma<int64_t>("PRAGMA user_version"); + switch (userVersion) { + case 0: + case 1: + // Newly created database, or old cache-only database; remove old table if it exists. + removeOldCacheTable(); + createSchema(); + return; + case 2: + migrateToVersion3(); + // fall through + case 3: + // Removed migration, see below. + // fall through + case 4: + migrateToVersion5(); + // fall through + case 5: + migrateToVersion6(); + // fall through + case 6: + // Happy path; we're done + return; + default: + // Downgrade: delete the database and try to reinitialize. + removeExisting(); + initialize(); } +} - try { - assert(result.is<mapbox::sqlite::Database>()); - db = std::make_unique<mapbox::sqlite::Database>(std::move(result.get<mapbox::sqlite::Database>())); - db->setBusyTimeout(Milliseconds::max()); - db->exec("PRAGMA foreign_keys = ON"); - - switch (userVersion()) { - case 0: - case 1: - // Newly created database, or old cache-only database; remove old table if it exists. - removeOldCacheTable(); - break; - case 2: - migrateToVersion3(); - // fall through - case 3: - case 4: - migrateToVersion5(); - // fall through - case 5: - migrateToVersion6(); - // fall through - case 6: - // happy path; we're done - return; - default: - // downgrade, delete the database - removeExisting(); - break; - } - } catch (const mapbox::sqlite::Exception& ex) { - // Unfortunately, SQLITE_NOTADB is not always reported upon opening the database. - // Apparently sometimes it is delayed until the first read operation. - if (ex.code == mapbox::sqlite::ResultCode::NotADB) { - removeExisting(); +void OfflineDatabase::handleError(const std::exception& ex, const char* action) { + if (auto sqliteEx = dynamic_cast<const mapbox::sqlite::Exception*>(&ex)) { + if (sqliteEx->code == mapbox::sqlite::ResultCode::NotADB || + sqliteEx->code == mapbox::sqlite::ResultCode::Corrupt) { + // Corrupted; delete database so that we have a clean slate for the next operation. + Log::Error(Event::Database, static_cast<int>(sqliteEx->code), "Can't %s: %s", action, sqliteEx->what()); + try { + removeExisting(); + } catch (const util::IOException& ioEx) { + Log::Error(Event::Database, ioEx.code, "Can't %s: %s", action, ioEx.what()); + } } else { - throw; - } - } - - try { - #include "offline_schema.cpp.include" - - // When downgrading the database, or when the database is corrupt, we've deleted the old database handle, - // so we need to reopen it. - if (!db) { - db = std::make_unique<mapbox::sqlite::Database>(mapbox::sqlite::Database::open(path, mapbox::sqlite::ReadWriteCreate)); - db->setBusyTimeout(Milliseconds::max()); - db->exec("PRAGMA foreign_keys = ON"); + // We treat the error as temporary, and pretend we have an inaccessible DB. + Log::Warning(Event::Database, static_cast<int>(sqliteEx->code), "Can't %s: %s", action, sqliteEx->what()); } - - db->exec("PRAGMA auto_vacuum = INCREMENTAL"); - db->exec("PRAGMA journal_mode = DELETE"); - db->exec("PRAGMA synchronous = FULL"); - db->exec(schema); - db->exec("PRAGMA user_version = 6"); - } catch (...) { - Log::Error(Event::Database, "Unexpected error creating database schema: %s", util::toString(std::current_exception()).c_str()); - throw; + } else if (auto ioEx = dynamic_cast<const util::IOException*>(&ex)) { + // We failed to delete the database file. + Log::Error(Event::Database, ioEx->code, "Can't %s: %s", action, ioEx->what()); + } else { + Log::Error(Event::Database, -1, "Can't %s: unknown error", action); } } -int OfflineDatabase::userVersion() { - return static_cast<int>(getPragma<int64_t>("PRAGMA user_version")); -} - void OfflineDatabase::removeExisting() { Log::Warning(Event::Database, "Removing existing incompatible offline database"); statements.clear(); db.reset(); - try { - util::deleteFile(path); - } catch (util::IOException& ex) { - Log::Error(Event::Database, ex.code, ex.what()); - } + util::deleteFile(path); } void OfflineDatabase::removeOldCacheTable() { + assert(db); db->exec("DROP TABLE IF EXISTS http_cache"); db->exec("VACUUM"); } +void OfflineDatabase::createSchema() { + #include "offline_schema.cpp.include" + assert(db); + db->exec("PRAGMA auto_vacuum = INCREMENTAL"); + db->exec("PRAGMA journal_mode = DELETE"); + db->exec("PRAGMA synchronous = FULL"); + mapbox::sqlite::Transaction transaction(*db); + db->exec(schema); + db->exec("PRAGMA user_version = 6"); + transaction.commit(); +} + void OfflineDatabase::migrateToVersion3() { + assert(db); db->exec("PRAGMA auto_vacuum = INCREMENTAL"); db->exec("VACUUM"); db->exec("PRAGMA user_version = 3"); @@ -138,12 +135,14 @@ void OfflineDatabase::migrateToVersion3() { // See: https://github.com/mapbox/mapbox-gl-native/pull/6320 void OfflineDatabase::migrateToVersion5() { + assert(db); db->exec("PRAGMA journal_mode = DELETE"); db->exec("PRAGMA synchronous = FULL"); db->exec("PRAGMA user_version = 5"); } void OfflineDatabase::migrateToVersion6() { + assert(db); mapbox::sqlite::Transaction transaction(*db); db->exec("ALTER TABLE resources ADD COLUMN must_revalidate INTEGER NOT NULL DEFAULT 0"); db->exec("ALTER TABLE tiles ADD COLUMN must_revalidate INTEGER NOT NULL DEFAULT 0"); @@ -152,6 +151,9 @@ void OfflineDatabase::migrateToVersion6() { } mapbox::sqlite::Statement& OfflineDatabase::getStatement(const char* sql) { + if (!db) { + initialize(); + } auto it = statements.find(sql); if (it == statements.end()) { it = statements.emplace(sql, std::make_unique<mapbox::sqlite::Statement>(*db, sql)).first; @@ -159,9 +161,12 @@ mapbox::sqlite::Statement& OfflineDatabase::getStatement(const char* sql) { return *it->second; } -optional<Response> OfflineDatabase::get(const Resource& resource) { +optional<Response> OfflineDatabase::get(const Resource& resource) try { auto result = getInternal(resource); - return result ? result->first : optional<Response>(); + return result ? optional<Response>{ result->first } : nullopt; +} catch (const std::exception& ex) { + handleError(ex, "read resource"); + return nullopt; } optional<std::pair<Response, uint64_t>> OfflineDatabase::getInternal(const Resource& resource) { @@ -182,11 +187,17 @@ optional<int64_t> OfflineDatabase::hasInternal(const Resource& resource) { } } -std::pair<bool, uint64_t> OfflineDatabase::put(const Resource& resource, const Response& response) { +std::pair<bool, uint64_t> OfflineDatabase::put(const Resource& resource, const Response& response) try { + if (!db) { + initialize(); + } mapbox::sqlite::Transaction transaction(*db, mapbox::sqlite::Transaction::Immediate); auto result = putInternal(resource, response, true); transaction.commit(); return result; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "write resource"); + return { false, 0 }; } std::pair<bool, uint64_t> OfflineDatabase::putInternal(const Resource& resource, const Response& response, bool evict_) { @@ -227,11 +238,19 @@ std::pair<bool, uint64_t> OfflineDatabase::putInternal(const Resource& resource, optional<std::pair<Response, uint64_t>> OfflineDatabase::getResource(const Resource& resource) { // Update accessed timestamp used for LRU eviction. - { + try { mapbox::sqlite::Query accessedQuery{ getStatement("UPDATE resources SET accessed = ?1 WHERE url = ?2") }; accessedQuery.bind(1, util::now()); accessedQuery.bind(2, resource.url); accessedQuery.run(); + } catch (const mapbox::sqlite::Exception& ex) { + if (ex.code == mapbox::sqlite::ResultCode::NotADB || + ex.code == mapbox::sqlite::ResultCode::Corrupt) { + throw; + } + + // If we don't have any indication that the database is corrupt, continue as usual. + Log::Warning(Event::Database, static_cast<int>(ex.code), "Can't update timestamp: %s", ex.what()); } // clang-format off @@ -245,7 +264,7 @@ optional<std::pair<Response, uint64_t>> OfflineDatabase::getResource(const Resou query.bind(1, resource.url); if (!query.run()) { - return {}; + return nullopt; } Response response; @@ -274,7 +293,7 @@ optional<int64_t> OfflineDatabase::hasResource(const Resource& resource) { mapbox::sqlite::Query query{ getStatement("SELECT length(data) FROM resources WHERE url = ?") }; query.bind(1, resource.url); if (!query.run()) { - return {}; + return nullopt; } return query.get<optional<int64_t>>(0); @@ -366,7 +385,8 @@ bool OfflineDatabase::putResource(const Resource& resource, } optional<std::pair<Response, uint64_t>> OfflineDatabase::getTile(const Resource::TileData& tile) { - { + // Update accessed timestamp used for LRU eviction. + try { // clang-format off mapbox::sqlite::Query accessedQuery{ getStatement( "UPDATE tiles " @@ -385,6 +405,13 @@ optional<std::pair<Response, uint64_t>> OfflineDatabase::getTile(const Resource: accessedQuery.bind(5, tile.y); accessedQuery.bind(6, tile.z); accessedQuery.run(); + } catch (const mapbox::sqlite::Exception& ex) { + if (ex.code == mapbox::sqlite::ResultCode::NotADB || ex.code == mapbox::sqlite::ResultCode::Corrupt) { + throw; + } + + // If we don't have any indication that the database is corrupt, continue as usual. + Log::Warning(Event::Database, static_cast<int>(ex.code), "Can't update timestamp: %s", ex.what()); } // clang-format off @@ -406,7 +433,7 @@ optional<std::pair<Response, uint64_t>> OfflineDatabase::getTile(const Resource: query.bind(5, tile.z); if (!query.run()) { - return {}; + return nullopt; } Response response; @@ -450,7 +477,7 @@ optional<int64_t> OfflineDatabase::hasTile(const Resource::TileData& tile) { size.bind(5, tile.z); if (!size.run()) { - return {}; + return nullopt; } return size.get<optional<int64_t>>(0); @@ -559,23 +586,30 @@ bool OfflineDatabase::putTile(const Resource::TileData& tile, return true; } -std::vector<OfflineRegion> OfflineDatabase::listRegions() { +std::vector<OfflineRegion> OfflineDatabase::listRegions() try { mapbox::sqlite::Query query{ getStatement("SELECT id, definition, description FROM regions") }; - std::vector<OfflineRegion> result; - while (query.run()) { - result.push_back(OfflineRegion( - query.get<int64_t>(0), - decodeOfflineRegionDefinition(query.get<std::string>(1)), - query.get<std::vector<uint8_t>>(2))); + const auto id = query.get<int64_t>(0); + const auto definition = query.get<std::string>(1); + const auto description = query.get<std::vector<uint8_t>>(2); + try { + OfflineRegion region(id, decodeOfflineRegionDefinition(definition), description); + result.emplace_back(std::move(region)); + } catch (const std::exception& ex) { + // Catch errors from malformed offline region definitions + // and skip them. + Log::Error(Event::General, "%s", ex.what()); + } } - return result; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "list regions"); + return {}; } -OfflineRegion OfflineDatabase::createRegion(const OfflineRegionDefinition& definition, - const OfflineRegionMetadata& metadata) { +optional<OfflineRegion> OfflineDatabase::createRegion(const OfflineRegionDefinition& definition, + const OfflineRegionMetadata& metadata) try { // clang-format off mapbox::sqlite::Query query{ getStatement( "INSERT INTO regions (definition, description) " @@ -585,11 +619,13 @@ OfflineRegion OfflineDatabase::createRegion(const OfflineRegionDefinition& defin query.bind(1, encodeOfflineRegionDefinition(definition)); query.bindBlob(2, metadata); query.run(); - return OfflineRegion(query.lastInsertRowId(), definition, metadata); +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "create region"); + return nullopt; } -OfflineRegionMetadata OfflineDatabase::updateMetadata(const int64_t regionID, const OfflineRegionMetadata& metadata) { +optional<OfflineRegionMetadata> OfflineDatabase::updateMetadata(const int64_t regionID, const OfflineRegionMetadata& metadata) try { // clang-format off mapbox::sqlite::Query query{ getStatement( "UPDATE regions SET description = ?1 " @@ -600,9 +636,12 @@ OfflineRegionMetadata OfflineDatabase::updateMetadata(const int64_t regionID, co query.run(); return metadata; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "update region metadata"); + return nullopt; } -void OfflineDatabase::deleteRegion(OfflineRegion&& region) { +void OfflineDatabase::deleteRegion(OfflineRegion&& region) try { { mapbox::sqlite::Query query{ getStatement("DELETE FROM regions WHERE id = ?") }; query.bind(1, region.getID()); @@ -610,13 +649,17 @@ void OfflineDatabase::deleteRegion(OfflineRegion&& region) { } evict(0); + assert(db); db->exec("PRAGMA incremental_vacuum"); // Ensure that the cached offlineTileCount value is recalculated. offlineMapboxTileCount = {}; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "delete region"); + return; } -optional<std::pair<Response, uint64_t>> OfflineDatabase::getRegionResource(int64_t regionID, const Resource& resource) { +optional<std::pair<Response, uint64_t>> OfflineDatabase::getRegionResource(int64_t regionID, const Resource& resource) try { auto response = getInternal(resource); if (response) { @@ -624,9 +667,12 @@ optional<std::pair<Response, uint64_t>> OfflineDatabase::getRegionResource(int64 } return response; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "read region resource"); + return nullopt; } -optional<int64_t> OfflineDatabase::hasRegionResource(int64_t regionID, const Resource& resource) { +optional<int64_t> OfflineDatabase::hasRegionResource(int64_t regionID, const Resource& resource) try { auto response = hasInternal(resource); if (response) { @@ -634,16 +680,32 @@ optional<int64_t> OfflineDatabase::hasRegionResource(int64_t regionID, const Res } return response; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "query region resource"); + return nullopt; } -uint64_t OfflineDatabase::putRegionResource(int64_t regionID, const Resource& resource, const Response& response) { +uint64_t OfflineDatabase::putRegionResource(int64_t regionID, + const Resource& resource, + const Response& response) try { + if (!db) { + initialize(); + } mapbox::sqlite::Transaction transaction(*db); auto size = putRegionResourceInternal(regionID, resource, response); transaction.commit(); return size; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "write region resource"); + return 0; } -void OfflineDatabase::putRegionResources(int64_t regionID, const std::list<std::tuple<Resource, Response>>& resources, OfflineRegionStatus& status) { +void OfflineDatabase::putRegionResources(int64_t regionID, + const std::list<std::tuple<Resource, Response>>& resources, + OfflineRegionStatus& status) try { + if (!db) { + initialize(); + } mapbox::sqlite::Transaction transaction(*db); for (const auto& elem : resources) { @@ -667,6 +729,8 @@ void OfflineDatabase::putRegionResources(int64_t regionID, const std::list<std:: // Commit the completed batch transaction.commit(); +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "write region resources"); } uint64_t OfflineDatabase::putRegionResourceInternal(int64_t regionID, const Resource& resource, const Response& response) { @@ -755,7 +819,7 @@ bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) { mapbox::sqlite::Query selectQuery{ getStatement( "SELECT region_id " "FROM region_resources, resources " - "WHERE region_id != ?1 " + "WHERE region_id != ?1 " " AND resources.url = ?2 " "LIMIT 1 ") }; // clang-format on @@ -766,15 +830,18 @@ bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) { } } -OfflineRegionDefinition OfflineDatabase::getRegionDefinition(int64_t regionID) { +optional<OfflineRegionDefinition> OfflineDatabase::getRegionDefinition(int64_t regionID) try { mapbox::sqlite::Query query{ getStatement("SELECT definition FROM regions WHERE id = ?1") }; query.bind(1, regionID); query.run(); return decodeOfflineRegionDefinition(query.get<std::string>(0)); +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "load region"); + return nullopt; } -OfflineRegionStatus OfflineDatabase::getRegionCompletedStatus(int64_t regionID) { +optional<OfflineRegionStatus> OfflineDatabase::getRegionCompletedStatus(int64_t regionID) try { OfflineRegionStatus result; std::tie(result.completedResourceCount, result.completedResourceSize) @@ -786,6 +853,9 @@ OfflineRegionStatus OfflineDatabase::getRegionCompletedStatus(int64_t regionID) result.completedResourceSize += result.completedTileSize; return result; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "get region status"); + return nullopt; } std::pair<int64_t, int64_t> OfflineDatabase::getCompletedResourceCountAndSize(int64_t regionID) { @@ -920,7 +990,7 @@ bool OfflineDatabase::offlineMapboxTileCountLimitExceeded() { return getOfflineMapboxTileCount() >= offlineMapboxTileCountLimit; } -uint64_t OfflineDatabase::getOfflineMapboxTileCount() { +uint64_t OfflineDatabase::getOfflineMapboxTileCount() try { // Calculating this on every call would be much simpler than caching and // manually updating the value, but it would make offline downloads an O(n²) // operation, because the database query below involves an index scan of @@ -942,6 +1012,9 @@ uint64_t OfflineDatabase::getOfflineMapboxTileCount() { offlineMapboxTileCount = query.get<int64_t>(0); return *offlineMapboxTileCount; +} catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "get offline Mapbox tile count"); + return std::numeric_limits<uint64_t>::max(); } bool OfflineDatabase::exceedsOfflineMapboxTileCountLimit(const Resource& resource) { |