From 543a5ea627984e20ccb0c3688165c4e579e1873d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Konstantin=20K=C3=A4fer?= Date: Wed, 7 Mar 2018 12:21:06 +0100 Subject: [core] harden OfflineDatabase - gracefully handles unreadable or corrupt files - gracefully handles temporary read or write errors, e.g. when the file system becomes unavailable - gracefully handles full disks - adds tests for all of these situations using a ramdisk for simulating full disks, or read only file systems --- cmake/test-files.cmake | 2 + include/mbgl/util/logging.hpp | 4 +- platform/default/mbgl/storage/offline_database.cpp | 1044 ++++++++++++-------- platform/default/mbgl/storage/offline_database.hpp | 18 +- platform/default/sqlite3.cpp | 110 +-- platform/default/sqlite3.hpp | 9 +- platform/qt/src/sqlite3.cpp | 29 +- src/mbgl/util/logging.cpp | 10 +- test/fixtures/offline_database/corrupt-delayed.db | Bin 0 -> 19456 bytes .../fixtures/offline_database/corrupt-immediate.db | Bin 0 -> 4096 bytes test/scripts/ramdisk_darwin.sh | 50 + test/src/mbgl/test/fixture_log_observer.cpp | 2 +- test/src/mbgl/test/ramdisk.cpp | 60 ++ test/src/mbgl/test/ramdisk.hpp | 39 + test/storage/offline_database.test.cpp | 372 ++++++- test/storage/sqlite.test.cpp | 2 +- 16 files changed, 1191 insertions(+), 560 deletions(-) create mode 100644 test/fixtures/offline_database/corrupt-delayed.db create mode 100644 test/fixtures/offline_database/corrupt-immediate.db create mode 100755 test/scripts/ramdisk_darwin.sh create mode 100644 test/src/mbgl/test/ramdisk.cpp create mode 100644 test/src/mbgl/test/ramdisk.hpp diff --git a/cmake/test-files.cmake b/cmake/test-files.cmake index 2aadfa8a7f..5aca26ada1 100644 --- a/cmake/test-files.cmake +++ b/cmake/test-files.cmake @@ -100,6 +100,8 @@ set(MBGL_TEST_FILES test/src/mbgl/test/fixture_log_observer.hpp test/src/mbgl/test/getrss.cpp test/src/mbgl/test/getrss.hpp + test/src/mbgl/test/ramdisk.cpp + test/src/mbgl/test/ramdisk.hpp test/src/mbgl/test/stub_file_source.cpp test/src/mbgl/test/stub_file_source.hpp test/src/mbgl/test/stub_geometry_tile_feature.hpp diff --git a/include/mbgl/util/logging.hpp b/include/mbgl/util/logging.hpp index d072673e76..03db2d7462 100644 --- a/include/mbgl/util/logging.hpp +++ b/include/mbgl/util/logging.hpp @@ -67,8 +67,8 @@ public: private: static void record(EventSeverity severity, Event event, const std::string &msg); - static void record(EventSeverity severity, Event event, const char* format, ...); - static void record(EventSeverity severity, Event event, int64_t code); + static void record(EventSeverity severity, Event event, const char* format = "", ...); + static void record(EventSeverity severity, Event event, int64_t code, const char* format = "", ...); static void record(EventSeverity severity, Event event, int64_t code, const std::string &msg); // This method is the data sink that must be implemented by each platform we diff --git a/platform/default/mbgl/storage/offline_database.cpp b/platform/default/mbgl/storage/offline_database.cpp index 6edd051c43..69768b1b0a 100644 --- a/platform/default/mbgl/storage/offline_database.cpp +++ b/platform/default/mbgl/storage/offline_database.cpp @@ -13,76 +13,72 @@ namespace mbgl { OfflineDatabase::OfflineDatabase(std::string path_, uint64_t maximumCacheSize_) : path(std::move(path_)), maximumCacheSize(maximumCacheSize_) { - ensureSchema(); + initialize(); } OfflineDatabase::~OfflineDatabase() { - // Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we - // can't throw anything. - try { - statements.clear(); - db.reset(); - } catch (mapbox::sqlite::Exception& ex) { - Log::Error(Event::Database, (int)ex.code, ex.what()); - } + closeDatabase(); } -void OfflineDatabase::connect(int flags) { - db = std::make_unique(path.c_str(), flags); - db->setBusyTimeout(Milliseconds::max()); - db->exec("PRAGMA foreign_keys = ON"); +void OfflineDatabase::handleError(const mapbox::sqlite::Exception& ex) { + Log::Error(Event::Database, (int)ex.code, ex.what()); + + if (ex.code == mapbox::sqlite::ResultCode::Corrupt || + ex.code == mapbox::sqlite::ResultCode::NotADB) { + removeExisting(); + } } -void OfflineDatabase::ensureSchema() { - if (path != ":memory:") { +bool OfflineDatabase::initialize() { + for (uint32_t tries = 0; tries < 2; tries++) { try { - connect(mapbox::sqlite::ReadWrite); - - switch (userVersion()) { - case 0: break; // cache-only database; ok to delete - case 1: break; // cache-only database; ok to delete - case 2: migrateToVersion3(); // fall through - case 3: // no-op and fall through - case 4: migrateToVersion5(); // fall through - case 5: migrateToVersion6(); // fall through - case 6: return; - default: break; // downgrade, delete the database - } - - removeExisting(); - connect(mapbox::sqlite::ReadWrite | mapbox::sqlite::Create); + openDatabase(); + return true; } catch (mapbox::sqlite::Exception& ex) { - if (ex.code != mapbox::sqlite::ResultCode::CantOpen && ex.code != mapbox::sqlite::ResultCode::NotADB) { - Log::Error(Event::Database, "Unexpected error connecting to database: %s", ex.what()); - throw; - } - - try { - if (ex.code == mapbox::sqlite::ResultCode::NotADB) { - removeExisting(); - } - connect(mapbox::sqlite::ReadWrite | mapbox::sqlite::Create); - } catch (...) { - Log::Error(Event::Database, "Unexpected error creating database: %s", util::toString(std::current_exception()).c_str()); - throw; - } + handleError(ex); + closeDatabase(); } } + return false; +} - try { - #include "offline_schema.cpp.include" +void OfflineDatabase::openDatabase() { + assert(!db); + db = std::make_unique(path.c_str(), mapbox::sqlite::ReadWriteCreate); + db->setBusyTimeout(Milliseconds::max()); + db->exec("PRAGMA foreign_keys = ON"); - connect(mapbox::sqlite::ReadWrite | mapbox::sqlite::Create); + const auto version = userVersion(); + switch (version) { + case 0: // New database, or a very old database that uses different tables. + createSchema(); + break; + + // Migrate existing database. We're using fall throughs. + case 2: + migrateToVersion3(); + case 3: + case 4: + migrateToVersion5(); + case 5: + migrateToVersion6(); + case 6: + // Current database version + break; + + // Delete unknown database version by treating it as a corrupt database. + default: + throw mapbox::sqlite::Exception{ mapbox::sqlite::ResultCode::Corrupt, + "Unknown database version " + util::toString(version) }; + } +} - // If you change the schema you must write a migration from the previous version. - db->exec("PRAGMA auto_vacuum = INCREMENTAL"); - db->exec("PRAGMA journal_mode = DELETE"); - db->exec("PRAGMA synchronous = FULL"); - db->exec(schema); - db->exec("PRAGMA user_version = 6"); - } catch (...) { - Log::Error(Event::Database, "Unexpected error creating database schema: %s", util::toString(std::current_exception()).c_str()); - throw; +void OfflineDatabase::closeDatabase() { + if (db) { + statements.clear(); + db.reset(); + } else { + assert(statements.empty()); } } @@ -93,8 +89,7 @@ int OfflineDatabase::userVersion() { void OfflineDatabase::removeExisting() { Log::Warning(Event::Database, "Removing existing incompatible offline database"); - statements.clear(); - db.reset(); + closeDatabase(); try { util::deleteFile(path); @@ -103,6 +98,19 @@ void OfflineDatabase::removeExisting() { } } +void OfflineDatabase::createSchema() { + db->exec("PRAGMA auto_vacuum = INCREMENTAL"); + db->exec("PRAGMA journal_mode = DELETE"); + db->exec("PRAGMA synchronous = FULL"); + + #include "offline_schema.cpp.include" + mapbox::sqlite::Transaction transaction(*db); + db->exec(schema); + transaction.commit(); + + db->exec("PRAGMA user_version = 6"); +} + void OfflineDatabase::migrateToVersion3() { db->exec("PRAGMA auto_vacuum = INCREMENTAL"); db->exec("VACUUM"); @@ -130,6 +138,7 @@ void OfflineDatabase::migrateToVersion6() { } mapbox::sqlite::Statement& OfflineDatabase::getStatement(const char* sql) { + assert(db); auto it = statements.find(sql); if (it == statements.end()) { it = statements.emplace(sql, std::make_unique(*db, sql)).first; @@ -169,14 +178,14 @@ std::pair OfflineDatabase::putInternal(const Resource& resource, return { false, 0 }; } - std::string compressedData; + std::shared_ptr data; bool compressed = false; - uint64_t size = 0; - + size_t size = 0; if (response.data) { - compressedData = util::compress(*response.data); + auto compressedData = util::compress(*response.data); compressed = compressedData.size() < response.data->size(); - size = compressed ? compressedData.size() : response.data->size(); + data = compressed ? std::make_shared(std::move(compressedData)) : response.data; + size = data->size(); } if (evict_ && !evict(size)) { @@ -184,171 +193,209 @@ std::pair OfflineDatabase::putInternal(const Resource& resource, return { false, 0 }; } - bool inserted; - if (resource.kind == Resource::Kind::Tile) { assert(resource.tileData); - inserted = putTile(*resource.tileData, response, - compressed ? compressedData : response.data ? *response.data : "", - compressed); + return putTile(*resource.tileData, response, data, compressed); } else { - inserted = putResource(resource, response, - compressed ? compressedData : response.data ? *response.data : "", - compressed); + return putResource(resource, response, data, compressed); } - - return { inserted, size }; } optional> OfflineDatabase::getResource(const Resource& resource) { + if (!db && !initialize()) { + return {}; + } + // Update accessed timestamp used for LRU eviction. - { + try { mapbox::sqlite::Query accessedQuery{ getStatement("UPDATE resources SET accessed = ?1 WHERE url = ?2") }; accessedQuery.bind(1, util::now()); accessedQuery.bind(2, resource.url); accessedQuery.run(); + } catch (mapbox::sqlite::Exception& ex) { + handleError(ex); + // Ignore access update failures so that we can still read from a database even if the disk + // is full (and we can't create a journal file), or when the file system is readonly. + + // handleError() may close the database, so ensure that we try to reopen/recreate it before + // proceeding. + if (!db && !initialize()) { + return {}; + } } - // clang-format off - mapbox::sqlite::Query query{ getStatement( - // 0 1 2 3 4 5 - "SELECT etag, expires, must_revalidate, modified, data, compressed " - "FROM resources " - "WHERE url = ?") }; - // clang-format on + try { + // clang-format off + mapbox::sqlite::Query query{ getStatement( + // 0 1 2 3 4 5 + "SELECT etag, expires, must_revalidate, modified, data, compressed " + "FROM resources " + "WHERE url = ?") }; + // clang-format on - query.bind(1, resource.url); + query.bind(1, resource.url); - if (!query.run()) { - return {}; - } - - Response response; - uint64_t size = 0; + if (!query.run()) { + return {}; + } - response.etag = query.get>(0); - response.expires = query.get>(1); - response.mustRevalidate = query.get(2); - response.modified = query.get>(3); + Response response; + uint64_t size = 0; + + response.etag = query.get>(0); + response.expires = query.get>(1); + response.mustRevalidate = query.get(2); + response.modified = query.get>(3); + + auto data = query.get>(4); + if (!data) { + response.noContent = true; + } else if (query.get(5)) { + response.data = std::make_shared(util::decompress(*data)); + size = data->length(); + } else { + response.data = std::make_shared(*data); + size = data->length(); + } - auto data = query.get>(4); - if (!data) { - response.noContent = true; - } else if (query.get(5)) { - response.data = std::make_shared(util::decompress(*data)); - size = data->length(); - } else { - response.data = std::make_shared(*data); - size = data->length(); + return std::make_pair(response, size); + } catch (mapbox::sqlite::Exception& ex) { + handleError(ex); + return {}; } - - return std::make_pair(response, size); } optional OfflineDatabase::hasResource(const Resource& resource) { - mapbox::sqlite::Query query{ getStatement("SELECT length(data) FROM resources WHERE url = ?") }; - query.bind(1, resource.url); - if (!query.run()) { + if (!db && !initialize()) { return {}; } - return query.get>(0); + try { + mapbox::sqlite::Query query{ getStatement("SELECT length(data) FROM resources WHERE url = ?") }; + query.bind(1, resource.url); + if (query.run()) { + return query.get>(0); + } + } catch (mapbox::sqlite::Exception& ex) { + handleError(ex); + } + + return {}; } -bool OfflineDatabase::putResource(const Resource& resource, - const Response& response, - const std::string& data, - bool compressed) { +std::pair OfflineDatabase::putResource(const Resource& resource, + const Response& response, + const std::shared_ptr& data, + bool compressed) { + if (!db && !initialize()) { + return { false, 0 }; + } + if (response.notModified) { + try { + // clang-format off + mapbox::sqlite::Query notModifiedQuery{ getStatement( + "UPDATE resources " + "SET accessed = ?1, " + " expires = ?2, " + " must_revalidate = ?3 " + "WHERE url = ?4 ") }; + // clang-format on + + notModifiedQuery.bind(1, util::now()); + notModifiedQuery.bind(2, response.expires); + notModifiedQuery.bind(3, response.mustRevalidate); + notModifiedQuery.bind(4, resource.url); + notModifiedQuery.run(); + return { false, data ? data->size() : 0 }; + } catch (mapbox::sqlite::Exception& ex) { + handleError(ex); + return { false, 0 }; + } + } + + try { + // We can't use REPLACE because it would change the id value. + + // Begin an immediate-mode transaction to ensure that two writers do not attempt + // to INSERT a resource at the same moment. + assert(db); + mapbox::sqlite::Transaction transaction(*db, mapbox::sqlite::Transaction::Immediate); + // clang-format off - mapbox::sqlite::Query notModifiedQuery{ getStatement( + mapbox::sqlite::Query updateQuery{ getStatement( "UPDATE resources " - "SET accessed = ?1, " - " expires = ?2, " - " must_revalidate = ?3 " - "WHERE url = ?4 ") }; + "SET kind = ?1, " + " etag = ?2, " + " expires = ?3, " + " must_revalidate = ?4, " + " modified = ?5, " + " accessed = ?6, " + " data = ?7, " + " compressed = ?8 " + "WHERE url = ?9 ") }; // clang-format on - notModifiedQuery.bind(1, util::now()); - notModifiedQuery.bind(2, response.expires); - notModifiedQuery.bind(3, response.mustRevalidate); - notModifiedQuery.bind(4, resource.url); - notModifiedQuery.run(); - return false; - } - - // We can't use REPLACE because it would change the id value. + updateQuery.bind(1, int(resource.kind)); + updateQuery.bind(2, response.etag); + updateQuery.bind(3, response.expires); + updateQuery.bind(4, response.mustRevalidate); + updateQuery.bind(5, response.modified); + updateQuery.bind(6, util::now()); + updateQuery.bind(9, resource.url); + + if (response.noContent || !data) { + updateQuery.bind(7, nullptr); + updateQuery.bind(8, false); + } else { + updateQuery.bindBlob(7, data->data(), data->size(), false); + updateQuery.bind(8, compressed); + } - // Begin an immediate-mode transaction to ensure that two writers do not attempt - // to INSERT a resource at the same moment. - mapbox::sqlite::Transaction transaction(*db, mapbox::sqlite::Transaction::Immediate); + updateQuery.run(); + if (updateQuery.changes() != 0) { + transaction.commit(); + return { false, data ? data->size() : 0 }; + } - // clang-format off - mapbox::sqlite::Query updateQuery{ getStatement( - "UPDATE resources " - "SET kind = ?1, " - " etag = ?2, " - " expires = ?3, " - " must_revalidate = ?4, " - " modified = ?5, " - " accessed = ?6, " - " data = ?7, " - " compressed = ?8 " - "WHERE url = ?9 ") }; - // clang-format on + // clang-format off + mapbox::sqlite::Query insertQuery{ getStatement( + "INSERT INTO resources (url, kind, etag, expires, must_revalidate, modified, accessed, data, compressed) " + "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) ") }; + // clang-format on - updateQuery.bind(1, int(resource.kind)); - updateQuery.bind(2, response.etag); - updateQuery.bind(3, response.expires); - updateQuery.bind(4, response.mustRevalidate); - updateQuery.bind(5, response.modified); - updateQuery.bind(6, util::now()); - updateQuery.bind(9, resource.url); - - if (response.noContent) { - updateQuery.bind(7, nullptr); - updateQuery.bind(8, false); - } else { - updateQuery.bindBlob(7, data.data(), data.size(), false); - updateQuery.bind(8, compressed); - } + insertQuery.bind(1, resource.url); + insertQuery.bind(2, int(resource.kind)); + insertQuery.bind(3, response.etag); + insertQuery.bind(4, response.expires); + insertQuery.bind(5, response.mustRevalidate); + insertQuery.bind(6, response.modified); + insertQuery.bind(7, util::now()); + + if (response.noContent || !data) { + insertQuery.bind(8, nullptr); + insertQuery.bind(9, false); + } else { + insertQuery.bindBlob(8, data->data(), data->size(), false); + insertQuery.bind(9, compressed); + } - updateQuery.run(); - if (updateQuery.changes() != 0) { + insertQuery.run(); transaction.commit(); - return false; - } - // clang-format off - mapbox::sqlite::Query insertQuery{ getStatement( - "INSERT INTO resources (url, kind, etag, expires, must_revalidate, modified, accessed, data, compressed) " - "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) ") }; - // clang-format on - - insertQuery.bind(1, resource.url); - insertQuery.bind(2, int(resource.kind)); - insertQuery.bind(3, response.etag); - insertQuery.bind(4, response.expires); - insertQuery.bind(5, response.mustRevalidate); - insertQuery.bind(6, response.modified); - insertQuery.bind(7, util::now()); - - if (response.noContent) { - insertQuery.bind(8, nullptr); - insertQuery.bind(9, false); - } else { - insertQuery.bindBlob(8, data.data(), data.size(), false); - insertQuery.bind(9, compressed); + return { true, data ? data->size() : 0 }; + } catch (mapbox::sqlite::Exception& ex) { + handleError(ex); + return { false, 0 }; } - - insertQuery.run(); - transaction.commit(); - - return true; } optional> OfflineDatabase::getTile(const Resource::TileData& tile) { - { + if (!db && !initialize()) { + return {}; + } + + try { // clang-format off mapbox::sqlite::Query accessedQuery{ getStatement( "UPDATE tiles " @@ -367,184 +414,222 @@ optional> OfflineDatabase::getTile(const Resource: accessedQuery.bind(5, tile.y); accessedQuery.bind(6, tile.z); accessedQuery.run(); + } catch (mapbox::sqlite::Exception& ex) { + handleError(ex); + // Ignore access update failures so that we can still read from a database even if the disk + // is full (and we can't create a journal file), or when the file system is readonly. + + // handleError() may close the database, so ensure that we try to reopen/recreate it before + // proceeding. + if (!db && !initialize()) { + return {}; + } } - // clang-format off - mapbox::sqlite::Query query{ getStatement( - // 0 1 2, 3, 4, 5 - "SELECT etag, expires, must_revalidate, modified, data, compressed " - "FROM tiles " - "WHERE url_template = ?1 " - " AND pixel_ratio = ?2 " - " AND x = ?3 " - " AND y = ?4 " - " AND z = ?5 ") }; - // clang-format on - - query.bind(1, tile.urlTemplate); - query.bind(2, tile.pixelRatio); - query.bind(3, tile.x); - query.bind(4, tile.y); - query.bind(5, tile.z); + try { + // clang-format off + mapbox::sqlite::Query query{ getStatement( + // 0 1 2, 3, 4, 5 + "SELECT etag, expires, must_revalidate, modified, data, compressed " + "FROM tiles " + "WHERE url_template = ?1 " + " AND pixel_ratio = ?2 " + " AND x = ?3 " + " AND y = ?4 " + " AND z = ?5 ") }; + // clang-format on - if (!query.run()) { - return {}; - } + query.bind(1, tile.urlTemplate); + query.bind(2, tile.pixelRatio); + query.bind(3, tile.x); + query.bind(4, tile.y); + query.bind(5, tile.z); - Response response; - uint64_t size = 0; + if (!query.run()) { + return {}; + } - response.etag = query.get>(0); - response.expires = query.get>(1); - response.mustRevalidate = query.get(2); - response.modified = query.get>(3); + Response response; + uint64_t size = 0; + + response.etag = query.get>(0); + response.expires = query.get>(1); + response.mustRevalidate = query.get(2); + response.modified = query.get>(3); + + optional data = query.get>(4); + if (!data) { + response.noContent = true; + } else if (query.get(5)) { + response.data = std::make_shared(util::decompress(*data)); + size = data->length(); + } else { + response.data = std::make_shared(*data); + size = data->length(); + } - optional data = query.get>(4); - if (!data) { - response.noContent = true; - } else if (query.get(5)) { - response.data = std::make_shared(util::decompress(*data)); - size = data->length(); - } else { - response.data = std::make_shared(*data); - size = data->length(); + return std::make_pair(response, size); + } catch (mapbox::sqlite::Exception& ex) { + handleError(ex); + return {}; } - - return std::make_pair(response, size); } optional OfflineDatabase::hasTile(const Resource::TileData& tile) { - // clang-format off - mapbox::sqlite::Query size{ getStatement( - "SELECT length(data) " - "FROM tiles " - "WHERE url_template = ?1 " - " AND pixel_ratio = ?2 " - " AND x = ?3 " - " AND y = ?4 " - " AND z = ?5 ") }; - // clang-format on + if (!db && !initialize()) { + return {}; + } + + try { + // clang-format off + mapbox::sqlite::Query size{ getStatement( + "SELECT length(data) " + "FROM tiles " + "WHERE url_template = ?1 " + " AND pixel_ratio = ?2 " + " AND x = ?3 " + " AND y = ?4 " + " AND z = ?5 ") }; + // clang-format on - size.bind(1, tile.urlTemplate); - size.bind(2, tile.pixelRatio); - size.bind(3, tile.x); - size.bind(4, tile.y); - size.bind(5, tile.z); + size.bind(1, tile.urlTemplate); + size.bind(2, tile.pixelRatio); + size.bind(3, tile.x); + size.bind(4, tile.y); + size.bind(5, tile.z); - if (!size.run()) { - return {}; + if (size.run()) { + return size.get>(0); + } + } catch (mapbox::sqlite::Exception& ex) { + handleError(ex); } - return size.get>(0); + return {}; } -bool OfflineDatabase::putTile(const Resource::TileData& tile, - const Response& response, - const std::string& data, - bool compressed) { +std::pair OfflineDatabase::putTile(const Resource::TileData& tile, + const Response& response, + const std::shared_ptr& data, + bool compressed) { + if (!db && !initialize()) { + return { false, 0 }; + } + if (response.notModified) { + try { + // clang-format off + mapbox::sqlite::Query notModifiedQuery{ getStatement( + "UPDATE tiles " + "SET accessed = ?1, " + " expires = ?2, " + " must_revalidate = ?3 " + "WHERE url_template = ?4 " + " AND pixel_ratio = ?5 " + " AND x = ?6 " + " AND y = ?7 " + " AND z = ?8 ") }; + // clang-format on + + notModifiedQuery.bind(1, util::now()); + notModifiedQuery.bind(2, response.expires); + notModifiedQuery.bind(3, response.mustRevalidate); + notModifiedQuery.bind(4, tile.urlTemplate); + notModifiedQuery.bind(5, tile.pixelRatio); + notModifiedQuery.bind(6, tile.x); + notModifiedQuery.bind(7, tile.y); + notModifiedQuery.bind(8, tile.z); + notModifiedQuery.run(); + return { false, data ? data->size() : 0 }; + } catch (mapbox::sqlite::Exception& ex) { + handleError(ex); + return { false, 0 }; + } + } + + try { + // We can't use REPLACE because it would change the id value. + + // Begin an immediate-mode transaction to ensure that two writers do not attempt + // to INSERT a resource at the same moment. + assert(db); + mapbox::sqlite::Transaction transaction(*db, mapbox::sqlite::Transaction::Immediate); + // clang-format off - mapbox::sqlite::Query notModifiedQuery{ getStatement( + mapbox::sqlite::Query updateQuery{ getStatement( "UPDATE tiles " - "SET accessed = ?1, " - " expires = ?2, " - " must_revalidate = ?3 " - "WHERE url_template = ?4 " - " AND pixel_ratio = ?5 " - " AND x = ?6 " - " AND y = ?7 " - " AND z = ?8 ") }; + "SET modified = ?1, " + " etag = ?2, " + " expires = ?3, " + " must_revalidate = ?4, " + " accessed = ?5, " + " data = ?6, " + " compressed = ?7 " + "WHERE url_template = ?8 " + " AND pixel_ratio = ?9 " + " AND x = ?10 " + " AND y = ?11 " + " AND z = ?12 ") }; // clang-format on - notModifiedQuery.bind(1, util::now()); - notModifiedQuery.bind(2, response.expires); - notModifiedQuery.bind(3, response.mustRevalidate); - notModifiedQuery.bind(4, tile.urlTemplate); - notModifiedQuery.bind(5, tile.pixelRatio); - notModifiedQuery.bind(6, tile.x); - notModifiedQuery.bind(7, tile.y); - notModifiedQuery.bind(8, tile.z); - notModifiedQuery.run(); - return false; - } - - // We can't use REPLACE because it would change the id value. + updateQuery.bind(1, response.modified); + updateQuery.bind(2, response.etag); + updateQuery.bind(3, response.expires); + updateQuery.bind(4, response.mustRevalidate); + updateQuery.bind(5, util::now()); + updateQuery.bind(8, tile.urlTemplate); + updateQuery.bind(9, tile.pixelRatio); + updateQuery.bind(10, tile.x); + updateQuery.bind(11, tile.y); + updateQuery.bind(12, tile.z); + + if (response.noContent || !data) { + updateQuery.bind(6, nullptr); + updateQuery.bind(7, false); + } else { + updateQuery.bindBlob(6, data->data(), data->size(), false); + updateQuery.bind(7, compressed); + } - // Begin an immediate-mode transaction to ensure that two writers do not attempt - // to INSERT a resource at the same moment. - mapbox::sqlite::Transaction transaction(*db, mapbox::sqlite::Transaction::Immediate); + updateQuery.run(); + if (updateQuery.changes() != 0) { + transaction.commit(); + return { false, data ? data->size() : 0 }; + } - // clang-format off - mapbox::sqlite::Query updateQuery{ getStatement( - "UPDATE tiles " - "SET modified = ?1, " - " etag = ?2, " - " expires = ?3, " - " must_revalidate = ?4, " - " accessed = ?5, " - " data = ?6, " - " compressed = ?7 " - "WHERE url_template = ?8 " - " AND pixel_ratio = ?9 " - " AND x = ?10 " - " AND y = ?11 " - " AND z = ?12 ") }; - // clang-format on + // clang-format off + mapbox::sqlite::Query insertQuery{ getStatement( + "INSERT INTO tiles (url_template, pixel_ratio, x, y, z, modified, must_revalidate, etag, expires, accessed, data, compressed) " + "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)") }; + // clang-format on - updateQuery.bind(1, response.modified); - updateQuery.bind(2, response.etag); - updateQuery.bind(3, response.expires); - updateQuery.bind(4, response.mustRevalidate); - updateQuery.bind(5, util::now()); - updateQuery.bind(8, tile.urlTemplate); - updateQuery.bind(9, tile.pixelRatio); - updateQuery.bind(10, tile.x); - updateQuery.bind(11, tile.y); - updateQuery.bind(12, tile.z); - - if (response.noContent) { - updateQuery.bind(6, nullptr); - updateQuery.bind(7, false); - } else { - updateQuery.bindBlob(6, data.data(), data.size(), false); - updateQuery.bind(7, compressed); - } + insertQuery.bind(1, tile.urlTemplate); + insertQuery.bind(2, tile.pixelRatio); + insertQuery.bind(3, tile.x); + insertQuery.bind(4, tile.y); + insertQuery.bind(5, tile.z); + insertQuery.bind(6, response.modified); + insertQuery.bind(7, response.mustRevalidate); + insertQuery.bind(8, response.etag); + insertQuery.bind(9, response.expires); + insertQuery.bind(10, util::now()); + + if (response.noContent || !data) { + insertQuery.bind(11, nullptr); + insertQuery.bind(12, false); + } else { + insertQuery.bindBlob(11, data->data(), data->size(), false); + insertQuery.bind(12, compressed); + } - updateQuery.run(); - if (updateQuery.changes() != 0) { + insertQuery.run(); transaction.commit(); - return false; - } - - // clang-format off - mapbox::sqlite::Query insertQuery{ getStatement( - "INSERT INTO tiles (url_template, pixel_ratio, x, y, z, modified, must_revalidate, etag, expires, accessed, data, compressed) " - "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)") }; - // clang-format on - insertQuery.bind(1, tile.urlTemplate); - insertQuery.bind(2, tile.pixelRatio); - insertQuery.bind(3, tile.x); - insertQuery.bind(4, tile.y); - insertQuery.bind(5, tile.z); - insertQuery.bind(6, response.modified); - insertQuery.bind(7, response.mustRevalidate); - insertQuery.bind(8, response.etag); - insertQuery.bind(9, response.expires); - insertQuery.bind(10, util::now()); - - if (response.noContent) { - insertQuery.bind(11, nullptr); - insertQuery.bind(12, false); - } else { - insertQuery.bindBlob(11, data.data(), data.size(), false); - insertQuery.bind(12, compressed); + return { true, data ? data->size() : 0 }; + } catch (mapbox::sqlite::Exception& ex) { + handleError(ex); + return { false, 0 }; } - - insertQuery.run(); - transaction.commit(); - - return true; } std::vector OfflineDatabase::listRegions() { @@ -605,6 +690,10 @@ void OfflineDatabase::deleteRegion(OfflineRegion&& region) { } optional> OfflineDatabase::getRegionResource(int64_t regionID, const Resource& resource) { + if (!db && !initialize()) { + return {}; + } + auto response = getInternal(resource); if (response) { @@ -638,8 +727,22 @@ uint64_t OfflineDatabase::putRegionResource(int64_t regionID, const Resource& re return size; } -bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) { +bool OfflineDatabase::markUsed(int64_t regionID, const mbgl::Resource& resource) { if (resource.kind == Resource::Kind::Tile) { + return markTileUsed(regionID, resource); + } else { + return markResourceUsed(regionID, resource); + } +} + +bool OfflineDatabase::markTileUsed(int64_t regionID, const Resource& resource) { + assert(resource.kind == Resource::Kind::Tile); + + if (!db && !initialize()) { + return false; + } + + try { // clang-format off mapbox::sqlite::Query insertQuery{ getStatement( "INSERT OR IGNORE INTO region_tiles (region_id, tile_id) " @@ -664,7 +767,17 @@ bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) { if (insertQuery.changes() == 0) { return false; } + } catch (mapbox::sqlite::Exception& ex) { + handleError(ex); + // handleError() may close the database, so ensure that we try to reopen/recreate it + // before proceeding. + if (!db && !initialize()) { + return false; + } + } + + try { // clang-format off mapbox::sqlite::Query selectQuery{ getStatement( "SELECT region_id " @@ -678,6 +791,7 @@ bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) { "LIMIT 1 ") }; // clang-format on + const Resource::TileData& tile = *resource.tileData; selectQuery.bind(1, regionID); selectQuery.bind(2, tile.urlTemplate); selectQuery.bind(3, tile.pixelRatio); @@ -685,7 +799,20 @@ bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) { selectQuery.bind(5, tile.y); selectQuery.bind(6, tile.z); return !selectQuery.run(); - } else { + } catch (mapbox::sqlite::Exception& ex) { + handleError(ex); + return false; + } +} + +bool OfflineDatabase::markResourceUsed(int64_t regionID, const Resource& resource) { + assert(resource.kind != Resource::Kind::Tile); + + if (!db && !initialize()) { + return false; + } + + try { // clang-format off mapbox::sqlite::Query insertQuery{ getStatement( "INSERT OR IGNORE INTO region_resources (region_id, resource_id) " @@ -701,7 +828,17 @@ bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) { if (insertQuery.changes() == 0) { return false; } + } catch (mapbox::sqlite::Exception& ex) { + handleError(ex); + + // handleError() may close the database, so ensure that we try to reopen/recreate it + // before proceeding. + if (!db && !initialize()) { + return false; + } + } + try { // clang-format off mapbox::sqlite::Query selectQuery{ getStatement( "SELECT region_id " @@ -714,15 +851,28 @@ bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) { selectQuery.bind(1, regionID); selectQuery.bind(2, resource.url); return !selectQuery.run(); + } catch (mapbox::sqlite::Exception& ex) { + handleError(ex); + return false; } } OfflineRegionDefinition OfflineDatabase::getRegionDefinition(int64_t regionID) { - mapbox::sqlite::Query query{ getStatement("SELECT definition FROM regions WHERE id = ?1") }; - query.bind(1, regionID); - query.run(); + if (!db && !initialize()) { + throw std::runtime_error("Could not find offline region"); + } + + try { + mapbox::sqlite::Query query{ getStatement("SELECT definition FROM regions WHERE id = ?1") }; + query.bind(1, regionID); + if (query.run()) { + return decodeOfflineRegionDefinition(query.get(0)); + } + } catch (mapbox::sqlite::Exception& ex) { + handleError(ex); + } - return decodeOfflineRegionDefinition(query.get(0)); + throw std::runtime_error("Could not find offline region"); } OfflineRegionStatus OfflineDatabase::getRegionCompletedStatus(int64_t regionID) { @@ -740,29 +890,50 @@ OfflineRegionStatus OfflineDatabase::getRegionCompletedStatus(int64_t regionID) } std::pair OfflineDatabase::getCompletedResourceCountAndSize(int64_t regionID) { - // clang-format off - mapbox::sqlite::Query query{ getStatement( - "SELECT COUNT(*), SUM(LENGTH(data)) " - "FROM region_resources, resources " - "WHERE region_id = ?1 " - "AND resource_id = resources.id ") }; - // clang-format on - query.bind(1, regionID); - query.run(); - return { query.get(0), query.get(1) }; + if (!db && !initialize()) { + return { 0, 0 }; + } + + try { + // clang-format off + mapbox::sqlite::Query query{ getStatement( + "SELECT COUNT(*), SUM(LENGTH(data)) " + "FROM region_resources, resources " + "WHERE region_id = ?1 " + "AND resource_id = resources.id ") }; + // clang-format on + query.bind(1, regionID); + if (query.run()) { + return { query.get(0), query.get(1) }; + } + } catch (mapbox::sqlite::Exception& ex) { + handleError(ex); + } + + return { 0, 0 }; } std::pair OfflineDatabase::getCompletedTileCountAndSize(int64_t regionID) { - // clang-format off - mapbox::sqlite::Query query{ getStatement( - "SELECT COUNT(*), SUM(LENGTH(data)) " - "FROM region_tiles, tiles " - "WHERE region_id = ?1 " - "AND tile_id = tiles.id ") }; - // clang-format on - query.bind(1, regionID); - query.run(); - return { query.get(0), query.get(1) }; + if (!db && !initialize()) { + return { 0, 0 }; + } + + try { + // clang-format off + mapbox::sqlite::Query query{ getStatement( + "SELECT COUNT(*), SUM(LENGTH(data)) " + "FROM region_tiles, tiles " + "WHERE region_id = ?1 " + "AND tile_id = tiles.id ") }; + // clang-format on + query.bind(1, regionID); + query.run(); + return { query.get(0), query.get(1) }; + } catch (mapbox::sqlite::Exception& ex) { + handleError(ex); + } + + return { 0, 0 }; } template @@ -783,80 +954,89 @@ T OfflineDatabase::getPragma(const char* sql) { // delete an arbitrary number of old cache entries. The free pages approach saves // us from calling VACCUM or keeping a running total, which can be costly. bool OfflineDatabase::evict(uint64_t neededFreeSize) { - uint64_t pageSize = getPragma("PRAGMA page_size"); - uint64_t pageCount = getPragma("PRAGMA page_count"); - - auto usedSize = [&] { - return pageSize * (pageCount - getPragma("PRAGMA freelist_count")); - }; + if (!db && !initialize()) { + return false; + } - // The addition of pageSize is a fudge factor to account for non `data` column - // size, and because pages can get fragmented on the database. - while (usedSize() + neededFreeSize + pageSize > maximumCacheSize) { - // clang-format off - mapbox::sqlite::Query accessedQuery{ getStatement( - "SELECT max(accessed) " - "FROM ( " - " SELECT accessed " - " FROM resources " - " LEFT JOIN region_resources " - " ON resource_id = resources.id " - " WHERE resource_id IS NULL " - " UNION ALL " - " SELECT accessed " - " FROM tiles " - " LEFT JOIN region_tiles " - " ON tile_id = tiles.id " - " WHERE tile_id IS NULL " - " ORDER BY accessed ASC LIMIT ?1 " - ") " - ) }; - accessedQuery.bind(1, 50); - // clang-format on - if (!accessedQuery.run()) { - return false; + try { + uint64_t pageSize = getPragma("PRAGMA page_size"); + uint64_t pageCount = getPragma("PRAGMA page_count"); + + auto usedSize = [&] { + return pageSize * (pageCount - getPragma("PRAGMA freelist_count")); + }; + + // The addition of pageSize is a fudge factor to account for non `data` column + // size, and because pages can get fragmented on the database. + while (usedSize() + neededFreeSize + pageSize > maximumCacheSize) { + // clang-format off + mapbox::sqlite::Query accessedQuery{ getStatement( + "SELECT max(accessed) " + "FROM ( " + " SELECT accessed " + " FROM resources " + " LEFT JOIN region_resources " + " ON resource_id = resources.id " + " WHERE resource_id IS NULL " + " UNION ALL " + " SELECT accessed " + " FROM tiles " + " LEFT JOIN region_tiles " + " ON tile_id = tiles.id " + " WHERE tile_id IS NULL " + " ORDER BY accessed ASC LIMIT ?1 " + ") " + ) }; + accessedQuery.bind(1, 50); + // clang-format on + if (!accessedQuery.run()) { + return false; + } + Timestamp accessed = accessedQuery.get(0); + + // clang-format off + mapbox::sqlite::Query resourceQuery{ getStatement( + "DELETE FROM resources " + "WHERE id IN ( " + " SELECT id FROM resources " + " LEFT JOIN region_resources " + " ON resource_id = resources.id " + " WHERE resource_id IS NULL " + " AND accessed <= ?1 " + ") ") }; + // clang-format on + resourceQuery.bind(1, accessed); + resourceQuery.run(); + const uint64_t resourceChanges = resourceQuery.changes(); + + // clang-format off + mapbox::sqlite::Query tileQuery{ getStatement( + "DELETE FROM tiles " + "WHERE id IN ( " + " SELECT id FROM tiles " + " LEFT JOIN region_tiles " + " ON tile_id = tiles.id " + " WHERE tile_id IS NULL " + " AND accessed <= ?1 " + ") ") }; + // clang-format on + tileQuery.bind(1, accessed); + tileQuery.run(); + const uint64_t tileChanges = tileQuery.changes(); + + // The cached value of offlineTileCount does not need to be updated + // here because only non-offline tiles can be removed by eviction. + + if (resourceChanges == 0 && tileChanges == 0) { + return false; + } } - Timestamp accessed = accessedQuery.get(0); - - // clang-format off - mapbox::sqlite::Query resourceQuery{ getStatement( - "DELETE FROM resources " - "WHERE id IN ( " - " SELECT id FROM resources " - " LEFT JOIN region_resources " - " ON resource_id = resources.id " - " WHERE resource_id IS NULL " - " AND accessed <= ?1 " - ") ") }; - // clang-format on - resourceQuery.bind(1, accessed); - resourceQuery.run(); - const uint64_t resourceChanges = resourceQuery.changes(); - - // clang-format off - mapbox::sqlite::Query tileQuery{ getStatement( - "DELETE FROM tiles " - "WHERE id IN ( " - " SELECT id FROM tiles " - " LEFT JOIN region_tiles " - " ON tile_id = tiles.id " - " WHERE tile_id IS NULL " - " AND accessed <= ?1 " - ") ") }; - // clang-format on - tileQuery.bind(1, accessed); - tileQuery.run(); - const uint64_t tileChanges = tileQuery.changes(); - // The cached value of offlineTileCount does not need to be updated - // here because only non-offline tiles can be removed by eviction. - - if (resourceChanges == 0 && tileChanges == 0) { - return false; - } + return true; + } catch (mapbox::sqlite::Exception& ex) { + handleError(ex); + return false; } - - return true; } void OfflineDatabase::setOfflineMapboxTileCountLimit(uint64_t limit) { diff --git a/platform/default/mbgl/storage/offline_database.hpp b/platform/default/mbgl/storage/offline_database.hpp index 9673ad8212..3ad91671e1 100644 --- a/platform/default/mbgl/storage/offline_database.hpp +++ b/platform/default/mbgl/storage/offline_database.hpp @@ -16,6 +16,7 @@ namespace sqlite { class Database; class Statement; class Query; +class Exception; } // namespace sqlite } // namespace mapbox @@ -59,9 +60,12 @@ public: uint64_t getOfflineMapboxTileCount(); private: - void connect(int flags); + void handleError(const mapbox::sqlite::Exception&); + bool initialize(); + void openDatabase(); + void createSchema(); + void closeDatabase(); int userVersion(); - void ensureSchema(); void removeExisting(); void migrateToVersion3(); void migrateToVersion5(); @@ -71,13 +75,13 @@ private: optional> getTile(const Resource::TileData&); optional hasTile(const Resource::TileData&); - bool putTile(const Resource::TileData&, const Response&, - const std::string&, bool compressed); + std::pair + putTile(const Resource::TileData&, const Response&, const std::shared_ptr&, bool compressed); optional> getResource(const Resource&); optional hasResource(const Resource&); - bool putResource(const Resource&, const Response&, - const std::string&, bool compressed); + std::pair + putResource(const Resource&, const Response&, const std::shared_ptr&, bool compressed); optional> getInternal(const Resource&); optional hasInternal(const Resource&); @@ -85,6 +89,8 @@ private: // Return value is true iff the resource was previously unused by any other regions. bool markUsed(int64_t regionID, const Resource&); + bool markTileUsed(int64_t regionID, const Resource&); + bool markResourceUsed(int64_t regionID, const Resource&); std::pair getCompletedResourceCountAndSize(int64_t regionID); std::pair getCompletedTileCountAndSize(int64_t regionID); diff --git a/platform/default/sqlite3.cpp b/platform/default/sqlite3.cpp index 8a567d602e..ffa32db985 100644 --- a/platform/default/sqlite3.cpp +++ b/platform/default/sqlite3.cpp @@ -1,5 +1,7 @@ #include "sqlite3.hpp" -#include + +#include +#include #include #include @@ -7,11 +9,37 @@ #include #include -#include +#include namespace mapbox { namespace sqlite { +static_assert(mbgl::underlying_type(ResultCode::OK) == SQLITE_OK, "error"); +static_assert(mbgl::underlying_type(ResultCode::Error) == SQLITE_ERROR, "error"); +static_assert(mbgl::underlying_type(ResultCode::Internal) == SQLITE_INTERNAL, "error"); +static_assert(mbgl::underlying_type(ResultCode::Perm) == SQLITE_PERM, "error"); +static_assert(mbgl::underlying_type(ResultCode::Abort) == SQLITE_ABORT, "error"); +static_assert(mbgl::underlying_type(ResultCode::Busy) == SQLITE_BUSY, "error"); +static_assert(mbgl::underlying_type(ResultCode::Locked) == SQLITE_LOCKED, "error"); +static_assert(mbgl::underlying_type(ResultCode::NoMem) == SQLITE_NOMEM, "error"); +static_assert(mbgl::underlying_type(ResultCode::ReadOnly) == SQLITE_READONLY, "error"); +static_assert(mbgl::underlying_type(ResultCode::Interrupt) == SQLITE_INTERRUPT, "error"); +static_assert(mbgl::underlying_type(ResultCode::IOErr) == SQLITE_IOERR, "error"); +static_assert(mbgl::underlying_type(ResultCode::Corrupt) == SQLITE_CORRUPT, "error"); +static_assert(mbgl::underlying_type(ResultCode::NotFound) == SQLITE_NOTFOUND, "error"); +static_assert(mbgl::underlying_type(ResultCode::Full) == SQLITE_FULL, "error"); +static_assert(mbgl::underlying_type(ResultCode::CantOpen) == SQLITE_CANTOPEN, "error"); +static_assert(mbgl::underlying_type(ResultCode::Protocol) == SQLITE_PROTOCOL, "error"); +static_assert(mbgl::underlying_type(ResultCode::Schema) == SQLITE_SCHEMA, "error"); +static_assert(mbgl::underlying_type(ResultCode::TooBig) == SQLITE_TOOBIG, "error"); +static_assert(mbgl::underlying_type(ResultCode::Constraint) == SQLITE_CONSTRAINT, "error"); +static_assert(mbgl::underlying_type(ResultCode::Mismatch) == SQLITE_MISMATCH, "error"); +static_assert(mbgl::underlying_type(ResultCode::Misuse) == SQLITE_MISUSE, "error"); +static_assert(mbgl::underlying_type(ResultCode::NoLFS) == SQLITE_NOLFS, "error"); +static_assert(mbgl::underlying_type(ResultCode::Auth) == SQLITE_AUTH, "error"); +static_assert(mbgl::underlying_type(ResultCode::Range) == SQLITE_RANGE, "error"); +static_assert(mbgl::underlying_type(ResultCode::NotADB) == SQLITE_NOTADB, "error"); + class DatabaseImpl { public: DatabaseImpl(const char* filename, int flags) @@ -69,84 +97,8 @@ public: template using optional = std::experimental::optional; -static const char* codeToString(const int err) { - switch (err) { - case SQLITE_OK: return "SQLITE_OK"; - case SQLITE_ERROR: return "SQLITE_ERROR"; - case SQLITE_INTERNAL: return "SQLITE_INTERNAL"; - case SQLITE_PERM: return "SQLITE_PERM"; - case SQLITE_ABORT: return "SQLITE_ABORT"; - case SQLITE_BUSY: return "SQLITE_BUSY"; - case SQLITE_LOCKED: return "SQLITE_LOCKED"; - case SQLITE_NOMEM: return "SQLITE_NOMEM"; - case SQLITE_READONLY: return "SQLITE_READONLY"; - case SQLITE_INTERRUPT: return "SQLITE_INTERRUPT"; - case SQLITE_IOERR: return "SQLITE_IOERR"; - case SQLITE_CORRUPT: return "SQLITE_CORRUPT"; - case SQLITE_NOTFOUND: return "SQLITE_NOTFOUND"; - case SQLITE_FULL: return "SQLITE_FULL"; - case SQLITE_CANTOPEN: return "SQLITE_CANTOPEN"; - case SQLITE_PROTOCOL: return "SQLITE_PROTOCOL"; - case SQLITE_EMPTY: return "SQLITE_EMPTY"; - case SQLITE_SCHEMA: return "SQLITE_SCHEMA"; - case SQLITE_TOOBIG: return "SQLITE_TOOBIG"; - case SQLITE_CONSTRAINT: return "SQLITE_CONSTRAINT"; - case SQLITE_MISMATCH: return "SQLITE_MISMATCH"; - case SQLITE_MISUSE: return "SQLITE_MISUSE"; - case SQLITE_NOLFS: return "SQLITE_NOLFS"; - case SQLITE_AUTH: return "SQLITE_AUTH"; - case SQLITE_FORMAT: return "SQLITE_FORMAT"; - case SQLITE_RANGE: return "SQLITE_RANGE"; - case SQLITE_NOTADB: return "SQLITE_NOTADB"; - case SQLITE_NOTICE: return "SQLITE_NOTICE"; - case SQLITE_WARNING: return "SQLITE_WARNING"; - case SQLITE_ROW: return "SQLITE_ROW"; - case SQLITE_DONE: return "SQLITE_DONE"; - default: return ""; - } -} - static void errorLogCallback(void *, const int err, const char *msg) { - auto severity = mbgl::EventSeverity::Info; - - switch (err) { - case SQLITE_ERROR: // Generic error - case SQLITE_INTERNAL: // Internal logic error in SQLite - case SQLITE_PERM: // Access permission denied - case SQLITE_ABORT: // Callback routine requested an abort - case SQLITE_BUSY: // The database file is locked - case SQLITE_LOCKED: // A table in the database is locked - case SQLITE_NOMEM: // A malloc() failed - case SQLITE_READONLY: // Attempt to write a readonly database - case SQLITE_INTERRUPT: // Operation terminated by sqlite3_interrupt( - case SQLITE_IOERR: // Some kind of disk I/O error occurred - case SQLITE_CORRUPT: // The database disk image is malformed - case SQLITE_NOTFOUND: // Unknown opcode in sqlite3_file_control() - case SQLITE_FULL: // Insertion failed because database is full - case SQLITE_CANTOPEN: // Unable to open the database file - case SQLITE_PROTOCOL: // Database lock protocol error - case SQLITE_EMPTY: // Internal use only - case SQLITE_SCHEMA: // The database schema changed - case SQLITE_TOOBIG: // String or BLOB exceeds size limit - case SQLITE_CONSTRAINT: // Abort due to constraint violation - case SQLITE_MISMATCH: // Data type mismatch - case SQLITE_MISUSE: // Library used incorrectly - case SQLITE_NOLFS: // Uses OS features not supported on host - case SQLITE_AUTH: // Authorization denied - case SQLITE_FORMAT: // Not used - case SQLITE_RANGE: // 2nd parameter to sqlite3_bind out of range - case SQLITE_NOTADB: // File opened that is not a database file - severity = mbgl::EventSeverity::Error; - break; - case SQLITE_WARNING: // Warnings from sqlite3_log() - severity = mbgl::EventSeverity::Warning; - break; - case SQLITE_NOTICE: // Notifications from sqlite3_log() - default: - break; - } - - mbgl::Log::Record(severity, mbgl::Event::Database, "%s (%s)", msg, codeToString(err)); + mbgl::Log::Record(mbgl::EventSeverity::Debug, mbgl::Event::Database, err, "%s", msg); } const static bool sqliteVersionCheck __attribute__((unused)) = []() { diff --git a/platform/default/sqlite3.hpp b/platform/default/sqlite3.hpp index 20d09b550c..6f54afb077 100644 --- a/platform/default/sqlite3.hpp +++ b/platform/default/sqlite3.hpp @@ -10,13 +10,8 @@ namespace mapbox { namespace sqlite { enum OpenFlag : int { - ReadOnly = 0x00000001, - ReadWrite = 0x00000002, - Create = 0x00000004, - NoMutex = 0x00008000, - FullMutex = 0x00010000, - SharedCache = 0x00020000, - PrivateCache = 0x00040000, + ReadOnly = 0b001, + ReadWriteCreate = 0b110, }; enum class ResultCode : int { diff --git a/platform/qt/src/sqlite3.cpp b/platform/qt/src/sqlite3.cpp index 4bcaea0e31..21ad681bd9 100644 --- a/platform/qt/src/sqlite3.cpp +++ b/platform/qt/src/sqlite3.cpp @@ -23,11 +23,30 @@ namespace mapbox { namespace sqlite { -// https://www.sqlite.org/rescode.html#ok static_assert(mbgl::underlying_type(ResultCode::OK) == 0, "error"); -// https://www.sqlite.org/rescode.html#cantopen +static_assert(mbgl::underlying_type(ResultCode::Error) == 1, "error"); +static_assert(mbgl::underlying_type(ResultCode::Internal) == 2, "error"); +static_assert(mbgl::underlying_type(ResultCode::Perm) == 3, "error"); +static_assert(mbgl::underlying_type(ResultCode::Abort) == 4, "error"); +static_assert(mbgl::underlying_type(ResultCode::Busy) == 5, "error"); +static_assert(mbgl::underlying_type(ResultCode::Locked) == 6, "error"); +static_assert(mbgl::underlying_type(ResultCode::NoMem) == 7, "error"); +static_assert(mbgl::underlying_type(ResultCode::ReadOnly) == 8, "error"); +static_assert(mbgl::underlying_type(ResultCode::Interrupt) == 9, "error"); +static_assert(mbgl::underlying_type(ResultCode::IOErr) == 10, "error"); +static_assert(mbgl::underlying_type(ResultCode::Corrupt) == 11, "error"); +static_assert(mbgl::underlying_type(ResultCode::NotFound) == 12, "error"); +static_assert(mbgl::underlying_type(ResultCode::Full) == 13, "error"); static_assert(mbgl::underlying_type(ResultCode::CantOpen) == 14, "error"); -// https://www.sqlite.org/rescode.html#notadb +static_assert(mbgl::underlying_type(ResultCode::Protocol) == 15, "error"); +static_assert(mbgl::underlying_type(ResultCode::Schema) == 17, "error"); +static_assert(mbgl::underlying_type(ResultCode::TooBig) == 18, "error"); +static_assert(mbgl::underlying_type(ResultCode::Constraint) == 19, "error"); +static_assert(mbgl::underlying_type(ResultCode::Mismatch) == 20, "error"); +static_assert(mbgl::underlying_type(ResultCode::Misuse) == 21, "error"); +static_assert(mbgl::underlying_type(ResultCode::NoLFS) == 22, "error"); +static_assert(mbgl::underlying_type(ResultCode::Auth) == 23, "error"); +static_assert(mbgl::underlying_type(ResultCode::Range) == 25, "error"); static_assert(mbgl::underlying_type(ResultCode::NotADB) == 26, "error"); void checkQueryError(const QSqlQuery& query) { @@ -85,10 +104,6 @@ public: if (!connectOptions.isEmpty()) connectOptions.append(';'); connectOptions.append("QSQLITE_OPEN_READONLY"); } - if (flags & OpenFlag::SharedCache) { - if (!connectOptions.isEmpty()) connectOptions.append(';'); - connectOptions.append("QSQLITE_ENABLE_SHARED_CACHE"); - } db.setConnectOptions(connectOptions); db.setDatabaseName(QString(filename)); diff --git a/src/mbgl/util/logging.cpp b/src/mbgl/util/logging.cpp index 672dcec665..3c73ac0bc1 100644 --- a/src/mbgl/util/logging.cpp +++ b/src/mbgl/util/logging.cpp @@ -38,8 +38,14 @@ void Log::record(EventSeverity severity, Event event, const char* format, ...) { record(severity, event, -1, msg); } -void Log::record(EventSeverity severity, Event event, int64_t code) { - record(severity, event, code, std::string()); +void Log::record(EventSeverity severity, Event event, int64_t code, const char* format, ...) { + va_list args; + va_start(args, format); + char msg[4096]; + vsnprintf(msg, sizeof(msg), format, args); + va_end(args); + + record(severity, event, code, std::string{ msg }); } void Log::record(EventSeverity severity, Event event, int64_t code, const std::string &msg) { diff --git a/test/fixtures/offline_database/corrupt-delayed.db b/test/fixtures/offline_database/corrupt-delayed.db new file mode 100644 index 0000000000..04989dbf36 Binary files /dev/null and b/test/fixtures/offline_database/corrupt-delayed.db differ diff --git a/test/fixtures/offline_database/corrupt-immediate.db b/test/fixtures/offline_database/corrupt-immediate.db new file mode 100644 index 0000000000..8909c402b2 Binary files /dev/null and b/test/fixtures/offline_database/corrupt-immediate.db differ diff --git a/test/scripts/ramdisk_darwin.sh b/test/scripts/ramdisk_darwin.sh new file mode 100755 index 0000000000..1622ea0c4a --- /dev/null +++ b/test/scripts/ramdisk_darwin.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env sh + +set -euo pipefail + +# Unmount a leftover mount point in case something went really wrong. +if [ -e test/ramdisk ]; then + diskutil eject test/ramdisk -force 2>/dev/null || true +fi + +# Create the ramdisk. +DEVICE=$(hdid -nomount ram://$1) +export BLOCKSIZE=4096 +newfs_hfs -v 'ramdisk' -b ${BLOCKSIZE} ${DEVICE} 2>/dev/null + +# Mount the ramdisk. +rm -rf test/ramdisk +mkdir -p test/ramdisk +function remount { mount $@ -o nobrowse -t hfs ${DEVICE} test/ramdisk; } +remount + +# Ensure that we're always unmounting the ramdisk to prevent them from piling up. +function finish { + diskutil eject test/ramdisk -force + rm -rf test/ramdisk +} +trap finish EXIT + +# Remount with different options until we get a signal to terminate. +while read -r LINE; do + case ${LINE} in + readonly) + remount -u -r + ;; + readwrite) + remount -u -w + ;; + filldisk*) + # Creates a file that fills up all sectors on the disk, but leaves room for the specified + # amount of sectors + rm -f test/ramdisk/fill + REMAINING=$((${LINE#filldisk})) + COUNT=$(df test/ramdisk | awk "!/Used/{print \$4 - ${REMAINING}}") + dd of=test/ramdisk/fill bs=${BLOCKSIZE} count=${COUNT} /dev/null + ;; + emptydisk) + rm -f test/ramdisk/fill + esac + # df test/ramdisk >&2 + echo "\n\0" +done diff --git a/test/src/mbgl/test/fixture_log_observer.cpp b/test/src/mbgl/test/fixture_log_observer.cpp index 587a010c3e..e03e05eb42 100644 --- a/test/src/mbgl/test/fixture_log_observer.cpp +++ b/test/src/mbgl/test/fixture_log_observer.cpp @@ -48,7 +48,7 @@ size_t FixtureLog::Observer::count(const Message& message, bool substring) const size_t message_count = 0; for (const auto& msg : messages) { - if (msg.matches(message, substring)) { + if (!msg.checked && msg.matches(message, substring)) { message_count++; msg.checked = true; } diff --git a/test/src/mbgl/test/ramdisk.cpp b/test/src/mbgl/test/ramdisk.cpp new file mode 100644 index 0000000000..063c09954b --- /dev/null +++ b/test/src/mbgl/test/ramdisk.cpp @@ -0,0 +1,60 @@ +#include + +#include + +#include + +namespace mbgl { +namespace test { + +#if TEST_HAS_RAMDISK + +RamDisk::RamDisk(const size_t size) { + char executable[64]; + snprintf(executable, sizeof executable, "test/scripts/ramdisk_darwin.sh %lu", size / 512); + if (!(child = popen(executable, "r+"))) { + throw std::runtime_error("Could not start child process"); + } + write("\n"); +} + +void RamDisk::write(const char* command) { + const size_t length = strlen(command); + if (fwrite(command, 1, length, child) != length) { + throw std::runtime_error("Could not write to child process"); + } + // Read lines until we get a line with a single NUL byte, which serves as command completion marker. + char line[256]; + do { + if (!fgets(line, sizeof line, child)) { + throw std::runtime_error("Could not read from child process"); + } + } while (strncmp(line, "\0\n", sizeof line) != 0); +} + +void RamDisk::setReadOnly() { + write("readonly\n"); +} + +void RamDisk::setReadWrite() { + write("readwrite\n"); +} + +void RamDisk::fillDiskExceptFor(size_t remaining) { + char command[32]; + snprintf(command, sizeof command, "filldisk %lu\n", remaining); + write(command); +} + +void RamDisk::emptyDisk() { + write("emptydisk\n"); +} + +RamDisk::~RamDisk() { + pclose(child); +} + +#endif // TEST_HAS_RAMDISK + +} // namespace test +} // namespace mbgl diff --git a/test/src/mbgl/test/ramdisk.hpp b/test/src/mbgl/test/ramdisk.hpp new file mode 100644 index 0000000000..728a87d6e4 --- /dev/null +++ b/test/src/mbgl/test/ramdisk.hpp @@ -0,0 +1,39 @@ +#pragma once + +#include + +#ifdef __APPLE__ +#include +#endif + +#if TARGET_OS_OSX +#define TEST_HAS_RAMDISK 1 +#else +#define TEST_HAS_RAMDISK 0 +#endif + +namespace mbgl { +namespace test { + +#if TEST_HAS_RAMDISK + +class RamDisk { +public: + RamDisk(size_t size = 1024 * 1024); + ~RamDisk(); + void setReadOnly(); + void setReadWrite(); + void fillDiskExceptFor(size_t remaining = 0); + void emptyDisk(); + +private: + void write(const char* command); + +private: + FILE* child; +}; + +#endif + +} // namespace test +} // namespace mbgl diff --git a/test/storage/offline_database.test.cpp b/test/storage/offline_database.test.cpp index c749584dc4..a7b480b821 100644 --- a/test/storage/offline_database.test.cpp +++ b/test/storage/offline_database.test.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -58,33 +59,291 @@ protected: static constexpr const char* filename = "test/fixtures/offline_database/offline.db"; std::unique_ptr log; + + struct { + const Resource style = Resource::style("mapbox://test"); + const Resource tile = Resource::tile("mapbox://test", 1, 0, 0, 0, Tileset::Scheme::XYZ); + const Response response = []{ + Response res; + res.data = std::make_shared("response"); + return res; + }(); + } mock; + }; TEST_F(OfflineDatabaseTest, TEST_REQUIRES_WRITE(Create)) { OfflineDatabase db(filename); EXPECT_FALSE(bool(db.get({ Resource::Unknown, "mapbox://test" }))); +} +#if TEST_HAS_RAMDISK +TEST_F(OfflineDatabaseTest, TEST_REQUIRES_WRITE(CreateDiskFull)) { + const Resource resource { Resource::Unknown, "mapbox://test" }; + const Resource tile = Resource::tile("mapbox://test", 1, 0, 0, 0, Tileset::Scheme::XYZ); + Response response; + response.data = std::make_shared("first"); + + test::RamDisk disk; + disk.fillDiskExceptFor(0); + + OfflineDatabase db("test/ramdisk/offline.db"); +#ifdef __QT__ + EXPECT_EQ(2u, log->count({ EventSeverity::Error, Event::Database, 14, "Error opening the database." })); +#else + EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 14, "cannot open file" }, true)); + EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 14, "test/ramdisk/offline.db) - No such file or directory" }, true)); + EXPECT_EQ(2u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file" })); +#endif + EXPECT_EQ(0u, log->uncheckedCount()); + + // We can try to insert things into the cache, but since the cache database isn't open, it won't be stored. + for (const auto& res : { resource, tile }) { + EXPECT_EQ(std::make_pair(false, uint64_t(0)), db.put(res, response)); +#ifdef __QT__ + EXPECT_EQ(2u, log->count({ EventSeverity::Error, Event::Database, 14, "Error opening the database." })); +#else + EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 14, "cannot open file" }, true)); + EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 14, "test/ramdisk/offline.db) - No such file or directory" }, true)); + EXPECT_EQ(2u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file" })); +#endif + EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, -1, "Unable to make space for entry" })); + EXPECT_EQ(0u, log->uncheckedCount()); + } + + // We can also still "query" the database even though it is not open, and we will always get an empty result. + for (const auto& res : { resource, tile }) { + EXPECT_FALSE(bool(db.get(res))); +#ifdef __QT__ + EXPECT_EQ(2u, log->count({ EventSeverity::Error, Event::Database, 14, "Error opening the database." })); +#else + EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 14, "cannot open file" }, true)); + EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 14, "test/ramdisk/offline.db) - No such file or directory" }, true)); + EXPECT_EQ(2u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file" })); +#endif + EXPECT_EQ(0u, log->uncheckedCount()); + } + + // Now, we're freeing up some space on the disk, and try to insert and query again. This time, we should + // be opening the datbase, creating the schema, and writing the data so that we can retrieve it again. + disk.emptyDisk(); + for (const auto& res : { resource, tile }) { + EXPECT_EQ(std::make_pair(true, uint64_t(5)), db.put(res, response)); + auto result = db.get(resource); + EXPECT_EQ(0u, log->uncheckedCount()); + ASSERT_TRUE(result && result->data); + EXPECT_EQ("first", *result->data); + } +} +#endif // TEST_HAS_RAMDISK + +#if TEST_HAS_RAMDISK +TEST_F(OfflineDatabaseTest, TEST_REQUIRES_WRITE(CreateDiskNotWritable)) { + const Resource resource { Resource::Unknown, "mapbox://test" }; + const Resource tile = Resource::tile("mapbox://test", 1, 0, 0, 0, Tileset::Scheme::XYZ); + Response response; + response.data = std::make_shared("first"); + + test::RamDisk disk; + disk.setReadOnly(); + + auto db = std::make_unique("test/ramdisk/offline.db"); +#ifdef __QT__ + EXPECT_EQ(2u, log->count({ EventSeverity::Error, Event::Database, 14, "Error opening the database." })); +#else + EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 14, "cannot open file" }, true)); + EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 14, "test/ramdisk/offline.db) - No such file or directory" }, true)); + EXPECT_EQ(2u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file" })); +#endif + EXPECT_EQ(0u, log->uncheckedCount()); + + // We can try to insert things into the cache, but since the cache database isn't open, it won't be stored. + for (const auto& res : { resource, tile }) { + EXPECT_EQ(std::make_pair(false, uint64_t(0)), db->put(res, response)); +#ifdef __QT__ + EXPECT_EQ(2u, log->count({ EventSeverity::Error, Event::Database, 14, "Error opening the database." })); +#else + EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 14, "cannot open file" }, true)); + EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 14, "test/ramdisk/offline.db) - No such file or directory" }, true)); + EXPECT_EQ(2u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file" })); +#endif + EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, -1, "Unable to make space for entry" })); + EXPECT_EQ(0u, log->uncheckedCount()); + } + + // We can also still "query" the database even though it is not open, and we will always get an empty result. + for (const auto& res : { resource, tile }) { + EXPECT_FALSE(bool(db->get(res))); +#ifdef __QT__ + EXPECT_EQ(2u, log->count({ EventSeverity::Error, Event::Database, 14, "Error opening the database." })); +#else + EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 14, "cannot open file" }, true)); + EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 14, "test/ramdisk/offline.db) - No such file or directory" }, true)); + EXPECT_EQ(2u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file" })); +#endif + EXPECT_EQ(0u, log->uncheckedCount()); + } + + // Now, the file system becomes writable, which means that we should be opening the datbase, creating the + // schema, and writing the data so that we can retrieve it again. + disk.setReadWrite(); + for (const auto& res : { resource, tile }) { + EXPECT_EQ(std::make_pair(true, uint64_t(5)), db->put(res, response)); + auto result = db->get(resource); + EXPECT_EQ(0u, log->uncheckedCount()); + ASSERT_TRUE(result && result->data); + EXPECT_EQ("first", *result->data); + } + + // Now fill up the disk, and try to write new data. This will fail because SQLite tries to create a journal file on this device. + // Verify that writing to the db consequently fails and that retrieving the resource will continue to return the old resource. + response.data = std::make_shared("second, longer string"); + disk.fillDiskExceptFor(0); + for (const auto& res : { resource, tile }) { + EXPECT_EQ(std::make_pair(false, uint64_t(0)), db->put(res, response)); +#ifdef __QT__ + EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file Unable to fetch row" })); +#else + EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 14, "cannot open file" }, true)); + EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 14, "test/ramdisk/offline.db-journal) - No such file or directory" }, true)); + EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 14, "statement aborts" }, true)); + EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file" })); +#endif + EXPECT_EQ(0u, log->uncheckedCount()); + } + + // Reading from the database should still work, except for journal file creation warnings. + for (const auto& res : { resource, tile }) { + auto result = db->get(res); +#ifdef __QT__ + EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file Unable to fetch row" })); +#else + EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 14, "cannot open file" }, true)); + EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 14, "test/ramdisk/offline.db-journal) - No such file or directory" }, true)); + EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 14, "statement aborts" }, true)); + EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file" })); +#endif + EXPECT_EQ(0u, log->uncheckedCount()); + ASSERT_TRUE(result && result->data); + EXPECT_EQ("first", *result->data); + } + + // Next, leave two blocks empty so that SQLite can create a journal file, but not so much that it can actually do the + // transaction. We should get a SQLITE_FULL error. + disk.fillDiskExceptFor(2); + for (const auto& res : { resource, tile }) { + EXPECT_EQ(std::make_pair(false, uint64_t(0)), db->put(res, response)); +#ifdef __QT__ + EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file Unable to fetch row" })); +#else + EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 778, "test/ramdisk/offline.db-journal) - No space left on device" }, true)); + EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 13, "test/ramdisk/offline.db-journal) - No space left on device" }, true)); + EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 13, "statement aborts" }, true)); + EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 13, "database or disk is full" })); +#endif + EXPECT_EQ(0u, log->uncheckedCount()); + } + + // Reading should still work, except for journal file disk full warnings. + for (const auto& res : { resource, tile }) { + auto result = db->get(res); +#ifdef __QT__ + EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file Unable to fetch row" })); +#else + EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 778, "test/ramdisk/offline.db-journal) - No space left on device" }, true)); + EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 13, "test/ramdisk/offline.db-journal) - No space left on device" }, true)); + EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 13, "statement aborts" }, true)); + EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 13, "database or disk is full" })); +#endif + EXPECT_EQ(0u, log->uncheckedCount()); + ASSERT_TRUE(result && result->data); + EXPECT_EQ("first", *result->data); + } - EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, -1, "cannot open file" }, true)); - EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, -1, "No such file or directory" }, true)); + // Empty the disk again, and redo the query to verify that we now don't get journal file creation messages. + disk.emptyDisk(); + for (const auto& res : { resource, tile }) { + auto result = db->get(res); +#ifdef __QT__ + EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file Unable to fetch row" })); +#endif + EXPECT_EQ(0u, log->uncheckedCount()); + ASSERT_TRUE(result && result->data); + EXPECT_EQ("first", *result->data); + } + + // Close the database, remount the ramdisk to read-only, and try to query the database. + db.reset(); + disk.setReadOnly(); + db = std::make_unique("test/ramdisk/offline.db"); + + for (const auto& res : { resource, tile }) { + EXPECT_EQ(std::make_pair(false, uint64_t(0)), db->put(res, response)); +#ifdef __QT__ + EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 8, "attempt to write a readonly database Unable to fetch row" })); +#else + EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 8, "statement aborts" }, true)); + EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 8, "attempt to write a readonly database" })); +#endif + EXPECT_EQ(0u, log->uncheckedCount()); + } + + for (const auto& res : { resource, tile }) { + auto result = db->get(res); +#ifdef __QT__ + EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 8, "attempt to write a readonly database Unable to fetch row" })); +#else + EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 8, "statement aborts" }, true)); + EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 8, "attempt to write a readonly database" })); +#endif + EXPECT_EQ(0u, log->uncheckedCount()); + ASSERT_TRUE(result && result->data); + EXPECT_EQ("first", *result->data); + } } +#endif // TEST_HAS_RAMDISK -TEST_F(OfflineDatabaseTest, TEST_REQUIRES_WRITE(SchemaVersion)) { +TEST_F(OfflineDatabaseTest, TEST_REQUIRES_WRITE(MigrateIncompatible)) { { - mapbox::sqlite::Database db{ filename, mapbox::sqlite::Create | mapbox::sqlite::ReadWrite }; + mapbox::sqlite::Database db{ filename, mapbox::sqlite::ReadWriteCreate }; db.exec("PRAGMA user_version = 1"); } OfflineDatabase db(filename); - + EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 11, "Unknown database version 1" })); EXPECT_EQ(1u, log->count({ EventSeverity::Warning, Event::Database, -1, "Removing existing incompatible offline database" })); + EXPECT_EQ(0u, log->uncheckedCount()); + + // Test insertion. This means that we have successfully recreated the database. + for (const auto& res : { mock.style, mock.tile }) { + EXPECT_EQ((std::pair{ true, mock.response.data->length() }), db.put(res, mock.response)); + auto result = db.get(res); + EXPECT_EQ(0u, log->uncheckedCount()); + ASSERT_TRUE(result && result->data); + EXPECT_EQ("response", *result->data); + } } TEST_F(OfflineDatabaseTest, TEST_REQUIRES_WRITE(Invalid)) { util::write_file(filename, "this is an invalid file"); - OfflineDatabase db(filename); + OfflineDatabase db(filename); +#ifdef __QT__ + EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 26, "file is encrypted or is not a database Unable to fetch row" })); +#else + EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 26, "file is encrypted or is not a database"}, true)); + EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 26, "file is encrypted or is not a database"})); +#endif EXPECT_EQ(1u, log->count({ EventSeverity::Warning, Event::Database, -1, "Removing existing incompatible offline database" })); - EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, -1, "file is encrypted or is not a database" }, true)); + EXPECT_EQ(0u, log->uncheckedCount()); + + // Test insertion. This means that we have successfully recreated the database. + for (const auto& res : { mock.style, mock.tile }) { + EXPECT_EQ((std::pair{ true, mock.response.data->length() }), db.put(res, mock.response)); + auto result = db.get(res); + EXPECT_EQ(0u, log->uncheckedCount()); + ASSERT_TRUE(result && result->data); + EXPECT_EQ("response", *result->data); + } } TEST_F(OfflineDatabaseTest, PutDoesNotStoreConnectionErrors) { @@ -310,32 +569,31 @@ TEST_F(OfflineDatabaseTest, CreateRegionInfiniteMaxZoom) { } TEST_F(OfflineDatabaseTest, TEST_REQUIRES_WRITE(ConcurrentUse)) { - OfflineDatabase db1(filename); - OfflineDatabase db2(filename); - Resource resource { Resource::Style, "http://example.com/" }; Response response; response.noContent = true; - std::thread thread1([&] { + auto fn = [&] { + OfflineDatabase db(filename); for (auto i = 0; i < 100; i++) { - db1.put(resource, response); - EXPECT_TRUE(bool(db1.get(resource))); + db.put(resource, response); + EXPECT_TRUE(bool(db.get(resource))); } - }); - - std::thread thread2([&] { - for (auto i = 0; i < 100; i++) { - db2.put(resource, response); - EXPECT_TRUE(bool(db2.get(resource))); - } - }); + }; + std::thread thread1(fn); + std::thread thread2(fn); thread1.join(); thread2.join(); - EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, -1, "cannot open file" }, true)); - EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, -1, "No such file or directory" }, true)); + // These messages are optional. Depending on how the threads are started they may or may not appear. +#ifdef __QT__ + EXPECT_GE(1u, log->count({ EventSeverity::Error, Event::Database, 1, "table resources already exists Unable to fetch row"})); +#else + EXPECT_GE(1u, log->count({ EventSeverity::Debug, Event::Database, 17, "database schema has changed"}, true)); + EXPECT_GE(1u, log->count({ EventSeverity::Debug, Event::Database, 1, "table resources already exists"})); + EXPECT_GE(1u, log->count({ EventSeverity::Error, Event::Database, 1, "table resources already exists"})); +#endif } static std::shared_ptr randomString(size_t size) { @@ -599,6 +857,7 @@ TEST_F(OfflineDatabaseTest, MigrateFromV2Schema) { db.deleteRegion(std::move(region)); } } + EXPECT_EQ(0u, log->uncheckedCount()); EXPECT_EQ(6, databaseUserVersion(filename)); EXPECT_LT(databasePageCount(filename), @@ -616,6 +875,7 @@ TEST_F(OfflineDatabaseTest, MigrateFromV3Schema) { db.deleteRegion(std::move(region)); } } + EXPECT_EQ(0u, log->uncheckedCount()); EXPECT_EQ(6, databaseUserVersion(filename)); } @@ -631,6 +891,7 @@ TEST_F(OfflineDatabaseTest, MigrateFromV4Schema) { db.deleteRegion(std::move(region)); } } + EXPECT_EQ(0u, log->uncheckedCount()); EXPECT_EQ(6, databaseUserVersion(filename)); @@ -653,6 +914,7 @@ TEST_F(OfflineDatabaseTest, MigrateFromV5Schema) { db.deleteRegion(std::move(region)); } } + EXPECT_EQ(0u, log->uncheckedCount()); EXPECT_EQ(6, databaseUserVersion(filename)); @@ -673,6 +935,9 @@ TEST_F(OfflineDatabaseTest, DowngradeSchema) { { OfflineDatabase db(filename, 0); } + EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 11, "Unknown database version 999" })); + EXPECT_EQ(1u, log->count({ EventSeverity::Warning, Event::Database, -1, "Removing existing incompatible offline database" })); + EXPECT_EQ(0u, log->uncheckedCount()); EXPECT_EQ(6, databaseUserVersion(filename)); @@ -683,6 +948,67 @@ TEST_F(OfflineDatabaseTest, DowngradeSchema) { EXPECT_EQ((std::vector{ "id", "url", "kind", "expires", "modified", "etag", "data", "compressed", "accessed", "must_revalidate" }), databaseTableColumns("test/fixtures/offline_database/migrated.db", "resources")); +} + +TEST_F(OfflineDatabaseTest, CorruptDatabaseOnOpen) { + copyFile("test/fixtures/offline_database/corrupt-immediate.db", filename); + // This database is corrupt in a way that will prevent opening the database. + OfflineDatabase db(filename); +#ifdef __QT__ + EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 11, "database disk image is malformed Unable to fetch row" })); +#else + EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 11, "database corruption at line" }, true)); + EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 11, "database disk image is malformed" }, true)); + EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 11, "database disk image is malformed" })); +#endif EXPECT_EQ(1u, log->count({ EventSeverity::Warning, Event::Database, -1, "Removing existing incompatible offline database" })); + EXPECT_EQ(0u, log->uncheckedCount()); + + // The first request fails because the corrupted database was deleted and we recreated it when + // opening it. + EXPECT_EQ(nullopt, db.get(mock.tile)); + + // Insert into the database and make sure that we can read it back. + for (const auto& res : { mock.style, mock.tile }) { + EXPECT_EQ((std::make_pair(true, uint64_t(mock.response.data->length()))), db.put(res, mock.response)); + auto result = db.get(res); + EXPECT_EQ(0u, log->uncheckedCount()); + ASSERT_TRUE(result &&result->data); + EXPECT_EQ(*mock.response.data, *result->data); + } } + +TEST_F(OfflineDatabaseTest, CorruptDatabaseOnQuery) { + copyFile("test/fixtures/offline_database/corrupt-delayed.db", filename); + + // This database is corrupt in a way that won't manifest itself until we start querying it, + // so just opening it will not cause an error. + OfflineDatabase db(filename); + + // Just opening this corrupt database should not have produced an error yet, since + // PRAGMA user_version still succeeds with this database. + EXPECT_EQ(0u, log->uncheckedCount()); + + // The first request fails because the database is corrupt and has to be recreated. + EXPECT_EQ(nullopt, db.get(mock.tile)); +#ifdef __QT__ + EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 11, "database disk image is malformed Unable to execute statement" })); +#else + EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 11, "database corruption at line" }, true)); + EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 11, "database disk image is malformed" }, true)); + EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 11, "database disk image is malformed" })); +#endif + EXPECT_EQ(1u, log->count({ EventSeverity::Warning, Event::Database, -1, "Removing existing incompatible offline database" })); + EXPECT_EQ(0u, log->uncheckedCount()); + + // Insert into the database and make sure that we can read it back. + for (const auto& res : { mock.style, mock.tile }) { + EXPECT_EQ((std::make_pair(true, uint64_t(mock.response.data->length()))), db.put(res, mock.response)); + auto result = db.get(res); + EXPECT_EQ(0u, log->uncheckedCount()); + ASSERT_TRUE(result &&result->data); + EXPECT_EQ(*mock.response.data, *result->data); + } +} + diff --git a/test/storage/sqlite.test.cpp b/test/storage/sqlite.test.cpp index 918200181f..0930f7375d 100644 --- a/test/storage/sqlite.test.cpp +++ b/test/storage/sqlite.test.cpp @@ -6,7 +6,7 @@ TEST(SQLite, Statement) { using namespace mbgl; - mapbox::sqlite::Database db(":memory:", mapbox::sqlite::Create | mapbox::sqlite::ReadWrite); + mapbox::sqlite::Database db(":memory:", mapbox::sqlite::ReadWriteCreate); db.exec("CREATE TABLE test (id INTEGER);"); mapbox::sqlite::Statement stmt1{ db, "INSERT INTO test (id) VALUES (?1);" }; -- cgit v1.2.1