summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKonstantin Käfer <mail@kkaefer.com>2018-03-07 12:21:06 +0100
committerKonstantin Käfer <mail@kkaefer.com>2018-03-07 13:51:52 +0100
commit543a5ea627984e20ccb0c3688165c4e579e1873d (patch)
tree276fcadb842fa35e61d884be00a47f20056c241f
parente75814fb9a1a6056a9d36f6f148637c56a21411e (diff)
downloadqtlocation-mapboxgl-upstream/offlinedb-test.tar.gz
[core] harden OfflineDatabaseupstream/offlinedb-test
- gracefully handles unreadable or corrupt files - gracefully handles temporary read or write errors, e.g. when the file system becomes unavailable - gracefully handles full disks - adds tests for all of these situations using a ramdisk for simulating full disks, or read only file systems
-rw-r--r--cmake/test-files.cmake2
-rw-r--r--include/mbgl/util/logging.hpp4
-rw-r--r--platform/default/mbgl/storage/offline_database.cpp1044
-rw-r--r--platform/default/mbgl/storage/offline_database.hpp18
-rw-r--r--platform/default/sqlite3.cpp110
-rw-r--r--platform/default/sqlite3.hpp9
-rw-r--r--platform/qt/src/sqlite3.cpp29
-rw-r--r--src/mbgl/util/logging.cpp10
-rw-r--r--test/fixtures/offline_database/corrupt-delayed.dbbin0 -> 19456 bytes
-rw-r--r--test/fixtures/offline_database/corrupt-immediate.dbbin0 -> 4096 bytes
-rwxr-xr-xtest/scripts/ramdisk_darwin.sh50
-rw-r--r--test/src/mbgl/test/fixture_log_observer.cpp2
-rw-r--r--test/src/mbgl/test/ramdisk.cpp60
-rw-r--r--test/src/mbgl/test/ramdisk.hpp39
-rw-r--r--test/storage/offline_database.test.cpp372
-rw-r--r--test/storage/sqlite.test.cpp2
16 files changed, 1191 insertions, 560 deletions
diff --git a/cmake/test-files.cmake b/cmake/test-files.cmake
index 2aadfa8a7f..5aca26ada1 100644
--- a/cmake/test-files.cmake
+++ b/cmake/test-files.cmake
@@ -100,6 +100,8 @@ set(MBGL_TEST_FILES
test/src/mbgl/test/fixture_log_observer.hpp
test/src/mbgl/test/getrss.cpp
test/src/mbgl/test/getrss.hpp
+ test/src/mbgl/test/ramdisk.cpp
+ test/src/mbgl/test/ramdisk.hpp
test/src/mbgl/test/stub_file_source.cpp
test/src/mbgl/test/stub_file_source.hpp
test/src/mbgl/test/stub_geometry_tile_feature.hpp
diff --git a/include/mbgl/util/logging.hpp b/include/mbgl/util/logging.hpp
index d072673e76..03db2d7462 100644
--- a/include/mbgl/util/logging.hpp
+++ b/include/mbgl/util/logging.hpp
@@ -67,8 +67,8 @@ public:
private:
static void record(EventSeverity severity, Event event, const std::string &msg);
- static void record(EventSeverity severity, Event event, const char* format, ...);
- static void record(EventSeverity severity, Event event, int64_t code);
+ static void record(EventSeverity severity, Event event, const char* format = "", ...);
+ static void record(EventSeverity severity, Event event, int64_t code, const char* format = "", ...);
static void record(EventSeverity severity, Event event, int64_t code, const std::string &msg);
// This method is the data sink that must be implemented by each platform we
diff --git a/platform/default/mbgl/storage/offline_database.cpp b/platform/default/mbgl/storage/offline_database.cpp
index 6edd051c43..69768b1b0a 100644
--- a/platform/default/mbgl/storage/offline_database.cpp
+++ b/platform/default/mbgl/storage/offline_database.cpp
@@ -13,76 +13,72 @@ namespace mbgl {
OfflineDatabase::OfflineDatabase(std::string path_, uint64_t maximumCacheSize_)
: path(std::move(path_)),
maximumCacheSize(maximumCacheSize_) {
- ensureSchema();
+ initialize();
}
OfflineDatabase::~OfflineDatabase() {
- // Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we
- // can't throw anything.
- try {
- statements.clear();
- db.reset();
- } catch (mapbox::sqlite::Exception& ex) {
- Log::Error(Event::Database, (int)ex.code, ex.what());
- }
+ closeDatabase();
}
-void OfflineDatabase::connect(int flags) {
- db = std::make_unique<mapbox::sqlite::Database>(path.c_str(), flags);
- db->setBusyTimeout(Milliseconds::max());
- db->exec("PRAGMA foreign_keys = ON");
+void OfflineDatabase::handleError(const mapbox::sqlite::Exception& ex) {
+ Log::Error(Event::Database, (int)ex.code, ex.what());
+
+ if (ex.code == mapbox::sqlite::ResultCode::Corrupt ||
+ ex.code == mapbox::sqlite::ResultCode::NotADB) {
+ removeExisting();
+ }
}
-void OfflineDatabase::ensureSchema() {
- if (path != ":memory:") {
+bool OfflineDatabase::initialize() {
+ for (uint32_t tries = 0; tries < 2; tries++) {
try {
- connect(mapbox::sqlite::ReadWrite);
-
- switch (userVersion()) {
- case 0: break; // cache-only database; ok to delete
- case 1: break; // cache-only database; ok to delete
- case 2: migrateToVersion3(); // fall through
- case 3: // no-op and fall through
- case 4: migrateToVersion5(); // fall through
- case 5: migrateToVersion6(); // fall through
- case 6: return;
- default: break; // downgrade, delete the database
- }
-
- removeExisting();
- connect(mapbox::sqlite::ReadWrite | mapbox::sqlite::Create);
+ openDatabase();
+ return true;
} catch (mapbox::sqlite::Exception& ex) {
- if (ex.code != mapbox::sqlite::ResultCode::CantOpen && ex.code != mapbox::sqlite::ResultCode::NotADB) {
- Log::Error(Event::Database, "Unexpected error connecting to database: %s", ex.what());
- throw;
- }
-
- try {
- if (ex.code == mapbox::sqlite::ResultCode::NotADB) {
- removeExisting();
- }
- connect(mapbox::sqlite::ReadWrite | mapbox::sqlite::Create);
- } catch (...) {
- Log::Error(Event::Database, "Unexpected error creating database: %s", util::toString(std::current_exception()).c_str());
- throw;
- }
+ handleError(ex);
+ closeDatabase();
}
}
+ return false;
+}
- try {
- #include "offline_schema.cpp.include"
+void OfflineDatabase::openDatabase() {
+ assert(!db);
+ db = std::make_unique<mapbox::sqlite::Database>(path.c_str(), mapbox::sqlite::ReadWriteCreate);
+ db->setBusyTimeout(Milliseconds::max());
+ db->exec("PRAGMA foreign_keys = ON");
- connect(mapbox::sqlite::ReadWrite | mapbox::sqlite::Create);
+ const auto version = userVersion();
+ switch (version) {
+ case 0: // New database, or a very old database that uses different tables.
+ createSchema();
+ break;
+
+ // Migrate existing database. We're using fall throughs.
+ case 2:
+ migrateToVersion3();
+ case 3:
+ case 4:
+ migrateToVersion5();
+ case 5:
+ migrateToVersion6();
+ case 6:
+ // Current database version
+ break;
+
+ // Delete unknown database version by treating it as a corrupt database.
+ default:
+ throw mapbox::sqlite::Exception{ mapbox::sqlite::ResultCode::Corrupt,
+ "Unknown database version " + util::toString(version) };
+ }
+}
- // If you change the schema you must write a migration from the previous version.
- db->exec("PRAGMA auto_vacuum = INCREMENTAL");
- db->exec("PRAGMA journal_mode = DELETE");
- db->exec("PRAGMA synchronous = FULL");
- db->exec(schema);
- db->exec("PRAGMA user_version = 6");
- } catch (...) {
- Log::Error(Event::Database, "Unexpected error creating database schema: %s", util::toString(std::current_exception()).c_str());
- throw;
+void OfflineDatabase::closeDatabase() {
+ if (db) {
+ statements.clear();
+ db.reset();
+ } else {
+ assert(statements.empty());
}
}
@@ -93,8 +89,7 @@ int OfflineDatabase::userVersion() {
void OfflineDatabase::removeExisting() {
Log::Warning(Event::Database, "Removing existing incompatible offline database");
- statements.clear();
- db.reset();
+ closeDatabase();
try {
util::deleteFile(path);
@@ -103,6 +98,19 @@ void OfflineDatabase::removeExisting() {
}
}
+void OfflineDatabase::createSchema() {
+ db->exec("PRAGMA auto_vacuum = INCREMENTAL");
+ db->exec("PRAGMA journal_mode = DELETE");
+ db->exec("PRAGMA synchronous = FULL");
+
+ #include "offline_schema.cpp.include"
+ mapbox::sqlite::Transaction transaction(*db);
+ db->exec(schema);
+ transaction.commit();
+
+ db->exec("PRAGMA user_version = 6");
+}
+
void OfflineDatabase::migrateToVersion3() {
db->exec("PRAGMA auto_vacuum = INCREMENTAL");
db->exec("VACUUM");
@@ -130,6 +138,7 @@ void OfflineDatabase::migrateToVersion6() {
}
mapbox::sqlite::Statement& OfflineDatabase::getStatement(const char* sql) {
+ assert(db);
auto it = statements.find(sql);
if (it == statements.end()) {
it = statements.emplace(sql, std::make_unique<mapbox::sqlite::Statement>(*db, sql)).first;
@@ -169,14 +178,14 @@ std::pair<bool, uint64_t> OfflineDatabase::putInternal(const Resource& resource,
return { false, 0 };
}
- std::string compressedData;
+ std::shared_ptr<const std::string> data;
bool compressed = false;
- uint64_t size = 0;
-
+ size_t size = 0;
if (response.data) {
- compressedData = util::compress(*response.data);
+ auto compressedData = util::compress(*response.data);
compressed = compressedData.size() < response.data->size();
- size = compressed ? compressedData.size() : response.data->size();
+ data = compressed ? std::make_shared<std::string>(std::move(compressedData)) : response.data;
+ size = data->size();
}
if (evict_ && !evict(size)) {
@@ -184,171 +193,209 @@ std::pair<bool, uint64_t> OfflineDatabase::putInternal(const Resource& resource,
return { false, 0 };
}
- bool inserted;
-
if (resource.kind == Resource::Kind::Tile) {
assert(resource.tileData);
- inserted = putTile(*resource.tileData, response,
- compressed ? compressedData : response.data ? *response.data : "",
- compressed);
+ return putTile(*resource.tileData, response, data, compressed);
} else {
- inserted = putResource(resource, response,
- compressed ? compressedData : response.data ? *response.data : "",
- compressed);
+ return putResource(resource, response, data, compressed);
}
-
- return { inserted, size };
}
optional<std::pair<Response, uint64_t>> OfflineDatabase::getResource(const Resource& resource) {
+ if (!db && !initialize()) {
+ return {};
+ }
+
// Update accessed timestamp used for LRU eviction.
- {
+ try {
mapbox::sqlite::Query accessedQuery{ getStatement("UPDATE resources SET accessed = ?1 WHERE url = ?2") };
accessedQuery.bind(1, util::now());
accessedQuery.bind(2, resource.url);
accessedQuery.run();
+ } catch (mapbox::sqlite::Exception& ex) {
+ handleError(ex);
+ // Ignore access update failures so that we can still read from a database even if the disk
+ // is full (and we can't create a journal file), or when the file system is readonly.
+
+ // handleError() may close the database, so ensure that we try to reopen/recreate it before
+ // proceeding.
+ if (!db && !initialize()) {
+ return {};
+ }
}
- // clang-format off
- mapbox::sqlite::Query query{ getStatement(
- // 0 1 2 3 4 5
- "SELECT etag, expires, must_revalidate, modified, data, compressed "
- "FROM resources "
- "WHERE url = ?") };
- // clang-format on
+ try {
+ // clang-format off
+ mapbox::sqlite::Query query{ getStatement(
+ // 0 1 2 3 4 5
+ "SELECT etag, expires, must_revalidate, modified, data, compressed "
+ "FROM resources "
+ "WHERE url = ?") };
+ // clang-format on
- query.bind(1, resource.url);
+ query.bind(1, resource.url);
- if (!query.run()) {
- return {};
- }
-
- Response response;
- uint64_t size = 0;
+ if (!query.run()) {
+ return {};
+ }
- response.etag = query.get<optional<std::string>>(0);
- response.expires = query.get<optional<Timestamp>>(1);
- response.mustRevalidate = query.get<bool>(2);
- response.modified = query.get<optional<Timestamp>>(3);
+ Response response;
+ uint64_t size = 0;
+
+ response.etag = query.get<optional<std::string>>(0);
+ response.expires = query.get<optional<Timestamp>>(1);
+ response.mustRevalidate = query.get<bool>(2);
+ response.modified = query.get<optional<Timestamp>>(3);
+
+ auto data = query.get<optional<std::string>>(4);
+ if (!data) {
+ response.noContent = true;
+ } else if (query.get<bool>(5)) {
+ response.data = std::make_shared<std::string>(util::decompress(*data));
+ size = data->length();
+ } else {
+ response.data = std::make_shared<std::string>(*data);
+ size = data->length();
+ }
- auto data = query.get<optional<std::string>>(4);
- if (!data) {
- response.noContent = true;
- } else if (query.get<bool>(5)) {
- response.data = std::make_shared<std::string>(util::decompress(*data));
- size = data->length();
- } else {
- response.data = std::make_shared<std::string>(*data);
- size = data->length();
+ return std::make_pair(response, size);
+ } catch (mapbox::sqlite::Exception& ex) {
+ handleError(ex);
+ return {};
}
-
- return std::make_pair(response, size);
}
optional<int64_t> OfflineDatabase::hasResource(const Resource& resource) {
- mapbox::sqlite::Query query{ getStatement("SELECT length(data) FROM resources WHERE url = ?") };
- query.bind(1, resource.url);
- if (!query.run()) {
+ if (!db && !initialize()) {
return {};
}
- return query.get<optional<int64_t>>(0);
+ try {
+ mapbox::sqlite::Query query{ getStatement("SELECT length(data) FROM resources WHERE url = ?") };
+ query.bind(1, resource.url);
+ if (query.run()) {
+ return query.get<optional<int64_t>>(0);
+ }
+ } catch (mapbox::sqlite::Exception& ex) {
+ handleError(ex);
+ }
+
+ return {};
}
-bool OfflineDatabase::putResource(const Resource& resource,
- const Response& response,
- const std::string& data,
- bool compressed) {
+std::pair<bool, uint64_t> OfflineDatabase::putResource(const Resource& resource,
+ const Response& response,
+ const std::shared_ptr<const std::string>& data,
+ bool compressed) {
+ if (!db && !initialize()) {
+ return { false, 0 };
+ }
+
if (response.notModified) {
+ try {
+ // clang-format off
+ mapbox::sqlite::Query notModifiedQuery{ getStatement(
+ "UPDATE resources "
+ "SET accessed = ?1, "
+ " expires = ?2, "
+ " must_revalidate = ?3 "
+ "WHERE url = ?4 ") };
+ // clang-format on
+
+ notModifiedQuery.bind(1, util::now());
+ notModifiedQuery.bind(2, response.expires);
+ notModifiedQuery.bind(3, response.mustRevalidate);
+ notModifiedQuery.bind(4, resource.url);
+ notModifiedQuery.run();
+ return { false, data ? data->size() : 0 };
+ } catch (mapbox::sqlite::Exception& ex) {
+ handleError(ex);
+ return { false, 0 };
+ }
+ }
+
+ try {
+ // We can't use REPLACE because it would change the id value.
+
+ // Begin an immediate-mode transaction to ensure that two writers do not attempt
+ // to INSERT a resource at the same moment.
+ assert(db);
+ mapbox::sqlite::Transaction transaction(*db, mapbox::sqlite::Transaction::Immediate);
+
// clang-format off
- mapbox::sqlite::Query notModifiedQuery{ getStatement(
+ mapbox::sqlite::Query updateQuery{ getStatement(
"UPDATE resources "
- "SET accessed = ?1, "
- " expires = ?2, "
- " must_revalidate = ?3 "
- "WHERE url = ?4 ") };
+ "SET kind = ?1, "
+ " etag = ?2, "
+ " expires = ?3, "
+ " must_revalidate = ?4, "
+ " modified = ?5, "
+ " accessed = ?6, "
+ " data = ?7, "
+ " compressed = ?8 "
+ "WHERE url = ?9 ") };
// clang-format on
- notModifiedQuery.bind(1, util::now());
- notModifiedQuery.bind(2, response.expires);
- notModifiedQuery.bind(3, response.mustRevalidate);
- notModifiedQuery.bind(4, resource.url);
- notModifiedQuery.run();
- return false;
- }
-
- // We can't use REPLACE because it would change the id value.
+ updateQuery.bind(1, int(resource.kind));
+ updateQuery.bind(2, response.etag);
+ updateQuery.bind(3, response.expires);
+ updateQuery.bind(4, response.mustRevalidate);
+ updateQuery.bind(5, response.modified);
+ updateQuery.bind(6, util::now());
+ updateQuery.bind(9, resource.url);
+
+ if (response.noContent || !data) {
+ updateQuery.bind(7, nullptr);
+ updateQuery.bind(8, false);
+ } else {
+ updateQuery.bindBlob(7, data->data(), data->size(), false);
+ updateQuery.bind(8, compressed);
+ }
- // Begin an immediate-mode transaction to ensure that two writers do not attempt
- // to INSERT a resource at the same moment.
- mapbox::sqlite::Transaction transaction(*db, mapbox::sqlite::Transaction::Immediate);
+ updateQuery.run();
+ if (updateQuery.changes() != 0) {
+ transaction.commit();
+ return { false, data ? data->size() : 0 };
+ }
- // clang-format off
- mapbox::sqlite::Query updateQuery{ getStatement(
- "UPDATE resources "
- "SET kind = ?1, "
- " etag = ?2, "
- " expires = ?3, "
- " must_revalidate = ?4, "
- " modified = ?5, "
- " accessed = ?6, "
- " data = ?7, "
- " compressed = ?8 "
- "WHERE url = ?9 ") };
- // clang-format on
+ // clang-format off
+ mapbox::sqlite::Query insertQuery{ getStatement(
+ "INSERT INTO resources (url, kind, etag, expires, must_revalidate, modified, accessed, data, compressed) "
+ "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) ") };
+ // clang-format on
- updateQuery.bind(1, int(resource.kind));
- updateQuery.bind(2, response.etag);
- updateQuery.bind(3, response.expires);
- updateQuery.bind(4, response.mustRevalidate);
- updateQuery.bind(5, response.modified);
- updateQuery.bind(6, util::now());
- updateQuery.bind(9, resource.url);
-
- if (response.noContent) {
- updateQuery.bind(7, nullptr);
- updateQuery.bind(8, false);
- } else {
- updateQuery.bindBlob(7, data.data(), data.size(), false);
- updateQuery.bind(8, compressed);
- }
+ insertQuery.bind(1, resource.url);
+ insertQuery.bind(2, int(resource.kind));
+ insertQuery.bind(3, response.etag);
+ insertQuery.bind(4, response.expires);
+ insertQuery.bind(5, response.mustRevalidate);
+ insertQuery.bind(6, response.modified);
+ insertQuery.bind(7, util::now());
+
+ if (response.noContent || !data) {
+ insertQuery.bind(8, nullptr);
+ insertQuery.bind(9, false);
+ } else {
+ insertQuery.bindBlob(8, data->data(), data->size(), false);
+ insertQuery.bind(9, compressed);
+ }
- updateQuery.run();
- if (updateQuery.changes() != 0) {
+ insertQuery.run();
transaction.commit();
- return false;
- }
- // clang-format off
- mapbox::sqlite::Query insertQuery{ getStatement(
- "INSERT INTO resources (url, kind, etag, expires, must_revalidate, modified, accessed, data, compressed) "
- "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) ") };
- // clang-format on
-
- insertQuery.bind(1, resource.url);
- insertQuery.bind(2, int(resource.kind));
- insertQuery.bind(3, response.etag);
- insertQuery.bind(4, response.expires);
- insertQuery.bind(5, response.mustRevalidate);
- insertQuery.bind(6, response.modified);
- insertQuery.bind(7, util::now());
-
- if (response.noContent) {
- insertQuery.bind(8, nullptr);
- insertQuery.bind(9, false);
- } else {
- insertQuery.bindBlob(8, data.data(), data.size(), false);
- insertQuery.bind(9, compressed);
+ return { true, data ? data->size() : 0 };
+ } catch (mapbox::sqlite::Exception& ex) {
+ handleError(ex);
+ return { false, 0 };
}
-
- insertQuery.run();
- transaction.commit();
-
- return true;
}
optional<std::pair<Response, uint64_t>> OfflineDatabase::getTile(const Resource::TileData& tile) {
- {
+ if (!db && !initialize()) {
+ return {};
+ }
+
+ try {
// clang-format off
mapbox::sqlite::Query accessedQuery{ getStatement(
"UPDATE tiles "
@@ -367,184 +414,222 @@ optional<std::pair<Response, uint64_t>> OfflineDatabase::getTile(const Resource:
accessedQuery.bind(5, tile.y);
accessedQuery.bind(6, tile.z);
accessedQuery.run();
+ } catch (mapbox::sqlite::Exception& ex) {
+ handleError(ex);
+ // Ignore access update failures so that we can still read from a database even if the disk
+ // is full (and we can't create a journal file), or when the file system is readonly.
+
+ // handleError() may close the database, so ensure that we try to reopen/recreate it before
+ // proceeding.
+ if (!db && !initialize()) {
+ return {};
+ }
}
- // clang-format off
- mapbox::sqlite::Query query{ getStatement(
- // 0 1 2, 3, 4, 5
- "SELECT etag, expires, must_revalidate, modified, data, compressed "
- "FROM tiles "
- "WHERE url_template = ?1 "
- " AND pixel_ratio = ?2 "
- " AND x = ?3 "
- " AND y = ?4 "
- " AND z = ?5 ") };
- // clang-format on
-
- query.bind(1, tile.urlTemplate);
- query.bind(2, tile.pixelRatio);
- query.bind(3, tile.x);
- query.bind(4, tile.y);
- query.bind(5, tile.z);
+ try {
+ // clang-format off
+ mapbox::sqlite::Query query{ getStatement(
+ // 0 1 2, 3, 4, 5
+ "SELECT etag, expires, must_revalidate, modified, data, compressed "
+ "FROM tiles "
+ "WHERE url_template = ?1 "
+ " AND pixel_ratio = ?2 "
+ " AND x = ?3 "
+ " AND y = ?4 "
+ " AND z = ?5 ") };
+ // clang-format on
- if (!query.run()) {
- return {};
- }
+ query.bind(1, tile.urlTemplate);
+ query.bind(2, tile.pixelRatio);
+ query.bind(3, tile.x);
+ query.bind(4, tile.y);
+ query.bind(5, tile.z);
- Response response;
- uint64_t size = 0;
+ if (!query.run()) {
+ return {};
+ }
- response.etag = query.get<optional<std::string>>(0);
- response.expires = query.get<optional<Timestamp>>(1);
- response.mustRevalidate = query.get<bool>(2);
- response.modified = query.get<optional<Timestamp>>(3);
+ Response response;
+ uint64_t size = 0;
+
+ response.etag = query.get<optional<std::string>>(0);
+ response.expires = query.get<optional<Timestamp>>(1);
+ response.mustRevalidate = query.get<bool>(2);
+ response.modified = query.get<optional<Timestamp>>(3);
+
+ optional<std::string> data = query.get<optional<std::string>>(4);
+ if (!data) {
+ response.noContent = true;
+ } else if (query.get<bool>(5)) {
+ response.data = std::make_shared<std::string>(util::decompress(*data));
+ size = data->length();
+ } else {
+ response.data = std::make_shared<std::string>(*data);
+ size = data->length();
+ }
- optional<std::string> data = query.get<optional<std::string>>(4);
- if (!data) {
- response.noContent = true;
- } else if (query.get<bool>(5)) {
- response.data = std::make_shared<std::string>(util::decompress(*data));
- size = data->length();
- } else {
- response.data = std::make_shared<std::string>(*data);
- size = data->length();
+ return std::make_pair(response, size);
+ } catch (mapbox::sqlite::Exception& ex) {
+ handleError(ex);
+ return {};
}
-
- return std::make_pair(response, size);
}
optional<int64_t> OfflineDatabase::hasTile(const Resource::TileData& tile) {
- // clang-format off
- mapbox::sqlite::Query size{ getStatement(
- "SELECT length(data) "
- "FROM tiles "
- "WHERE url_template = ?1 "
- " AND pixel_ratio = ?2 "
- " AND x = ?3 "
- " AND y = ?4 "
- " AND z = ?5 ") };
- // clang-format on
+ if (!db && !initialize()) {
+ return {};
+ }
+
+ try {
+ // clang-format off
+ mapbox::sqlite::Query size{ getStatement(
+ "SELECT length(data) "
+ "FROM tiles "
+ "WHERE url_template = ?1 "
+ " AND pixel_ratio = ?2 "
+ " AND x = ?3 "
+ " AND y = ?4 "
+ " AND z = ?5 ") };
+ // clang-format on
- size.bind(1, tile.urlTemplate);
- size.bind(2, tile.pixelRatio);
- size.bind(3, tile.x);
- size.bind(4, tile.y);
- size.bind(5, tile.z);
+ size.bind(1, tile.urlTemplate);
+ size.bind(2, tile.pixelRatio);
+ size.bind(3, tile.x);
+ size.bind(4, tile.y);
+ size.bind(5, tile.z);
- if (!size.run()) {
- return {};
+ if (size.run()) {
+ return size.get<optional<int64_t>>(0);
+ }
+ } catch (mapbox::sqlite::Exception& ex) {
+ handleError(ex);
}
- return size.get<optional<int64_t>>(0);
+ return {};
}
-bool OfflineDatabase::putTile(const Resource::TileData& tile,
- const Response& response,
- const std::string& data,
- bool compressed) {
+std::pair<bool, uint64_t> OfflineDatabase::putTile(const Resource::TileData& tile,
+ const Response& response,
+ const std::shared_ptr<const std::string>& data,
+ bool compressed) {
+ if (!db && !initialize()) {
+ return { false, 0 };
+ }
+
if (response.notModified) {
+ try {
+ // clang-format off
+ mapbox::sqlite::Query notModifiedQuery{ getStatement(
+ "UPDATE tiles "
+ "SET accessed = ?1, "
+ " expires = ?2, "
+ " must_revalidate = ?3 "
+ "WHERE url_template = ?4 "
+ " AND pixel_ratio = ?5 "
+ " AND x = ?6 "
+ " AND y = ?7 "
+ " AND z = ?8 ") };
+ // clang-format on
+
+ notModifiedQuery.bind(1, util::now());
+ notModifiedQuery.bind(2, response.expires);
+ notModifiedQuery.bind(3, response.mustRevalidate);
+ notModifiedQuery.bind(4, tile.urlTemplate);
+ notModifiedQuery.bind(5, tile.pixelRatio);
+ notModifiedQuery.bind(6, tile.x);
+ notModifiedQuery.bind(7, tile.y);
+ notModifiedQuery.bind(8, tile.z);
+ notModifiedQuery.run();
+ return { false, data ? data->size() : 0 };
+ } catch (mapbox::sqlite::Exception& ex) {
+ handleError(ex);
+ return { false, 0 };
+ }
+ }
+
+ try {
+ // We can't use REPLACE because it would change the id value.
+
+ // Begin an immediate-mode transaction to ensure that two writers do not attempt
+ // to INSERT a resource at the same moment.
+ assert(db);
+ mapbox::sqlite::Transaction transaction(*db, mapbox::sqlite::Transaction::Immediate);
+
// clang-format off
- mapbox::sqlite::Query notModifiedQuery{ getStatement(
+ mapbox::sqlite::Query updateQuery{ getStatement(
"UPDATE tiles "
- "SET accessed = ?1, "
- " expires = ?2, "
- " must_revalidate = ?3 "
- "WHERE url_template = ?4 "
- " AND pixel_ratio = ?5 "
- " AND x = ?6 "
- " AND y = ?7 "
- " AND z = ?8 ") };
+ "SET modified = ?1, "
+ " etag = ?2, "
+ " expires = ?3, "
+ " must_revalidate = ?4, "
+ " accessed = ?5, "
+ " data = ?6, "
+ " compressed = ?7 "
+ "WHERE url_template = ?8 "
+ " AND pixel_ratio = ?9 "
+ " AND x = ?10 "
+ " AND y = ?11 "
+ " AND z = ?12 ") };
// clang-format on
- notModifiedQuery.bind(1, util::now());
- notModifiedQuery.bind(2, response.expires);
- notModifiedQuery.bind(3, response.mustRevalidate);
- notModifiedQuery.bind(4, tile.urlTemplate);
- notModifiedQuery.bind(5, tile.pixelRatio);
- notModifiedQuery.bind(6, tile.x);
- notModifiedQuery.bind(7, tile.y);
- notModifiedQuery.bind(8, tile.z);
- notModifiedQuery.run();
- return false;
- }
-
- // We can't use REPLACE because it would change the id value.
+ updateQuery.bind(1, response.modified);
+ updateQuery.bind(2, response.etag);
+ updateQuery.bind(3, response.expires);
+ updateQuery.bind(4, response.mustRevalidate);
+ updateQuery.bind(5, util::now());
+ updateQuery.bind(8, tile.urlTemplate);
+ updateQuery.bind(9, tile.pixelRatio);
+ updateQuery.bind(10, tile.x);
+ updateQuery.bind(11, tile.y);
+ updateQuery.bind(12, tile.z);
+
+ if (response.noContent || !data) {
+ updateQuery.bind(6, nullptr);
+ updateQuery.bind(7, false);
+ } else {
+ updateQuery.bindBlob(6, data->data(), data->size(), false);
+ updateQuery.bind(7, compressed);
+ }
- // Begin an immediate-mode transaction to ensure that two writers do not attempt
- // to INSERT a resource at the same moment.
- mapbox::sqlite::Transaction transaction(*db, mapbox::sqlite::Transaction::Immediate);
+ updateQuery.run();
+ if (updateQuery.changes() != 0) {
+ transaction.commit();
+ return { false, data ? data->size() : 0 };
+ }
- // clang-format off
- mapbox::sqlite::Query updateQuery{ getStatement(
- "UPDATE tiles "
- "SET modified = ?1, "
- " etag = ?2, "
- " expires = ?3, "
- " must_revalidate = ?4, "
- " accessed = ?5, "
- " data = ?6, "
- " compressed = ?7 "
- "WHERE url_template = ?8 "
- " AND pixel_ratio = ?9 "
- " AND x = ?10 "
- " AND y = ?11 "
- " AND z = ?12 ") };
- // clang-format on
+ // clang-format off
+ mapbox::sqlite::Query insertQuery{ getStatement(
+ "INSERT INTO tiles (url_template, pixel_ratio, x, y, z, modified, must_revalidate, etag, expires, accessed, data, compressed) "
+ "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)") };
+ // clang-format on
- updateQuery.bind(1, response.modified);
- updateQuery.bind(2, response.etag);
- updateQuery.bind(3, response.expires);
- updateQuery.bind(4, response.mustRevalidate);
- updateQuery.bind(5, util::now());
- updateQuery.bind(8, tile.urlTemplate);
- updateQuery.bind(9, tile.pixelRatio);
- updateQuery.bind(10, tile.x);
- updateQuery.bind(11, tile.y);
- updateQuery.bind(12, tile.z);
-
- if (response.noContent) {
- updateQuery.bind(6, nullptr);
- updateQuery.bind(7, false);
- } else {
- updateQuery.bindBlob(6, data.data(), data.size(), false);
- updateQuery.bind(7, compressed);
- }
+ insertQuery.bind(1, tile.urlTemplate);
+ insertQuery.bind(2, tile.pixelRatio);
+ insertQuery.bind(3, tile.x);
+ insertQuery.bind(4, tile.y);
+ insertQuery.bind(5, tile.z);
+ insertQuery.bind(6, response.modified);
+ insertQuery.bind(7, response.mustRevalidate);
+ insertQuery.bind(8, response.etag);
+ insertQuery.bind(9, response.expires);
+ insertQuery.bind(10, util::now());
+
+ if (response.noContent || !data) {
+ insertQuery.bind(11, nullptr);
+ insertQuery.bind(12, false);
+ } else {
+ insertQuery.bindBlob(11, data->data(), data->size(), false);
+ insertQuery.bind(12, compressed);
+ }
- updateQuery.run();
- if (updateQuery.changes() != 0) {
+ insertQuery.run();
transaction.commit();
- return false;
- }
-
- // clang-format off
- mapbox::sqlite::Query insertQuery{ getStatement(
- "INSERT INTO tiles (url_template, pixel_ratio, x, y, z, modified, must_revalidate, etag, expires, accessed, data, compressed) "
- "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)") };
- // clang-format on
- insertQuery.bind(1, tile.urlTemplate);
- insertQuery.bind(2, tile.pixelRatio);
- insertQuery.bind(3, tile.x);
- insertQuery.bind(4, tile.y);
- insertQuery.bind(5, tile.z);
- insertQuery.bind(6, response.modified);
- insertQuery.bind(7, response.mustRevalidate);
- insertQuery.bind(8, response.etag);
- insertQuery.bind(9, response.expires);
- insertQuery.bind(10, util::now());
-
- if (response.noContent) {
- insertQuery.bind(11, nullptr);
- insertQuery.bind(12, false);
- } else {
- insertQuery.bindBlob(11, data.data(), data.size(), false);
- insertQuery.bind(12, compressed);
+ return { true, data ? data->size() : 0 };
+ } catch (mapbox::sqlite::Exception& ex) {
+ handleError(ex);
+ return { false, 0 };
}
-
- insertQuery.run();
- transaction.commit();
-
- return true;
}
std::vector<OfflineRegion> OfflineDatabase::listRegions() {
@@ -605,6 +690,10 @@ void OfflineDatabase::deleteRegion(OfflineRegion&& region) {
}
optional<std::pair<Response, uint64_t>> OfflineDatabase::getRegionResource(int64_t regionID, const Resource& resource) {
+ if (!db && !initialize()) {
+ return {};
+ }
+
auto response = getInternal(resource);
if (response) {
@@ -638,8 +727,22 @@ uint64_t OfflineDatabase::putRegionResource(int64_t regionID, const Resource& re
return size;
}
-bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) {
+bool OfflineDatabase::markUsed(int64_t regionID, const mbgl::Resource& resource) {
if (resource.kind == Resource::Kind::Tile) {
+ return markTileUsed(regionID, resource);
+ } else {
+ return markResourceUsed(regionID, resource);
+ }
+}
+
+bool OfflineDatabase::markTileUsed(int64_t regionID, const Resource& resource) {
+ assert(resource.kind == Resource::Kind::Tile);
+
+ if (!db && !initialize()) {
+ return false;
+ }
+
+ try {
// clang-format off
mapbox::sqlite::Query insertQuery{ getStatement(
"INSERT OR IGNORE INTO region_tiles (region_id, tile_id) "
@@ -664,7 +767,17 @@ bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) {
if (insertQuery.changes() == 0) {
return false;
}
+ } catch (mapbox::sqlite::Exception& ex) {
+ handleError(ex);
+ // handleError() may close the database, so ensure that we try to reopen/recreate it
+ // before proceeding.
+ if (!db && !initialize()) {
+ return false;
+ }
+ }
+
+ try {
// clang-format off
mapbox::sqlite::Query selectQuery{ getStatement(
"SELECT region_id "
@@ -678,6 +791,7 @@ bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) {
"LIMIT 1 ") };
// clang-format on
+ const Resource::TileData& tile = *resource.tileData;
selectQuery.bind(1, regionID);
selectQuery.bind(2, tile.urlTemplate);
selectQuery.bind(3, tile.pixelRatio);
@@ -685,7 +799,20 @@ bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) {
selectQuery.bind(5, tile.y);
selectQuery.bind(6, tile.z);
return !selectQuery.run();
- } else {
+ } catch (mapbox::sqlite::Exception& ex) {
+ handleError(ex);
+ return false;
+ }
+}
+
+bool OfflineDatabase::markResourceUsed(int64_t regionID, const Resource& resource) {
+ assert(resource.kind != Resource::Kind::Tile);
+
+ if (!db && !initialize()) {
+ return false;
+ }
+
+ try {
// clang-format off
mapbox::sqlite::Query insertQuery{ getStatement(
"INSERT OR IGNORE INTO region_resources (region_id, resource_id) "
@@ -701,7 +828,17 @@ bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) {
if (insertQuery.changes() == 0) {
return false;
}
+ } catch (mapbox::sqlite::Exception& ex) {
+ handleError(ex);
+
+ // handleError() may close the database, so ensure that we try to reopen/recreate it
+ // before proceeding.
+ if (!db && !initialize()) {
+ return false;
+ }
+ }
+ try {
// clang-format off
mapbox::sqlite::Query selectQuery{ getStatement(
"SELECT region_id "
@@ -714,15 +851,28 @@ bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) {
selectQuery.bind(1, regionID);
selectQuery.bind(2, resource.url);
return !selectQuery.run();
+ } catch (mapbox::sqlite::Exception& ex) {
+ handleError(ex);
+ return false;
}
}
OfflineRegionDefinition OfflineDatabase::getRegionDefinition(int64_t regionID) {
- mapbox::sqlite::Query query{ getStatement("SELECT definition FROM regions WHERE id = ?1") };
- query.bind(1, regionID);
- query.run();
+ if (!db && !initialize()) {
+ throw std::runtime_error("Could not find offline region");
+ }
+
+ try {
+ mapbox::sqlite::Query query{ getStatement("SELECT definition FROM regions WHERE id = ?1") };
+ query.bind(1, regionID);
+ if (query.run()) {
+ return decodeOfflineRegionDefinition(query.get<std::string>(0));
+ }
+ } catch (mapbox::sqlite::Exception& ex) {
+ handleError(ex);
+ }
- return decodeOfflineRegionDefinition(query.get<std::string>(0));
+ throw std::runtime_error("Could not find offline region");
}
OfflineRegionStatus OfflineDatabase::getRegionCompletedStatus(int64_t regionID) {
@@ -740,29 +890,50 @@ OfflineRegionStatus OfflineDatabase::getRegionCompletedStatus(int64_t regionID)
}
std::pair<int64_t, int64_t> OfflineDatabase::getCompletedResourceCountAndSize(int64_t regionID) {
- // clang-format off
- mapbox::sqlite::Query query{ getStatement(
- "SELECT COUNT(*), SUM(LENGTH(data)) "
- "FROM region_resources, resources "
- "WHERE region_id = ?1 "
- "AND resource_id = resources.id ") };
- // clang-format on
- query.bind(1, regionID);
- query.run();
- return { query.get<int64_t>(0), query.get<int64_t>(1) };
+ if (!db && !initialize()) {
+ return { 0, 0 };
+ }
+
+ try {
+ // clang-format off
+ mapbox::sqlite::Query query{ getStatement(
+ "SELECT COUNT(*), SUM(LENGTH(data)) "
+ "FROM region_resources, resources "
+ "WHERE region_id = ?1 "
+ "AND resource_id = resources.id ") };
+ // clang-format on
+ query.bind(1, regionID);
+ if (query.run()) {
+ return { query.get<int64_t>(0), query.get<int64_t>(1) };
+ }
+ } catch (mapbox::sqlite::Exception& ex) {
+ handleError(ex);
+ }
+
+ return { 0, 0 };
}
std::pair<int64_t, int64_t> OfflineDatabase::getCompletedTileCountAndSize(int64_t regionID) {
- // clang-format off
- mapbox::sqlite::Query query{ getStatement(
- "SELECT COUNT(*), SUM(LENGTH(data)) "
- "FROM region_tiles, tiles "
- "WHERE region_id = ?1 "
- "AND tile_id = tiles.id ") };
- // clang-format on
- query.bind(1, regionID);
- query.run();
- return { query.get<int64_t>(0), query.get<int64_t>(1) };
+ if (!db && !initialize()) {
+ return { 0, 0 };
+ }
+
+ try {
+ // clang-format off
+ mapbox::sqlite::Query query{ getStatement(
+ "SELECT COUNT(*), SUM(LENGTH(data)) "
+ "FROM region_tiles, tiles "
+ "WHERE region_id = ?1 "
+ "AND tile_id = tiles.id ") };
+ // clang-format on
+ query.bind(1, regionID);
+ query.run();
+ return { query.get<int64_t>(0), query.get<int64_t>(1) };
+ } catch (mapbox::sqlite::Exception& ex) {
+ handleError(ex);
+ }
+
+ return { 0, 0 };
}
template <class T>
@@ -783,80 +954,89 @@ T OfflineDatabase::getPragma(const char* sql) {
// delete an arbitrary number of old cache entries. The free pages approach saves
// us from calling VACCUM or keeping a running total, which can be costly.
bool OfflineDatabase::evict(uint64_t neededFreeSize) {
- uint64_t pageSize = getPragma<int64_t>("PRAGMA page_size");
- uint64_t pageCount = getPragma<int64_t>("PRAGMA page_count");
-
- auto usedSize = [&] {
- return pageSize * (pageCount - getPragma<int64_t>("PRAGMA freelist_count"));
- };
+ if (!db && !initialize()) {
+ return false;
+ }
- // The addition of pageSize is a fudge factor to account for non `data` column
- // size, and because pages can get fragmented on the database.
- while (usedSize() + neededFreeSize + pageSize > maximumCacheSize) {
- // clang-format off
- mapbox::sqlite::Query accessedQuery{ getStatement(
- "SELECT max(accessed) "
- "FROM ( "
- " SELECT accessed "
- " FROM resources "
- " LEFT JOIN region_resources "
- " ON resource_id = resources.id "
- " WHERE resource_id IS NULL "
- " UNION ALL "
- " SELECT accessed "
- " FROM tiles "
- " LEFT JOIN region_tiles "
- " ON tile_id = tiles.id "
- " WHERE tile_id IS NULL "
- " ORDER BY accessed ASC LIMIT ?1 "
- ") "
- ) };
- accessedQuery.bind(1, 50);
- // clang-format on
- if (!accessedQuery.run()) {
- return false;
+ try {
+ uint64_t pageSize = getPragma<int64_t>("PRAGMA page_size");
+ uint64_t pageCount = getPragma<int64_t>("PRAGMA page_count");
+
+ auto usedSize = [&] {
+ return pageSize * (pageCount - getPragma<int64_t>("PRAGMA freelist_count"));
+ };
+
+ // The addition of pageSize is a fudge factor to account for non `data` column
+ // size, and because pages can get fragmented on the database.
+ while (usedSize() + neededFreeSize + pageSize > maximumCacheSize) {
+ // clang-format off
+ mapbox::sqlite::Query accessedQuery{ getStatement(
+ "SELECT max(accessed) "
+ "FROM ( "
+ " SELECT accessed "
+ " FROM resources "
+ " LEFT JOIN region_resources "
+ " ON resource_id = resources.id "
+ " WHERE resource_id IS NULL "
+ " UNION ALL "
+ " SELECT accessed "
+ " FROM tiles "
+ " LEFT JOIN region_tiles "
+ " ON tile_id = tiles.id "
+ " WHERE tile_id IS NULL "
+ " ORDER BY accessed ASC LIMIT ?1 "
+ ") "
+ ) };
+ accessedQuery.bind(1, 50);
+ // clang-format on
+ if (!accessedQuery.run()) {
+ return false;
+ }
+ Timestamp accessed = accessedQuery.get<Timestamp>(0);
+
+ // clang-format off
+ mapbox::sqlite::Query resourceQuery{ getStatement(
+ "DELETE FROM resources "
+ "WHERE id IN ( "
+ " SELECT id FROM resources "
+ " LEFT JOIN region_resources "
+ " ON resource_id = resources.id "
+ " WHERE resource_id IS NULL "
+ " AND accessed <= ?1 "
+ ") ") };
+ // clang-format on
+ resourceQuery.bind(1, accessed);
+ resourceQuery.run();
+ const uint64_t resourceChanges = resourceQuery.changes();
+
+ // clang-format off
+ mapbox::sqlite::Query tileQuery{ getStatement(
+ "DELETE FROM tiles "
+ "WHERE id IN ( "
+ " SELECT id FROM tiles "
+ " LEFT JOIN region_tiles "
+ " ON tile_id = tiles.id "
+ " WHERE tile_id IS NULL "
+ " AND accessed <= ?1 "
+ ") ") };
+ // clang-format on
+ tileQuery.bind(1, accessed);
+ tileQuery.run();
+ const uint64_t tileChanges = tileQuery.changes();
+
+ // The cached value of offlineTileCount does not need to be updated
+ // here because only non-offline tiles can be removed by eviction.
+
+ if (resourceChanges == 0 && tileChanges == 0) {
+ return false;
+ }
}
- Timestamp accessed = accessedQuery.get<Timestamp>(0);
-
- // clang-format off
- mapbox::sqlite::Query resourceQuery{ getStatement(
- "DELETE FROM resources "
- "WHERE id IN ( "
- " SELECT id FROM resources "
- " LEFT JOIN region_resources "
- " ON resource_id = resources.id "
- " WHERE resource_id IS NULL "
- " AND accessed <= ?1 "
- ") ") };
- // clang-format on
- resourceQuery.bind(1, accessed);
- resourceQuery.run();
- const uint64_t resourceChanges = resourceQuery.changes();
-
- // clang-format off
- mapbox::sqlite::Query tileQuery{ getStatement(
- "DELETE FROM tiles "
- "WHERE id IN ( "
- " SELECT id FROM tiles "
- " LEFT JOIN region_tiles "
- " ON tile_id = tiles.id "
- " WHERE tile_id IS NULL "
- " AND accessed <= ?1 "
- ") ") };
- // clang-format on
- tileQuery.bind(1, accessed);
- tileQuery.run();
- const uint64_t tileChanges = tileQuery.changes();
- // The cached value of offlineTileCount does not need to be updated
- // here because only non-offline tiles can be removed by eviction.
-
- if (resourceChanges == 0 && tileChanges == 0) {
- return false;
- }
+ return true;
+ } catch (mapbox::sqlite::Exception& ex) {
+ handleError(ex);
+ return false;
}
-
- return true;
}
void OfflineDatabase::setOfflineMapboxTileCountLimit(uint64_t limit) {
diff --git a/platform/default/mbgl/storage/offline_database.hpp b/platform/default/mbgl/storage/offline_database.hpp
index 9673ad8212..3ad91671e1 100644
--- a/platform/default/mbgl/storage/offline_database.hpp
+++ b/platform/default/mbgl/storage/offline_database.hpp
@@ -16,6 +16,7 @@ namespace sqlite {
class Database;
class Statement;
class Query;
+class Exception;
} // namespace sqlite
} // namespace mapbox
@@ -59,9 +60,12 @@ public:
uint64_t getOfflineMapboxTileCount();
private:
- void connect(int flags);
+ void handleError(const mapbox::sqlite::Exception&);
+ bool initialize();
+ void openDatabase();
+ void createSchema();
+ void closeDatabase();
int userVersion();
- void ensureSchema();
void removeExisting();
void migrateToVersion3();
void migrateToVersion5();
@@ -71,13 +75,13 @@ private:
optional<std::pair<Response, uint64_t>> getTile(const Resource::TileData&);
optional<int64_t> hasTile(const Resource::TileData&);
- bool putTile(const Resource::TileData&, const Response&,
- const std::string&, bool compressed);
+ std::pair<bool, uint64_t>
+ putTile(const Resource::TileData&, const Response&, const std::shared_ptr<const std::string>&, bool compressed);
optional<std::pair<Response, uint64_t>> getResource(const Resource&);
optional<int64_t> hasResource(const Resource&);
- bool putResource(const Resource&, const Response&,
- const std::string&, bool compressed);
+ std::pair<bool, uint64_t>
+ putResource(const Resource&, const Response&, const std::shared_ptr<const std::string>&, bool compressed);
optional<std::pair<Response, uint64_t>> getInternal(const Resource&);
optional<int64_t> hasInternal(const Resource&);
@@ -85,6 +89,8 @@ private:
// Return value is true iff the resource was previously unused by any other regions.
bool markUsed(int64_t regionID, const Resource&);
+ bool markTileUsed(int64_t regionID, const Resource&);
+ bool markResourceUsed(int64_t regionID, const Resource&);
std::pair<int64_t, int64_t> getCompletedResourceCountAndSize(int64_t regionID);
std::pair<int64_t, int64_t> getCompletedTileCountAndSize(int64_t regionID);
diff --git a/platform/default/sqlite3.cpp b/platform/default/sqlite3.cpp
index 8a567d602e..ffa32db985 100644
--- a/platform/default/sqlite3.cpp
+++ b/platform/default/sqlite3.cpp
@@ -1,5 +1,7 @@
#include "sqlite3.hpp"
-#include <sqlite3.h>
+
+#include <mbgl/util/logging.hpp>
+#include <mbgl/util/traits.hpp>
#include <cassert>
#include <cstring>
@@ -7,11 +9,37 @@
#include <chrono>
#include <experimental/optional>
-#include <mbgl/util/logging.hpp>
+#include <sqlite3.h>
namespace mapbox {
namespace sqlite {
+static_assert(mbgl::underlying_type(ResultCode::OK) == SQLITE_OK, "error");
+static_assert(mbgl::underlying_type(ResultCode::Error) == SQLITE_ERROR, "error");
+static_assert(mbgl::underlying_type(ResultCode::Internal) == SQLITE_INTERNAL, "error");
+static_assert(mbgl::underlying_type(ResultCode::Perm) == SQLITE_PERM, "error");
+static_assert(mbgl::underlying_type(ResultCode::Abort) == SQLITE_ABORT, "error");
+static_assert(mbgl::underlying_type(ResultCode::Busy) == SQLITE_BUSY, "error");
+static_assert(mbgl::underlying_type(ResultCode::Locked) == SQLITE_LOCKED, "error");
+static_assert(mbgl::underlying_type(ResultCode::NoMem) == SQLITE_NOMEM, "error");
+static_assert(mbgl::underlying_type(ResultCode::ReadOnly) == SQLITE_READONLY, "error");
+static_assert(mbgl::underlying_type(ResultCode::Interrupt) == SQLITE_INTERRUPT, "error");
+static_assert(mbgl::underlying_type(ResultCode::IOErr) == SQLITE_IOERR, "error");
+static_assert(mbgl::underlying_type(ResultCode::Corrupt) == SQLITE_CORRUPT, "error");
+static_assert(mbgl::underlying_type(ResultCode::NotFound) == SQLITE_NOTFOUND, "error");
+static_assert(mbgl::underlying_type(ResultCode::Full) == SQLITE_FULL, "error");
+static_assert(mbgl::underlying_type(ResultCode::CantOpen) == SQLITE_CANTOPEN, "error");
+static_assert(mbgl::underlying_type(ResultCode::Protocol) == SQLITE_PROTOCOL, "error");
+static_assert(mbgl::underlying_type(ResultCode::Schema) == SQLITE_SCHEMA, "error");
+static_assert(mbgl::underlying_type(ResultCode::TooBig) == SQLITE_TOOBIG, "error");
+static_assert(mbgl::underlying_type(ResultCode::Constraint) == SQLITE_CONSTRAINT, "error");
+static_assert(mbgl::underlying_type(ResultCode::Mismatch) == SQLITE_MISMATCH, "error");
+static_assert(mbgl::underlying_type(ResultCode::Misuse) == SQLITE_MISUSE, "error");
+static_assert(mbgl::underlying_type(ResultCode::NoLFS) == SQLITE_NOLFS, "error");
+static_assert(mbgl::underlying_type(ResultCode::Auth) == SQLITE_AUTH, "error");
+static_assert(mbgl::underlying_type(ResultCode::Range) == SQLITE_RANGE, "error");
+static_assert(mbgl::underlying_type(ResultCode::NotADB) == SQLITE_NOTADB, "error");
+
class DatabaseImpl {
public:
DatabaseImpl(const char* filename, int flags)
@@ -69,84 +97,8 @@ public:
template <typename T>
using optional = std::experimental::optional<T>;
-static const char* codeToString(const int err) {
- switch (err) {
- case SQLITE_OK: return "SQLITE_OK";
- case SQLITE_ERROR: return "SQLITE_ERROR";
- case SQLITE_INTERNAL: return "SQLITE_INTERNAL";
- case SQLITE_PERM: return "SQLITE_PERM";
- case SQLITE_ABORT: return "SQLITE_ABORT";
- case SQLITE_BUSY: return "SQLITE_BUSY";
- case SQLITE_LOCKED: return "SQLITE_LOCKED";
- case SQLITE_NOMEM: return "SQLITE_NOMEM";
- case SQLITE_READONLY: return "SQLITE_READONLY";
- case SQLITE_INTERRUPT: return "SQLITE_INTERRUPT";
- case SQLITE_IOERR: return "SQLITE_IOERR";
- case SQLITE_CORRUPT: return "SQLITE_CORRUPT";
- case SQLITE_NOTFOUND: return "SQLITE_NOTFOUND";
- case SQLITE_FULL: return "SQLITE_FULL";
- case SQLITE_CANTOPEN: return "SQLITE_CANTOPEN";
- case SQLITE_PROTOCOL: return "SQLITE_PROTOCOL";
- case SQLITE_EMPTY: return "SQLITE_EMPTY";
- case SQLITE_SCHEMA: return "SQLITE_SCHEMA";
- case SQLITE_TOOBIG: return "SQLITE_TOOBIG";
- case SQLITE_CONSTRAINT: return "SQLITE_CONSTRAINT";
- case SQLITE_MISMATCH: return "SQLITE_MISMATCH";
- case SQLITE_MISUSE: return "SQLITE_MISUSE";
- case SQLITE_NOLFS: return "SQLITE_NOLFS";
- case SQLITE_AUTH: return "SQLITE_AUTH";
- case SQLITE_FORMAT: return "SQLITE_FORMAT";
- case SQLITE_RANGE: return "SQLITE_RANGE";
- case SQLITE_NOTADB: return "SQLITE_NOTADB";
- case SQLITE_NOTICE: return "SQLITE_NOTICE";
- case SQLITE_WARNING: return "SQLITE_WARNING";
- case SQLITE_ROW: return "SQLITE_ROW";
- case SQLITE_DONE: return "SQLITE_DONE";
- default: return "<unknown>";
- }
-}
-
static void errorLogCallback(void *, const int err, const char *msg) {
- auto severity = mbgl::EventSeverity::Info;
-
- switch (err) {
- case SQLITE_ERROR: // Generic error
- case SQLITE_INTERNAL: // Internal logic error in SQLite
- case SQLITE_PERM: // Access permission denied
- case SQLITE_ABORT: // Callback routine requested an abort
- case SQLITE_BUSY: // The database file is locked
- case SQLITE_LOCKED: // A table in the database is locked
- case SQLITE_NOMEM: // A malloc() failed
- case SQLITE_READONLY: // Attempt to write a readonly database
- case SQLITE_INTERRUPT: // Operation terminated by sqlite3_interrupt(
- case SQLITE_IOERR: // Some kind of disk I/O error occurred
- case SQLITE_CORRUPT: // The database disk image is malformed
- case SQLITE_NOTFOUND: // Unknown opcode in sqlite3_file_control()
- case SQLITE_FULL: // Insertion failed because database is full
- case SQLITE_CANTOPEN: // Unable to open the database file
- case SQLITE_PROTOCOL: // Database lock protocol error
- case SQLITE_EMPTY: // Internal use only
- case SQLITE_SCHEMA: // The database schema changed
- case SQLITE_TOOBIG: // String or BLOB exceeds size limit
- case SQLITE_CONSTRAINT: // Abort due to constraint violation
- case SQLITE_MISMATCH: // Data type mismatch
- case SQLITE_MISUSE: // Library used incorrectly
- case SQLITE_NOLFS: // Uses OS features not supported on host
- case SQLITE_AUTH: // Authorization denied
- case SQLITE_FORMAT: // Not used
- case SQLITE_RANGE: // 2nd parameter to sqlite3_bind out of range
- case SQLITE_NOTADB: // File opened that is not a database file
- severity = mbgl::EventSeverity::Error;
- break;
- case SQLITE_WARNING: // Warnings from sqlite3_log()
- severity = mbgl::EventSeverity::Warning;
- break;
- case SQLITE_NOTICE: // Notifications from sqlite3_log()
- default:
- break;
- }
-
- mbgl::Log::Record(severity, mbgl::Event::Database, "%s (%s)", msg, codeToString(err));
+ mbgl::Log::Record(mbgl::EventSeverity::Debug, mbgl::Event::Database, err, "%s", msg);
}
const static bool sqliteVersionCheck __attribute__((unused)) = []() {
diff --git a/platform/default/sqlite3.hpp b/platform/default/sqlite3.hpp
index 20d09b550c..6f54afb077 100644
--- a/platform/default/sqlite3.hpp
+++ b/platform/default/sqlite3.hpp
@@ -10,13 +10,8 @@ namespace mapbox {
namespace sqlite {
enum OpenFlag : int {
- ReadOnly = 0x00000001,
- ReadWrite = 0x00000002,
- Create = 0x00000004,
- NoMutex = 0x00008000,
- FullMutex = 0x00010000,
- SharedCache = 0x00020000,
- PrivateCache = 0x00040000,
+ ReadOnly = 0b001,
+ ReadWriteCreate = 0b110,
};
enum class ResultCode : int {
diff --git a/platform/qt/src/sqlite3.cpp b/platform/qt/src/sqlite3.cpp
index 4bcaea0e31..21ad681bd9 100644
--- a/platform/qt/src/sqlite3.cpp
+++ b/platform/qt/src/sqlite3.cpp
@@ -23,11 +23,30 @@
namespace mapbox {
namespace sqlite {
-// https://www.sqlite.org/rescode.html#ok
static_assert(mbgl::underlying_type(ResultCode::OK) == 0, "error");
-// https://www.sqlite.org/rescode.html#cantopen
+static_assert(mbgl::underlying_type(ResultCode::Error) == 1, "error");
+static_assert(mbgl::underlying_type(ResultCode::Internal) == 2, "error");
+static_assert(mbgl::underlying_type(ResultCode::Perm) == 3, "error");
+static_assert(mbgl::underlying_type(ResultCode::Abort) == 4, "error");
+static_assert(mbgl::underlying_type(ResultCode::Busy) == 5, "error");
+static_assert(mbgl::underlying_type(ResultCode::Locked) == 6, "error");
+static_assert(mbgl::underlying_type(ResultCode::NoMem) == 7, "error");
+static_assert(mbgl::underlying_type(ResultCode::ReadOnly) == 8, "error");
+static_assert(mbgl::underlying_type(ResultCode::Interrupt) == 9, "error");
+static_assert(mbgl::underlying_type(ResultCode::IOErr) == 10, "error");
+static_assert(mbgl::underlying_type(ResultCode::Corrupt) == 11, "error");
+static_assert(mbgl::underlying_type(ResultCode::NotFound) == 12, "error");
+static_assert(mbgl::underlying_type(ResultCode::Full) == 13, "error");
static_assert(mbgl::underlying_type(ResultCode::CantOpen) == 14, "error");
-// https://www.sqlite.org/rescode.html#notadb
+static_assert(mbgl::underlying_type(ResultCode::Protocol) == 15, "error");
+static_assert(mbgl::underlying_type(ResultCode::Schema) == 17, "error");
+static_assert(mbgl::underlying_type(ResultCode::TooBig) == 18, "error");
+static_assert(mbgl::underlying_type(ResultCode::Constraint) == 19, "error");
+static_assert(mbgl::underlying_type(ResultCode::Mismatch) == 20, "error");
+static_assert(mbgl::underlying_type(ResultCode::Misuse) == 21, "error");
+static_assert(mbgl::underlying_type(ResultCode::NoLFS) == 22, "error");
+static_assert(mbgl::underlying_type(ResultCode::Auth) == 23, "error");
+static_assert(mbgl::underlying_type(ResultCode::Range) == 25, "error");
static_assert(mbgl::underlying_type(ResultCode::NotADB) == 26, "error");
void checkQueryError(const QSqlQuery& query) {
@@ -85,10 +104,6 @@ public:
if (!connectOptions.isEmpty()) connectOptions.append(';');
connectOptions.append("QSQLITE_OPEN_READONLY");
}
- if (flags & OpenFlag::SharedCache) {
- if (!connectOptions.isEmpty()) connectOptions.append(';');
- connectOptions.append("QSQLITE_ENABLE_SHARED_CACHE");
- }
db.setConnectOptions(connectOptions);
db.setDatabaseName(QString(filename));
diff --git a/src/mbgl/util/logging.cpp b/src/mbgl/util/logging.cpp
index 672dcec665..3c73ac0bc1 100644
--- a/src/mbgl/util/logging.cpp
+++ b/src/mbgl/util/logging.cpp
@@ -38,8 +38,14 @@ void Log::record(EventSeverity severity, Event event, const char* format, ...) {
record(severity, event, -1, msg);
}
-void Log::record(EventSeverity severity, Event event, int64_t code) {
- record(severity, event, code, std::string());
+void Log::record(EventSeverity severity, Event event, int64_t code, const char* format, ...) {
+ va_list args;
+ va_start(args, format);
+ char msg[4096];
+ vsnprintf(msg, sizeof(msg), format, args);
+ va_end(args);
+
+ record(severity, event, code, std::string{ msg });
}
void Log::record(EventSeverity severity, Event event, int64_t code, const std::string &msg) {
diff --git a/test/fixtures/offline_database/corrupt-delayed.db b/test/fixtures/offline_database/corrupt-delayed.db
new file mode 100644
index 0000000000..04989dbf36
--- /dev/null
+++ b/test/fixtures/offline_database/corrupt-delayed.db
Binary files differ
diff --git a/test/fixtures/offline_database/corrupt-immediate.db b/test/fixtures/offline_database/corrupt-immediate.db
new file mode 100644
index 0000000000..8909c402b2
--- /dev/null
+++ b/test/fixtures/offline_database/corrupt-immediate.db
Binary files differ
diff --git a/test/scripts/ramdisk_darwin.sh b/test/scripts/ramdisk_darwin.sh
new file mode 100755
index 0000000000..1622ea0c4a
--- /dev/null
+++ b/test/scripts/ramdisk_darwin.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env sh
+
+set -euo pipefail
+
+# Unmount a leftover mount point in case something went really wrong.
+if [ -e test/ramdisk ]; then
+ diskutil eject test/ramdisk -force 2>/dev/null || true
+fi
+
+# Create the ramdisk.
+DEVICE=$(hdid -nomount ram://$1)
+export BLOCKSIZE=4096
+newfs_hfs -v 'ramdisk' -b ${BLOCKSIZE} ${DEVICE} 2>/dev/null
+
+# Mount the ramdisk.
+rm -rf test/ramdisk
+mkdir -p test/ramdisk
+function remount { mount $@ -o nobrowse -t hfs ${DEVICE} test/ramdisk; }
+remount
+
+# Ensure that we're always unmounting the ramdisk to prevent them from piling up.
+function finish {
+ diskutil eject test/ramdisk -force
+ rm -rf test/ramdisk
+}
+trap finish EXIT
+
+# Remount with different options until we get a signal to terminate.
+while read -r LINE; do
+ case ${LINE} in
+ readonly)
+ remount -u -r
+ ;;
+ readwrite)
+ remount -u -w
+ ;;
+ filldisk*)
+ # Creates a file that fills up all sectors on the disk, but leaves room for the specified
+ # amount of sectors
+ rm -f test/ramdisk/fill
+ REMAINING=$((${LINE#filldisk}))
+ COUNT=$(df test/ramdisk | awk "!/Used/{print \$4 - ${REMAINING}}")
+ dd of=test/ramdisk/fill bs=${BLOCKSIZE} count=${COUNT} </dev/zero 2>/dev/null
+ ;;
+ emptydisk)
+ rm -f test/ramdisk/fill
+ esac
+ # df test/ramdisk >&2
+ echo "\n\0"
+done
diff --git a/test/src/mbgl/test/fixture_log_observer.cpp b/test/src/mbgl/test/fixture_log_observer.cpp
index 587a010c3e..e03e05eb42 100644
--- a/test/src/mbgl/test/fixture_log_observer.cpp
+++ b/test/src/mbgl/test/fixture_log_observer.cpp
@@ -48,7 +48,7 @@ size_t FixtureLog::Observer::count(const Message& message, bool substring) const
size_t message_count = 0;
for (const auto& msg : messages) {
- if (msg.matches(message, substring)) {
+ if (!msg.checked && msg.matches(message, substring)) {
message_count++;
msg.checked = true;
}
diff --git a/test/src/mbgl/test/ramdisk.cpp b/test/src/mbgl/test/ramdisk.cpp
new file mode 100644
index 0000000000..063c09954b
--- /dev/null
+++ b/test/src/mbgl/test/ramdisk.cpp
@@ -0,0 +1,60 @@
+#include <mbgl/test/ramdisk.hpp>
+
+#include <stdexcept>
+
+#include <cstring>
+
+namespace mbgl {
+namespace test {
+
+#if TEST_HAS_RAMDISK
+
+RamDisk::RamDisk(const size_t size) {
+ char executable[64];
+ snprintf(executable, sizeof executable, "test/scripts/ramdisk_darwin.sh %lu", size / 512);
+ if (!(child = popen(executable, "r+"))) {
+ throw std::runtime_error("Could not start child process");
+ }
+ write("\n");
+}
+
+void RamDisk::write(const char* command) {
+ const size_t length = strlen(command);
+ if (fwrite(command, 1, length, child) != length) {
+ throw std::runtime_error("Could not write to child process");
+ }
+ // Read lines until we get a line with a single NUL byte, which serves as command completion marker.
+ char line[256];
+ do {
+ if (!fgets(line, sizeof line, child)) {
+ throw std::runtime_error("Could not read from child process");
+ }
+ } while (strncmp(line, "\0\n", sizeof line) != 0);
+}
+
+void RamDisk::setReadOnly() {
+ write("readonly\n");
+}
+
+void RamDisk::setReadWrite() {
+ write("readwrite\n");
+}
+
+void RamDisk::fillDiskExceptFor(size_t remaining) {
+ char command[32];
+ snprintf(command, sizeof command, "filldisk %lu\n", remaining);
+ write(command);
+}
+
+void RamDisk::emptyDisk() {
+ write("emptydisk\n");
+}
+
+RamDisk::~RamDisk() {
+ pclose(child);
+}
+
+#endif // TEST_HAS_RAMDISK
+
+} // namespace test
+} // namespace mbgl
diff --git a/test/src/mbgl/test/ramdisk.hpp b/test/src/mbgl/test/ramdisk.hpp
new file mode 100644
index 0000000000..728a87d6e4
--- /dev/null
+++ b/test/src/mbgl/test/ramdisk.hpp
@@ -0,0 +1,39 @@
+#pragma once
+
+#include <cstdio>
+
+#ifdef __APPLE__
+#include <TargetConditionals.h>
+#endif
+
+#if TARGET_OS_OSX
+#define TEST_HAS_RAMDISK 1
+#else
+#define TEST_HAS_RAMDISK 0
+#endif
+
+namespace mbgl {
+namespace test {
+
+#if TEST_HAS_RAMDISK
+
+class RamDisk {
+public:
+ RamDisk(size_t size = 1024 * 1024);
+ ~RamDisk();
+ void setReadOnly();
+ void setReadWrite();
+ void fillDiskExceptFor(size_t remaining = 0);
+ void emptyDisk();
+
+private:
+ void write(const char* command);
+
+private:
+ FILE* child;
+};
+
+#endif
+
+} // namespace test
+} // namespace mbgl
diff --git a/test/storage/offline_database.test.cpp b/test/storage/offline_database.test.cpp
index c749584dc4..a7b480b821 100644
--- a/test/storage/offline_database.test.cpp
+++ b/test/storage/offline_database.test.cpp
@@ -1,4 +1,5 @@
#include <mbgl/test/util.hpp>
+#include <mbgl/test/ramdisk.hpp>
#include <mbgl/test/fixture_log_observer.hpp>
#include <mbgl/storage/offline_database.hpp>
@@ -58,33 +59,291 @@ protected:
static constexpr const char* filename = "test/fixtures/offline_database/offline.db";
std::unique_ptr<FixtureLog> log;
+
+ struct {
+ const Resource style = Resource::style("mapbox://test");
+ const Resource tile = Resource::tile("mapbox://test", 1, 0, 0, 0, Tileset::Scheme::XYZ);
+ const Response response = []{
+ Response res;
+ res.data = std::make_shared<std::string>("response");
+ return res;
+ }();
+ } mock;
+
};
TEST_F(OfflineDatabaseTest, TEST_REQUIRES_WRITE(Create)) {
OfflineDatabase db(filename);
EXPECT_FALSE(bool(db.get({ Resource::Unknown, "mapbox://test" })));
+}
+#if TEST_HAS_RAMDISK
+TEST_F(OfflineDatabaseTest, TEST_REQUIRES_WRITE(CreateDiskFull)) {
+ const Resource resource { Resource::Unknown, "mapbox://test" };
+ const Resource tile = Resource::tile("mapbox://test", 1, 0, 0, 0, Tileset::Scheme::XYZ);
+ Response response;
+ response.data = std::make_shared<std::string>("first");
+
+ test::RamDisk disk;
+ disk.fillDiskExceptFor(0);
+
+ OfflineDatabase db("test/ramdisk/offline.db");
+#ifdef __QT__
+ EXPECT_EQ(2u, log->count({ EventSeverity::Error, Event::Database, 14, "Error opening the database." }));
+#else
+ EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 14, "cannot open file" }, true));
+ EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 14, "test/ramdisk/offline.db) - No such file or directory" }, true));
+ EXPECT_EQ(2u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file" }));
+#endif
+ EXPECT_EQ(0u, log->uncheckedCount());
+
+ // We can try to insert things into the cache, but since the cache database isn't open, it won't be stored.
+ for (const auto& res : { resource, tile }) {
+ EXPECT_EQ(std::make_pair(false, uint64_t(0)), db.put(res, response));
+#ifdef __QT__
+ EXPECT_EQ(2u, log->count({ EventSeverity::Error, Event::Database, 14, "Error opening the database." }));
+#else
+ EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 14, "cannot open file" }, true));
+ EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 14, "test/ramdisk/offline.db) - No such file or directory" }, true));
+ EXPECT_EQ(2u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file" }));
+#endif
+ EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, -1, "Unable to make space for entry" }));
+ EXPECT_EQ(0u, log->uncheckedCount());
+ }
+
+ // We can also still "query" the database even though it is not open, and we will always get an empty result.
+ for (const auto& res : { resource, tile }) {
+ EXPECT_FALSE(bool(db.get(res)));
+#ifdef __QT__
+ EXPECT_EQ(2u, log->count({ EventSeverity::Error, Event::Database, 14, "Error opening the database." }));
+#else
+ EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 14, "cannot open file" }, true));
+ EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 14, "test/ramdisk/offline.db) - No such file or directory" }, true));
+ EXPECT_EQ(2u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file" }));
+#endif
+ EXPECT_EQ(0u, log->uncheckedCount());
+ }
+
+ // Now, we're freeing up some space on the disk, and try to insert and query again. This time, we should
+ // be opening the datbase, creating the schema, and writing the data so that we can retrieve it again.
+ disk.emptyDisk();
+ for (const auto& res : { resource, tile }) {
+ EXPECT_EQ(std::make_pair(true, uint64_t(5)), db.put(res, response));
+ auto result = db.get(resource);
+ EXPECT_EQ(0u, log->uncheckedCount());
+ ASSERT_TRUE(result && result->data);
+ EXPECT_EQ("first", *result->data);
+ }
+}
+#endif // TEST_HAS_RAMDISK
+
+#if TEST_HAS_RAMDISK
+TEST_F(OfflineDatabaseTest, TEST_REQUIRES_WRITE(CreateDiskNotWritable)) {
+ const Resource resource { Resource::Unknown, "mapbox://test" };
+ const Resource tile = Resource::tile("mapbox://test", 1, 0, 0, 0, Tileset::Scheme::XYZ);
+ Response response;
+ response.data = std::make_shared<std::string>("first");
+
+ test::RamDisk disk;
+ disk.setReadOnly();
+
+ auto db = std::make_unique<OfflineDatabase>("test/ramdisk/offline.db");
+#ifdef __QT__
+ EXPECT_EQ(2u, log->count({ EventSeverity::Error, Event::Database, 14, "Error opening the database." }));
+#else
+ EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 14, "cannot open file" }, true));
+ EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 14, "test/ramdisk/offline.db) - No such file or directory" }, true));
+ EXPECT_EQ(2u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file" }));
+#endif
+ EXPECT_EQ(0u, log->uncheckedCount());
+
+ // We can try to insert things into the cache, but since the cache database isn't open, it won't be stored.
+ for (const auto& res : { resource, tile }) {
+ EXPECT_EQ(std::make_pair(false, uint64_t(0)), db->put(res, response));
+#ifdef __QT__
+ EXPECT_EQ(2u, log->count({ EventSeverity::Error, Event::Database, 14, "Error opening the database." }));
+#else
+ EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 14, "cannot open file" }, true));
+ EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 14, "test/ramdisk/offline.db) - No such file or directory" }, true));
+ EXPECT_EQ(2u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file" }));
+#endif
+ EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, -1, "Unable to make space for entry" }));
+ EXPECT_EQ(0u, log->uncheckedCount());
+ }
+
+ // We can also still "query" the database even though it is not open, and we will always get an empty result.
+ for (const auto& res : { resource, tile }) {
+ EXPECT_FALSE(bool(db->get(res)));
+#ifdef __QT__
+ EXPECT_EQ(2u, log->count({ EventSeverity::Error, Event::Database, 14, "Error opening the database." }));
+#else
+ EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 14, "cannot open file" }, true));
+ EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 14, "test/ramdisk/offline.db) - No such file or directory" }, true));
+ EXPECT_EQ(2u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file" }));
+#endif
+ EXPECT_EQ(0u, log->uncheckedCount());
+ }
+
+ // Now, the file system becomes writable, which means that we should be opening the datbase, creating the
+ // schema, and writing the data so that we can retrieve it again.
+ disk.setReadWrite();
+ for (const auto& res : { resource, tile }) {
+ EXPECT_EQ(std::make_pair(true, uint64_t(5)), db->put(res, response));
+ auto result = db->get(resource);
+ EXPECT_EQ(0u, log->uncheckedCount());
+ ASSERT_TRUE(result && result->data);
+ EXPECT_EQ("first", *result->data);
+ }
+
+ // Now fill up the disk, and try to write new data. This will fail because SQLite tries to create a journal file on this device.
+ // Verify that writing to the db consequently fails and that retrieving the resource will continue to return the old resource.
+ response.data = std::make_shared<std::string>("second, longer string");
+ disk.fillDiskExceptFor(0);
+ for (const auto& res : { resource, tile }) {
+ EXPECT_EQ(std::make_pair(false, uint64_t(0)), db->put(res, response));
+#ifdef __QT__
+ EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file Unable to fetch row" }));
+#else
+ EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 14, "cannot open file" }, true));
+ EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 14, "test/ramdisk/offline.db-journal) - No such file or directory" }, true));
+ EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 14, "statement aborts" }, true));
+ EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file" }));
+#endif
+ EXPECT_EQ(0u, log->uncheckedCount());
+ }
+
+ // Reading from the database should still work, except for journal file creation warnings.
+ for (const auto& res : { resource, tile }) {
+ auto result = db->get(res);
+#ifdef __QT__
+ EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file Unable to fetch row" }));
+#else
+ EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 14, "cannot open file" }, true));
+ EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 14, "test/ramdisk/offline.db-journal) - No such file or directory" }, true));
+ EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 14, "statement aborts" }, true));
+ EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file" }));
+#endif
+ EXPECT_EQ(0u, log->uncheckedCount());
+ ASSERT_TRUE(result && result->data);
+ EXPECT_EQ("first", *result->data);
+ }
+
+ // Next, leave two blocks empty so that SQLite can create a journal file, but not so much that it can actually do the
+ // transaction. We should get a SQLITE_FULL error.
+ disk.fillDiskExceptFor(2);
+ for (const auto& res : { resource, tile }) {
+ EXPECT_EQ(std::make_pair(false, uint64_t(0)), db->put(res, response));
+#ifdef __QT__
+ EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file Unable to fetch row" }));
+#else
+ EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 778, "test/ramdisk/offline.db-journal) - No space left on device" }, true));
+ EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 13, "test/ramdisk/offline.db-journal) - No space left on device" }, true));
+ EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 13, "statement aborts" }, true));
+ EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 13, "database or disk is full" }));
+#endif
+ EXPECT_EQ(0u, log->uncheckedCount());
+ }
+
+ // Reading should still work, except for journal file disk full warnings.
+ for (const auto& res : { resource, tile }) {
+ auto result = db->get(res);
+#ifdef __QT__
+ EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file Unable to fetch row" }));
+#else
+ EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 778, "test/ramdisk/offline.db-journal) - No space left on device" }, true));
+ EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 13, "test/ramdisk/offline.db-journal) - No space left on device" }, true));
+ EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 13, "statement aborts" }, true));
+ EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 13, "database or disk is full" }));
+#endif
+ EXPECT_EQ(0u, log->uncheckedCount());
+ ASSERT_TRUE(result && result->data);
+ EXPECT_EQ("first", *result->data);
+ }
- EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, -1, "cannot open file" }, true));
- EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, -1, "No such file or directory" }, true));
+ // Empty the disk again, and redo the query to verify that we now don't get journal file creation messages.
+ disk.emptyDisk();
+ for (const auto& res : { resource, tile }) {
+ auto result = db->get(res);
+#ifdef __QT__
+ EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file Unable to fetch row" }));
+#endif
+ EXPECT_EQ(0u, log->uncheckedCount());
+ ASSERT_TRUE(result && result->data);
+ EXPECT_EQ("first", *result->data);
+ }
+
+ // Close the database, remount the ramdisk to read-only, and try to query the database.
+ db.reset();
+ disk.setReadOnly();
+ db = std::make_unique<OfflineDatabase>("test/ramdisk/offline.db");
+
+ for (const auto& res : { resource, tile }) {
+ EXPECT_EQ(std::make_pair(false, uint64_t(0)), db->put(res, response));
+#ifdef __QT__
+ EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 8, "attempt to write a readonly database Unable to fetch row" }));
+#else
+ EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 8, "statement aborts" }, true));
+ EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 8, "attempt to write a readonly database" }));
+#endif
+ EXPECT_EQ(0u, log->uncheckedCount());
+ }
+
+ for (const auto& res : { resource, tile }) {
+ auto result = db->get(res);
+#ifdef __QT__
+ EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 8, "attempt to write a readonly database Unable to fetch row" }));
+#else
+ EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 8, "statement aborts" }, true));
+ EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 8, "attempt to write a readonly database" }));
+#endif
+ EXPECT_EQ(0u, log->uncheckedCount());
+ ASSERT_TRUE(result && result->data);
+ EXPECT_EQ("first", *result->data);
+ }
}
+#endif // TEST_HAS_RAMDISK
-TEST_F(OfflineDatabaseTest, TEST_REQUIRES_WRITE(SchemaVersion)) {
+TEST_F(OfflineDatabaseTest, TEST_REQUIRES_WRITE(MigrateIncompatible)) {
{
- mapbox::sqlite::Database db{ filename, mapbox::sqlite::Create | mapbox::sqlite::ReadWrite };
+ mapbox::sqlite::Database db{ filename, mapbox::sqlite::ReadWriteCreate };
db.exec("PRAGMA user_version = 1");
}
OfflineDatabase db(filename);
-
+ EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 11, "Unknown database version 1" }));
EXPECT_EQ(1u, log->count({ EventSeverity::Warning, Event::Database, -1, "Removing existing incompatible offline database" }));
+ EXPECT_EQ(0u, log->uncheckedCount());
+
+ // Test insertion. This means that we have successfully recreated the database.
+ for (const auto& res : { mock.style, mock.tile }) {
+ EXPECT_EQ((std::pair<bool, uint64_t>{ true, mock.response.data->length() }), db.put(res, mock.response));
+ auto result = db.get(res);
+ EXPECT_EQ(0u, log->uncheckedCount());
+ ASSERT_TRUE(result && result->data);
+ EXPECT_EQ("response", *result->data);
+ }
}
TEST_F(OfflineDatabaseTest, TEST_REQUIRES_WRITE(Invalid)) {
util::write_file(filename, "this is an invalid file");
- OfflineDatabase db(filename);
+ OfflineDatabase db(filename);
+#ifdef __QT__
+ EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 26, "file is encrypted or is not a database Unable to fetch row" }));
+#else
+ EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 26, "file is encrypted or is not a database"}, true));
+ EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 26, "file is encrypted or is not a database"}));
+#endif
EXPECT_EQ(1u, log->count({ EventSeverity::Warning, Event::Database, -1, "Removing existing incompatible offline database" }));
- EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, -1, "file is encrypted or is not a database" }, true));
+ EXPECT_EQ(0u, log->uncheckedCount());
+
+ // Test insertion. This means that we have successfully recreated the database.
+ for (const auto& res : { mock.style, mock.tile }) {
+ EXPECT_EQ((std::pair<bool, uint64_t>{ true, mock.response.data->length() }), db.put(res, mock.response));
+ auto result = db.get(res);
+ EXPECT_EQ(0u, log->uncheckedCount());
+ ASSERT_TRUE(result && result->data);
+ EXPECT_EQ("response", *result->data);
+ }
}
TEST_F(OfflineDatabaseTest, PutDoesNotStoreConnectionErrors) {
@@ -310,32 +569,31 @@ TEST_F(OfflineDatabaseTest, CreateRegionInfiniteMaxZoom) {
}
TEST_F(OfflineDatabaseTest, TEST_REQUIRES_WRITE(ConcurrentUse)) {
- OfflineDatabase db1(filename);
- OfflineDatabase db2(filename);
-
Resource resource { Resource::Style, "http://example.com/" };
Response response;
response.noContent = true;
- std::thread thread1([&] {
+ auto fn = [&] {
+ OfflineDatabase db(filename);
for (auto i = 0; i < 100; i++) {
- db1.put(resource, response);
- EXPECT_TRUE(bool(db1.get(resource)));
+ db.put(resource, response);
+ EXPECT_TRUE(bool(db.get(resource)));
}
- });
-
- std::thread thread2([&] {
- for (auto i = 0; i < 100; i++) {
- db2.put(resource, response);
- EXPECT_TRUE(bool(db2.get(resource)));
- }
- });
+ };
+ std::thread thread1(fn);
+ std::thread thread2(fn);
thread1.join();
thread2.join();
- EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, -1, "cannot open file" }, true));
- EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, -1, "No such file or directory" }, true));
+ // These messages are optional. Depending on how the threads are started they may or may not appear.
+#ifdef __QT__
+ EXPECT_GE(1u, log->count({ EventSeverity::Error, Event::Database, 1, "table resources already exists Unable to fetch row"}));
+#else
+ EXPECT_GE(1u, log->count({ EventSeverity::Debug, Event::Database, 17, "database schema has changed"}, true));
+ EXPECT_GE(1u, log->count({ EventSeverity::Debug, Event::Database, 1, "table resources already exists"}));
+ EXPECT_GE(1u, log->count({ EventSeverity::Error, Event::Database, 1, "table resources already exists"}));
+#endif
}
static std::shared_ptr<std::string> randomString(size_t size) {
@@ -599,6 +857,7 @@ TEST_F(OfflineDatabaseTest, MigrateFromV2Schema) {
db.deleteRegion(std::move(region));
}
}
+ EXPECT_EQ(0u, log->uncheckedCount());
EXPECT_EQ(6, databaseUserVersion(filename));
EXPECT_LT(databasePageCount(filename),
@@ -616,6 +875,7 @@ TEST_F(OfflineDatabaseTest, MigrateFromV3Schema) {
db.deleteRegion(std::move(region));
}
}
+ EXPECT_EQ(0u, log->uncheckedCount());
EXPECT_EQ(6, databaseUserVersion(filename));
}
@@ -631,6 +891,7 @@ TEST_F(OfflineDatabaseTest, MigrateFromV4Schema) {
db.deleteRegion(std::move(region));
}
}
+ EXPECT_EQ(0u, log->uncheckedCount());
EXPECT_EQ(6, databaseUserVersion(filename));
@@ -653,6 +914,7 @@ TEST_F(OfflineDatabaseTest, MigrateFromV5Schema) {
db.deleteRegion(std::move(region));
}
}
+ EXPECT_EQ(0u, log->uncheckedCount());
EXPECT_EQ(6, databaseUserVersion(filename));
@@ -673,6 +935,9 @@ TEST_F(OfflineDatabaseTest, DowngradeSchema) {
{
OfflineDatabase db(filename, 0);
}
+ EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 11, "Unknown database version 999" }));
+ EXPECT_EQ(1u, log->count({ EventSeverity::Warning, Event::Database, -1, "Removing existing incompatible offline database" }));
+ EXPECT_EQ(0u, log->uncheckedCount());
EXPECT_EQ(6, databaseUserVersion(filename));
@@ -683,6 +948,67 @@ TEST_F(OfflineDatabaseTest, DowngradeSchema) {
EXPECT_EQ((std::vector<std::string>{ "id", "url", "kind", "expires", "modified", "etag", "data",
"compressed", "accessed", "must_revalidate" }),
databaseTableColumns("test/fixtures/offline_database/migrated.db", "resources"));
+}
+
+TEST_F(OfflineDatabaseTest, CorruptDatabaseOnOpen) {
+ copyFile("test/fixtures/offline_database/corrupt-immediate.db", filename);
+ // This database is corrupt in a way that will prevent opening the database.
+ OfflineDatabase db(filename);
+#ifdef __QT__
+ EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 11, "database disk image is malformed Unable to fetch row" }));
+#else
+ EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 11, "database corruption at line" }, true));
+ EXPECT_EQ(1u, log->count({ EventSeverity::Debug, Event::Database, 11, "database disk image is malformed" }, true));
+ EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 11, "database disk image is malformed" }));
+#endif
EXPECT_EQ(1u, log->count({ EventSeverity::Warning, Event::Database, -1, "Removing existing incompatible offline database" }));
+ EXPECT_EQ(0u, log->uncheckedCount());
+
+ // The first request fails because the corrupted database was deleted and we recreated it when
+ // opening it.
+ EXPECT_EQ(nullopt, db.get(mock.tile));
+
+ // Insert into the database and make sure that we can read it back.
+ for (const auto& res : { mock.style, mock.tile }) {
+ EXPECT_EQ((std::make_pair(true, uint64_t(mock.response.data->length()))), db.put(res, mock.response));
+ auto result = db.get(res);
+ EXPECT_EQ(0u, log->uncheckedCount());
+ ASSERT_TRUE(result &&result->data);
+ EXPECT_EQ(*mock.response.data, *result->data);
+ }
}
+
+TEST_F(OfflineDatabaseTest, CorruptDatabaseOnQuery) {
+ copyFile("test/fixtures/offline_database/corrupt-delayed.db", filename);
+
+ // This database is corrupt in a way that won't manifest itself until we start querying it,
+ // so just opening it will not cause an error.
+ OfflineDatabase db(filename);
+
+ // Just opening this corrupt database should not have produced an error yet, since
+ // PRAGMA user_version still succeeds with this database.
+ EXPECT_EQ(0u, log->uncheckedCount());
+
+ // The first request fails because the database is corrupt and has to be recreated.
+ EXPECT_EQ(nullopt, db.get(mock.tile));
+#ifdef __QT__
+ EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 11, "database disk image is malformed Unable to execute statement" }));
+#else
+ EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 11, "database corruption at line" }, true));
+ EXPECT_EQ(2u, log->count({ EventSeverity::Debug, Event::Database, 11, "database disk image is malformed" }, true));
+ EXPECT_EQ(1u, log->count({ EventSeverity::Error, Event::Database, 11, "database disk image is malformed" }));
+#endif
+ EXPECT_EQ(1u, log->count({ EventSeverity::Warning, Event::Database, -1, "Removing existing incompatible offline database" }));
+ EXPECT_EQ(0u, log->uncheckedCount());
+
+ // Insert into the database and make sure that we can read it back.
+ for (const auto& res : { mock.style, mock.tile }) {
+ EXPECT_EQ((std::make_pair(true, uint64_t(mock.response.data->length()))), db.put(res, mock.response));
+ auto result = db.get(res);
+ EXPECT_EQ(0u, log->uncheckedCount());
+ ASSERT_TRUE(result &&result->data);
+ EXPECT_EQ(*mock.response.data, *result->data);
+ }
+}
+
diff --git a/test/storage/sqlite.test.cpp b/test/storage/sqlite.test.cpp
index 918200181f..0930f7375d 100644
--- a/test/storage/sqlite.test.cpp
+++ b/test/storage/sqlite.test.cpp
@@ -6,7 +6,7 @@
TEST(SQLite, Statement) {
using namespace mbgl;
- mapbox::sqlite::Database db(":memory:", mapbox::sqlite::Create | mapbox::sqlite::ReadWrite);
+ mapbox::sqlite::Database db(":memory:", mapbox::sqlite::ReadWriteCreate);
db.exec("CREATE TABLE test (id INTEGER);");
mapbox::sqlite::Statement stmt1{ db, "INSERT INTO test (id) VALUES (?1);" };