diff options
Diffstat (limited to 'platform/default/sqlite_cache.cpp')
-rw-r--r-- | platform/default/sqlite_cache.cpp | 272 |
1 files changed, 237 insertions, 35 deletions
diff --git a/platform/default/sqlite_cache.cpp b/platform/default/sqlite_cache.cpp index 43b67cfd7f..4d8cb8180c 100644 --- a/platform/default/sqlite_cache.cpp +++ b/platform/default/sqlite_cache.cpp @@ -11,6 +11,21 @@ #include "sqlite3.hpp" #include <sqlite3.h> +namespace { + +// The cache won't accept entries larger than this arbitrary size +// and will silently discard request for adding them to the cache. +// Large entries can cause the database to grow in disk size and +// never shrink again. +const uint64_t kMaximumCacheEntrySize = 5 * 1024 * 1024; // 5 MB + +// Number of records we delete when we are close to the maximum +// database size, if set. The current criteria is to prune +// the least used entries based on `accessed` time. +const int kPrunedEntriesLimit = 100; + +} // namespace + namespace mbgl { using namespace mapbox::sqlite; @@ -22,7 +37,9 @@ SQLiteCache::SQLiteCache(const std::string& path_) SQLiteCache::~SQLiteCache() = default; SQLiteCache::Impl::Impl(const std::string& path_) - : path(path_) { + : maximumCacheSize(0), // Unlimited + maximumCacheEntrySize(kMaximumCacheEntrySize), + path(path_) { } SQLiteCache::Impl::~Impl() { @@ -32,6 +49,10 @@ SQLiteCache::Impl::~Impl() { getStmt.reset(); putStmt.reset(); refreshStmt.reset(); + countStmt.reset(); + freeStmt.reset(); + pruneStmt.reset(); + accessedStmt.reset(); db.reset(); } catch (mapbox::sqlite::Exception& ex) { Log::Error(Event::Database, ex.code, ex.what()); @@ -51,10 +72,12 @@ void SQLiteCache::Impl::createSchema() { " `modified` INTEGER," // Timestamp when the file was last modified. " `etag` TEXT," " `expires` INTEGER," // Timestamp when the server says the file expires. + " `accessed` INTEGER," // Timestamp when the database record was last accessed. " `data` BLOB," " `compressed` INTEGER NOT NULL DEFAULT 0" // Whether the data is compressed. ");" - "CREATE INDEX IF NOT EXISTS `http_cache_kind_idx` ON `http_cache` (`kind`);"; + "CREATE INDEX IF NOT EXISTS `http_cache_kind_idx` ON `http_cache` (`kind`);" + "CREATE INDEX IF NOT EXISTS `http_cache_accessed_idx` ON `http_cache` (`modified`);"; try { db->exec(sql); @@ -80,6 +103,167 @@ void SQLiteCache::Impl::createSchema() { } } +void SQLiteCache::setMaximumCacheSize(uint64_t size) { + thread->invoke(&Impl::setMaximumCacheSize, size); +} + +void SQLiteCache::Impl::setMaximumCacheSize(uint64_t size) { + maximumCacheSize = size; + + // Unlimited. + if (size == 0) { + return; + } + + uint64_t lastSoftSize = cacheSoftSize(); + + // Keep pruning until we fit in the new + // size limit. + while (lastSoftSize > maximumCacheSize) { + pruneEntries(); + + if (lastSoftSize != cacheSoftSize()) { + lastSoftSize = cacheSoftSize(); + } else { + break; + } + } + + if (cacheHardSize() > size) { + Log::Warning(mbgl::Event::Database, + "Current cache hard size is bigger than the defined " + "maximum size. Database won't get truncated."); + } +} + +void SQLiteCache::setMaximumCacheEntrySize(uint64_t size) { + thread->invoke(&Impl::setMaximumCacheEntrySize, size); +} + +void SQLiteCache::Impl::setMaximumCacheEntrySize(uint64_t size) { + maximumCacheEntrySize = size; +} + +void SQLiteCache::Impl::initializeDatabase() { + if (!db) { + createDatabase(); + } + + if (!schema) { + createSchema(); + } +} + +int SQLiteCache::Impl::cachePageSize() { + try { + if (!pageSize) { + Statement pageSizeStmt(db->prepare("PRAGMA page_size")); + if (pageSizeStmt.run()) { + pageSize = pageSizeStmt.get<int>(0); + } + } + } catch (mapbox::sqlite::Exception& ex) { + Log::Error(Event::Database, ex.code, ex.what()); + } + + return pageSize; +} + +uint64_t SQLiteCache::Impl::cacheHardSize() { + try { + initializeDatabase(); + + if (!countStmt) { + countStmt = std::make_unique<Statement>(db->prepare("PRAGMA page_count")); + } else { + countStmt->reset(); + } + + if (countStmt->run()) { + return cachePageSize() * countStmt->get<int>(0); + } + } catch (mapbox::sqlite::Exception& ex) { + Log::Error(Event::Database, ex.code, ex.what()); + } + + return 0; +} + +uint64_t SQLiteCache::Impl::cacheSoftSize() { + if (!softSizeDirty) { + return softSize; + } + + try { + initializeDatabase(); + + if (!freeStmt) { + freeStmt = std::make_unique<Statement>(db->prepare("PRAGMA freelist_count")); + } else { + freeStmt->reset(); + } + + uint64_t hardSize = cacheHardSize(); + if (!hardSize) { + return 0; + } + + if (freeStmt->run()) { + return hardSize - cachePageSize() * freeStmt->get<int>(0); + } + + softSizeDirty = false; + } catch (mapbox::sqlite::Exception& ex) { + Log::Error(Event::Database, ex.code, ex.what()); + } + + return 0; +} + +bool SQLiteCache::Impl::needsPruning() { + // SQLite database never shrinks in size unless we call VACCUM. We here + // are monitoring the soft limit (i.e. number of free pages in the file) + // and as it approaches to the hard limit (i.e. the actual file size) we + // delete an arbitrary number of old cache entries. + // + // The free pages approach saves us from calling VACCUM or keeping a + // running total, which can be costly. We need a buffer because pages can + // get fragmented on the database. + if (cacheSoftSize() + maximumCacheEntrySize * 2 < maximumCacheSize) { + return false; + } else { + return true; + } +} + +void SQLiteCache::Impl::pruneEntries() { + if (!maximumCacheSize) { + return; + } + + if (!needsPruning()) { + return; + } + + try { + if (!pruneStmt) { + pruneStmt = std::make_unique<Statement>(db->prepare( + "DELETE FROM `http_cache` WHERE `rowid` IN (SELECT `rowid` FROM " + // 1 + "`http_cache` ORDER BY `accessed` ASC LIMIT ?)")); + } else { + pruneStmt->reset(); + } + + pruneStmt->bind(1, kPrunedEntriesLimit); + + pruneStmt->run(); + softSizeDirty = true; + } catch (mapbox::sqlite::Exception& ex) { + Log::Error(Event::Database, ex.code, ex.what()); + } +} + std::unique_ptr<WorkRequest> SQLiteCache::get(const Resource &resource, Callback callback) { // Can be called from any thread, but most likely from the file source thread. // Will try to load the URL from the SQLite database and call the callback when done. @@ -90,14 +274,7 @@ std::unique_ptr<WorkRequest> SQLiteCache::get(const Resource &resource, Callback void SQLiteCache::Impl::get(const Resource &resource, Callback callback) { try { - // This is called in the SQLite event loop. - if (!db) { - createDatabase(); - } - - if (!schema) { - createSchema(); - } + initializeDatabase(); if (!getStmt) { // Initialize the statement 0 1 @@ -130,6 +307,23 @@ void SQLiteCache::Impl::get(const Resource &resource, Callback callback) { // There is no data. callback(nullptr); } + + // We do an extra query for refreshing the last time + // the record was accessed that can be costly and is only + // worth doing if we are monitoring the database size. + if (maximumCacheSize) { + if (!accessedStmt) { + accessedStmt = std::make_unique<Statement>( + // 1 2 + db->prepare("UPDATE `http_cache` SET `accessed` = ? WHERE `url` = ?")); + } else { + accessedStmt->reset(); + } + + accessedStmt->bind(1, toSeconds(SystemClock::now()).count()); + accessedStmt->bind(2, canonicalURL.c_str()); + accessedStmt->run(); + } } catch (mapbox::sqlite::Exception& ex) { Log::Error(Event::Database, ex.code, ex.what()); callback(nullptr); @@ -152,19 +346,28 @@ void SQLiteCache::put(const Resource &resource, std::shared_ptr<const Response> void SQLiteCache::Impl::put(const Resource& resource, std::shared_ptr<const Response> response) { try { - if (!db) { - createDatabase(); - } + initializeDatabase(); + pruneEntries(); + + if (response->data) { + auto entrySize = response->data->size(); - if (!schema) { - createSchema(); + if (entrySize > maximumCacheEntrySize) { + Log::Warning(Event::Database, "Entry too big for caching."); + return; + } + + if (maximumCacheSize && entrySize + cacheSoftSize() > maximumCacheSize) { + Log::Warning(Event::Database, "Unable to make space for new entries."); + return; + } } if (!putStmt) { putStmt = std::make_unique<Statement>(db->prepare("REPLACE INTO `http_cache` (" - // 1 2 3 4 5 6 7 8 - "`url`, `status`, `kind`, `modified`, `etag`, `expires`, `data`, `compressed`" - ") VALUES(?, ?, ?, ?, ?, ?, ?, ?)")); + // 1 2 3 4 5 6 7 8 9 + "`url`, `status`, `kind`, `modified`, `etag`, `expires`, `accessed`, `data`, `compressed`" + ") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)")); } else { putStmt->reset(); } @@ -180,6 +383,7 @@ void SQLiteCache::Impl::put(const Resource& resource, std::shared_ptr<const Resp putStmt->bind(4 /* modified */, response->modified.count()); putStmt->bind(5 /* etag */, response->etag.c_str()); putStmt->bind(6 /* expires */, response->expires.count()); + putStmt->bind(7 /* accessed */, toSeconds(SystemClock::now()).count()); std::string data; if (resource.kind != Resource::SpriteImage && response->data) { @@ -190,17 +394,18 @@ void SQLiteCache::Impl::put(const Resource& resource, std::shared_ptr<const Resp if (!data.empty() && data.size() < response->data->size()) { // Store the compressed data when it is smaller than the original // uncompressed data. - putStmt->bind(7 /* data */, data, false); // do not retain the string internally. - putStmt->bind(8 /* compressed */, true); + putStmt->bind(8 /* data */, data, false); // do not retain the string internally. + putStmt->bind(9 /* compressed */, true); } else if (response->data) { - putStmt->bind(7 /* data */, *response->data, false); // do not retain the string internally. - putStmt->bind(8 /* compressed */, false); + putStmt->bind(8 /* data */, *response->data, false); // do not retain the string internally. + putStmt->bind(9 /* compressed */, false); } else { - putStmt->bind(7 /* data */, "", false); - putStmt->bind(8 /* compressed */, false); + putStmt->bind(8 /* data */, "", false); + putStmt->bind(9 /* compressed */, false); } putStmt->run(); + softSizeDirty = true; } catch (mapbox::sqlite::Exception& ex) { Log::Error(Event::Database, ex.code, ex.what()); } catch (std::runtime_error& ex) { @@ -210,24 +415,21 @@ void SQLiteCache::Impl::put(const Resource& resource, std::shared_ptr<const Resp void SQLiteCache::Impl::refresh(const Resource& resource, Seconds expires) { try { - if (!db) { - createDatabase(); - } - - if (!schema) { - createSchema(); - } + initializeDatabase(); if (!refreshStmt) { - refreshStmt = std::make_unique<Statement>( // 1 2 - db->prepare("UPDATE `http_cache` SET `expires` = ? WHERE `url` = ?")); + refreshStmt = std::make_unique<Statement>( + db->prepare("UPDATE `http_cache` SET " + // 1 2 3 + "`accessed` = ?, `expires` = ? WHERE `url` = ?")); } else { refreshStmt->reset(); } const auto canonicalURL = util::mapbox::canonicalURL(resource.url); - refreshStmt->bind(1, expires.count()); - refreshStmt->bind(2, canonicalURL.c_str()); + refreshStmt->bind(1, toSeconds(SystemClock::now()).count()); + refreshStmt->bind(2, expires.count()); + refreshStmt->bind(3, canonicalURL.c_str()); refreshStmt->run(); } catch (mapbox::sqlite::Exception& ex) { Log::Error(Event::Database, ex.code, ex.what()); |