diff options
author | Thiago Marcos P. Santos <thiago@mapbox.com> | 2015-12-14 11:16:01 -0800 |
---|---|---|
committer | Thiago Marcos P. Santos <thiago@mapbox.com> | 2015-12-27 18:15:33 +0200 |
commit | 6d0b01c5fb1e0740ea48566fea200f03b979c52c (patch) | |
tree | 1ef2b3f41a35188079c766d47bf8abaffa9945ff | |
parent | 225eda4fff542918580c838f1cafa301ca7ea6fb (diff) | |
download | qtlocation-mapboxgl-6d0b01c5fb1e0740ea48566fea200f03b979c52c.tar.gz |
[core] Make it possible to limit the size of the SQLite cache
-rw-r--r-- | include/mbgl/storage/sqlite_cache.hpp | 3 | ||||
-rw-r--r-- | platform/default/sqlite_cache.cpp | 272 | ||||
-rw-r--r-- | platform/default/sqlite_cache_impl.hpp | 28 | ||||
-rw-r--r-- | platform/linux/main.cpp | 3 |
4 files changed, 269 insertions, 37 deletions
diff --git a/include/mbgl/storage/sqlite_cache.hpp b/include/mbgl/storage/sqlite_cache.hpp index 4b51d94920..6e79d44a33 100644 --- a/include/mbgl/storage/sqlite_cache.hpp +++ b/include/mbgl/storage/sqlite_cache.hpp @@ -16,6 +16,9 @@ public: SQLiteCache(const std::string &path = ":memory:"); ~SQLiteCache() override; + void setMaximumCacheSize(uint64_t size); + void setMaximumCacheEntrySize(uint64_t size); + // FileCache API std::unique_ptr<WorkRequest> get(const Resource &resource, Callback callback) override; void put(const Resource &resource, std::shared_ptr<const Response> response, Hint hint) override; diff --git a/platform/default/sqlite_cache.cpp b/platform/default/sqlite_cache.cpp index 43b67cfd7f..4d8cb8180c 100644 --- a/platform/default/sqlite_cache.cpp +++ b/platform/default/sqlite_cache.cpp @@ -11,6 +11,21 @@ #include "sqlite3.hpp" #include <sqlite3.h> +namespace { + +// The cache won't accept entries larger than this arbitrary size +// and will silently discard request for adding them to the cache. +// Large entries can cause the database to grow in disk size and +// never shrink again. +const uint64_t kMaximumCacheEntrySize = 5 * 1024 * 1024; // 5 MB + +// Number of records we delete when we are close to the maximum +// database size, if set. The current criteria is to prune +// the least used entries based on `accessed` time. +const int kPrunedEntriesLimit = 100; + +} // namespace + namespace mbgl { using namespace mapbox::sqlite; @@ -22,7 +37,9 @@ SQLiteCache::SQLiteCache(const std::string& path_) SQLiteCache::~SQLiteCache() = default; SQLiteCache::Impl::Impl(const std::string& path_) - : path(path_) { + : maximumCacheSize(0), // Unlimited + maximumCacheEntrySize(kMaximumCacheEntrySize), + path(path_) { } SQLiteCache::Impl::~Impl() { @@ -32,6 +49,10 @@ SQLiteCache::Impl::~Impl() { getStmt.reset(); putStmt.reset(); refreshStmt.reset(); + countStmt.reset(); + freeStmt.reset(); + pruneStmt.reset(); + accessedStmt.reset(); db.reset(); } catch (mapbox::sqlite::Exception& ex) { Log::Error(Event::Database, ex.code, ex.what()); @@ -51,10 +72,12 @@ void SQLiteCache::Impl::createSchema() { " `modified` INTEGER," // Timestamp when the file was last modified. " `etag` TEXT," " `expires` INTEGER," // Timestamp when the server says the file expires. + " `accessed` INTEGER," // Timestamp when the database record was last accessed. " `data` BLOB," " `compressed` INTEGER NOT NULL DEFAULT 0" // Whether the data is compressed. ");" - "CREATE INDEX IF NOT EXISTS `http_cache_kind_idx` ON `http_cache` (`kind`);"; + "CREATE INDEX IF NOT EXISTS `http_cache_kind_idx` ON `http_cache` (`kind`);" + "CREATE INDEX IF NOT EXISTS `http_cache_accessed_idx` ON `http_cache` (`modified`);"; try { db->exec(sql); @@ -80,6 +103,167 @@ void SQLiteCache::Impl::createSchema() { } } +void SQLiteCache::setMaximumCacheSize(uint64_t size) { + thread->invoke(&Impl::setMaximumCacheSize, size); +} + +void SQLiteCache::Impl::setMaximumCacheSize(uint64_t size) { + maximumCacheSize = size; + + // Unlimited. + if (size == 0) { + return; + } + + uint64_t lastSoftSize = cacheSoftSize(); + + // Keep pruning until we fit in the new + // size limit. + while (lastSoftSize > maximumCacheSize) { + pruneEntries(); + + if (lastSoftSize != cacheSoftSize()) { + lastSoftSize = cacheSoftSize(); + } else { + break; + } + } + + if (cacheHardSize() > size) { + Log::Warning(mbgl::Event::Database, + "Current cache hard size is bigger than the defined " + "maximum size. Database won't get truncated."); + } +} + +void SQLiteCache::setMaximumCacheEntrySize(uint64_t size) { + thread->invoke(&Impl::setMaximumCacheEntrySize, size); +} + +void SQLiteCache::Impl::setMaximumCacheEntrySize(uint64_t size) { + maximumCacheEntrySize = size; +} + +void SQLiteCache::Impl::initializeDatabase() { + if (!db) { + createDatabase(); + } + + if (!schema) { + createSchema(); + } +} + +int SQLiteCache::Impl::cachePageSize() { + try { + if (!pageSize) { + Statement pageSizeStmt(db->prepare("PRAGMA page_size")); + if (pageSizeStmt.run()) { + pageSize = pageSizeStmt.get<int>(0); + } + } + } catch (mapbox::sqlite::Exception& ex) { + Log::Error(Event::Database, ex.code, ex.what()); + } + + return pageSize; +} + +uint64_t SQLiteCache::Impl::cacheHardSize() { + try { + initializeDatabase(); + + if (!countStmt) { + countStmt = std::make_unique<Statement>(db->prepare("PRAGMA page_count")); + } else { + countStmt->reset(); + } + + if (countStmt->run()) { + return cachePageSize() * countStmt->get<int>(0); + } + } catch (mapbox::sqlite::Exception& ex) { + Log::Error(Event::Database, ex.code, ex.what()); + } + + return 0; +} + +uint64_t SQLiteCache::Impl::cacheSoftSize() { + if (!softSizeDirty) { + return softSize; + } + + try { + initializeDatabase(); + + if (!freeStmt) { + freeStmt = std::make_unique<Statement>(db->prepare("PRAGMA freelist_count")); + } else { + freeStmt->reset(); + } + + uint64_t hardSize = cacheHardSize(); + if (!hardSize) { + return 0; + } + + if (freeStmt->run()) { + return hardSize - cachePageSize() * freeStmt->get<int>(0); + } + + softSizeDirty = false; + } catch (mapbox::sqlite::Exception& ex) { + Log::Error(Event::Database, ex.code, ex.what()); + } + + return 0; +} + +bool SQLiteCache::Impl::needsPruning() { + // SQLite database never shrinks in size unless we call VACCUM. We here + // are monitoring the soft limit (i.e. number of free pages in the file) + // and as it approaches to the hard limit (i.e. the actual file size) we + // delete an arbitrary number of old cache entries. + // + // The free pages approach saves us from calling VACCUM or keeping a + // running total, which can be costly. We need a buffer because pages can + // get fragmented on the database. + if (cacheSoftSize() + maximumCacheEntrySize * 2 < maximumCacheSize) { + return false; + } else { + return true; + } +} + +void SQLiteCache::Impl::pruneEntries() { + if (!maximumCacheSize) { + return; + } + + if (!needsPruning()) { + return; + } + + try { + if (!pruneStmt) { + pruneStmt = std::make_unique<Statement>(db->prepare( + "DELETE FROM `http_cache` WHERE `rowid` IN (SELECT `rowid` FROM " + // 1 + "`http_cache` ORDER BY `accessed` ASC LIMIT ?)")); + } else { + pruneStmt->reset(); + } + + pruneStmt->bind(1, kPrunedEntriesLimit); + + pruneStmt->run(); + softSizeDirty = true; + } catch (mapbox::sqlite::Exception& ex) { + Log::Error(Event::Database, ex.code, ex.what()); + } +} + std::unique_ptr<WorkRequest> SQLiteCache::get(const Resource &resource, Callback callback) { // Can be called from any thread, but most likely from the file source thread. // Will try to load the URL from the SQLite database and call the callback when done. @@ -90,14 +274,7 @@ std::unique_ptr<WorkRequest> SQLiteCache::get(const Resource &resource, Callback void SQLiteCache::Impl::get(const Resource &resource, Callback callback) { try { - // This is called in the SQLite event loop. - if (!db) { - createDatabase(); - } - - if (!schema) { - createSchema(); - } + initializeDatabase(); if (!getStmt) { // Initialize the statement 0 1 @@ -130,6 +307,23 @@ void SQLiteCache::Impl::get(const Resource &resource, Callback callback) { // There is no data. callback(nullptr); } + + // We do an extra query for refreshing the last time + // the record was accessed that can be costly and is only + // worth doing if we are monitoring the database size. + if (maximumCacheSize) { + if (!accessedStmt) { + accessedStmt = std::make_unique<Statement>( + // 1 2 + db->prepare("UPDATE `http_cache` SET `accessed` = ? WHERE `url` = ?")); + } else { + accessedStmt->reset(); + } + + accessedStmt->bind(1, toSeconds(SystemClock::now()).count()); + accessedStmt->bind(2, canonicalURL.c_str()); + accessedStmt->run(); + } } catch (mapbox::sqlite::Exception& ex) { Log::Error(Event::Database, ex.code, ex.what()); callback(nullptr); @@ -152,19 +346,28 @@ void SQLiteCache::put(const Resource &resource, std::shared_ptr<const Response> void SQLiteCache::Impl::put(const Resource& resource, std::shared_ptr<const Response> response) { try { - if (!db) { - createDatabase(); - } + initializeDatabase(); + pruneEntries(); + + if (response->data) { + auto entrySize = response->data->size(); - if (!schema) { - createSchema(); + if (entrySize > maximumCacheEntrySize) { + Log::Warning(Event::Database, "Entry too big for caching."); + return; + } + + if (maximumCacheSize && entrySize + cacheSoftSize() > maximumCacheSize) { + Log::Warning(Event::Database, "Unable to make space for new entries."); + return; + } } if (!putStmt) { putStmt = std::make_unique<Statement>(db->prepare("REPLACE INTO `http_cache` (" - // 1 2 3 4 5 6 7 8 - "`url`, `status`, `kind`, `modified`, `etag`, `expires`, `data`, `compressed`" - ") VALUES(?, ?, ?, ?, ?, ?, ?, ?)")); + // 1 2 3 4 5 6 7 8 9 + "`url`, `status`, `kind`, `modified`, `etag`, `expires`, `accessed`, `data`, `compressed`" + ") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)")); } else { putStmt->reset(); } @@ -180,6 +383,7 @@ void SQLiteCache::Impl::put(const Resource& resource, std::shared_ptr<const Resp putStmt->bind(4 /* modified */, response->modified.count()); putStmt->bind(5 /* etag */, response->etag.c_str()); putStmt->bind(6 /* expires */, response->expires.count()); + putStmt->bind(7 /* accessed */, toSeconds(SystemClock::now()).count()); std::string data; if (resource.kind != Resource::SpriteImage && response->data) { @@ -190,17 +394,18 @@ void SQLiteCache::Impl::put(const Resource& resource, std::shared_ptr<const Resp if (!data.empty() && data.size() < response->data->size()) { // Store the compressed data when it is smaller than the original // uncompressed data. - putStmt->bind(7 /* data */, data, false); // do not retain the string internally. - putStmt->bind(8 /* compressed */, true); + putStmt->bind(8 /* data */, data, false); // do not retain the string internally. + putStmt->bind(9 /* compressed */, true); } else if (response->data) { - putStmt->bind(7 /* data */, *response->data, false); // do not retain the string internally. - putStmt->bind(8 /* compressed */, false); + putStmt->bind(8 /* data */, *response->data, false); // do not retain the string internally. + putStmt->bind(9 /* compressed */, false); } else { - putStmt->bind(7 /* data */, "", false); - putStmt->bind(8 /* compressed */, false); + putStmt->bind(8 /* data */, "", false); + putStmt->bind(9 /* compressed */, false); } putStmt->run(); + softSizeDirty = true; } catch (mapbox::sqlite::Exception& ex) { Log::Error(Event::Database, ex.code, ex.what()); } catch (std::runtime_error& ex) { @@ -210,24 +415,21 @@ void SQLiteCache::Impl::put(const Resource& resource, std::shared_ptr<const Resp void SQLiteCache::Impl::refresh(const Resource& resource, Seconds expires) { try { - if (!db) { - createDatabase(); - } - - if (!schema) { - createSchema(); - } + initializeDatabase(); if (!refreshStmt) { - refreshStmt = std::make_unique<Statement>( // 1 2 - db->prepare("UPDATE `http_cache` SET `expires` = ? WHERE `url` = ?")); + refreshStmt = std::make_unique<Statement>( + db->prepare("UPDATE `http_cache` SET " + // 1 2 3 + "`accessed` = ?, `expires` = ? WHERE `url` = ?")); } else { refreshStmt->reset(); } const auto canonicalURL = util::mapbox::canonicalURL(resource.url); - refreshStmt->bind(1, expires.count()); - refreshStmt->bind(2, canonicalURL.c_str()); + refreshStmt->bind(1, toSeconds(SystemClock::now()).count()); + refreshStmt->bind(2, expires.count()); + refreshStmt->bind(3, canonicalURL.c_str()); refreshStmt->run(); } catch (mapbox::sqlite::Exception& ex) { Log::Error(Event::Database, ex.code, ex.what()); diff --git a/platform/default/sqlite_cache_impl.hpp b/platform/default/sqlite_cache_impl.hpp index f557666e3f..be2db5df43 100644 --- a/platform/default/sqlite_cache_impl.hpp +++ b/platform/default/sqlite_cache_impl.hpp @@ -18,23 +18,47 @@ public: explicit Impl(const std::string &path = ":memory:"); ~Impl(); + void setMaximumCacheSize(uint64_t size); + void setMaximumCacheEntrySize(uint64_t size); + void get(const Resource&, Callback); void put(const Resource& resource, std::shared_ptr<const Response> response); void refresh(const Resource& resource, Seconds expires); private: + void initializeDatabase(); + + int cachePageSize(); + + uint64_t cacheHardSize(); + uint64_t cacheSoftSize(); + + uint64_t softSize = 0; + bool softSizeDirty = true; + + bool needsPruning(); + void pruneEntries(); + void createDatabase(); void createSchema(); + int pageSize = 0; + + uint64_t maximumCacheSize; + uint64_t maximumCacheEntrySize; + const std::string path; std::unique_ptr<::mapbox::sqlite::Database> db; std::unique_ptr<::mapbox::sqlite::Statement> getStmt; std::unique_ptr<::mapbox::sqlite::Statement> putStmt; std::unique_ptr<::mapbox::sqlite::Statement> refreshStmt; + std::unique_ptr<::mapbox::sqlite::Statement> countStmt; + std::unique_ptr<::mapbox::sqlite::Statement> freeStmt; + std::unique_ptr<::mapbox::sqlite::Statement> pruneStmt; + std::unique_ptr<::mapbox::sqlite::Statement> accessedStmt; bool schema = false; }; - -} +} // namespace mbgl #endif diff --git a/platform/linux/main.cpp b/platform/linux/main.cpp index bb571a0093..45f2b8defa 100644 --- a/platform/linux/main.cpp +++ b/platform/linux/main.cpp @@ -107,6 +107,9 @@ int main(int argc, char *argv[]) { view = std::make_unique<GLFWView>(fullscreen, benchmark); mbgl::SQLiteCache cache("/tmp/mbgl-cache.db"); + cache.setMaximumCacheEntrySize(1 * 1024 * 1024); // 1 MB + cache.setMaximumCacheSize(50 * 1024 * 1024); // 50 MB + mbgl::DefaultFileSource fileSource(&cache); // Set access token if present |