summaryrefslogtreecommitdiff
path: root/platform/default
diff options
context:
space:
mode:
Diffstat (limited to 'platform/default')
-rw-r--r--platform/default/default_file_source.cpp33
-rw-r--r--platform/default/mbgl/storage/offline_database.cpp136
-rw-r--r--platform/default/mbgl/storage/offline_database.hpp17
-rw-r--r--platform/default/sqlite3.cpp5
-rw-r--r--platform/default/sqlite3.hpp1
5 files changed, 142 insertions, 50 deletions
diff --git a/platform/default/default_file_source.cpp b/platform/default/default_file_source.cpp
index cdf5e1e442..db0f4ccac6 100644
--- a/platform/default/default_file_source.cpp
+++ b/platform/default/default_file_source.cpp
@@ -50,8 +50,8 @@ public:
std::unique_ptr<FileRequest> onlineRequest;
};
- Impl(const std::string& cachePath)
- : offlineDatabase(cachePath) {
+ Impl(const std::string& cachePath, uint64_t maximumCacheSize, uint64_t maximumCacheEntrySize)
+ : offlineDatabase(cachePath, maximumCacheSize, maximumCacheEntrySize) {
}
void setAccessToken(const std::string& accessToken) {
@@ -97,15 +97,6 @@ public:
}
}
- void removeUnusedOfflineResources(std::function<void (std::exception_ptr)> callback) {
- try {
- offlineDatabase.removeUnusedResources();
- callback({});
- } catch (...) {
- callback(std::current_exception());
- }
- }
-
void setRegionObserver(int64_t regionID, std::unique_ptr<OfflineRegionObserver> observer) {
getDownload(regionID).setObserver(std::move(observer));
}
@@ -147,8 +138,12 @@ private:
bool offline = false;
};
-DefaultFileSource::DefaultFileSource(const std::string& cachePath, const std::string& assetRoot)
- : thread(std::make_unique<util::Thread<DefaultFileSource::Impl>>(util::ThreadContext{"DefaultFileSource", util::ThreadType::Unknown, util::ThreadPriority::Low}, cachePath)),
+DefaultFileSource::DefaultFileSource(const std::string& cachePath,
+ const std::string& assetRoot,
+ uint64_t maximumCacheSize,
+ uint64_t maximumCacheEntrySize)
+ : thread(std::make_unique<util::Thread<Impl>>(util::ThreadContext{"DefaultFileSource", util::ThreadType::Unknown, util::ThreadPriority::Low},
+ cachePath, maximumCacheSize, maximumCacheEntrySize)),
assetFileSource(std::make_unique<AssetFileSource>(assetRoot)) {
}
@@ -162,14 +157,6 @@ std::string DefaultFileSource::getAccessToken() const {
return thread->invokeSync<std::string>(&Impl::getAccessToken);
}
-void DefaultFileSource::setMaximumCacheSize(uint64_t) {
- // TODO
-}
-
-void DefaultFileSource::setMaximumCacheEntrySize(uint64_t) {
- // TODO
-}
-
std::unique_ptr<FileRequest> DefaultFileSource::request(const Resource& resource, Callback callback) {
class DefaultFileRequest : public FileRequest {
public:
@@ -219,10 +206,6 @@ void DefaultFileSource::getOfflineRegionStatus(OfflineRegion& region, std::funct
thread->invoke(&Impl::getRegionStatus, region.getID(), callback);
}
-void DefaultFileSource::removeUnusedOfflineResources(std::function<void (std::exception_ptr)> callback) {
- thread->invoke(&Impl::removeUnusedOfflineResources, callback);
-}
-
// For testing only:
void DefaultFileSource::put(const Resource& resource, const Response& response) {
diff --git a/platform/default/mbgl/storage/offline_database.cpp b/platform/default/mbgl/storage/offline_database.cpp
index baaa1628d5..d775c40759 100644
--- a/platform/default/mbgl/storage/offline_database.cpp
+++ b/platform/default/mbgl/storage/offline_database.cpp
@@ -22,8 +22,10 @@ OfflineDatabase::Statement::~Statement() {
stmt.clearBindings();
}
-OfflineDatabase::OfflineDatabase(const std::string& path_)
- : path(path_) {
+OfflineDatabase::OfflineDatabase(const std::string& path_, uint64_t maximumCacheSize_, uint64_t maximumCacheEntrySize_)
+ : path(path_),
+ maximumCacheSize(maximumCacheSize_),
+ maximumCacheEntrySize(maximumCacheEntrySize_) {
ensureSchema();
}
@@ -110,6 +112,18 @@ optional<Response> OfflineDatabase::get(const Resource& resource) {
}
uint64_t OfflineDatabase::put(const Resource& resource, const Response& response) {
+ if (response.data && response.data->size() > maximumCacheEntrySize) {
+ Log::Warning(Event::Database, "Entry too big for caching");
+ return 0;
+ } else if (!evict()) {
+ Log::Warning(Event::Database, "Unable to make space for new entries");
+ return 0;
+ } else {
+ return putInternal(resource, response);
+ }
+}
+
+uint64_t OfflineDatabase::putInternal(const Resource& resource, const Response& response) {
// Don't store errors in the cache.
if (response.error) {
return 0;
@@ -124,6 +138,13 @@ uint64_t OfflineDatabase::put(const Resource& resource, const Response& response
}
optional<Response> OfflineDatabase::getResource(const Resource& resource) {
+ Statement accessedStmt = getStatement(
+ "UPDATE resources SET accessed = ?1 WHERE url = ?2");
+
+ accessedStmt->bind(1, SystemClock::now());
+ accessedStmt->bind(2, resource.url);
+ accessedStmt->run();
+
Statement stmt = getStatement(
// 0 1 2 3 4
"SELECT etag, expires, modified, data, compressed "
@@ -203,6 +224,24 @@ uint64_t OfflineDatabase::putResource(const Resource& resource, const Response&
}
optional<Response> OfflineDatabase::getTile(const Resource::TileData& tile) {
+ Statement accessedStmt = getStatement(
+ "UPDATE tiles SET accessed = ?1 "
+ "WHERE tileset_id = ( "
+ " SELECT id FROM tilesets "
+ " WHERE url_template = ?2 "
+ " AND pixel_ratio = ?3) "
+ "AND tiles.x = ?4 "
+ "AND tiles.y = ?5 "
+ "AND tiles.z = ?6 ");
+
+ accessedStmt->bind(1, SystemClock::now());
+ accessedStmt->bind(2, tile.urlTemplate);
+ accessedStmt->bind(3, tile.pixelRatio);
+ accessedStmt->bind(4, tile.x);
+ accessedStmt->bind(5, tile.y);
+ accessedStmt->bind(6, tile.z);
+ accessedStmt->run();
+
Statement stmt = getStatement(
// 0 1 2 3 4
"SELECT etag, expires, modified, data, compressed "
@@ -348,6 +387,8 @@ void OfflineDatabase::deleteRegion(OfflineRegion&& region) {
stmt->bind(1, region.getID());
stmt->run();
+
+ evict();
}
optional<Response> OfflineDatabase::getRegionResource(int64_t regionID, const Resource& resource) {
@@ -361,7 +402,7 @@ optional<Response> OfflineDatabase::getRegionResource(int64_t regionID, const Re
}
uint64_t OfflineDatabase::putRegionResource(int64_t regionID, const Resource& resource, const Response& response) {
- uint64_t result = put(resource, response);
+ uint64_t result = putInternal(resource, response);
markUsed(regionID, resource);
return result;
}
@@ -431,27 +472,76 @@ OfflineRegionStatus OfflineDatabase::getRegionCompletedStatus(int64_t regionID)
return result;
}
-void OfflineDatabase::removeUnusedResources() {
- Statement stmt1 = getStatement(
- "DELETE FROM resources "
- "WHERE ROWID NOT IN ( "
- " SELECT resources.ROWID "
- " FROM resources, region_resources "
- " WHERE resources.url = region_resources.resource_url "
- ") ");
- stmt1->run();
+template <class T>
+T OfflineDatabase::getPragma(const char * sql) {
+ Statement stmt = getStatement(sql);
+ stmt->run();
+ return stmt->get<T>(0);
+}
- Statement stmt2 = getStatement(
- "DELETE FROM tiles "
- "WHERE ROWID NOT IN ( "
- " SELECT tiles.ROWID "
- " FROM tiles, region_tiles "
- " AND tiles.tileset_id = region_tiles.tileset_id "
- " AND tiles.z = region_tiles.z "
- " AND tiles.x = region_tiles.x "
- " AND tiles.y = region_tiles.y "
- ") ");
- stmt2->run();
+// Remove least-recently used resources and tiles until the used database size,
+// as calculated by multiplying the number of in-use pages by the page size, is
+// less than the maximum cache size. Returns false if this condition cannot be
+// satisfied.
+//
+// SQLite database never shrinks in size unless we call VACCUM. We here
+// are monitoring the soft limit (i.e. number of free pages in the file)
+// and as it approaches to the hard limit (i.e. the actual file size) we
+// delete an arbitrary number of old cache entries.
+//
+// The free pages approach saves us from calling VACCUM or keeping a
+// running total, which can be costly. We need a buffer because pages can
+// get fragmented on the database.
+bool OfflineDatabase::evict() {
+ uint64_t pageSize = getPragma<int64_t>("PRAGMA page_size");
+ uint64_t pageCount = getPragma<int64_t>("PRAGMA page_count");
+
+ if (pageSize * pageCount > maximumCacheSize) {
+ Log::Warning(mbgl::Event::Database, "Current size is larger than the maximum size. Database won't get truncated.");
+ }
+
+ auto usedSize = [&] {
+ return pageSize * (pageCount - getPragma<int64_t>("PRAGMA freelist_count"));
+ };
+
+ while (usedSize() > maximumCacheSize - 5 * pageSize) {
+ Statement stmt1 = getStatement(
+ "DELETE FROM resources "
+ "WHERE ROWID IN ( "
+ " SELECT resources.ROWID "
+ " FROM resources "
+ " LEFT JOIN region_resources "
+ " ON resources.url = region_resources.resource_url "
+ " WHERE region_resources.resource_url IS NULL "
+ " ORDER BY accessed ASC LIMIT ?1 "
+ ") ");
+ stmt1->bind(1, 50);
+ stmt1->run();
+ uint64_t changes1 = db->changes();
+
+ Statement stmt2 = getStatement(
+ "DELETE FROM tiles "
+ "WHERE ROWID IN ( "
+ " SELECT tiles.ROWID "
+ " FROM tiles "
+ " LEFT JOIN region_tiles "
+ " ON tiles.tileset_id = region_tiles.tileset_id "
+ " AND tiles.z = region_tiles.z "
+ " AND tiles.x = region_tiles.x "
+ " AND tiles.y = region_tiles.y "
+ " WHERE region_tiles.tileset_id IS NULL "
+ " ORDER BY accessed ASC LIMIT ?1 "
+ ") ");
+ stmt2->bind(1, 50);
+ stmt2->run();
+ uint64_t changes2 = db->changes();
+
+ if (changes1 == 0 && changes2 == 0) {
+ return false;
+ }
+ }
+
+ return true;
}
} // namespace mbgl
diff --git a/platform/default/mbgl/storage/offline_database.hpp b/platform/default/mbgl/storage/offline_database.hpp
index b0b02f871f..650cf6a16c 100644
--- a/platform/default/mbgl/storage/offline_database.hpp
+++ b/platform/default/mbgl/storage/offline_database.hpp
@@ -5,6 +5,7 @@
#include <mbgl/storage/offline.hpp>
#include <mbgl/util/noncopyable.hpp>
#include <mbgl/util/optional.hpp>
+#include <mbgl/util/constants.hpp>
#include <unordered_map>
#include <memory>
@@ -24,7 +25,11 @@ class TileID;
class OfflineDatabase : private util::noncopyable {
public:
- OfflineDatabase(const std::string& path);
+ // Limits affect ambient caching (put) only; resources required by offline
+ // regions are exempt.
+ OfflineDatabase(const std::string& path,
+ uint64_t maximumCacheSize = util::DEFAULT_MAX_CACHE_SIZE,
+ uint64_t maximumCacheEntrySize = util::DEFAULT_MAX_CACHE_ENTRY_SIZE);
~OfflineDatabase();
optional<Response> get(const Resource&);
@@ -36,7 +41,6 @@ public:
const OfflineRegionMetadata&);
void deleteRegion(OfflineRegion&&);
- void removeUnusedResources();
optional<Response> getRegionResource(int64_t regionID, const Resource&);
uint64_t putRegionResource(int64_t regionID, const Resource&, const Response&);
@@ -62,6 +66,7 @@ private:
};
Statement getStatement(const char *);
+ uint64_t putInternal(const Resource&, const Response&);
optional<Response> getTile(const Resource::TileData&);
uint64_t putTile(const Resource::TileData&, const Response&);
@@ -74,6 +79,14 @@ private:
const std::string path;
std::unique_ptr<::mapbox::sqlite::Database> db;
std::unordered_map<const char *, std::unique_ptr<::mapbox::sqlite::Statement>> statements;
+
+ template <class T>
+ T getPragma(const char *);
+
+ uint64_t maximumCacheSize;
+ uint64_t maximumCacheEntrySize;
+
+ bool evict();
};
} // namespace mbgl
diff --git a/platform/default/sqlite3.cpp b/platform/default/sqlite3.cpp
index 5059c85c80..72296c6843 100644
--- a/platform/default/sqlite3.cpp
+++ b/platform/default/sqlite3.cpp
@@ -91,6 +91,11 @@ int64_t Database::lastInsertRowid() const {
return sqlite3_last_insert_rowid(db);
}
+uint64_t Database::changes() const {
+ assert(db);
+ return sqlite3_changes(db);
+}
+
Statement::Statement(sqlite3 *db, const char *sql) {
const int err = sqlite3_prepare_v2(db, sql, -1, &stmt, nullptr);
if (err != SQLITE_OK) {
diff --git a/platform/default/sqlite3.hpp b/platform/default/sqlite3.hpp
index 13e92bf07f..abe83a2d44 100644
--- a/platform/default/sqlite3.hpp
+++ b/platform/default/sqlite3.hpp
@@ -47,6 +47,7 @@ public:
Statement prepare(const char *query);
int64_t lastInsertRowid() const;
+ uint64_t changes() const;
private:
sqlite3 *db = nullptr;
View Lorry Controller Status