summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJohn Firebaugh <john.firebaugh@gmail.com>2016-01-26 12:19:04 -0800
committerJohn Firebaugh <john.firebaugh@gmail.com>2016-02-10 15:40:20 -0800
commitcdedb66387065680efd318dacb61572c920b81b1 (patch)
tree3cc2676e15fac4c7b024cd17603c46e341f5a589
parent025375ad0b365a06e0742b92fecc9bc538b5a6e0 (diff)
downloadqtlocation-mapboxgl-cdedb66387065680efd318dacb61572c920b81b1.tar.gz
[core] Reimplement existing caching within an offline-capable database schema
-rw-r--r--.gitignore4
-rw-r--r--gyp/platform-android.gypi4
-rw-r--r--gyp/platform-ios.gypi4
-rw-r--r--gyp/platform-linux.gypi4
-rw-r--r--gyp/platform-osx.gypi4
-rw-r--r--include/mbgl/storage/default_file_source.hpp15
-rw-r--r--platform/default/default_file_source.cpp136
-rw-r--r--platform/default/mbgl/storage/offline_database.cpp297
-rw-r--r--platform/default/mbgl/storage/offline_database.hpp50
-rw-r--r--platform/default/mbgl/storage/offline_schema.cpp.include51
-rw-r--r--platform/default/mbgl/storage/offline_schema.js24
-rw-r--r--platform/default/mbgl/storage/offline_schema.sql62
-rw-r--r--platform/default/sqlite3.cpp34
-rw-r--r--platform/default/sqlite_cache.cpp502
-rw-r--r--platform/default/sqlite_cache_impl.hpp68
-rw-r--r--src/mbgl/storage/sqlite_cache.hpp44
-rw-r--r--test/api/offline.cpp50
-rw-r--r--test/fixtures/offline/0-0-0.vector.pbf (renamed from test/fixtures/stale/0-0-0.vector.pbf)bin482553 -> 482553 bytes
-rw-r--r--test/fixtures/offline/expected.pngbin0 -> 47842 bytes
-rw-r--r--test/fixtures/offline/glyph.pbf (renamed from test/fixtures/stale/glyph.pbf)bin74722 -> 74722 bytes
-rw-r--r--test/fixtures/offline/sprite.json (renamed from test/fixtures/stale/sprite.json)0
-rw-r--r--test/fixtures/offline/sprite.png (renamed from test/fixtures/stale/sprite.png)bin3351 -> 3351 bytes
-rw-r--r--test/fixtures/offline/streets.json (renamed from test/fixtures/stale/streets.json)2
-rw-r--r--test/fixtures/offline/style.json (renamed from test/fixtures/stale/style_and_glyphs.json)10
-rw-r--r--test/fixtures/stale/stale_style/expected.pngbin13447 -> 0 bytes
-rw-r--r--test/fixtures/stale/stale_style_and_glyphs/expected.pngbin36120 -> 0 bytes
-rw-r--r--test/fixtures/stale/stale_style_and_sprite/expected.pngbin26859 -> 0 bytes
-rw-r--r--test/fixtures/stale/stale_style_and_tilejson/expected.pngbin13447 -> 0 bytes
-rw-r--r--test/fixtures/stale/style.json25
-rw-r--r--test/fixtures/stale/style_and_sprite.json26
-rw-r--r--test/fixtures/stale/style_and_tilejson.json25
-rw-r--r--test/fixtures/util.cpp12
-rw-r--r--test/fixtures/util.hpp4
-rw-r--r--test/storage/asset_file_source.cpp1
-rw-r--r--test/storage/cache_response.cpp215
-rw-r--r--test/storage/cache_shared.cpp34
-rw-r--r--test/storage/cache_size.cpp241
-rw-r--r--test/storage/cache_stale.cpp131
-rw-r--r--test/storage/database.cpp349
-rw-r--r--test/storage/default_file_source.cpp (renamed from test/storage/cache_revalidate.cpp)52
-rw-r--r--test/storage/offline_database.cpp374
-rw-r--r--test/test.gypi9
42 files changed, 1114 insertions, 1749 deletions
diff --git a/.gitignore b/.gitignore
index f9fd16cd2b..f750e2f876 100644
--- a/.gitignore
+++ b/.gitignore
@@ -23,8 +23,8 @@
/test/fixtures/api/1.png
/test/fixtures/api/2.png
/test/fixtures/database/*.db
-/test/fixtures/*/*/actual.png
-/test/fixtures/*/*/diff.png
+/test/fixtures/**/actual.png
+/test/fixtures/**/diff.png
/test/output
/include/mbgl/shader/shaders.hpp
/src/shader/shaders_gl.cpp
diff --git a/gyp/platform-android.gypi b/gyp/platform-android.gypi
index 8bb046a588..2c1e06a715 100644
--- a/gyp/platform-android.gypi
+++ b/gyp/platform-android.gypi
@@ -21,7 +21,8 @@
'../platform/default/timer.cpp',
'../platform/default/default_file_source.cpp',
'../platform/default/online_file_source.cpp',
- '../platform/default/sqlite_cache.cpp',
+ '../platform/default/mbgl/storage/offline_database.hpp',
+ '../platform/default/mbgl/storage/offline_database.cpp',
'../platform/default/sqlite3.hpp',
'../platform/default/sqlite3.cpp',
],
@@ -56,6 +57,7 @@
'include_dirs': [
'../include',
'../src',
+ '../platform/default',
],
'conditions': [
diff --git a/gyp/platform-ios.gypi b/gyp/platform-ios.gypi
index f6d94b0e1c..806f103a17 100644
--- a/gyp/platform-ios.gypi
+++ b/gyp/platform-ios.gypi
@@ -16,7 +16,8 @@
'../platform/default/timer.cpp',
'../platform/default/default_file_source.cpp',
'../platform/default/online_file_source.cpp',
- '../platform/default/sqlite_cache.cpp',
+ '../platform/default/mbgl/storage/offline_database.hpp',
+ '../platform/default/mbgl/storage/offline_database.cpp',
'../platform/default/sqlite3.hpp',
'../platform/default/sqlite3.cpp',
'../platform/darwin/log_nslog.mm',
@@ -98,6 +99,7 @@
'../include/mbgl/darwin',
'../include',
'../src',
+ '../platform/default',
],
'xcode_settings': {
diff --git a/gyp/platform-linux.gypi b/gyp/platform-linux.gypi
index 1513828821..2eafe3e821 100644
--- a/gyp/platform-linux.gypi
+++ b/gyp/platform-linux.gypi
@@ -23,7 +23,8 @@
'../platform/default/timer.cpp',
'../platform/default/default_file_source.cpp',
'../platform/default/online_file_source.cpp',
- '../platform/default/sqlite_cache.cpp',
+ '../platform/default/mbgl/storage/offline_database.hpp',
+ '../platform/default/mbgl/storage/offline_database.cpp',
'../platform/default/sqlite3.hpp',
'../platform/default/sqlite3.cpp',
],
@@ -61,6 +62,7 @@
'include_dirs': [
'../include',
'../src',
+ '../platform/default',
],
'conditions': [
diff --git a/gyp/platform-osx.gypi b/gyp/platform-osx.gypi
index 2aedce8538..53e7047492 100644
--- a/gyp/platform-osx.gypi
+++ b/gyp/platform-osx.gypi
@@ -15,7 +15,8 @@
'../platform/default/timer.cpp',
'../platform/default/default_file_source.cpp',
'../platform/default/online_file_source.cpp',
- '../platform/default/sqlite_cache.cpp',
+ '../platform/default/mbgl/storage/offline_database.hpp',
+ '../platform/default/mbgl/storage/offline_database.cpp',
'../platform/default/sqlite3.hpp',
'../platform/default/sqlite3.cpp',
'../platform/darwin/log_nslog.mm',
@@ -83,6 +84,7 @@
'../include/mbgl/darwin',
'../include',
'../src',
+ '../platform/default',
],
'xcode_settings': {
diff --git a/include/mbgl/storage/default_file_source.hpp b/include/mbgl/storage/default_file_source.hpp
index e7f95aa238..d8fc4b98a3 100644
--- a/include/mbgl/storage/default_file_source.hpp
+++ b/include/mbgl/storage/default_file_source.hpp
@@ -5,7 +5,9 @@
namespace mbgl {
-class SQLiteCache;
+namespace util {
+template <typename T> class Thread;
+} // namespace util
class DefaultFileSource : public FileSource {
public:
@@ -20,14 +22,15 @@ public:
std::unique_ptr<FileRequest> request(const Resource&, Callback) override;
- // For testing purposes only.
- SQLiteCache& getCache();
+ // For testing only.
+ void put(const Resource&, const Response&);
+ void goOffline();
-private:
class Impl;
- friend class DefaultFileRequest;
- const std::unique_ptr<Impl> impl;
+private:
+ const std::unique_ptr<util::Thread<Impl>> thread;
+ const std::unique_ptr<FileSource> assetFileSource;
};
} // namespace mbgl
diff --git a/platform/default/default_file_source.cpp b/platform/default/default_file_source.cpp
index 56b47539b1..efe893d49b 100644
--- a/platform/default/default_file_source.cpp
+++ b/platform/default/default_file_source.cpp
@@ -1,10 +1,11 @@
#include <mbgl/storage/default_file_source.hpp>
#include <mbgl/storage/asset_file_source.hpp>
#include <mbgl/storage/online_file_source.hpp>
-#include <mbgl/storage/sqlite_cache.hpp>
+#include <mbgl/storage/offline_database.hpp>
#include <mbgl/platform/platform.hpp>
#include <mbgl/util/url.hpp>
+#include <mbgl/util/thread.hpp>
#include <mbgl/util/work_request.hpp>
#include <cassert>
@@ -23,76 +24,117 @@ namespace mbgl {
class DefaultFileSource::Impl {
public:
- Impl(const std::string& cachePath, const std::string& assetRoot)
- : assetFileSource(assetRoot),
- cache(SQLiteCache::getShared(cachePath)) {
+ class Task {
+ public:
+ Task(Resource resource, FileSource::Callback callback, DefaultFileSource::Impl* impl) {
+ auto offlineResponse = impl->offlineDatabase.get(resource);
+
+ Resource revalidation = resource;
+
+ if (offlineResponse) {
+ revalidation.priorModified = offlineResponse->modified;
+ revalidation.priorExpires = offlineResponse->expires;
+ revalidation.priorEtag = offlineResponse->etag;
+ callback(*offlineResponse);
+ }
+
+ if (!impl->offline) {
+ onlineRequest = impl->onlineFileSource.request(revalidation, [=] (Response onlineResponse) {
+ impl->offlineDatabase.put(revalidation, onlineResponse);
+ callback(onlineResponse);
+ });
+ }
+ }
+
+ std::unique_ptr<FileRequest> onlineRequest;
+ };
+
+ Impl(const std::string& cachePath)
+ : offlineDatabase(cachePath) {
+ }
+
+ void setAccessToken(const std::string& accessToken) {
+ onlineFileSource.setAccessToken(accessToken);
+ }
+
+ std::string getAccessToken() const {
+ return onlineFileSource.getAccessToken();
+ }
+
+ void add(FileRequest* req, Resource resource, Callback callback) {
+ tasks[req] = std::make_unique<Task>(resource, callback, this);
}
- AssetFileSource assetFileSource;
- std::shared_ptr<SQLiteCache> cache;
+ void cancel(FileRequest* req) {
+ tasks.erase(req);
+ }
+
+ void put(const Resource& resource, const Response& response) {
+ offlineDatabase.put(resource, response);
+ }
+
+ void goOffline() {
+ offline = true;
+ }
+
+ OfflineDatabase offlineDatabase;
OnlineFileSource onlineFileSource;
+ std::unordered_map<FileRequest*, std::unique_ptr<Task>> tasks;
+ bool offline = false;
+};
+
+class DefaultFileRequest : public FileRequest {
+public:
+ DefaultFileRequest(Resource resource, FileSource::Callback callback, util::Thread<DefaultFileSource::Impl>& thread_)
+ : thread(thread_),
+ workRequest(thread.invokeWithCallback(&DefaultFileSource::Impl::add, callback, this, resource)) {
+ }
+
+ ~DefaultFileRequest() {
+ thread.invoke(&DefaultFileSource::Impl::cancel, this);
+ }
+
+ util::Thread<DefaultFileSource::Impl>& thread;
+ std::unique_ptr<WorkRequest> workRequest;
};
DefaultFileSource::DefaultFileSource(const std::string& cachePath, const std::string& assetRoot)
- : impl(std::make_unique<DefaultFileSource::Impl>(cachePath, assetRoot)) {
+ : thread(std::make_unique<util::Thread<DefaultFileSource::Impl>>(util::ThreadContext{"DefaultFileSource", util::ThreadType::Unknown, util::ThreadPriority::Low}, cachePath)),
+ assetFileSource(std::make_unique<AssetFileSource>(assetRoot)) {
}
DefaultFileSource::~DefaultFileSource() = default;
void DefaultFileSource::setAccessToken(const std::string& accessToken) {
- impl->onlineFileSource.setAccessToken(accessToken);
+ thread->invokeSync(&Impl::setAccessToken, accessToken);
}
std::string DefaultFileSource::getAccessToken() const {
- return impl->onlineFileSource.getAccessToken();
-}
-
-void DefaultFileSource::setMaximumCacheSize(uint64_t size) {
- impl->cache->setMaximumCacheSize(size);
+ return thread->invokeSync<std::string>(&Impl::getAccessToken);
}
-void DefaultFileSource::setMaximumCacheEntrySize(uint64_t size) {
- impl->cache->setMaximumCacheEntrySize(size);
+void DefaultFileSource::setMaximumCacheSize(uint64_t) {
+ // TODO
}
-SQLiteCache& DefaultFileSource::getCache() {
- return *impl->cache;
+void DefaultFileSource::setMaximumCacheEntrySize(uint64_t) {
+ // TODO
}
-class DefaultFileRequest : public FileRequest {
-public:
- DefaultFileRequest(Resource resource, FileSource::Callback callback, DefaultFileSource::Impl* impl) {
- cacheRequest = impl->cache->get(resource, [=](std::shared_ptr<Response> cacheResponse) mutable {
- cacheRequest.reset();
-
- if (cacheResponse) {
- resource.priorModified = cacheResponse->modified;
- resource.priorExpires = cacheResponse->expires;
- resource.priorEtag = cacheResponse->etag;
- }
-
- onlineRequest = impl->onlineFileSource.request(resource, [=] (Response onlineResponse) {
- impl->cache->put(resource, onlineResponse);
- callback(onlineResponse);
- });
-
- // Do this last because it may result in deleting this DefaultFileRequest.
- if (cacheResponse) {
- callback(*cacheResponse);
- }
- });
- }
-
- std::unique_ptr<WorkRequest> cacheRequest;
- std::unique_ptr<FileRequest> onlineRequest;
-};
-
std::unique_ptr<FileRequest> DefaultFileSource::request(const Resource& resource, Callback callback) {
if (isAssetURL(resource.url)) {
- return impl->assetFileSource.request(resource, callback);
+ return assetFileSource->request(resource, callback);
} else {
- return std::make_unique<DefaultFileRequest>(resource, callback, impl.get());
+ return std::make_unique<DefaultFileRequest>(resource, callback, *thread);
}
}
+void DefaultFileSource::put(const Resource& resource, const Response& response) {
+ thread->invokeSync(&Impl::put, resource, response);
+}
+
+void DefaultFileSource::goOffline() {
+ thread->invokeSync(&Impl::goOffline);
+}
+
} // namespace mbgl
diff --git a/platform/default/mbgl/storage/offline_database.cpp b/platform/default/mbgl/storage/offline_database.cpp
new file mode 100644
index 0000000000..cd0f88b7fe
--- /dev/null
+++ b/platform/default/mbgl/storage/offline_database.cpp
@@ -0,0 +1,297 @@
+#include <mbgl/storage/offline_database.hpp>
+#include <mbgl/storage/response.hpp>
+#include <mbgl/util/compression.hpp>
+#include <mbgl/util/io.hpp>
+#include <mbgl/util/string.hpp>
+#include <mbgl/util/chrono.hpp>
+#include <mbgl/map/tile_id.hpp>
+#include <mbgl/platform/log.hpp>
+
+#include "sqlite3.hpp"
+#include <sqlite3.h>
+
+namespace mbgl {
+
+using namespace mapbox::sqlite;
+
+// If you change the schema you must write a migration from the previous version.
+static const uint32_t schemaVersion = 2;
+
+OfflineDatabase::OfflineDatabase(const std::string& path_)
+ : path(path_) {
+ ensureSchema();
+}
+
+OfflineDatabase::~OfflineDatabase() {
+ // Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we
+ // can't throw anything.
+ try {
+ statements.clear();
+ db.reset();
+ } catch (mapbox::sqlite::Exception& ex) {
+ Log::Error(Event::Database, ex.code, ex.what());
+ }
+}
+
+void OfflineDatabase::ensureSchema() {
+ if (path != ":memory:") {
+ try {
+ db = std::make_unique<Database>(path.c_str(), ReadWrite);
+
+ {
+ Statement userVersionStmt(db->prepare("PRAGMA user_version"));
+ userVersionStmt.run();
+ switch (userVersionStmt.get<int>(0)) {
+ case 0: break; // cache-only database; ok to delete
+ case 1: break; // cache-only database; ok to delete
+ case 2: return;
+ default: throw std::runtime_error("unknown schema version");
+ }
+ }
+
+ removeExisting();
+ db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
+ } catch (mapbox::sqlite::Exception& ex) {
+ if (ex.code == SQLITE_CANTOPEN) {
+ db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
+ } else if (ex.code == SQLITE_NOTADB) {
+ removeExisting();
+ db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
+ }
+ }
+ }
+
+ #include "offline_schema.cpp.include"
+
+ db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
+ db->exec(schema);
+ db->exec("PRAGMA user_version = " + util::toString(schemaVersion));
+}
+
+void OfflineDatabase::removeExisting() {
+ Log::Warning(Event::Database, "Removing existing incompatible offline database");
+
+ db.reset();
+
+ try {
+ util::deleteFile(path);
+ } catch (util::IOException& ex) {
+ Log::Error(Event::Database, ex.code, ex.what());
+ }
+}
+
+mapbox::sqlite::Statement& OfflineDatabase::getStatement(const char * sql) {
+ auto it = statements.find(sql);
+
+ if (it != statements.end()) {
+ it->second->reset();
+ return *it->second;
+ }
+
+ return *(statements[sql] = std::make_unique<Statement>(db->prepare(sql)));
+}
+
+optional<Response> OfflineDatabase::get(const Resource& resource) {
+ if (resource.kind == Resource::Kind::Tile) {
+ assert(resource.tileData);
+ return getTile(*resource.tileData);
+ } else {
+ return getResource(resource);
+ }
+}
+
+void OfflineDatabase::put(const Resource& resource, const Response& response) {
+ // Except for 404s, don't store errors in the cache.
+ if (response.error && response.error->reason != Response::Error::Reason::NotFound) {
+ return;
+ }
+
+ if (resource.kind == Resource::Kind::Tile) {
+ assert(resource.tileData);
+ putTile(*resource.tileData, response);
+ } else {
+ putResource(resource, response);
+ }
+}
+
+optional<Response> OfflineDatabase::getResource(const Resource& resource) {
+ mapbox::sqlite::Statement& stmt = getStatement(
+ // 0 1 2 3 4
+ "SELECT etag, expires, modified, data, compressed "
+ "FROM resources "
+ "WHERE url = ?");
+
+ stmt.bind(1, resource.url);
+
+ if (!stmt.run()) {
+ return {};
+ }
+
+ Response response;
+
+ response.etag = stmt.get<optional<std::string>>(0);
+ response.expires = stmt.get<optional<SystemTimePoint>>(1);
+ response.modified = stmt.get<optional<SystemTimePoint>>(2);
+
+ optional<std::string> data = stmt.get<optional<std::string>>(3);
+ if (!data) {
+ response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
+ } else if (stmt.get<int>(4)) {
+ response.data = std::make_shared<std::string>(util::decompress(*data));
+ } else {
+ response.data = std::make_shared<std::string>(*data);
+ }
+
+ return response;
+}
+
+void OfflineDatabase::putResource(const Resource& resource, const Response& response) {
+ if (response.notModified) {
+ mapbox::sqlite::Statement& stmt = getStatement(
+ // 1 2 3
+ "UPDATE resources SET accessed = ?, expires = ? WHERE url = ?");
+
+ stmt.bind(1, SystemClock::now());
+ stmt.bind(2, response.expires);
+ stmt.bind(3, resource.url);
+ stmt.run();
+ } else {
+ mapbox::sqlite::Statement& stmt = getStatement(
+ // 1 2 3 4 5 6 7 8
+ "REPLACE INTO resources (url, kind, etag, expires, modified, accessed, data, compressed) "
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?)");
+
+ stmt.bind(1 /* url */, resource.url);
+ stmt.bind(2 /* kind */, int(resource.kind));
+ stmt.bind(3 /* etag */, response.etag);
+ stmt.bind(4 /* expires */, response.expires);
+ stmt.bind(5 /* modified */, response.modified);
+ stmt.bind(6 /* accessed */, SystemClock::now());
+
+ std::string data;
+
+ if (response.error) { // Can only be NotFound
+ stmt.bind(7 /* data */, nullptr);
+ stmt.bind(8 /* compressed */, false);
+ } else {
+ data = util::compress(*response.data);
+ if (data.size() < response.data->size()) {
+ stmt.bind(7 /* data */, data, false); // do not retain the string internally.
+ stmt.bind(8 /* compressed */, true);
+ } else {
+ stmt.bind(7 /* data */, *response.data, false); // do not retain the string internally.
+ stmt.bind(8 /* compressed */, false);
+ }
+ }
+
+ stmt.run();
+ }
+}
+
+optional<Response> OfflineDatabase::getTile(const Resource::TileData& tile) {
+ mapbox::sqlite::Statement& stmt = getStatement(
+ // 0 1 2 3 4
+ "SELECT etag, expires, modified, data, compressed "
+ "FROM tilesets, tiles "
+ "WHERE tilesets.url_template = ? " // 1
+ "AND tilesets.pixel_ratio = ? " // 2
+ "AND tiles.x = ? " // 3
+ "AND tiles.y = ? " // 4
+ "AND tiles.z = ? " // 5
+ "AND tilesets.id = tiles.tileset_id ");
+
+ stmt.bind(1, tile.urlTemplate);
+ stmt.bind(2, tile.pixelRatio);
+ stmt.bind(3, tile.x);
+ stmt.bind(4, tile.y);
+ stmt.bind(5, tile.z);
+
+ if (!stmt.run()) {
+ return {};
+ }
+
+ Response response;
+
+ response.etag = stmt.get<optional<std::string>>(0);
+ response.expires = stmt.get<optional<SystemTimePoint>>(1);
+ response.modified = stmt.get<optional<SystemTimePoint>>(2);
+
+ optional<std::string> data = stmt.get<optional<std::string>>(3);
+ if (!data) {
+ response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
+ } else if (stmt.get<int>(4)) {
+ response.data = std::make_shared<std::string>(util::decompress(*data));
+ } else {
+ response.data = std::make_shared<std::string>(*data);
+ }
+
+ return response;
+}
+
+void OfflineDatabase::putTile(const Resource::TileData& tile, const Response& response) {
+ if (response.notModified) {
+ mapbox::sqlite::Statement& stmt = getStatement(
+ "UPDATE tiles SET accessed = ?1, expires = ?2 "
+ "WHERE tileset_id = ( "
+ " SELECT id FROM tilesets "
+ " WHERE url_template = ?3 "
+ " AND pixel_ratio = ?4) "
+ "AND tiles.x = ?5 "
+ "AND tiles.y = ?6 "
+ "AND tiles.z = ?7 ");
+
+ stmt.bind(1, SystemClock::now());
+ stmt.bind(2, response.expires);
+ stmt.bind(3, tile.urlTemplate);
+ stmt.bind(4, tile.pixelRatio);
+ stmt.bind(5, tile.x);
+ stmt.bind(6, tile.y);
+ stmt.bind(7, tile.z);
+ stmt.run();
+ } else {
+ mapbox::sqlite::Statement& stmt1 = getStatement(
+ "REPLACE INTO tilesets (url_template, pixel_ratio) "
+ "VALUES (?1, ?2) ");
+
+ stmt1.bind(1 /* url_template */, tile.urlTemplate);
+ stmt1.bind(2 /* pixel_ratio */, tile.pixelRatio);
+ stmt1.run();
+
+ mapbox::sqlite::Statement& stmt2 = getStatement(
+ "REPLACE INTO tiles (tileset_id, x, y, z, modified, etag, expires, accessed, data, compressed) "
+ "SELECT tilesets.id, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11 "
+ "FROM tilesets "
+ "WHERE url_template = ?1 "
+ "AND pixel_ratio = ?2 ");
+
+ stmt2.bind(1 /* url_template */, tile.urlTemplate);
+ stmt2.bind(2 /* pixel_ratio */, tile.pixelRatio);
+ stmt2.bind(3 /* x */, tile.x);
+ stmt2.bind(4 /* y */, tile.y);
+ stmt2.bind(5 /* z */, tile.z);
+ stmt2.bind(6 /* modified */, response.modified);
+ stmt2.bind(7 /* etag */, response.etag);
+ stmt2.bind(8 /* expires */, response.expires);
+ stmt2.bind(9 /* accessed */, SystemClock::now());
+
+ std::string data;
+
+ if (response.error) { // Can only be NotFound
+ stmt2.bind(10 /* data */, nullptr);
+ stmt2.bind(11 /* compressed */, false);
+ } else {
+ data = util::compress(*response.data);
+ if (data.size() < response.data->size()) {
+ stmt2.bind(10 /* data */, data, false); // do not retain the string internally.
+ stmt2.bind(11 /* compressed */, true);
+ } else {
+ stmt2.bind(10 /* data */, *response.data, false); // do not retain the string internally.
+ stmt2.bind(11 /* compressed */, false);
+ }
+ }
+
+ stmt2.run();
+ }
+}
+
+} // namespace mbgl
diff --git a/platform/default/mbgl/storage/offline_database.hpp b/platform/default/mbgl/storage/offline_database.hpp
new file mode 100644
index 0000000000..bc6f784d50
--- /dev/null
+++ b/platform/default/mbgl/storage/offline_database.hpp
@@ -0,0 +1,50 @@
+#ifndef MBGL_OFFLINE_DATABASE
+#define MBGL_OFFLINE_DATABASE
+
+#include <mbgl/storage/resource.hpp>
+#include <mbgl/util/noncopyable.hpp>
+#include <mbgl/util/optional.hpp>
+
+#include <unordered_map>
+#include <memory>
+#include <string>
+
+namespace mapbox {
+namespace sqlite {
+class Database;
+class Statement;
+}
+}
+
+namespace mbgl {
+
+class Response;
+class TileID;
+
+class OfflineDatabase : private util::noncopyable {
+public:
+ OfflineDatabase(const std::string& path);
+ ~OfflineDatabase();
+
+ optional<Response> get(const Resource&);
+ void put(const Resource&, const Response&);
+
+private:
+ void ensureSchema();
+ void removeExisting();
+ mapbox::sqlite::Statement& getStatement(const char *);
+
+ optional<Response> getTile(const Resource::TileData&);
+ void putTile(const Resource::TileData&, const Response&);
+
+ optional<Response> getResource(const Resource&);
+ void putResource(const Resource&, const Response&);
+
+ const std::string path;
+ std::unique_ptr<::mapbox::sqlite::Database> db;
+ std::unordered_map<const char *, std::unique_ptr<::mapbox::sqlite::Statement>> statements;
+};
+
+} // namespace mbgl
+
+#endif
diff --git a/platform/default/mbgl/storage/offline_schema.cpp.include b/platform/default/mbgl/storage/offline_schema.cpp.include
new file mode 100644
index 0000000000..31b11d5f12
--- /dev/null
+++ b/platform/default/mbgl/storage/offline_schema.cpp.include
@@ -0,0 +1,51 @@
+/* THIS IS A GENERATED FILE; EDIT offline_schema.sql INSTEAD */
+static const char * schema =
+"CREATE TABLE resources (\n"
+" url TEXT NOT NULL PRIMARY KEY,\n"
+" kind INTEGER NOT NULL,\n"
+" expires INTEGER,\n"
+" modified INTEGER,\n"
+" accessed INTEGER,\n"
+" etag TEXT,\n"
+" data BLOB,\n"
+" compressed INTEGER NOT NULL DEFAULT 0\n"
+");\n"
+"CREATE TABLE tilesets (\n"
+" id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,\n"
+" url_template TEXT NOT NULL,\n"
+" pixel_ratio INTEGER,\n"
+" UNIQUE (url_template, pixel_ratio)\n"
+");\n"
+"CREATE TABLE tiles (\n"
+" tileset_id INTEGER NOT NULL REFERENCES tilesets(id),\n"
+" z INTEGER NOT NULL,\n"
+" x INTEGER NOT NULL,\n"
+" y INTEGER NOT NULL,\n"
+" expires INTEGER,\n"
+" modified INTEGER,\n"
+" accessed INTEGER,\n"
+" etag TEXT,\n"
+" data BLOB,\n"
+" compressed INTEGER NOT NULL DEFAULT 0,\n"
+" PRIMARY KEY (tileset_id, z, x, y)\n"
+");\n"
+"CREATE TABLE regions (\n"
+" id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,\n"
+" definition TEXT NOT NULL,\n"
+" description BLOB\n"
+");\n"
+"CREATE TABLE region_resources (\n"
+" region_id INTEGER NOT NULL REFERENCES regions(id),\n"
+" resource_url TEXT NOT NULL REFERENCES resources(url),\n"
+" PRIMARY KEY (region_id, resource_url)\n"
+");\n"
+"CREATE TABLE region_tiles (\n"
+" region_id INTEGER NOT NULL REFERENCES regions(id),\n"
+" tileset_id INTEGER NOT NULL REFERENCES tilesets(id),\n"
+" z INTEGER NOT NULL,\n"
+" x INTEGER NOT NULL,\n"
+" y INTEGER NOT NULL,\n"
+" PRIMARY KEY (region_id, tileset_id, z, x, y),\n"
+" FOREIGN KEY (tileset_id, z, x, y) REFERENCES tiles (tileset_id, z, x, y)\n"
+");\n"
+;
diff --git a/platform/default/mbgl/storage/offline_schema.js b/platform/default/mbgl/storage/offline_schema.js
new file mode 100644
index 0000000000..153ba34e38
--- /dev/null
+++ b/platform/default/mbgl/storage/offline_schema.js
@@ -0,0 +1,24 @@
+// To regenerate:
+// (cd platform/default/mbgl/storage && node offline_schema.js)
+
+var fs = require('fs');
+var readline = require('readline');
+
+var lineReader = readline.createInterface({
+ input: fs.createReadStream('offline_schema.sql')
+});
+
+var lines = [
+ "/* THIS IS A GENERATED FILE; EDIT offline_schema.sql INSTEAD */",
+ "static const char * schema = ",
+];
+
+lineReader
+ .on('line', function (line) {
+ line = line.replace(/ *--.*/, '');
+ if (line) lines.push('"' + line + '\\n"');
+ })
+ .on('close', function () {
+ lines.push(';\n');
+ fs.writeFileSync('offline_schema.cpp.include', lines.join('\n'));
+ });
diff --git a/platform/default/mbgl/storage/offline_schema.sql b/platform/default/mbgl/storage/offline_schema.sql
new file mode 100644
index 0000000000..e55517c489
--- /dev/null
+++ b/platform/default/mbgl/storage/offline_schema.sql
@@ -0,0 +1,62 @@
+CREATE TABLE resources ( -- Generic table for style, source, sprite, and glyph resources.
+ url TEXT NOT NULL PRIMARY KEY, -- Same schema as http_cache table.
+ kind INTEGER NOT NULL,
+ expires INTEGER,
+ modified INTEGER,
+ accessed INTEGER,
+ etag TEXT,
+ data BLOB, -- NULL if the response was a 404
+ compressed INTEGER NOT NULL DEFAULT 0
+);
+
+CREATE TABLE tilesets (
+ id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+ url_template TEXT NOT NULL, -- As it would appear in TileJSON (but no support for host sharding).
+ pixel_ratio INTEGER, -- If NULL, 1 is assumed for raster sources.
+ UNIQUE (url_template, pixel_ratio) -- Capable of caching the same tileset at multiple resolutions.
+);
+
+CREATE TABLE tiles (
+ tileset_id INTEGER NOT NULL REFERENCES tilesets(id),
+ z INTEGER NOT NULL, -- Fully abandon TMS coordinates in favor of ZXY.
+ x INTEGER NOT NULL,
+ y INTEGER NOT NULL,
+ expires INTEGER,
+ modified INTEGER,
+ accessed INTEGER,
+ etag TEXT,
+ data BLOB, -- NULL if the response was a 404
+ compressed INTEGER NOT NULL DEFAULT 0,
+ PRIMARY KEY (tileset_id, z, x, y)
+);
+
+CREATE TABLE regions (
+ id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+ definition TEXT NOT NULL, -- JSON formatted definition of region. Regions may be of variant types:
+ -- e.g. bbox and zoom range, route path, flyTo parameters, etc. Note that
+ -- the set of tiles required for a region may span multiple sources.
+ description BLOB -- User provided data in user-defined format
+);
+
+CREATE TABLE region_resources (
+ region_id INTEGER NOT NULL REFERENCES regions(id),
+ resource_url TEXT NOT NULL REFERENCES resources(url),
+ PRIMARY KEY (region_id, resource_url)
+);
+
+CREATE TABLE region_tiles (
+ region_id INTEGER NOT NULL REFERENCES regions(id),
+ tileset_id INTEGER NOT NULL REFERENCES tilesets(id),
+ z INTEGER NOT NULL,
+ x INTEGER NOT NULL,
+ y INTEGER NOT NULL,
+ PRIMARY KEY (region_id, tileset_id, z, x, y),
+ FOREIGN KEY (tileset_id, z, x, y) REFERENCES tiles (tileset_id, z, x, y)
+);
+
+-- `region_resources` and `region_tiles` are used for deduplication and deletion logic.
+-- A row in `tiles` exists IFF one or more corresponding rows exist in `region_tiles`. If
+-- more than one corresponding row exists, it indicates multiple regions contain the tile, and
+-- storage for the tile is being deduplicated. When a region is deleted, corresponding rows in
+-- `region_tiles` must also be deleted, and then rows in `tiles` and `tilesets` without a
+-- corresponding `region_tiles` row must be deleted. Similarly for `resources` / `region_resources`.
diff --git a/platform/default/sqlite3.cpp b/platform/default/sqlite3.cpp
index 5122e01015..09301bc4d9 100644
--- a/platform/default/sqlite3.cpp
+++ b/platform/default/sqlite3.cpp
@@ -115,9 +115,19 @@ template <> void Statement::bind(int offset, std::nullptr_t) {
check(sqlite3_bind_null(stmt, offset));
}
-template <> void Statement::bind(int offset, int value) {
+template <> void Statement::bind(int offset, int8_t value) {
assert(stmt);
- check(sqlite3_bind_int(stmt, offset, value));
+ check(sqlite3_bind_int64(stmt, offset, value));
+}
+
+template <> void Statement::bind(int offset, int16_t value) {
+ assert(stmt);
+ check(sqlite3_bind_int64(stmt, offset, value));
+}
+
+template <> void Statement::bind(int offset, int32_t value) {
+ assert(stmt);
+ check(sqlite3_bind_int64(stmt, offset, value));
}
template <> void Statement::bind(int offset, int64_t value) {
@@ -125,6 +135,26 @@ template <> void Statement::bind(int offset, int64_t value) {
check(sqlite3_bind_int64(stmt, offset, value));
}
+template <> void Statement::bind(int offset, uint8_t value) {
+ assert(stmt);
+ check(sqlite3_bind_int64(stmt, offset, value));
+}
+
+template <> void Statement::bind(int offset, uint16_t value) {
+ assert(stmt);
+ check(sqlite3_bind_int64(stmt, offset, value));
+}
+
+template <> void Statement::bind(int offset, uint32_t value) {
+ assert(stmt);
+ check(sqlite3_bind_int64(stmt, offset, value));
+}
+
+template <> void Statement::bind(int offset, float value) {
+ assert(stmt);
+ check(sqlite3_bind_double(stmt, offset, value));
+}
+
template <> void Statement::bind(int offset, double value) {
assert(stmt);
check(sqlite3_bind_double(stmt, offset, value));
diff --git a/platform/default/sqlite_cache.cpp b/platform/default/sqlite_cache.cpp
deleted file mode 100644
index b3f528ac92..0000000000
--- a/platform/default/sqlite_cache.cpp
+++ /dev/null
@@ -1,502 +0,0 @@
-#include "sqlite_cache_impl.hpp"
-#include <mbgl/storage/resource.hpp>
-#include <mbgl/storage/response.hpp>
-
-#include <mbgl/util/compression.hpp>
-#include <mbgl/util/io.hpp>
-#include <mbgl/util/string.hpp>
-#include <mbgl/util/thread.hpp>
-#include <mbgl/util/mapbox.hpp>
-#include <mbgl/platform/log.hpp>
-
-#include "sqlite3.hpp"
-#include <sqlite3.h>
-
-#include <unordered_map>
-#include <mutex>
-
-namespace {
-
-// The cache won't accept entries larger than this arbitrary size
-// and will silently discard request for adding them to the cache.
-// Large entries can cause the database to grow in disk size and
-// never shrink again.
-const uint64_t kMaximumCacheEntrySize = 5 * 1024 * 1024; // 5 MB
-
-// Number of records we delete when we are close to the maximum
-// database size, if set. The current criteria is to prune
-// the least used entries based on `accessed` time.
-const int kPrunedEntriesLimit = 100;
-
-} // namespace
-
-namespace mbgl {
-
-using namespace mapbox::sqlite;
-
-SQLiteCache::SQLiteCache(const std::string& path_)
- : thread(std::make_unique<util::Thread<Impl>>(util::ThreadContext{"SQLiteCache", util::ThreadType::Unknown, util::ThreadPriority::Low}, path_)) {
-}
-
-SQLiteCache::~SQLiteCache() = default;
-
-SQLiteCache::Impl::Impl(const std::string& path_)
- : maximumCacheSize(0), // Unlimited
- maximumCacheEntrySize(kMaximumCacheEntrySize),
- path(path_) {
-}
-
-SQLiteCache::Impl::~Impl() {
- // Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we
- // can't throw anything.
- try {
- getStmt.reset();
- putStmt.reset();
- refreshStmt.reset();
- countStmt.reset();
- freeStmt.reset();
- pruneStmt.reset();
- accessedStmt.reset();
- db.reset();
- } catch (mapbox::sqlite::Exception& ex) {
- Log::Error(Event::Database, ex.code, ex.what());
- }
-}
-
-void SQLiteCache::Impl::createDatabase() {
- db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
-}
-
-int SQLiteCache::Impl::schemaVersion() const {
- // WARNING: Bump the version when changing the cache
- // scheme to force the table to be recreated.
- return 1;
-}
-
-void SQLiteCache::Impl::createSchema() {
- constexpr const char *const sql = ""
- "CREATE TABLE IF NOT EXISTS `http_cache` ("
- " `url` TEXT PRIMARY KEY NOT NULL,"
- " `status` INTEGER NOT NULL," // The response status (Successful or Error).
- " `kind` INTEGER NOT NULL," // The kind of file.
- " `modified` INTEGER," // Timestamp when the file was last modified.
- " `etag` TEXT,"
- " `expires` INTEGER," // Timestamp when the server says the file expires.
- " `accessed` INTEGER," // Timestamp when the database record was last accessed.
- " `data` BLOB,"
- " `compressed` INTEGER NOT NULL DEFAULT 0" // Whether the data is compressed.
- ");"
- "CREATE INDEX IF NOT EXISTS `http_cache_kind_idx` ON `http_cache` (`kind`);"
- "CREATE INDEX IF NOT EXISTS `http_cache_accessed_idx` ON `http_cache` (`accessed`);";
-
- ensureSchemaVersion();
-
- try {
- db->exec(sql);
- db->exec("PRAGMA user_version = " + util::toString(schemaVersion()));
- schema = true;
- } catch (mapbox::sqlite::Exception &ex) {
- if (ex.code == SQLITE_NOTADB) {
- Log::Warning(Event::Database, "Trashing invalid database");
- db.reset();
- try {
- util::deleteFile(path);
- } catch (util::IOException& ioEx) {
- Log::Error(Event::Database, ex.code, ex.what());
- }
- db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
- } else {
- Log::Error(Event::Database, ex.code, ex.what());
- }
-
- // Creating the database table + index failed. That means there may already be one, likely
- // with different columns. Drop it and try to create a new one.
- db->exec("DROP TABLE IF EXISTS `http_cache`");
- db->exec(sql);
- db->exec("PRAGMA user_version = " + util::toString(schemaVersion()));
- }
-}
-
-void SQLiteCache::Impl::ensureSchemaVersion() {
- try {
- Statement userVersionStmt(db->prepare("PRAGMA user_version"));
- if (userVersionStmt.run() && userVersionStmt.get<int>(0) == schemaVersion()) {
- return;
- }
- } catch (mapbox::sqlite::Exception& ex) {
- if (ex.code == SQLITE_NOTADB) {
- return;
- }
-
- Log::Error(Event::Database, ex.code, ex.what());
- }
-
- // Version mismatch, drop the table so it will
- // get recreated by `createSchema()`.
- try {
- db->exec("DROP TABLE IF EXISTS `http_cache`");
- } catch (mapbox::sqlite::Exception& ex) {
- Log::Error(Event::Database, ex.code, ex.what());
- }
-}
-
-void SQLiteCache::setMaximumCacheSize(uint64_t size) {
- thread->invoke(&Impl::setMaximumCacheSize, size);
-}
-
-void SQLiteCache::Impl::setMaximumCacheSize(uint64_t size) {
- maximumCacheSize = size;
-
- // Unlimited.
- if (size == 0) {
- return;
- }
-
- uint64_t lastSoftSize = cacheSoftSize();
-
- // Keep pruning until we fit in the new
- // size limit.
- while (lastSoftSize > maximumCacheSize) {
- pruneEntries();
-
- if (lastSoftSize != cacheSoftSize()) {
- lastSoftSize = cacheSoftSize();
- } else {
- break;
- }
- }
-
- if (cacheHardSize() > size) {
- Log::Warning(mbgl::Event::Database,
- "Current cache hard size is bigger than the defined "
- "maximum size. Database won't get truncated.");
- }
-}
-
-void SQLiteCache::setMaximumCacheEntrySize(uint64_t size) {
- thread->invoke(&Impl::setMaximumCacheEntrySize, size);
-}
-
-void SQLiteCache::Impl::setMaximumCacheEntrySize(uint64_t size) {
- maximumCacheEntrySize = size;
-}
-
-void SQLiteCache::Impl::initializeDatabase() {
- if (!db) {
- createDatabase();
- }
-
- if (!schema) {
- createSchema();
- }
-}
-
-int SQLiteCache::Impl::cachePageSize() {
- try {
- if (!pageSize) {
- Statement pageSizeStmt(db->prepare("PRAGMA page_size"));
- if (pageSizeStmt.run()) {
- pageSize = pageSizeStmt.get<int>(0);
- }
- }
- } catch (mapbox::sqlite::Exception& ex) {
- Log::Error(Event::Database, ex.code, ex.what());
- }
-
- return pageSize;
-}
-
-uint64_t SQLiteCache::Impl::cacheHardSize() {
- try {
- initializeDatabase();
-
- if (!countStmt) {
- countStmt = std::make_unique<Statement>(db->prepare("PRAGMA page_count"));
- } else {
- countStmt->reset();
- }
-
- if (countStmt->run()) {
- return cachePageSize() * countStmt->get<int>(0);
- }
- } catch (mapbox::sqlite::Exception& ex) {
- Log::Error(Event::Database, ex.code, ex.what());
- }
-
- return 0;
-}
-
-uint64_t SQLiteCache::Impl::cacheSoftSize() {
- if (!softSizeDirty) {
- return softSize;
- }
-
- try {
- initializeDatabase();
-
- if (!freeStmt) {
- freeStmt = std::make_unique<Statement>(db->prepare("PRAGMA freelist_count"));
- } else {
- freeStmt->reset();
- }
-
- uint64_t hardSize = cacheHardSize();
- if (!hardSize) {
- return 0;
- }
-
- if (freeStmt->run()) {
- return hardSize - cachePageSize() * freeStmt->get<int>(0);
- }
-
- softSizeDirty = false;
- } catch (mapbox::sqlite::Exception& ex) {
- Log::Error(Event::Database, ex.code, ex.what());
- }
-
- return 0;
-}
-
-bool SQLiteCache::Impl::needsPruning() {
- // SQLite database never shrinks in size unless we call VACCUM. We here
- // are monitoring the soft limit (i.e. number of free pages in the file)
- // and as it approaches to the hard limit (i.e. the actual file size) we
- // delete an arbitrary number of old cache entries.
- //
- // The free pages approach saves us from calling VACCUM or keeping a
- // running total, which can be costly. We need a buffer because pages can
- // get fragmented on the database.
- if (cacheSoftSize() + maximumCacheEntrySize * 2 < maximumCacheSize) {
- return false;
- } else {
- return true;
- }
-}
-
-void SQLiteCache::Impl::pruneEntries() {
- if (!maximumCacheSize) {
- return;
- }
-
- if (!needsPruning()) {
- return;
- }
-
- try {
- if (!pruneStmt) {
- pruneStmt = std::make_unique<Statement>(db->prepare(
- "DELETE FROM `http_cache` WHERE `rowid` IN (SELECT `rowid` FROM "
- // 1
- "`http_cache` ORDER BY `accessed` ASC LIMIT ?)"));
- } else {
- pruneStmt->reset();
- }
-
- pruneStmt->bind(1, kPrunedEntriesLimit);
-
- pruneStmt->run();
- softSizeDirty = true;
- } catch (mapbox::sqlite::Exception& ex) {
- Log::Error(Event::Database, ex.code, ex.what());
- }
-}
-
-std::unique_ptr<WorkRequest> SQLiteCache::get(const Resource &resource, Callback callback) {
- // Can be called from any thread, but most likely from the file source thread.
- // Will try to load the URL from the SQLite database and call the callback when done.
- // Note that the callback is probably going to invoked from another thread, so the caller
- // must make sure that it can run in that thread.
- return thread->invokeWithCallback(&Impl::get, callback, resource);
-}
-
-void SQLiteCache::Impl::get(const Resource &resource, Callback callback) {
- try {
- initializeDatabase();
-
- if (!getStmt) {
- // Initialize the statement 0 1
- getStmt = std::make_unique<Statement>(db->prepare("SELECT `status`, `modified`, "
- // 2 3 4 5 1
- "`etag`, `expires`, `data`, `compressed` FROM `http_cache` WHERE `url` = ?"));
- } else {
- getStmt->reset();
- }
-
- getStmt->bind(1, resource.url);
- if (getStmt->run()) {
- // There is data.
- auto response = std::make_unique<Response>();
- const auto status = getStmt->get<int>(0);
- if (status > 1) {
- // Status codes > 1 indicate an error
- response->error = std::make_unique<Response::Error>(Response::Error::Reason(status));
- }
- response->modified = getStmt->get<optional<SystemTimePoint>>(1);
- response->etag = getStmt->get<optional<std::string>>(2);
- response->expires = getStmt->get<optional<SystemTimePoint>>(3);
- response->data = std::make_shared<std::string>(getStmt->get<std::string>(4));
- if (getStmt->get<int>(5)) { // == compressed
- response->data = std::make_shared<std::string>(util::decompress(*response->data));
- }
- callback(std::move(response));
- } else {
- // There is no data.
- callback(nullptr);
- }
-
- // We do an extra query for refreshing the last time
- // the record was accessed that can be costly and is only
- // worth doing if we are monitoring the database size.
- if (maximumCacheSize) {
- if (!accessedStmt) {
- accessedStmt = std::make_unique<Statement>(
- // 1 2
- db->prepare("UPDATE `http_cache` SET `accessed` = ? WHERE `url` = ?"));
- } else {
- accessedStmt->reset();
- }
-
- accessedStmt->bind(1, SystemClock::now());
- accessedStmt->bind(2, resource.url);
- accessedStmt->run();
- }
- } catch (mapbox::sqlite::Exception& ex) {
- Log::Error(Event::Database, ex.code, ex.what());
- callback(nullptr);
- } catch (std::runtime_error& ex) {
- Log::Error(Event::Database, "%s", ex.what());
- callback(nullptr);
- }
-}
-
-void SQLiteCache::put(const Resource& resource, const Response& response) {
- // Except for 404s, don't store errors in the cache.
- if (response.error && response.error->reason != Response::Error::Reason::NotFound) {
- return;
- }
-
- if (response.notModified) {
- thread->invoke(&Impl::refresh, resource, response.expires);
- } else {
- thread->invoke(&Impl::put, resource, response);
- }
-}
-
-void SQLiteCache::Impl::put(const Resource& resource, const Response& response) {
- try {
- initializeDatabase();
- pruneEntries();
-
- if (response.data) {
- auto entrySize = response.data->size();
-
- if (entrySize > maximumCacheEntrySize) {
- Log::Warning(Event::Database, "Entry too big for caching.");
- return;
- }
-
- if (maximumCacheSize && entrySize + cacheSoftSize() > maximumCacheSize) {
- Log::Warning(Event::Database, "Unable to make space for new entries.");
- return;
- }
- }
-
- if (!putStmt) {
- putStmt = std::make_unique<Statement>(db->prepare("REPLACE INTO `http_cache` ("
- // 1 2 3 4 5 6 7 8 9
- "`url`, `status`, `kind`, `modified`, `etag`, `expires`, `accessed`, `data`, `compressed`"
- ") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)"));
- } else {
- putStmt->reset();
- }
-
- putStmt->bind(1 /* url */, resource.url);
- if (response.error) {
- putStmt->bind(2 /* status */, int(response.error->reason));
- } else {
- putStmt->bind(2 /* status */, 1 /* success */);
- }
- putStmt->bind(3 /* kind */, int(resource.kind));
- putStmt->bind(4 /* modified */, response.modified);
- putStmt->bind(5 /* etag */, response.etag);
- putStmt->bind(6 /* expires */, response.expires);
- putStmt->bind(7 /* accessed */, SystemClock::now());
-
- std::string data;
- if (resource.kind != Resource::SpriteImage && response.data) {
- // Do not compress images, since they are typically compressed already.
- data = util::compress(*response.data);
- }
-
- if (!data.empty() && data.size() < response.data->size()) {
- // Store the compressed data when it is smaller than the original
- // uncompressed data.
- putStmt->bind(8 /* data */, data, false); // do not retain the string internally.
- putStmt->bind(9 /* compressed */, true);
- } else if (response.data) {
- putStmt->bind(8 /* data */, *response.data, false); // do not retain the string internally.
- putStmt->bind(9 /* compressed */, false);
- } else {
- putStmt->bind(8 /* data */, "", false);
- putStmt->bind(9 /* compressed */, false);
- }
-
- putStmt->run();
- softSizeDirty = true;
- } catch (mapbox::sqlite::Exception& ex) {
- Log::Error(Event::Database, ex.code, ex.what());
- } catch (std::runtime_error& ex) {
- Log::Error(Event::Database, "%s", ex.what());
- }
-}
-
-void SQLiteCache::Impl::refresh(const Resource& resource, optional<SystemTimePoint> expires) {
- try {
- initializeDatabase();
-
- if (!refreshStmt) {
- refreshStmt = std::make_unique<Statement>(
- db->prepare("UPDATE `http_cache` SET "
- // 1 2 3
- "`accessed` = ?, `expires` = ? WHERE `url` = ?"));
- } else {
- refreshStmt->reset();
- }
-
- refreshStmt->bind(1, SystemClock::now());
- refreshStmt->bind(2, expires);
- refreshStmt->bind(3, resource.url);
- refreshStmt->run();
- } catch (mapbox::sqlite::Exception& ex) {
- Log::Error(Event::Database, ex.code, ex.what());
- }
-}
-
-namespace {
-
-static std::mutex sharedMutex;
-static std::unordered_map<std::string, std::weak_ptr<SQLiteCache>> shared;
-
-} // namespace
-
-std::shared_ptr<SQLiteCache> SQLiteCache::getShared(const std::string &path) {
- std::lock_guard<std::mutex> lock(sharedMutex);
-
- std::shared_ptr<SQLiteCache> cache;
-
- auto it = shared.find(path);
- if (it != shared.end()) {
- cache = it->second.lock();
- if (!cache) {
- cache = std::make_shared<SQLiteCache>(path);
- it->second = cache;
- }
- } else {
- cache = std::make_shared<SQLiteCache>(path);
- shared.emplace(path, cache);
- }
-
- return cache;
-}
-
-} // namespace mbgl
diff --git a/platform/default/sqlite_cache_impl.hpp b/platform/default/sqlite_cache_impl.hpp
deleted file mode 100644
index e156532402..0000000000
--- a/platform/default/sqlite_cache_impl.hpp
+++ /dev/null
@@ -1,68 +0,0 @@
-#ifndef MBGL_STORAGE_DEFAULT_SQLITE_CACHE_IMPL
-#define MBGL_STORAGE_DEFAULT_SQLITE_CACHE_IMPL
-
-#include <mbgl/storage/sqlite_cache.hpp>
-#include <mbgl/util/chrono.hpp>
-#include <mbgl/util/optional.hpp>
-
-namespace mapbox {
-namespace sqlite {
-class Database;
-class Statement;
-}
-}
-
-namespace mbgl {
-
-class SQLiteCache::Impl {
-public:
- explicit Impl(const std::string &path = ":memory:");
- ~Impl();
-
- void setMaximumCacheSize(uint64_t size);
- void setMaximumCacheEntrySize(uint64_t size);
-
- void get(const Resource&, Callback);
- void put(const Resource&, const Response&);
- void refresh(const Resource&, optional<SystemTimePoint> expires);
-
-private:
- void initializeDatabase();
-
- int cachePageSize();
-
- uint64_t cacheHardSize();
- uint64_t cacheSoftSize();
-
- uint64_t softSize = 0;
- bool softSizeDirty = true;
-
- bool needsPruning();
- void pruneEntries();
-
- void createDatabase();
- void createSchema();
-
- int schemaVersion() const;
- void ensureSchemaVersion();
-
- int pageSize = 0;
-
- uint64_t maximumCacheSize;
- uint64_t maximumCacheEntrySize;
-
- const std::string path;
- std::unique_ptr<::mapbox::sqlite::Database> db;
- std::unique_ptr<::mapbox::sqlite::Statement> getStmt;
- std::unique_ptr<::mapbox::sqlite::Statement> putStmt;
- std::unique_ptr<::mapbox::sqlite::Statement> refreshStmt;
- std::unique_ptr<::mapbox::sqlite::Statement> countStmt;
- std::unique_ptr<::mapbox::sqlite::Statement> freeStmt;
- std::unique_ptr<::mapbox::sqlite::Statement> pruneStmt;
- std::unique_ptr<::mapbox::sqlite::Statement> accessedStmt;
- bool schema = false;
-};
-
-} // namespace mbgl
-
-#endif
diff --git a/src/mbgl/storage/sqlite_cache.hpp b/src/mbgl/storage/sqlite_cache.hpp
deleted file mode 100644
index b5a7cbcc07..0000000000
--- a/src/mbgl/storage/sqlite_cache.hpp
+++ /dev/null
@@ -1,44 +0,0 @@
-#ifndef MBGL_STORAGE_DEFAULT_SQLITE_CACHE
-#define MBGL_STORAGE_DEFAULT_SQLITE_CACHE
-
-#include <mbgl/util/noncopyable.hpp>
-#include <mbgl/util/chrono.hpp>
-
-#include <functional>
-#include <memory>
-#include <string>
-
-namespace mbgl {
-
-class Resource;
-class Response;
-class WorkRequest;
-
-namespace util {
-template <typename T> class Thread;
-} // namespace util
-
-class SQLiteCache : private util::noncopyable {
-public:
- static std::shared_ptr<SQLiteCache> getShared(const std::string &path = ":memory:");
-
- SQLiteCache(const std::string &path = ":memory:");
- ~SQLiteCache();
-
- void setMaximumCacheSize(uint64_t size);
- void setMaximumCacheEntrySize(uint64_t size);
-
- using Callback = std::function<void(std::unique_ptr<Response>)>;
-
- std::unique_ptr<WorkRequest> get(const Resource&, Callback);
- void put(const Resource&, const Response&);
-
- class Impl;
-
-private:
- const std::unique_ptr<util::Thread<Impl>> thread;
-};
-
-} // namespace mbgl
-
-#endif
diff --git a/test/api/offline.cpp b/test/api/offline.cpp
new file mode 100644
index 0000000000..f168ff71f5
--- /dev/null
+++ b/test/api/offline.cpp
@@ -0,0 +1,50 @@
+#include "../fixtures/util.hpp"
+
+#include <mbgl/platform/default/headless_display.hpp>
+#include <mbgl/platform/default/headless_view.hpp>
+#include <mbgl/storage/offline_database.hpp>
+#include <mbgl/storage/default_file_source.hpp>
+
+#include <mbgl/platform/log.hpp>
+#include <mbgl/util/work_request.hpp>
+#include <mbgl/util/io.hpp>
+
+using namespace mbgl;
+using namespace std::literals::chrono_literals;
+using namespace std::literals::string_literals;
+
+namespace {
+
+Response expiredItem(const std::string& path) {
+ Response response;
+ response.data = std::make_shared<std::string>(util::read_file("test/fixtures/"s + path));
+ response.expires = SystemClock::from_time_t(0);
+ return response;
+}
+
+const std::string prefix = "http://127.0.0.1:3000";
+
+}
+
+auto display = std::make_shared<mbgl::HeadlessDisplay>();
+
+TEST(API, Offline) {
+ HeadlessView view(display, 1);
+ DefaultFileSource fileSource(":memory:", ".");
+
+ fileSource.put(Resource::style(prefix + "/offline/style.json"), expiredItem("offline/style.json"));
+ fileSource.put(Resource::source(prefix + "/offline/streets.json"), expiredItem("offline/streets.json"));
+ fileSource.put(Resource::spriteJSON(prefix + "/offline/sprite", 1.0), expiredItem("offline/sprite.json"));
+ fileSource.put(Resource::spriteImage(prefix + "/offline/sprite", 1.0), expiredItem("offline/sprite.png"));
+ fileSource.put(Resource::tile(prefix + "/offline/{z}-{x}-{y}.vector.pbf", 1.0, 0, 0, 0), expiredItem("offline/0-0-0.vector.pbf"));
+ fileSource.put(Resource::glyphs(prefix + "/offline/{fontstack}/{range}.pbf", "Helvetica", {0, 255}), expiredItem("offline/glyph.pbf"));
+ fileSource.goOffline();
+
+ Map map(view, fileSource, MapMode::Still);
+ map.setStyleURL(prefix + "/offline/style.json");
+
+ test::checkImage("test/fixtures/offline"s,
+ test::render(map),
+ 0.0015,
+ 0.1);
+}
diff --git a/test/fixtures/stale/0-0-0.vector.pbf b/test/fixtures/offline/0-0-0.vector.pbf
index a0f049ad43..a0f049ad43 100644
--- a/test/fixtures/stale/0-0-0.vector.pbf
+++ b/test/fixtures/offline/0-0-0.vector.pbf
Binary files differ
diff --git a/test/fixtures/offline/expected.png b/test/fixtures/offline/expected.png
new file mode 100644
index 0000000000..1b1c2be4c6
--- /dev/null
+++ b/test/fixtures/offline/expected.png
Binary files differ
diff --git a/test/fixtures/stale/glyph.pbf b/test/fixtures/offline/glyph.pbf
index 0d160f7898..0d160f7898 100644
--- a/test/fixtures/stale/glyph.pbf
+++ b/test/fixtures/offline/glyph.pbf
Binary files differ
diff --git a/test/fixtures/stale/sprite.json b/test/fixtures/offline/sprite.json
index e640365519..e640365519 100644
--- a/test/fixtures/stale/sprite.json
+++ b/test/fixtures/offline/sprite.json
diff --git a/test/fixtures/stale/sprite.png b/test/fixtures/offline/sprite.png
index a02d8eb542..a02d8eb542 100644
--- a/test/fixtures/stale/sprite.png
+++ b/test/fixtures/offline/sprite.png
Binary files differ
diff --git a/test/fixtures/stale/streets.json b/test/fixtures/offline/streets.json
index 11cb5c8557..805bd68f45 100644
--- a/test/fixtures/stale/streets.json
+++ b/test/fixtures/offline/streets.json
@@ -10,5 +10,5 @@
"name": "Streets",
"scheme": "xyz",
"tilejson": "2.0.0",
- "tiles": [ "asset://test/fixtures/stale/{z}-{x}-{y}.vector.pbf" ]
+ "tiles": [ "http://127.0.0.1:3000/offline/{z}-{x}-{y}.vector.pbf" ]
}
diff --git a/test/fixtures/stale/style_and_glyphs.json b/test/fixtures/offline/style.json
index adf91705fb..c34ee59893 100644
--- a/test/fixtures/stale/style_and_glyphs.json
+++ b/test/fixtures/offline/style.json
@@ -4,10 +4,11 @@
"sources": {
"mapbox": {
"type": "vector",
- "url": "asset://test/fixtures/stale/streets.json"
+ "url": "http://127.0.0.1:3000/offline/streets.json"
}
},
- "glyphs": "http://127.0.0.1:3000/stale/{fontstack}/{range}.pbf",
+ "glyphs": "http://127.0.0.1:3000/offline/{fontstack}/{range}.pbf",
+ "sprite": "http://127.0.0.1:3000/offline/sprite",
"layers": [{
"id": "background",
"type": "background",
@@ -20,7 +21,7 @@
"source": "mapbox",
"source-layer": "water",
"paint": {
- "fill-color": "blue"
+ "fill-pattern": "noise"
}
}, {
"id": "admin",
@@ -28,7 +29,8 @@
"source": "mapbox",
"source-layer": "admin",
"layout": {
- "text-field": "Text"
+ "text-font": ["Helvetica"],
+ "text-field": "Text"
},
"paint": {
"text-color": "black",
diff --git a/test/fixtures/stale/stale_style/expected.png b/test/fixtures/stale/stale_style/expected.png
deleted file mode 100644
index d3c6ef3cd5..0000000000
--- a/test/fixtures/stale/stale_style/expected.png
+++ /dev/null
Binary files differ
diff --git a/test/fixtures/stale/stale_style_and_glyphs/expected.png b/test/fixtures/stale/stale_style_and_glyphs/expected.png
deleted file mode 100644
index d8dbda1092..0000000000
--- a/test/fixtures/stale/stale_style_and_glyphs/expected.png
+++ /dev/null
Binary files differ
diff --git a/test/fixtures/stale/stale_style_and_sprite/expected.png b/test/fixtures/stale/stale_style_and_sprite/expected.png
deleted file mode 100644
index 171599bf5c..0000000000
--- a/test/fixtures/stale/stale_style_and_sprite/expected.png
+++ /dev/null
Binary files differ
diff --git a/test/fixtures/stale/stale_style_and_tilejson/expected.png b/test/fixtures/stale/stale_style_and_tilejson/expected.png
deleted file mode 100644
index d3c6ef3cd5..0000000000
--- a/test/fixtures/stale/stale_style_and_tilejson/expected.png
+++ /dev/null
Binary files differ
diff --git a/test/fixtures/stale/style.json b/test/fixtures/stale/style.json
deleted file mode 100644
index ca86503d4f..0000000000
--- a/test/fixtures/stale/style.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "version": 8,
- "name": "Water",
- "sources": {
- "mapbox": {
- "type": "vector",
- "url": "asset://test/fixtures/stale/streets.json"
- }
- },
- "layers": [{
- "id": "background",
- "type": "background",
- "paint": {
- "background-color": "red"
- }
- }, {
- "id": "water",
- "type": "fill",
- "source": "mapbox",
- "source-layer": "water",
- "paint": {
- "fill-color": "blue"
- }
- }]
-}
diff --git a/test/fixtures/stale/style_and_sprite.json b/test/fixtures/stale/style_and_sprite.json
deleted file mode 100644
index 61e3214a5e..0000000000
--- a/test/fixtures/stale/style_and_sprite.json
+++ /dev/null
@@ -1,26 +0,0 @@
-{
- "version": 8,
- "name": "Water",
- "sources": {
- "mapbox": {
- "type": "vector",
- "url": "asset://test/fixtures/stale/streets.json"
- }
- },
- "sprite": "http://127.0.0.1:3000/stale/sprite",
- "layers": [{
- "id": "background",
- "type": "background",
- "paint": {
- "background-color": "red"
- }
- }, {
- "id": "water",
- "type": "fill",
- "source": "mapbox",
- "source-layer": "water",
- "paint": {
- "fill-pattern": "noise"
- }
- }]
-}
diff --git a/test/fixtures/stale/style_and_tilejson.json b/test/fixtures/stale/style_and_tilejson.json
deleted file mode 100644
index f2a4fddd0b..0000000000
--- a/test/fixtures/stale/style_and_tilejson.json
+++ /dev/null
@@ -1,25 +0,0 @@
-{
- "version": 8,
- "name": "Water",
- "sources": {
- "mapbox": {
- "type": "vector",
- "url": "http://127.0.0.1:3000/stale/streets.json"
- }
- },
- "layers": [{
- "id": "background",
- "type": "background",
- "paint": {
- "background-color": "red"
- }
- }, {
- "id": "water",
- "type": "fill",
- "source": "mapbox",
- "source-layer": "water",
- "paint": {
- "fill-color": "blue"
- }
- }]
-}
diff --git a/test/fixtures/util.cpp b/test/fixtures/util.cpp
index fcb6abed4e..5617b15e8b 100644
--- a/test/fixtures/util.cpp
+++ b/test/fixtures/util.cpp
@@ -106,20 +106,12 @@ uint64_t crc64(const PremultipliedImage &image) {
return crc64(reinterpret_cast<const char*>(image.data.get()), image.size());
}
-PremultipliedImage render(Map& map, Milliseconds timeout) {
+PremultipliedImage render(Map& map) {
std::promise<PremultipliedImage> promise;
map.renderStill([&](std::exception_ptr, PremultipliedImage&& image) {
promise.set_value(std::move(image));
});
-
- // Limit maximum wait time.
- auto future = promise.get_future();
- if (future.wait_for(timeout) != std::future_status::ready) {
- // Alas, we didn't get the promised future :(
- Log::Error(Event::Image, "Failed to generate image within %dms", timeout.count());
- return { 0, 0 };
- }
- return future.get();
+ return promise.get_future().get();
}
void checkImage(const std::string& base,
diff --git a/test/fixtures/util.hpp b/test/fixtures/util.hpp
index b93d822a83..2e27e5bf0f 100644
--- a/test/fixtures/util.hpp
+++ b/test/fixtures/util.hpp
@@ -4,7 +4,6 @@
#include <mbgl/util/image.hpp>
#include <mbgl/util/chrono.hpp>
-#include <chrono>
#include <cstdint>
#include <gtest/gtest.h>
@@ -36,12 +35,13 @@ uint64_t crc64(const char*, size_t);
uint64_t crc64(const std::string&);
uint64_t crc64(const PremultipliedImage&);
-PremultipliedImage render(Map&, Milliseconds timeout = Milliseconds(1000));
+PremultipliedImage render(Map&);
void checkImage(const std::string& base,
const PremultipliedImage& actual,
double imageThreshold = 0,
double pixelThreshold = 0);
+
}
}
diff --git a/test/storage/asset_file_source.cpp b/test/storage/asset_file_source.cpp
index 6734cc693c..a9261ee8a2 100644
--- a/test/storage/asset_file_source.cpp
+++ b/test/storage/asset_file_source.cpp
@@ -1,7 +1,6 @@
#include "storage.hpp"
#include <mbgl/storage/asset_file_source.hpp>
-#include <mbgl/storage/sqlite_cache.hpp>
#include <mbgl/platform/platform.hpp>
#include <mbgl/util/chrono.hpp>
#include <mbgl/util/run_loop.hpp>
diff --git a/test/storage/cache_response.cpp b/test/storage/cache_response.cpp
deleted file mode 100644
index adeb45727e..0000000000
--- a/test/storage/cache_response.cpp
+++ /dev/null
@@ -1,215 +0,0 @@
-#include "storage.hpp"
-
-#include <mbgl/storage/default_file_source.hpp>
-#include <mbgl/storage/sqlite_cache.hpp>
-#include <mbgl/util/chrono.hpp>
-#include <mbgl/util/run_loop.hpp>
-
-TEST_F(Storage, CacheResponse) {
- SCOPED_TEST(CacheResponse);
-
- using namespace mbgl;
-
- util::RunLoop loop;
- DefaultFileSource fs(":memory:", ".");
-
- const Resource resource { Resource::Unknown, "http://127.0.0.1:3000/cache" };
- Response response;
-
- std::unique_ptr<FileRequest> req1;
- std::unique_ptr<FileRequest> req2;
-
- req1 = fs.request(resource, [&](Response res) {
- req1.reset();
- EXPECT_EQ(nullptr, res.error);
- ASSERT_TRUE(res.data.get());
- EXPECT_EQ("Response 1", *res.data);
- EXPECT_TRUE(bool(res.expires));
- EXPECT_FALSE(bool(res.modified));
- EXPECT_FALSE(bool(res.etag));
- response = res;
-
- // Now test that we get the same values as in the previous request. If we'd go to the server
- // again, we'd get different values.
- req2 = fs.request(resource, [&](Response res2) {
- req2.reset();
- EXPECT_EQ(response.error, res2.error);
- ASSERT_TRUE(res2.data.get());
- EXPECT_EQ(*response.data, *res2.data);
- EXPECT_EQ(response.expires, res2.expires);
- EXPECT_EQ(response.modified, res2.modified);
- EXPECT_EQ(response.etag, res2.etag);
-
- loop.stop();
- CacheResponse.finish();
- });
- });
-
- loop.run();
-}
-
-// Make sure we /do/ store 404 Not Found responses into the cache
-TEST_F(Storage, CacheNotFound) {
- SCOPED_TEST(CacheNotFound);
-
- using namespace mbgl;
-
- util::RunLoop loop;
- DefaultFileSource fs(":memory:", ".");
-
- const Resource resource{ Resource::Unknown, "http://127.0.0.1:3000/not-found" };
-
- // Insert existing data into the cache that will be marked as stale.
- Response response;
- response.data = std::make_shared<const std::string>("existing data");
- fs.getCache().put(resource, response);
-
- std::unique_ptr<FileRequest> req1;
- std::unique_ptr<WorkRequest> req2;
-
- int counter = 0;
-
- // Then, request the actual URL and check that we're getting the rigged cache response first,
- // then the connection error message.
- req1 = fs.request(resource, [&](Response res) {
- if (counter == 0) {
- EXPECT_EQ(nullptr, res.error);
- ASSERT_TRUE(res.data.get());
- EXPECT_EQ("existing data", *res.data);
- EXPECT_FALSE(bool(res.expires));
- EXPECT_FALSE(bool(res.modified));
- EXPECT_FALSE(bool(res.etag));
- } else if (counter == 1) {
- EXPECT_NE(nullptr, res.error);
- EXPECT_EQ(Response::Error::Reason::NotFound, res.error->reason);
- ASSERT_TRUE(res.data.get());
- EXPECT_EQ("Not Found!", *res.data);
- req1.reset();
-
- // Finally, check the cache to make sure we cached the 404 response.
- req2 = fs.getCache().get(resource, [&](std::unique_ptr<Response> res2) {
- EXPECT_NE(nullptr, res2->error);
- EXPECT_EQ(Response::Error::Reason::NotFound, res2->error->reason);
- ASSERT_TRUE(res2->data.get());
- EXPECT_EQ("Not Found!", *res2->data);
- CacheNotFound.finish();
- loop.stop();
- });
- } else {
- FAIL() << "Got too many responses";
- }
- counter++;
- });
-
- loop.run();
-}
-
-// Make sure we don't store a connection error into the cache
-TEST_F(Storage, DontCacheConnectionErrors) {
- SCOPED_TEST(DontCacheConnectionErrors);
-
- using namespace mbgl;
-
- util::RunLoop loop;
- DefaultFileSource fs(":memory:", ".");
-
- const Resource resource{ Resource::Unknown, "http://127.0.0.1:3001" };
-
- // Insert existing data into the cache that will be marked as stale.
- Response response;
- response.data = std::make_shared<const std::string>("existing data");
- fs.getCache().put(resource, response);
-
- std::unique_ptr<FileRequest> req1;
- std::unique_ptr<WorkRequest> req2;
-
- int counter = 0;
-
- // Then, request the actual URL and check that we're getting the rigged cache response first,
- // then the connection error message.
- req1 = fs.request(resource, [&](Response res) {
- if (counter == 0) {
- EXPECT_EQ(nullptr, res.error);
- ASSERT_TRUE(res.data.get());
- EXPECT_EQ("existing data", *res.data);
- EXPECT_FALSE(bool(res.expires));
- EXPECT_FALSE(bool(res.modified));
- EXPECT_FALSE(bool(res.etag));
- } else if (counter == 1) {
- EXPECT_NE(nullptr, res.error);
- EXPECT_EQ(Response::Error::Reason::Connection, res.error->reason);
- req1.reset();
-
- // Finally, check the cache to make sure we still have our original data in there rather
- // than the failed connection attempt.
- req2 = fs.getCache().get(resource, [&](std::unique_ptr<Response> res2) {
- EXPECT_EQ(nullptr, res2->error);
- ASSERT_TRUE(res2->data.get());
- EXPECT_EQ("existing data", *res2->data);
- DontCacheConnectionErrors.finish();
- loop.stop();
- });
- } else {
- FAIL() << "Got too many responses";
- }
- counter++;
- });
-
- loop.run();
-}
-
-// Make sure we don't store a bad server response into the cache
-TEST_F(Storage, DontCacheServerErrors) {
- SCOPED_TEST(DontCacheServerErrors);
-
- using namespace mbgl;
-
- util::RunLoop loop;
- DefaultFileSource fs(":memory:", ".");
-
- const Resource resource{ Resource::Unknown, "http://127.0.0.1:3000/permanent-error" };
-
- // Insert existing data into the cache that will be marked as stale.
- Response response;
- response.data = std::make_shared<const std::string>("existing data");
- fs.getCache().put(resource, response);
-
- std::unique_ptr<FileRequest> req1;
- std::unique_ptr<WorkRequest> req2;
-
- int counter = 0;
-
- // Then, request the actual URL and check that we're getting the rigged cache response first,
- // then the server error message.
- req1 = fs.request(resource, [&](Response res) {
- if (counter == 0) {
- EXPECT_EQ(nullptr, res.error);
- ASSERT_TRUE(res.data.get());
- EXPECT_EQ("existing data", *res.data);
- EXPECT_FALSE(bool(res.expires));
- EXPECT_FALSE(bool(res.modified));
- EXPECT_FALSE(bool(res.etag));
- } else if (counter == 1) {
- EXPECT_NE(nullptr, res.error);
- EXPECT_EQ(Response::Error::Reason::Server, res.error->reason);
- ASSERT_TRUE(res.data.get());
- EXPECT_EQ("Server Error!", *res.data);
- req1.reset();
-
- // Finally, check the cache to make sure we still have our original data in there rather
- // than the failed connection attempt.
- req2 = fs.getCache().get(resource, [&](std::unique_ptr<Response> res2) {
- EXPECT_EQ(nullptr, res2->error);
- ASSERT_TRUE(res2->data.get());
- EXPECT_EQ("existing data", *res2->data);
- DontCacheServerErrors.finish();
- loop.stop();
- });
- } else {
- FAIL() << "Got too many responses";
- }
- counter++;
- });
-
- loop.run();
-}
diff --git a/test/storage/cache_shared.cpp b/test/storage/cache_shared.cpp
deleted file mode 100644
index 89ef3bfb84..0000000000
--- a/test/storage/cache_shared.cpp
+++ /dev/null
@@ -1,34 +0,0 @@
-#include "storage.hpp"
-
-#include <mbgl/storage/sqlite_cache.hpp>
-#include <mbgl/storage/resource.hpp>
-#include <mbgl/util/run_loop.hpp>
-
-TEST_F(Storage, CacheShared) {
- SCOPED_TEST(CacheShared)
- using namespace mbgl;
-
- util::RunLoop loop;
-
- // Check that we're getting two different caches when we request different paths.
- auto memory = SQLiteCache::getShared();
- auto file = SQLiteCache::getShared("test/fixtures/database/cache.db");
- EXPECT_NE(memory.get(), file.get());
- EXPECT_EQ(memory.get(), SQLiteCache::getShared().get());
-
- // Store a response into the memory file cache, then delete the last reference.
- const Resource resource { Resource::Kind::Unknown, "http://example.com" };
- memory->put(resource, Response());
- memory.reset();
-
- // Now check that the original memory file cache has been destructed and that it doesn't contain
- // the information we put into it.
- memory = SQLiteCache::getShared();
- auto req = memory->get(resource, [&](std::unique_ptr<Response> res) {
- EXPECT_FALSE(res.get());
- CacheShared.finish();
- loop.stop();
- });
-
- loop.run();
-}
diff --git a/test/storage/cache_size.cpp b/test/storage/cache_size.cpp
deleted file mode 100644
index b0d59d5934..0000000000
--- a/test/storage/cache_size.cpp
+++ /dev/null
@@ -1,241 +0,0 @@
-#include "storage.hpp"
-
-#include <mbgl/storage/resource.hpp>
-#include <mbgl/storage/response.hpp>
-#include <mbgl/storage/sqlite_cache.hpp>
-#include <mbgl/util/run_loop.hpp>
-#include <mbgl/util/string.hpp>
-#include <mbgl/util/timer.hpp>
-#include <mbgl/util/chrono.hpp>
-
-#include <memory>
-#include <random>
-
-bool tileIsCached(mbgl::SQLiteCache* cache, unsigned id) {
- using namespace mbgl;
-
- auto url = std::string("http://tile") + mbgl::util::toString(id);
- bool replied = false;
-
- std::unique_ptr<Response> response;
- auto callback = [&] (std::unique_ptr<Response> res) {
- replied = true;
- response = std::move(res);
- };
-
- Resource resource{ Resource::Kind::Tile, url };
- auto req = cache->get(resource, callback);
-
- while (!replied) {
- util::RunLoop::Get()->runOnce();
- }
-
- return response != nullptr;
-}
-
-void insertTile(mbgl::SQLiteCache* cache, unsigned id, uint64_t size) {
- using namespace mbgl;
-
- auto url = std::string("http://tile") + mbgl::util::toString(id);
-
- Response response;
- response.modified = SystemClock::now();
- response.expires = SystemClock::now() + Seconds(5000);
- response.etag = url;
-
- auto data = std::make_shared<std::string>(size, 0);
-
- // Fill data with garbage so SQLite won't try to
- // optimize allocation by reusing pages.
- static std::mt19937 generator;
- std::generate_n(data->begin(), size, generator);
-
- response.data = data;
-
- Resource resource{ Resource::Kind::Tile, url };
- cache->put(resource, response);
-}
-
-void refreshTile(mbgl::SQLiteCache* cache, unsigned id) {
- using namespace mbgl;
-
- auto url = std::string("http://tile") + mbgl::util::toString(id);
-
- Response response;
- response.modified = SystemClock::now();
- response.expires = SystemClock::now() + Seconds(5000);
- response.notModified = true;
-
- Resource resource{ Resource::Kind::Tile, url };
- cache->put(resource, response);
-}
-
-uint64_t cacheSize(mbgl::SQLiteCache* cache, unsigned entryCount, uint64_t entrySize) {
- uint64_t total = 0;
-
- for (unsigned i = 0; i < entryCount; ++i) {
- if (tileIsCached(cache, i)) {
- total += entrySize;
- }
- }
-
- return total;
-}
-
-TEST_F(Storage, CacheEntrySizeLimit) {
- using namespace mbgl;
-
- util::RunLoop loop;
- SQLiteCache cache(":memory:");
-
- const uint64_t entrySize = 5 * 1024 * 1024; // 5 MB
-
- insertTile(&cache, 0, entrySize);
- EXPECT_TRUE(tileIsCached(&cache, 0));
-
- insertTile(&cache, 1, entrySize + 1);
- EXPECT_FALSE(tileIsCached(&cache, 1));
-
- insertTile(&cache, 2, entrySize - 1);
- EXPECT_TRUE(tileIsCached(&cache, 2));
-
- // Setting a new size should not delete existing entries.
- cache.setMaximumCacheEntrySize(entrySize / 2);
- EXPECT_TRUE(tileIsCached(&cache, 2));
-
- insertTile(&cache, 3, entrySize / 2 - 1);
- EXPECT_TRUE(tileIsCached(&cache, 3));
-
- insertTile(&cache, 4, entrySize);
- EXPECT_FALSE(tileIsCached(&cache, 4));
-
- cache.setMaximumCacheEntrySize(entrySize * 2);
- insertTile(&cache, 5, entrySize);
- EXPECT_TRUE(tileIsCached(&cache, 5));
-}
-
-TEST_F(Storage, CacheSizeSetNewLimit) {
- using namespace mbgl;
-
- util::RunLoop loop;
- SQLiteCache cache(":memory:");
-
- const unsigned entryCount = 800;
- const uint64_t entrySize = 10 * 1024; // 10 KB
-
- cache.setMaximumCacheEntrySize(entrySize + 1);
-
- // Cache size defaults to unlimited, all these
- // inserts should work.
- for (unsigned i = 0; i < entryCount; ++i) {
- insertTile(&cache, i, entrySize);
- }
-
- for (unsigned i = 0; i < entryCount; ++i) {
- EXPECT_TRUE(tileIsCached(&cache, i));
- }
-
- uint64_t expectedCacheSize = entryCount * entrySize;
- EXPECT_EQ(cacheSize(&cache, entryCount, entrySize), expectedCacheSize);
-
- // Setting a new size should remove records until the new
- // size limit is satisfied.
- cache.setMaximumCacheSize(expectedCacheSize / 2);
- EXPECT_LT(cacheSize(&cache, entryCount, entrySize), expectedCacheSize / 2);
-
- // Cache size 1 should practically clean the cache and
- // prevent adding any record, although it makes no sense
- // to use such size limit IRL.
- cache.setMaximumCacheSize(1);
- EXPECT_EQ(cacheSize(&cache, entryCount, entrySize), 0);
-
- insertTile(&cache, 1000, entrySize);
- EXPECT_FALSE(tileIsCached(&cache, 1000));
-
- // Zero should be treated as unlimited.
- cache.setMaximumCacheSize(0);
-
- for (unsigned i = 0; i < entryCount; ++i) {
- insertTile(&cache, i, entrySize);
- }
-
- EXPECT_EQ(cacheSize(&cache, entryCount, entrySize), expectedCacheSize);
-}
-
-TEST_F(Storage, CacheSizePruneLeastAccessed) {
- using namespace mbgl;
-
- util::RunLoop loop;
- SQLiteCache cache(":memory:");
-
- const unsigned entryCount = 400;
- const uint64_t entrySize = 10 * 1024; // 10 KB
-
- cache.setMaximumCacheEntrySize(entrySize + 1);
- cache.setMaximumCacheSize(entrySize * 350);
-
- for (unsigned i = 0; i < entryCount; ++i) {
- insertTile(&cache, i, entrySize);
-
- if (i == entryCount / 2) {
- // We need to sleep for 1s here because
- // that is the time resolution for the
- // `accessed` time. Then we 'ping' the
- // entry, that should update the
- // `accessed` time, so it won't get
- // pruned when we need more space.
- bool done = false;
-
- util::Timer timer;
- timer.start(Milliseconds(1300),
- Duration::zero(),
- [&done] { done = true; });
-
- while (!done) {
- loop.runOnce();
- }
-
- EXPECT_TRUE(tileIsCached(&cache, 7));
-
- // Refresh should also update the `accessed`
- // time of a tile.
- refreshTile(&cache, 9);
- }
- }
-
- EXPECT_FALSE(tileIsCached(&cache, 6));
- EXPECT_FALSE(tileIsCached(&cache, 8));
- EXPECT_FALSE(tileIsCached(&cache, 10));
-
- EXPECT_TRUE(tileIsCached(&cache, 7));
- EXPECT_TRUE(tileIsCached(&cache, 9));
-}
-
-TEST_F(Storage, CacheSizeStress) {
- using namespace mbgl;
-
- util::RunLoop loop;
- SQLiteCache cache(":memory:");
-
- const unsigned entryCount = 2000;
- const uint64_t entrySize = 10 * 1024; // 10 KB
-
- cache.setMaximumCacheEntrySize(entrySize + 1);
- cache.setMaximumCacheSize(entrySize * 300);
-
- for (unsigned i = 0; i < entryCount; ++i) {
- insertTile(&cache, i, entrySize);
- }
-
- // Should not be in the cache as they were
- // first inserted.
- EXPECT_FALSE(tileIsCached(&cache, 0));
- EXPECT_FALSE(tileIsCached(&cache, 99));
- EXPECT_FALSE(tileIsCached(&cache, 199));
- EXPECT_FALSE(tileIsCached(&cache, 299));
- EXPECT_FALSE(tileIsCached(&cache, 399));
-
- EXPECT_TRUE(tileIsCached(&cache, entryCount - 1));
-
- EXPECT_LT(cacheSize(&cache, entryCount, entrySize), entrySize * 300);
-}
diff --git a/test/storage/cache_stale.cpp b/test/storage/cache_stale.cpp
deleted file mode 100644
index 9ad0d1c06e..0000000000
--- a/test/storage/cache_stale.cpp
+++ /dev/null
@@ -1,131 +0,0 @@
-#include "storage.hpp"
-
-#include <mbgl/platform/default/headless_display.hpp>
-#include <mbgl/platform/default/headless_view.hpp>
-#include <mbgl/storage/sqlite_cache.hpp>
-#include <mbgl/storage/default_file_source.hpp>
-
-#include <mbgl/platform/log.hpp>
-#include <mbgl/util/work_request.hpp>
-#include <mbgl/util/io.hpp>
-
-using namespace mbgl;
-using namespace std::literals::chrono_literals;
-using namespace std::literals::string_literals;
-
-namespace {
-
-void checkRendering(Map& map,
- const char* name,
- std::chrono::milliseconds timeout,
- double imageThreshold = 0.001,
- double pixelThreshold = 0.1) {
- test::checkImage("test/fixtures/stale/"s + name, test::render(map, timeout), imageThreshold,
- pixelThreshold);
-}
-
-Response expiredItem(const std::string& path) {
- Response response;
- response.data = std::make_shared<std::string>(util::read_file("test/fixtures/"s + path));
- response.expires = SystemClock::from_time_t(0);
- return response;
-}
-
-const std::string prefix = "http://127.0.0.1:3000";
-
-}
-
-auto display = std::make_shared<mbgl::HeadlessDisplay>();
-
-TEST_F(Storage, CacheStaleStyle) {
- HeadlessView view(display, 1);
-
- auto cache = SQLiteCache::getShared(":memory:");
-
- // Rig the cache with an expired stylesheet.
- const std::string stylePath = "stale/style.json";
- const Resource styleResource{ Resource::Kind::Style, prefix + "/" + stylePath };
- cache->put(styleResource, expiredItem(stylePath));
-
- DefaultFileSource fileSource(":memory:", ".");
-
- Map map(view, fileSource, MapMode::Still);
- map.setStyleURL(styleResource.url);
-
- checkRendering(map, "stale_style", 1000ms);
-}
-
-TEST_F(Storage, CacheStaleStyleAndTileJSON) {
- HeadlessView view(display, 1);
-
- auto cache = SQLiteCache::getShared(":memory:");
-
- // Rig the cache with an expired stylesheet.
- const std::string stylePath = "stale/style_and_tilejson.json";
- const Resource styleResource{ Resource::Kind::Style, prefix + "/" + stylePath };
- cache->put(styleResource, expiredItem(stylePath));
-
- // Rig the cache with an expired TileJSON.
- const std::string tilejsonPath = "stale/streets.json";
- const Resource tilejsonResource{ Resource::Kind::Source, prefix + "/" + tilejsonPath };
- cache->put(tilejsonResource, expiredItem(tilejsonPath));
-
- DefaultFileSource fileSource(":memory:", ".");
-
- Map map(view, fileSource, MapMode::Still);
- map.setStyleURL(styleResource.url);
-
- checkRendering(map, "stale_style_and_tilejson", 1000ms);
-}
-
-TEST_F(Storage, CacheStaleStyleAndSprite) {
- HeadlessView view(display, 1);
-
- auto cache = SQLiteCache::getShared(":memory:");
-
- // Rig the cache with an expired stylesheet.
- const std::string stylePath = "stale/style_and_sprite.json";
- const Resource styleResource{ Resource::Kind::Style, prefix + "/" + stylePath };
- cache->put(styleResource, expiredItem(stylePath));
-
- // Rig the cache with an expired sprite JSON.
- const std::string spritejsonPath = "stale/sprite.json";
- const Resource spritejsonResource{ Resource::Kind::SpriteJSON, prefix + "/" + spritejsonPath };
- cache->put(spritejsonResource, expiredItem(spritejsonPath));
-
- // Rig the cache with an expired sprite JSON.
- const std::string spriteimagePath = "stale/sprite.png";
- const Resource spriteimageResource{ Resource::Kind::SpriteImage, prefix + "/" + spriteimagePath };
- cache->put(spriteimageResource, expiredItem(spriteimagePath));
-
- DefaultFileSource fileSource(":memory:", ".");
-
- Map map(view, fileSource, MapMode::Still);
- map.setStyleURL(styleResource.url);
-
- checkRendering(map, "stale_style_and_sprite", 1000ms);
-}
-
-TEST_F(Storage, CacheStaleStyleAndGlyphs) {
- HeadlessView view(display, 1);
-
- auto cache = SQLiteCache::getShared(":memory:");
-
- // Rig the cache with an expired stylesheet.
- const std::string stylePath = "stale/style_and_glyphs.json";
- const Resource styleResource{ Resource::Kind::Style, prefix + "/" + stylePath };
- cache->put(styleResource, expiredItem(stylePath));
-
- // Rig the cache with an expired glyph PBF.
- const std::string glyphPath = "stale/glyph.pbf";
- const Resource glyphResource{ Resource::Kind::Glyphs, prefix + "/stale/Open%20Sans%20Regular%2c%20Arial%20Unicode%20MS%20Regular/0-255.pbf" };
- cache->put(glyphResource, expiredItem(glyphPath));
-
- DefaultFileSource fileSource(":memory:", ".");
-
- Map map(view, fileSource, MapMode::Still);
- map.setStyleURL(styleResource.url);
-
- // TODO: this shouldn't take > 1 second
- checkRendering(map, "stale_style_and_glyphs", 2000ms, 0.0015);
-}
diff --git a/test/storage/database.cpp b/test/storage/database.cpp
deleted file mode 100644
index 19b46763b9..0000000000
--- a/test/storage/database.cpp
+++ /dev/null
@@ -1,349 +0,0 @@
-#include "storage.hpp"
-#include "../fixtures/fixture_log_observer.hpp"
-
-#include "sqlite_cache_impl.hpp"
-#include <mbgl/storage/resource.hpp>
-#include <mbgl/storage/response.hpp>
-#include <mbgl/util/io.hpp>
-
-#include <sqlite3.h>
-
-TEST_F(Storage, DatabaseDoesNotExist) {
- using namespace mbgl;
-
- Log::setObserver(std::make_unique<FixtureLogObserver>());
-
- SQLiteCache::Impl cache("test/fixtures/404/cache.db");
-
- cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
- EXPECT_EQ(nullptr, res.get());
- });
-
- auto observer = Log::removeObserver();
- EXPECT_EQ(1ul, dynamic_cast<FixtureLogObserver*>(observer.get())->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file" }));
-}
-
-void createDir(const char* name) {
- const int ret = mkdir(name, 0755);
- if (ret == -1) {
- ASSERT_EQ(EEXIST, errno);
- } else {
- ASSERT_EQ(0, ret);
- }
-}
-
-void deleteFile(const char* name) {
- const int ret = unlink(name);
- if (ret == -1) {
- ASSERT_EQ(ENOENT, errno);
- } else {
- ASSERT_EQ(0, ret);
- }
-}
-
-
-void writeFile(const char* name, const std::string& data) {
- mbgl::util::write_file(name, data);
-}
-
-
-TEST_F(Storage, DatabaseCreate) {
- using namespace mbgl;
-
- createDir("test/fixtures/database");
- deleteFile("test/fixtures/database/cache.db");
-
- Log::setObserver(std::make_unique<FixtureLogObserver>());
-
- SQLiteCache::Impl cache("test/fixtures/database/cache.db");
-
- cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
- EXPECT_EQ(nullptr, res.get());
- });
-
- Log::removeObserver();
-}
-
-TEST_F(Storage, DatabaseVersion) {
- using namespace mbgl;
-
- createDir("test/fixtures/database");
- deleteFile("test/fixtures/database/cache.db");
- std::string path("test/fixtures/database/cache.db");
-
- Log::setObserver(std::make_unique<FixtureLogObserver>());
-
- {
- SQLiteCache::Impl cache(path);
- cache.put({ Resource::Unknown, "mapbox://test" }, Response());
- }
-
- sqlite3* db;
- sqlite3_open_v2(path.c_str(), &db, SQLITE_OPEN_READWRITE, nullptr);
- sqlite3_exec(db, "PRAGMA user_version = 999999", nullptr, nullptr, nullptr);
- sqlite3_close_v2(db);
-
- // Changing the version will force the database to get recreated
- // thus removing every pre-existing cache entry.
- {
- SQLiteCache::Impl cache(path);
-
- cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
- EXPECT_EQ(nullptr, res.get());
- });
- }
-
- Log::removeObserver();
-}
-
-class FileLock {
-public:
- FileLock(const std::string& path) {
- const int err = sqlite3_open_v2(path.c_str(), &db, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE, nullptr);
- if (err != SQLITE_OK) {
- throw std::runtime_error("Could not open db");
- }
- lock();
- }
-
- void lock() {
- assert(!locked);
- const int err = sqlite3_exec(db, "begin exclusive transaction", nullptr, nullptr, nullptr);
- if (err != SQLITE_OK) {
- throw std::runtime_error("Could not lock db");
- }
- locked = true;
- }
-
- void unlock() {
- assert(locked);
- const int err = sqlite3_exec(db, "commit", nullptr, nullptr, nullptr);
- if (err != SQLITE_OK) {
- throw std::runtime_error("Could not unlock db");
- }
- locked = false;
- }
-
- ~FileLock() {
- if (locked) {
- unlock();
- }
- }
-
-private:
- sqlite3* db;
- bool locked = false;
-};
-
-TEST_F(Storage, DatabaseLockedRead) {
- using namespace mbgl;
-
- // Create a locked file.
- createDir("test/fixtures/database");
- deleteFile("test/fixtures/database/locked.db");
- FileLock guard("test/fixtures/database/locked.db");
-
- SQLiteCache::Impl cache("test/fixtures/database/locked.db");
-
- {
- // First request should fail.
- Log::setObserver(std::make_unique<FixtureLogObserver>());
-
- cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
- EXPECT_EQ(nullptr, res.get());
- });
-
- // Make sure that we got a few "database locked" errors
- auto observer = Log::removeObserver();
- auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
- EXPECT_EQ(4ul, flo->count({ EventSeverity::Error, Event::Database, 5, "database is locked" }));
- }
-
- // Then, unlock the file and try again.
- guard.unlock();
-
- {
- // First, try getting a file (the cache value should not exist).
- Log::setObserver(std::make_unique<FixtureLogObserver>());
-
- cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
- EXPECT_EQ(nullptr, res.get());
- });
-
- // Make sure that we got a no errors
- Log::removeObserver();
- }
-}
-
-
-
-TEST_F(Storage, DatabaseLockedWrite) {
- using namespace mbgl;
-
- // Create a locked file.
- createDir("test/fixtures/database");
- deleteFile("test/fixtures/database/locked.db");
- FileLock guard("test/fixtures/database/locked.db");
-
- SQLiteCache::Impl cache("test/fixtures/database/locked.db");
-
- {
- // Adds a file (which should fail).
- Log::setObserver(std::make_unique<FixtureLogObserver>());
-
- cache.put({ Resource::Unknown, "mapbox://test" }, Response());
- cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
- EXPECT_EQ(nullptr, res.get());
- });
-
- auto observer = Log::removeObserver();
- auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
- EXPECT_EQ(8ul, flo->count({ EventSeverity::Error, Event::Database, 5, "database is locked" }));
- }
-
- // Then, unlock the file and try again.
- guard.unlock();
-
- {
- // Then, set a file and obtain it again.
- Log::setObserver(std::make_unique<FixtureLogObserver>());
-
- Response response;
- response.data = std::make_shared<std::string>("Demo");
- cache.put({ Resource::Unknown, "mapbox://test" }, response);
- cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
- ASSERT_NE(nullptr, res.get());
- ASSERT_TRUE(res->data.get());
- EXPECT_EQ("Demo", *res->data);
- });
-
- // Make sure that we got a no errors
- Log::removeObserver();
- }
-}
-
-
-
-
-TEST_F(Storage, DatabaseLockedRefresh) {
- using namespace mbgl;
-
- // Create a locked file.
- createDir("test/fixtures/database");
- deleteFile("test/fixtures/database/locked.db");
-
- SQLiteCache::Impl cache("test/fixtures/database/locked.db");
-
- // Then, lock the file and try again.
- FileLock guard("test/fixtures/database/locked.db");
-
- {
- // Adds a file.
- Log::setObserver(std::make_unique<FixtureLogObserver>());
-
- Response response;
- response.data = std::make_shared<std::string>("Demo");
- cache.put({ Resource::Unknown, "mapbox://test" }, response);
- cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
- EXPECT_EQ(nullptr, res.get());
- });
-
- auto observer = Log::removeObserver();
- auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
- EXPECT_EQ(8ul, flo->count({ EventSeverity::Error, Event::Database, 5, "database is locked" }));
- }
-
- {
- // Then, try to refresh it.
- Log::setObserver(std::make_unique<FixtureLogObserver>());
-
- cache.refresh({ Resource::Unknown, "mapbox://test" }, {});
- cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
- EXPECT_EQ(nullptr, res.get());
- });
-
- // Make sure that we got the right errors.
- auto observer = Log::removeObserver();
- auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
- EXPECT_EQ(8ul, flo->count({ EventSeverity::Error, Event::Database, 5, "database is locked" }));
- }
-}
-
-
-
-TEST_F(Storage, DatabaseDeleted) {
- using namespace mbgl;
-
- // Create a locked file.
- createDir("test/fixtures/database");
- deleteFile("test/fixtures/database/locked.db");
-
- SQLiteCache::Impl cache("test/fixtures/database/locked.db");
-
- {
- // Adds a file.
- Log::setObserver(std::make_unique<FixtureLogObserver>());
-
- Response response;
- response.data = std::make_shared<std::string>("Demo");
- cache.put({ Resource::Unknown, "mapbox://test" }, response);
- cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
- ASSERT_NE(nullptr, res.get());
- ASSERT_TRUE(res->data.get());
- EXPECT_EQ("Demo", *res->data);
- });
-
- Log::removeObserver();
- }
-
- deleteFile("test/fixtures/database/locked.db");
-
- {
- // Adds a file.
- Log::setObserver(std::make_unique<FixtureLogObserver>());
-
- Response response;
- response.data = std::make_shared<std::string>("Demo");
- cache.put({ Resource::Unknown, "mapbox://test" }, response);
- cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
- ASSERT_NE(nullptr, res.get());
- ASSERT_TRUE(res->data.get());
- EXPECT_EQ("Demo", *res->data);
- });
-
- auto observer = Log::removeObserver();
- auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
- EXPECT_EQ(1ul, flo->count({ EventSeverity::Error, Event::Database, 8, "attempt to write a readonly database" }));
- }
-}
-
-
-
-TEST_F(Storage, DatabaseInvalid) {
- using namespace mbgl;
-
- // Create a locked file.
- createDir("test/fixtures/database");
- deleteFile("test/fixtures/database/invalid.db");
- writeFile("test/fixtures/database/invalid.db", "this is an invalid file");
-
- SQLiteCache::Impl cache("test/fixtures/database/invalid.db");
-
- {
- // Adds a file.
- Log::setObserver(std::make_unique<FixtureLogObserver>());
-
- Response response;
- response.data = std::make_shared<std::string>("Demo");
- cache.put({ Resource::Unknown, "mapbox://test" }, response);
- cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) {
- ASSERT_NE(nullptr, res.get());
- ASSERT_TRUE(res->data.get());
- EXPECT_EQ("Demo", *res->data);
- });
-
- auto observer = Log::removeObserver();
- auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
- EXPECT_EQ(1ul, flo->count({ EventSeverity::Warning, Event::Database, -1, "Trashing invalid database" }));
- }
-}
diff --git a/test/storage/cache_revalidate.cpp b/test/storage/default_file_source.cpp
index e769e934d5..26fb164d3f 100644
--- a/test/storage/cache_revalidate.cpp
+++ b/test/storage/default_file_source.cpp
@@ -1,10 +1,54 @@
#include "storage.hpp"
#include <mbgl/storage/default_file_source.hpp>
-#include <mbgl/util/chrono.hpp>
#include <mbgl/util/run_loop.hpp>
-TEST_F(Storage, CacheRevalidateSame) {
+class DefaultFileSourceTest : public Storage {};
+
+TEST_F(DefaultFileSourceTest, CacheResponse) {
+ SCOPED_TEST(CacheResponse);
+
+ using namespace mbgl;
+
+ util::RunLoop loop;
+ DefaultFileSource fs(":memory:", ".");
+
+ const Resource resource { Resource::Unknown, "http://127.0.0.1:3000/cache" };
+ Response response;
+
+ std::unique_ptr<FileRequest> req1;
+ std::unique_ptr<FileRequest> req2;
+
+ req1 = fs.request(resource, [&](Response res) {
+ req1.reset();
+ EXPECT_EQ(nullptr, res.error);
+ ASSERT_TRUE(res.data.get());
+ EXPECT_EQ("Response 1", *res.data);
+ EXPECT_TRUE(bool(res.expires));
+ EXPECT_FALSE(bool(res.modified));
+ EXPECT_FALSE(bool(res.etag));
+ response = res;
+
+ // Now test that we get the same values as in the previous request. If we'd go to the server
+ // again, we'd get different values.
+ req2 = fs.request(resource, [&](Response res2) {
+ req2.reset();
+ EXPECT_EQ(response.error, res2.error);
+ ASSERT_TRUE(res2.data.get());
+ EXPECT_EQ(*response.data, *res2.data);
+ EXPECT_EQ(response.expires, res2.expires);
+ EXPECT_EQ(response.modified, res2.modified);
+ EXPECT_EQ(response.etag, res2.etag);
+
+ loop.stop();
+ CacheResponse.finish();
+ });
+ });
+
+ loop.run();
+}
+
+TEST_F(DefaultFileSourceTest, CacheRevalidateSame) {
SCOPED_TEST(CacheRevalidateSame)
using namespace mbgl;
@@ -53,7 +97,7 @@ TEST_F(Storage, CacheRevalidateSame) {
loop.run();
}
-TEST_F(Storage, CacheRevalidateModified) {
+TEST_F(DefaultFileSourceTest, CacheRevalidateModified) {
SCOPED_TEST(CacheRevalidateModified)
using namespace mbgl;
@@ -102,7 +146,7 @@ TEST_F(Storage, CacheRevalidateModified) {
loop.run();
}
-TEST_F(Storage, CacheRevalidateEtag) {
+TEST_F(DefaultFileSourceTest, CacheRevalidateEtag) {
SCOPED_TEST(CacheRevalidateEtag)
using namespace mbgl;
diff --git a/test/storage/offline_database.cpp b/test/storage/offline_database.cpp
new file mode 100644
index 0000000000..e2e32ee36b
--- /dev/null
+++ b/test/storage/offline_database.cpp
@@ -0,0 +1,374 @@
+#include "../fixtures/fixture_log_observer.hpp"
+
+#include <mbgl/storage/offline_database.hpp>
+#include <mbgl/storage/resource.hpp>
+#include <mbgl/storage/response.hpp>
+#include <mbgl/util/io.hpp>
+
+#include <gtest/gtest.h>
+#include <sqlite3.h>
+
+namespace {
+
+void createDir(const char* name) {
+ const int ret = mkdir(name, 0755);
+ if (ret == -1) {
+ ASSERT_EQ(EEXIST, errno);
+ } else {
+ ASSERT_EQ(0, ret);
+ }
+}
+
+void deleteFile(const char* name) {
+ const int ret = unlink(name);
+ if (ret == -1) {
+ ASSERT_EQ(ENOENT, errno);
+ } else {
+ ASSERT_EQ(0, ret);
+ }
+}
+
+void writeFile(const char* name, const std::string& data) {
+ mbgl::util::write_file(name, data);
+}
+
+class FileLock {
+public:
+ FileLock(const std::string& path) {
+ const int err = sqlite3_open_v2(path.c_str(), &db, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE, nullptr);
+ if (err != SQLITE_OK) {
+ throw std::runtime_error("Could not open db");
+ }
+ lock();
+ }
+
+ void lock() {
+ assert(!locked);
+ const int err = sqlite3_exec(db, "begin exclusive transaction", nullptr, nullptr, nullptr);
+ if (err != SQLITE_OK) {
+ throw std::runtime_error("Could not lock db");
+ }
+ locked = true;
+ }
+
+ void unlock() {
+ assert(locked);
+ const int err = sqlite3_exec(db, "commit", nullptr, nullptr, nullptr);
+ if (err != SQLITE_OK) {
+ throw std::runtime_error("Could not unlock db");
+ }
+ locked = false;
+ }
+
+ ~FileLock() {
+ if (locked) {
+ unlock();
+ }
+ }
+
+private:
+ sqlite3* db;
+ bool locked = false;
+};
+
+}
+
+//TEST(OfflineDatabase, NonexistentDirectory) {
+// using namespace mbgl;
+//
+// Log::setObserver(std::make_unique<FixtureLogObserver>());
+//
+// OfflineDatabase db("test/fixtures/404/offline.db");
+//
+// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) {
+// EXPECT_FALSE(bool(res));
+// });
+//
+// auto observer = Log::removeObserver();
+// EXPECT_EQ(1ul, dynamic_cast<FixtureLogObserver*>(observer.get())->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file" }));
+//}
+
+TEST(OfflineDatabase, Create) {
+ using namespace mbgl;
+
+ createDir("test/fixtures/database");
+ deleteFile("test/fixtures/database/offline.db");
+
+ Log::setObserver(std::make_unique<FixtureLogObserver>());
+
+ OfflineDatabase db("test/fixtures/database/offline.db");
+ EXPECT_FALSE(bool(db.get({ Resource::Unknown, "mapbox://test" })));
+
+ Log::removeObserver();
+}
+
+TEST(OfflineDatabase, SchemaVersion) {
+ using namespace mbgl;
+
+ createDir("test/fixtures/database");
+ deleteFile("test/fixtures/database/offline.db");
+ std::string path("test/fixtures/database/offline.db");
+
+ {
+ sqlite3* db;
+ sqlite3_open_v2(path.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nullptr);
+ sqlite3_exec(db, "PRAGMA user_version = 1", nullptr, nullptr, nullptr);
+ sqlite3_close_v2(db);
+ }
+
+ Log::setObserver(std::make_unique<FixtureLogObserver>());
+ OfflineDatabase db(path);
+
+ auto observer = Log::removeObserver();
+ auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
+ EXPECT_EQ(1ul, flo->count({ EventSeverity::Warning, Event::Database, -1, "Removing existing incompatible offline database" }));
+}
+
+TEST(OfflineDatabase, Invalid) {
+ using namespace mbgl;
+
+ createDir("test/fixtures/database");
+ deleteFile("test/fixtures/database/invalid.db");
+ writeFile("test/fixtures/database/invalid.db", "this is an invalid file");
+
+ Log::setObserver(std::make_unique<FixtureLogObserver>());
+
+ OfflineDatabase db("test/fixtures/database/invalid.db");
+
+ auto observer = Log::removeObserver();
+ auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
+ EXPECT_EQ(1ul, flo->count({ EventSeverity::Warning, Event::Database, -1, "Removing existing incompatible offline database" }));
+}
+
+//TEST(OfflineDatabase, DatabaseLockedRead) {
+// using namespace mbgl;
+//
+// // Create a locked file.
+// createDir("test/fixtures/database");
+// deleteFile("test/fixtures/database/locked.db");
+// FileLock guard("test/fixtures/database/locked.db");
+//
+// OfflineDatabase db("test/fixtures/database/locked.db");
+//
+// {
+// // First request should fail.
+// Log::setObserver(std::make_unique<FixtureLogObserver>());
+//
+// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) {
+// EXPECT_FALSE(bool(res));
+// });
+//
+// // Make sure that we got a few "database locked" errors
+// auto observer = Log::removeObserver();
+// auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
+// EXPECT_EQ(4ul, flo->count({ EventSeverity::Error, Event::Database, 5, "database is locked" }));
+// }
+//
+// // Then, unlock the file and try again.
+// guard.unlock();
+//
+// {
+// // First, try getting a file (the cache value should not exist).
+// Log::setObserver(std::make_unique<FixtureLogObserver>());
+//
+// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) {
+// EXPECT_FALSE(bool(res));
+// });
+//
+// // Make sure that we got a no errors
+// Log::removeObserver();
+// }
+//}
+//
+//TEST(OfflineDatabase, DatabaseLockedWrite) {
+// using namespace mbgl;
+//
+// // Create a locked file.
+// createDir("test/fixtures/database");
+// deleteFile("test/fixtures/database/locked.db");
+// FileLock guard("test/fixtures/database/locked.db");
+//
+// OfflineDatabase db("test/fixtures/database/locked.db");
+//
+// {
+// // Adds a file (which should fail).
+// Log::setObserver(std::make_unique<FixtureLogObserver>());
+//
+// db.put({ Resource::Unknown, "mapbox://test" }, Response());
+// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) {
+// EXPECT_FALSE(bool(res));
+// });
+//
+// auto observer = Log::removeObserver();
+// auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
+// EXPECT_EQ(8ul, flo->count({ EventSeverity::Error, Event::Database, 5, "database is locked" }));
+// }
+//
+// // Then, unlock the file and try again.
+// guard.unlock();
+//
+// {
+// // Then, set a file and obtain it again.
+// Log::setObserver(std::make_unique<FixtureLogObserver>());
+//
+// Response response;
+// response.data = std::make_shared<std::string>("Demo");
+// db.put({ Resource::Unknown, "mapbox://test" }, response);
+// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) {
+// ASSERT_TRUE(bool(res));
+// ASSERT_TRUE(res->data.get());
+// EXPECT_EQ("Demo", *res->data);
+// });
+//
+// // Make sure that we got a no errors
+// Log::removeObserver();
+// }
+//}
+//
+//TEST(OfflineDatabase, DatabaseDeleted) {
+// using namespace mbgl;
+//
+// // Create a locked file.
+// createDir("test/fixtures/database");
+// deleteFile("test/fixtures/database/locked.db");
+//
+// OfflineDatabase db("test/fixtures/database/locked.db");
+//
+// {
+// // Adds a file.
+// Log::setObserver(std::make_unique<FixtureLogObserver>());
+//
+// Response response;
+// response.data = std::make_shared<std::string>("Demo");
+// db.put({ Resource::Unknown, "mapbox://test" }, response);
+// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) {
+// ASSERT_TRUE(bool(res));
+// ASSERT_TRUE(res->data.get());
+// EXPECT_EQ("Demo", *res->data);
+// });
+//
+// Log::removeObserver();
+// }
+//
+// deleteFile("test/fixtures/database/locked.db");
+//
+// {
+// // Adds a file.
+// Log::setObserver(std::make_unique<FixtureLogObserver>());
+//
+// Response response;
+// response.data = std::make_shared<std::string>("Demo");
+// db.put({ Resource::Unknown, "mapbox://test" }, response);
+// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) {
+// ASSERT_TRUE(bool(res));
+// ASSERT_TRUE(res->data.get());
+// EXPECT_EQ("Demo", *res->data);
+// });
+//
+// auto observer = Log::removeObserver();
+// auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
+// EXPECT_EQ(1ul, flo->count({ EventSeverity::Error, Event::Database, 8, "attempt to write a readonly database" }));
+// }
+//}
+
+TEST(OfflineDatabase, PutDoesNotStoreConnectionErrors) {
+ using namespace mbgl;
+
+ OfflineDatabase db(":memory:");
+
+ Resource resource { Resource::Unknown, "http://example.com/" };
+ Response response;
+ response.error = std::make_unique<Response::Error>(Response::Error::Reason::Connection);
+
+ db.put(resource, response);
+ EXPECT_FALSE(bool(db.get(resource)));
+}
+
+TEST(OfflineDatabase, PutDoesNotStoreServerErrors) {
+ using namespace mbgl;
+
+ OfflineDatabase db(":memory:");
+
+ Resource resource { Resource::Unknown, "http://example.com/" };
+ Response response;
+ response.error = std::make_unique<Response::Error>(Response::Error::Reason::Server);
+
+ db.put(resource, response);
+ EXPECT_FALSE(bool(db.get(resource)));
+}
+
+TEST(OfflineDatabase, PutResource) {
+ using namespace mbgl;
+
+ OfflineDatabase db(":memory:");
+
+ Resource resource { Resource::Style, "http://example.com/" };
+ Response response;
+ response.data = std::make_shared<std::string>("data");
+
+ db.put(resource, response);
+ auto res = db.get(resource);
+ EXPECT_EQ(nullptr, res->error.get());
+ EXPECT_EQ("data", *res->data);
+}
+
+TEST(OfflineDatabase, PutTile) {
+ using namespace mbgl;
+
+ OfflineDatabase db(":memory:");
+
+ Resource resource { Resource::Tile, "http://example.com/" };
+ resource.tileData = Resource::TileData {
+ "http://example.com/",
+ 1,
+ 0,
+ 0,
+ 0
+ };
+ Response response;
+ response.data = std::make_shared<std::string>("data");
+
+ db.put(resource, response);
+ auto res = db.get(resource);
+ EXPECT_EQ(nullptr, res->error.get());
+ EXPECT_EQ("data", *res->data);
+}
+
+TEST(OfflineDatabase, PutResourceNotFound) {
+ using namespace mbgl;
+
+ OfflineDatabase db(":memory:");
+
+ Resource resource { Resource::Style, "http://example.com/" };
+ Response response;
+ response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
+
+ db.put(resource, response);
+ auto res = db.get(resource);
+ EXPECT_NE(nullptr, res->error);
+ EXPECT_EQ(Response::Error::Reason::NotFound, res->error->reason);
+ EXPECT_FALSE(res->data.get());
+}
+
+TEST(OfflineDatabase, PutTileNotFound) {
+ using namespace mbgl;
+
+ OfflineDatabase db(":memory:");
+
+ Resource resource { Resource::Tile, "http://example.com/" };
+ resource.tileData = Resource::TileData {
+ "http://example.com/",
+ 1,
+ 0,
+ 0,
+ 0
+ };
+ Response response;
+ response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
+
+ db.put(resource, response);
+ auto res = db.get(resource);
+ EXPECT_NE(nullptr, res->error);
+ EXPECT_EQ(Response::Error::Reason::NotFound, res->error->reason);
+ EXPECT_FALSE(res->data.get());
+}
diff --git a/test/test.gypi b/test/test.gypi
index 22e3d0184e..883b954a35 100644
--- a/test/test.gypi
+++ b/test/test.gypi
@@ -58,6 +58,7 @@
'api/render_missing.cpp',
'api/set_style.cpp',
'api/custom_layer.cpp',
+ 'api/offline.cpp',
'geometry/binpack.cpp',
@@ -68,12 +69,8 @@
'storage/storage.hpp',
'storage/storage.cpp',
- 'storage/cache_response.cpp',
- 'storage/cache_revalidate.cpp',
- 'storage/cache_shared.cpp',
- 'storage/cache_size.cpp',
- 'storage/cache_stale.cpp',
- 'storage/database.cpp',
+ 'storage/default_file_source.cpp',
+ 'storage/offline_database.cpp',
'storage/asset_file_source.cpp',
'storage/headers.cpp',
'storage/http_cancel.cpp',