diff options
42 files changed, 1114 insertions, 1749 deletions
diff --git a/.gitignore b/.gitignore index f9fd16cd2b..f750e2f876 100644 --- a/.gitignore +++ b/.gitignore @@ -23,8 +23,8 @@ /test/fixtures/api/1.png /test/fixtures/api/2.png /test/fixtures/database/*.db -/test/fixtures/*/*/actual.png -/test/fixtures/*/*/diff.png +/test/fixtures/**/actual.png +/test/fixtures/**/diff.png /test/output /include/mbgl/shader/shaders.hpp /src/shader/shaders_gl.cpp diff --git a/gyp/platform-android.gypi b/gyp/platform-android.gypi index 8bb046a588..2c1e06a715 100644 --- a/gyp/platform-android.gypi +++ b/gyp/platform-android.gypi @@ -21,7 +21,8 @@ '../platform/default/timer.cpp', '../platform/default/default_file_source.cpp', '../platform/default/online_file_source.cpp', - '../platform/default/sqlite_cache.cpp', + '../platform/default/mbgl/storage/offline_database.hpp', + '../platform/default/mbgl/storage/offline_database.cpp', '../platform/default/sqlite3.hpp', '../platform/default/sqlite3.cpp', ], @@ -56,6 +57,7 @@ 'include_dirs': [ '../include', '../src', + '../platform/default', ], 'conditions': [ diff --git a/gyp/platform-ios.gypi b/gyp/platform-ios.gypi index f6d94b0e1c..806f103a17 100644 --- a/gyp/platform-ios.gypi +++ b/gyp/platform-ios.gypi @@ -16,7 +16,8 @@ '../platform/default/timer.cpp', '../platform/default/default_file_source.cpp', '../platform/default/online_file_source.cpp', - '../platform/default/sqlite_cache.cpp', + '../platform/default/mbgl/storage/offline_database.hpp', + '../platform/default/mbgl/storage/offline_database.cpp', '../platform/default/sqlite3.hpp', '../platform/default/sqlite3.cpp', '../platform/darwin/log_nslog.mm', @@ -98,6 +99,7 @@ '../include/mbgl/darwin', '../include', '../src', + '../platform/default', ], 'xcode_settings': { diff --git a/gyp/platform-linux.gypi b/gyp/platform-linux.gypi index 1513828821..2eafe3e821 100644 --- a/gyp/platform-linux.gypi +++ b/gyp/platform-linux.gypi @@ -23,7 +23,8 @@ '../platform/default/timer.cpp', '../platform/default/default_file_source.cpp', '../platform/default/online_file_source.cpp', - '../platform/default/sqlite_cache.cpp', + '../platform/default/mbgl/storage/offline_database.hpp', + '../platform/default/mbgl/storage/offline_database.cpp', '../platform/default/sqlite3.hpp', '../platform/default/sqlite3.cpp', ], @@ -61,6 +62,7 @@ 'include_dirs': [ '../include', '../src', + '../platform/default', ], 'conditions': [ diff --git a/gyp/platform-osx.gypi b/gyp/platform-osx.gypi index 2aedce8538..53e7047492 100644 --- a/gyp/platform-osx.gypi +++ b/gyp/platform-osx.gypi @@ -15,7 +15,8 @@ '../platform/default/timer.cpp', '../platform/default/default_file_source.cpp', '../platform/default/online_file_source.cpp', - '../platform/default/sqlite_cache.cpp', + '../platform/default/mbgl/storage/offline_database.hpp', + '../platform/default/mbgl/storage/offline_database.cpp', '../platform/default/sqlite3.hpp', '../platform/default/sqlite3.cpp', '../platform/darwin/log_nslog.mm', @@ -83,6 +84,7 @@ '../include/mbgl/darwin', '../include', '../src', + '../platform/default', ], 'xcode_settings': { diff --git a/include/mbgl/storage/default_file_source.hpp b/include/mbgl/storage/default_file_source.hpp index e7f95aa238..d8fc4b98a3 100644 --- a/include/mbgl/storage/default_file_source.hpp +++ b/include/mbgl/storage/default_file_source.hpp @@ -5,7 +5,9 @@ namespace mbgl { -class SQLiteCache; +namespace util { +template <typename T> class Thread; +} // namespace util class DefaultFileSource : public FileSource { public: @@ -20,14 +22,15 @@ public: std::unique_ptr<FileRequest> request(const Resource&, Callback) override; - // For testing purposes only. - SQLiteCache& getCache(); + // For testing only. + void put(const Resource&, const Response&); + void goOffline(); -private: class Impl; - friend class DefaultFileRequest; - const std::unique_ptr<Impl> impl; +private: + const std::unique_ptr<util::Thread<Impl>> thread; + const std::unique_ptr<FileSource> assetFileSource; }; } // namespace mbgl diff --git a/platform/default/default_file_source.cpp b/platform/default/default_file_source.cpp index 56b47539b1..efe893d49b 100644 --- a/platform/default/default_file_source.cpp +++ b/platform/default/default_file_source.cpp @@ -1,10 +1,11 @@ #include <mbgl/storage/default_file_source.hpp> #include <mbgl/storage/asset_file_source.hpp> #include <mbgl/storage/online_file_source.hpp> -#include <mbgl/storage/sqlite_cache.hpp> +#include <mbgl/storage/offline_database.hpp> #include <mbgl/platform/platform.hpp> #include <mbgl/util/url.hpp> +#include <mbgl/util/thread.hpp> #include <mbgl/util/work_request.hpp> #include <cassert> @@ -23,76 +24,117 @@ namespace mbgl { class DefaultFileSource::Impl { public: - Impl(const std::string& cachePath, const std::string& assetRoot) - : assetFileSource(assetRoot), - cache(SQLiteCache::getShared(cachePath)) { + class Task { + public: + Task(Resource resource, FileSource::Callback callback, DefaultFileSource::Impl* impl) { + auto offlineResponse = impl->offlineDatabase.get(resource); + + Resource revalidation = resource; + + if (offlineResponse) { + revalidation.priorModified = offlineResponse->modified; + revalidation.priorExpires = offlineResponse->expires; + revalidation.priorEtag = offlineResponse->etag; + callback(*offlineResponse); + } + + if (!impl->offline) { + onlineRequest = impl->onlineFileSource.request(revalidation, [=] (Response onlineResponse) { + impl->offlineDatabase.put(revalidation, onlineResponse); + callback(onlineResponse); + }); + } + } + + std::unique_ptr<FileRequest> onlineRequest; + }; + + Impl(const std::string& cachePath) + : offlineDatabase(cachePath) { + } + + void setAccessToken(const std::string& accessToken) { + onlineFileSource.setAccessToken(accessToken); + } + + std::string getAccessToken() const { + return onlineFileSource.getAccessToken(); + } + + void add(FileRequest* req, Resource resource, Callback callback) { + tasks[req] = std::make_unique<Task>(resource, callback, this); } - AssetFileSource assetFileSource; - std::shared_ptr<SQLiteCache> cache; + void cancel(FileRequest* req) { + tasks.erase(req); + } + + void put(const Resource& resource, const Response& response) { + offlineDatabase.put(resource, response); + } + + void goOffline() { + offline = true; + } + + OfflineDatabase offlineDatabase; OnlineFileSource onlineFileSource; + std::unordered_map<FileRequest*, std::unique_ptr<Task>> tasks; + bool offline = false; +}; + +class DefaultFileRequest : public FileRequest { +public: + DefaultFileRequest(Resource resource, FileSource::Callback callback, util::Thread<DefaultFileSource::Impl>& thread_) + : thread(thread_), + workRequest(thread.invokeWithCallback(&DefaultFileSource::Impl::add, callback, this, resource)) { + } + + ~DefaultFileRequest() { + thread.invoke(&DefaultFileSource::Impl::cancel, this); + } + + util::Thread<DefaultFileSource::Impl>& thread; + std::unique_ptr<WorkRequest> workRequest; }; DefaultFileSource::DefaultFileSource(const std::string& cachePath, const std::string& assetRoot) - : impl(std::make_unique<DefaultFileSource::Impl>(cachePath, assetRoot)) { + : thread(std::make_unique<util::Thread<DefaultFileSource::Impl>>(util::ThreadContext{"DefaultFileSource", util::ThreadType::Unknown, util::ThreadPriority::Low}, cachePath)), + assetFileSource(std::make_unique<AssetFileSource>(assetRoot)) { } DefaultFileSource::~DefaultFileSource() = default; void DefaultFileSource::setAccessToken(const std::string& accessToken) { - impl->onlineFileSource.setAccessToken(accessToken); + thread->invokeSync(&Impl::setAccessToken, accessToken); } std::string DefaultFileSource::getAccessToken() const { - return impl->onlineFileSource.getAccessToken(); -} - -void DefaultFileSource::setMaximumCacheSize(uint64_t size) { - impl->cache->setMaximumCacheSize(size); + return thread->invokeSync<std::string>(&Impl::getAccessToken); } -void DefaultFileSource::setMaximumCacheEntrySize(uint64_t size) { - impl->cache->setMaximumCacheEntrySize(size); +void DefaultFileSource::setMaximumCacheSize(uint64_t) { + // TODO } -SQLiteCache& DefaultFileSource::getCache() { - return *impl->cache; +void DefaultFileSource::setMaximumCacheEntrySize(uint64_t) { + // TODO } -class DefaultFileRequest : public FileRequest { -public: - DefaultFileRequest(Resource resource, FileSource::Callback callback, DefaultFileSource::Impl* impl) { - cacheRequest = impl->cache->get(resource, [=](std::shared_ptr<Response> cacheResponse) mutable { - cacheRequest.reset(); - - if (cacheResponse) { - resource.priorModified = cacheResponse->modified; - resource.priorExpires = cacheResponse->expires; - resource.priorEtag = cacheResponse->etag; - } - - onlineRequest = impl->onlineFileSource.request(resource, [=] (Response onlineResponse) { - impl->cache->put(resource, onlineResponse); - callback(onlineResponse); - }); - - // Do this last because it may result in deleting this DefaultFileRequest. - if (cacheResponse) { - callback(*cacheResponse); - } - }); - } - - std::unique_ptr<WorkRequest> cacheRequest; - std::unique_ptr<FileRequest> onlineRequest; -}; - std::unique_ptr<FileRequest> DefaultFileSource::request(const Resource& resource, Callback callback) { if (isAssetURL(resource.url)) { - return impl->assetFileSource.request(resource, callback); + return assetFileSource->request(resource, callback); } else { - return std::make_unique<DefaultFileRequest>(resource, callback, impl.get()); + return std::make_unique<DefaultFileRequest>(resource, callback, *thread); } } +void DefaultFileSource::put(const Resource& resource, const Response& response) { + thread->invokeSync(&Impl::put, resource, response); +} + +void DefaultFileSource::goOffline() { + thread->invokeSync(&Impl::goOffline); +} + } // namespace mbgl diff --git a/platform/default/mbgl/storage/offline_database.cpp b/platform/default/mbgl/storage/offline_database.cpp new file mode 100644 index 0000000000..cd0f88b7fe --- /dev/null +++ b/platform/default/mbgl/storage/offline_database.cpp @@ -0,0 +1,297 @@ +#include <mbgl/storage/offline_database.hpp> +#include <mbgl/storage/response.hpp> +#include <mbgl/util/compression.hpp> +#include <mbgl/util/io.hpp> +#include <mbgl/util/string.hpp> +#include <mbgl/util/chrono.hpp> +#include <mbgl/map/tile_id.hpp> +#include <mbgl/platform/log.hpp> + +#include "sqlite3.hpp" +#include <sqlite3.h> + +namespace mbgl { + +using namespace mapbox::sqlite; + +// If you change the schema you must write a migration from the previous version. +static const uint32_t schemaVersion = 2; + +OfflineDatabase::OfflineDatabase(const std::string& path_) + : path(path_) { + ensureSchema(); +} + +OfflineDatabase::~OfflineDatabase() { + // Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we + // can't throw anything. + try { + statements.clear(); + db.reset(); + } catch (mapbox::sqlite::Exception& ex) { + Log::Error(Event::Database, ex.code, ex.what()); + } +} + +void OfflineDatabase::ensureSchema() { + if (path != ":memory:") { + try { + db = std::make_unique<Database>(path.c_str(), ReadWrite); + + { + Statement userVersionStmt(db->prepare("PRAGMA user_version")); + userVersionStmt.run(); + switch (userVersionStmt.get<int>(0)) { + case 0: break; // cache-only database; ok to delete + case 1: break; // cache-only database; ok to delete + case 2: return; + default: throw std::runtime_error("unknown schema version"); + } + } + + removeExisting(); + db = std::make_unique<Database>(path.c_str(), ReadWrite | Create); + } catch (mapbox::sqlite::Exception& ex) { + if (ex.code == SQLITE_CANTOPEN) { + db = std::make_unique<Database>(path.c_str(), ReadWrite | Create); + } else if (ex.code == SQLITE_NOTADB) { + removeExisting(); + db = std::make_unique<Database>(path.c_str(), ReadWrite | Create); + } + } + } + + #include "offline_schema.cpp.include" + + db = std::make_unique<Database>(path.c_str(), ReadWrite | Create); + db->exec(schema); + db->exec("PRAGMA user_version = " + util::toString(schemaVersion)); +} + +void OfflineDatabase::removeExisting() { + Log::Warning(Event::Database, "Removing existing incompatible offline database"); + + db.reset(); + + try { + util::deleteFile(path); + } catch (util::IOException& ex) { + Log::Error(Event::Database, ex.code, ex.what()); + } +} + +mapbox::sqlite::Statement& OfflineDatabase::getStatement(const char * sql) { + auto it = statements.find(sql); + + if (it != statements.end()) { + it->second->reset(); + return *it->second; + } + + return *(statements[sql] = std::make_unique<Statement>(db->prepare(sql))); +} + +optional<Response> OfflineDatabase::get(const Resource& resource) { + if (resource.kind == Resource::Kind::Tile) { + assert(resource.tileData); + return getTile(*resource.tileData); + } else { + return getResource(resource); + } +} + +void OfflineDatabase::put(const Resource& resource, const Response& response) { + // Except for 404s, don't store errors in the cache. + if (response.error && response.error->reason != Response::Error::Reason::NotFound) { + return; + } + + if (resource.kind == Resource::Kind::Tile) { + assert(resource.tileData); + putTile(*resource.tileData, response); + } else { + putResource(resource, response); + } +} + +optional<Response> OfflineDatabase::getResource(const Resource& resource) { + mapbox::sqlite::Statement& stmt = getStatement( + // 0 1 2 3 4 + "SELECT etag, expires, modified, data, compressed " + "FROM resources " + "WHERE url = ?"); + + stmt.bind(1, resource.url); + + if (!stmt.run()) { + return {}; + } + + Response response; + + response.etag = stmt.get<optional<std::string>>(0); + response.expires = stmt.get<optional<SystemTimePoint>>(1); + response.modified = stmt.get<optional<SystemTimePoint>>(2); + + optional<std::string> data = stmt.get<optional<std::string>>(3); + if (!data) { + response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound); + } else if (stmt.get<int>(4)) { + response.data = std::make_shared<std::string>(util::decompress(*data)); + } else { + response.data = std::make_shared<std::string>(*data); + } + + return response; +} + +void OfflineDatabase::putResource(const Resource& resource, const Response& response) { + if (response.notModified) { + mapbox::sqlite::Statement& stmt = getStatement( + // 1 2 3 + "UPDATE resources SET accessed = ?, expires = ? WHERE url = ?"); + + stmt.bind(1, SystemClock::now()); + stmt.bind(2, response.expires); + stmt.bind(3, resource.url); + stmt.run(); + } else { + mapbox::sqlite::Statement& stmt = getStatement( + // 1 2 3 4 5 6 7 8 + "REPLACE INTO resources (url, kind, etag, expires, modified, accessed, data, compressed) " + "VALUES (?, ?, ?, ?, ?, ?, ?, ?)"); + + stmt.bind(1 /* url */, resource.url); + stmt.bind(2 /* kind */, int(resource.kind)); + stmt.bind(3 /* etag */, response.etag); + stmt.bind(4 /* expires */, response.expires); + stmt.bind(5 /* modified */, response.modified); + stmt.bind(6 /* accessed */, SystemClock::now()); + + std::string data; + + if (response.error) { // Can only be NotFound + stmt.bind(7 /* data */, nullptr); + stmt.bind(8 /* compressed */, false); + } else { + data = util::compress(*response.data); + if (data.size() < response.data->size()) { + stmt.bind(7 /* data */, data, false); // do not retain the string internally. + stmt.bind(8 /* compressed */, true); + } else { + stmt.bind(7 /* data */, *response.data, false); // do not retain the string internally. + stmt.bind(8 /* compressed */, false); + } + } + + stmt.run(); + } +} + +optional<Response> OfflineDatabase::getTile(const Resource::TileData& tile) { + mapbox::sqlite::Statement& stmt = getStatement( + // 0 1 2 3 4 + "SELECT etag, expires, modified, data, compressed " + "FROM tilesets, tiles " + "WHERE tilesets.url_template = ? " // 1 + "AND tilesets.pixel_ratio = ? " // 2 + "AND tiles.x = ? " // 3 + "AND tiles.y = ? " // 4 + "AND tiles.z = ? " // 5 + "AND tilesets.id = tiles.tileset_id "); + + stmt.bind(1, tile.urlTemplate); + stmt.bind(2, tile.pixelRatio); + stmt.bind(3, tile.x); + stmt.bind(4, tile.y); + stmt.bind(5, tile.z); + + if (!stmt.run()) { + return {}; + } + + Response response; + + response.etag = stmt.get<optional<std::string>>(0); + response.expires = stmt.get<optional<SystemTimePoint>>(1); + response.modified = stmt.get<optional<SystemTimePoint>>(2); + + optional<std::string> data = stmt.get<optional<std::string>>(3); + if (!data) { + response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound); + } else if (stmt.get<int>(4)) { + response.data = std::make_shared<std::string>(util::decompress(*data)); + } else { + response.data = std::make_shared<std::string>(*data); + } + + return response; +} + +void OfflineDatabase::putTile(const Resource::TileData& tile, const Response& response) { + if (response.notModified) { + mapbox::sqlite::Statement& stmt = getStatement( + "UPDATE tiles SET accessed = ?1, expires = ?2 " + "WHERE tileset_id = ( " + " SELECT id FROM tilesets " + " WHERE url_template = ?3 " + " AND pixel_ratio = ?4) " + "AND tiles.x = ?5 " + "AND tiles.y = ?6 " + "AND tiles.z = ?7 "); + + stmt.bind(1, SystemClock::now()); + stmt.bind(2, response.expires); + stmt.bind(3, tile.urlTemplate); + stmt.bind(4, tile.pixelRatio); + stmt.bind(5, tile.x); + stmt.bind(6, tile.y); + stmt.bind(7, tile.z); + stmt.run(); + } else { + mapbox::sqlite::Statement& stmt1 = getStatement( + "REPLACE INTO tilesets (url_template, pixel_ratio) " + "VALUES (?1, ?2) "); + + stmt1.bind(1 /* url_template */, tile.urlTemplate); + stmt1.bind(2 /* pixel_ratio */, tile.pixelRatio); + stmt1.run(); + + mapbox::sqlite::Statement& stmt2 = getStatement( + "REPLACE INTO tiles (tileset_id, x, y, z, modified, etag, expires, accessed, data, compressed) " + "SELECT tilesets.id, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11 " + "FROM tilesets " + "WHERE url_template = ?1 " + "AND pixel_ratio = ?2 "); + + stmt2.bind(1 /* url_template */, tile.urlTemplate); + stmt2.bind(2 /* pixel_ratio */, tile.pixelRatio); + stmt2.bind(3 /* x */, tile.x); + stmt2.bind(4 /* y */, tile.y); + stmt2.bind(5 /* z */, tile.z); + stmt2.bind(6 /* modified */, response.modified); + stmt2.bind(7 /* etag */, response.etag); + stmt2.bind(8 /* expires */, response.expires); + stmt2.bind(9 /* accessed */, SystemClock::now()); + + std::string data; + + if (response.error) { // Can only be NotFound + stmt2.bind(10 /* data */, nullptr); + stmt2.bind(11 /* compressed */, false); + } else { + data = util::compress(*response.data); + if (data.size() < response.data->size()) { + stmt2.bind(10 /* data */, data, false); // do not retain the string internally. + stmt2.bind(11 /* compressed */, true); + } else { + stmt2.bind(10 /* data */, *response.data, false); // do not retain the string internally. + stmt2.bind(11 /* compressed */, false); + } + } + + stmt2.run(); + } +} + +} // namespace mbgl diff --git a/platform/default/mbgl/storage/offline_database.hpp b/platform/default/mbgl/storage/offline_database.hpp new file mode 100644 index 0000000000..bc6f784d50 --- /dev/null +++ b/platform/default/mbgl/storage/offline_database.hpp @@ -0,0 +1,50 @@ +#ifndef MBGL_OFFLINE_DATABASE +#define MBGL_OFFLINE_DATABASE + +#include <mbgl/storage/resource.hpp> +#include <mbgl/util/noncopyable.hpp> +#include <mbgl/util/optional.hpp> + +#include <unordered_map> +#include <memory> +#include <string> + +namespace mapbox { +namespace sqlite { +class Database; +class Statement; +} +} + +namespace mbgl { + +class Response; +class TileID; + +class OfflineDatabase : private util::noncopyable { +public: + OfflineDatabase(const std::string& path); + ~OfflineDatabase(); + + optional<Response> get(const Resource&); + void put(const Resource&, const Response&); + +private: + void ensureSchema(); + void removeExisting(); + mapbox::sqlite::Statement& getStatement(const char *); + + optional<Response> getTile(const Resource::TileData&); + void putTile(const Resource::TileData&, const Response&); + + optional<Response> getResource(const Resource&); + void putResource(const Resource&, const Response&); + + const std::string path; + std::unique_ptr<::mapbox::sqlite::Database> db; + std::unordered_map<const char *, std::unique_ptr<::mapbox::sqlite::Statement>> statements; +}; + +} // namespace mbgl + +#endif diff --git a/platform/default/mbgl/storage/offline_schema.cpp.include b/platform/default/mbgl/storage/offline_schema.cpp.include new file mode 100644 index 0000000000..31b11d5f12 --- /dev/null +++ b/platform/default/mbgl/storage/offline_schema.cpp.include @@ -0,0 +1,51 @@ +/* THIS IS A GENERATED FILE; EDIT offline_schema.sql INSTEAD */ +static const char * schema = +"CREATE TABLE resources (\n" +" url TEXT NOT NULL PRIMARY KEY,\n" +" kind INTEGER NOT NULL,\n" +" expires INTEGER,\n" +" modified INTEGER,\n" +" accessed INTEGER,\n" +" etag TEXT,\n" +" data BLOB,\n" +" compressed INTEGER NOT NULL DEFAULT 0\n" +");\n" +"CREATE TABLE tilesets (\n" +" id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,\n" +" url_template TEXT NOT NULL,\n" +" pixel_ratio INTEGER,\n" +" UNIQUE (url_template, pixel_ratio)\n" +");\n" +"CREATE TABLE tiles (\n" +" tileset_id INTEGER NOT NULL REFERENCES tilesets(id),\n" +" z INTEGER NOT NULL,\n" +" x INTEGER NOT NULL,\n" +" y INTEGER NOT NULL,\n" +" expires INTEGER,\n" +" modified INTEGER,\n" +" accessed INTEGER,\n" +" etag TEXT,\n" +" data BLOB,\n" +" compressed INTEGER NOT NULL DEFAULT 0,\n" +" PRIMARY KEY (tileset_id, z, x, y)\n" +");\n" +"CREATE TABLE regions (\n" +" id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,\n" +" definition TEXT NOT NULL,\n" +" description BLOB\n" +");\n" +"CREATE TABLE region_resources (\n" +" region_id INTEGER NOT NULL REFERENCES regions(id),\n" +" resource_url TEXT NOT NULL REFERENCES resources(url),\n" +" PRIMARY KEY (region_id, resource_url)\n" +");\n" +"CREATE TABLE region_tiles (\n" +" region_id INTEGER NOT NULL REFERENCES regions(id),\n" +" tileset_id INTEGER NOT NULL REFERENCES tilesets(id),\n" +" z INTEGER NOT NULL,\n" +" x INTEGER NOT NULL,\n" +" y INTEGER NOT NULL,\n" +" PRIMARY KEY (region_id, tileset_id, z, x, y),\n" +" FOREIGN KEY (tileset_id, z, x, y) REFERENCES tiles (tileset_id, z, x, y)\n" +");\n" +; diff --git a/platform/default/mbgl/storage/offline_schema.js b/platform/default/mbgl/storage/offline_schema.js new file mode 100644 index 0000000000..153ba34e38 --- /dev/null +++ b/platform/default/mbgl/storage/offline_schema.js @@ -0,0 +1,24 @@ +// To regenerate: +// (cd platform/default/mbgl/storage && node offline_schema.js) + +var fs = require('fs'); +var readline = require('readline'); + +var lineReader = readline.createInterface({ + input: fs.createReadStream('offline_schema.sql') +}); + +var lines = [ + "/* THIS IS A GENERATED FILE; EDIT offline_schema.sql INSTEAD */", + "static const char * schema = ", +]; + +lineReader + .on('line', function (line) { + line = line.replace(/ *--.*/, ''); + if (line) lines.push('"' + line + '\\n"'); + }) + .on('close', function () { + lines.push(';\n'); + fs.writeFileSync('offline_schema.cpp.include', lines.join('\n')); + }); diff --git a/platform/default/mbgl/storage/offline_schema.sql b/platform/default/mbgl/storage/offline_schema.sql new file mode 100644 index 0000000000..e55517c489 --- /dev/null +++ b/platform/default/mbgl/storage/offline_schema.sql @@ -0,0 +1,62 @@ +CREATE TABLE resources ( -- Generic table for style, source, sprite, and glyph resources. + url TEXT NOT NULL PRIMARY KEY, -- Same schema as http_cache table. + kind INTEGER NOT NULL, + expires INTEGER, + modified INTEGER, + accessed INTEGER, + etag TEXT, + data BLOB, -- NULL if the response was a 404 + compressed INTEGER NOT NULL DEFAULT 0 +); + +CREATE TABLE tilesets ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + url_template TEXT NOT NULL, -- As it would appear in TileJSON (but no support for host sharding). + pixel_ratio INTEGER, -- If NULL, 1 is assumed for raster sources. + UNIQUE (url_template, pixel_ratio) -- Capable of caching the same tileset at multiple resolutions. +); + +CREATE TABLE tiles ( + tileset_id INTEGER NOT NULL REFERENCES tilesets(id), + z INTEGER NOT NULL, -- Fully abandon TMS coordinates in favor of ZXY. + x INTEGER NOT NULL, + y INTEGER NOT NULL, + expires INTEGER, + modified INTEGER, + accessed INTEGER, + etag TEXT, + data BLOB, -- NULL if the response was a 404 + compressed INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (tileset_id, z, x, y) +); + +CREATE TABLE regions ( + id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT, + definition TEXT NOT NULL, -- JSON formatted definition of region. Regions may be of variant types: + -- e.g. bbox and zoom range, route path, flyTo parameters, etc. Note that + -- the set of tiles required for a region may span multiple sources. + description BLOB -- User provided data in user-defined format +); + +CREATE TABLE region_resources ( + region_id INTEGER NOT NULL REFERENCES regions(id), + resource_url TEXT NOT NULL REFERENCES resources(url), + PRIMARY KEY (region_id, resource_url) +); + +CREATE TABLE region_tiles ( + region_id INTEGER NOT NULL REFERENCES regions(id), + tileset_id INTEGER NOT NULL REFERENCES tilesets(id), + z INTEGER NOT NULL, + x INTEGER NOT NULL, + y INTEGER NOT NULL, + PRIMARY KEY (region_id, tileset_id, z, x, y), + FOREIGN KEY (tileset_id, z, x, y) REFERENCES tiles (tileset_id, z, x, y) +); + +-- `region_resources` and `region_tiles` are used for deduplication and deletion logic. +-- A row in `tiles` exists IFF one or more corresponding rows exist in `region_tiles`. If +-- more than one corresponding row exists, it indicates multiple regions contain the tile, and +-- storage for the tile is being deduplicated. When a region is deleted, corresponding rows in +-- `region_tiles` must also be deleted, and then rows in `tiles` and `tilesets` without a +-- corresponding `region_tiles` row must be deleted. Similarly for `resources` / `region_resources`. diff --git a/platform/default/sqlite3.cpp b/platform/default/sqlite3.cpp index 5122e01015..09301bc4d9 100644 --- a/platform/default/sqlite3.cpp +++ b/platform/default/sqlite3.cpp @@ -115,9 +115,19 @@ template <> void Statement::bind(int offset, std::nullptr_t) { check(sqlite3_bind_null(stmt, offset)); } -template <> void Statement::bind(int offset, int value) { +template <> void Statement::bind(int offset, int8_t value) { assert(stmt); - check(sqlite3_bind_int(stmt, offset, value)); + check(sqlite3_bind_int64(stmt, offset, value)); +} + +template <> void Statement::bind(int offset, int16_t value) { + assert(stmt); + check(sqlite3_bind_int64(stmt, offset, value)); +} + +template <> void Statement::bind(int offset, int32_t value) { + assert(stmt); + check(sqlite3_bind_int64(stmt, offset, value)); } template <> void Statement::bind(int offset, int64_t value) { @@ -125,6 +135,26 @@ template <> void Statement::bind(int offset, int64_t value) { check(sqlite3_bind_int64(stmt, offset, value)); } +template <> void Statement::bind(int offset, uint8_t value) { + assert(stmt); + check(sqlite3_bind_int64(stmt, offset, value)); +} + +template <> void Statement::bind(int offset, uint16_t value) { + assert(stmt); + check(sqlite3_bind_int64(stmt, offset, value)); +} + +template <> void Statement::bind(int offset, uint32_t value) { + assert(stmt); + check(sqlite3_bind_int64(stmt, offset, value)); +} + +template <> void Statement::bind(int offset, float value) { + assert(stmt); + check(sqlite3_bind_double(stmt, offset, value)); +} + template <> void Statement::bind(int offset, double value) { assert(stmt); check(sqlite3_bind_double(stmt, offset, value)); diff --git a/platform/default/sqlite_cache.cpp b/platform/default/sqlite_cache.cpp deleted file mode 100644 index b3f528ac92..0000000000 --- a/platform/default/sqlite_cache.cpp +++ /dev/null @@ -1,502 +0,0 @@ -#include "sqlite_cache_impl.hpp" -#include <mbgl/storage/resource.hpp> -#include <mbgl/storage/response.hpp> - -#include <mbgl/util/compression.hpp> -#include <mbgl/util/io.hpp> -#include <mbgl/util/string.hpp> -#include <mbgl/util/thread.hpp> -#include <mbgl/util/mapbox.hpp> -#include <mbgl/platform/log.hpp> - -#include "sqlite3.hpp" -#include <sqlite3.h> - -#include <unordered_map> -#include <mutex> - -namespace { - -// The cache won't accept entries larger than this arbitrary size -// and will silently discard request for adding them to the cache. -// Large entries can cause the database to grow in disk size and -// never shrink again. -const uint64_t kMaximumCacheEntrySize = 5 * 1024 * 1024; // 5 MB - -// Number of records we delete when we are close to the maximum -// database size, if set. The current criteria is to prune -// the least used entries based on `accessed` time. -const int kPrunedEntriesLimit = 100; - -} // namespace - -namespace mbgl { - -using namespace mapbox::sqlite; - -SQLiteCache::SQLiteCache(const std::string& path_) - : thread(std::make_unique<util::Thread<Impl>>(util::ThreadContext{"SQLiteCache", util::ThreadType::Unknown, util::ThreadPriority::Low}, path_)) { -} - -SQLiteCache::~SQLiteCache() = default; - -SQLiteCache::Impl::Impl(const std::string& path_) - : maximumCacheSize(0), // Unlimited - maximumCacheEntrySize(kMaximumCacheEntrySize), - path(path_) { -} - -SQLiteCache::Impl::~Impl() { - // Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we - // can't throw anything. - try { - getStmt.reset(); - putStmt.reset(); - refreshStmt.reset(); - countStmt.reset(); - freeStmt.reset(); - pruneStmt.reset(); - accessedStmt.reset(); - db.reset(); - } catch (mapbox::sqlite::Exception& ex) { - Log::Error(Event::Database, ex.code, ex.what()); - } -} - -void SQLiteCache::Impl::createDatabase() { - db = std::make_unique<Database>(path.c_str(), ReadWrite | Create); -} - -int SQLiteCache::Impl::schemaVersion() const { - // WARNING: Bump the version when changing the cache - // scheme to force the table to be recreated. - return 1; -} - -void SQLiteCache::Impl::createSchema() { - constexpr const char *const sql = "" - "CREATE TABLE IF NOT EXISTS `http_cache` (" - " `url` TEXT PRIMARY KEY NOT NULL," - " `status` INTEGER NOT NULL," // The response status (Successful or Error). - " `kind` INTEGER NOT NULL," // The kind of file. - " `modified` INTEGER," // Timestamp when the file was last modified. - " `etag` TEXT," - " `expires` INTEGER," // Timestamp when the server says the file expires. - " `accessed` INTEGER," // Timestamp when the database record was last accessed. - " `data` BLOB," - " `compressed` INTEGER NOT NULL DEFAULT 0" // Whether the data is compressed. - ");" - "CREATE INDEX IF NOT EXISTS `http_cache_kind_idx` ON `http_cache` (`kind`);" - "CREATE INDEX IF NOT EXISTS `http_cache_accessed_idx` ON `http_cache` (`accessed`);"; - - ensureSchemaVersion(); - - try { - db->exec(sql); - db->exec("PRAGMA user_version = " + util::toString(schemaVersion())); - schema = true; - } catch (mapbox::sqlite::Exception &ex) { - if (ex.code == SQLITE_NOTADB) { - Log::Warning(Event::Database, "Trashing invalid database"); - db.reset(); - try { - util::deleteFile(path); - } catch (util::IOException& ioEx) { - Log::Error(Event::Database, ex.code, ex.what()); - } - db = std::make_unique<Database>(path.c_str(), ReadWrite | Create); - } else { - Log::Error(Event::Database, ex.code, ex.what()); - } - - // Creating the database table + index failed. That means there may already be one, likely - // with different columns. Drop it and try to create a new one. - db->exec("DROP TABLE IF EXISTS `http_cache`"); - db->exec(sql); - db->exec("PRAGMA user_version = " + util::toString(schemaVersion())); - } -} - -void SQLiteCache::Impl::ensureSchemaVersion() { - try { - Statement userVersionStmt(db->prepare("PRAGMA user_version")); - if (userVersionStmt.run() && userVersionStmt.get<int>(0) == schemaVersion()) { - return; - } - } catch (mapbox::sqlite::Exception& ex) { - if (ex.code == SQLITE_NOTADB) { - return; - } - - Log::Error(Event::Database, ex.code, ex.what()); - } - - // Version mismatch, drop the table so it will - // get recreated by `createSchema()`. - try { - db->exec("DROP TABLE IF EXISTS `http_cache`"); - } catch (mapbox::sqlite::Exception& ex) { - Log::Error(Event::Database, ex.code, ex.what()); - } -} - -void SQLiteCache::setMaximumCacheSize(uint64_t size) { - thread->invoke(&Impl::setMaximumCacheSize, size); -} - -void SQLiteCache::Impl::setMaximumCacheSize(uint64_t size) { - maximumCacheSize = size; - - // Unlimited. - if (size == 0) { - return; - } - - uint64_t lastSoftSize = cacheSoftSize(); - - // Keep pruning until we fit in the new - // size limit. - while (lastSoftSize > maximumCacheSize) { - pruneEntries(); - - if (lastSoftSize != cacheSoftSize()) { - lastSoftSize = cacheSoftSize(); - } else { - break; - } - } - - if (cacheHardSize() > size) { - Log::Warning(mbgl::Event::Database, - "Current cache hard size is bigger than the defined " - "maximum size. Database won't get truncated."); - } -} - -void SQLiteCache::setMaximumCacheEntrySize(uint64_t size) { - thread->invoke(&Impl::setMaximumCacheEntrySize, size); -} - -void SQLiteCache::Impl::setMaximumCacheEntrySize(uint64_t size) { - maximumCacheEntrySize = size; -} - -void SQLiteCache::Impl::initializeDatabase() { - if (!db) { - createDatabase(); - } - - if (!schema) { - createSchema(); - } -} - -int SQLiteCache::Impl::cachePageSize() { - try { - if (!pageSize) { - Statement pageSizeStmt(db->prepare("PRAGMA page_size")); - if (pageSizeStmt.run()) { - pageSize = pageSizeStmt.get<int>(0); - } - } - } catch (mapbox::sqlite::Exception& ex) { - Log::Error(Event::Database, ex.code, ex.what()); - } - - return pageSize; -} - -uint64_t SQLiteCache::Impl::cacheHardSize() { - try { - initializeDatabase(); - - if (!countStmt) { - countStmt = std::make_unique<Statement>(db->prepare("PRAGMA page_count")); - } else { - countStmt->reset(); - } - - if (countStmt->run()) { - return cachePageSize() * countStmt->get<int>(0); - } - } catch (mapbox::sqlite::Exception& ex) { - Log::Error(Event::Database, ex.code, ex.what()); - } - - return 0; -} - -uint64_t SQLiteCache::Impl::cacheSoftSize() { - if (!softSizeDirty) { - return softSize; - } - - try { - initializeDatabase(); - - if (!freeStmt) { - freeStmt = std::make_unique<Statement>(db->prepare("PRAGMA freelist_count")); - } else { - freeStmt->reset(); - } - - uint64_t hardSize = cacheHardSize(); - if (!hardSize) { - return 0; - } - - if (freeStmt->run()) { - return hardSize - cachePageSize() * freeStmt->get<int>(0); - } - - softSizeDirty = false; - } catch (mapbox::sqlite::Exception& ex) { - Log::Error(Event::Database, ex.code, ex.what()); - } - - return 0; -} - -bool SQLiteCache::Impl::needsPruning() { - // SQLite database never shrinks in size unless we call VACCUM. We here - // are monitoring the soft limit (i.e. number of free pages in the file) - // and as it approaches to the hard limit (i.e. the actual file size) we - // delete an arbitrary number of old cache entries. - // - // The free pages approach saves us from calling VACCUM or keeping a - // running total, which can be costly. We need a buffer because pages can - // get fragmented on the database. - if (cacheSoftSize() + maximumCacheEntrySize * 2 < maximumCacheSize) { - return false; - } else { - return true; - } -} - -void SQLiteCache::Impl::pruneEntries() { - if (!maximumCacheSize) { - return; - } - - if (!needsPruning()) { - return; - } - - try { - if (!pruneStmt) { - pruneStmt = std::make_unique<Statement>(db->prepare( - "DELETE FROM `http_cache` WHERE `rowid` IN (SELECT `rowid` FROM " - // 1 - "`http_cache` ORDER BY `accessed` ASC LIMIT ?)")); - } else { - pruneStmt->reset(); - } - - pruneStmt->bind(1, kPrunedEntriesLimit); - - pruneStmt->run(); - softSizeDirty = true; - } catch (mapbox::sqlite::Exception& ex) { - Log::Error(Event::Database, ex.code, ex.what()); - } -} - -std::unique_ptr<WorkRequest> SQLiteCache::get(const Resource &resource, Callback callback) { - // Can be called from any thread, but most likely from the file source thread. - // Will try to load the URL from the SQLite database and call the callback when done. - // Note that the callback is probably going to invoked from another thread, so the caller - // must make sure that it can run in that thread. - return thread->invokeWithCallback(&Impl::get, callback, resource); -} - -void SQLiteCache::Impl::get(const Resource &resource, Callback callback) { - try { - initializeDatabase(); - - if (!getStmt) { - // Initialize the statement 0 1 - getStmt = std::make_unique<Statement>(db->prepare("SELECT `status`, `modified`, " - // 2 3 4 5 1 - "`etag`, `expires`, `data`, `compressed` FROM `http_cache` WHERE `url` = ?")); - } else { - getStmt->reset(); - } - - getStmt->bind(1, resource.url); - if (getStmt->run()) { - // There is data. - auto response = std::make_unique<Response>(); - const auto status = getStmt->get<int>(0); - if (status > 1) { - // Status codes > 1 indicate an error - response->error = std::make_unique<Response::Error>(Response::Error::Reason(status)); - } - response->modified = getStmt->get<optional<SystemTimePoint>>(1); - response->etag = getStmt->get<optional<std::string>>(2); - response->expires = getStmt->get<optional<SystemTimePoint>>(3); - response->data = std::make_shared<std::string>(getStmt->get<std::string>(4)); - if (getStmt->get<int>(5)) { // == compressed - response->data = std::make_shared<std::string>(util::decompress(*response->data)); - } - callback(std::move(response)); - } else { - // There is no data. - callback(nullptr); - } - - // We do an extra query for refreshing the last time - // the record was accessed that can be costly and is only - // worth doing if we are monitoring the database size. - if (maximumCacheSize) { - if (!accessedStmt) { - accessedStmt = std::make_unique<Statement>( - // 1 2 - db->prepare("UPDATE `http_cache` SET `accessed` = ? WHERE `url` = ?")); - } else { - accessedStmt->reset(); - } - - accessedStmt->bind(1, SystemClock::now()); - accessedStmt->bind(2, resource.url); - accessedStmt->run(); - } - } catch (mapbox::sqlite::Exception& ex) { - Log::Error(Event::Database, ex.code, ex.what()); - callback(nullptr); - } catch (std::runtime_error& ex) { - Log::Error(Event::Database, "%s", ex.what()); - callback(nullptr); - } -} - -void SQLiteCache::put(const Resource& resource, const Response& response) { - // Except for 404s, don't store errors in the cache. - if (response.error && response.error->reason != Response::Error::Reason::NotFound) { - return; - } - - if (response.notModified) { - thread->invoke(&Impl::refresh, resource, response.expires); - } else { - thread->invoke(&Impl::put, resource, response); - } -} - -void SQLiteCache::Impl::put(const Resource& resource, const Response& response) { - try { - initializeDatabase(); - pruneEntries(); - - if (response.data) { - auto entrySize = response.data->size(); - - if (entrySize > maximumCacheEntrySize) { - Log::Warning(Event::Database, "Entry too big for caching."); - return; - } - - if (maximumCacheSize && entrySize + cacheSoftSize() > maximumCacheSize) { - Log::Warning(Event::Database, "Unable to make space for new entries."); - return; - } - } - - if (!putStmt) { - putStmt = std::make_unique<Statement>(db->prepare("REPLACE INTO `http_cache` (" - // 1 2 3 4 5 6 7 8 9 - "`url`, `status`, `kind`, `modified`, `etag`, `expires`, `accessed`, `data`, `compressed`" - ") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)")); - } else { - putStmt->reset(); - } - - putStmt->bind(1 /* url */, resource.url); - if (response.error) { - putStmt->bind(2 /* status */, int(response.error->reason)); - } else { - putStmt->bind(2 /* status */, 1 /* success */); - } - putStmt->bind(3 /* kind */, int(resource.kind)); - putStmt->bind(4 /* modified */, response.modified); - putStmt->bind(5 /* etag */, response.etag); - putStmt->bind(6 /* expires */, response.expires); - putStmt->bind(7 /* accessed */, SystemClock::now()); - - std::string data; - if (resource.kind != Resource::SpriteImage && response.data) { - // Do not compress images, since they are typically compressed already. - data = util::compress(*response.data); - } - - if (!data.empty() && data.size() < response.data->size()) { - // Store the compressed data when it is smaller than the original - // uncompressed data. - putStmt->bind(8 /* data */, data, false); // do not retain the string internally. - putStmt->bind(9 /* compressed */, true); - } else if (response.data) { - putStmt->bind(8 /* data */, *response.data, false); // do not retain the string internally. - putStmt->bind(9 /* compressed */, false); - } else { - putStmt->bind(8 /* data */, "", false); - putStmt->bind(9 /* compressed */, false); - } - - putStmt->run(); - softSizeDirty = true; - } catch (mapbox::sqlite::Exception& ex) { - Log::Error(Event::Database, ex.code, ex.what()); - } catch (std::runtime_error& ex) { - Log::Error(Event::Database, "%s", ex.what()); - } -} - -void SQLiteCache::Impl::refresh(const Resource& resource, optional<SystemTimePoint> expires) { - try { - initializeDatabase(); - - if (!refreshStmt) { - refreshStmt = std::make_unique<Statement>( - db->prepare("UPDATE `http_cache` SET " - // 1 2 3 - "`accessed` = ?, `expires` = ? WHERE `url` = ?")); - } else { - refreshStmt->reset(); - } - - refreshStmt->bind(1, SystemClock::now()); - refreshStmt->bind(2, expires); - refreshStmt->bind(3, resource.url); - refreshStmt->run(); - } catch (mapbox::sqlite::Exception& ex) { - Log::Error(Event::Database, ex.code, ex.what()); - } -} - -namespace { - -static std::mutex sharedMutex; -static std::unordered_map<std::string, std::weak_ptr<SQLiteCache>> shared; - -} // namespace - -std::shared_ptr<SQLiteCache> SQLiteCache::getShared(const std::string &path) { - std::lock_guard<std::mutex> lock(sharedMutex); - - std::shared_ptr<SQLiteCache> cache; - - auto it = shared.find(path); - if (it != shared.end()) { - cache = it->second.lock(); - if (!cache) { - cache = std::make_shared<SQLiteCache>(path); - it->second = cache; - } - } else { - cache = std::make_shared<SQLiteCache>(path); - shared.emplace(path, cache); - } - - return cache; -} - -} // namespace mbgl diff --git a/platform/default/sqlite_cache_impl.hpp b/platform/default/sqlite_cache_impl.hpp deleted file mode 100644 index e156532402..0000000000 --- a/platform/default/sqlite_cache_impl.hpp +++ /dev/null @@ -1,68 +0,0 @@ -#ifndef MBGL_STORAGE_DEFAULT_SQLITE_CACHE_IMPL -#define MBGL_STORAGE_DEFAULT_SQLITE_CACHE_IMPL - -#include <mbgl/storage/sqlite_cache.hpp> -#include <mbgl/util/chrono.hpp> -#include <mbgl/util/optional.hpp> - -namespace mapbox { -namespace sqlite { -class Database; -class Statement; -} -} - -namespace mbgl { - -class SQLiteCache::Impl { -public: - explicit Impl(const std::string &path = ":memory:"); - ~Impl(); - - void setMaximumCacheSize(uint64_t size); - void setMaximumCacheEntrySize(uint64_t size); - - void get(const Resource&, Callback); - void put(const Resource&, const Response&); - void refresh(const Resource&, optional<SystemTimePoint> expires); - -private: - void initializeDatabase(); - - int cachePageSize(); - - uint64_t cacheHardSize(); - uint64_t cacheSoftSize(); - - uint64_t softSize = 0; - bool softSizeDirty = true; - - bool needsPruning(); - void pruneEntries(); - - void createDatabase(); - void createSchema(); - - int schemaVersion() const; - void ensureSchemaVersion(); - - int pageSize = 0; - - uint64_t maximumCacheSize; - uint64_t maximumCacheEntrySize; - - const std::string path; - std::unique_ptr<::mapbox::sqlite::Database> db; - std::unique_ptr<::mapbox::sqlite::Statement> getStmt; - std::unique_ptr<::mapbox::sqlite::Statement> putStmt; - std::unique_ptr<::mapbox::sqlite::Statement> refreshStmt; - std::unique_ptr<::mapbox::sqlite::Statement> countStmt; - std::unique_ptr<::mapbox::sqlite::Statement> freeStmt; - std::unique_ptr<::mapbox::sqlite::Statement> pruneStmt; - std::unique_ptr<::mapbox::sqlite::Statement> accessedStmt; - bool schema = false; -}; - -} // namespace mbgl - -#endif diff --git a/src/mbgl/storage/sqlite_cache.hpp b/src/mbgl/storage/sqlite_cache.hpp deleted file mode 100644 index b5a7cbcc07..0000000000 --- a/src/mbgl/storage/sqlite_cache.hpp +++ /dev/null @@ -1,44 +0,0 @@ -#ifndef MBGL_STORAGE_DEFAULT_SQLITE_CACHE -#define MBGL_STORAGE_DEFAULT_SQLITE_CACHE - -#include <mbgl/util/noncopyable.hpp> -#include <mbgl/util/chrono.hpp> - -#include <functional> -#include <memory> -#include <string> - -namespace mbgl { - -class Resource; -class Response; -class WorkRequest; - -namespace util { -template <typename T> class Thread; -} // namespace util - -class SQLiteCache : private util::noncopyable { -public: - static std::shared_ptr<SQLiteCache> getShared(const std::string &path = ":memory:"); - - SQLiteCache(const std::string &path = ":memory:"); - ~SQLiteCache(); - - void setMaximumCacheSize(uint64_t size); - void setMaximumCacheEntrySize(uint64_t size); - - using Callback = std::function<void(std::unique_ptr<Response>)>; - - std::unique_ptr<WorkRequest> get(const Resource&, Callback); - void put(const Resource&, const Response&); - - class Impl; - -private: - const std::unique_ptr<util::Thread<Impl>> thread; -}; - -} // namespace mbgl - -#endif diff --git a/test/api/offline.cpp b/test/api/offline.cpp new file mode 100644 index 0000000000..f168ff71f5 --- /dev/null +++ b/test/api/offline.cpp @@ -0,0 +1,50 @@ +#include "../fixtures/util.hpp" + +#include <mbgl/platform/default/headless_display.hpp> +#include <mbgl/platform/default/headless_view.hpp> +#include <mbgl/storage/offline_database.hpp> +#include <mbgl/storage/default_file_source.hpp> + +#include <mbgl/platform/log.hpp> +#include <mbgl/util/work_request.hpp> +#include <mbgl/util/io.hpp> + +using namespace mbgl; +using namespace std::literals::chrono_literals; +using namespace std::literals::string_literals; + +namespace { + +Response expiredItem(const std::string& path) { + Response response; + response.data = std::make_shared<std::string>(util::read_file("test/fixtures/"s + path)); + response.expires = SystemClock::from_time_t(0); + return response; +} + +const std::string prefix = "http://127.0.0.1:3000"; + +} + +auto display = std::make_shared<mbgl::HeadlessDisplay>(); + +TEST(API, Offline) { + HeadlessView view(display, 1); + DefaultFileSource fileSource(":memory:", "."); + + fileSource.put(Resource::style(prefix + "/offline/style.json"), expiredItem("offline/style.json")); + fileSource.put(Resource::source(prefix + "/offline/streets.json"), expiredItem("offline/streets.json")); + fileSource.put(Resource::spriteJSON(prefix + "/offline/sprite", 1.0), expiredItem("offline/sprite.json")); + fileSource.put(Resource::spriteImage(prefix + "/offline/sprite", 1.0), expiredItem("offline/sprite.png")); + fileSource.put(Resource::tile(prefix + "/offline/{z}-{x}-{y}.vector.pbf", 1.0, 0, 0, 0), expiredItem("offline/0-0-0.vector.pbf")); + fileSource.put(Resource::glyphs(prefix + "/offline/{fontstack}/{range}.pbf", "Helvetica", {0, 255}), expiredItem("offline/glyph.pbf")); + fileSource.goOffline(); + + Map map(view, fileSource, MapMode::Still); + map.setStyleURL(prefix + "/offline/style.json"); + + test::checkImage("test/fixtures/offline"s, + test::render(map), + 0.0015, + 0.1); +} diff --git a/test/fixtures/stale/0-0-0.vector.pbf b/test/fixtures/offline/0-0-0.vector.pbf Binary files differindex a0f049ad43..a0f049ad43 100644 --- a/test/fixtures/stale/0-0-0.vector.pbf +++ b/test/fixtures/offline/0-0-0.vector.pbf diff --git a/test/fixtures/offline/expected.png b/test/fixtures/offline/expected.png Binary files differnew file mode 100644 index 0000000000..1b1c2be4c6 --- /dev/null +++ b/test/fixtures/offline/expected.png diff --git a/test/fixtures/stale/glyph.pbf b/test/fixtures/offline/glyph.pbf Binary files differindex 0d160f7898..0d160f7898 100644 --- a/test/fixtures/stale/glyph.pbf +++ b/test/fixtures/offline/glyph.pbf diff --git a/test/fixtures/stale/sprite.json b/test/fixtures/offline/sprite.json index e640365519..e640365519 100644 --- a/test/fixtures/stale/sprite.json +++ b/test/fixtures/offline/sprite.json diff --git a/test/fixtures/stale/sprite.png b/test/fixtures/offline/sprite.png Binary files differindex a02d8eb542..a02d8eb542 100644 --- a/test/fixtures/stale/sprite.png +++ b/test/fixtures/offline/sprite.png diff --git a/test/fixtures/stale/streets.json b/test/fixtures/offline/streets.json index 11cb5c8557..805bd68f45 100644 --- a/test/fixtures/stale/streets.json +++ b/test/fixtures/offline/streets.json @@ -10,5 +10,5 @@ "name": "Streets", "scheme": "xyz", "tilejson": "2.0.0", - "tiles": [ "asset://test/fixtures/stale/{z}-{x}-{y}.vector.pbf" ] + "tiles": [ "http://127.0.0.1:3000/offline/{z}-{x}-{y}.vector.pbf" ] } diff --git a/test/fixtures/stale/style_and_glyphs.json b/test/fixtures/offline/style.json index adf91705fb..c34ee59893 100644 --- a/test/fixtures/stale/style_and_glyphs.json +++ b/test/fixtures/offline/style.json @@ -4,10 +4,11 @@ "sources": { "mapbox": { "type": "vector", - "url": "asset://test/fixtures/stale/streets.json" + "url": "http://127.0.0.1:3000/offline/streets.json" } }, - "glyphs": "http://127.0.0.1:3000/stale/{fontstack}/{range}.pbf", + "glyphs": "http://127.0.0.1:3000/offline/{fontstack}/{range}.pbf", + "sprite": "http://127.0.0.1:3000/offline/sprite", "layers": [{ "id": "background", "type": "background", @@ -20,7 +21,7 @@ "source": "mapbox", "source-layer": "water", "paint": { - "fill-color": "blue" + "fill-pattern": "noise" } }, { "id": "admin", @@ -28,7 +29,8 @@ "source": "mapbox", "source-layer": "admin", "layout": { - "text-field": "Text" + "text-font": ["Helvetica"], + "text-field": "Text" }, "paint": { "text-color": "black", diff --git a/test/fixtures/stale/stale_style/expected.png b/test/fixtures/stale/stale_style/expected.png Binary files differdeleted file mode 100644 index d3c6ef3cd5..0000000000 --- a/test/fixtures/stale/stale_style/expected.png +++ /dev/null diff --git a/test/fixtures/stale/stale_style_and_glyphs/expected.png b/test/fixtures/stale/stale_style_and_glyphs/expected.png Binary files differdeleted file mode 100644 index d8dbda1092..0000000000 --- a/test/fixtures/stale/stale_style_and_glyphs/expected.png +++ /dev/null diff --git a/test/fixtures/stale/stale_style_and_sprite/expected.png b/test/fixtures/stale/stale_style_and_sprite/expected.png Binary files differdeleted file mode 100644 index 171599bf5c..0000000000 --- a/test/fixtures/stale/stale_style_and_sprite/expected.png +++ /dev/null diff --git a/test/fixtures/stale/stale_style_and_tilejson/expected.png b/test/fixtures/stale/stale_style_and_tilejson/expected.png Binary files differdeleted file mode 100644 index d3c6ef3cd5..0000000000 --- a/test/fixtures/stale/stale_style_and_tilejson/expected.png +++ /dev/null diff --git a/test/fixtures/stale/style.json b/test/fixtures/stale/style.json deleted file mode 100644 index ca86503d4f..0000000000 --- a/test/fixtures/stale/style.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "version": 8, - "name": "Water", - "sources": { - "mapbox": { - "type": "vector", - "url": "asset://test/fixtures/stale/streets.json" - } - }, - "layers": [{ - "id": "background", - "type": "background", - "paint": { - "background-color": "red" - } - }, { - "id": "water", - "type": "fill", - "source": "mapbox", - "source-layer": "water", - "paint": { - "fill-color": "blue" - } - }] -} diff --git a/test/fixtures/stale/style_and_sprite.json b/test/fixtures/stale/style_and_sprite.json deleted file mode 100644 index 61e3214a5e..0000000000 --- a/test/fixtures/stale/style_and_sprite.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "version": 8, - "name": "Water", - "sources": { - "mapbox": { - "type": "vector", - "url": "asset://test/fixtures/stale/streets.json" - } - }, - "sprite": "http://127.0.0.1:3000/stale/sprite", - "layers": [{ - "id": "background", - "type": "background", - "paint": { - "background-color": "red" - } - }, { - "id": "water", - "type": "fill", - "source": "mapbox", - "source-layer": "water", - "paint": { - "fill-pattern": "noise" - } - }] -} diff --git a/test/fixtures/stale/style_and_tilejson.json b/test/fixtures/stale/style_and_tilejson.json deleted file mode 100644 index f2a4fddd0b..0000000000 --- a/test/fixtures/stale/style_and_tilejson.json +++ /dev/null @@ -1,25 +0,0 @@ -{ - "version": 8, - "name": "Water", - "sources": { - "mapbox": { - "type": "vector", - "url": "http://127.0.0.1:3000/stale/streets.json" - } - }, - "layers": [{ - "id": "background", - "type": "background", - "paint": { - "background-color": "red" - } - }, { - "id": "water", - "type": "fill", - "source": "mapbox", - "source-layer": "water", - "paint": { - "fill-color": "blue" - } - }] -} diff --git a/test/fixtures/util.cpp b/test/fixtures/util.cpp index fcb6abed4e..5617b15e8b 100644 --- a/test/fixtures/util.cpp +++ b/test/fixtures/util.cpp @@ -106,20 +106,12 @@ uint64_t crc64(const PremultipliedImage &image) { return crc64(reinterpret_cast<const char*>(image.data.get()), image.size()); } -PremultipliedImage render(Map& map, Milliseconds timeout) { +PremultipliedImage render(Map& map) { std::promise<PremultipliedImage> promise; map.renderStill([&](std::exception_ptr, PremultipliedImage&& image) { promise.set_value(std::move(image)); }); - - // Limit maximum wait time. - auto future = promise.get_future(); - if (future.wait_for(timeout) != std::future_status::ready) { - // Alas, we didn't get the promised future :( - Log::Error(Event::Image, "Failed to generate image within %dms", timeout.count()); - return { 0, 0 }; - } - return future.get(); + return promise.get_future().get(); } void checkImage(const std::string& base, diff --git a/test/fixtures/util.hpp b/test/fixtures/util.hpp index b93d822a83..2e27e5bf0f 100644 --- a/test/fixtures/util.hpp +++ b/test/fixtures/util.hpp @@ -4,7 +4,6 @@ #include <mbgl/util/image.hpp> #include <mbgl/util/chrono.hpp> -#include <chrono> #include <cstdint> #include <gtest/gtest.h> @@ -36,12 +35,13 @@ uint64_t crc64(const char*, size_t); uint64_t crc64(const std::string&); uint64_t crc64(const PremultipliedImage&); -PremultipliedImage render(Map&, Milliseconds timeout = Milliseconds(1000)); +PremultipliedImage render(Map&); void checkImage(const std::string& base, const PremultipliedImage& actual, double imageThreshold = 0, double pixelThreshold = 0); + } } diff --git a/test/storage/asset_file_source.cpp b/test/storage/asset_file_source.cpp index 6734cc693c..a9261ee8a2 100644 --- a/test/storage/asset_file_source.cpp +++ b/test/storage/asset_file_source.cpp @@ -1,7 +1,6 @@ #include "storage.hpp" #include <mbgl/storage/asset_file_source.hpp> -#include <mbgl/storage/sqlite_cache.hpp> #include <mbgl/platform/platform.hpp> #include <mbgl/util/chrono.hpp> #include <mbgl/util/run_loop.hpp> diff --git a/test/storage/cache_response.cpp b/test/storage/cache_response.cpp deleted file mode 100644 index adeb45727e..0000000000 --- a/test/storage/cache_response.cpp +++ /dev/null @@ -1,215 +0,0 @@ -#include "storage.hpp" - -#include <mbgl/storage/default_file_source.hpp> -#include <mbgl/storage/sqlite_cache.hpp> -#include <mbgl/util/chrono.hpp> -#include <mbgl/util/run_loop.hpp> - -TEST_F(Storage, CacheResponse) { - SCOPED_TEST(CacheResponse); - - using namespace mbgl; - - util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); - - const Resource resource { Resource::Unknown, "http://127.0.0.1:3000/cache" }; - Response response; - - std::unique_ptr<FileRequest> req1; - std::unique_ptr<FileRequest> req2; - - req1 = fs.request(resource, [&](Response res) { - req1.reset(); - EXPECT_EQ(nullptr, res.error); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Response 1", *res.data); - EXPECT_TRUE(bool(res.expires)); - EXPECT_FALSE(bool(res.modified)); - EXPECT_FALSE(bool(res.etag)); - response = res; - - // Now test that we get the same values as in the previous request. If we'd go to the server - // again, we'd get different values. - req2 = fs.request(resource, [&](Response res2) { - req2.reset(); - EXPECT_EQ(response.error, res2.error); - ASSERT_TRUE(res2.data.get()); - EXPECT_EQ(*response.data, *res2.data); - EXPECT_EQ(response.expires, res2.expires); - EXPECT_EQ(response.modified, res2.modified); - EXPECT_EQ(response.etag, res2.etag); - - loop.stop(); - CacheResponse.finish(); - }); - }); - - loop.run(); -} - -// Make sure we /do/ store 404 Not Found responses into the cache -TEST_F(Storage, CacheNotFound) { - SCOPED_TEST(CacheNotFound); - - using namespace mbgl; - - util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); - - const Resource resource{ Resource::Unknown, "http://127.0.0.1:3000/not-found" }; - - // Insert existing data into the cache that will be marked as stale. - Response response; - response.data = std::make_shared<const std::string>("existing data"); - fs.getCache().put(resource, response); - - std::unique_ptr<FileRequest> req1; - std::unique_ptr<WorkRequest> req2; - - int counter = 0; - - // Then, request the actual URL and check that we're getting the rigged cache response first, - // then the connection error message. - req1 = fs.request(resource, [&](Response res) { - if (counter == 0) { - EXPECT_EQ(nullptr, res.error); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("existing data", *res.data); - EXPECT_FALSE(bool(res.expires)); - EXPECT_FALSE(bool(res.modified)); - EXPECT_FALSE(bool(res.etag)); - } else if (counter == 1) { - EXPECT_NE(nullptr, res.error); - EXPECT_EQ(Response::Error::Reason::NotFound, res.error->reason); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Not Found!", *res.data); - req1.reset(); - - // Finally, check the cache to make sure we cached the 404 response. - req2 = fs.getCache().get(resource, [&](std::unique_ptr<Response> res2) { - EXPECT_NE(nullptr, res2->error); - EXPECT_EQ(Response::Error::Reason::NotFound, res2->error->reason); - ASSERT_TRUE(res2->data.get()); - EXPECT_EQ("Not Found!", *res2->data); - CacheNotFound.finish(); - loop.stop(); - }); - } else { - FAIL() << "Got too many responses"; - } - counter++; - }); - - loop.run(); -} - -// Make sure we don't store a connection error into the cache -TEST_F(Storage, DontCacheConnectionErrors) { - SCOPED_TEST(DontCacheConnectionErrors); - - using namespace mbgl; - - util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); - - const Resource resource{ Resource::Unknown, "http://127.0.0.1:3001" }; - - // Insert existing data into the cache that will be marked as stale. - Response response; - response.data = std::make_shared<const std::string>("existing data"); - fs.getCache().put(resource, response); - - std::unique_ptr<FileRequest> req1; - std::unique_ptr<WorkRequest> req2; - - int counter = 0; - - // Then, request the actual URL and check that we're getting the rigged cache response first, - // then the connection error message. - req1 = fs.request(resource, [&](Response res) { - if (counter == 0) { - EXPECT_EQ(nullptr, res.error); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("existing data", *res.data); - EXPECT_FALSE(bool(res.expires)); - EXPECT_FALSE(bool(res.modified)); - EXPECT_FALSE(bool(res.etag)); - } else if (counter == 1) { - EXPECT_NE(nullptr, res.error); - EXPECT_EQ(Response::Error::Reason::Connection, res.error->reason); - req1.reset(); - - // Finally, check the cache to make sure we still have our original data in there rather - // than the failed connection attempt. - req2 = fs.getCache().get(resource, [&](std::unique_ptr<Response> res2) { - EXPECT_EQ(nullptr, res2->error); - ASSERT_TRUE(res2->data.get()); - EXPECT_EQ("existing data", *res2->data); - DontCacheConnectionErrors.finish(); - loop.stop(); - }); - } else { - FAIL() << "Got too many responses"; - } - counter++; - }); - - loop.run(); -} - -// Make sure we don't store a bad server response into the cache -TEST_F(Storage, DontCacheServerErrors) { - SCOPED_TEST(DontCacheServerErrors); - - using namespace mbgl; - - util::RunLoop loop; - DefaultFileSource fs(":memory:", "."); - - const Resource resource{ Resource::Unknown, "http://127.0.0.1:3000/permanent-error" }; - - // Insert existing data into the cache that will be marked as stale. - Response response; - response.data = std::make_shared<const std::string>("existing data"); - fs.getCache().put(resource, response); - - std::unique_ptr<FileRequest> req1; - std::unique_ptr<WorkRequest> req2; - - int counter = 0; - - // Then, request the actual URL and check that we're getting the rigged cache response first, - // then the server error message. - req1 = fs.request(resource, [&](Response res) { - if (counter == 0) { - EXPECT_EQ(nullptr, res.error); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("existing data", *res.data); - EXPECT_FALSE(bool(res.expires)); - EXPECT_FALSE(bool(res.modified)); - EXPECT_FALSE(bool(res.etag)); - } else if (counter == 1) { - EXPECT_NE(nullptr, res.error); - EXPECT_EQ(Response::Error::Reason::Server, res.error->reason); - ASSERT_TRUE(res.data.get()); - EXPECT_EQ("Server Error!", *res.data); - req1.reset(); - - // Finally, check the cache to make sure we still have our original data in there rather - // than the failed connection attempt. - req2 = fs.getCache().get(resource, [&](std::unique_ptr<Response> res2) { - EXPECT_EQ(nullptr, res2->error); - ASSERT_TRUE(res2->data.get()); - EXPECT_EQ("existing data", *res2->data); - DontCacheServerErrors.finish(); - loop.stop(); - }); - } else { - FAIL() << "Got too many responses"; - } - counter++; - }); - - loop.run(); -} diff --git a/test/storage/cache_shared.cpp b/test/storage/cache_shared.cpp deleted file mode 100644 index 89ef3bfb84..0000000000 --- a/test/storage/cache_shared.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include "storage.hpp" - -#include <mbgl/storage/sqlite_cache.hpp> -#include <mbgl/storage/resource.hpp> -#include <mbgl/util/run_loop.hpp> - -TEST_F(Storage, CacheShared) { - SCOPED_TEST(CacheShared) - using namespace mbgl; - - util::RunLoop loop; - - // Check that we're getting two different caches when we request different paths. - auto memory = SQLiteCache::getShared(); - auto file = SQLiteCache::getShared("test/fixtures/database/cache.db"); - EXPECT_NE(memory.get(), file.get()); - EXPECT_EQ(memory.get(), SQLiteCache::getShared().get()); - - // Store a response into the memory file cache, then delete the last reference. - const Resource resource { Resource::Kind::Unknown, "http://example.com" }; - memory->put(resource, Response()); - memory.reset(); - - // Now check that the original memory file cache has been destructed and that it doesn't contain - // the information we put into it. - memory = SQLiteCache::getShared(); - auto req = memory->get(resource, [&](std::unique_ptr<Response> res) { - EXPECT_FALSE(res.get()); - CacheShared.finish(); - loop.stop(); - }); - - loop.run(); -} diff --git a/test/storage/cache_size.cpp b/test/storage/cache_size.cpp deleted file mode 100644 index b0d59d5934..0000000000 --- a/test/storage/cache_size.cpp +++ /dev/null @@ -1,241 +0,0 @@ -#include "storage.hpp" - -#include <mbgl/storage/resource.hpp> -#include <mbgl/storage/response.hpp> -#include <mbgl/storage/sqlite_cache.hpp> -#include <mbgl/util/run_loop.hpp> -#include <mbgl/util/string.hpp> -#include <mbgl/util/timer.hpp> -#include <mbgl/util/chrono.hpp> - -#include <memory> -#include <random> - -bool tileIsCached(mbgl::SQLiteCache* cache, unsigned id) { - using namespace mbgl; - - auto url = std::string("http://tile") + mbgl::util::toString(id); - bool replied = false; - - std::unique_ptr<Response> response; - auto callback = [&] (std::unique_ptr<Response> res) { - replied = true; - response = std::move(res); - }; - - Resource resource{ Resource::Kind::Tile, url }; - auto req = cache->get(resource, callback); - - while (!replied) { - util::RunLoop::Get()->runOnce(); - } - - return response != nullptr; -} - -void insertTile(mbgl::SQLiteCache* cache, unsigned id, uint64_t size) { - using namespace mbgl; - - auto url = std::string("http://tile") + mbgl::util::toString(id); - - Response response; - response.modified = SystemClock::now(); - response.expires = SystemClock::now() + Seconds(5000); - response.etag = url; - - auto data = std::make_shared<std::string>(size, 0); - - // Fill data with garbage so SQLite won't try to - // optimize allocation by reusing pages. - static std::mt19937 generator; - std::generate_n(data->begin(), size, generator); - - response.data = data; - - Resource resource{ Resource::Kind::Tile, url }; - cache->put(resource, response); -} - -void refreshTile(mbgl::SQLiteCache* cache, unsigned id) { - using namespace mbgl; - - auto url = std::string("http://tile") + mbgl::util::toString(id); - - Response response; - response.modified = SystemClock::now(); - response.expires = SystemClock::now() + Seconds(5000); - response.notModified = true; - - Resource resource{ Resource::Kind::Tile, url }; - cache->put(resource, response); -} - -uint64_t cacheSize(mbgl::SQLiteCache* cache, unsigned entryCount, uint64_t entrySize) { - uint64_t total = 0; - - for (unsigned i = 0; i < entryCount; ++i) { - if (tileIsCached(cache, i)) { - total += entrySize; - } - } - - return total; -} - -TEST_F(Storage, CacheEntrySizeLimit) { - using namespace mbgl; - - util::RunLoop loop; - SQLiteCache cache(":memory:"); - - const uint64_t entrySize = 5 * 1024 * 1024; // 5 MB - - insertTile(&cache, 0, entrySize); - EXPECT_TRUE(tileIsCached(&cache, 0)); - - insertTile(&cache, 1, entrySize + 1); - EXPECT_FALSE(tileIsCached(&cache, 1)); - - insertTile(&cache, 2, entrySize - 1); - EXPECT_TRUE(tileIsCached(&cache, 2)); - - // Setting a new size should not delete existing entries. - cache.setMaximumCacheEntrySize(entrySize / 2); - EXPECT_TRUE(tileIsCached(&cache, 2)); - - insertTile(&cache, 3, entrySize / 2 - 1); - EXPECT_TRUE(tileIsCached(&cache, 3)); - - insertTile(&cache, 4, entrySize); - EXPECT_FALSE(tileIsCached(&cache, 4)); - - cache.setMaximumCacheEntrySize(entrySize * 2); - insertTile(&cache, 5, entrySize); - EXPECT_TRUE(tileIsCached(&cache, 5)); -} - -TEST_F(Storage, CacheSizeSetNewLimit) { - using namespace mbgl; - - util::RunLoop loop; - SQLiteCache cache(":memory:"); - - const unsigned entryCount = 800; - const uint64_t entrySize = 10 * 1024; // 10 KB - - cache.setMaximumCacheEntrySize(entrySize + 1); - - // Cache size defaults to unlimited, all these - // inserts should work. - for (unsigned i = 0; i < entryCount; ++i) { - insertTile(&cache, i, entrySize); - } - - for (unsigned i = 0; i < entryCount; ++i) { - EXPECT_TRUE(tileIsCached(&cache, i)); - } - - uint64_t expectedCacheSize = entryCount * entrySize; - EXPECT_EQ(cacheSize(&cache, entryCount, entrySize), expectedCacheSize); - - // Setting a new size should remove records until the new - // size limit is satisfied. - cache.setMaximumCacheSize(expectedCacheSize / 2); - EXPECT_LT(cacheSize(&cache, entryCount, entrySize), expectedCacheSize / 2); - - // Cache size 1 should practically clean the cache and - // prevent adding any record, although it makes no sense - // to use such size limit IRL. - cache.setMaximumCacheSize(1); - EXPECT_EQ(cacheSize(&cache, entryCount, entrySize), 0); - - insertTile(&cache, 1000, entrySize); - EXPECT_FALSE(tileIsCached(&cache, 1000)); - - // Zero should be treated as unlimited. - cache.setMaximumCacheSize(0); - - for (unsigned i = 0; i < entryCount; ++i) { - insertTile(&cache, i, entrySize); - } - - EXPECT_EQ(cacheSize(&cache, entryCount, entrySize), expectedCacheSize); -} - -TEST_F(Storage, CacheSizePruneLeastAccessed) { - using namespace mbgl; - - util::RunLoop loop; - SQLiteCache cache(":memory:"); - - const unsigned entryCount = 400; - const uint64_t entrySize = 10 * 1024; // 10 KB - - cache.setMaximumCacheEntrySize(entrySize + 1); - cache.setMaximumCacheSize(entrySize * 350); - - for (unsigned i = 0; i < entryCount; ++i) { - insertTile(&cache, i, entrySize); - - if (i == entryCount / 2) { - // We need to sleep for 1s here because - // that is the time resolution for the - // `accessed` time. Then we 'ping' the - // entry, that should update the - // `accessed` time, so it won't get - // pruned when we need more space. - bool done = false; - - util::Timer timer; - timer.start(Milliseconds(1300), - Duration::zero(), - [&done] { done = true; }); - - while (!done) { - loop.runOnce(); - } - - EXPECT_TRUE(tileIsCached(&cache, 7)); - - // Refresh should also update the `accessed` - // time of a tile. - refreshTile(&cache, 9); - } - } - - EXPECT_FALSE(tileIsCached(&cache, 6)); - EXPECT_FALSE(tileIsCached(&cache, 8)); - EXPECT_FALSE(tileIsCached(&cache, 10)); - - EXPECT_TRUE(tileIsCached(&cache, 7)); - EXPECT_TRUE(tileIsCached(&cache, 9)); -} - -TEST_F(Storage, CacheSizeStress) { - using namespace mbgl; - - util::RunLoop loop; - SQLiteCache cache(":memory:"); - - const unsigned entryCount = 2000; - const uint64_t entrySize = 10 * 1024; // 10 KB - - cache.setMaximumCacheEntrySize(entrySize + 1); - cache.setMaximumCacheSize(entrySize * 300); - - for (unsigned i = 0; i < entryCount; ++i) { - insertTile(&cache, i, entrySize); - } - - // Should not be in the cache as they were - // first inserted. - EXPECT_FALSE(tileIsCached(&cache, 0)); - EXPECT_FALSE(tileIsCached(&cache, 99)); - EXPECT_FALSE(tileIsCached(&cache, 199)); - EXPECT_FALSE(tileIsCached(&cache, 299)); - EXPECT_FALSE(tileIsCached(&cache, 399)); - - EXPECT_TRUE(tileIsCached(&cache, entryCount - 1)); - - EXPECT_LT(cacheSize(&cache, entryCount, entrySize), entrySize * 300); -} diff --git a/test/storage/cache_stale.cpp b/test/storage/cache_stale.cpp deleted file mode 100644 index 9ad0d1c06e..0000000000 --- a/test/storage/cache_stale.cpp +++ /dev/null @@ -1,131 +0,0 @@ -#include "storage.hpp" - -#include <mbgl/platform/default/headless_display.hpp> -#include <mbgl/platform/default/headless_view.hpp> -#include <mbgl/storage/sqlite_cache.hpp> -#include <mbgl/storage/default_file_source.hpp> - -#include <mbgl/platform/log.hpp> -#include <mbgl/util/work_request.hpp> -#include <mbgl/util/io.hpp> - -using namespace mbgl; -using namespace std::literals::chrono_literals; -using namespace std::literals::string_literals; - -namespace { - -void checkRendering(Map& map, - const char* name, - std::chrono::milliseconds timeout, - double imageThreshold = 0.001, - double pixelThreshold = 0.1) { - test::checkImage("test/fixtures/stale/"s + name, test::render(map, timeout), imageThreshold, - pixelThreshold); -} - -Response expiredItem(const std::string& path) { - Response response; - response.data = std::make_shared<std::string>(util::read_file("test/fixtures/"s + path)); - response.expires = SystemClock::from_time_t(0); - return response; -} - -const std::string prefix = "http://127.0.0.1:3000"; - -} - -auto display = std::make_shared<mbgl::HeadlessDisplay>(); - -TEST_F(Storage, CacheStaleStyle) { - HeadlessView view(display, 1); - - auto cache = SQLiteCache::getShared(":memory:"); - - // Rig the cache with an expired stylesheet. - const std::string stylePath = "stale/style.json"; - const Resource styleResource{ Resource::Kind::Style, prefix + "/" + stylePath }; - cache->put(styleResource, expiredItem(stylePath)); - - DefaultFileSource fileSource(":memory:", "."); - - Map map(view, fileSource, MapMode::Still); - map.setStyleURL(styleResource.url); - - checkRendering(map, "stale_style", 1000ms); -} - -TEST_F(Storage, CacheStaleStyleAndTileJSON) { - HeadlessView view(display, 1); - - auto cache = SQLiteCache::getShared(":memory:"); - - // Rig the cache with an expired stylesheet. - const std::string stylePath = "stale/style_and_tilejson.json"; - const Resource styleResource{ Resource::Kind::Style, prefix + "/" + stylePath }; - cache->put(styleResource, expiredItem(stylePath)); - - // Rig the cache with an expired TileJSON. - const std::string tilejsonPath = "stale/streets.json"; - const Resource tilejsonResource{ Resource::Kind::Source, prefix + "/" + tilejsonPath }; - cache->put(tilejsonResource, expiredItem(tilejsonPath)); - - DefaultFileSource fileSource(":memory:", "."); - - Map map(view, fileSource, MapMode::Still); - map.setStyleURL(styleResource.url); - - checkRendering(map, "stale_style_and_tilejson", 1000ms); -} - -TEST_F(Storage, CacheStaleStyleAndSprite) { - HeadlessView view(display, 1); - - auto cache = SQLiteCache::getShared(":memory:"); - - // Rig the cache with an expired stylesheet. - const std::string stylePath = "stale/style_and_sprite.json"; - const Resource styleResource{ Resource::Kind::Style, prefix + "/" + stylePath }; - cache->put(styleResource, expiredItem(stylePath)); - - // Rig the cache with an expired sprite JSON. - const std::string spritejsonPath = "stale/sprite.json"; - const Resource spritejsonResource{ Resource::Kind::SpriteJSON, prefix + "/" + spritejsonPath }; - cache->put(spritejsonResource, expiredItem(spritejsonPath)); - - // Rig the cache with an expired sprite JSON. - const std::string spriteimagePath = "stale/sprite.png"; - const Resource spriteimageResource{ Resource::Kind::SpriteImage, prefix + "/" + spriteimagePath }; - cache->put(spriteimageResource, expiredItem(spriteimagePath)); - - DefaultFileSource fileSource(":memory:", "."); - - Map map(view, fileSource, MapMode::Still); - map.setStyleURL(styleResource.url); - - checkRendering(map, "stale_style_and_sprite", 1000ms); -} - -TEST_F(Storage, CacheStaleStyleAndGlyphs) { - HeadlessView view(display, 1); - - auto cache = SQLiteCache::getShared(":memory:"); - - // Rig the cache with an expired stylesheet. - const std::string stylePath = "stale/style_and_glyphs.json"; - const Resource styleResource{ Resource::Kind::Style, prefix + "/" + stylePath }; - cache->put(styleResource, expiredItem(stylePath)); - - // Rig the cache with an expired glyph PBF. - const std::string glyphPath = "stale/glyph.pbf"; - const Resource glyphResource{ Resource::Kind::Glyphs, prefix + "/stale/Open%20Sans%20Regular%2c%20Arial%20Unicode%20MS%20Regular/0-255.pbf" }; - cache->put(glyphResource, expiredItem(glyphPath)); - - DefaultFileSource fileSource(":memory:", "."); - - Map map(view, fileSource, MapMode::Still); - map.setStyleURL(styleResource.url); - - // TODO: this shouldn't take > 1 second - checkRendering(map, "stale_style_and_glyphs", 2000ms, 0.0015); -} diff --git a/test/storage/database.cpp b/test/storage/database.cpp deleted file mode 100644 index 19b46763b9..0000000000 --- a/test/storage/database.cpp +++ /dev/null @@ -1,349 +0,0 @@ -#include "storage.hpp" -#include "../fixtures/fixture_log_observer.hpp" - -#include "sqlite_cache_impl.hpp" -#include <mbgl/storage/resource.hpp> -#include <mbgl/storage/response.hpp> -#include <mbgl/util/io.hpp> - -#include <sqlite3.h> - -TEST_F(Storage, DatabaseDoesNotExist) { - using namespace mbgl; - - Log::setObserver(std::make_unique<FixtureLogObserver>()); - - SQLiteCache::Impl cache("test/fixtures/404/cache.db"); - - cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) { - EXPECT_EQ(nullptr, res.get()); - }); - - auto observer = Log::removeObserver(); - EXPECT_EQ(1ul, dynamic_cast<FixtureLogObserver*>(observer.get())->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file" })); -} - -void createDir(const char* name) { - const int ret = mkdir(name, 0755); - if (ret == -1) { - ASSERT_EQ(EEXIST, errno); - } else { - ASSERT_EQ(0, ret); - } -} - -void deleteFile(const char* name) { - const int ret = unlink(name); - if (ret == -1) { - ASSERT_EQ(ENOENT, errno); - } else { - ASSERT_EQ(0, ret); - } -} - - -void writeFile(const char* name, const std::string& data) { - mbgl::util::write_file(name, data); -} - - -TEST_F(Storage, DatabaseCreate) { - using namespace mbgl; - - createDir("test/fixtures/database"); - deleteFile("test/fixtures/database/cache.db"); - - Log::setObserver(std::make_unique<FixtureLogObserver>()); - - SQLiteCache::Impl cache("test/fixtures/database/cache.db"); - - cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) { - EXPECT_EQ(nullptr, res.get()); - }); - - Log::removeObserver(); -} - -TEST_F(Storage, DatabaseVersion) { - using namespace mbgl; - - createDir("test/fixtures/database"); - deleteFile("test/fixtures/database/cache.db"); - std::string path("test/fixtures/database/cache.db"); - - Log::setObserver(std::make_unique<FixtureLogObserver>()); - - { - SQLiteCache::Impl cache(path); - cache.put({ Resource::Unknown, "mapbox://test" }, Response()); - } - - sqlite3* db; - sqlite3_open_v2(path.c_str(), &db, SQLITE_OPEN_READWRITE, nullptr); - sqlite3_exec(db, "PRAGMA user_version = 999999", nullptr, nullptr, nullptr); - sqlite3_close_v2(db); - - // Changing the version will force the database to get recreated - // thus removing every pre-existing cache entry. - { - SQLiteCache::Impl cache(path); - - cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) { - EXPECT_EQ(nullptr, res.get()); - }); - } - - Log::removeObserver(); -} - -class FileLock { -public: - FileLock(const std::string& path) { - const int err = sqlite3_open_v2(path.c_str(), &db, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE, nullptr); - if (err != SQLITE_OK) { - throw std::runtime_error("Could not open db"); - } - lock(); - } - - void lock() { - assert(!locked); - const int err = sqlite3_exec(db, "begin exclusive transaction", nullptr, nullptr, nullptr); - if (err != SQLITE_OK) { - throw std::runtime_error("Could not lock db"); - } - locked = true; - } - - void unlock() { - assert(locked); - const int err = sqlite3_exec(db, "commit", nullptr, nullptr, nullptr); - if (err != SQLITE_OK) { - throw std::runtime_error("Could not unlock db"); - } - locked = false; - } - - ~FileLock() { - if (locked) { - unlock(); - } - } - -private: - sqlite3* db; - bool locked = false; -}; - -TEST_F(Storage, DatabaseLockedRead) { - using namespace mbgl; - - // Create a locked file. - createDir("test/fixtures/database"); - deleteFile("test/fixtures/database/locked.db"); - FileLock guard("test/fixtures/database/locked.db"); - - SQLiteCache::Impl cache("test/fixtures/database/locked.db"); - - { - // First request should fail. - Log::setObserver(std::make_unique<FixtureLogObserver>()); - - cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) { - EXPECT_EQ(nullptr, res.get()); - }); - - // Make sure that we got a few "database locked" errors - auto observer = Log::removeObserver(); - auto flo = dynamic_cast<FixtureLogObserver*>(observer.get()); - EXPECT_EQ(4ul, flo->count({ EventSeverity::Error, Event::Database, 5, "database is locked" })); - } - - // Then, unlock the file and try again. - guard.unlock(); - - { - // First, try getting a file (the cache value should not exist). - Log::setObserver(std::make_unique<FixtureLogObserver>()); - - cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) { - EXPECT_EQ(nullptr, res.get()); - }); - - // Make sure that we got a no errors - Log::removeObserver(); - } -} - - - -TEST_F(Storage, DatabaseLockedWrite) { - using namespace mbgl; - - // Create a locked file. - createDir("test/fixtures/database"); - deleteFile("test/fixtures/database/locked.db"); - FileLock guard("test/fixtures/database/locked.db"); - - SQLiteCache::Impl cache("test/fixtures/database/locked.db"); - - { - // Adds a file (which should fail). - Log::setObserver(std::make_unique<FixtureLogObserver>()); - - cache.put({ Resource::Unknown, "mapbox://test" }, Response()); - cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) { - EXPECT_EQ(nullptr, res.get()); - }); - - auto observer = Log::removeObserver(); - auto flo = dynamic_cast<FixtureLogObserver*>(observer.get()); - EXPECT_EQ(8ul, flo->count({ EventSeverity::Error, Event::Database, 5, "database is locked" })); - } - - // Then, unlock the file and try again. - guard.unlock(); - - { - // Then, set a file and obtain it again. - Log::setObserver(std::make_unique<FixtureLogObserver>()); - - Response response; - response.data = std::make_shared<std::string>("Demo"); - cache.put({ Resource::Unknown, "mapbox://test" }, response); - cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) { - ASSERT_NE(nullptr, res.get()); - ASSERT_TRUE(res->data.get()); - EXPECT_EQ("Demo", *res->data); - }); - - // Make sure that we got a no errors - Log::removeObserver(); - } -} - - - - -TEST_F(Storage, DatabaseLockedRefresh) { - using namespace mbgl; - - // Create a locked file. - createDir("test/fixtures/database"); - deleteFile("test/fixtures/database/locked.db"); - - SQLiteCache::Impl cache("test/fixtures/database/locked.db"); - - // Then, lock the file and try again. - FileLock guard("test/fixtures/database/locked.db"); - - { - // Adds a file. - Log::setObserver(std::make_unique<FixtureLogObserver>()); - - Response response; - response.data = std::make_shared<std::string>("Demo"); - cache.put({ Resource::Unknown, "mapbox://test" }, response); - cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) { - EXPECT_EQ(nullptr, res.get()); - }); - - auto observer = Log::removeObserver(); - auto flo = dynamic_cast<FixtureLogObserver*>(observer.get()); - EXPECT_EQ(8ul, flo->count({ EventSeverity::Error, Event::Database, 5, "database is locked" })); - } - - { - // Then, try to refresh it. - Log::setObserver(std::make_unique<FixtureLogObserver>()); - - cache.refresh({ Resource::Unknown, "mapbox://test" }, {}); - cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) { - EXPECT_EQ(nullptr, res.get()); - }); - - // Make sure that we got the right errors. - auto observer = Log::removeObserver(); - auto flo = dynamic_cast<FixtureLogObserver*>(observer.get()); - EXPECT_EQ(8ul, flo->count({ EventSeverity::Error, Event::Database, 5, "database is locked" })); - } -} - - - -TEST_F(Storage, DatabaseDeleted) { - using namespace mbgl; - - // Create a locked file. - createDir("test/fixtures/database"); - deleteFile("test/fixtures/database/locked.db"); - - SQLiteCache::Impl cache("test/fixtures/database/locked.db"); - - { - // Adds a file. - Log::setObserver(std::make_unique<FixtureLogObserver>()); - - Response response; - response.data = std::make_shared<std::string>("Demo"); - cache.put({ Resource::Unknown, "mapbox://test" }, response); - cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) { - ASSERT_NE(nullptr, res.get()); - ASSERT_TRUE(res->data.get()); - EXPECT_EQ("Demo", *res->data); - }); - - Log::removeObserver(); - } - - deleteFile("test/fixtures/database/locked.db"); - - { - // Adds a file. - Log::setObserver(std::make_unique<FixtureLogObserver>()); - - Response response; - response.data = std::make_shared<std::string>("Demo"); - cache.put({ Resource::Unknown, "mapbox://test" }, response); - cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) { - ASSERT_NE(nullptr, res.get()); - ASSERT_TRUE(res->data.get()); - EXPECT_EQ("Demo", *res->data); - }); - - auto observer = Log::removeObserver(); - auto flo = dynamic_cast<FixtureLogObserver*>(observer.get()); - EXPECT_EQ(1ul, flo->count({ EventSeverity::Error, Event::Database, 8, "attempt to write a readonly database" })); - } -} - - - -TEST_F(Storage, DatabaseInvalid) { - using namespace mbgl; - - // Create a locked file. - createDir("test/fixtures/database"); - deleteFile("test/fixtures/database/invalid.db"); - writeFile("test/fixtures/database/invalid.db", "this is an invalid file"); - - SQLiteCache::Impl cache("test/fixtures/database/invalid.db"); - - { - // Adds a file. - Log::setObserver(std::make_unique<FixtureLogObserver>()); - - Response response; - response.data = std::make_shared<std::string>("Demo"); - cache.put({ Resource::Unknown, "mapbox://test" }, response); - cache.get({ Resource::Unknown, "mapbox://test" }, [] (std::unique_ptr<Response> res) { - ASSERT_NE(nullptr, res.get()); - ASSERT_TRUE(res->data.get()); - EXPECT_EQ("Demo", *res->data); - }); - - auto observer = Log::removeObserver(); - auto flo = dynamic_cast<FixtureLogObserver*>(observer.get()); - EXPECT_EQ(1ul, flo->count({ EventSeverity::Warning, Event::Database, -1, "Trashing invalid database" })); - } -} diff --git a/test/storage/cache_revalidate.cpp b/test/storage/default_file_source.cpp index e769e934d5..26fb164d3f 100644 --- a/test/storage/cache_revalidate.cpp +++ b/test/storage/default_file_source.cpp @@ -1,10 +1,54 @@ #include "storage.hpp" #include <mbgl/storage/default_file_source.hpp> -#include <mbgl/util/chrono.hpp> #include <mbgl/util/run_loop.hpp> -TEST_F(Storage, CacheRevalidateSame) { +class DefaultFileSourceTest : public Storage {}; + +TEST_F(DefaultFileSourceTest, CacheResponse) { + SCOPED_TEST(CacheResponse); + + using namespace mbgl; + + util::RunLoop loop; + DefaultFileSource fs(":memory:", "."); + + const Resource resource { Resource::Unknown, "http://127.0.0.1:3000/cache" }; + Response response; + + std::unique_ptr<FileRequest> req1; + std::unique_ptr<FileRequest> req2; + + req1 = fs.request(resource, [&](Response res) { + req1.reset(); + EXPECT_EQ(nullptr, res.error); + ASSERT_TRUE(res.data.get()); + EXPECT_EQ("Response 1", *res.data); + EXPECT_TRUE(bool(res.expires)); + EXPECT_FALSE(bool(res.modified)); + EXPECT_FALSE(bool(res.etag)); + response = res; + + // Now test that we get the same values as in the previous request. If we'd go to the server + // again, we'd get different values. + req2 = fs.request(resource, [&](Response res2) { + req2.reset(); + EXPECT_EQ(response.error, res2.error); + ASSERT_TRUE(res2.data.get()); + EXPECT_EQ(*response.data, *res2.data); + EXPECT_EQ(response.expires, res2.expires); + EXPECT_EQ(response.modified, res2.modified); + EXPECT_EQ(response.etag, res2.etag); + + loop.stop(); + CacheResponse.finish(); + }); + }); + + loop.run(); +} + +TEST_F(DefaultFileSourceTest, CacheRevalidateSame) { SCOPED_TEST(CacheRevalidateSame) using namespace mbgl; @@ -53,7 +97,7 @@ TEST_F(Storage, CacheRevalidateSame) { loop.run(); } -TEST_F(Storage, CacheRevalidateModified) { +TEST_F(DefaultFileSourceTest, CacheRevalidateModified) { SCOPED_TEST(CacheRevalidateModified) using namespace mbgl; @@ -102,7 +146,7 @@ TEST_F(Storage, CacheRevalidateModified) { loop.run(); } -TEST_F(Storage, CacheRevalidateEtag) { +TEST_F(DefaultFileSourceTest, CacheRevalidateEtag) { SCOPED_TEST(CacheRevalidateEtag) using namespace mbgl; diff --git a/test/storage/offline_database.cpp b/test/storage/offline_database.cpp new file mode 100644 index 0000000000..e2e32ee36b --- /dev/null +++ b/test/storage/offline_database.cpp @@ -0,0 +1,374 @@ +#include "../fixtures/fixture_log_observer.hpp" + +#include <mbgl/storage/offline_database.hpp> +#include <mbgl/storage/resource.hpp> +#include <mbgl/storage/response.hpp> +#include <mbgl/util/io.hpp> + +#include <gtest/gtest.h> +#include <sqlite3.h> + +namespace { + +void createDir(const char* name) { + const int ret = mkdir(name, 0755); + if (ret == -1) { + ASSERT_EQ(EEXIST, errno); + } else { + ASSERT_EQ(0, ret); + } +} + +void deleteFile(const char* name) { + const int ret = unlink(name); + if (ret == -1) { + ASSERT_EQ(ENOENT, errno); + } else { + ASSERT_EQ(0, ret); + } +} + +void writeFile(const char* name, const std::string& data) { + mbgl::util::write_file(name, data); +} + +class FileLock { +public: + FileLock(const std::string& path) { + const int err = sqlite3_open_v2(path.c_str(), &db, SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE, nullptr); + if (err != SQLITE_OK) { + throw std::runtime_error("Could not open db"); + } + lock(); + } + + void lock() { + assert(!locked); + const int err = sqlite3_exec(db, "begin exclusive transaction", nullptr, nullptr, nullptr); + if (err != SQLITE_OK) { + throw std::runtime_error("Could not lock db"); + } + locked = true; + } + + void unlock() { + assert(locked); + const int err = sqlite3_exec(db, "commit", nullptr, nullptr, nullptr); + if (err != SQLITE_OK) { + throw std::runtime_error("Could not unlock db"); + } + locked = false; + } + + ~FileLock() { + if (locked) { + unlock(); + } + } + +private: + sqlite3* db; + bool locked = false; +}; + +} + +//TEST(OfflineDatabase, NonexistentDirectory) { +// using namespace mbgl; +// +// Log::setObserver(std::make_unique<FixtureLogObserver>()); +// +// OfflineDatabase db("test/fixtures/404/offline.db"); +// +// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) { +// EXPECT_FALSE(bool(res)); +// }); +// +// auto observer = Log::removeObserver(); +// EXPECT_EQ(1ul, dynamic_cast<FixtureLogObserver*>(observer.get())->count({ EventSeverity::Error, Event::Database, 14, "unable to open database file" })); +//} + +TEST(OfflineDatabase, Create) { + using namespace mbgl; + + createDir("test/fixtures/database"); + deleteFile("test/fixtures/database/offline.db"); + + Log::setObserver(std::make_unique<FixtureLogObserver>()); + + OfflineDatabase db("test/fixtures/database/offline.db"); + EXPECT_FALSE(bool(db.get({ Resource::Unknown, "mapbox://test" }))); + + Log::removeObserver(); +} + +TEST(OfflineDatabase, SchemaVersion) { + using namespace mbgl; + + createDir("test/fixtures/database"); + deleteFile("test/fixtures/database/offline.db"); + std::string path("test/fixtures/database/offline.db"); + + { + sqlite3* db; + sqlite3_open_v2(path.c_str(), &db, SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, nullptr); + sqlite3_exec(db, "PRAGMA user_version = 1", nullptr, nullptr, nullptr); + sqlite3_close_v2(db); + } + + Log::setObserver(std::make_unique<FixtureLogObserver>()); + OfflineDatabase db(path); + + auto observer = Log::removeObserver(); + auto flo = dynamic_cast<FixtureLogObserver*>(observer.get()); + EXPECT_EQ(1ul, flo->count({ EventSeverity::Warning, Event::Database, -1, "Removing existing incompatible offline database" })); +} + +TEST(OfflineDatabase, Invalid) { + using namespace mbgl; + + createDir("test/fixtures/database"); + deleteFile("test/fixtures/database/invalid.db"); + writeFile("test/fixtures/database/invalid.db", "this is an invalid file"); + + Log::setObserver(std::make_unique<FixtureLogObserver>()); + + OfflineDatabase db("test/fixtures/database/invalid.db"); + + auto observer = Log::removeObserver(); + auto flo = dynamic_cast<FixtureLogObserver*>(observer.get()); + EXPECT_EQ(1ul, flo->count({ EventSeverity::Warning, Event::Database, -1, "Removing existing incompatible offline database" })); +} + +//TEST(OfflineDatabase, DatabaseLockedRead) { +// using namespace mbgl; +// +// // Create a locked file. +// createDir("test/fixtures/database"); +// deleteFile("test/fixtures/database/locked.db"); +// FileLock guard("test/fixtures/database/locked.db"); +// +// OfflineDatabase db("test/fixtures/database/locked.db"); +// +// { +// // First request should fail. +// Log::setObserver(std::make_unique<FixtureLogObserver>()); +// +// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) { +// EXPECT_FALSE(bool(res)); +// }); +// +// // Make sure that we got a few "database locked" errors +// auto observer = Log::removeObserver(); +// auto flo = dynamic_cast<FixtureLogObserver*>(observer.get()); +// EXPECT_EQ(4ul, flo->count({ EventSeverity::Error, Event::Database, 5, "database is locked" })); +// } +// +// // Then, unlock the file and try again. +// guard.unlock(); +// +// { +// // First, try getting a file (the cache value should not exist). +// Log::setObserver(std::make_unique<FixtureLogObserver>()); +// +// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) { +// EXPECT_FALSE(bool(res)); +// }); +// +// // Make sure that we got a no errors +// Log::removeObserver(); +// } +//} +// +//TEST(OfflineDatabase, DatabaseLockedWrite) { +// using namespace mbgl; +// +// // Create a locked file. +// createDir("test/fixtures/database"); +// deleteFile("test/fixtures/database/locked.db"); +// FileLock guard("test/fixtures/database/locked.db"); +// +// OfflineDatabase db("test/fixtures/database/locked.db"); +// +// { +// // Adds a file (which should fail). +// Log::setObserver(std::make_unique<FixtureLogObserver>()); +// +// db.put({ Resource::Unknown, "mapbox://test" }, Response()); +// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) { +// EXPECT_FALSE(bool(res)); +// }); +// +// auto observer = Log::removeObserver(); +// auto flo = dynamic_cast<FixtureLogObserver*>(observer.get()); +// EXPECT_EQ(8ul, flo->count({ EventSeverity::Error, Event::Database, 5, "database is locked" })); +// } +// +// // Then, unlock the file and try again. +// guard.unlock(); +// +// { +// // Then, set a file and obtain it again. +// Log::setObserver(std::make_unique<FixtureLogObserver>()); +// +// Response response; +// response.data = std::make_shared<std::string>("Demo"); +// db.put({ Resource::Unknown, "mapbox://test" }, response); +// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) { +// ASSERT_TRUE(bool(res)); +// ASSERT_TRUE(res->data.get()); +// EXPECT_EQ("Demo", *res->data); +// }); +// +// // Make sure that we got a no errors +// Log::removeObserver(); +// } +//} +// +//TEST(OfflineDatabase, DatabaseDeleted) { +// using namespace mbgl; +// +// // Create a locked file. +// createDir("test/fixtures/database"); +// deleteFile("test/fixtures/database/locked.db"); +// +// OfflineDatabase db("test/fixtures/database/locked.db"); +// +// { +// // Adds a file. +// Log::setObserver(std::make_unique<FixtureLogObserver>()); +// +// Response response; +// response.data = std::make_shared<std::string>("Demo"); +// db.put({ Resource::Unknown, "mapbox://test" }, response); +// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) { +// ASSERT_TRUE(bool(res)); +// ASSERT_TRUE(res->data.get()); +// EXPECT_EQ("Demo", *res->data); +// }); +// +// Log::removeObserver(); +// } +// +// deleteFile("test/fixtures/database/locked.db"); +// +// { +// // Adds a file. +// Log::setObserver(std::make_unique<FixtureLogObserver>()); +// +// Response response; +// response.data = std::make_shared<std::string>("Demo"); +// db.put({ Resource::Unknown, "mapbox://test" }, response); +// db.get({ Resource::Unknown, "mapbox://test" }, [] (optional<Response> res) { +// ASSERT_TRUE(bool(res)); +// ASSERT_TRUE(res->data.get()); +// EXPECT_EQ("Demo", *res->data); +// }); +// +// auto observer = Log::removeObserver(); +// auto flo = dynamic_cast<FixtureLogObserver*>(observer.get()); +// EXPECT_EQ(1ul, flo->count({ EventSeverity::Error, Event::Database, 8, "attempt to write a readonly database" })); +// } +//} + +TEST(OfflineDatabase, PutDoesNotStoreConnectionErrors) { + using namespace mbgl; + + OfflineDatabase db(":memory:"); + + Resource resource { Resource::Unknown, "http://example.com/" }; + Response response; + response.error = std::make_unique<Response::Error>(Response::Error::Reason::Connection); + + db.put(resource, response); + EXPECT_FALSE(bool(db.get(resource))); +} + +TEST(OfflineDatabase, PutDoesNotStoreServerErrors) { + using namespace mbgl; + + OfflineDatabase db(":memory:"); + + Resource resource { Resource::Unknown, "http://example.com/" }; + Response response; + response.error = std::make_unique<Response::Error>(Response::Error::Reason::Server); + + db.put(resource, response); + EXPECT_FALSE(bool(db.get(resource))); +} + +TEST(OfflineDatabase, PutResource) { + using namespace mbgl; + + OfflineDatabase db(":memory:"); + + Resource resource { Resource::Style, "http://example.com/" }; + Response response; + response.data = std::make_shared<std::string>("data"); + + db.put(resource, response); + auto res = db.get(resource); + EXPECT_EQ(nullptr, res->error.get()); + EXPECT_EQ("data", *res->data); +} + +TEST(OfflineDatabase, PutTile) { + using namespace mbgl; + + OfflineDatabase db(":memory:"); + + Resource resource { Resource::Tile, "http://example.com/" }; + resource.tileData = Resource::TileData { + "http://example.com/", + 1, + 0, + 0, + 0 + }; + Response response; + response.data = std::make_shared<std::string>("data"); + + db.put(resource, response); + auto res = db.get(resource); + EXPECT_EQ(nullptr, res->error.get()); + EXPECT_EQ("data", *res->data); +} + +TEST(OfflineDatabase, PutResourceNotFound) { + using namespace mbgl; + + OfflineDatabase db(":memory:"); + + Resource resource { Resource::Style, "http://example.com/" }; + Response response; + response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound); + + db.put(resource, response); + auto res = db.get(resource); + EXPECT_NE(nullptr, res->error); + EXPECT_EQ(Response::Error::Reason::NotFound, res->error->reason); + EXPECT_FALSE(res->data.get()); +} + +TEST(OfflineDatabase, PutTileNotFound) { + using namespace mbgl; + + OfflineDatabase db(":memory:"); + + Resource resource { Resource::Tile, "http://example.com/" }; + resource.tileData = Resource::TileData { + "http://example.com/", + 1, + 0, + 0, + 0 + }; + Response response; + response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound); + + db.put(resource, response); + auto res = db.get(resource); + EXPECT_NE(nullptr, res->error); + EXPECT_EQ(Response::Error::Reason::NotFound, res->error->reason); + EXPECT_FALSE(res->data.get()); +} diff --git a/test/test.gypi b/test/test.gypi index 22e3d0184e..883b954a35 100644 --- a/test/test.gypi +++ b/test/test.gypi @@ -58,6 +58,7 @@ 'api/render_missing.cpp', 'api/set_style.cpp', 'api/custom_layer.cpp', + 'api/offline.cpp', 'geometry/binpack.cpp', @@ -68,12 +69,8 @@ 'storage/storage.hpp', 'storage/storage.cpp', - 'storage/cache_response.cpp', - 'storage/cache_revalidate.cpp', - 'storage/cache_shared.cpp', - 'storage/cache_size.cpp', - 'storage/cache_stale.cpp', - 'storage/database.cpp', + 'storage/default_file_source.cpp', + 'storage/offline_database.cpp', 'storage/asset_file_source.cpp', 'storage/headers.cpp', 'storage/http_cancel.cpp', |
