diff options
author | John Firebaugh <john.firebaugh@gmail.com> | 2016-02-05 17:10:13 -0800 |
---|---|---|
committer | John Firebaugh <john.firebaugh@gmail.com> | 2016-02-10 15:40:20 -0800 |
commit | c3c4c7b9a695ad1dbebe57242ba071103fe9a567 (patch) | |
tree | e205ecdc6a2f6318c6ba6308b5aa8baacc42f481 | |
parent | e9302c797f68c7e48b908b87b126045c8c5e5209 (diff) | |
download | qtlocation-mapboxgl-c3c4c7b9a695ad1dbebe57242ba071103fe9a567.tar.gz |
[core] Interface and implementation for offline
23 files changed, 1524 insertions, 16 deletions
diff --git a/gyp/platform-android.gypi b/gyp/platform-android.gypi index 2c1e06a715..91962ed856 100644 --- a/gyp/platform-android.gypi +++ b/gyp/platform-android.gypi @@ -21,8 +21,12 @@ '../platform/default/timer.cpp', '../platform/default/default_file_source.cpp', '../platform/default/online_file_source.cpp', + '../platform/default/mbgl/storage/offline.hpp', + '../platform/default/mbgl/storage/offline.cpp', '../platform/default/mbgl/storage/offline_database.hpp', '../platform/default/mbgl/storage/offline_database.cpp', + '../platform/default/mbgl/storage/offline_download.hpp', + '../platform/default/mbgl/storage/offline_download.cpp', '../platform/default/sqlite3.hpp', '../platform/default/sqlite3.cpp', ], @@ -35,6 +39,7 @@ '<@(nunicode_cflags)', '<@(boost_cflags)', '<@(sqlite_cflags)', + '<@(rapidjson_cflags)', ], 'ldflags': [ '<@(libpng_ldflags)', diff --git a/gyp/platform-ios.gypi b/gyp/platform-ios.gypi index 806f103a17..b70abd3a13 100644 --- a/gyp/platform-ios.gypi +++ b/gyp/platform-ios.gypi @@ -16,8 +16,12 @@ '../platform/default/timer.cpp', '../platform/default/default_file_source.cpp', '../platform/default/online_file_source.cpp', + '../platform/default/mbgl/storage/offline.hpp', + '../platform/default/mbgl/storage/offline.cpp', '../platform/default/mbgl/storage/offline_database.hpp', '../platform/default/mbgl/storage/offline_database.cpp', + '../platform/default/mbgl/storage/offline_download.hpp', + '../platform/default/mbgl/storage/offline_download.cpp', '../platform/default/sqlite3.hpp', '../platform/default/sqlite3.cpp', '../platform/darwin/log_nslog.mm', @@ -72,6 +76,7 @@ '<@(boost_cflags)', '<@(sqlite_cflags)', '<@(zlib_cflags)', + '<@(rapidjson_cflags)', ], 'ldflags': [ '<@(sqlite_ldflags)', diff --git a/gyp/platform-linux.gypi b/gyp/platform-linux.gypi index 2eafe3e821..0fae25d4b8 100644 --- a/gyp/platform-linux.gypi +++ b/gyp/platform-linux.gypi @@ -23,8 +23,12 @@ '../platform/default/timer.cpp', '../platform/default/default_file_source.cpp', '../platform/default/online_file_source.cpp', + '../platform/default/mbgl/storage/offline.hpp', + '../platform/default/mbgl/storage/offline.cpp', '../platform/default/mbgl/storage/offline_database.hpp', '../platform/default/mbgl/storage/offline_database.cpp', + '../platform/default/mbgl/storage/offline_download.hpp', + '../platform/default/mbgl/storage/offline_download.cpp', '../platform/default/sqlite3.hpp', '../platform/default/sqlite3.cpp', ], @@ -38,6 +42,7 @@ '<@(boost_cflags)', '<@(sqlite_cflags)', '<@(webp_cflags)', + '<@(rapidjson_cflags)', ], 'ldflags': [ '<@(libpng_ldflags)', diff --git a/gyp/platform-osx.gypi b/gyp/platform-osx.gypi index 53e7047492..970fd1b5f6 100644 --- a/gyp/platform-osx.gypi +++ b/gyp/platform-osx.gypi @@ -15,8 +15,12 @@ '../platform/default/timer.cpp', '../platform/default/default_file_source.cpp', '../platform/default/online_file_source.cpp', + '../platform/default/mbgl/storage/offline.hpp', + '../platform/default/mbgl/storage/offline.cpp', '../platform/default/mbgl/storage/offline_database.hpp', '../platform/default/mbgl/storage/offline_database.cpp', + '../platform/default/mbgl/storage/offline_download.hpp', + '../platform/default/mbgl/storage/offline_download.cpp', '../platform/default/sqlite3.hpp', '../platform/default/sqlite3.cpp', '../platform/darwin/log_nslog.mm', @@ -63,6 +67,7 @@ '<@(boost_cflags)', '<@(sqlite_cflags)', '<@(zlib_cflags)', + '<@(rapidjson_cflags)', ], 'ldflags': [ '<@(zlib_ldflags)', diff --git a/include/mbgl/storage/default_file_source.hpp b/include/mbgl/storage/default_file_source.hpp index d8fc4b98a3..e7ffe125b1 100644 --- a/include/mbgl/storage/default_file_source.hpp +++ b/include/mbgl/storage/default_file_source.hpp @@ -2,6 +2,9 @@ #define MBGL_STORAGE_DEFAULT_FILE_SOURCE #include <mbgl/storage/file_source.hpp> +#include <mbgl/storage/offline.hpp> + +#include <vector> namespace mbgl { @@ -22,6 +25,80 @@ public: std::unique_ptr<FileRequest> request(const Resource&, Callback) override; + /* + * Retrieve all regions in the offline database. + * + * The query will be executed asynchronously and the results passed to the given + * callback, which will be executed on the database thread; it is the responsibility + * of the SDK bindings to re-execute a user-provided callback on the main thread. + */ + void listOfflineRegions(std::function<void (std::exception_ptr, + optional<std::vector<OfflineRegion>>)>); + + /* + * Create an offline region in the database. + * + * When the initial database queries have completed, the provided callback will be + * executed on the database thread; it is the responsibility of the SDK bindings + * to re-execute a user-provided callback on the main thread. + * + * Note that the resulting region will be in an inactive download state; to begin + * downloading resources, call `setOfflineRegionDownloadState(OfflineRegionDownloadState::Active)`, + * optionally registering an `OfflineRegionObserver` beforehand. + */ + void createOfflineRegion(const OfflineRegionDefinition& definition, + const OfflineRegionMetadata& metadata, + std::function<void (std::exception_ptr, + optional<OfflineRegion>)>); + + /* + * Register an observer to be notified when the state of the region changes. + */ + void setOfflineRegionObserver(OfflineRegion&, std::unique_ptr<OfflineRegionObserver>); + + /* + * Pause or resume downloading of regional resources. + */ + void setOfflineRegionDownloadState(OfflineRegion&, OfflineRegionDownloadState); + + /* + * Retrieve the current status of the region. The query will be executed + * asynchronously and the results passed to the given callback, which will be + * executed on the database thread; it is the responsibility of the SDK bindings + * to re-execute a user-provided callback on the main thread. + */ + void getOfflineRegionStatus(OfflineRegion&, std::function<void (std::exception_ptr, + optional<OfflineRegionStatus>)>) const; + + /* + * Initiate the removal of offline region from the database. + * + * All resources required by the region, but not also required by other regions, will + * become eligible for removal for space-optimization. Because the offline database is + * also used for ambient usage-based caching, and offline resources may still be useful + * for ambient usage, they are not immediately removed. To immediately remove resources + * not used by any extant region, call removeUnusedOfflineResources(). + * + * Note that this method takes ownership of the input, reflecting the fact that once + * region deletion is initiated, it is not legal to perform further actions with the + * region. + * + * When the operation is complete or encounters an error, the given callback will be + * executed on the database thread; it is the responsibility of the SDK bindings + * to re-execute a user-provided callback on the main thread. + */ + void deleteOfflineRegion(OfflineRegion&&, std::function<void (std::exception_ptr)>); + + /* + * Remove all resources in the database that are not required by any region, thus + * optimizing the disk space used by the offline database. + * + * When the operation is complete or encounters an error, the given callback will be + * executed on the database thread; it is the responsibility of the SDK bindings + * to re-execute a user-provided callback on the main thread. + */ + void removeUnusedOfflineResources(std::function<void (std::exception_ptr)>); + // For testing only. void put(const Resource&, const Response&); void goOffline(); diff --git a/include/mbgl/storage/offline.hpp b/include/mbgl/storage/offline.hpp new file mode 100644 index 0000000000..dd63cf968f --- /dev/null +++ b/include/mbgl/storage/offline.hpp @@ -0,0 +1,181 @@ +#pragma once + +#include <mbgl/util/geo.hpp> +#include <mbgl/util/optional.hpp> +#include <mbgl/style/types.hpp> +#include <mbgl/storage/response.hpp> + +#include <string> +#include <vector> +#include <functional> + +namespace mbgl { + +class TileID; +class SourceInfo; + +/* + * An offline region defined by a style URL, geographic bounding box, zoom range, and + * device pixel ratio. + * + * Both minZoom and maxZoom must be ≥ 0, and maxZoom must be ≥ minZoom. + * + * maxZoom may be ∞, in which case for each tile source, the region will include + * tiles from minZoom up to the maximum zoom level provided by that source. + * + * pixelRatio must be ≥ 0 and should typically be 1.0 or 2.0. + */ +class OfflineTilePyramidRegionDefinition { +public: + OfflineTilePyramidRegionDefinition(const std::string&, const LatLngBounds&, double, double, float); + + /* Private */ + std::vector<TileID> tileCover(SourceType, uint16_t tileSize, const SourceInfo&) const; + + const std::string styleURL; + const LatLngBounds bounds; + const double minZoom; + const double maxZoom; + const float pixelRatio; +}; + +/* + * For the present, a tile pyramid is the only type of offline region. In the future, + * other definition types will be available and this will be a variant type. + */ +using OfflineRegionDefinition = OfflineTilePyramidRegionDefinition; + +/* + * The encoded format is private. + */ +std::string encodeOfflineRegionDefinition(const OfflineRegionDefinition&); +OfflineRegionDefinition decodeOfflineRegionDefinition(const std::string&); + +/* + * Arbitrary binary region metadata. The contents are opaque to the mbgl implementation; + * it just stores and retrieves a BLOB. SDK bindings should leave the interpretation of + * this data up to the application; they _should not_ enforce a higher-level data format. + * In the future we want offline database to be portable across target platforms, and a + * platform-specific metadata format would prevent that. + */ +using OfflineRegionMetadata = std::vector<uint8_t>; + +/* + * A region is either inactive (not downloading, but previously-downloaded + * resources are available for use), or active (resources are being downloaded + * or will be downloaded, if necessary, when network access is available). + * + * This state is independent of whether or not the complete set of resources + * is currently available for offline use. To check if that is the case, use + * `OfflineRegionStatus::complete()`. + */ +enum class OfflineRegionDownloadState { + Inactive, + Active +}; + +/* + * A region's status includes its active/inactive state as well as counts + * of the number of resources that have completed downloading, their total + * size in bytes, and the total number of resources that are required. + * + * Note that the total required size in bytes is not currently available. A + * future API release may provide an estimate of this number. + */ +class OfflineRegionStatus { +public: + OfflineRegionDownloadState downloadState = OfflineRegionDownloadState::Inactive; + + /** + * The number of resources that have been fully downloaded and are ready for + * offline access. + */ + uint64_t completedResourceCount = 0; + + /** + * The cumulative size, in bytes, of all resources that have been fully downloaded. + */ + uint64_t completedResourceSize = 0; + + /** + * The number of resources that are known to be required for this region. See the + * documentation for `requiredResourceCountIsIndeterminate` for an important caveat + * about this number. + */ + uint64_t requiredResourceCount = 0; + + /** + * This property is true during early phases of an offline download, when the total + * required resource count is unknown and requiredResourceCount is merely a lower + * bound. + * + * Specifically, it is true before until the style and tile sources have been + * downloaded, and false thereafter. + */ + bool requiredResourceCountIsIndeterminate = true; + + bool complete() const { + return completedResourceCount == requiredResourceCount; + } +}; + +/* + * A region can have a single observer, which gets notified whenever a change + * to the region's status occurs. + */ +class OfflineRegionObserver { +public: + virtual ~OfflineRegionObserver() = default; + + /* + * Implement this method to be notified of a change in the status of an + * offline region. Status changes include any change in state of the members + * of OfflineRegionStatus. + * + * Note that this method will be executed on the database thread; it is the + * responsibility of the SDK bindings to wrap this object in an interface that + * re-executes the user-provided implementation on the main thread. + */ + virtual void statusChanged(OfflineRegionStatus) {} + + /* + * Implement this method to be notified of errors encountered while downloading + * regional resources. Such errors may be recoverable; for example the implementation + * will attempt to re-request failed resources based on an exponential backoff + * algorithm, or when it detects that network access has been restored. + * + * Note that this method will be executed on the database thread; it is the + * responsibility of the SDK bindings to wrap this object in an interface that + * re-executes the user-provided implementation on the main thread. + */ + virtual void responseError(Response::Error) {} +}; + +class OfflineRegion { +public: + // Move-only; not publicly constructible. + OfflineRegion(OfflineRegion&&); + OfflineRegion& operator=(OfflineRegion&&); + ~OfflineRegion(); + + OfflineRegion() = delete; + OfflineRegion(const OfflineRegion&) = delete; + OfflineRegion& operator=(const OfflineRegion&) = delete; + + int64_t getID() const; + const OfflineRegionDefinition& getDefinition() const; + const OfflineRegionMetadata& getMetadata() const; + +private: + friend class OfflineDatabase; + + OfflineRegion(int64_t id, + const OfflineRegionDefinition&, + const OfflineRegionMetadata&); + + const int64_t id; + const OfflineRegionDefinition definition; + const OfflineRegionMetadata metadata; +}; + +} // namespace mbgl diff --git a/platform/default/default_file_source.cpp b/platform/default/default_file_source.cpp index efe893d49b..5bb171f7c2 100644 --- a/platform/default/default_file_source.cpp +++ b/platform/default/default_file_source.cpp @@ -2,6 +2,7 @@ #include <mbgl/storage/asset_file_source.hpp> #include <mbgl/storage/online_file_source.hpp> #include <mbgl/storage/offline_database.hpp> +#include <mbgl/storage/offline_download.hpp> #include <mbgl/platform/platform.hpp> #include <mbgl/util/url.hpp> @@ -61,6 +62,58 @@ public: return onlineFileSource.getAccessToken(); } + void listRegions(std::function<void (std::exception_ptr, optional<std::vector<OfflineRegion>>)> callback) { + try { + callback({}, offlineDatabase.listRegions()); + } catch (...) { + callback(std::current_exception(), {}); + } + } + + void createRegion(const OfflineRegionDefinition& definition, + const OfflineRegionMetadata& metadata, + std::function<void (std::exception_ptr, optional<OfflineRegion>)> callback) { + try { + callback({}, offlineDatabase.createRegion(definition, metadata)); + } catch (...) { + callback(std::current_exception(), {}); + } + } + + void getRegionStatus(int64_t regionID, std::function<void (std::exception_ptr, optional<OfflineRegionStatus>)> callback) { + try { + callback({}, getDownload(regionID).getStatus()); + } catch (...) { + callback(std::current_exception(), {}); + } + } + + void deleteRegion(OfflineRegion&& region, std::function<void (std::exception_ptr)> callback) { + try { + offlineDatabase.deleteRegion(std::move(region)); + callback({}); + } catch (...) { + callback(std::current_exception()); + } + } + + void removeUnusedOfflineResources(std::function<void (std::exception_ptr)> callback) { + try { + offlineDatabase.removeUnusedResources(); + callback({}); + } catch (...) { + callback(std::current_exception()); + } + } + + void setRegionObserver(int64_t regionID, std::unique_ptr<OfflineRegionObserver> observer) { + getDownload(regionID).setObserver(std::move(observer)); + } + + void setRegionDownloadState(int64_t regionID, OfflineRegionDownloadState state) { + getDownload(regionID).setState(state); + } + void add(FileRequest* req, Resource resource, Callback callback) { tasks[req] = std::make_unique<Task>(resource, callback, this); } @@ -77,15 +130,26 @@ public: offline = true; } +private: + OfflineDownload& getDownload(int64_t regionID) { + auto it = downloads.find(regionID); + if (it != downloads.end()) { + return *it->second; + } + return *downloads.emplace(regionID, + std::make_unique<OfflineDownload>(regionID, offlineDatabase.getRegionDefinition(regionID), offlineDatabase, onlineFileSource)).first->second; + } + OfflineDatabase offlineDatabase; OnlineFileSource onlineFileSource; std::unordered_map<FileRequest*, std::unique_ptr<Task>> tasks; + std::unordered_map<int64_t, std::unique_ptr<OfflineDownload>> downloads; bool offline = false; }; class DefaultFileRequest : public FileRequest { public: - DefaultFileRequest(Resource resource, FileSource::Callback callback, util::Thread<DefaultFileSource::Impl>& thread_) + DefaultFileRequest(Resource resource, FileSource::Callback callback, util::Thread<DefaultFileSource::Impl>& thread_) : thread(thread_), workRequest(thread.invokeWithCallback(&DefaultFileSource::Impl::add, callback, this, resource)) { } @@ -129,6 +193,38 @@ std::unique_ptr<FileRequest> DefaultFileSource::request(const Resource& resource } } +void DefaultFileSource::listOfflineRegions(std::function<void (std::exception_ptr, optional<std::vector<OfflineRegion>>)> callback) { + thread->invoke(&Impl::listRegions, callback); +} + +void DefaultFileSource::createOfflineRegion(const OfflineRegionDefinition& definition, + const OfflineRegionMetadata& metadata, + std::function<void (std::exception_ptr, optional<OfflineRegion>)> callback) { + thread->invoke(&Impl::createRegion, definition, metadata, callback); +} + +void DefaultFileSource::deleteOfflineRegion(OfflineRegion&& region, std::function<void (std::exception_ptr)> callback) { + thread->invoke(&Impl::deleteRegion, std::move(region), callback); +} + +void DefaultFileSource::setOfflineRegionObserver(OfflineRegion& region, std::unique_ptr<OfflineRegionObserver> observer) { + thread->invoke(&Impl::setRegionObserver, region.getID(), std::move(observer)); +} + +void DefaultFileSource::setOfflineRegionDownloadState(OfflineRegion& region, OfflineRegionDownloadState state) { + thread->invoke(&Impl::setRegionDownloadState, region.getID(), state); +} + +void DefaultFileSource::getOfflineRegionStatus(OfflineRegion& region, std::function<void (std::exception_ptr, optional<OfflineRegionStatus>)> callback) const { + thread->invoke(&Impl::getRegionStatus, region.getID(), callback); +} + +void DefaultFileSource::removeUnusedOfflineResources(std::function<void (std::exception_ptr)> callback) { + thread->invoke(&Impl::removeUnusedOfflineResources, callback); +} + +// For testing only: + void DefaultFileSource::put(const Resource& resource, const Response& response) { thread->invokeSync(&Impl::put, resource, response); } diff --git a/platform/default/mbgl/storage/offline.cpp b/platform/default/mbgl/storage/offline.cpp new file mode 100644 index 0000000000..7311474bcf --- /dev/null +++ b/platform/default/mbgl/storage/offline.cpp @@ -0,0 +1,122 @@ +#include <mbgl/storage/offline.hpp> +#include <mbgl/util/tile_cover.hpp> +#include <mbgl/map/source_info.hpp> + +#include <rapidjson/document.h> +#include <rapidjson/stringbuffer.h> +#include <rapidjson/writer.h> + +#include <cmath> + +namespace mbgl { + +OfflineTilePyramidRegionDefinition::OfflineTilePyramidRegionDefinition( + const std::string& styleURL_, const LatLngBounds& bounds_, double minZoom_, double maxZoom_, float pixelRatio_) + : styleURL(styleURL_), + bounds(bounds_), + minZoom(minZoom_), + maxZoom(maxZoom_), + pixelRatio(pixelRatio_) { + if (minZoom < 0 || maxZoom < 0 || maxZoom < minZoom || pixelRatio < 0 || + !std::isfinite(minZoom) || std::isnan(maxZoom) || !std::isfinite(pixelRatio)) { + throw std::invalid_argument("Invalid offline region definition"); + } +} + +std::vector<TileID> OfflineTilePyramidRegionDefinition::tileCover(SourceType type, uint16_t tileSize, const SourceInfo& info) const { + double minZ = std::max<double>(coveringZoomLevel(minZoom, type, tileSize), info.minZoom); + double maxZ = std::min<double>(coveringZoomLevel(maxZoom, type, tileSize), info.maxZoom); + + assert(minZ >= 0); + assert(maxZ >= 0); + assert(minZ < std::numeric_limits<uint8_t>::max()); + assert(maxZ < std::numeric_limits<uint8_t>::max()); + + std::vector<TileID> result; + + for (uint8_t z = minZ; z <= maxZ; z++) { + for (const auto& tile : mbgl::tileCover(bounds, z, z)) { + result.push_back(tile.normalized()); + } + } + + return result; +} + +OfflineRegionDefinition decodeOfflineRegionDefinition(const std::string& region) { + rapidjson::GenericDocument<rapidjson::UTF8<>, rapidjson::CrtAllocator> doc; + doc.Parse<0>(region.c_str()); + + if (doc.HasParseError() || + !doc.HasMember("style_url") || !doc["style_url"].IsString() || + !doc.HasMember("bounds") || !doc["bounds"].IsArray() || doc["bounds"].Size() != 4 || + !doc["bounds"][0].IsDouble() || !doc["bounds"][1].IsDouble() || + !doc["bounds"][2].IsDouble() || !doc["bounds"][3].IsDouble() || + !doc.HasMember("min_zoom") || !doc["min_zoom"].IsDouble() || + (doc.HasMember("max_zoom") && !doc["max_zoom"].IsDouble()) || + !doc.HasMember("pixel_ratio") || !doc["pixel_ratio"].IsDouble()) { + throw std::runtime_error("Malformed offline region definition"); + } + + std::string styleURL { doc["style_url"].GetString(), doc["style_url"].GetStringLength() }; + LatLngBounds bounds = LatLngBounds::hull( + LatLng(doc["bounds"][0].GetDouble(), doc["bounds"][1].GetDouble()), + LatLng(doc["bounds"][2].GetDouble(), doc["bounds"][3].GetDouble())); + double minZoom = doc["min_zoom"].GetDouble(); + double maxZoom = doc.HasMember("max_zoom") ? doc["max_zoom"].GetDouble() : INFINITY; + float pixelRatio = doc["pixel_ratio"].GetDouble(); + + return { styleURL, bounds, minZoom, maxZoom, pixelRatio }; +} + +std::string encodeOfflineRegionDefinition(const OfflineRegionDefinition& region) { + rapidjson::GenericDocument<rapidjson::UTF8<>, rapidjson::CrtAllocator> doc; + doc.SetObject(); + + doc.AddMember("style_url", rapidjson::StringRef(region.styleURL.data(), region.styleURL.length()), doc.GetAllocator()); + + rapidjson::GenericValue<rapidjson::UTF8<>, rapidjson::CrtAllocator> bounds(rapidjson::kArrayType); + bounds.PushBack(region.bounds.south(), doc.GetAllocator()); + bounds.PushBack(region.bounds.west(), doc.GetAllocator()); + bounds.PushBack(region.bounds.north(), doc.GetAllocator()); + bounds.PushBack(region.bounds.east(), doc.GetAllocator()); + doc.AddMember("bounds", bounds, doc.GetAllocator()); + + doc.AddMember("min_zoom", region.minZoom, doc.GetAllocator()); + if (std::isfinite(region.maxZoom)) { + doc.AddMember("max_zoom", region.maxZoom, doc.GetAllocator()); + } + + doc.AddMember("pixel_ratio", region.pixelRatio, doc.GetAllocator()); + + rapidjson::StringBuffer buffer; + rapidjson::Writer<rapidjson::StringBuffer> writer(buffer); + doc.Accept(writer); + + return buffer.GetString(); +} + +OfflineRegion::OfflineRegion(int64_t id_, + const OfflineRegionDefinition& definition_, + const OfflineRegionMetadata& metadata_) + : id(id_), + definition(definition_), + metadata(metadata_) { +} + +OfflineRegion::OfflineRegion(OfflineRegion&&) = default; +OfflineRegion::~OfflineRegion() = default; + +const OfflineRegionDefinition& OfflineRegion::getDefinition() const { + return definition; +} + +const OfflineRegionMetadata& OfflineRegion::getMetadata() const { + return metadata; +} + +int64_t OfflineRegion::getID() const { + return id; +} + +} // namespace mbgl diff --git a/platform/default/mbgl/storage/offline_database.cpp b/platform/default/mbgl/storage/offline_database.cpp index 67fcc435c6..c6510157bc 100644 --- a/platform/default/mbgl/storage/offline_database.cpp +++ b/platform/default/mbgl/storage/offline_database.cpp @@ -100,17 +100,17 @@ optional<Response> OfflineDatabase::get(const Resource& resource) { } } -void OfflineDatabase::put(const Resource& resource, const Response& response) { +uint64_t OfflineDatabase::put(const Resource& resource, const Response& response) { // Don't store errors in the cache. if (response.error) { - return; + return 0; } if (resource.kind == Resource::Kind::Tile) { assert(resource.tileData); - putTile(*resource.tileData, response); + return putTile(*resource.tileData, response); } else { - putResource(resource, response); + return putResource(resource, response); } } @@ -145,7 +145,7 @@ optional<Response> OfflineDatabase::getResource(const Resource& resource) { return response; } -void OfflineDatabase::putResource(const Resource& resource, const Response& response) { +uint64_t OfflineDatabase::putResource(const Resource& resource, const Response& response) { if (response.notModified) { mapbox::sqlite::Statement& stmt = getStatement( // 1 2 3 @@ -155,6 +155,7 @@ void OfflineDatabase::putResource(const Resource& resource, const Response& resp stmt.bind(2, response.expires); stmt.bind(3, resource.url); stmt.run(); + return 0; } else { mapbox::sqlite::Statement& stmt = getStatement( // 1 2 3 4 5 6 7 8 @@ -169,6 +170,7 @@ void OfflineDatabase::putResource(const Resource& resource, const Response& resp stmt.bind(6 /* accessed */, SystemClock::now()); std::string data; + uint64_t size = 0; if (response.noContent) { stmt.bind(7 /* data */, nullptr); @@ -176,15 +178,18 @@ void OfflineDatabase::putResource(const Resource& resource, const Response& resp } else { data = util::compress(*response.data); if (data.size() < response.data->size()) { - stmt.bind(7 /* data */, data, false); // do not retain the string internally. + size = data.size(); + stmt.bindBlob(7 /* data */, data.data(), size, false); stmt.bind(8 /* compressed */, true); } else { - stmt.bind(7 /* data */, *response.data, false); // do not retain the string internally. + size = response.data->size(); + stmt.bindBlob(7 /* data */, response.data->data(), size, false); stmt.bind(8 /* compressed */, false); } } stmt.run(); + return size; } } @@ -228,7 +233,7 @@ optional<Response> OfflineDatabase::getTile(const Resource::TileData& tile) { return response; } -void OfflineDatabase::putTile(const Resource::TileData& tile, const Response& response) { +uint64_t OfflineDatabase::putTile(const Resource::TileData& tile, const Response& response) { if (response.notModified) { mapbox::sqlite::Statement& stmt = getStatement( "UPDATE tiles SET accessed = ?1, expires = ?2 " @@ -248,6 +253,7 @@ void OfflineDatabase::putTile(const Resource::TileData& tile, const Response& re stmt.bind(6, tile.y); stmt.bind(7, tile.z); stmt.run(); + return 0; } else { mapbox::sqlite::Statement& stmt1 = getStatement( "REPLACE INTO tilesets (url_template, pixel_ratio) " @@ -275,6 +281,7 @@ void OfflineDatabase::putTile(const Resource::TileData& tile, const Response& re stmt2.bind(9 /* accessed */, SystemClock::now()); std::string data; + uint64_t size = 0; if (response.noContent) { stmt2.bind(10 /* data */, nullptr); @@ -282,16 +289,160 @@ void OfflineDatabase::putTile(const Resource::TileData& tile, const Response& re } else { data = util::compress(*response.data); if (data.size() < response.data->size()) { - stmt2.bind(10 /* data */, data, false); // do not retain the string internally. + size = data.size(); + stmt2.bindBlob(10 /* data */, data.data(), size, false); stmt2.bind(11 /* compressed */, true); } else { - stmt2.bind(10 /* data */, *response.data, false); // do not retain the string internally. + size = response.data->size(); + stmt2.bindBlob(10 /* data */, response.data->data(), size, false); stmt2.bind(11 /* compressed */, false); } } stmt2.run(); + return size; } } +std::vector<OfflineRegion> OfflineDatabase::listRegions() { + mapbox::sqlite::Statement& stmt = getStatement( + "SELECT id, definition, description FROM regions"); + + std::vector<OfflineRegion> result; + + while (stmt.run()) { + result.push_back(OfflineRegion( + stmt.get<int64_t>(0), + decodeOfflineRegionDefinition(stmt.get<std::string>(1)), + stmt.get<std::vector<uint8_t>>(2))); + } + + return std::move(result); +} + +OfflineRegion OfflineDatabase::createRegion(const OfflineRegionDefinition& definition, + const OfflineRegionMetadata& metadata) { + mapbox::sqlite::Statement& stmt = getStatement( + "INSERT INTO regions (definition, description) " + "VALUES (?1, ?2) "); + + stmt.bind(1, encodeOfflineRegionDefinition(definition)); + stmt.bindBlob(2, metadata); + stmt.run(); + + return OfflineRegion(db->lastInsertRowid(), definition, metadata); +} + +void OfflineDatabase::deleteRegion(OfflineRegion&& region) { + mapbox::sqlite::Statement& stmt = getStatement( + "DELETE FROM regions WHERE id = ?"); + + stmt.bind(1, region.getID()); + stmt.run(); +} + +optional<Response> OfflineDatabase::getRegionResource(int64_t regionID, const Resource& resource) { + auto response = get(resource); + + if (response) { + markUsed(regionID, resource); + } + + return response; +} + +uint64_t OfflineDatabase::putRegionResource(int64_t regionID, const Resource& resource, const Response& response) { + uint64_t result = put(resource, response); + markUsed(regionID, resource); + return result; +} + +void OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) { + if (resource.kind == Resource::Kind::Tile) { + mapbox::sqlite::Statement& stmt1 = getStatement( + "REPLACE INTO region_tiles (region_id, tileset_id, x, y, z) " + "SELECT ?1, tilesets.id, ?4, ?5, ?6 " + "FROM tilesets " + "WHERE url_template = ?2 " + "AND pixel_ratio = ?3 "); + + stmt1.bind(1, regionID); + stmt1.bind(2, (*resource.tileData).urlTemplate); + stmt1.bind(3, (*resource.tileData).pixelRatio); + stmt1.bind(4, (*resource.tileData).x); + stmt1.bind(5, (*resource.tileData).y); + stmt1.bind(6, (*resource.tileData).z); + stmt1.run(); + } else { + mapbox::sqlite::Statement& stmt1 = getStatement( + "REPLACE INTO region_resources (region_id, resource_url) " + "VALUES (?1, ?2) "); + + stmt1.bind(1, regionID); + stmt1.bind(2, resource.url); + stmt1.run(); + } +} + +OfflineRegionDefinition OfflineDatabase::getRegionDefinition(int64_t regionID) { + mapbox::sqlite::Statement& stmt = getStatement( + "SELECT definition FROM regions WHERE id = ?1"); + + stmt.bind(1, regionID); + stmt.run(); + + return decodeOfflineRegionDefinition(stmt.get<std::string>(0)); +} + +OfflineRegionStatus OfflineDatabase::getRegionCompletedStatus(int64_t regionID) { + OfflineRegionStatus result; + + mapbox::sqlite::Statement& stmt = getStatement( + "SELECT COUNT(*), SUM(size) FROM ( " + " SELECT LENGTH(data) as size " + " FROM region_resources, resources " + " WHERE region_id = ?1 " + " AND resources.url = region_resources.resource_url " + " UNION ALL " + " SELECT LENGTH(data) as size " + " FROM region_tiles, tiles " + " WHERE region_id = ?1 " + " AND tiles.tileset_id = region_tiles.tileset_id " + " AND tiles.z = region_tiles.z " + " AND tiles.x = region_tiles.x " + " AND tiles.y = region_tiles.y " + ") "); + + stmt.bind(1, regionID); + stmt.run(); + + result.completedResourceCount = stmt.get<int64_t>(0); + result.completedResourceSize = stmt.get<int64_t>(1); + + return result; +} + +void OfflineDatabase::removeUnusedResources() { + mapbox::sqlite::Statement& stmt1 = getStatement( + "DELETE FROM resources " + "WHERE ROWID NOT IN ( " + " SELECT resources.ROWID " + " FROM resources, region_resources " + " WHERE resources.url = region_resources.resource_url " + ") "); + stmt1.run(); + + mapbox::sqlite::Statement& stmt2 = getStatement( + "DELETE FROM tiles " + "WHERE ROWID NOT IN ( " + " SELECT tiles.ROWID " + " FROM tiles, region_tiles " + " AND tiles.tileset_id = region_tiles.tileset_id " + " AND tiles.z = region_tiles.z " + " AND tiles.x = region_tiles.x " + " AND tiles.y = region_tiles.y " + ") "); + stmt2.run(); +} + } // namespace mbgl diff --git a/platform/default/mbgl/storage/offline_database.hpp b/platform/default/mbgl/storage/offline_database.hpp index bc6f784d50..554bb16068 100644 --- a/platform/default/mbgl/storage/offline_database.hpp +++ b/platform/default/mbgl/storage/offline_database.hpp @@ -2,6 +2,7 @@ #define MBGL_OFFLINE_DATABASE #include <mbgl/storage/resource.hpp> +#include <mbgl/storage/offline.hpp> #include <mbgl/util/noncopyable.hpp> #include <mbgl/util/optional.hpp> @@ -27,7 +28,21 @@ public: ~OfflineDatabase(); optional<Response> get(const Resource&); - void put(const Resource&, const Response&); + uint64_t put(const Resource&, const Response&); + + std::vector<OfflineRegion> listRegions(); + + OfflineRegion createRegion(const OfflineRegionDefinition&, + const OfflineRegionMetadata&); + + void deleteRegion(OfflineRegion&&); + void removeUnusedResources(); + + optional<Response> getRegionResource(int64_t regionID, const Resource&); + uint64_t putRegionResource(int64_t regionID, const Resource&, const Response&); + + OfflineRegionDefinition getRegionDefinition(int64_t regionID); + OfflineRegionStatus getRegionCompletedStatus(int64_t regionID); private: void ensureSchema(); @@ -35,10 +50,12 @@ private: mapbox::sqlite::Statement& getStatement(const char *); optional<Response> getTile(const Resource::TileData&); - void putTile(const Resource::TileData&, const Response&); + uint64_t putTile(const Resource::TileData&, const Response&); optional<Response> getResource(const Resource&); - void putResource(const Resource&, const Response&); + uint64_t putResource(const Resource&, const Response&); + + void markUsed(int64_t regionID, const Resource&); const std::string path; std::unique_ptr<::mapbox::sqlite::Database> db; diff --git a/platform/default/mbgl/storage/offline_download.cpp b/platform/default/mbgl/storage/offline_download.cpp new file mode 100644 index 0000000000..1559895be2 --- /dev/null +++ b/platform/default/mbgl/storage/offline_download.cpp @@ -0,0 +1,241 @@ +#include <mbgl/storage/offline_download.hpp> +#include <mbgl/storage/offline_database.hpp> +#include <mbgl/storage/file_source.hpp> +#include <mbgl/storage/resource.hpp> +#include <mbgl/storage/response.hpp> +#include <mbgl/style/style_parser.hpp> +#include <mbgl/layer/symbol_layer.hpp> +#include <mbgl/text/glyph.hpp> +#include <mbgl/util/tile_cover.hpp> + +#include <set> + +namespace mbgl { + +OfflineDownload::OfflineDownload(int64_t id_, + OfflineRegionDefinition&& definition_, + OfflineDatabase& offlineDatabase_, + FileSource& onlineFileSource_) + : id(id_), + definition(definition_), + offlineDatabase(offlineDatabase_), + onlineFileSource(onlineFileSource_) { + setObserver(nullptr); +} + +OfflineDownload::~OfflineDownload() = default; + +void OfflineDownload::setObserver(std::unique_ptr<OfflineRegionObserver> observer_) { + observer = observer_ ? std::move(observer_) : std::make_unique<OfflineRegionObserver>(); +} + +void OfflineDownload::setState(OfflineRegionDownloadState state) { + if (status.downloadState == state) { + return; + } + + status.downloadState = state; + + if (status.downloadState == OfflineRegionDownloadState::Active) { + activateDownload(); + } else { + deactivateDownload(); + } +} + +std::vector<Resource> OfflineDownload::spriteResources(const StyleParser& parser) const { + std::vector<Resource> result; + + if (!parser.spriteURL.empty()) { + result.push_back(Resource::spriteImage(parser.spriteURL, definition.pixelRatio)); + result.push_back(Resource::spriteJSON(parser.spriteURL, definition.pixelRatio)); + } + + return result; +} + +std::vector<Resource> OfflineDownload::glyphResources(const StyleParser& parser) const { + std::vector<Resource> result; + + if (!parser.glyphURL.empty()) { + for (const auto& fontStack : parser.fontStacks()) { + for (uint32_t i = 0; i < 256; i++) { + result.push_back(Resource::glyphs(parser.glyphURL, fontStack, getGlyphRange(i * 256))); + } + } + } + + return result; +} + +std::vector<Resource> OfflineDownload::tileResources(SourceType type, uint16_t tileSize, const SourceInfo& info) const { + std::vector<Resource> result; + + for (const auto& tile : definition.tileCover(type, tileSize, info)) { + result.push_back(Resource::tile(info.tiles[0], definition.pixelRatio, tile.x, tile.y, tile.z)); + } + + return result; +} + +OfflineRegionStatus OfflineDownload::getStatus() const { + if (status.downloadState == OfflineRegionDownloadState::Active) { + return status; + } + + OfflineRegionStatus result = offlineDatabase.getRegionCompletedStatus(id); + + result.requiredResourceCount++; + optional<Response> styleResponse = offlineDatabase.get(Resource::style(definition.styleURL)); + if (!styleResponse) { + return result; + } + + StyleParser parser; + parser.parse(*styleResponse->data); + + result.requiredResourceCountIsIndeterminate = false; + + for (const auto& source : parser.sources) { + switch (source->type) { + case SourceType::Vector: + case SourceType::Raster: + if (source->getInfo()) { + result.requiredResourceCount += tileResources(source->type, source->tileSize, *source->getInfo()).size(); + } else { + result.requiredResourceCount += 1; + optional<Response> sourceResponse = offlineDatabase.get(Resource::source(source->url)); + if (sourceResponse) { + result.requiredResourceCount += tileResources(source->type, source->tileSize, + *StyleParser::parseTileJSON(*sourceResponse->data, source->url, source->type)).size(); + } else { + result.requiredResourceCountIsIndeterminate = true; + } + } + break; + + case SourceType::GeoJSON: + if (!source->url.empty()) { + result.requiredResourceCount += 1; + } + break; + + case SourceType::Video: + case SourceType::Annotations: + break; + } + } + + result.requiredResourceCount += spriteResources(parser).size(); + result.requiredResourceCount += glyphResources(parser).size(); + + return result; +} + +void OfflineDownload::activateDownload() { + status = offlineDatabase.getRegionCompletedStatus(id); + requiredSourceURLs.clear(); + + ensureResource(Resource::style(definition.styleURL), [&] (Response styleResponse) { + status.requiredResourceCountIsIndeterminate = false; + + StyleParser parser; + parser.parse(*styleResponse.data); + + for (const auto& source : parser.sources) { + SourceType type = source->type; + uint16_t tileSize = source->tileSize; + std::string url = source->url; + + switch (type) { + case SourceType::Vector: + case SourceType::Raster: + if (source->getInfo()) { + ensureTiles(type, tileSize, *source->getInfo()); + } else { + status.requiredResourceCountIsIndeterminate = true; + requiredSourceURLs.insert(url); + + ensureResource(Resource::source(url), [=] (Response sourceResponse) { + ensureTiles(type, tileSize, *StyleParser::parseTileJSON(*sourceResponse.data, url, type)); + + requiredSourceURLs.erase(url); + if (requiredSourceURLs.empty()) { + status.requiredResourceCountIsIndeterminate = false; + } + }); + } + break; + + case SourceType::GeoJSON: + if (!source->url.empty()) { + ensureResource(Resource::source(source->url)); + } + break; + + case SourceType::Video: + case SourceType::Annotations: + break; + } + } + + for (const auto& resource : spriteResources(parser)) { + ensureResource(resource); + } + + for (const auto& resource : glyphResources(parser)) { + ensureResource(resource); + } + }); + + // This will be the initial notification, after we've incremented requiredResourceCount + // to the reflect the extent to which required resources are already in the database. + observer->statusChanged(status); +} + +void OfflineDownload::deactivateDownload() { + requests.clear(); +} + +void OfflineDownload::ensureTiles(SourceType type, uint16_t tileSize, const SourceInfo& info) { + for (const auto& resource : tileResources(type, tileSize, info)) { + ensureResource(resource); + } +} + +void OfflineDownload::ensureResource(const Resource& resource, std::function<void (Response)> callback) { + status.requiredResourceCount++; + + optional<Response> offlineResponse = offlineDatabase.getRegionResource(id, resource); + if (offlineResponse) { + if (callback) { + callback(*offlineResponse); + } + + // Not incrementing status.completedResource{Size,Count} here because previously-existing + // resources are already accounted for by offlineDatabase.getRegionCompletedStatus(); + + return; + } + + auto it = requests.insert(requests.begin(), nullptr); + *it = onlineFileSource.request(resource, [=] (Response onlineResponse) { + requests.erase(it); + + if (onlineResponse.error) { + observer->responseError(*onlineResponse.error); + return; + } + + if (callback) { + callback(onlineResponse); + } + + status.completedResourceCount++; + status.completedResourceSize += offlineDatabase.putRegionResource(id, resource, onlineResponse); + + observer->statusChanged(status); + }); +} + +} // namespace mbgl diff --git a/platform/default/mbgl/storage/offline_download.hpp b/platform/default/mbgl/storage/offline_download.hpp new file mode 100644 index 0000000000..4200020487 --- /dev/null +++ b/platform/default/mbgl/storage/offline_download.hpp @@ -0,0 +1,62 @@ + #pragma once + +#include <mbgl/storage/offline.hpp> +#include <mbgl/style/types.hpp> + +#include <list> +#include <set> +#include <memory> + +namespace mbgl { + +class OfflineDatabase; +class FileSource; +class FileRequest; +class Resource; +class Response; +class SourceInfo; +class StyleParser; +class Source; + +/** + * Coordinates the request and storage of all resources for an offline region. + + * @private + */ +class OfflineDownload { +public: + OfflineDownload(int64_t id, OfflineRegionDefinition&&, OfflineDatabase& offline, FileSource& online); + ~OfflineDownload(); + + void setObserver(std::unique_ptr<OfflineRegionObserver>); + void setState(OfflineRegionDownloadState); + + OfflineRegionStatus getStatus() const; + +private: + void activateDownload(); + void deactivateDownload(); + + std::vector<Resource> spriteResources(const StyleParser&) const; + std::vector<Resource> glyphResources(const StyleParser&) const; + std::vector<Resource> tileResources(SourceType, uint16_t, const SourceInfo&) const; + + /* + * Ensure that the resource is stored in the database, requesting it if necessary. + * While the request is in progress, it is recorded in `requests`. If the download + * is deactivated, all in progress requests are cancelled. + */ + void ensureResource(const Resource&, std::function<void (Response)> = {}); + void ensureTiles(SourceType, uint16_t, const SourceInfo&); + + int64_t id; + OfflineRegionDefinition definition; + OfflineDatabase& offlineDatabase; + FileSource& onlineFileSource; + OfflineRegionStatus status; + std::unique_ptr<OfflineRegionObserver> observer; + std::list<std::unique_ptr<FileRequest>> requests; + std::set<std::string> requiredSourceURLs; +}; + +} // namespace mbgl diff --git a/platform/default/sqlite3.cpp b/platform/default/sqlite3.cpp index 09301bc4d9..2cc0e9f001 100644 --- a/platform/default/sqlite3.cpp +++ b/platform/default/sqlite3.cpp @@ -77,6 +77,11 @@ Statement Database::prepare(const char *query) { return Statement(db, query); } +int64_t Database::lastInsertRowid() const { + assert(db); + return sqlite3_last_insert_rowid(db); +} + Statement::Statement(sqlite3 *db, const char *sql) { const int err = sqlite3_prepare_v2(db, sql, -1, &stmt, nullptr); if (err != SQLITE_OK) { @@ -170,12 +175,39 @@ template <> void Statement::bind(int offset, const char *value) { check(sqlite3_bind_text(stmt, offset, value, -1, SQLITE_STATIC)); } +// We currently cannot use sqlite3_bind_blob64 / sqlite3_bind_text64 because they +// was introduced in SQLite 3.8.7, and we need to support earlier versions: +// iOS 7.0: 3.7.13 +// iOS 8.2: 3.8.5 +// According to http://stackoverflow.com/questions/14288128/what-version-of-sqlite-does-ios-provide, +// the first iOS version with 3.8.7+ was 9.0, with 3.8.10.2. + +void Statement::bind(int offset, const char * value, std::size_t length, bool retain) { + assert(stmt); + if (length > std::numeric_limits<int>::max()) { + throw std::range_error("value too long for sqlite3_bind_text"); + } + check(sqlite3_bind_text(stmt, offset, value, int(length), + retain ? SQLITE_TRANSIENT : SQLITE_STATIC)); +} + void Statement::bind(int offset, const std::string& value, bool retain) { + bind(offset, value.data(), value.size(), retain); +} + +void Statement::bindBlob(int offset, const void * value, std::size_t length, bool retain) { assert(stmt); - check(sqlite3_bind_blob(stmt, offset, value.data(), int(value.size()), + if (length > std::numeric_limits<int>::max()) { + throw std::range_error("value too long for sqlite3_bind_text"); + } + check(sqlite3_bind_blob(stmt, offset, value, int(length), retain ? SQLITE_TRANSIENT : SQLITE_STATIC)); } +void Statement::bindBlob(int offset, const std::vector<uint8_t>& value, bool retain) { + bindBlob(offset, value.data(), value.size(), retain); +} + template <> void Statement::bind(int offset, std::chrono::system_clock::time_point value) { assert(stmt); check(sqlite3_bind_int64(stmt, offset, std::chrono::system_clock::to_time_t(value))); @@ -234,6 +266,13 @@ template <> std::string Statement::get(int offset) { }; } +template <> std::vector<uint8_t> Statement::get(int offset) { + assert(stmt); + const uint8_t* begin = reinterpret_cast<const uint8_t*>(sqlite3_column_blob(stmt, offset)); + const uint8_t* end = begin + sqlite3_column_bytes(stmt, offset); + return { begin, end }; +} + template <> std::chrono::system_clock::time_point Statement::get(int offset) { assert(stmt); return std::chrono::system_clock::from_time_t(sqlite3_column_int64(stmt, offset)); diff --git a/platform/default/sqlite3.hpp b/platform/default/sqlite3.hpp index 29e8967db3..cdfd5dc8de 100644 --- a/platform/default/sqlite3.hpp +++ b/platform/default/sqlite3.hpp @@ -1,6 +1,7 @@ #pragma once #include <string> +#include <vector> #include <stdexcept> typedef struct sqlite3 sqlite3; @@ -43,6 +44,8 @@ public: void exec(const std::string &sql); Statement prepare(const char *query); + int64_t lastInsertRowid() const; + private: sqlite3 *db = nullptr; }; @@ -63,7 +66,15 @@ public: operator bool() const; template <typename T> void bind(int offset, T value); - void bind(int offset, const std::string &value, bool retain = true); + + // Text + void bind(int offset, const char *, std::size_t length, bool retain = true); + void bind(int offset, const std::string&, bool retain = true); + + // Blob + void bindBlob(int offset, const void *, std::size_t length, bool retain = true); + void bindBlob(int offset, const std::vector<uint8_t>&, bool retain = true); + template <typename T> T get(int offset); bool run(); diff --git a/src/mbgl/map/source.hpp b/src/mbgl/map/source.hpp index 7f69fc2881..d3f4abbca8 100644 --- a/src/mbgl/map/source.hpp +++ b/src/mbgl/map/source.hpp @@ -53,6 +53,8 @@ public: bool isLoading() const; bool isLoaded() const; + const SourceInfo* getInfo() const { return info.get(); } + // Request or parse all the tiles relevant for the "TransformState". This method // will return true if all the tiles were scheduled for updating of false if // they were not. shouldReparsePartialTiles must be set to "true" if there is diff --git a/test/fixtures/offline/empty.style.json b/test/fixtures/offline/empty.style.json new file mode 100644 index 0000000000..61a8fadcdb --- /dev/null +++ b/test/fixtures/offline/empty.style.json @@ -0,0 +1,5 @@ +{ + "version": 8, + "sources": {}, + "layers": [] +} diff --git a/test/fixtures/offline/geojson.json b/test/fixtures/offline/geojson.json new file mode 100644 index 0000000000..8b3698faf7 --- /dev/null +++ b/test/fixtures/offline/geojson.json @@ -0,0 +1,4 @@ +{ + "type": "FeatureCollection", + "features": [] +} diff --git a/test/fixtures/offline/geojson_source.style.json b/test/fixtures/offline/geojson_source.style.json new file mode 100644 index 0000000000..511fca9fd0 --- /dev/null +++ b/test/fixtures/offline/geojson_source.style.json @@ -0,0 +1,10 @@ +{ + "version": 8, + "sources": { + "geojson": { + "type": "geojson", + "data": "http://127.0.0.1:3000/offline/geojson.json" + } + }, + "layers": [] +} diff --git a/test/fixtures/offline/inline_source.style.json b/test/fixtures/offline/inline_source.style.json new file mode 100644 index 0000000000..87155d07d8 --- /dev/null +++ b/test/fixtures/offline/inline_source.style.json @@ -0,0 +1,17 @@ +{ + "version": 8, + "sources": { + "inline": { + "type": "vector", + "maxzoom": 15, + "minzoom": 0, + "tiles": [ "http://127.0.0.1:3000/offline/{z}-{x}-{y}.vector.pbf" ] + } + }, + "layers": [{ + "id": "fill", + "type": "fill", + "source": "inline", + "source-layer": "water" + }] +} diff --git a/test/storage/offline.cpp b/test/storage/offline.cpp new file mode 100644 index 0000000000..b34aa02c22 --- /dev/null +++ b/test/storage/offline.cpp @@ -0,0 +1,73 @@ +#include <mbgl/storage/offline.hpp> +#include <mbgl/map/source_info.hpp> +#include <mbgl/map/tile_id.hpp> + +#include <gtest/gtest.h> + +using namespace mbgl; + +static const LatLngBounds sanFrancisco = LatLngBounds::hull( + { 37.6609, -122.5744 }, + { 37.8271, -122.3204 }); + +static const LatLngBounds sanFranciscoWrapped = LatLngBounds::hull( + { 37.6609, 238.5744 }, + { 37.8271, 238.3204 }); + +TEST(OfflineTilePyramidRegionDefinition, TileCoverEmpty) { + OfflineTilePyramidRegionDefinition region("", LatLngBounds::empty(), 0, 20, 1.0); + SourceInfo info; + + auto result = region.tileCover(SourceType::Vector, 512, info); + ASSERT_TRUE(result.empty()); +} + +TEST(OfflineTilePyramidRegionDefinition, TileCoverZoomIntersection) { + OfflineTilePyramidRegionDefinition region("", sanFrancisco, 2, 2, 1.0); + SourceInfo info; + + info.minZoom = 0; + auto resultIntersection = region.tileCover(SourceType::Vector, 512, info); + ASSERT_EQ(1, resultIntersection.size()); + + info.minZoom = 3; + auto resultNoIntersection = region.tileCover(SourceType::Vector, 512, info); + ASSERT_TRUE(resultNoIntersection.empty()); +} + +TEST(OfflineTilePyramidRegionDefinition, TileCoverTileSize) { + OfflineTilePyramidRegionDefinition region("", LatLngBounds::world(), 0, 0, 1.0); + SourceInfo info; + + auto result512 = region.tileCover(SourceType::Vector, 512, info); + ASSERT_EQ(1, result512.size()); + ASSERT_EQ(0, result512[0].z); + + auto result256 = region.tileCover(SourceType::Vector, 256, info); + ASSERT_EQ(4, result256.size()); + ASSERT_EQ(1, result256[0].z); +} + +TEST(OfflineTilePyramidRegionDefinition, TileCoverZoomRounding) { + OfflineTilePyramidRegionDefinition region("", sanFrancisco, 0.6, 0.7, 1.0); + SourceInfo info; + + auto resultVector = region.tileCover(SourceType::Vector, 512, info); + ASSERT_EQ(1, resultVector.size()); + ASSERT_EQ(0, resultVector[0].z); + + auto resultRaster = region.tileCover(SourceType::Raster, 512, info); + ASSERT_EQ(1, resultRaster.size()); + ASSERT_EQ(1, resultRaster[0].z); +} + +TEST(OfflineTilePyramidRegionDefinition, TileCoverWrapped) { + OfflineTilePyramidRegionDefinition region("", sanFranciscoWrapped, 0, 0, 1.0); + SourceInfo info; + + auto result = region.tileCover(SourceType::Vector, 512, info); + ASSERT_EQ(1, result.size()); + ASSERT_EQ(0, result[0].z); + ASSERT_EQ(0, result[0].x); + ASSERT_EQ(0, result[0].y); +} diff --git a/test/storage/offline_database.cpp b/test/storage/offline_database.cpp index 1c43506066..2018f6a40b 100644 --- a/test/storage/offline_database.cpp +++ b/test/storage/offline_database.cpp @@ -372,3 +372,79 @@ TEST(OfflineDatabase, PutTileNotFound) { EXPECT_TRUE(res->noContent); EXPECT_FALSE(res->data.get()); } + +TEST(OfflineDatabase, CreateRegion) { + using namespace mbgl; + + OfflineDatabase db(":memory:"); + OfflineRegionDefinition definition { "http://example.com/style", LatLngBounds::hull({1, 2}, {3, 4}), 5, 6, 2.0 }; + OfflineRegionMetadata metadata {{ 1, 2, 3 }}; + OfflineRegion region = db.createRegion(definition, metadata); + + EXPECT_EQ(definition.styleURL, region.getDefinition().styleURL); + EXPECT_EQ(definition.bounds, region.getDefinition().bounds); + EXPECT_EQ(definition.minZoom, region.getDefinition().minZoom); + EXPECT_EQ(definition.maxZoom, region.getDefinition().maxZoom); + EXPECT_EQ(definition.pixelRatio, region.getDefinition().pixelRatio); + EXPECT_EQ(metadata, region.getMetadata()); +} + +TEST(OfflineDatabase, ListRegions) { + using namespace mbgl; + + OfflineDatabase db(":memory:"); + OfflineRegionDefinition definition { "http://example.com/style", LatLngBounds::hull({1, 2}, {3, 4}), 5, 6, 2.0 }; + OfflineRegionMetadata metadata {{ 1, 2, 3 }}; + + OfflineRegion region = db.createRegion(definition, metadata); + std::vector<OfflineRegion> regions = db.listRegions(); + + ASSERT_EQ(1, regions.size()); + EXPECT_EQ(region.getID(), regions.at(0).getID()); + EXPECT_EQ(definition.styleURL, regions.at(0).getDefinition().styleURL); + EXPECT_EQ(definition.bounds, regions.at(0).getDefinition().bounds); + EXPECT_EQ(definition.minZoom, regions.at(0).getDefinition().minZoom); + EXPECT_EQ(definition.maxZoom, regions.at(0).getDefinition().maxZoom); + EXPECT_EQ(definition.pixelRatio, regions.at(0).getDefinition().pixelRatio); + EXPECT_EQ(metadata, regions.at(0).getMetadata()); +} + +TEST(OfflineDatabase, GetRegionDefinition) { + using namespace mbgl; + + OfflineDatabase db(":memory:"); + OfflineRegionDefinition definition { "http://example.com/style", LatLngBounds::hull({1, 2}, {3, 4}), 5, 6, 2.0 }; + OfflineRegionMetadata metadata {{ 1, 2, 3 }}; + + OfflineRegion region = db.createRegion(definition, metadata); + OfflineRegionDefinition result = db.getRegionDefinition(region.getID()); + + EXPECT_EQ(definition.styleURL, result.styleURL); + EXPECT_EQ(definition.bounds, result.bounds); + EXPECT_EQ(definition.minZoom, result.minZoom); + EXPECT_EQ(definition.maxZoom, result.maxZoom); + EXPECT_EQ(definition.pixelRatio, result.pixelRatio); +} + +TEST(OfflineDatabase, DeleteRegion) { + using namespace mbgl; + + OfflineDatabase db(":memory:"); + OfflineRegionDefinition definition { "http://example.com/style", LatLngBounds::hull({1, 2}, {3, 4}), 5, 6, 2.0 }; + OfflineRegionMetadata metadata {{ 1, 2, 3 }}; + db.deleteRegion(db.createRegion(definition, metadata)); + + ASSERT_EQ(0, db.listRegions().size()); +} + +TEST(OfflineDatabase, CreateRegionInfiniteMaxZoom) { + using namespace mbgl; + + OfflineDatabase db(":memory:"); + OfflineRegionDefinition definition { "", LatLngBounds::world(), 0, INFINITY, 1.0 }; + OfflineRegionMetadata metadata; + OfflineRegion region = db.createRegion(definition, metadata); + + EXPECT_EQ(0, region.getDefinition().minZoom); + EXPECT_EQ(INFINITY, region.getDefinition().maxZoom); +} diff --git a/test/storage/offline_download.cpp b/test/storage/offline_download.cpp new file mode 100644 index 0000000000..d89d4a035c --- /dev/null +++ b/test/storage/offline_download.cpp @@ -0,0 +1,302 @@ +#include "../fixtures/stub_file_source.hpp" + +#include <mbgl/storage/offline.hpp> +#include <mbgl/storage/offline_database.hpp> +#include <mbgl/storage/offline_download.hpp> +#include <mbgl/util/run_loop.hpp> +#include <mbgl/util/io.hpp> +#include <mbgl/util/compression.hpp> +#include <mbgl/util/string.hpp> + +#include <gtest/gtest.h> +#include <iostream> + +using namespace mbgl; +using namespace std::literals::string_literals; + +class MockObserver : public OfflineRegionObserver { +public: + void statusChanged(OfflineRegionStatus status) override { + if (statusChangedFn) statusChangedFn(status); + } + + void responseError(Response::Error error) override { + if (responseErrorFn) responseErrorFn(error); + } + + std::function<void (OfflineRegionStatus)> statusChangedFn; + std::function<void (Response::Error)> responseErrorFn; +}; + +class OfflineTest { +public: + util::RunLoop loop; + StubFileSource fileSource; + OfflineDatabase db { ":memory:" }; + std::size_t size = 0; + + Response response(const std::string& path) { + Response result; + result.data = std::make_shared<std::string>(util::read_file("test/fixtures/"s + path)); + size_t uncompressed = result.data->size(); + size_t compressed = util::compress(*result.data).size(); + size += std::min(uncompressed, compressed); + return result; + } +}; + +TEST(OfflineDownload, NoSubresources) { + OfflineTest test; + OfflineDownload download( + 1, + OfflineTilePyramidRegionDefinition("http://127.0.0.1:3000/offline/style.json", LatLngBounds::world(), 0.0, 0.0, 1.0), + test.db, test.fileSource); + + test.fileSource.styleResponse = [&] (const Resource& resource) { + EXPECT_EQ("http://127.0.0.1:3000/offline/style.json", resource.url); + return test.response("offline/empty.style.json"); + }; + + auto observer = std::make_unique<MockObserver>(); + + observer->statusChangedFn = [&] (OfflineRegionStatus status) { + if (status.complete()) { + EXPECT_EQ(1, status.completedResourceCount); + EXPECT_EQ(test.size, status.completedResourceSize); + EXPECT_FALSE(status.requiredResourceCountIsIndeterminate); + test.loop.stop(); + } + }; + + download.setObserver(std::move(observer)); + download.setState(OfflineRegionDownloadState::Active); + + test.loop.run(); +} + +TEST(OfflineDownload, InlineSource) { + OfflineTest test; + OfflineDownload download( + 1, + OfflineTilePyramidRegionDefinition("http://127.0.0.1:3000/offline/style.json", LatLngBounds::world(), 0.0, 0.0, 1.0), + test.db, test.fileSource); + + test.fileSource.styleResponse = [&] (const Resource& resource) { + EXPECT_EQ("http://127.0.0.1:3000/offline/style.json", resource.url); + return test.response("offline/inline_source.style.json"); + }; + + test.fileSource.tileResponse = [&] (const Resource& resource) { + const Resource::TileData& tile = *resource.tileData; + EXPECT_EQ("http://127.0.0.1:3000/offline/{z}-{x}-{y}.vector.pbf", tile.urlTemplate); + EXPECT_EQ(1, tile.pixelRatio); + EXPECT_EQ(0, tile.x); + EXPECT_EQ(0, tile.y); + EXPECT_EQ(0, tile.z); + return test.response("offline/0-0-0.vector.pbf"); + }; + + auto observer = std::make_unique<MockObserver>(); + + observer->statusChangedFn = [&] (OfflineRegionStatus status) { + if (status.complete()) { + EXPECT_EQ(2, status.completedResourceCount); + EXPECT_EQ(test.size, status.completedResourceSize); + EXPECT_FALSE(status.requiredResourceCountIsIndeterminate); + test.loop.stop(); + } + }; + + download.setObserver(std::move(observer)); + download.setState(OfflineRegionDownloadState::Active); + + test.loop.run(); +} + +TEST(OfflineDownload, GeoJSONSource) { + OfflineTest test; + OfflineDownload download( + 1, + OfflineTilePyramidRegionDefinition("http://127.0.0.1:3000/offline/style.json", LatLngBounds::world(), 0.0, 0.0, 1.0), + test.db, test.fileSource); + + test.fileSource.styleResponse = [&] (const Resource& resource) { + EXPECT_EQ("http://127.0.0.1:3000/offline/style.json", resource.url); + return test.response("offline/geojson_source.style.json"); + }; + + test.fileSource.sourceResponse = [&] (const Resource& resource) { + EXPECT_EQ("http://127.0.0.1:3000/offline/geojson.json", resource.url); + return test.response("offline/geojson.json"); + }; + + auto observer = std::make_unique<MockObserver>(); + + observer->statusChangedFn = [&] (OfflineRegionStatus status) { + if (status.complete()) { + EXPECT_EQ(2, status.completedResourceCount); + EXPECT_EQ(test.size, status.completedResourceSize); + EXPECT_FALSE(status.requiredResourceCountIsIndeterminate); + test.loop.stop(); + } + }; + + download.setObserver(std::move(observer)); + download.setState(OfflineRegionDownloadState::Active); + + test.loop.run(); +} + +TEST(OfflineDownload, Activate) { + OfflineTest test; + OfflineDownload download( + 1, + OfflineTilePyramidRegionDefinition("http://127.0.0.1:3000/offline/style.json", LatLngBounds::world(), 0.0, 0.0, 1.0), + test.db, test.fileSource); + + test.fileSource.styleResponse = [&] (const Resource& resource) { + EXPECT_EQ("http://127.0.0.1:3000/offline/style.json", resource.url); + return test.response("offline/style.json"); + }; + + test.fileSource.spriteImageResponse = [&] (const Resource& resource) { + EXPECT_EQ("http://127.0.0.1:3000/offline/sprite.png", resource.url); + return test.response("offline/sprite.png"); + }; + + test.fileSource.spriteJSONResponse = [&] (const Resource& resource) { + EXPECT_EQ("http://127.0.0.1:3000/offline/sprite.json", resource.url); + return test.response("offline/sprite.json"); + }; + + test.fileSource.glyphsResponse = [&] (const Resource&) { + return test.response("offline/glyph.pbf"); + }; + + test.fileSource.sourceResponse = [&] (const Resource& resource) { + EXPECT_EQ("http://127.0.0.1:3000/offline/streets.json", resource.url); + return test.response("offline/streets.json"); + }; + + test.fileSource.tileResponse = [&] (const Resource& resource) { + const Resource::TileData& tile = *resource.tileData; + EXPECT_EQ("http://127.0.0.1:3000/offline/{z}-{x}-{y}.vector.pbf", tile.urlTemplate); + EXPECT_EQ(1, tile.pixelRatio); + EXPECT_EQ(0, tile.x); + EXPECT_EQ(0, tile.y); + EXPECT_EQ(0, tile.z); + return test.response("offline/0-0-0.vector.pbf"); + }; + + auto observer = std::make_unique<MockObserver>(); + + observer->statusChangedFn = [&] (OfflineRegionStatus status) { + if (status.complete()) { + EXPECT_EQ(261, status.completedResourceCount); // 256 glyphs, 1 tile, 1 style, source, sprite image, and sprite json + EXPECT_EQ(test.size, status.completedResourceSize); + + download.setState(OfflineRegionDownloadState::Inactive); + OfflineRegionStatus computedStatus = download.getStatus(); + EXPECT_EQ(status.requiredResourceCount, computedStatus.requiredResourceCount); + EXPECT_EQ(status.completedResourceCount, computedStatus.completedResourceCount); + EXPECT_EQ(status.completedResourceSize, computedStatus.completedResourceSize); + EXPECT_FALSE(status.requiredResourceCountIsIndeterminate); + + test.loop.stop(); + } + }; + + download.setObserver(std::move(observer)); + download.setState(OfflineRegionDownloadState::Active); + + test.loop.run(); +} + +TEST(OfflineDownload, GetStatusNoResources) { + OfflineTest test; + OfflineDownload download( + 1, + OfflineTilePyramidRegionDefinition("http://127.0.0.1:3000/offline/style.json", LatLngBounds::world(), 0.0, 0.0, 1.0), + test.db, test.fileSource); + OfflineRegionStatus status = download.getStatus(); + + EXPECT_EQ(OfflineRegionDownloadState::Inactive, status.downloadState); + EXPECT_EQ(0, status.completedResourceCount); + EXPECT_EQ(0, status.completedResourceSize); + EXPECT_EQ(1, status.requiredResourceCount); + EXPECT_TRUE(status.requiredResourceCountIsIndeterminate); + EXPECT_FALSE(status.complete()); +} + +TEST(OfflineDownload, GetStatusStyleComplete) { + OfflineTest test; + OfflineDownload download( + 1, + OfflineTilePyramidRegionDefinition("http://127.0.0.1:3000/offline/style.json", LatLngBounds::world(), 0.0, 0.0, 1.0), + test.db, test.fileSource); + + test.db.putRegionResource(1, + Resource::style("http://127.0.0.1:3000/offline/style.json"), + test.response("offline/style.json")); + + OfflineRegionStatus status = download.getStatus(); + + EXPECT_EQ(OfflineRegionDownloadState::Inactive, status.downloadState); + EXPECT_EQ(1, status.completedResourceCount); + EXPECT_EQ(test.size, status.completedResourceSize); + EXPECT_EQ(260, status.requiredResourceCount); + EXPECT_TRUE(status.requiredResourceCountIsIndeterminate); + EXPECT_FALSE(status.complete()); +} + +TEST(OfflineDownload, GetStatusStyleAndSourceComplete) { + OfflineTest test; + OfflineDownload download( + 1, + OfflineTilePyramidRegionDefinition("http://127.0.0.1:3000/offline/style.json", LatLngBounds::world(), 0.0, 0.0, 1.0), + test.db, test.fileSource); + + test.db.putRegionResource(1, + Resource::style("http://127.0.0.1:3000/offline/style.json"), + test.response("offline/style.json")); + + test.db.putRegionResource(1, + Resource::source("http://127.0.0.1:3000/offline/streets.json"), + test.response("offline/streets.json")); + + OfflineRegionStatus status = download.getStatus(); + + EXPECT_EQ(OfflineRegionDownloadState::Inactive, status.downloadState); + EXPECT_EQ(2, status.completedResourceCount); + EXPECT_EQ(test.size, status.completedResourceSize); + EXPECT_EQ(261, status.requiredResourceCount); + EXPECT_FALSE(status.requiredResourceCountIsIndeterminate); + EXPECT_FALSE(status.complete()); +} + +TEST(OfflineDownload, RequestError) { + OfflineTest test; + OfflineDownload download( + 1, + OfflineTilePyramidRegionDefinition("http://127.0.0.1:3000/offline/style.json", LatLngBounds::world(), 0.0, 0.0, 1.0), + test.db, test.fileSource); + + test.fileSource.styleResponse = [&] (const Resource&) { + Response response; + response.error = std::make_unique<Response::Error>(Response::Error::Reason::Connection, "connection error"); + return response; + }; + + auto observer = std::make_unique<MockObserver>(); + + observer->responseErrorFn = [&] (Response::Error error) { + EXPECT_EQ(Response::Error::Reason::Connection, error.reason); + EXPECT_EQ("connection error", error.message); + test.loop.stop(); + }; + + download.setObserver(std::move(observer)); + download.setState(OfflineRegionDownloadState::Active); + + test.loop.run(); +} diff --git a/test/test.gypi b/test/test.gypi index 883b954a35..9b5bceea72 100644 --- a/test/test.gypi +++ b/test/test.gypi @@ -70,7 +70,9 @@ 'storage/storage.hpp', 'storage/storage.cpp', 'storage/default_file_source.cpp', + 'storage/offline.cpp', 'storage/offline_database.cpp', + 'storage/offline_download.cpp', 'storage/asset_file_source.cpp', 'storage/headers.cpp', 'storage/http_cancel.cpp', |