From 065f5ca87e96f2280959b9fd3ad3d6e63a99bae6 Mon Sep 17 00:00:00 2001 From: Asheem Mamoowala Date: Tue, 21 Aug 2018 14:11:44 -0700 Subject: [core] Add DefaultFileSource::mergeRegions API and CLI support in the mbgl-offline tool. --- bin/offline.cpp | 43 +++++++++++++++--- include/mbgl/storage/default_file_source.hpp | 17 +++++++ platform/default/default_file_source.cpp | 13 +++++- platform/default/mbgl/storage/offline_database.cpp | 53 ++++++++++++++++++++++ platform/default/mbgl/storage/offline_database.hpp | 3 ++ 5 files changed, 122 insertions(+), 7 deletions(-) diff --git a/bin/offline.cpp b/bin/offline.cpp index 2da47a4476..5123e7b926 100644 --- a/bin/offline.cpp +++ b/bin/offline.cpp @@ -62,18 +62,21 @@ int main(int argc, char *argv[]) { args::ValueFlag outputValue(argumentParser, "file", "Output database file name", {'o', "output"}); args::ValueFlag apiBaseValue(argumentParser, "URL", "API Base URL", {'a', "apiBaseURL"}); + args::Group mergeGroup(argumentParser, "Merge databases:", args::Group::Validators::AllOrNone); + args::ValueFlag mergePathValue(mergeGroup, "merge", "Database to merge from", {'m', "merge"}); + args::ValueFlag inputValue(mergeGroup, "input", "Database to merge into. Use with --merge option.", {'i', "input"}); + // LatLngBounds args::Group latLngBoundsGroup(argumentParser, "LatLng bounds:", args::Group::Validators::AllOrNone); args::ValueFlag northValue(latLngBoundsGroup, "degrees", "North latitude", {"north"}); args::ValueFlag westValue(latLngBoundsGroup, "degrees", "West longitude", {"west"}); args::ValueFlag southValue(latLngBoundsGroup, "degrees", "South latitude", {"south"}); args::ValueFlag eastValue(latLngBoundsGroup, "degrees", "East longitude", {"east"}); - + // Geometry args::Group geoJSONGroup(argumentParser, "GeoJson geometry:", args::Group::Validators::AllOrNone); args::ValueFlag geometryValue(geoJSONGroup, "file", "GeoJSON file containing the region geometry", {"geojson"}); - - + args::ValueFlag minZoomValue(argumentParser, "number", "Min zoom level", {"minZoom"}); args::ValueFlag maxZoomValue(argumentParser, "number", "Max zoom level", {"maxZoom"}); args::ValueFlag pixelRatioValue(argumentParser, "number", "Pixel ratio", {"pixelRatio"}); @@ -95,6 +98,11 @@ int main(int argc, char *argv[]) { std::string style = styleValue ? args::get(styleValue) : mbgl::util::default_styles::streets.url; + mbgl::optional mergePath = {}; + if (mergePathValue) mergePath = args::get(mergePathValue); + mbgl::optional inputDb = {}; + if (inputValue) inputDb = args::get(inputValue); + const double minZoom = minZoomValue ? args::get(minZoomValue) : 0.0; const double maxZoom = maxZoomValue ? args::get(maxZoomValue) : 15.0; const double pixelRatio = pixelRatioValue ? args::get(pixelRatioValue) : 1.0; @@ -136,15 +144,37 @@ int main(int argc, char *argv[]) { fileSource.setAccessToken(token); fileSource.setAPIBaseURL(apiBaseURL); + if (inputDb && mergePath) { + DefaultFileSource inputSource(*inputDb, "."); + inputSource.setAccessToken(token); + inputSource.setAPIBaseURL(apiBaseURL); + + int retCode = 0; + std::cout << "Start Merge" << std::endl; + inputSource.mergeOfflineRegions(*mergePath, [&] (mbgl::expected, std::exception_ptr> result) { + + if (!result) { + std::cerr << "Error merging database: " << util::toString(result.error()) << std::endl; + retCode = 1; + } else { + std::cout << " Added " << result->size() << " Regions" << std::endl; + std::cout << "Finished Merge" << std::endl; + } + loop.stop(); + }); + loop.run(); + return retCode; + } OfflineRegionMetadata metadata; class Observer : public OfflineRegionObserver { public: - Observer(OfflineRegion& region_, DefaultFileSource& fileSource_, util::RunLoop& loop_) + Observer(OfflineRegion& region_, DefaultFileSource& fileSource_, util::RunLoop& loop_, mbgl::optional mergePath_) : region(region_), fileSource(fileSource_), loop(loop_), + mergePath(mergePath_), start(util::now()) { } @@ -170,7 +200,7 @@ int main(int argc, char *argv[]) { << std::endl; if (status.complete()) { - std::cout << "Finished" << std::endl; + std::cout << "Finished Download" << std::endl; loop.stop(); } } @@ -186,6 +216,7 @@ int main(int argc, char *argv[]) { OfflineRegion& region; DefaultFileSource& fileSource; util::RunLoop& loop; + mbgl::optional mergePath; Timestamp start; }; @@ -206,7 +237,7 @@ int main(int argc, char *argv[]) { } else { assert(region_); region = std::make_unique(std::move(*region_)); - fileSource.setOfflineRegionObserver(*region, std::make_unique(*region, fileSource, loop)); + fileSource.setOfflineRegionObserver(*region, std::make_unique(*region, fileSource, loop, mergePath)); fileSource.setOfflineRegionDownloadState(*region, OfflineRegionDownloadState::Active); } }); diff --git a/include/mbgl/storage/default_file_source.hpp b/include/mbgl/storage/default_file_source.hpp index e048d82af2..942749fc00 100644 --- a/include/mbgl/storage/default_file_source.hpp +++ b/include/mbgl/storage/default_file_source.hpp @@ -99,6 +99,23 @@ public: OfflineRegion&, std::function)>) const; + /* + * Merge offline regions from a secondary database into the main offline database. + * + * When the database merge is completed, the provided callback will be + * executed on the database thread; it is the responsibility of the SDK bindings + * to re-execute a user-provided callback on the main thread. + * + * Only resources and tiles that belong to a region will be copied over. Identical + * regions will be flattened into a single new region in the main database. + * + * Note that the resulting new regions may not be in a completed status if the + * secondary database does not contain all the tiles or resources required by the + * region definition. + */ + void mergeOfflineRegions(const std::string& sideDatabasePath, + std::function)>); + /* * Remove an offline region from the database and perform any resources evictions * necessary as a result. diff --git a/platform/default/default_file_source.cpp b/platform/default/default_file_source.cpp index 93f10eea72..99e5c4dff3 100644 --- a/platform/default/default_file_source.cpp +++ b/platform/default/default_file_source.cpp @@ -55,6 +55,11 @@ public: callback(offlineDatabase->createRegion(definition, metadata)); } + void mergeOfflineRegions(const std::string& sideDatabasePath, + std::function)> callback) { + callback(offlineDatabase->mergeDatabase(sideDatabasePath)); + } + void updateMetadata(const int64_t regionID, const OfflineRegionMetadata& metadata, std::function)> callback) { @@ -258,9 +263,15 @@ void DefaultFileSource::createOfflineRegion(const OfflineRegionDefinition& defin impl->actor().invoke(&Impl::createRegion, definition, metadata, callback); } +void DefaultFileSource::mergeOfflineRegions(const std::string& sideDatabasePath, + std::function)> callback) { + impl->actor().invoke(&Impl::mergeOfflineRegions, sideDatabasePath, callback); +} + void DefaultFileSource::updateOfflineMetadata(const int64_t regionID, const OfflineRegionMetadata& metadata, - std::function)> callback) { + std::function)> callback) { impl->actor().invoke(&Impl::updateMetadata, regionID, metadata, callback); } diff --git a/platform/default/mbgl/storage/offline_database.cpp b/platform/default/mbgl/storage/offline_database.cpp index 79bc3c8f27..30b76d1666 100644 --- a/platform/default/mbgl/storage/offline_database.cpp +++ b/platform/default/mbgl/storage/offline_database.cpp @@ -7,6 +7,7 @@ #include #include "offline_schema.hpp" +#include "merge_sideloaded.hpp" #include "sqlite3.hpp" @@ -637,6 +638,58 @@ OfflineDatabase::createRegion(const OfflineRegionDefinition& definition, return unexpected(std::current_exception()); } +expected +OfflineDatabase::mergeDatabase(const std::string& sideDatabasePath) { + try { + // clang-format off + mapbox::sqlite::Query query{ getStatement("ATTACH DATABASE ?1 AS side") }; + // clang-format on + + query.bind(1, sideDatabasePath); + query.run(); + } catch (const mapbox::sqlite::Exception& ex) { + handleError(ex, "merge databse attach"); + return unexpected(std::current_exception()); + } + try { + // Attaching an accessible path without a db file creates a new temporary + //database and attaches it. Check for matching schema version. + auto sideUserVersion = static_cast(getPragma("PRAGMA side.user_version")); + if (sideUserVersion != 6) { + Log::Warning(Event::Database, "Merge database does not match user_version of main database"); + throw std::runtime_error("merge database does not match schema or has incorrect user_version"); + } + + mapbox::sqlite::Transaction transaction(*db); + db->exec(mergeSideloadedDatabaseSQL); + transaction.commit(); + + // clang-format off + mapbox::sqlite::Query query{ getStatement( + "SELECT r.id, r.definition, r.description " + "FROM side.regions sr " + "JOIN regions r ON sr.definition = r.definition") }; + // clang-format on + + OfflineRegions result; + while (query.run()) { + // Construct, then move because this constructor is private. + OfflineRegion region(query.get(0), + decodeOfflineRegionDefinition(query.get(1)), + query.get>(2)); + result.emplace_back(std::move(region)); + } + db->exec("DETACH DATABASE side"); + // Explicit move to avoid triggering the copy constructor. + return { std::move(result) }; + } catch (const mapbox::sqlite::Exception& ex) { + db->exec("DETACH DATABASE side"); + handleError(ex, "merge databse post merge sql"); + return unexpected(std::current_exception()); + } + return {}; +} + expected OfflineDatabase::updateMetadata(const int64_t regionID, const OfflineRegionMetadata& metadata) try { // clang-format off diff --git a/platform/default/mbgl/storage/offline_database.hpp b/platform/default/mbgl/storage/offline_database.hpp index fbe6c707e1..993f36a606 100644 --- a/platform/default/mbgl/storage/offline_database.hpp +++ b/platform/default/mbgl/storage/offline_database.hpp @@ -53,6 +53,9 @@ public: expected createRegion(const OfflineRegionDefinition&, const OfflineRegionMetadata&); + expected + mergeDatabase(const std::string& sideDatabasePath); + expected updateMetadata(const int64_t regionID, const OfflineRegionMetadata&); -- cgit v1.2.1