[core] Add DefaultFileSource::mergeRegions API and CLI support in the mbgl-offline tool.

5 files changed, 122 insertions, 7 deletions

diff --git a/bin/offline.cpp b/bin/offline.cpp
index 2da47a4476..5123e7b926 100644
--- a/bin/offline.cpp
+++ b/bin/offline.cpp
@@ -62,18 +62,21 @@ int main(int argc, char *argv[]) {
     args::ValueFlag<std::string> outputValue(argumentParser, "file", "Output database file name", {'o', "output"});
     args::ValueFlag<std::string> apiBaseValue(argumentParser, "URL", "API Base URL", {'a', "apiBaseURL"});
     
+    args::Group mergeGroup(argumentParser, "Merge databases:", args::Group::Validators::AllOrNone);
+    args::ValueFlag<std::string> mergePathValue(mergeGroup, "merge", "Database to merge from", {'m', "merge"});
+    args::ValueFlag<std::string> inputValue(mergeGroup, "input", "Database to merge into. Use with --merge option.", {'i', "input"});
+
     // LatLngBounds
     args::Group latLngBoundsGroup(argumentParser, "LatLng bounds:", args::Group::Validators::AllOrNone);
     args::ValueFlag<double> northValue(latLngBoundsGroup, "degrees", "North latitude", {"north"});
     args::ValueFlag<double> westValue(latLngBoundsGroup, "degrees", "West longitude", {"west"});
     args::ValueFlag<double> southValue(latLngBoundsGroup, "degrees", "South latitude", {"south"});
     args::ValueFlag<double> eastValue(latLngBoundsGroup, "degrees", "East longitude", {"east"});
-    
+
     // Geometry
     args::Group geoJSONGroup(argumentParser, "GeoJson geometry:", args::Group::Validators::AllOrNone);
     args::ValueFlag<std::string> geometryValue(geoJSONGroup, "file", "GeoJSON file containing the region geometry", {"geojson"});
-    
-    
+
     args::ValueFlag<double> minZoomValue(argumentParser, "number", "Min zoom level", {"minZoom"});
     args::ValueFlag<double> maxZoomValue(argumentParser, "number", "Max zoom level", {"maxZoom"});
     args::ValueFlag<double> pixelRatioValue(argumentParser, "number", "Pixel ratio", {"pixelRatio"});
@@ -95,6 +98,11 @@ int main(int argc, char *argv[]) {
 
     std::string style = styleValue ? args::get(styleValue) : mbgl::util::default_styles::streets.url;
 
+    mbgl::optional<std::string> mergePath = {};
+    if (mergePathValue) mergePath = args::get(mergePathValue);
+    mbgl::optional<std::string> inputDb = {};
+    if (inputValue) inputDb = args::get(inputValue);
+
     const double minZoom = minZoomValue ? args::get(minZoomValue) : 0.0;
     const double maxZoom = maxZoomValue ? args::get(maxZoomValue) : 15.0;
     const double pixelRatio = pixelRatioValue ? args::get(pixelRatioValue) : 1.0;
@@ -136,15 +144,37 @@ int main(int argc, char *argv[]) {
     fileSource.setAccessToken(token);
     fileSource.setAPIBaseURL(apiBaseURL);
 
+    if (inputDb && mergePath) {
+        DefaultFileSource inputSource(*inputDb, ".");
+        inputSource.setAccessToken(token);
+        inputSource.setAPIBaseURL(apiBaseURL);
+        
+        int retCode = 0;
+        std::cout << "Start Merge" << std::endl;
+        inputSource.mergeOfflineRegions(*mergePath,  [&] (mbgl::expected<std::vector<OfflineRegion>, std::exception_ptr> result) {
+
+            if (!result) {
+                std::cerr << "Error merging database: " << util::toString(result.error()) << std::endl;
+                retCode = 1;
+            } else {
+                std::cout << " Added " << result->size() << " Regions" << std::endl;
+                std::cout << "Finished Merge" << std::endl;
+            }
+            loop.stop();
+        });
+        loop.run();
+        return retCode;
+    }
 
     OfflineRegionMetadata metadata;
 
     class Observer : public OfflineRegionObserver {
     public:
-        Observer(OfflineRegion& region_, DefaultFileSource& fileSource_, util::RunLoop& loop_)
+        Observer(OfflineRegion& region_, DefaultFileSource& fileSource_, util::RunLoop& loop_, mbgl::optional<std::string> mergePath_)
             : region(region_),
               fileSource(fileSource_),
               loop(loop_),
+              mergePath(mergePath_),
               start(util::now()) {
         }
 
@@ -170,7 +200,7 @@ int main(int argc, char *argv[]) {
                       << std::endl;
 
             if (status.complete()) {
-                std::cout << "Finished" << std::endl;
+                std::cout << "Finished Download" << std::endl;
                 loop.stop();
             }
         }
@@ -186,6 +216,7 @@ int main(int argc, char *argv[]) {
         OfflineRegion& region;
         DefaultFileSource& fileSource;
         util::RunLoop& loop;
+        mbgl::optional<std::string> mergePath;
         Timestamp start;
     };
 
@@ -206,7 +237,7 @@ int main(int argc, char *argv[]) {
         } else {
             assert(region_);
             region = std::make_unique<OfflineRegion>(std::move(*region_));
-            fileSource.setOfflineRegionObserver(*region, std::make_unique<Observer>(*region, fileSource, loop));
+            fileSource.setOfflineRegionObserver(*region, std::make_unique<Observer>(*region, fileSource, loop, mergePath));
             fileSource.setOfflineRegionDownloadState(*region, OfflineRegionDownloadState::Active);
         }
     });
diff --git a/include/mbgl/storage/default_file_source.hpp b/include/mbgl/storage/default_file_source.hpp
index e048d82af2..942749fc00 100644
--- a/include/mbgl/storage/default_file_source.hpp
+++ b/include/mbgl/storage/default_file_source.hpp
@@ -100,6 +100,23 @@ public:
         std::function<void (expected<OfflineRegionStatus, std::exception_ptr>)>) const;
 
     /*
+     * Merge offline regions from a secondary database into the main offline database.
+     *
+     * When the database merge is completed, the provided callback will be
+     * executed on the database thread; it is the responsibility of the SDK bindings
+     * to re-execute a user-provided callback on the main thread.
+     *
+     * Only resources and tiles that belong to a region will be copied over. Identical
+     * regions will be flattened into a single new region in the main database.
+     *
+     * Note that the resulting new regions may not be in a completed status if the
+     * secondary database does not contain all the tiles or resources required by the
+     * region definition.
+     */
+    void mergeOfflineRegions(const std::string& sideDatabasePath,
+                            std::function<void (expected<OfflineRegions, std::exception_ptr>)>);
+
+    /*
      * Remove an offline region from the database and perform any resources evictions
      * necessary as a result.
      *
diff --git a/platform/default/default_file_source.cpp b/platform/default/default_file_source.cpp
index 93f10eea72..99e5c4dff3 100644
--- a/platform/default/default_file_source.cpp
+++ b/platform/default/default_file_source.cpp
@@ -55,6 +55,11 @@ public:
         callback(offlineDatabase->createRegion(definition, metadata));
     }
 
+    void mergeOfflineRegions(const std::string& sideDatabasePath,
+                             std::function<void (expected<OfflineRegions, std::exception_ptr>)> callback) {
+        callback(offlineDatabase->mergeDatabase(sideDatabasePath));
+     }
+
     void updateMetadata(const int64_t regionID,
                       const OfflineRegionMetadata& metadata,
                       std::function<void (expected<OfflineRegionMetadata, std::exception_ptr>)> callback) {
@@ -258,9 +263,15 @@ void DefaultFileSource::createOfflineRegion(const OfflineRegionDefinition& defin
     impl->actor().invoke(&Impl::createRegion, definition, metadata, callback);
 }
 
+void DefaultFileSource::mergeOfflineRegions(const std::string& sideDatabasePath,
+                                            std::function<void (expected<OfflineRegions, std::exception_ptr>)> callback) {
+    impl->actor().invoke(&Impl::mergeOfflineRegions, sideDatabasePath, callback);
+}
+
 void DefaultFileSource::updateOfflineMetadata(const int64_t regionID,
                                             const OfflineRegionMetadata& metadata,
-                                            std::function<void (expected<OfflineRegionMetadata, std::exception_ptr>)> callback) {
+                                            std::function<void (expected<OfflineRegionMetadata,
+                                             std::exception_ptr>)> callback) {
     impl->actor().invoke(&Impl::updateMetadata, regionID, metadata, callback);
 }
 
diff --git a/platform/default/mbgl/storage/offline_database.cpp b/platform/default/mbgl/storage/offline_database.cpp
index 79bc3c8f27..30b76d1666 100644
--- a/platform/default/mbgl/storage/offline_database.cpp
+++ b/platform/default/mbgl/storage/offline_database.cpp
@@ -7,6 +7,7 @@
 #include <mbgl/util/logging.hpp>
 
 #include "offline_schema.hpp"
+#include "merge_sideloaded.hpp"
 
 #include "sqlite3.hpp"
 
@@ -637,6 +638,58 @@ OfflineDatabase::createRegion(const OfflineRegionDefinition& definition,
     return unexpected<std::exception_ptr>(std::current_exception());
 }
 
+expected<OfflineRegions, std::exception_ptr>
+OfflineDatabase::mergeDatabase(const std::string& sideDatabasePath) {
+    try {
+        // clang-format off
+        mapbox::sqlite::Query query{ getStatement("ATTACH DATABASE ?1 AS side") };
+        // clang-format on
+
+        query.bind(1, sideDatabasePath);
+        query.run();
+    } catch (const mapbox::sqlite::Exception& ex) {
+        handleError(ex, "merge databse attach");
+        return unexpected<std::exception_ptr>(std::current_exception());
+    }
+    try {
+        // Attaching an accessible path without a db file creates a new temporary
+        //database and attaches it. Check for matching schema version.
+        auto sideUserVersion = static_cast<int>(getPragma<int64_t>("PRAGMA side.user_version"));
+        if (sideUserVersion != 6) {
+            Log::Warning(Event::Database, "Merge database does not match user_version of main database");
+            throw std::runtime_error("merge database does not match schema or has incorrect user_version");
+        }
+
+        mapbox::sqlite::Transaction transaction(*db);
+        db->exec(mergeSideloadedDatabaseSQL);
+        transaction.commit();
+
+        // clang-format off
+        mapbox::sqlite::Query query{ getStatement(
+            "SELECT r.id, r.definition, r.description "
+            "FROM side.regions sr "
+            "JOIN regions r ON sr.definition = r.definition") };
+        // clang-format on
+
+        OfflineRegions result;
+        while (query.run()) {
+            // Construct, then move because this constructor is private.
+            OfflineRegion region(query.get<int64_t>(0),
+                decodeOfflineRegionDefinition(query.get<std::string>(1)),
+                query.get<std::vector<uint8_t>>(2));
+            result.emplace_back(std::move(region));
+        }
+        db->exec("DETACH DATABASE side");
+        // Explicit move to avoid triggering the copy constructor.
+        return { std::move(result) };
+    } catch (const mapbox::sqlite::Exception& ex) {
+        db->exec("DETACH DATABASE side");
+        handleError(ex, "merge databse post merge sql");
+        return unexpected<std::exception_ptr>(std::current_exception());
+    }
+    return {};
+}
+
 expected<OfflineRegionMetadata, std::exception_ptr>
 OfflineDatabase::updateMetadata(const int64_t regionID, const OfflineRegionMetadata& metadata) try {
     // clang-format off
diff --git a/platform/default/mbgl/storage/offline_database.hpp b/platform/default/mbgl/storage/offline_database.hpp
index fbe6c707e1..993f36a606 100644
--- a/platform/default/mbgl/storage/offline_database.hpp
+++ b/platform/default/mbgl/storage/offline_database.hpp
@@ -53,6 +53,9 @@ public:
     expected<OfflineRegion, std::exception_ptr> createRegion(const OfflineRegionDefinition&,
                                                              const OfflineRegionMetadata&);
 
+    expected<OfflineRegions, std::exception_ptr>
+    mergeDatabase(const std::string& sideDatabasePath);
+
     expected<OfflineRegionMetadata, std::exception_ptr>
     updateMetadata(const int64_t regionID, const OfflineRegionMetadata&);