summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat
-rw-r--r--include/mbgl/storage/default_file_source.hpp6
-rw-r--r--include/mbgl/storage/offline.hpp16
-rw-r--r--platform/default/default_file_source.cpp8
-rw-r--r--platform/default/mbgl/storage/offline_database.cpp114
-rw-r--r--platform/default/mbgl/storage/offline_database.hpp13
-rw-r--r--platform/default/mbgl/storage/offline_download.cpp8
-rw-r--r--src/mbgl/util/mapbox.cpp2
-rw-r--r--src/mbgl/util/mapbox.hpp2
-rw-r--r--test/fixtures/offline/mapbox_source.style.json17
-rw-r--r--test/storage/offline_database.cpp57
-rw-r--r--test/storage/offline_download.cpp33
11 files changed, 261 insertions, 15 deletions
diff --git a/include/mbgl/storage/default_file_source.hpp b/include/mbgl/storage/default_file_source.hpp
index 3e839bbc18..93d6297651 100644
--- a/include/mbgl/storage/default_file_source.hpp
+++ b/include/mbgl/storage/default_file_source.hpp
@@ -94,6 +94,12 @@ public:
*/
void deleteOfflineRegion(OfflineRegion&&, std::function<void (std::exception_ptr)>);
+ /*
+ * Changing or bypassing this limit without permission from Mapbox is prohibited
+ * by the Mapbox Terms of Service.
+ */
+ void setOfflineMapboxTileCountLimit(uint64_t) const;
+
// For testing only.
void put(const Resource&, const Response&);
void goOffline();
diff --git a/include/mbgl/storage/offline.hpp b/include/mbgl/storage/offline.hpp
index 2bc286dbab..a3153adc28 100644
--- a/include/mbgl/storage/offline.hpp
+++ b/include/mbgl/storage/offline.hpp
@@ -150,6 +150,22 @@ public:
* re-executes the user-provided implementation on the main thread.
*/
virtual void responseError(Response::Error) {}
+
+ /*
+ * Implement this method to be notified when the limit on the number of Mapbox
+ * tiles stored for offline regions has been reached.
+ *
+ * Once the limit has been reached, the SDK will not download further offline
+ * tiles from Mapbox APIs until existing tiles have been removed. Contact your
+ * Mapbox sales representative to raise the limit.
+ *
+ * This limit does not apply to non-Mapbox tile sources.
+ *
+ * Note that this method will be executed on the database thread; it is the
+ * responsibility of the SDK bindings to wrap this object in an interface that
+ * re-executes the user-provided implementation on the main thread.
+ */
+ virtual void mapboxTileCountLimitExceeded(uint64_t /* limit */) {}
};
class OfflineRegion {
diff --git a/platform/default/default_file_source.cpp b/platform/default/default_file_source.cpp
index bdf02ee69a..24297374c9 100644
--- a/platform/default/default_file_source.cpp
+++ b/platform/default/default_file_source.cpp
@@ -113,6 +113,10 @@ public:
tasks.erase(req);
}
+ void setOfflineMapboxTileCountLimit(uint64_t limit) {
+ offlineDatabase.setOfflineMapboxTileCountLimit(limit);
+ }
+
void put(const Resource& resource, const Response& response) {
offlineDatabase.put(resource, response);
}
@@ -205,6 +209,10 @@ void DefaultFileSource::getOfflineRegionStatus(OfflineRegion& region, std::funct
thread->invoke(&Impl::getRegionStatus, region.getID(), callback);
}
+void DefaultFileSource::setOfflineMapboxTileCountLimit(uint64_t limit) const {
+ thread->invokeSync(&Impl::setOfflineMapboxTileCountLimit, limit);
+}
+
// For testing only:
void DefaultFileSource::put(const Resource& resource, const Response& response) {
diff --git a/platform/default/mbgl/storage/offline_database.cpp b/platform/default/mbgl/storage/offline_database.cpp
index 983b84a3b4..df3e3f4dc3 100644
--- a/platform/default/mbgl/storage/offline_database.cpp
+++ b/platform/default/mbgl/storage/offline_database.cpp
@@ -447,6 +447,9 @@ void OfflineDatabase::deleteRegion(OfflineRegion&& region) {
stmt->run();
evict(0);
+
+ // Ensure that the cached offlineTileCount value is recalculated.
+ offlineMapboxTileCount = {};
}
optional<Response> OfflineDatabase::getRegionResource(int64_t regionID, const Resource& resource) {
@@ -461,13 +464,21 @@ optional<Response> OfflineDatabase::getRegionResource(int64_t regionID, const Re
uint64_t OfflineDatabase::putRegionResource(int64_t regionID, const Resource& resource, const Response& response) {
uint64_t size = putInternal(resource, response, false).second;
- markUsed(regionID, resource);
+ bool previouslyUnused = markUsed(regionID, resource);
+
+ if (offlineMapboxTileCount
+ && resource.kind == Resource::Kind::Tile
+ && util::mapbox::isMapboxURL(resource.url)
+ && previouslyUnused) {
+ *offlineMapboxTileCount += 1;
+ }
+
return size;
}
-void OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) {
+bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) {
if (resource.kind == Resource::Kind::Tile) {
- Statement stmt = getStatement(
+ Statement insert = getStatement(
"INSERT OR IGNORE INTO region_tiles (region_id, tile_id) "
"SELECT ?1, tiles.id "
"FROM tiles "
@@ -478,23 +489,61 @@ void OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) {
" AND z = ?6 ");
const Resource::TileData& tile = *resource.tileData;
- stmt->bind(1, regionID);
- stmt->bind(2, tile.urlTemplate);
- stmt->bind(3, tile.pixelRatio);
- stmt->bind(4, tile.x);
- stmt->bind(5, tile.y);
- stmt->bind(6, tile.z);
- stmt->run();
+ insert->bind(1, regionID);
+ insert->bind(2, tile.urlTemplate);
+ insert->bind(3, tile.pixelRatio);
+ insert->bind(4, tile.x);
+ insert->bind(5, tile.y);
+ insert->bind(6, tile.z);
+ insert->run();
+
+ if (db->changes() == 0) {
+ return false;
+ }
+
+ Statement select = getStatement(
+ "SELECT region_id "
+ "FROM region_tiles, tiles "
+ "WHERE region_id != ?1 "
+ " AND url_template = ?2 "
+ " AND pixel_ratio = ?3 "
+ " AND x = ?4 "
+ " AND y = ?5 "
+ " AND z = ?6 "
+ "LIMIT 1 ");
+
+ select->bind(1, regionID);
+ select->bind(2, tile.urlTemplate);
+ select->bind(3, tile.pixelRatio);
+ select->bind(4, tile.x);
+ select->bind(5, tile.y);
+ select->bind(6, tile.z);
+ return !select->run();
} else {
- Statement stmt = getStatement(
+ Statement insert = getStatement(
"INSERT OR IGNORE INTO region_resources (region_id, resource_id) "
"SELECT ?1, resources.id "
"FROM resources "
"WHERE resources.url = ?2 ");
- stmt->bind(1, regionID);
- stmt->bind(2, resource.url);
- stmt->run();
+ insert->bind(1, regionID);
+ insert->bind(2, resource.url);
+ insert->run();
+
+ if (db->changes() == 0) {
+ return false;
+ }
+
+ Statement select = getStatement(
+ "SELECT region_id "
+ "FROM region_resources, resources "
+ "WHERE region_id != ?1 "
+ " AND resources.url = ?2 "
+ "LIMIT 1 ");
+
+ select->bind(1, regionID);
+ select->bind(2, resource.url);
+ return !select->run();
}
}
@@ -591,6 +640,9 @@ bool OfflineDatabase::evict(uint64_t neededFreeSize) {
stmt2->run();
uint64_t changes2 = db->changes();
+ // The cached value of offlineTileCount does not need to be updated
+ // here because only non-offline tiles can be removed by eviction.
+
if (changes1 == 0 && changes2 == 0) {
return false;
}
@@ -599,4 +651,38 @@ bool OfflineDatabase::evict(uint64_t neededFreeSize) {
return true;
}
+void OfflineDatabase::setOfflineMapboxTileCountLimit(uint64_t limit) {
+ offlineMapboxTileCountLimit = limit;
+}
+
+uint64_t OfflineDatabase::getOfflineMapboxTileCountLimit() {
+ return offlineMapboxTileCountLimit;
+}
+
+bool OfflineDatabase::offlineMapboxTileCountLimitExceeded() {
+ return getOfflineMapboxTileCount() >= offlineMapboxTileCountLimit;
+}
+
+uint64_t OfflineDatabase::getOfflineMapboxTileCount() {
+ // Calculating this on every call would be much simpler than caching and
+ // manually updating the value, but it would make offline downloads an O(n²)
+ // operation, because the database query below involves an index scan of
+ // region_tiles.
+
+ if (offlineMapboxTileCount) {
+ return *offlineMapboxTileCount;
+ }
+
+ Statement stmt = getStatement(
+ "SELECT COUNT(DISTINCT id) "
+ "FROM region_tiles, tiles "
+ "WHERE tile_id = tiles.id "
+ "AND url_template LIKE 'mapbox://%' ");
+
+ stmt->run();
+
+ offlineMapboxTileCount = stmt->get<int64_t>(0);
+ return *offlineMapboxTileCount;
+}
+
} // namespace mbgl
diff --git a/platform/default/mbgl/storage/offline_database.hpp b/platform/default/mbgl/storage/offline_database.hpp
index f207ab406a..fc3f729bff 100644
--- a/platform/default/mbgl/storage/offline_database.hpp
+++ b/platform/default/mbgl/storage/offline_database.hpp
@@ -6,6 +6,7 @@
#include <mbgl/util/noncopyable.hpp>
#include <mbgl/util/optional.hpp>
#include <mbgl/util/constants.hpp>
+#include <mbgl/util/mapbox.hpp>
#include <unordered_map>
#include <memory>
@@ -49,6 +50,11 @@ public:
OfflineRegionDefinition getRegionDefinition(int64_t regionID);
OfflineRegionStatus getRegionCompletedStatus(int64_t regionID);
+ void setOfflineMapboxTileCountLimit(uint64_t);
+ uint64_t getOfflineMapboxTileCountLimit();
+ bool offlineMapboxTileCountLimitExceeded();
+ uint64_t getOfflineMapboxTileCount();
+
private:
void connect(int flags);
void ensureSchema();
@@ -78,7 +84,9 @@ private:
const std::string&, bool compressed);
std::pair<bool, uint64_t> putInternal(const Resource&, const Response&, bool evict);
- void markUsed(int64_t regionID, const Resource&);
+
+ // Return value is true iff the resource was previously unused by any other regions.
+ bool markUsed(int64_t regionID, const Resource&);
const std::string path;
std::unique_ptr<::mapbox::sqlite::Database> db;
@@ -89,6 +97,9 @@ private:
uint64_t maximumCacheSize;
+ uint64_t offlineMapboxTileCountLimit = util::mapbox::DEFAULT_OFFLINE_TILE_COUNT_LIMIT;
+ optional<uint64_t> offlineMapboxTileCount;
+
bool evict(uint64_t neededFreeSize);
};
diff --git a/platform/default/mbgl/storage/offline_download.cpp b/platform/default/mbgl/storage/offline_download.cpp
index 228942f1f0..8aa58d6c34 100644
--- a/platform/default/mbgl/storage/offline_download.cpp
+++ b/platform/default/mbgl/storage/offline_download.cpp
@@ -7,6 +7,7 @@
#include <mbgl/layer/symbol_layer.hpp>
#include <mbgl/text/glyph.hpp>
#include <mbgl/util/tile_cover.hpp>
+#include <mbgl/util/mapbox.hpp>
#include <set>
@@ -218,6 +219,13 @@ void OfflineDownload::ensureResource(const Resource& resource, std::function<voi
return;
}
+ if (resource.kind == Resource::Kind::Tile
+ && util::mapbox::isMapboxURL(resource.url)
+ && offlineDatabase.offlineMapboxTileCountLimitExceeded()) {
+ observer->mapboxTileCountLimitExceeded(offlineDatabase.getOfflineMapboxTileCountLimit());
+ return;
+ }
+
auto it = requests.insert(requests.begin(), nullptr);
*it = onlineFileSource.request(resource, [=] (Response onlineResponse) {
if (onlineResponse.error) {
diff --git a/src/mbgl/util/mapbox.cpp b/src/mbgl/util/mapbox.cpp
index 7ee0f279b6..bcc7601446 100644
--- a/src/mbgl/util/mapbox.cpp
+++ b/src/mbgl/util/mapbox.cpp
@@ -169,6 +169,8 @@ std::string canonicalizeTileURL(const std::string& url, SourceType type, uint16_
return result;
}
+const uint64_t DEFAULT_OFFLINE_TILE_COUNT_LIMIT = std::numeric_limits<uint64_t>::max();
+
} // end namespace mapbox
} // end namespace util
} // end namespace mbgl
diff --git a/src/mbgl/util/mapbox.hpp b/src/mbgl/util/mapbox.hpp
index bb0536cfa2..56c40df7ca 100644
--- a/src/mbgl/util/mapbox.hpp
+++ b/src/mbgl/util/mapbox.hpp
@@ -19,6 +19,8 @@ std::string normalizeTileURL(const std::string& url, const std::string& accessTo
// Return a "mapbox://tiles/..." URL (suitable for normalizeTileURL) for the given Mapbox tile URL.
std::string canonicalizeTileURL(const std::string& url, SourceType, uint16_t tileSize);
+extern const uint64_t DEFAULT_OFFLINE_TILE_COUNT_LIMIT;
+
} // namespace mapbox
} // namespace util
} // namespace mbgl
diff --git a/test/fixtures/offline/mapbox_source.style.json b/test/fixtures/offline/mapbox_source.style.json
new file mode 100644
index 0000000000..75bfd6667d
--- /dev/null
+++ b/test/fixtures/offline/mapbox_source.style.json
@@ -0,0 +1,17 @@
+{
+ "version": 8,
+ "sources": {
+ "inline": {
+ "type": "vector",
+ "maxzoom": 15,
+ "minzoom": 0,
+ "tiles": [ "mapbox://{z}-{x}-{y}.vector.pbf" ]
+ }
+ },
+ "layers": [{
+ "id": "fill",
+ "type": "fill",
+ "source": "inline",
+ "source-layer": "water"
+ }]
+}
diff --git a/test/storage/offline_database.cpp b/test/storage/offline_database.cpp
index d93945fb40..82e07c208a 100644
--- a/test/storage/offline_database.cpp
+++ b/test/storage/offline_database.cpp
@@ -603,3 +603,60 @@ TEST(OfflineDatabase, PutFailsWhenEvictionInsuffices) {
auto flo = dynamic_cast<FixtureLogObserver*>(observer.get());
EXPECT_EQ(1ul, flo->count({ EventSeverity::Warning, Event::Database, -1, "Unable to make space for entry" }));
}
+
+TEST(OfflineDatabase, OfflineMapboxTileCount) {
+ using namespace mbgl;
+
+ OfflineDatabase db(":memory:");
+ OfflineRegionDefinition definition { "http://example.com/style", LatLngBounds::hull({1, 2}, {3, 4}), 5, 6, 2.0 };
+ OfflineRegionMetadata metadata;
+
+ OfflineRegion region1 = db.createRegion(definition, metadata);
+ OfflineRegion region2 = db.createRegion(definition, metadata);
+
+ Resource nonMapboxTile = Resource::tile("http://example.com/", 1.0, 0, 0, 0);
+ Resource mapboxTile1 = Resource::tile("mapbox://tiles/1", 1.0, 0, 0, 0);
+ Resource mapboxTile2 = Resource::tile("mapbox://tiles/2", 1.0, 0, 0, 1);
+
+ Response response;
+ response.data = std::make_shared<std::string>("data");
+
+ // Count is initially zero.
+ EXPECT_EQ(0, db.getOfflineMapboxTileCount());
+
+ // Count stays the same after putting a non-tile resource.
+ db.putRegionResource(region1.getID(), Resource::style("http://example.com/"), response);
+ EXPECT_EQ(0, db.getOfflineMapboxTileCount());
+
+ // Count stays the same after putting a non-Mapbox tile.
+ db.putRegionResource(region1.getID(), nonMapboxTile, response);
+ EXPECT_EQ(0, db.getOfflineMapboxTileCount());
+
+ // Count increases after putting a Mapbox tile not used by another region.
+ db.putRegionResource(region1.getID(), mapboxTile1, response);
+ EXPECT_EQ(1, db.getOfflineMapboxTileCount());
+
+ // Count stays the same after putting a Mapbox tile used by another region.
+ db.putRegionResource(region2.getID(), mapboxTile1, response);
+ EXPECT_EQ(1, db.getOfflineMapboxTileCount());
+
+ // Count stays the same after putting a Mapbox tile used by the same region.
+ db.putRegionResource(region2.getID(), mapboxTile1, response);
+ EXPECT_EQ(1, db.getOfflineMapboxTileCount());
+
+ // Count stays the same after deleting a region when the tile is still used by another region.
+ db.deleteRegion(std::move(region2));
+ EXPECT_EQ(1, db.getOfflineMapboxTileCount());
+
+ // Count stays the same after the putting a non-offline Mapbox tile.
+ db.put(mapboxTile2, response);
+ EXPECT_EQ(1, db.getOfflineMapboxTileCount());
+
+ // Count increases after putting a pre-existing, but non-offline Mapbox tile.
+ db.putRegionResource(region1.getID(), mapboxTile2, response);
+ EXPECT_EQ(2, db.getOfflineMapboxTileCount());
+
+ // Count decreases after deleting a region when the tiles are not used by other regions.
+ db.deleteRegion(std::move(region1));
+ EXPECT_EQ(0, db.getOfflineMapboxTileCount());
+}
diff --git a/test/storage/offline_download.cpp b/test/storage/offline_download.cpp
index 73a6f80fcc..fce081b8ab 100644
--- a/test/storage/offline_download.cpp
+++ b/test/storage/offline_download.cpp
@@ -24,8 +24,13 @@ public:
if (responseErrorFn) responseErrorFn(error);
}
+ void mapboxTileCountLimitExceeded(uint64_t limit) override {
+ if (mapboxTileCountLimitExceededFn) mapboxTileCountLimitExceededFn(limit);
+ }
+
std::function<void (OfflineRegionStatus)> statusChangedFn;
std::function<void (Response::Error)> responseErrorFn;
+ std::function<void (uint64_t)> mapboxTileCountLimitExceededFn;
};
class OfflineTest {
@@ -347,3 +352,31 @@ TEST(OfflineDownload, RequestErrorsAreRetried) {
test.loop.run();
}
+
+TEST(OfflineDownload, TileCountLimitExceeded) {
+ OfflineTest test;
+ OfflineRegion region = test.createRegion();
+ OfflineDownload download(
+ region.getID(),
+ OfflineTilePyramidRegionDefinition("http://127.0.0.1:3000/offline/style.json", LatLngBounds::world(), 0.0, 0.0, 1.0),
+ test.db, test.fileSource);
+
+ test.db.setOfflineMapboxTileCountLimit(0);
+
+ test.fileSource.styleResponse = [&] (const Resource& resource) {
+ EXPECT_EQ("http://127.0.0.1:3000/offline/style.json", resource.url);
+ return test.response("offline/mapbox_source.style.json");
+ };
+
+ auto observer = std::make_unique<MockObserver>();
+
+ observer->mapboxTileCountLimitExceededFn = [&] (uint64_t limit) {
+ EXPECT_EQ(0, limit);
+ test.loop.stop();
+ };
+
+ download.setObserver(std::move(observer));
+ download.setState(OfflineRegionDownloadState::Active);
+
+ test.loop.run();
+}
View Lorry Controller Status