summaryrefslogtreecommitdiff
path: root/platform/default/mbgl/storage/offline_database.cpp
diff options
context:
space:
mode:
authorKonstantin Käfer <mail@kkaefer.com>2018-12-13 18:45:29 +0100
committerKonstantin Käfer <mail@kkaefer.com>2018-12-14 11:03:03 +0100
commit1d8235f5b899a2cd8414522b2d72b96fab91577b (patch)
tree2ab56dce064de872525db7f24ba150a9065c4757 /platform/default/mbgl/storage/offline_database.cpp
parentc2a4a8822ce9577c972975da61034a30fb0fe3e9 (diff)
downloadqtlocation-mapboxgl-1d8235f5b899a2cd8414522b2d72b96fab91577b.tar.gz
[build] rework platform/default directory and add -files.txt for vendored libs
Diffstat (limited to 'platform/default/mbgl/storage/offline_database.cpp')
-rw-r--r--platform/default/mbgl/storage/offline_database.cpp1129
1 files changed, 0 insertions, 1129 deletions
diff --git a/platform/default/mbgl/storage/offline_database.cpp b/platform/default/mbgl/storage/offline_database.cpp
deleted file mode 100644
index 28c1359106..0000000000
--- a/platform/default/mbgl/storage/offline_database.cpp
+++ /dev/null
@@ -1,1129 +0,0 @@
-#include <mbgl/storage/offline_database.hpp>
-#include <mbgl/storage/response.hpp>
-#include <mbgl/util/compression.hpp>
-#include <mbgl/util/io.hpp>
-#include <mbgl/util/string.hpp>
-#include <mbgl/util/chrono.hpp>
-#include <mbgl/util/logging.hpp>
-
-#include "offline_schema.hpp"
-#include "merge_sideloaded.hpp"
-
-#include "sqlite3.hpp"
-
-namespace mbgl {
-
-OfflineDatabase::OfflineDatabase(std::string path_, uint64_t maximumCacheSize_)
- : path(std::move(path_)),
- maximumCacheSize(maximumCacheSize_) {
- try {
- initialize();
- } catch (const util::IOException& ex) {
- handleError(ex, "open database");
- } catch (const mapbox::sqlite::Exception& ex) {
- handleError(ex, "open database");
- }
- // Assume that we can't open the database right now and work with an empty database object.
-}
-
-OfflineDatabase::~OfflineDatabase() {
- // Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we
- // can't throw anything.
- try {
- statements.clear();
- db.reset();
- } catch (const util::IOException& ex) {
- handleError(ex, "close database");
- } catch (const mapbox::sqlite::Exception& ex) {
- handleError(ex, "close database");
- }
-}
-
-void OfflineDatabase::initialize() {
- assert(!db);
- assert(statements.empty());
-
- db = std::make_unique<mapbox::sqlite::Database>(
- mapbox::sqlite::Database::open(path, mapbox::sqlite::ReadWriteCreate));
- db->setBusyTimeout(Milliseconds::max());
- db->exec("PRAGMA foreign_keys = ON");
-
- const auto userVersion = getPragma<int64_t>("PRAGMA user_version");
- switch (userVersion) {
- case 0:
- case 1:
- // Newly created database, or old cache-only database; remove old table if it exists.
- removeOldCacheTable();
- createSchema();
- return;
- case 2:
- migrateToVersion3();
- // fall through
- case 3:
- // Removed migration, see below.
- // fall through
- case 4:
- migrateToVersion5();
- // fall through
- case 5:
- migrateToVersion6();
- // fall through
- case 6:
- // Happy path; we're done
- return;
- default:
- // Downgrade: delete the database and try to reinitialize.
- removeExisting();
- initialize();
- }
-}
-
-void OfflineDatabase::handleError(const mapbox::sqlite::Exception& ex, const char* action) {
- if (ex.code == mapbox::sqlite::ResultCode::NotADB ||
- ex.code == mapbox::sqlite::ResultCode::Corrupt ||
- (ex.code == mapbox::sqlite::ResultCode::ReadOnly &&
- ex.extendedCode == mapbox::sqlite::ExtendedResultCode::ReadOnlyDBMoved)) {
- // The database was corruped, moved away, or deleted. We're going to start fresh with a
- // clean slate for the next operation.
- Log::Error(Event::Database, static_cast<int>(ex.code), "Can't %s: %s", action, ex.what());
- try {
- removeExisting();
- } catch (const util::IOException& ioEx) {
- handleError(ioEx, action);
- }
- } else {
- // We treat the error as temporary, and pretend we have an inaccessible DB.
- Log::Warning(Event::Database, static_cast<int>(ex.code), "Can't %s: %s", action, ex.what());
- }
-}
-
-void OfflineDatabase::handleError(const util::IOException& ex, const char* action) {
- // We failed to delete the database file.
- Log::Error(Event::Database, ex.code, "Can't %s: %s", action, ex.what());
-}
-
-void OfflineDatabase::removeExisting() {
- Log::Warning(Event::Database, "Removing existing incompatible offline database");
-
- statements.clear();
- db.reset();
-
- util::deleteFile(path);
-}
-
-void OfflineDatabase::removeOldCacheTable() {
- assert(db);
- db->exec("DROP TABLE IF EXISTS http_cache");
- db->exec("VACUUM");
-}
-
-void OfflineDatabase::createSchema() {
- assert(db);
- db->exec("PRAGMA auto_vacuum = INCREMENTAL");
- db->exec("PRAGMA journal_mode = DELETE");
- db->exec("PRAGMA synchronous = FULL");
- mapbox::sqlite::Transaction transaction(*db);
- db->exec(offlineDatabaseSchema);
- db->exec("PRAGMA user_version = 6");
- transaction.commit();
-}
-
-void OfflineDatabase::migrateToVersion3() {
- assert(db);
- db->exec("PRAGMA auto_vacuum = INCREMENTAL");
- db->exec("VACUUM");
- db->exec("PRAGMA user_version = 3");
-}
-
-// Schema version 4 was WAL journal + NORMAL sync. It was reverted during pre-
-// release development and the migration was removed entirely to avoid potential
-// conflicts from quickly (and needlessly) switching journal and sync modes.
-//
-// See: https://github.com/mapbox/mapbox-gl-native/pull/6320
-
-void OfflineDatabase::migrateToVersion5() {
- assert(db);
- db->exec("PRAGMA journal_mode = DELETE");
- db->exec("PRAGMA synchronous = FULL");
- db->exec("PRAGMA user_version = 5");
-}
-
-void OfflineDatabase::migrateToVersion6() {
- assert(db);
- mapbox::sqlite::Transaction transaction(*db);
- db->exec("ALTER TABLE resources ADD COLUMN must_revalidate INTEGER NOT NULL DEFAULT 0");
- db->exec("ALTER TABLE tiles ADD COLUMN must_revalidate INTEGER NOT NULL DEFAULT 0");
- db->exec("PRAGMA user_version = 6");
- transaction.commit();
-}
-
-mapbox::sqlite::Statement& OfflineDatabase::getStatement(const char* sql) {
- if (!db) {
- initialize();
- }
- auto it = statements.find(sql);
- if (it == statements.end()) {
- it = statements.emplace(sql, std::make_unique<mapbox::sqlite::Statement>(*db, sql)).first;
- }
- return *it->second;
-}
-
-optional<Response> OfflineDatabase::get(const Resource& resource) try {
- auto result = getInternal(resource);
- return result ? optional<Response>{ result->first } : nullopt;
-} catch (const util::IOException& ex) {
- handleError(ex, "read resource");
- return nullopt;
-} catch (const mapbox::sqlite::Exception& ex) {
- handleError(ex, "read resource");
- return nullopt;
-}
-
-optional<std::pair<Response, uint64_t>> OfflineDatabase::getInternal(const Resource& resource) {
- if (resource.kind == Resource::Kind::Tile) {
- assert(resource.tileData);
- return getTile(*resource.tileData);
- } else {
- return getResource(resource);
- }
-}
-
-optional<int64_t> OfflineDatabase::hasInternal(const Resource& resource) {
- if (resource.kind == Resource::Kind::Tile) {
- assert(resource.tileData);
- return hasTile(*resource.tileData);
- } else {
- return hasResource(resource);
- }
-}
-
-std::pair<bool, uint64_t> OfflineDatabase::put(const Resource& resource, const Response& response) try {
- if (!db) {
- initialize();
- }
- mapbox::sqlite::Transaction transaction(*db, mapbox::sqlite::Transaction::Immediate);
- auto result = putInternal(resource, response, true);
- transaction.commit();
- return result;
-} catch (const mapbox::sqlite::Exception& ex) {
- handleError(ex, "write resource");
- return { false, 0 };
-}
-
-std::pair<bool, uint64_t> OfflineDatabase::putInternal(const Resource& resource, const Response& response, bool evict_) {
- if (response.error) {
- return { false, 0 };
- }
-
- std::string compressedData;
- bool compressed = false;
- uint64_t size = 0;
-
- if (response.data) {
- compressedData = util::compress(*response.data);
- compressed = compressedData.size() < response.data->size();
- size = compressed ? compressedData.size() : response.data->size();
- }
-
- if (evict_ && !evict(size)) {
- Log::Info(Event::Database, "Unable to make space for entry");
- return { false, 0 };
- }
-
- bool inserted;
-
- if (resource.kind == Resource::Kind::Tile) {
- assert(resource.tileData);
- inserted = putTile(*resource.tileData, response,
- compressed ? compressedData : response.data ? *response.data : "",
- compressed);
- } else {
- inserted = putResource(resource, response,
- compressed ? compressedData : response.data ? *response.data : "",
- compressed);
- }
-
- return { inserted, size };
-}
-
-optional<std::pair<Response, uint64_t>> OfflineDatabase::getResource(const Resource& resource) {
- // Update accessed timestamp used for LRU eviction.
- try {
- mapbox::sqlite::Query accessedQuery{ getStatement("UPDATE resources SET accessed = ?1 WHERE url = ?2") };
- accessedQuery.bind(1, util::now());
- accessedQuery.bind(2, resource.url);
- accessedQuery.run();
- } catch (const mapbox::sqlite::Exception& ex) {
- if (ex.code == mapbox::sqlite::ResultCode::NotADB ||
- ex.code == mapbox::sqlite::ResultCode::Corrupt) {
- throw;
- }
-
- // If we don't have any indication that the database is corrupt, continue as usual.
- Log::Warning(Event::Database, static_cast<int>(ex.code), "Can't update timestamp: %s", ex.what());
- }
-
- // clang-format off
- mapbox::sqlite::Query query{ getStatement(
- // 0 1 2 3 4 5
- "SELECT etag, expires, must_revalidate, modified, data, compressed "
- "FROM resources "
- "WHERE url = ?") };
- // clang-format on
-
- query.bind(1, resource.url);
-
- if (!query.run()) {
- return nullopt;
- }
-
- Response response;
- uint64_t size = 0;
-
- response.etag = query.get<optional<std::string>>(0);
- response.expires = query.get<optional<Timestamp>>(1);
- response.mustRevalidate = query.get<bool>(2);
- response.modified = query.get<optional<Timestamp>>(3);
-
- auto data = query.get<optional<std::string>>(4);
- if (!data) {
- response.noContent = true;
- } else if (query.get<bool>(5)) {
- response.data = std::make_shared<std::string>(util::decompress(*data));
- size = data->length();
- } else {
- response.data = std::make_shared<std::string>(*data);
- size = data->length();
- }
-
- return std::make_pair(response, size);
-}
-
-optional<int64_t> OfflineDatabase::hasResource(const Resource& resource) {
- mapbox::sqlite::Query query{ getStatement("SELECT length(data) FROM resources WHERE url = ?") };
- query.bind(1, resource.url);
- if (!query.run()) {
- return nullopt;
- }
-
- return query.get<optional<int64_t>>(0);
-}
-
-bool OfflineDatabase::putResource(const Resource& resource,
- const Response& response,
- const std::string& data,
- bool compressed) {
- if (response.notModified) {
- // clang-format off
- mapbox::sqlite::Query notModifiedQuery{ getStatement(
- "UPDATE resources "
- "SET accessed = ?1, "
- " expires = ?2, "
- " must_revalidate = ?3 "
- "WHERE url = ?4 ") };
- // clang-format on
-
- notModifiedQuery.bind(1, util::now());
- notModifiedQuery.bind(2, response.expires);
- notModifiedQuery.bind(3, response.mustRevalidate);
- notModifiedQuery.bind(4, resource.url);
- notModifiedQuery.run();
- return false;
- }
-
- // We can't use REPLACE because it would change the id value.
- // clang-format off
- mapbox::sqlite::Query updateQuery{ getStatement(
- "UPDATE resources "
- "SET kind = ?1, "
- " etag = ?2, "
- " expires = ?3, "
- " must_revalidate = ?4, "
- " modified = ?5, "
- " accessed = ?6, "
- " data = ?7, "
- " compressed = ?8 "
- "WHERE url = ?9 ") };
- // clang-format on
-
- updateQuery.bind(1, int(resource.kind));
- updateQuery.bind(2, response.etag);
- updateQuery.bind(3, response.expires);
- updateQuery.bind(4, response.mustRevalidate);
- updateQuery.bind(5, response.modified);
- updateQuery.bind(6, util::now());
- updateQuery.bind(9, resource.url);
-
- if (response.noContent) {
- updateQuery.bind(7, nullptr);
- updateQuery.bind(8, false);
- } else {
- updateQuery.bindBlob(7, data.data(), data.size(), false);
- updateQuery.bind(8, compressed);
- }
-
- updateQuery.run();
- if (updateQuery.changes() != 0) {
- return false;
- }
-
- // clang-format off
- mapbox::sqlite::Query insertQuery{ getStatement(
- "INSERT INTO resources (url, kind, etag, expires, must_revalidate, modified, accessed, data, compressed) "
- "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) ") };
- // clang-format on
-
- insertQuery.bind(1, resource.url);
- insertQuery.bind(2, int(resource.kind));
- insertQuery.bind(3, response.etag);
- insertQuery.bind(4, response.expires);
- insertQuery.bind(5, response.mustRevalidate);
- insertQuery.bind(6, response.modified);
- insertQuery.bind(7, util::now());
-
- if (response.noContent) {
- insertQuery.bind(8, nullptr);
- insertQuery.bind(9, false);
- } else {
- insertQuery.bindBlob(8, data.data(), data.size(), false);
- insertQuery.bind(9, compressed);
- }
-
- insertQuery.run();
-
- return true;
-}
-
-optional<std::pair<Response, uint64_t>> OfflineDatabase::getTile(const Resource::TileData& tile) {
- // Update accessed timestamp used for LRU eviction.
- try {
- // clang-format off
- mapbox::sqlite::Query accessedQuery{ getStatement(
- "UPDATE tiles "
- "SET accessed = ?1 "
- "WHERE url_template = ?2 "
- " AND pixel_ratio = ?3 "
- " AND x = ?4 "
- " AND y = ?5 "
- " AND z = ?6 ") };
- // clang-format on
-
- accessedQuery.bind(1, util::now());
- accessedQuery.bind(2, tile.urlTemplate);
- accessedQuery.bind(3, tile.pixelRatio);
- accessedQuery.bind(4, tile.x);
- accessedQuery.bind(5, tile.y);
- accessedQuery.bind(6, tile.z);
- accessedQuery.run();
- } catch (const mapbox::sqlite::Exception& ex) {
- if (ex.code == mapbox::sqlite::ResultCode::NotADB || ex.code == mapbox::sqlite::ResultCode::Corrupt) {
- throw;
- }
-
- // If we don't have any indication that the database is corrupt, continue as usual.
- Log::Warning(Event::Database, static_cast<int>(ex.code), "Can't update timestamp: %s", ex.what());
- }
-
- // clang-format off
- mapbox::sqlite::Query query{ getStatement(
- // 0 1 2, 3, 4, 5
- "SELECT etag, expires, must_revalidate, modified, data, compressed "
- "FROM tiles "
- "WHERE url_template = ?1 "
- " AND pixel_ratio = ?2 "
- " AND x = ?3 "
- " AND y = ?4 "
- " AND z = ?5 ") };
- // clang-format on
-
- query.bind(1, tile.urlTemplate);
- query.bind(2, tile.pixelRatio);
- query.bind(3, tile.x);
- query.bind(4, tile.y);
- query.bind(5, tile.z);
-
- if (!query.run()) {
- return nullopt;
- }
-
- Response response;
- uint64_t size = 0;
-
- response.etag = query.get<optional<std::string>>(0);
- response.expires = query.get<optional<Timestamp>>(1);
- response.mustRevalidate = query.get<bool>(2);
- response.modified = query.get<optional<Timestamp>>(3);
-
- optional<std::string> data = query.get<optional<std::string>>(4);
- if (!data) {
- response.noContent = true;
- } else if (query.get<bool>(5)) {
- response.data = std::make_shared<std::string>(util::decompress(*data));
- size = data->length();
- } else {
- response.data = std::make_shared<std::string>(*data);
- size = data->length();
- }
-
- return std::make_pair(response, size);
-}
-
-optional<int64_t> OfflineDatabase::hasTile(const Resource::TileData& tile) {
- // clang-format off
- mapbox::sqlite::Query size{ getStatement(
- "SELECT length(data) "
- "FROM tiles "
- "WHERE url_template = ?1 "
- " AND pixel_ratio = ?2 "
- " AND x = ?3 "
- " AND y = ?4 "
- " AND z = ?5 ") };
- // clang-format on
-
- size.bind(1, tile.urlTemplate);
- size.bind(2, tile.pixelRatio);
- size.bind(3, tile.x);
- size.bind(4, tile.y);
- size.bind(5, tile.z);
-
- if (!size.run()) {
- return nullopt;
- }
-
- return size.get<optional<int64_t>>(0);
-}
-
-bool OfflineDatabase::putTile(const Resource::TileData& tile,
- const Response& response,
- const std::string& data,
- bool compressed) {
- if (response.notModified) {
- // clang-format off
- mapbox::sqlite::Query notModifiedQuery{ getStatement(
- "UPDATE tiles "
- "SET accessed = ?1, "
- " expires = ?2, "
- " must_revalidate = ?3 "
- "WHERE url_template = ?4 "
- " AND pixel_ratio = ?5 "
- " AND x = ?6 "
- " AND y = ?7 "
- " AND z = ?8 ") };
- // clang-format on
-
- notModifiedQuery.bind(1, util::now());
- notModifiedQuery.bind(2, response.expires);
- notModifiedQuery.bind(3, response.mustRevalidate);
- notModifiedQuery.bind(4, tile.urlTemplate);
- notModifiedQuery.bind(5, tile.pixelRatio);
- notModifiedQuery.bind(6, tile.x);
- notModifiedQuery.bind(7, tile.y);
- notModifiedQuery.bind(8, tile.z);
- notModifiedQuery.run();
- return false;
- }
-
- // We can't use REPLACE because it would change the id value.
-
- // clang-format off
- mapbox::sqlite::Query updateQuery{ getStatement(
- "UPDATE tiles "
- "SET modified = ?1, "
- " etag = ?2, "
- " expires = ?3, "
- " must_revalidate = ?4, "
- " accessed = ?5, "
- " data = ?6, "
- " compressed = ?7 "
- "WHERE url_template = ?8 "
- " AND pixel_ratio = ?9 "
- " AND x = ?10 "
- " AND y = ?11 "
- " AND z = ?12 ") };
- // clang-format on
-
- updateQuery.bind(1, response.modified);
- updateQuery.bind(2, response.etag);
- updateQuery.bind(3, response.expires);
- updateQuery.bind(4, response.mustRevalidate);
- updateQuery.bind(5, util::now());
- updateQuery.bind(8, tile.urlTemplate);
- updateQuery.bind(9, tile.pixelRatio);
- updateQuery.bind(10, tile.x);
- updateQuery.bind(11, tile.y);
- updateQuery.bind(12, tile.z);
-
- if (response.noContent) {
- updateQuery.bind(6, nullptr);
- updateQuery.bind(7, false);
- } else {
- updateQuery.bindBlob(6, data.data(), data.size(), false);
- updateQuery.bind(7, compressed);
- }
-
- updateQuery.run();
- if (updateQuery.changes() != 0) {
- return false;
- }
-
- // clang-format off
- mapbox::sqlite::Query insertQuery{ getStatement(
- "INSERT INTO tiles (url_template, pixel_ratio, x, y, z, modified, must_revalidate, etag, expires, accessed, data, compressed) "
- "VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)") };
- // clang-format on
-
- insertQuery.bind(1, tile.urlTemplate);
- insertQuery.bind(2, tile.pixelRatio);
- insertQuery.bind(3, tile.x);
- insertQuery.bind(4, tile.y);
- insertQuery.bind(5, tile.z);
- insertQuery.bind(6, response.modified);
- insertQuery.bind(7, response.mustRevalidate);
- insertQuery.bind(8, response.etag);
- insertQuery.bind(9, response.expires);
- insertQuery.bind(10, util::now());
-
- if (response.noContent) {
- insertQuery.bind(11, nullptr);
- insertQuery.bind(12, false);
- } else {
- insertQuery.bindBlob(11, data.data(), data.size(), false);
- insertQuery.bind(12, compressed);
- }
-
- insertQuery.run();
-
- return true;
-}
-
-expected<OfflineRegions, std::exception_ptr> OfflineDatabase::listRegions() try {
- mapbox::sqlite::Query query{ getStatement("SELECT id, definition, description FROM regions") };
- OfflineRegions result;
- while (query.run()) {
- const auto id = query.get<int64_t>(0);
- const auto definition = query.get<std::string>(1);
- const auto description = query.get<std::vector<uint8_t>>(2);
- try {
- // Construct, then move because this constructor is private.
- OfflineRegion region(id, decodeOfflineRegionDefinition(definition), description);
- result.emplace_back(std::move(region));
- } catch (const std::exception& ex) {
- // Catch errors from malformed offline region definitions
- // and skip them.
- Log::Error(Event::General, "%s", ex.what());
- }
- }
- // Explicit move to avoid triggering the copy constructor.
- return { std::move(result) };
-} catch (const mapbox::sqlite::Exception& ex) {
- handleError(ex, "list regions");
- return unexpected<std::exception_ptr>(std::current_exception());
-}
-
-expected<OfflineRegion, std::exception_ptr>
-OfflineDatabase::createRegion(const OfflineRegionDefinition& definition,
- const OfflineRegionMetadata& metadata) try {
- // clang-format off
- mapbox::sqlite::Query query{ getStatement(
- "INSERT INTO regions (definition, description) "
- "VALUES (?1, ?2) ") };
- // clang-format on
-
- query.bind(1, encodeOfflineRegionDefinition(definition));
- query.bindBlob(2, metadata);
- query.run();
- return OfflineRegion(query.lastInsertRowId(), definition, metadata);
-} catch (const mapbox::sqlite::Exception& ex) {
- handleError(ex, "create region");
- return unexpected<std::exception_ptr>(std::current_exception());
-}
-
-expected<OfflineRegions, std::exception_ptr>
-OfflineDatabase::mergeDatabase(const std::string& sideDatabasePath) {
- try {
- // clang-format off
- mapbox::sqlite::Query query{ getStatement("ATTACH DATABASE ?1 AS side") };
- // clang-format on
-
- query.bind(1, sideDatabasePath);
- query.run();
- } catch (const mapbox::sqlite::Exception& ex) {
- Log::Error(Event::Database, static_cast<int>(ex.code), "Can't attach database (%s) for merge: %s", sideDatabasePath.c_str(), ex.what());
- return unexpected<std::exception_ptr>(std::current_exception());
- }
- try {
- // Support sideloaded databases at user_version = 6. Future schema version
- // changes will need to implement migration paths for sideloaded databases at
- // version 6.
- auto sideUserVersion = static_cast<int>(getPragma<int64_t>("PRAGMA side.user_version"));
- const auto mainUserVersion = getPragma<int64_t>("PRAGMA user_version");
- if (sideUserVersion < 6 || sideUserVersion != mainUserVersion) {
- throw std::runtime_error("Merge database has incorrect user_version");
- }
-
- auto currentTileCount = getOfflineMapboxTileCount();
- // clang-format off
- mapbox::sqlite::Query queryTiles{ getStatement(
- "SELECT COUNT(DISTINCT st.id) "
- "FROM side.tiles st "
- //only consider region tiles, and not ambient tiles.
- "JOIN side.region_tiles srt ON srt.tile_id = st.id "
- "LEFT JOIN tiles t ON st.url_template = t.url_template AND "
- "st.pixel_ratio = t.pixel_ratio AND "
- "st.z = t.z AND "
- "st.x = t.x AND "
- "st.y = t.y "
- "WHERE t.id IS NULL "
- "AND st.url_template LIKE 'mapbox://%' ") };
- // clang-format on
- queryTiles.run();
- auto countOfTilesToMerge = queryTiles.get<int64_t>(0);
- if ((countOfTilesToMerge + currentTileCount) > offlineMapboxTileCountLimit) {
- throw MapboxTileLimitExceededException();
- }
- queryTiles.reset();
-
- mapbox::sqlite::Transaction transaction(*db);
- db->exec(mergeSideloadedDatabaseSQL);
- transaction.commit();
-
- // clang-format off
- mapbox::sqlite::Query queryRegions{ getStatement(
- "SELECT DISTINCT r.id, r.definition, r.description "
- "FROM side.regions sr "
- "JOIN regions r ON sr.definition = r.definition AND sr.description IS r.description") };
- // clang-format on
-
- OfflineRegions result;
- while (queryRegions.run()) {
- // Construct, then move because this constructor is private.
- OfflineRegion region(queryRegions.get<int64_t>(0),
- decodeOfflineRegionDefinition(queryRegions.get<std::string>(1)),
- queryRegions.get<std::vector<uint8_t>>(2));
- result.emplace_back(std::move(region));
- }
- db->exec("DETACH DATABASE side");
- // Explicit move to avoid triggering the copy constructor.
- return { std::move(result) };
- } catch (const std::runtime_error& ex) {
- db->exec("DETACH DATABASE side");
- Log::Error(Event::Database, "%s", ex.what());
-
- return unexpected<std::exception_ptr>(std::current_exception());
- }
- return {};
-}
-
-expected<OfflineRegionMetadata, std::exception_ptr>
-OfflineDatabase::updateMetadata(const int64_t regionID, const OfflineRegionMetadata& metadata) try {
- // clang-format off
- mapbox::sqlite::Query query{ getStatement(
- "UPDATE regions SET description = ?1 "
- "WHERE id = ?2") };
- // clang-format on
- query.bindBlob(1, metadata);
- query.bind(2, regionID);
- query.run();
-
- return metadata;
-} catch (const mapbox::sqlite::Exception& ex) {
- handleError(ex, "update region metadata");
- return unexpected<std::exception_ptr>(std::current_exception());
-}
-
-std::exception_ptr OfflineDatabase::deleteRegion(OfflineRegion&& region) try {
- {
- mapbox::sqlite::Query query{ getStatement("DELETE FROM regions WHERE id = ?") };
- query.bind(1, region.getID());
- query.run();
- }
-
- evict(0);
- assert(db);
- db->exec("PRAGMA incremental_vacuum");
-
- // Ensure that the cached offlineTileCount value is recalculated.
- offlineMapboxTileCount = {};
- return nullptr;
-} catch (const mapbox::sqlite::Exception& ex) {
- handleError(ex, "delete region");
- return std::current_exception();
-}
-
-optional<std::pair<Response, uint64_t>> OfflineDatabase::getRegionResource(int64_t regionID, const Resource& resource) try {
- auto response = getInternal(resource);
-
- if (response) {
- markUsed(regionID, resource);
- }
-
- return response;
-} catch (const mapbox::sqlite::Exception& ex) {
- handleError(ex, "read region resource");
- return nullopt;
-}
-
-optional<int64_t> OfflineDatabase::hasRegionResource(int64_t regionID, const Resource& resource) try {
- auto response = hasInternal(resource);
-
- if (response) {
- markUsed(regionID, resource);
- }
-
- return response;
-} catch (const mapbox::sqlite::Exception& ex) {
- handleError(ex, "query region resource");
- return nullopt;
-}
-
-uint64_t OfflineDatabase::putRegionResource(int64_t regionID,
- const Resource& resource,
- const Response& response) try {
- if (!db) {
- initialize();
- }
- mapbox::sqlite::Transaction transaction(*db);
- auto size = putRegionResourceInternal(regionID, resource, response);
- transaction.commit();
- return size;
-} catch (const mapbox::sqlite::Exception& ex) {
- handleError(ex, "write region resource");
- return 0;
-}
-
-void OfflineDatabase::putRegionResources(int64_t regionID,
- const std::list<std::tuple<Resource, Response>>& resources,
- OfflineRegionStatus& status) try {
- if (!db) {
- initialize();
- }
- mapbox::sqlite::Transaction transaction(*db);
-
- // Accumulate all statistics locally first before adding them to the OfflineRegionStatus object
- // to ensure correctness when the transaction fails.
- uint64_t completedResourceCount = 0;
- uint64_t completedResourceSize = 0;
- uint64_t completedTileCount = 0;
- uint64_t completedTileSize = 0;
-
- for (const auto& elem : resources) {
- const auto& resource = std::get<0>(elem);
- const auto& response = std::get<1>(elem);
-
- try {
- uint64_t resourceSize = putRegionResourceInternal(regionID, resource, response);
- completedResourceCount++;
- completedResourceSize += resourceSize;
- if (resource.kind == Resource::Kind::Tile) {
- completedTileCount += 1;
- completedTileSize += resourceSize;
- }
- } catch (const MapboxTileLimitExceededException&) {
- // Commit the rest of the batch and rethrow
- transaction.commit();
- throw;
- }
- }
-
- // Commit the completed batch
- transaction.commit();
-
- status.completedResourceCount += completedResourceCount;
- status.completedResourceSize += completedResourceSize;
- status.completedTileCount += completedTileCount;
- status.completedTileSize += completedTileSize;
-} catch (const mapbox::sqlite::Exception& ex) {
- handleError(ex, "write region resources");
-}
-
-uint64_t OfflineDatabase::putRegionResourceInternal(int64_t regionID, const Resource& resource, const Response& response) {
- if (exceedsOfflineMapboxTileCountLimit(resource)) {
- throw MapboxTileLimitExceededException();
- }
-
- uint64_t size = putInternal(resource, response, false).second;
- bool previouslyUnused = markUsed(regionID, resource);
-
- if (offlineMapboxTileCount
- && resource.kind == Resource::Kind::Tile
- && util::mapbox::isMapboxURL(resource.url)
- && previouslyUnused) {
- *offlineMapboxTileCount += 1;
- }
-
- return size;
-}
-
-bool OfflineDatabase::markUsed(int64_t regionID, const Resource& resource) {
- if (resource.kind == Resource::Kind::Tile) {
- // clang-format off
- mapbox::sqlite::Query insertQuery{ getStatement(
- "INSERT OR IGNORE INTO region_tiles (region_id, tile_id) "
- "SELECT ?1, tiles.id "
- "FROM tiles "
- "WHERE url_template = ?2 "
- " AND pixel_ratio = ?3 "
- " AND x = ?4 "
- " AND y = ?5 "
- " AND z = ?6 ") };
- // clang-format on
-
- const Resource::TileData& tile = *resource.tileData;
- insertQuery.bind(1, regionID);
- insertQuery.bind(2, tile.urlTemplate);
- insertQuery.bind(3, tile.pixelRatio);
- insertQuery.bind(4, tile.x);
- insertQuery.bind(5, tile.y);
- insertQuery.bind(6, tile.z);
- insertQuery.run();
-
- if (insertQuery.changes() == 0) {
- return false;
- }
-
- // clang-format off
- mapbox::sqlite::Query selectQuery{ getStatement(
- "SELECT region_id "
- "FROM region_tiles, tiles "
- "WHERE region_id != ?1 "
- " AND url_template = ?2 "
- " AND pixel_ratio = ?3 "
- " AND x = ?4 "
- " AND y = ?5 "
- " AND z = ?6 "
- "LIMIT 1 ") };
- // clang-format on
-
- selectQuery.bind(1, regionID);
- selectQuery.bind(2, tile.urlTemplate);
- selectQuery.bind(3, tile.pixelRatio);
- selectQuery.bind(4, tile.x);
- selectQuery.bind(5, tile.y);
- selectQuery.bind(6, tile.z);
- return !selectQuery.run();
- } else {
- // clang-format off
- mapbox::sqlite::Query insertQuery{ getStatement(
- "INSERT OR IGNORE INTO region_resources (region_id, resource_id) "
- "SELECT ?1, resources.id "
- "FROM resources "
- "WHERE resources.url = ?2 ") };
- // clang-format on
-
- insertQuery.bind(1, regionID);
- insertQuery.bind(2, resource.url);
- insertQuery.run();
-
- if (insertQuery.changes() == 0) {
- return false;
- }
-
- // clang-format off
- mapbox::sqlite::Query selectQuery{ getStatement(
- "SELECT region_id "
- "FROM region_resources, resources "
- "WHERE region_id != ?1 "
- " AND resources.url = ?2 "
- "LIMIT 1 ") };
- // clang-format on
-
- selectQuery.bind(1, regionID);
- selectQuery.bind(2, resource.url);
- return !selectQuery.run();
- }
-}
-
-expected<OfflineRegionDefinition, std::exception_ptr> OfflineDatabase::getRegionDefinition(int64_t regionID) try {
- mapbox::sqlite::Query query{ getStatement("SELECT definition FROM regions WHERE id = ?1") };
- query.bind(1, regionID);
- query.run();
-
- return decodeOfflineRegionDefinition(query.get<std::string>(0));
-} catch (const mapbox::sqlite::Exception& ex) {
- handleError(ex, "load region");
- return unexpected<std::exception_ptr>(std::current_exception());
-}
-
-expected<OfflineRegionStatus, std::exception_ptr> OfflineDatabase::getRegionCompletedStatus(int64_t regionID) try {
- OfflineRegionStatus result;
-
- std::tie(result.completedResourceCount, result.completedResourceSize)
- = getCompletedResourceCountAndSize(regionID);
- std::tie(result.completedTileCount, result.completedTileSize)
- = getCompletedTileCountAndSize(regionID);
-
- result.completedResourceCount += result.completedTileCount;
- result.completedResourceSize += result.completedTileSize;
-
- return result;
-} catch (const mapbox::sqlite::Exception& ex) {
- handleError(ex, "get region status");
- return unexpected<std::exception_ptr>(std::current_exception());
-}
-
-std::pair<int64_t, int64_t> OfflineDatabase::getCompletedResourceCountAndSize(int64_t regionID) {
- // clang-format off
- mapbox::sqlite::Query query{ getStatement(
- "SELECT COUNT(*), SUM(LENGTH(data)) "
- "FROM region_resources, resources "
- "WHERE region_id = ?1 "
- "AND resource_id = resources.id ") };
- // clang-format on
- query.bind(1, regionID);
- query.run();
- return { query.get<int64_t>(0), query.get<int64_t>(1) };
-}
-
-std::pair<int64_t, int64_t> OfflineDatabase::getCompletedTileCountAndSize(int64_t regionID) {
- // clang-format off
- mapbox::sqlite::Query query{ getStatement(
- "SELECT COUNT(*), SUM(LENGTH(data)) "
- "FROM region_tiles, tiles "
- "WHERE region_id = ?1 "
- "AND tile_id = tiles.id ") };
- // clang-format on
- query.bind(1, regionID);
- query.run();
- return { query.get<int64_t>(0), query.get<int64_t>(1) };
-}
-
-template <class T>
-T OfflineDatabase::getPragma(const char* sql) {
- mapbox::sqlite::Query query{ getStatement(sql) };
- query.run();
- return query.get<T>(0);
-}
-
-// Remove least-recently used resources and tiles until the used database size,
-// as calculated by multiplying the number of in-use pages by the page size, is
-// less than the maximum cache size. Returns false if this condition cannot be
-// satisfied.
-//
-// SQLite database never shrinks in size unless we call VACCUM. We here
-// are monitoring the soft limit (i.e. number of free pages in the file)
-// and as it approaches to the hard limit (i.e. the actual file size) we
-// delete an arbitrary number of old cache entries. The free pages approach saves
-// us from calling VACCUM or keeping a running total, which can be costly.
-bool OfflineDatabase::evict(uint64_t neededFreeSize) {
- uint64_t pageSize = getPragma<int64_t>("PRAGMA page_size");
- uint64_t pageCount = getPragma<int64_t>("PRAGMA page_count");
-
- auto usedSize = [&] {
- return pageSize * (pageCount - getPragma<int64_t>("PRAGMA freelist_count"));
- };
-
- // The addition of pageSize is a fudge factor to account for non `data` column
- // size, and because pages can get fragmented on the database.
- while (usedSize() + neededFreeSize + pageSize > maximumCacheSize) {
- // clang-format off
- mapbox::sqlite::Query accessedQuery{ getStatement(
- "SELECT max(accessed) "
- "FROM ( "
- " SELECT accessed "
- " FROM resources "
- " LEFT JOIN region_resources "
- " ON resource_id = resources.id "
- " WHERE resource_id IS NULL "
- " UNION ALL "
- " SELECT accessed "
- " FROM tiles "
- " LEFT JOIN region_tiles "
- " ON tile_id = tiles.id "
- " WHERE tile_id IS NULL "
- " ORDER BY accessed ASC LIMIT ?1 "
- ") "
- ) };
- accessedQuery.bind(1, 50);
- // clang-format on
- if (!accessedQuery.run()) {
- return false;
- }
- Timestamp accessed = accessedQuery.get<Timestamp>(0);
-
- // clang-format off
- mapbox::sqlite::Query resourceQuery{ getStatement(
- "DELETE FROM resources "
- "WHERE id IN ( "
- " SELECT id FROM resources "
- " LEFT JOIN region_resources "
- " ON resource_id = resources.id "
- " WHERE resource_id IS NULL "
- " AND accessed <= ?1 "
- ") ") };
- // clang-format on
- resourceQuery.bind(1, accessed);
- resourceQuery.run();
- const uint64_t resourceChanges = resourceQuery.changes();
-
- // clang-format off
- mapbox::sqlite::Query tileQuery{ getStatement(
- "DELETE FROM tiles "
- "WHERE id IN ( "
- " SELECT id FROM tiles "
- " LEFT JOIN region_tiles "
- " ON tile_id = tiles.id "
- " WHERE tile_id IS NULL "
- " AND accessed <= ?1 "
- ") ") };
- // clang-format on
- tileQuery.bind(1, accessed);
- tileQuery.run();
- const uint64_t tileChanges = tileQuery.changes();
-
- // The cached value of offlineTileCount does not need to be updated
- // here because only non-offline tiles can be removed by eviction.
-
- if (resourceChanges == 0 && tileChanges == 0) {
- return false;
- }
- }
-
- return true;
-}
-
-void OfflineDatabase::setOfflineMapboxTileCountLimit(uint64_t limit) {
- offlineMapboxTileCountLimit = limit;
-}
-
-uint64_t OfflineDatabase::getOfflineMapboxTileCountLimit() {
- return offlineMapboxTileCountLimit;
-}
-
-bool OfflineDatabase::offlineMapboxTileCountLimitExceeded() {
- return getOfflineMapboxTileCount() >= offlineMapboxTileCountLimit;
-}
-
-uint64_t OfflineDatabase::getOfflineMapboxTileCount() try {
- // Calculating this on every call would be much simpler than caching and
- // manually updating the value, but it would make offline downloads an O(n²)
- // operation, because the database query below involves an index scan of
- // region_tiles.
-
- if (offlineMapboxTileCount) {
- return *offlineMapboxTileCount;
- }
-
- // clang-format off
- mapbox::sqlite::Query query{ getStatement(
- "SELECT COUNT(DISTINCT id) "
- "FROM region_tiles, tiles "
- "WHERE tile_id = tiles.id "
- "AND url_template LIKE 'mapbox://%' ") };
- // clang-format on
-
- query.run();
-
- offlineMapboxTileCount = query.get<int64_t>(0);
- return *offlineMapboxTileCount;
-} catch (const mapbox::sqlite::Exception& ex) {
- handleError(ex, "get offline Mapbox tile count");
- return std::numeric_limits<uint64_t>::max();
-}
-
-bool OfflineDatabase::exceedsOfflineMapboxTileCountLimit(const Resource& resource) {
- return resource.kind == Resource::Kind::Tile
- && util::mapbox::isMapboxURL(resource.url)
- && offlineMapboxTileCountLimitExceeded();
-}
-
-} // namespace mbgl