[core] Reimplement existing caching within an offline-capable database schema

9 files changed, 605 insertions, 619 deletions

diff --git a/platform/default/default_file_source.cpp b/platform/default/default_file_source.cpp
index 56b47539b1..efe893d49b 100644
--- a/platform/default/default_file_source.cpp
+++ b/platform/default/default_file_source.cpp
@@ -1,10 +1,11 @@
 #include <mbgl/storage/default_file_source.hpp>
 #include <mbgl/storage/asset_file_source.hpp>
 #include <mbgl/storage/online_file_source.hpp>
-#include <mbgl/storage/sqlite_cache.hpp>
+#include <mbgl/storage/offline_database.hpp>
 
 #include <mbgl/platform/platform.hpp>
 #include <mbgl/util/url.hpp>
+#include <mbgl/util/thread.hpp>
 #include <mbgl/util/work_request.hpp>
 
 #include <cassert>
@@ -23,76 +24,117 @@ namespace mbgl {
 
 class DefaultFileSource::Impl {
 public:
-    Impl(const std::string& cachePath, const std::string& assetRoot)
-        : assetFileSource(assetRoot),
-          cache(SQLiteCache::getShared(cachePath)) {
+    class Task {
+    public:
+        Task(Resource resource, FileSource::Callback callback, DefaultFileSource::Impl* impl) {
+            auto offlineResponse = impl->offlineDatabase.get(resource);
+
+            Resource revalidation = resource;
+
+            if (offlineResponse) {
+                revalidation.priorModified = offlineResponse->modified;
+                revalidation.priorExpires = offlineResponse->expires;
+                revalidation.priorEtag = offlineResponse->etag;
+                callback(*offlineResponse);
+            }
+
+            if (!impl->offline) {
+                onlineRequest = impl->onlineFileSource.request(revalidation, [=] (Response onlineResponse) {
+                    impl->offlineDatabase.put(revalidation, onlineResponse);
+                    callback(onlineResponse);
+                });
+            }
+        }
+
+        std::unique_ptr<FileRequest> onlineRequest;
+    };
+
+    Impl(const std::string& cachePath)
+        : offlineDatabase(cachePath) {
+    }
+
+    void setAccessToken(const std::string& accessToken) {
+        onlineFileSource.setAccessToken(accessToken);
+    }
+
+    std::string getAccessToken() const {
+        return onlineFileSource.getAccessToken();
+    }
+
+    void add(FileRequest* req, Resource resource, Callback callback) {
+        tasks[req] = std::make_unique<Task>(resource, callback, this);
     }
 
-    AssetFileSource assetFileSource;
-    std::shared_ptr<SQLiteCache> cache;
+    void cancel(FileRequest* req) {
+        tasks.erase(req);
+    }
+
+    void put(const Resource& resource, const Response& response) {
+        offlineDatabase.put(resource, response);
+    }
+
+    void goOffline() {
+        offline = true;
+    }
+
+    OfflineDatabase offlineDatabase;
     OnlineFileSource onlineFileSource;
+    std::unordered_map<FileRequest*, std::unique_ptr<Task>> tasks;
+    bool offline = false;
+};
+
+class DefaultFileRequest : public FileRequest {
+public:
+    DefaultFileRequest(Resource     resource, FileSource::Callback callback, util::Thread<DefaultFileSource::Impl>& thread_)
+        : thread(thread_),
+          workRequest(thread.invokeWithCallback(&DefaultFileSource::Impl::add, callback, this, resource)) {
+    }
+
+    ~DefaultFileRequest() {
+        thread.invoke(&DefaultFileSource::Impl::cancel, this);
+    }
+
+    util::Thread<DefaultFileSource::Impl>& thread;
+    std::unique_ptr<WorkRequest> workRequest;
 };
 
 DefaultFileSource::DefaultFileSource(const std::string& cachePath, const std::string& assetRoot)
-    : impl(std::make_unique<DefaultFileSource::Impl>(cachePath, assetRoot)) {
+    : thread(std::make_unique<util::Thread<DefaultFileSource::Impl>>(util::ThreadContext{"DefaultFileSource", util::ThreadType::Unknown, util::ThreadPriority::Low}, cachePath)),
+      assetFileSource(std::make_unique<AssetFileSource>(assetRoot)) {
 }
 
 DefaultFileSource::~DefaultFileSource() = default;
 
 void DefaultFileSource::setAccessToken(const std::string& accessToken) {
-    impl->onlineFileSource.setAccessToken(accessToken);
+    thread->invokeSync(&Impl::setAccessToken, accessToken);
 }
 
 std::string DefaultFileSource::getAccessToken() const {
-    return impl->onlineFileSource.getAccessToken();
-}
-
-void DefaultFileSource::setMaximumCacheSize(uint64_t size) {
-    impl->cache->setMaximumCacheSize(size);
+    return thread->invokeSync<std::string>(&Impl::getAccessToken);
 }
 
-void DefaultFileSource::setMaximumCacheEntrySize(uint64_t size) {
-    impl->cache->setMaximumCacheEntrySize(size);
+void DefaultFileSource::setMaximumCacheSize(uint64_t) {
+    // TODO
 }
 
-SQLiteCache& DefaultFileSource::getCache() {
-    return *impl->cache;
+void DefaultFileSource::setMaximumCacheEntrySize(uint64_t) {
+    // TODO
 }
 
-class DefaultFileRequest : public FileRequest {
-public:
-    DefaultFileRequest(Resource resource, FileSource::Callback callback, DefaultFileSource::Impl* impl) {
-        cacheRequest = impl->cache->get(resource, [=](std::shared_ptr<Response> cacheResponse) mutable {
-            cacheRequest.reset();
-
-            if (cacheResponse) {
-                resource.priorModified = cacheResponse->modified;
-                resource.priorExpires = cacheResponse->expires;
-                resource.priorEtag = cacheResponse->etag;
-            }
-
-            onlineRequest = impl->onlineFileSource.request(resource, [=] (Response onlineResponse) {
-                impl->cache->put(resource, onlineResponse);
-                callback(onlineResponse);
-            });
-
-            // Do this last because it may result in deleting this DefaultFileRequest.
-            if (cacheResponse) {
-                callback(*cacheResponse);
-            }
-        });
-    }
-
-    std::unique_ptr<WorkRequest> cacheRequest;
-    std::unique_ptr<FileRequest> onlineRequest;
-};
-
 std::unique_ptr<FileRequest> DefaultFileSource::request(const Resource& resource, Callback callback) {
     if (isAssetURL(resource.url)) {
-        return impl->assetFileSource.request(resource, callback);
+        return assetFileSource->request(resource, callback);
     } else {
-        return std::make_unique<DefaultFileRequest>(resource, callback, impl.get());
+        return std::make_unique<DefaultFileRequest>(resource, callback, *thread);
     }
 }
 
+void DefaultFileSource::put(const Resource& resource, const Response& response) {
+    thread->invokeSync(&Impl::put, resource, response);
+}
+
+void DefaultFileSource::goOffline() {
+    thread->invokeSync(&Impl::goOffline);
+}
+
 } // namespace mbgl
diff --git a/platform/default/mbgl/storage/offline_database.cpp b/platform/default/mbgl/storage/offline_database.cpp
new file mode 100644
index 0000000000..cd0f88b7fe
--- /dev/null
+++ b/platform/default/mbgl/storage/offline_database.cpp
@@ -0,0 +1,297 @@
+#include <mbgl/storage/offline_database.hpp>
+#include <mbgl/storage/response.hpp>
+#include <mbgl/util/compression.hpp>
+#include <mbgl/util/io.hpp>
+#include <mbgl/util/string.hpp>
+#include <mbgl/util/chrono.hpp>
+#include <mbgl/map/tile_id.hpp>
+#include <mbgl/platform/log.hpp>
+
+#include "sqlite3.hpp"
+#include <sqlite3.h>
+
+namespace mbgl {
+
+using namespace mapbox::sqlite;
+
+// If you change the schema you must write a migration from the previous version.
+static const uint32_t schemaVersion = 2;
+
+OfflineDatabase::OfflineDatabase(const std::string& path_)
+    : path(path_) {
+    ensureSchema();
+}
+
+OfflineDatabase::~OfflineDatabase() {
+    // Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we
+    // can't throw anything.
+    try {
+        statements.clear();
+        db.reset();
+    } catch (mapbox::sqlite::Exception& ex) {
+        Log::Error(Event::Database, ex.code, ex.what());
+    }
+}
+
+void OfflineDatabase::ensureSchema() {
+    if (path != ":memory:") {
+        try {
+            db = std::make_unique<Database>(path.c_str(), ReadWrite);
+
+            {
+                Statement userVersionStmt(db->prepare("PRAGMA user_version"));
+                userVersionStmt.run();
+                switch (userVersionStmt.get<int>(0)) {
+                case 0: break; // cache-only database; ok to delete
+                case 1: break; // cache-only database; ok to delete
+                case 2: return;
+                default: throw std::runtime_error("unknown schema version");
+                }
+            }
+
+            removeExisting();
+            db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
+        } catch (mapbox::sqlite::Exception& ex) {
+            if (ex.code == SQLITE_CANTOPEN) {
+                db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
+            } else if (ex.code == SQLITE_NOTADB) {
+                removeExisting();
+                db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
+            }
+        }
+    }
+
+    #include "offline_schema.cpp.include"
+
+    db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
+    db->exec(schema);
+    db->exec("PRAGMA user_version = " + util::toString(schemaVersion));
+}
+
+void OfflineDatabase::removeExisting() {
+    Log::Warning(Event::Database, "Removing existing incompatible offline database");
+
+    db.reset();
+
+    try {
+        util::deleteFile(path);
+    } catch (util::IOException& ex) {
+        Log::Error(Event::Database, ex.code, ex.what());
+    }
+}
+
+mapbox::sqlite::Statement& OfflineDatabase::getStatement(const char * sql) {
+    auto it = statements.find(sql);
+
+    if (it != statements.end()) {
+        it->second->reset();
+        return *it->second;
+    }
+
+    return *(statements[sql] = std::make_unique<Statement>(db->prepare(sql)));
+}
+
+optional<Response> OfflineDatabase::get(const Resource& resource) {
+    if (resource.kind == Resource::Kind::Tile) {
+        assert(resource.tileData);
+        return getTile(*resource.tileData);
+    } else {
+        return getResource(resource);
+    }
+}
+
+void OfflineDatabase::put(const Resource& resource, const Response& response) {
+    // Except for 404s, don't store errors in the cache.
+    if (response.error && response.error->reason != Response::Error::Reason::NotFound) {
+        return;
+    }
+
+    if (resource.kind == Resource::Kind::Tile) {
+        assert(resource.tileData);
+        putTile(*resource.tileData, response);
+    } else {
+        putResource(resource, response);
+    }
+}
+
+optional<Response> OfflineDatabase::getResource(const Resource& resource) {
+    mapbox::sqlite::Statement& stmt = getStatement(
+        //        0      1        2       3        4
+        "SELECT etag, expires, modified, data, compressed "
+        "FROM resources "
+        "WHERE url = ?");
+
+    stmt.bind(1, resource.url);
+
+    if (!stmt.run()) {
+        return {};
+    }
+
+    Response response;
+
+    response.etag     = stmt.get<optional<std::string>>(0);
+    response.expires  = stmt.get<optional<SystemTimePoint>>(1);
+    response.modified = stmt.get<optional<SystemTimePoint>>(2);
+
+    optional<std::string> data = stmt.get<optional<std::string>>(3);
+    if (!data) {
+        response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
+    } else if (stmt.get<int>(4)) {
+        response.data = std::make_shared<std::string>(util::decompress(*data));
+    } else {
+        response.data = std::make_shared<std::string>(*data);
+    }
+
+    return response;
+}
+
+void OfflineDatabase::putResource(const Resource& resource, const Response& response) {
+    if (response.notModified) {
+        mapbox::sqlite::Statement& stmt = getStatement(
+            //                               1            2             3
+            "UPDATE resources SET accessed = ?, expires = ? WHERE url = ?");
+
+        stmt.bind(1, SystemClock::now());
+        stmt.bind(2, response.expires);
+        stmt.bind(3, resource.url);
+        stmt.run();
+    } else {
+        mapbox::sqlite::Statement& stmt = getStatement(
+            //                        1     2    3      4        5         6        7       8
+            "REPLACE INTO resources (url, kind, etag, expires, modified, accessed, data, compressed) "
+            "VALUES                 (?,   ?,    ?,    ?,       ?,        ?,        ?,    ?)");
+
+        stmt.bind(1 /* url */, resource.url);
+        stmt.bind(2 /* kind */, int(resource.kind));
+        stmt.bind(3 /* etag */, response.etag);
+        stmt.bind(4 /* expires */, response.expires);
+        stmt.bind(5 /* modified */, response.modified);
+        stmt.bind(6 /* accessed */, SystemClock::now());
+
+        std::string data;
+
+        if (response.error) { // Can only be NotFound
+            stmt.bind(7 /* data */, nullptr);
+            stmt.bind(8 /* compressed */, false);
+        } else {
+            data = util::compress(*response.data);
+            if (data.size() < response.data->size()) {
+                stmt.bind(7 /* data */, data, false); // do not retain the string internally.
+                stmt.bind(8 /* compressed */, true);
+            } else {
+                stmt.bind(7 /* data */, *response.data, false); // do not retain the string internally.
+                stmt.bind(8 /* compressed */, false);
+            }
+        }
+
+        stmt.run();
+    }
+}
+
+optional<Response> OfflineDatabase::getTile(const Resource::TileData& tile) {
+    mapbox::sqlite::Statement& stmt = getStatement(
+        //        0      1        2       3        4
+        "SELECT etag, expires, modified, data, compressed "
+        "FROM tilesets, tiles "
+        "WHERE tilesets.url_template = ? " // 1
+        "AND tilesets.pixel_ratio = ? "    // 2
+        "AND tiles.x = ? "                 // 3
+        "AND tiles.y = ? "                 // 4
+        "AND tiles.z = ? "                 // 5
+        "AND tilesets.id = tiles.tileset_id ");
+
+    stmt.bind(1, tile.urlTemplate);
+    stmt.bind(2, tile.pixelRatio);
+    stmt.bind(3, tile.x);
+    stmt.bind(4, tile.y);
+    stmt.bind(5, tile.z);
+
+    if (!stmt.run()) {
+        return {};
+    }
+
+    Response response;
+
+    response.etag     = stmt.get<optional<std::string>>(0);
+    response.expires  = stmt.get<optional<SystemTimePoint>>(1);
+    response.modified = stmt.get<optional<SystemTimePoint>>(2);
+
+    optional<std::string> data = stmt.get<optional<std::string>>(3);
+    if (!data) {
+        response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
+    } else if (stmt.get<int>(4)) {
+        response.data = std::make_shared<std::string>(util::decompress(*data));
+    } else {
+        response.data = std::make_shared<std::string>(*data);
+    }
+
+    return response;
+}
+
+void OfflineDatabase::putTile(const Resource::TileData& tile, const Response& response) {
+    if (response.notModified) {
+        mapbox::sqlite::Statement& stmt = getStatement(
+            "UPDATE tiles SET accessed = ?1, expires = ?2 "
+            "WHERE tileset_id = ( "
+            "  SELECT id FROM tilesets "
+            "  WHERE url_template = ?3 "
+            "  AND pixel_ratio = ?4) "
+            "AND tiles.x = ?5 "
+            "AND tiles.y = ?6 "
+            "AND tiles.z = ?7 ");
+
+        stmt.bind(1, SystemClock::now());
+        stmt.bind(2, response.expires);
+        stmt.bind(3, tile.urlTemplate);
+        stmt.bind(4, tile.pixelRatio);
+        stmt.bind(5, tile.x);
+        stmt.bind(6, tile.y);
+        stmt.bind(7, tile.z);
+        stmt.run();
+    } else {
+        mapbox::sqlite::Statement& stmt1 = getStatement(
+            "REPLACE INTO tilesets (url_template, pixel_ratio) "
+            "VALUES                (?1,           ?2) ");
+
+        stmt1.bind(1 /* url_template */, tile.urlTemplate);
+        stmt1.bind(2 /* pixel_ratio */, tile.pixelRatio);
+        stmt1.run();
+
+        mapbox::sqlite::Statement& stmt2 = getStatement(
+            "REPLACE INTO tiles (tileset_id,  x,  y,  z,  modified,  etag,  expires,  accessed,  data, compressed) "
+            "SELECT              tilesets.id, ?3, ?4, ?5, ?6,        ?7,    ?8,       ?9,        ?10,  ?11 "
+            "FROM tilesets "
+            "WHERE url_template = ?1 "
+            "AND pixel_ratio    = ?2 ");
+
+        stmt2.bind(1 /* url_template */, tile.urlTemplate);
+        stmt2.bind(2 /* pixel_ratio */, tile.pixelRatio);
+        stmt2.bind(3 /* x */, tile.x);
+        stmt2.bind(4 /* y */, tile.y);
+        stmt2.bind(5 /* z */, tile.z);
+        stmt2.bind(6 /* modified */, response.modified);
+        stmt2.bind(7 /* etag */, response.etag);
+        stmt2.bind(8 /* expires */, response.expires);
+        stmt2.bind(9 /* accessed */, SystemClock::now());
+
+        std::string data;
+
+        if (response.error) { // Can only be NotFound
+            stmt2.bind(10 /* data */, nullptr);
+            stmt2.bind(11 /* compressed */, false);
+        } else {
+            data = util::compress(*response.data);
+            if (data.size() < response.data->size()) {
+                stmt2.bind(10 /* data */, data, false); // do not retain the string internally.
+                stmt2.bind(11 /* compressed */, true);
+            } else {
+                stmt2.bind(10 /* data */, *response.data, false); // do not retain the string internally.
+                stmt2.bind(11 /* compressed */, false);
+            }
+        }
+
+        stmt2.run();
+    }
+}
+
+} // namespace mbgl
diff --git a/platform/default/mbgl/storage/offline_database.hpp b/platform/default/mbgl/storage/offline_database.hpp
new file mode 100644
index 0000000000..bc6f784d50
--- /dev/null
+++ b/platform/default/mbgl/storage/offline_database.hpp
@@ -0,0 +1,50 @@
+#ifndef MBGL_OFFLINE_DATABASE
+#define MBGL_OFFLINE_DATABASE
+
+#include <mbgl/storage/resource.hpp>
+#include <mbgl/util/noncopyable.hpp>
+#include <mbgl/util/optional.hpp>
+
+#include <unordered_map>
+#include <memory>
+#include <string>
+
+namespace mapbox {
+namespace sqlite {
+class Database;
+class Statement;
+}
+}
+
+namespace mbgl {
+
+class Response;
+class TileID;
+
+class OfflineDatabase : private util::noncopyable {
+public:
+    OfflineDatabase(const std::string& path);
+    ~OfflineDatabase();
+
+    optional<Response> get(const Resource&);
+    void put(const Resource&, const Response&);
+
+private:
+    void ensureSchema();
+    void removeExisting();
+    mapbox::sqlite::Statement& getStatement(const char *);
+
+    optional<Response> getTile(const Resource::TileData&);
+    void putTile(const Resource::TileData&, const Response&);
+
+    optional<Response> getResource(const Resource&);
+    void putResource(const Resource&, const Response&);
+
+    const std::string path;
+    std::unique_ptr<::mapbox::sqlite::Database> db;
+    std::unordered_map<const char *, std::unique_ptr<::mapbox::sqlite::Statement>> statements;
+};
+
+} // namespace mbgl
+
+#endif
diff --git a/platform/default/mbgl/storage/offline_schema.cpp.include b/platform/default/mbgl/storage/offline_schema.cpp.include
new file mode 100644
index 0000000000..31b11d5f12
--- /dev/null
+++ b/platform/default/mbgl/storage/offline_schema.cpp.include
@@ -0,0 +1,51 @@
+/* THIS IS A GENERATED FILE; EDIT offline_schema.sql INSTEAD */
+static const char * schema = 
+"CREATE TABLE resources (\n"
+"  url TEXT NOT NULL PRIMARY KEY,\n"
+"  kind INTEGER NOT NULL,\n"
+"  expires INTEGER,\n"
+"  modified INTEGER,\n"
+"  accessed INTEGER,\n"
+"  etag TEXT,\n"
+"  data BLOB,\n"
+"  compressed INTEGER NOT NULL DEFAULT 0\n"
+");\n"
+"CREATE TABLE tilesets (\n"
+"  id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,\n"
+"  url_template TEXT NOT NULL,\n"
+"  pixel_ratio INTEGER,\n"
+"  UNIQUE (url_template, pixel_ratio)\n"
+");\n"
+"CREATE TABLE tiles (\n"
+"  tileset_id INTEGER NOT NULL REFERENCES tilesets(id),\n"
+"  z INTEGER NOT NULL,\n"
+"  x INTEGER NOT NULL,\n"
+"  y INTEGER NOT NULL,\n"
+"  expires INTEGER,\n"
+"  modified INTEGER,\n"
+"  accessed INTEGER,\n"
+"  etag TEXT,\n"
+"  data BLOB,\n"
+"  compressed INTEGER NOT NULL DEFAULT 0,\n"
+"  PRIMARY KEY (tileset_id, z, x, y)\n"
+");\n"
+"CREATE TABLE regions (\n"
+"  id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,\n"
+"  definition TEXT NOT NULL,\n"
+"  description BLOB\n"
+");\n"
+"CREATE TABLE region_resources (\n"
+"  region_id INTEGER NOT NULL REFERENCES regions(id),\n"
+"  resource_url TEXT NOT NULL REFERENCES resources(url),\n"
+"  PRIMARY KEY (region_id, resource_url)\n"
+");\n"
+"CREATE TABLE region_tiles (\n"
+"  region_id INTEGER NOT NULL REFERENCES regions(id),\n"
+"  tileset_id INTEGER NOT NULL REFERENCES tilesets(id),\n"
+"  z INTEGER NOT NULL,\n"
+"  x INTEGER NOT NULL,\n"
+"  y INTEGER NOT NULL,\n"
+"  PRIMARY KEY (region_id, tileset_id, z, x, y),\n"
+"  FOREIGN KEY (tileset_id, z, x, y) REFERENCES tiles (tileset_id, z, x, y)\n"
+");\n"
+;
diff --git a/platform/default/mbgl/storage/offline_schema.js b/platform/default/mbgl/storage/offline_schema.js
new file mode 100644
index 0000000000..153ba34e38
--- /dev/null
+++ b/platform/default/mbgl/storage/offline_schema.js
@@ -0,0 +1,24 @@
+// To regenerate:
+// (cd platform/default/mbgl/storage && node offline_schema.js)
+
+var fs = require('fs');
+var readline = require('readline');
+
+var lineReader = readline.createInterface({
+    input: fs.createReadStream('offline_schema.sql')
+});
+
+var lines = [
+    "/* THIS IS A GENERATED FILE; EDIT offline_schema.sql INSTEAD */",
+    "static const char * schema = ",
+];
+
+lineReader
+    .on('line', function (line) {
+        line = line.replace(/ *--.*/, '');
+        if (line) lines.push('"' + line + '\\n"');
+    })
+    .on('close', function () {
+        lines.push(';\n');
+        fs.writeFileSync('offline_schema.cpp.include', lines.join('\n'));
+    });
diff --git a/platform/default/mbgl/storage/offline_schema.sql b/platform/default/mbgl/storage/offline_schema.sql
new file mode 100644
index 0000000000..e55517c489
--- /dev/null
+++ b/platform/default/mbgl/storage/offline_schema.sql
@@ -0,0 +1,62 @@
+CREATE TABLE resources (                   -- Generic table for style, source, sprite, and glyph resources.
+  url TEXT NOT NULL PRIMARY KEY,           -- Same schema as http_cache table.
+  kind INTEGER NOT NULL,
+  expires INTEGER,
+  modified INTEGER,
+  accessed INTEGER,
+  etag TEXT,
+  data BLOB,                               -- NULL if the response was a 404
+  compressed INTEGER NOT NULL DEFAULT 0
+);
+
+CREATE TABLE tilesets (
+  id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+  url_template TEXT NOT NULL,           -- As it would appear in TileJSON (but no support for host sharding).
+  pixel_ratio INTEGER,                  -- If NULL, 1 is assumed for raster sources.
+  UNIQUE (url_template, pixel_ratio)    -- Capable of caching the same tileset at multiple resolutions.
+);
+
+CREATE TABLE tiles (
+  tileset_id INTEGER NOT NULL REFERENCES tilesets(id),
+  z INTEGER NOT NULL,                   -- Fully abandon TMS coordinates in favor of ZXY.
+  x INTEGER NOT NULL,
+  y INTEGER NOT NULL,
+  expires INTEGER,
+  modified INTEGER,
+  accessed INTEGER,
+  etag TEXT,
+  data BLOB,                            -- NULL if the response was a 404
+  compressed INTEGER NOT NULL DEFAULT 0,
+  PRIMARY KEY (tileset_id, z, x, y)
+);
+
+CREATE TABLE regions (
+  id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+  definition TEXT NOT NULL,   -- JSON formatted definition of region. Regions may be of variant types:
+                              -- e.g. bbox and zoom range, route path, flyTo parameters, etc. Note that
+                              -- the set of tiles required for a region may span multiple sources.
+  description BLOB            -- User provided data in user-defined format
+);
+
+CREATE TABLE region_resources (
+  region_id INTEGER NOT NULL REFERENCES regions(id),
+  resource_url TEXT NOT NULL REFERENCES resources(url),
+  PRIMARY KEY (region_id, resource_url)
+);
+
+CREATE TABLE region_tiles (
+  region_id INTEGER NOT NULL REFERENCES regions(id),
+  tileset_id INTEGER NOT NULL REFERENCES tilesets(id),
+  z INTEGER NOT NULL,
+  x INTEGER NOT NULL,
+  y INTEGER NOT NULL,
+  PRIMARY KEY (region_id, tileset_id, z, x, y),
+  FOREIGN KEY (tileset_id, z, x, y) REFERENCES tiles (tileset_id, z, x, y)
+);
+
+-- `region_resources` and `region_tiles` are used for deduplication and deletion logic.
+-- A row in `tiles` exists IFF one or more corresponding rows exist in `region_tiles`. If
+-- more than one corresponding row exists, it indicates multiple regions contain the tile, and
+-- storage for the tile is being deduplicated. When a region is deleted, corresponding rows in
+-- `region_tiles` must also be deleted, and then rows in `tiles` and `tilesets` without a
+-- corresponding `region_tiles` row must be deleted. Similarly for `resources` / `region_resources`.
diff --git a/platform/default/sqlite3.cpp b/platform/default/sqlite3.cpp
index 5122e01015..09301bc4d9 100644
--- a/platform/default/sqlite3.cpp
+++ b/platform/default/sqlite3.cpp
@@ -115,9 +115,19 @@ template <> void Statement::bind(int offset, std::nullptr_t) {
     check(sqlite3_bind_null(stmt, offset));
 }
 
-template <> void Statement::bind(int offset, int value) {
+template <> void Statement::bind(int offset, int8_t value) {
     assert(stmt);
-    check(sqlite3_bind_int(stmt, offset, value));
+    check(sqlite3_bind_int64(stmt, offset, value));
+}
+
+template <> void Statement::bind(int offset, int16_t value) {
+    assert(stmt);
+    check(sqlite3_bind_int64(stmt, offset, value));
+}
+
+template <> void Statement::bind(int offset, int32_t value) {
+    assert(stmt);
+    check(sqlite3_bind_int64(stmt, offset, value));
 }
 
 template <> void Statement::bind(int offset, int64_t value) {
@@ -125,6 +135,26 @@ template <> void Statement::bind(int offset, int64_t value) {
     check(sqlite3_bind_int64(stmt, offset, value));
 }
 
+template <> void Statement::bind(int offset, uint8_t value) {
+    assert(stmt);
+    check(sqlite3_bind_int64(stmt, offset, value));
+}
+
+template <> void Statement::bind(int offset, uint16_t value) {
+    assert(stmt);
+    check(sqlite3_bind_int64(stmt, offset, value));
+}
+
+template <> void Statement::bind(int offset, uint32_t value) {
+    assert(stmt);
+    check(sqlite3_bind_int64(stmt, offset, value));
+}
+
+template <> void Statement::bind(int offset, float value) {
+    assert(stmt);
+    check(sqlite3_bind_double(stmt, offset, value));
+}
+
 template <> void Statement::bind(int offset, double value) {
     assert(stmt);
     check(sqlite3_bind_double(stmt, offset, value));
diff --git a/platform/default/sqlite_cache.cpp b/platform/default/sqlite_cache.cpp
deleted file mode 100644
index b3f528ac92..0000000000
--- a/platform/default/sqlite_cache.cpp
+++ /dev/null
@@ -1,502 +0,0 @@
-#include "sqlite_cache_impl.hpp"
-#include <mbgl/storage/resource.hpp>
-#include <mbgl/storage/response.hpp>
-
-#include <mbgl/util/compression.hpp>
-#include <mbgl/util/io.hpp>
-#include <mbgl/util/string.hpp>
-#include <mbgl/util/thread.hpp>
-#include <mbgl/util/mapbox.hpp>
-#include <mbgl/platform/log.hpp>
-
-#include "sqlite3.hpp"
-#include <sqlite3.h>
-
-#include <unordered_map>
-#include <mutex>
-
-namespace {
-
-// The cache won't accept entries larger than this arbitrary size
-// and will silently discard request for adding them to the cache.
-// Large entries can cause the database to grow in disk size and
-// never shrink again.
-const uint64_t kMaximumCacheEntrySize = 5 * 1024 * 1024; // 5 MB
-
-// Number of records we delete when we are close to the maximum
-// database size, if set. The current criteria is to prune
-// the least used entries based on `accessed` time.
-const int kPrunedEntriesLimit = 100;
-
-} // namespace
-
-namespace mbgl {
-
-using namespace mapbox::sqlite;
-
-SQLiteCache::SQLiteCache(const std::string& path_)
-    : thread(std::make_unique<util::Thread<Impl>>(util::ThreadContext{"SQLiteCache", util::ThreadType::Unknown, util::ThreadPriority::Low}, path_)) {
-}
-
-SQLiteCache::~SQLiteCache() = default;
-
-SQLiteCache::Impl::Impl(const std::string& path_)
-    : maximumCacheSize(0), // Unlimited
-      maximumCacheEntrySize(kMaximumCacheEntrySize),
-      path(path_) {
-}
-
-SQLiteCache::Impl::~Impl() {
-    // Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we
-    // can't throw anything.
-    try {
-        getStmt.reset();
-        putStmt.reset();
-        refreshStmt.reset();
-        countStmt.reset();
-        freeStmt.reset();
-        pruneStmt.reset();
-        accessedStmt.reset();
-        db.reset();
-    } catch (mapbox::sqlite::Exception& ex) {
-        Log::Error(Event::Database, ex.code, ex.what());
-    }
-}
-
-void SQLiteCache::Impl::createDatabase() {
-    db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
-}
-
-int SQLiteCache::Impl::schemaVersion() const {
-    // WARNING: Bump the version when changing the cache
-    // scheme to force the table to be recreated.
-    return 1;
-}
-
-void SQLiteCache::Impl::createSchema() {
-    constexpr const char *const sql = ""
-        "CREATE TABLE IF NOT EXISTS `http_cache` ("
-        "    `url` TEXT PRIMARY KEY NOT NULL,"
-        "    `status` INTEGER NOT NULL," // The response status (Successful or Error).
-        "    `kind` INTEGER NOT NULL," // The kind of file.
-        "    `modified` INTEGER," // Timestamp when the file was last modified.
-        "    `etag` TEXT,"
-        "    `expires` INTEGER," // Timestamp when the server says the file expires.
-        "    `accessed` INTEGER," // Timestamp when the database record was last accessed.
-        "    `data` BLOB,"
-        "    `compressed` INTEGER NOT NULL DEFAULT 0" // Whether the data is compressed.
-        ");"
-        "CREATE INDEX IF NOT EXISTS `http_cache_kind_idx` ON `http_cache` (`kind`);"
-        "CREATE INDEX IF NOT EXISTS `http_cache_accessed_idx` ON `http_cache` (`accessed`);";
-
-    ensureSchemaVersion();
-
-    try {
-        db->exec(sql);
-        db->exec("PRAGMA user_version = " + util::toString(schemaVersion()));
-        schema = true;
-    } catch (mapbox::sqlite::Exception &ex) {
-        if (ex.code == SQLITE_NOTADB) {
-            Log::Warning(Event::Database, "Trashing invalid database");
-            db.reset();
-            try {
-                util::deleteFile(path);
-            } catch (util::IOException& ioEx) {
-                Log::Error(Event::Database, ex.code, ex.what());
-            }
-            db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
-        } else {
-            Log::Error(Event::Database, ex.code, ex.what());
-        }
-
-        // Creating the database table + index failed. That means there may already be one, likely
-        // with different columns. Drop it and try to create a new one.
-        db->exec("DROP TABLE IF EXISTS `http_cache`");
-        db->exec(sql);
-        db->exec("PRAGMA user_version = " + util::toString(schemaVersion()));
-    }
-}
-
-void SQLiteCache::Impl::ensureSchemaVersion() {
-    try {
-       Statement userVersionStmt(db->prepare("PRAGMA user_version"));
-       if (userVersionStmt.run() && userVersionStmt.get<int>(0) == schemaVersion()) {
-           return;
-       }
-    } catch (mapbox::sqlite::Exception& ex) {
-        if (ex.code == SQLITE_NOTADB) {
-            return;
-        }
-
-        Log::Error(Event::Database, ex.code, ex.what());
-    }
-
-    // Version mismatch, drop the table so it will
-    // get recreated by `createSchema()`.
-    try {
-        db->exec("DROP TABLE IF EXISTS `http_cache`");
-    } catch (mapbox::sqlite::Exception& ex) {
-        Log::Error(Event::Database, ex.code, ex.what());
-    }
-}
-
-void SQLiteCache::setMaximumCacheSize(uint64_t size) {
-    thread->invoke(&Impl::setMaximumCacheSize, size);
-}
-
-void SQLiteCache::Impl::setMaximumCacheSize(uint64_t size) {
-    maximumCacheSize = size;
-
-    // Unlimited.
-    if (size == 0) {
-        return;
-    }
-
-    uint64_t lastSoftSize = cacheSoftSize();
-
-    // Keep pruning until we fit in the new
-    // size limit.
-    while (lastSoftSize > maximumCacheSize) {
-        pruneEntries();
-
-        if (lastSoftSize != cacheSoftSize()) {
-            lastSoftSize = cacheSoftSize();
-        } else {
-            break;
-        }
-    }
-
-    if (cacheHardSize() > size) {
-        Log::Warning(mbgl::Event::Database,
-            "Current cache hard size is bigger than the defined "
-            "maximum size. Database won't get truncated.");
-    }
-}
-
-void SQLiteCache::setMaximumCacheEntrySize(uint64_t size) {
-    thread->invoke(&Impl::setMaximumCacheEntrySize, size);
-}
-
-void SQLiteCache::Impl::setMaximumCacheEntrySize(uint64_t size) {
-    maximumCacheEntrySize = size;
-}
-
-void SQLiteCache::Impl::initializeDatabase() {
-    if (!db) {
-        createDatabase();
-    }
-
-    if (!schema) {
-        createSchema();
-    }
-}
-
-int SQLiteCache::Impl::cachePageSize() {
-    try {
-        if (!pageSize) {
-            Statement pageSizeStmt(db->prepare("PRAGMA page_size"));
-            if (pageSizeStmt.run()) {
-                pageSize = pageSizeStmt.get<int>(0);
-            }
-        }
-    } catch (mapbox::sqlite::Exception& ex) {
-        Log::Error(Event::Database, ex.code, ex.what());
-    }
-
-    return pageSize;
-}
-
-uint64_t SQLiteCache::Impl::cacheHardSize() {
-    try {
-        initializeDatabase();
-
-        if (!countStmt) {
-            countStmt = std::make_unique<Statement>(db->prepare("PRAGMA page_count"));
-        } else {
-            countStmt->reset();
-        }
-
-        if (countStmt->run()) {
-            return cachePageSize() * countStmt->get<int>(0);
-        }
-    } catch (mapbox::sqlite::Exception& ex) {
-        Log::Error(Event::Database, ex.code, ex.what());
-    }
-
-    return 0;
-}
-
-uint64_t SQLiteCache::Impl::cacheSoftSize() {
-    if (!softSizeDirty) {
-        return softSize;
-    }
-
-    try {
-        initializeDatabase();
-
-        if (!freeStmt) {
-            freeStmt = std::make_unique<Statement>(db->prepare("PRAGMA freelist_count"));
-        } else {
-            freeStmt->reset();
-        }
-
-        uint64_t hardSize = cacheHardSize();
-        if (!hardSize) {
-            return 0;
-        }
-
-        if (freeStmt->run()) {
-            return hardSize - cachePageSize() * freeStmt->get<int>(0);
-        }
-
-        softSizeDirty = false;
-    } catch (mapbox::sqlite::Exception& ex) {
-        Log::Error(Event::Database, ex.code, ex.what());
-    }
-
-    return 0;
-}
-
-bool SQLiteCache::Impl::needsPruning() {
-    // SQLite database never shrinks in size unless we call VACCUM. We here
-    // are monitoring the soft limit (i.e. number of free pages in the file)
-    // and as it approaches to the hard limit (i.e. the actual file size) we
-    // delete an arbitrary number of old cache entries.
-    //
-    // The free pages approach saves us from calling VACCUM or keeping a
-    // running total, which can be costly. We need a buffer because pages can
-    // get fragmented on the database.
-    if (cacheSoftSize() + maximumCacheEntrySize * 2 < maximumCacheSize) {
-        return false;
-    } else {
-        return true;
-    }
-}
-
-void SQLiteCache::Impl::pruneEntries() {
-    if (!maximumCacheSize) {
-        return;
-    }
-
-    if (!needsPruning()) {
-        return;
-    }
-
-    try {
-        if (!pruneStmt) {
-            pruneStmt = std::make_unique<Statement>(db->prepare(
-                "DELETE FROM `http_cache` WHERE `rowid` IN (SELECT `rowid` FROM "
-                //                                          1
-                "`http_cache` ORDER BY `accessed` ASC LIMIT ?)"));
-        } else {
-            pruneStmt->reset();
-        }
-
-        pruneStmt->bind(1, kPrunedEntriesLimit);
-
-        pruneStmt->run();
-        softSizeDirty = true;
-    } catch (mapbox::sqlite::Exception& ex) {
-        Log::Error(Event::Database, ex.code, ex.what());
-    }
-}
-
-std::unique_ptr<WorkRequest> SQLiteCache::get(const Resource &resource, Callback callback) {
-    // Can be called from any thread, but most likely from the file source thread.
-    // Will try to load the URL from the SQLite database and call the callback when done.
-    // Note that the callback is probably going to invoked from another thread, so the caller
-    // must make sure that it can run in that thread.
-    return thread->invokeWithCallback(&Impl::get, callback, resource);
-}
-
-void SQLiteCache::Impl::get(const Resource &resource, Callback callback) {
-    try {
-        initializeDatabase();
-
-        if (!getStmt) {
-            // Initialize the statement                                  0         1
-            getStmt = std::make_unique<Statement>(db->prepare("SELECT `status`, `modified`, "
-            //     2         3        4          5                                       1
-                "`etag`, `expires`, `data`, `compressed` FROM `http_cache` WHERE `url` = ?"));
-        } else {
-            getStmt->reset();
-        }
-
-        getStmt->bind(1, resource.url);
-        if (getStmt->run()) {
-            // There is data.
-            auto response = std::make_unique<Response>();
-            const auto status = getStmt->get<int>(0);
-            if (status > 1) {
-                // Status codes > 1 indicate an error
-                response->error = std::make_unique<Response::Error>(Response::Error::Reason(status));
-            }
-            response->modified = getStmt->get<optional<SystemTimePoint>>(1);
-            response->etag = getStmt->get<optional<std::string>>(2);
-            response->expires = getStmt->get<optional<SystemTimePoint>>(3);
-            response->data = std::make_shared<std::string>(getStmt->get<std::string>(4));
-            if (getStmt->get<int>(5)) { // == compressed
-                response->data = std::make_shared<std::string>(util::decompress(*response->data));
-            }
-            callback(std::move(response));
-        } else {
-            // There is no data.
-            callback(nullptr);
-        }
-
-        // We do an extra query for refreshing the last time
-        // the record was accessed that can be costly and is only
-        // worth doing if we are monitoring the database size.
-        if (maximumCacheSize) {
-            if (!accessedStmt) {
-                accessedStmt = std::make_unique<Statement>(
-                    //                                                1               2
-                    db->prepare("UPDATE `http_cache` SET `accessed` = ? WHERE `url` = ?"));
-            } else {
-                accessedStmt->reset();
-            }
-
-            accessedStmt->bind(1, SystemClock::now());
-            accessedStmt->bind(2, resource.url);
-            accessedStmt->run();
-        }
-    } catch (mapbox::sqlite::Exception& ex) {
-        Log::Error(Event::Database, ex.code, ex.what());
-        callback(nullptr);
-    } catch (std::runtime_error& ex) {
-        Log::Error(Event::Database, "%s", ex.what());
-        callback(nullptr);
-    }
-}
-
-void SQLiteCache::put(const Resource& resource, const Response& response) {
-    // Except for 404s, don't store errors in the cache.
-    if (response.error && response.error->reason != Response::Error::Reason::NotFound) {
-        return;
-    }
-
-    if (response.notModified) {
-        thread->invoke(&Impl::refresh, resource, response.expires);
-    } else {
-        thread->invoke(&Impl::put, resource, response);
-    }
-}
-
-void SQLiteCache::Impl::put(const Resource& resource, const Response& response) {
-    try {
-        initializeDatabase();
-        pruneEntries();
-
-        if (response.data) {
-            auto entrySize = response.data->size();
-
-            if (entrySize > maximumCacheEntrySize) {
-                Log::Warning(Event::Database, "Entry too big for caching.");
-                return;
-            }
-
-            if (maximumCacheSize && entrySize + cacheSoftSize() > maximumCacheSize) {
-                Log::Warning(Event::Database, "Unable to make space for new entries.");
-                return;
-            }
-        }
-
-        if (!putStmt) {
-            putStmt = std::make_unique<Statement>(db->prepare("REPLACE INTO `http_cache` ("
-                // 1        2       3         4         5         6          7         8          9
-                "`url`, `status`, `kind`, `modified`, `etag`, `expires`, `accessed`, `data`, `compressed`"
-                ") VALUES(?, ?, ?, ?, ?, ?, ?, ?, ?)"));
-        } else {
-            putStmt->reset();
-        }
-
-        putStmt->bind(1 /* url */, resource.url);
-        if (response.error) {
-            putStmt->bind(2 /* status */, int(response.error->reason));
-        } else {
-            putStmt->bind(2 /* status */, 1 /* success */);
-        }
-        putStmt->bind(3 /* kind */, int(resource.kind));
-        putStmt->bind(4 /* modified */, response.modified);
-        putStmt->bind(5 /* etag */, response.etag);
-        putStmt->bind(6 /* expires */, response.expires);
-        putStmt->bind(7 /* accessed */, SystemClock::now());
-
-        std::string data;
-        if (resource.kind != Resource::SpriteImage && response.data) {
-            // Do not compress images, since they are typically compressed already.
-            data = util::compress(*response.data);
-        }
-
-        if (!data.empty() && data.size() < response.data->size()) {
-            // Store the compressed data when it is smaller than the original
-            // uncompressed data.
-            putStmt->bind(8 /* data */, data, false); // do not retain the string internally.
-            putStmt->bind(9 /* compressed */, true);
-        } else if (response.data) {
-            putStmt->bind(8 /* data */, *response.data, false); // do not retain the string internally.
-            putStmt->bind(9 /* compressed */, false);
-        } else {
-            putStmt->bind(8 /* data */, "", false);
-            putStmt->bind(9 /* compressed */, false);
-        }
-
-        putStmt->run();
-        softSizeDirty = true;
-    } catch (mapbox::sqlite::Exception& ex) {
-        Log::Error(Event::Database, ex.code, ex.what());
-    } catch (std::runtime_error& ex) {
-        Log::Error(Event::Database, "%s", ex.what());
-    }
-}
-
-void SQLiteCache::Impl::refresh(const Resource& resource, optional<SystemTimePoint> expires) {
-    try {
-        initializeDatabase();
-
-        if (!refreshStmt) {
-            refreshStmt = std::make_unique<Statement>(
-                db->prepare("UPDATE `http_cache` SET "
-                    //            1              2               3
-                    "`accessed` = ?, `expires` = ? WHERE `url` = ?"));
-        } else {
-            refreshStmt->reset();
-        }
-
-        refreshStmt->bind(1, SystemClock::now());
-        refreshStmt->bind(2, expires);
-        refreshStmt->bind(3, resource.url);
-        refreshStmt->run();
-    } catch (mapbox::sqlite::Exception& ex) {
-        Log::Error(Event::Database, ex.code, ex.what());
-    }
-}
-
-namespace {
-
-static std::mutex sharedMutex;
-static std::unordered_map<std::string, std::weak_ptr<SQLiteCache>> shared;
-
-} // namespace
-
-std::shared_ptr<SQLiteCache> SQLiteCache::getShared(const std::string &path) {
-    std::lock_guard<std::mutex> lock(sharedMutex);
-
-    std::shared_ptr<SQLiteCache> cache;
-
-    auto it = shared.find(path);
-    if (it != shared.end()) {
-        cache = it->second.lock();
-        if (!cache) {
-            cache = std::make_shared<SQLiteCache>(path);
-            it->second = cache;
-        }
-    } else {
-        cache = std::make_shared<SQLiteCache>(path);
-        shared.emplace(path, cache);
-    }
-
-    return cache;
-}
-
-} // namespace mbgl
diff --git a/platform/default/sqlite_cache_impl.hpp b/platform/default/sqlite_cache_impl.hpp
deleted file mode 100644
index e156532402..0000000000
--- a/platform/default/sqlite_cache_impl.hpp
+++ /dev/null
@@ -1,68 +0,0 @@
-#ifndef MBGL_STORAGE_DEFAULT_SQLITE_CACHE_IMPL
-#define MBGL_STORAGE_DEFAULT_SQLITE_CACHE_IMPL
-
-#include <mbgl/storage/sqlite_cache.hpp>
-#include <mbgl/util/chrono.hpp>
-#include <mbgl/util/optional.hpp>
-
-namespace mapbox {
-namespace sqlite {
-class Database;
-class Statement;
-}
-}
-
-namespace mbgl {
-
-class SQLiteCache::Impl {
-public:
-    explicit Impl(const std::string &path = ":memory:");
-    ~Impl();
-
-    void setMaximumCacheSize(uint64_t size);
-    void setMaximumCacheEntrySize(uint64_t size);
-
-    void get(const Resource&, Callback);
-    void put(const Resource&, const Response&);
-    void refresh(const Resource&, optional<SystemTimePoint> expires);
-
-private:
-    void initializeDatabase();
-
-    int cachePageSize();
-
-    uint64_t cacheHardSize();
-    uint64_t cacheSoftSize();
-
-    uint64_t softSize = 0;
-    bool softSizeDirty = true;
-
-    bool needsPruning();
-    void pruneEntries();
-
-    void createDatabase();
-    void createSchema();
-
-    int schemaVersion() const;
-    void ensureSchemaVersion();
-
-    int pageSize = 0;
-
-    uint64_t maximumCacheSize;
-    uint64_t maximumCacheEntrySize;
-
-    const std::string path;
-    std::unique_ptr<::mapbox::sqlite::Database> db;
-    std::unique_ptr<::mapbox::sqlite::Statement> getStmt;
-    std::unique_ptr<::mapbox::sqlite::Statement> putStmt;
-    std::unique_ptr<::mapbox::sqlite::Statement> refreshStmt;
-    std::unique_ptr<::mapbox::sqlite::Statement> countStmt;
-    std::unique_ptr<::mapbox::sqlite::Statement> freeStmt;
-    std::unique_ptr<::mapbox::sqlite::Statement> pruneStmt;
-    std::unique_ptr<::mapbox::sqlite::Statement> accessedStmt;
-    bool schema = false;
-};
-
-} // namespace mbgl
-
-#endif