5 files changed, 484 insertions, 0 deletions
diff --git a/platform/default/mbgl/storage/offline_database.cpp b/platform/default/mbgl/storage/offline_database.cpp
new file mode 100644
index 0000000000..cd0f88b7fe
--- /dev/null
+++ b/platform/default/mbgl/storage/offline_database.cpp
@@ -0,0 +1,297 @@
+#include <mbgl/storage/offline_database.hpp>
+#include <mbgl/storage/response.hpp>
+#include <mbgl/util/compression.hpp>
+#include <mbgl/util/io.hpp>
+#include <mbgl/util/string.hpp>
+#include <mbgl/util/chrono.hpp>
+#include <mbgl/map/tile_id.hpp>
+#include <mbgl/platform/log.hpp>
+
+#include "sqlite3.hpp"
+#include <sqlite3.h>
+
+namespace mbgl {
+
+using namespace mapbox::sqlite;
+
+// If you change the schema you must write a migration from the previous version.
+static const uint32_t schemaVersion = 2;
+
+OfflineDatabase::OfflineDatabase(const std::string& path_)
+    : path(path_) {
+    ensureSchema();
+}
+
+OfflineDatabase::~OfflineDatabase() {
+    // Deleting these SQLite objects may result in exceptions, but we're in a destructor, so we
+    // can't throw anything.
+    try {
+        statements.clear();
+        db.reset();
+    } catch (mapbox::sqlite::Exception& ex) {
+        Log::Error(Event::Database, ex.code, ex.what());
+    }
+}
+
+void OfflineDatabase::ensureSchema() {
+    if (path != ":memory:") {
+        try {
+            db = std::make_unique<Database>(path.c_str(), ReadWrite);
+
+            {
+                Statement userVersionStmt(db->prepare("PRAGMA user_version"));
+                userVersionStmt.run();
+                switch (userVersionStmt.get<int>(0)) {
+                case 0: break; // cache-only database; ok to delete
+                case 1: break; // cache-only database; ok to delete
+                case 2: return;
+                default: throw std::runtime_error("unknown schema version");
+                }
+            }
+
+            removeExisting();
+            db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
+        } catch (mapbox::sqlite::Exception& ex) {
+            if (ex.code == SQLITE_CANTOPEN) {
+                db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
+            } else if (ex.code == SQLITE_NOTADB) {
+                removeExisting();
+                db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
+            }
+        }
+    }
+
+    #include "offline_schema.cpp.include"
+
+    db = std::make_unique<Database>(path.c_str(), ReadWrite | Create);
+    db->exec(schema);
+    db->exec("PRAGMA user_version = " + util::toString(schemaVersion));
+}
+
+void OfflineDatabase::removeExisting() {
+    Log::Warning(Event::Database, "Removing existing incompatible offline database");
+
+    db.reset();
+
+    try {
+        util::deleteFile(path);
+    } catch (util::IOException& ex) {
+        Log::Error(Event::Database, ex.code, ex.what());
+    }
+}
+
+mapbox::sqlite::Statement& OfflineDatabase::getStatement(const char * sql) {
+    auto it = statements.find(sql);
+
+    if (it != statements.end()) {
+        it->second->reset();
+        return *it->second;
+    }
+
+    return *(statements[sql] = std::make_unique<Statement>(db->prepare(sql)));
+}
+
+optional<Response> OfflineDatabase::get(const Resource& resource) {
+    if (resource.kind == Resource::Kind::Tile) {
+        assert(resource.tileData);
+        return getTile(*resource.tileData);
+    } else {
+        return getResource(resource);
+    }
+}
+
+void OfflineDatabase::put(const Resource& resource, const Response& response) {
+    // Except for 404s, don't store errors in the cache.
+    if (response.error && response.error->reason != Response::Error::Reason::NotFound) {
+        return;
+    }
+
+    if (resource.kind == Resource::Kind::Tile) {
+        assert(resource.tileData);
+        putTile(*resource.tileData, response);
+    } else {
+        putResource(resource, response);
+    }
+}
+
+optional<Response> OfflineDatabase::getResource(const Resource& resource) {
+    mapbox::sqlite::Statement& stmt = getStatement(
+        //        0      1        2       3        4
+        "SELECT etag, expires, modified, data, compressed "
+        "FROM resources "
+        "WHERE url = ?");
+
+    stmt.bind(1, resource.url);
+
+    if (!stmt.run()) {
+        return {};
+    }
+
+    Response response;
+
+    response.etag     = stmt.get<optional<std::string>>(0);
+    response.expires  = stmt.get<optional<SystemTimePoint>>(1);
+    response.modified = stmt.get<optional<SystemTimePoint>>(2);
+
+    optional<std::string> data = stmt.get<optional<std::string>>(3);
+    if (!data) {
+        response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
+    } else if (stmt.get<int>(4)) {
+        response.data = std::make_shared<std::string>(util::decompress(*data));
+    } else {
+        response.data = std::make_shared<std::string>(*data);
+    }
+
+    return response;
+}
+
+void OfflineDatabase::putResource(const Resource& resource, const Response& response) {
+    if (response.notModified) {
+        mapbox::sqlite::Statement& stmt = getStatement(
+            //                               1            2             3
+            "UPDATE resources SET accessed = ?, expires = ? WHERE url = ?");
+
+        stmt.bind(1, SystemClock::now());
+        stmt.bind(2, response.expires);
+        stmt.bind(3, resource.url);
+        stmt.run();
+    } else {
+        mapbox::sqlite::Statement& stmt = getStatement(
+            //                        1     2    3      4        5         6        7       8
+            "REPLACE INTO resources (url, kind, etag, expires, modified, accessed, data, compressed) "
+            "VALUES                 (?,   ?,    ?,    ?,       ?,        ?,        ?,    ?)");
+
+        stmt.bind(1 /* url */, resource.url);
+        stmt.bind(2 /* kind */, int(resource.kind));
+        stmt.bind(3 /* etag */, response.etag);
+        stmt.bind(4 /* expires */, response.expires);
+        stmt.bind(5 /* modified */, response.modified);
+        stmt.bind(6 /* accessed */, SystemClock::now());
+
+        std::string data;
+
+        if (response.error) { // Can only be NotFound
+            stmt.bind(7 /* data */, nullptr);
+            stmt.bind(8 /* compressed */, false);
+        } else {
+            data = util::compress(*response.data);
+            if (data.size() < response.data->size()) {
+                stmt.bind(7 /* data */, data, false); // do not retain the string internally.
+                stmt.bind(8 /* compressed */, true);
+            } else {
+                stmt.bind(7 /* data */, *response.data, false); // do not retain the string internally.
+                stmt.bind(8 /* compressed */, false);
+            }
+        }
+
+        stmt.run();
+    }
+}
+
+optional<Response> OfflineDatabase::getTile(const Resource::TileData& tile) {
+    mapbox::sqlite::Statement& stmt = getStatement(
+        //        0      1        2       3        4
+        "SELECT etag, expires, modified, data, compressed "
+        "FROM tilesets, tiles "
+        "WHERE tilesets.url_template = ? " // 1
+        "AND tilesets.pixel_ratio = ? "    // 2
+        "AND tiles.x = ? "                 // 3
+        "AND tiles.y = ? "                 // 4
+        "AND tiles.z = ? "                 // 5
+        "AND tilesets.id = tiles.tileset_id ");
+
+    stmt.bind(1, tile.urlTemplate);
+    stmt.bind(2, tile.pixelRatio);
+    stmt.bind(3, tile.x);
+    stmt.bind(4, tile.y);
+    stmt.bind(5, tile.z);
+
+    if (!stmt.run()) {
+        return {};
+    }
+
+    Response response;
+
+    response.etag     = stmt.get<optional<std::string>>(0);
+    response.expires  = stmt.get<optional<SystemTimePoint>>(1);
+    response.modified = stmt.get<optional<SystemTimePoint>>(2);
+
+    optional<std::string> data = stmt.get<optional<std::string>>(3);
+    if (!data) {
+        response.error = std::make_unique<Response::Error>(Response::Error::Reason::NotFound);
+    } else if (stmt.get<int>(4)) {
+        response.data = std::make_shared<std::string>(util::decompress(*data));
+    } else {
+        response.data = std::make_shared<std::string>(*data);
+    }
+
+    return response;
+}
+
+void OfflineDatabase::putTile(const Resource::TileData& tile, const Response& response) {
+    if (response.notModified) {
+        mapbox::sqlite::Statement& stmt = getStatement(
+            "UPDATE tiles SET accessed = ?1, expires = ?2 "
+            "WHERE tileset_id = ( "
+            "  SELECT id FROM tilesets "
+            "  WHERE url_template = ?3 "
+            "  AND pixel_ratio = ?4) "
+            "AND tiles.x = ?5 "
+            "AND tiles.y = ?6 "
+            "AND tiles.z = ?7 ");
+
+        stmt.bind(1, SystemClock::now());
+        stmt.bind(2, response.expires);
+        stmt.bind(3, tile.urlTemplate);
+        stmt.bind(4, tile.pixelRatio);
+        stmt.bind(5, tile.x);
+        stmt.bind(6, tile.y);
+        stmt.bind(7, tile.z);
+        stmt.run();
+    } else {
+        mapbox::sqlite::Statement& stmt1 = getStatement(
+            "REPLACE INTO tilesets (url_template, pixel_ratio) "
+            "VALUES                (?1,           ?2) ");
+
+        stmt1.bind(1 /* url_template */, tile.urlTemplate);
+        stmt1.bind(2 /* pixel_ratio */, tile.pixelRatio);
+        stmt1.run();
+
+        mapbox::sqlite::Statement& stmt2 = getStatement(
+            "REPLACE INTO tiles (tileset_id,  x,  y,  z,  modified,  etag,  expires,  accessed,  data, compressed) "
+            "SELECT              tilesets.id, ?3, ?4, ?5, ?6,        ?7,    ?8,       ?9,        ?10,  ?11 "
+            "FROM tilesets "
+            "WHERE url_template = ?1 "
+            "AND pixel_ratio    = ?2 ");
+
+        stmt2.bind(1 /* url_template */, tile.urlTemplate);
+        stmt2.bind(2 /* pixel_ratio */, tile.pixelRatio);
+        stmt2.bind(3 /* x */, tile.x);
+        stmt2.bind(4 /* y */, tile.y);
+        stmt2.bind(5 /* z */, tile.z);
+        stmt2.bind(6 /* modified */, response.modified);
+        stmt2.bind(7 /* etag */, response.etag);
+        stmt2.bind(8 /* expires */, response.expires);
+        stmt2.bind(9 /* accessed */, SystemClock::now());
+
+        std::string data;
+
+        if (response.error) { // Can only be NotFound
+            stmt2.bind(10 /* data */, nullptr);
+            stmt2.bind(11 /* compressed */, false);
+        } else {
+            data = util::compress(*response.data);
+            if (data.size() < response.data->size()) {
+                stmt2.bind(10 /* data */, data, false); // do not retain the string internally.
+                stmt2.bind(11 /* compressed */, true);
+            } else {
+                stmt2.bind(10 /* data */, *response.data, false); // do not retain the string internally.
+                stmt2.bind(11 /* compressed */, false);
+            }
+        }
+
+        stmt2.run();
+    }
+}
+
+} // namespace mbgl
diff --git a/platform/default/mbgl/storage/offline_database.hpp b/platform/default/mbgl/storage/offline_database.hpp
new file mode 100644
index 0000000000..bc6f784d50
--- /dev/null
+++ b/platform/default/mbgl/storage/offline_database.hpp
@@ -0,0 +1,50 @@
+#ifndef MBGL_OFFLINE_DATABASE
+#define MBGL_OFFLINE_DATABASE
+
+#include <mbgl/storage/resource.hpp>
+#include <mbgl/util/noncopyable.hpp>
+#include <mbgl/util/optional.hpp>
+
+#include <unordered_map>
+#include <memory>
+#include <string>
+
+namespace mapbox {
+namespace sqlite {
+class Database;
+class Statement;
+}
+}
+
+namespace mbgl {
+
+class Response;
+class TileID;
+
+class OfflineDatabase : private util::noncopyable {
+public:
+    OfflineDatabase(const std::string& path);
+    ~OfflineDatabase();
+
+    optional<Response> get(const Resource&);
+    void put(const Resource&, const Response&);
+
+private:
+    void ensureSchema();
+    void removeExisting();
+    mapbox::sqlite::Statement& getStatement(const char *);
+
+    optional<Response> getTile(const Resource::TileData&);
+    void putTile(const Resource::TileData&, const Response&);
+
+    optional<Response> getResource(const Resource&);
+    void putResource(const Resource&, const Response&);
+
+    const std::string path;
+    std::unique_ptr<::mapbox::sqlite::Database> db;
+    std::unordered_map<const char *, std::unique_ptr<::mapbox::sqlite::Statement>> statements;
+};
+
+} // namespace mbgl
+
+#endif
diff --git a/platform/default/mbgl/storage/offline_schema.cpp.include b/platform/default/mbgl/storage/offline_schema.cpp.include
new file mode 100644
index 0000000000..31b11d5f12
--- /dev/null
+++ b/platform/default/mbgl/storage/offline_schema.cpp.include
@@ -0,0 +1,51 @@
+/* THIS IS A GENERATED FILE; EDIT offline_schema.sql INSTEAD */
+static const char * schema = 
+"CREATE TABLE resources (\n"
+"  url TEXT NOT NULL PRIMARY KEY,\n"
+"  kind INTEGER NOT NULL,\n"
+"  expires INTEGER,\n"
+"  modified INTEGER,\n"
+"  accessed INTEGER,\n"
+"  etag TEXT,\n"
+"  data BLOB,\n"
+"  compressed INTEGER NOT NULL DEFAULT 0\n"
+");\n"
+"CREATE TABLE tilesets (\n"
+"  id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,\n"
+"  url_template TEXT NOT NULL,\n"
+"  pixel_ratio INTEGER,\n"
+"  UNIQUE (url_template, pixel_ratio)\n"
+");\n"
+"CREATE TABLE tiles (\n"
+"  tileset_id INTEGER NOT NULL REFERENCES tilesets(id),\n"
+"  z INTEGER NOT NULL,\n"
+"  x INTEGER NOT NULL,\n"
+"  y INTEGER NOT NULL,\n"
+"  expires INTEGER,\n"
+"  modified INTEGER,\n"
+"  accessed INTEGER,\n"
+"  etag TEXT,\n"
+"  data BLOB,\n"
+"  compressed INTEGER NOT NULL DEFAULT 0,\n"
+"  PRIMARY KEY (tileset_id, z, x, y)\n"
+");\n"
+"CREATE TABLE regions (\n"
+"  id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,\n"
+"  definition TEXT NOT NULL,\n"
+"  description BLOB\n"
+");\n"
+"CREATE TABLE region_resources (\n"
+"  region_id INTEGER NOT NULL REFERENCES regions(id),\n"
+"  resource_url TEXT NOT NULL REFERENCES resources(url),\n"
+"  PRIMARY KEY (region_id, resource_url)\n"
+");\n"
+"CREATE TABLE region_tiles (\n"
+"  region_id INTEGER NOT NULL REFERENCES regions(id),\n"
+"  tileset_id INTEGER NOT NULL REFERENCES tilesets(id),\n"
+"  z INTEGER NOT NULL,\n"
+"  x INTEGER NOT NULL,\n"
+"  y INTEGER NOT NULL,\n"
+"  PRIMARY KEY (region_id, tileset_id, z, x, y),\n"
+"  FOREIGN KEY (tileset_id, z, x, y) REFERENCES tiles (tileset_id, z, x, y)\n"
+");\n"
+;
diff --git a/platform/default/mbgl/storage/offline_schema.js b/platform/default/mbgl/storage/offline_schema.js
new file mode 100644
index 0000000000..153ba34e38
--- /dev/null
+++ b/platform/default/mbgl/storage/offline_schema.js
@@ -0,0 +1,24 @@
+// To regenerate:
+// (cd platform/default/mbgl/storage && node offline_schema.js)
+
+var fs = require('fs');
+var readline = require('readline');
+
+var lineReader = readline.createInterface({
+    input: fs.createReadStream('offline_schema.sql')
+});
+
+var lines = [
+    "/* THIS IS A GENERATED FILE; EDIT offline_schema.sql INSTEAD */",
+    "static const char * schema = ",
+];
+
+lineReader
+    .on('line', function (line) {
+        line = line.replace(/ *--.*/, '');
+        if (line) lines.push('"' + line + '\\n"');
+    })
+    .on('close', function () {
+        lines.push(';\n');
+        fs.writeFileSync('offline_schema.cpp.include', lines.join('\n'));
+    });
diff --git a/platform/default/mbgl/storage/offline_schema.sql b/platform/default/mbgl/storage/offline_schema.sql
new file mode 100644
index 0000000000..e55517c489
--- /dev/null
+++ b/platform/default/mbgl/storage/offline_schema.sql
@@ -0,0 +1,62 @@
+CREATE TABLE resources (                   -- Generic table for style, source, sprite, and glyph resources.
+  url TEXT NOT NULL PRIMARY KEY,           -- Same schema as http_cache table.
+  kind INTEGER NOT NULL,
+  expires INTEGER,
+  modified INTEGER,
+  accessed INTEGER,
+  etag TEXT,
+  data BLOB,                               -- NULL if the response was a 404
+  compressed INTEGER NOT NULL DEFAULT 0
+);
+
+CREATE TABLE tilesets (
+  id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+  url_template TEXT NOT NULL,           -- As it would appear in TileJSON (but no support for host sharding).
+  pixel_ratio INTEGER,                  -- If NULL, 1 is assumed for raster sources.
+  UNIQUE (url_template, pixel_ratio)    -- Capable of caching the same tileset at multiple resolutions.
+);
+
+CREATE TABLE tiles (
+  tileset_id INTEGER NOT NULL REFERENCES tilesets(id),
+  z INTEGER NOT NULL,                   -- Fully abandon TMS coordinates in favor of ZXY.
+  x INTEGER NOT NULL,
+  y INTEGER NOT NULL,
+  expires INTEGER,
+  modified INTEGER,
+  accessed INTEGER,
+  etag TEXT,
+  data BLOB,                            -- NULL if the response was a 404
+  compressed INTEGER NOT NULL DEFAULT 0,
+  PRIMARY KEY (tileset_id, z, x, y)
+);
+
+CREATE TABLE regions (
+  id INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT,
+  definition TEXT NOT NULL,   -- JSON formatted definition of region. Regions may be of variant types:
+                              -- e.g. bbox and zoom range, route path, flyTo parameters, etc. Note that
+                              -- the set of tiles required for a region may span multiple sources.
+  description BLOB            -- User provided data in user-defined format
+);
+
+CREATE TABLE region_resources (
+  region_id INTEGER NOT NULL REFERENCES regions(id),
+  resource_url TEXT NOT NULL REFERENCES resources(url),
+  PRIMARY KEY (region_id, resource_url)
+);
+
+CREATE TABLE region_tiles (
+  region_id INTEGER NOT NULL REFERENCES regions(id),
+  tileset_id INTEGER NOT NULL REFERENCES tilesets(id),
+  z INTEGER NOT NULL,
+  x INTEGER NOT NULL,
+  y INTEGER NOT NULL,
+  PRIMARY KEY (region_id, tileset_id, z, x, y),
+  FOREIGN KEY (tileset_id, z, x, y) REFERENCES tiles (tileset_id, z, x, y)
+);
+
+-- `region_resources` and `region_tiles` are used for deduplication and deletion logic.
+-- A row in `tiles` exists IFF one or more corresponding rows exist in `region_tiles`. If
+-- more than one corresponding row exists, it indicates multiple regions contain the tile, and
+-- storage for the tile is being deduplicated. When a region is deleted, corresponding rows in
+-- `region_tiles` must also be deleted, and then rows in `tiles` and `tilesets` without a
+-- corresponding `region_tiles` row must be deleted. Similarly for `resources` / `region_resources`.