[core] introduce Blob for compressed and uncompressed dataupstream/blob

- Blob is a wrapper type for a shared_ptr<const string> that has accessor functions for getting compressed and uncompressed data
- Moved util::writeFile, util::readFile, util::compress, util::uncompress, decodeImage, and encodePNG to the Blob interface
- Added Blob support to Request and file sources
- Added Blob support to VectorTile objects
- Added support for gzip decoding to util::uncompress
- We're no longer compressing WebP, PNG, and JPEG data when storing in the OfflineDatabase
- Android's HTTPRequest returns compressed Blobs by default

One caveat is that our previous decompress function didn't support gzip, so once users upgrade to this version, their offline cache may contain both zlib-compressed data and gzip-compressed data, but older versions won't be able to decompress gzip data. On the other hand, we don't support downgrading SDKs anyway, so this shouldn't be a problem. To be on the safe side, we could bump the user_version of the SQLite DB.

3 files changed, 65 insertions, 28 deletions

diff --git a/platform/default/mbgl/storage/offline_database.cpp b/platform/default/mbgl/storage/offline_database.cpp
index 65c2097182..d7022f1c80 100644
--- a/platform/default/mbgl/storage/offline_database.cpp
+++ b/platform/default/mbgl/storage/offline_database.cpp
@@ -1,10 +1,10 @@
 #include <mbgl/storage/offline_database.hpp>
 #include <mbgl/storage/response.hpp>
-#include <mbgl/util/compression.hpp>
 #include <mbgl/util/io.hpp>
 #include <mbgl/util/string.hpp>
 #include <mbgl/util/chrono.hpp>
 #include <mbgl/util/logging.hpp>
+#include <mbgl/util/compression.hpp>
 
 #include "sqlite3.hpp"
 
@@ -153,7 +153,7 @@ optional<Response> OfflineDatabase::get(const Resource& resource) {
 optional<std::pair<Response, uint64_t>> OfflineDatabase::getInternal(const Resource& resource) {
     if (resource.kind == Resource::Kind::Tile) {
         assert(resource.tileData);
-        return getTile(*resource.tileData);
+        return getTile(resource);
     } else {
         return getResource(resource);
     }
@@ -177,14 +177,31 @@ std::pair<bool, uint64_t> OfflineDatabase::putInternal(const Resource& resource,
         return { false, 0 };
     }
 
-    std::string compressedData;
+    std::shared_ptr<const std::string> data;
     bool compressed = false;
     uint64_t size = 0;
 
     if (response.data) {
-        compressedData = util::compress(*response.data);
-        compressed = compressedData.size() < response.data->size();
-        size = compressed ? compressedData.size() : response.data->size();
+        if (response.data.isCompressed()) {
+            // The response is already compressed; don't try to compare it against the uncompressed size.
+            compressed = true;
+            data = response.data.compressedData();
+        } else {
+            data = response.data.uncompressedData();
+
+            // Only try to compress the data when we have a good chance that the data can actually
+            // be considerably compressed.
+            if (util::isCompressible(*data)) {
+                const auto compressedData = response.data.compressedData();
+                if (compressedData->size() < data->size()) {
+                    compressed = true;
+                    data = compressedData;
+                }
+            }
+        }
+        size = data->size();
+    } else {
+        data = std::make_shared<const std::string>();
     }
 
     if (evict_ && !evict(size)) {
@@ -196,13 +213,9 @@ std::pair<bool, uint64_t> OfflineDatabase::putInternal(const Resource& resource,
 
     if (resource.kind == Resource::Kind::Tile) {
         assert(resource.tileData);
-        inserted = putTile(*resource.tileData, response,
-                compressed ? compressedData : response.data ? *response.data : "",
-                compressed);
+        inserted = putTile(*resource.tileData, response, *data, compressed);
     } else {
-        inserted = putResource(resource, response,
-                compressed ? compressedData : response.data ? *response.data : "",
-                compressed);
+        inserted = putResource(resource, response, *data, compressed);
     }
 
     return { inserted, size };
@@ -243,12 +256,20 @@ optional<std::pair<Response, uint64_t>> OfflineDatabase::getResource(const Resou
     optional<std::string> data = stmt->get<optional<std::string>>(4);
     if (!data) {
         response.noContent = true;
-    } else if (stmt->get<bool>(5)) {
-        response.data = std::make_shared<std::string>(util::decompress(*data));
-        size = data->length();
     } else {
-        response.data = std::make_shared<std::string>(*data);
+        response.data = { std::move(*data), stmt->get<bool>(5) };
         size = data->length();
+
+        // Make sure the data is decompressed when the user explicitly requested uncompressed data.
+        if (response.data.isCompressed() &&
+            resource.compression == Resource::Compression::Uncompressed) {
+            try {
+                response.data.uncompress();
+            } catch (std::exception& ex) {
+                response.error =
+                    std::make_unique<Response::Error>(Response::Error::Reason::Other, ex.what());
+            }
+        }
     }
 
     return std::make_pair(response, size);
@@ -359,7 +380,9 @@ bool OfflineDatabase::putResource(const Resource& resource,
     return true;
 }
 
-optional<std::pair<Response, uint64_t>> OfflineDatabase::getTile(const Resource::TileData& tile) {
+optional<std::pair<Response, uint64_t>> OfflineDatabase::getTile(const Resource& resource) {
+    const auto& tile = *resource.tileData;
+
     // clang-format off
     Statement accessedStmt = getStatement(
         "UPDATE tiles "
@@ -412,12 +435,20 @@ optional<std::pair<Response, uint64_t>> OfflineDatabase::getTile(const Resource:
     optional<std::string> data = stmt->get<optional<std::string>>(4);
     if (!data) {
         response.noContent = true;
-    } else if (stmt->get<bool>(5)) {
-        response.data = std::make_shared<std::string>(util::decompress(*data));
-        size = data->length();
     } else {
-        response.data = std::make_shared<std::string>(*data);
+        response.data = { std::move(*data), stmt->get<bool>(5) };
         size = data->length();
+
+        // Make sure the data is decompressed when the user explicitly requested uncompressed data.
+        if (response.data.isCompressed() &&
+            resource.compression == Resource::Compression::Uncompressed) {
+            try {
+                response.data.uncompress();
+            } catch (std::exception& ex) {
+                response.error =
+                    std::make_unique<Response::Error>(Response::Error::Reason::Other, ex.what());
+            }
+        }
     }
 
     return std::make_pair(response, size);
diff --git a/platform/default/mbgl/storage/offline_database.hpp b/platform/default/mbgl/storage/offline_database.hpp
index 91b544a9e0..fba1a3c230 100644
--- a/platform/default/mbgl/storage/offline_database.hpp
+++ b/platform/default/mbgl/storage/offline_database.hpp
@@ -81,7 +81,7 @@ private:
 
     Statement getStatement(const char *);
 
-    optional<std::pair<Response, uint64_t>> getTile(const Resource::TileData&);
+    optional<std::pair<Response, uint64_t>> getTile(const Resource&);
     optional<int64_t> hasTile(const Resource::TileData&);
     bool putTile(const Resource::TileData&, const Response&,
                  const std::string&, bool compressed);
diff --git a/platform/default/mbgl/storage/offline_download.cpp b/platform/default/mbgl/storage/offline_download.cpp
index ba504c1f9b..d8fe8c646b 100644
--- a/platform/default/mbgl/storage/offline_download.cpp
+++ b/platform/default/mbgl/storage/offline_download.cpp
@@ -71,7 +71,7 @@ OfflineRegionStatus OfflineDownload::getStatus() const {
     }
 
     style::Parser parser;
-    parser.parse(*styleResponse->data);
+    parser.parse(*styleResponse->data.uncompressedData());
 
     result.requiredResourceCountIsPrecise = true;
 
@@ -88,7 +88,8 @@ OfflineRegionStatus OfflineDownload::getStatus() const {
                 optional<Response> sourceResponse = offlineDatabase.get(Resource::source(url));
                 if (sourceResponse) {
                     style::conversion::Error error;
-                    optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(*sourceResponse->data, error);
+                    optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(
+                        *sourceResponse->data.uncompressedData(), error);
                     if (tileset) {
                         result.requiredResourceCount +=
                             definition.tileCount(type, tileSize, (*tileset).zoomRange);
@@ -160,7 +161,7 @@ void OfflineDownload::activateDownload() {
         status.requiredResourceCountIsPrecise = true;
 
         style::Parser parser;
-        parser.parse(*styleResponse.data);
+        parser.parse(*styleResponse.data.uncompressedData());
 
         for (const auto& source : parser.sources) {
             SourceType type = source->getType();
@@ -176,7 +177,8 @@ void OfflineDownload::activateDownload() {
 
                     ensureResource(Resource::source(url), [=](Response sourceResponse) {
                         style::conversion::Error error;
-                        optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(*sourceResponse.data, error);
+                        optional<Tileset> tileset = style::conversion::convertJSON<Tileset>(
+                            *sourceResponse.data.uncompressedData(), error);
                         if (tileset) {
                             util::mapbox::canonicalizeTileset(*tileset, url, type, tileSize);
                             queueTiles(type, tileSize, *tileset);
@@ -236,14 +238,18 @@ void OfflineDownload::activateDownload() {
         if (!parser.glyphURL.empty()) {
             for (const auto& fontStack : parser.fontStacks()) {
                 for (char16_t i = 0; i < GLYPH_RANGES_PER_FONT_STACK; i++) {
-                    queueResource(Resource::glyphs(parser.glyphURL, fontStack, getGlyphRange(i * GLYPHS_PER_GLYPH_RANGE)));
+                    auto resource = Resource::glyphs(parser.glyphURL, fontStack, getGlyphRange(i * GLYPHS_PER_GLYPH_RANGE));
+                    resource.compression = Resource::Compression::PreferCompressed;
+                    queueResource(resource);
                 }
             }
         }
 
         if (!parser.spriteURL.empty()) {
             queueResource(Resource::spriteImage(parser.spriteURL, definition.pixelRatio));
-            queueResource(Resource::spriteJSON(parser.spriteURL, definition.pixelRatio));
+            auto spriteJSON = Resource::spriteJSON(parser.spriteURL, definition.pixelRatio);
+            spriteJSON.compression = Resource::Compression::PreferCompressed;
+            queueResource(spriteJSON);
         }
 
         continueDownload();