[core] introduce Blob for compressed and uncompressed dataupstream/blob

- Blob is a wrapper type for a shared_ptr<const string> that has accessor functions for getting compressed and uncompressed data
- Moved util::writeFile, util::readFile, util::compress, util::uncompress, decodeImage, and encodePNG to the Blob interface
- Added Blob support to Request and file sources
- Added Blob support to VectorTile objects
- Added support for gzip decoding to util::uncompress
- We're no longer compressing WebP, PNG, and JPEG data when storing in the OfflineDatabase
- Android's HTTPRequest returns compressed Blobs by default

One caveat is that our previous decompress function didn't support gzip, so once users upgrade to this version, their offline cache may contain both zlib-compressed data and gzip-compressed data, but older versions won't be able to decompress gzip data. On the other hand, we don't support downgrading SDKs anyway, so this shouldn't be a problem. To be on the safe side, we could bump the user_version of the SQLite DB.

2 files changed, 30 insertions, 34 deletions

diff --git a/test/map/map.test.cpp b/test/map/map.test.cpp
index 9b34ea89b0..bb0ff2a201 100644
--- a/test/map/map.test.cpp
+++ b/test/map/map.test.cpp
@@ -21,7 +21,6 @@
 
 using namespace mbgl;
 using namespace mbgl::style;
-using namespace std::literals::string_literals;
 
 class StubMapObserver : public MapObserver {
 public:
@@ -162,7 +161,7 @@ TEST(Map, Offline) {
 
     auto expiredItem = [] (const std::string& path) {
         Response response;
-        response.data = std::make_shared<std::string>(util::read_file("test/fixtures/map/offline/"s + path));
+        response.data = util::readFile("test/fixtures/map/offline/" + path);
         response.expires = Timestamp{ Seconds(0) };
         return response;
     };
@@ -188,13 +187,13 @@ TEST(Map, Offline) {
 
 TEST(Map, SetStyleDefaultCamera) {
     MapTest<> test;
-    test.map.getStyle().loadJSON(util::read_file("test/fixtures/api/empty.json"));
+    test.map.getStyle().loadJSON(util::readFile("test/fixtures/api/empty.json"));
     EXPECT_DOUBLE_EQ(test.map.getZoom(), 0.0);
     EXPECT_DOUBLE_EQ(test.map.getPitch(), 0.0);
     EXPECT_DOUBLE_EQ(test.map.getBearing(), 0.0);
     EXPECT_EQ(test.map.getLatLng(), LatLng {});
 
-    test.map.getStyle().loadJSON(util::read_file("test/fixtures/api/empty-zoomed.json"));
+    test.map.getStyle().loadJSON(util::readFile("test/fixtures/api/empty-zoomed.json"));
     EXPECT_DOUBLE_EQ(test.map.getZoom(), 0.0);
 
     test.map.jumpTo(test.map.getStyle().getDefaultCamera());
@@ -211,7 +210,7 @@ TEST(Map, SetStyleInvalidJSON) {
         test.observer.didFailLoadingMapCallback = [&]() {
             fail = true;
         };
-        test.map.getStyle().loadJSON("invalid");
+        test.map.getStyle().loadJSON(Blob{ "invalid", false });
     }
 
     EXPECT_TRUE(fail);
@@ -247,8 +246,8 @@ TEST(Map, SetStyleInvalidURL) {
 TEST(Map, DoubleStyleLoad) {
     MapTest<> test;
 
-    test.map.getStyle().loadJSON("");
-    test.map.getStyle().loadJSON("");
+    test.map.getStyle().loadJSON(Blob{ "", false });
+    test.map.getStyle().loadJSON(Blob{ "", false });
 }
 
 TEST(Map, StyleFresh) {
@@ -260,7 +259,7 @@ TEST(Map, StyleFresh) {
     EXPECT_EQ(1u, test.fileSource.requests.size());
 
     Response response;
-    response.data = std::make_shared<std::string>(util::read_file("test/fixtures/api/empty.json"));
+    response.data = util::readFile("test/fixtures/api/empty.json");
     response.expires = Timestamp::max();
 
     test.fileSource.respond(Resource::Style, response);
@@ -278,7 +277,7 @@ TEST(Map, StyleExpired) {
     EXPECT_EQ(1u, test.fileSource.requests.size());
 
     Response response;
-    response.data = std::make_shared<std::string>(util::read_file("test/fixtures/api/empty.json"));
+    response.data = util::readFile("test/fixtures/api/empty.json");
     response.expires = util::now() - 1h;
 
     test.fileSource.respond(Resource::Style, response);
@@ -303,7 +302,7 @@ TEST(Map, StyleExpiredWithAnnotations) {
     EXPECT_EQ(1u, test.fileSource.requests.size());
 
     Response response;
-    response.data = std::make_shared<std::string>(util::read_file("test/fixtures/api/empty.json"));
+    response.data = util::readFile("test/fixtures/api/empty.json");
     response.expires = util::now() - 1h;
 
     test.fileSource.respond(Resource::Style, response);
@@ -327,7 +326,7 @@ TEST(Map, StyleExpiredWithRender) {
     EXPECT_EQ(1u, test.fileSource.requests.size());
 
     Response response;
-    response.data = std::make_shared<std::string>(util::read_file("test/fixtures/api/empty.json"));
+    response.data = util::readFile("test/fixtures/api/empty.json");
     response.expires = util::now() - 1h;
 
     test.fileSource.respond(Resource::Style, response);
@@ -349,7 +348,7 @@ TEST(Map, StyleEarlyMutation) {
     test.map.getStyle().addLayer(std::make_unique<style::BackgroundLayer>("bg"));
 
     Response response;
-    response.data = std::make_shared<std::string>(util::read_file("test/fixtures/api/water.json"));
+    response.data = util::readFile("test/fixtures/api/water.json");
     test.fileSource.respond(Resource::Style, response);
 
     EXPECT_EQ(0u, test.fileSource.requests.size());
@@ -363,7 +362,7 @@ TEST(Map, MapLoadingSignal) {
     test.observer.onWillStartLoadingMapCallback = [&]() {
         emitted = true;
     };
-    test.map.getStyle().loadJSON(util::read_file("test/fixtures/api/empty.json"));
+    test.map.getStyle().loadJSON(util::readFile("test/fixtures/api/empty.json"));
     EXPECT_TRUE(emitted);
 }
 
@@ -374,7 +373,7 @@ TEST(Map, MapLoadedSignal) {
         test.runLoop.stop();
     };
 
-    test.map.getStyle().loadJSON(util::read_file("test/fixtures/api/empty.json"));
+    test.map.getStyle().loadJSON(util::readFile("test/fixtures/api/empty.json"));
     test.runLoop.run();
 }
 
@@ -386,12 +385,12 @@ TEST(Map, StyleLoadedSignal) {
     test.observer.didFinishLoadingStyleCallback = [&]() {
         emitted = true;
     };
-    test.map.getStyle().loadJSON(util::read_file("test/fixtures/api/empty.json"));
+    test.map.getStyle().loadJSON(util::readFile("test/fixtures/api/empty.json"));
     EXPECT_TRUE(emitted);
 
     // But not when the style couldn't be parsed
     emitted = false;
-    test.map.getStyle().loadJSON("invalid");
+    test.map.getStyle().loadJSON(Blob{ "invalid", false });
     EXPECT_FALSE(emitted);
 }
 
@@ -437,7 +436,7 @@ TEST(Map, TEST_REQUIRES_SERVER(StyleNotFound)) {
 TEST(Map, AddLayer) {
     MapTest<> test;
 
-    test.map.getStyle().loadJSON(util::read_file("test/fixtures/api/empty.json"));
+    test.map.getStyle().loadJSON(util::readFile("test/fixtures/api/empty.json"));
 
     auto layer = std::make_unique<BackgroundLayer>("background");
     layer->setBackgroundColor({ { 1, 0, 0, 1 } });
@@ -452,7 +451,7 @@ TEST(Map, WithoutVAOExtension) {
     BackendScope scope { *test.frontend.getBackend() };
     test.frontend.getBackend()->getContext().disableVAOExtension = true;
 
-    test.map.getStyle().loadJSON(util::read_file("test/fixtures/api/water.json"));
+    test.map.getStyle().loadJSON(util::readFile("test/fixtures/api/water.json"));
 
     test::checkImage("test/fixtures/map/no_vao", test.frontend.render(test.map), 0.002);
 }
@@ -460,7 +459,7 @@ TEST(Map, WithoutVAOExtension) {
 TEST(Map, RemoveLayer) {
     MapTest<> test;
 
-    test.map.getStyle().loadJSON(util::read_file("test/fixtures/api/empty.json"));
+    test.map.getStyle().loadJSON(util::readFile("test/fixtures/api/empty.json"));
 
     auto layer = std::make_unique<BackgroundLayer>("background");
     layer->setBackgroundColor({{ 1, 0, 0, 1 }});
@@ -477,8 +476,7 @@ TEST(Map, DisabledSources) {
     test.fileSource.response = [] (const Resource& res) -> optional<Response> {
         if (res.url == "asset://tile.png") {
             Response response;
-            response.data = std::make_shared<std::string>(
-                util::read_file("test/fixtures/map/disabled_layers/tile.png"));
+            response.data = util::readFile("test/fixtures/map/disabled_layers/tile.png");
             return {std::move(response)};
         }
         return {};
@@ -491,7 +489,7 @@ TEST(Map, DisabledSources) {
     // to an opacity of 0.5). Then, we are zooming back out to a zoom level of 0.5 and rerender.
     // The "raster1" layer should not be visible anymore since it has minzoom 1, while "raster2"
     // should still be there. Both layers have a distinct color through "raster-hue-rotate".
-    test.map.getStyle().loadJSON(R"STYLE(
+    test.map.getStyle().loadJSON(Blob{ R"STYLE(
 {
   "version": 8,
   "name": "Test",
@@ -523,7 +521,7 @@ TEST(Map, DisabledSources) {
     }
   }]
 }
-)STYLE");
+)STYLE", false });
 
     test::checkImage("test/fixtures/map/disabled_layers/first", test.frontend.render(test.map));
     test.map.setZoom(0.5);
@@ -533,7 +531,7 @@ TEST(Map, DisabledSources) {
 TEST(Map, DontLoadUnneededTiles) {
     MapTest<> test;
 
-    test.map.getStyle().loadJSON(R"STYLE({
+    test.map.getStyle().loadJSON(Blob{ R"STYLE({
   "sources": {
     "a": { "type": "vector", "tiles": [ "a/{z}/{x}/{y}" ] }
   },
@@ -545,7 +543,7 @@ TEST(Map, DontLoadUnneededTiles) {
     "minzoom": 0.3,
     "maxzoom": 1.6
   }]
-})STYLE");
+})STYLE", false });
 
     using Tiles = std::unordered_set<std::string>;
     Tiles tiles;
@@ -606,7 +604,7 @@ TEST(Map, TEST_DISABLED_ON_CI(ContinuousRendering)) {
     };
 
     Map map(frontend, observer, frontend.getSize(), pixelRatio, fileSource, threadPool, MapMode::Continuous);
-    map.getStyle().loadJSON(util::read_file("test/fixtures/api/water.json"));
+    map.getStyle().loadJSON(util::readFile("test/fixtures/api/water.json"));
 
     runLoop.run();
 }
@@ -624,7 +622,7 @@ TEST(Map, NoContentTiles) {
                                        Tileset::Scheme::XYZ),
                         response);
 
-    test.map.getStyle().loadJSON(R"STYLE({
+    test.map.getStyle().loadJSON(Blob{ R"STYLE({
       "version": 8,
       "name": "Water",
       "sources": {
@@ -645,7 +643,7 @@ TEST(Map, NoContentTiles) {
         "source": "mapbox",
         "source-layer": "water"
       }]
-    })STYLE");
+    })STYLE", false });
 
     test::checkImage("test/fixtures/map/nocontent",
                      test.frontend.render(test.map),
diff --git a/test/map/prefetch.test.cpp b/test/map/prefetch.test.cpp
index 4c82b2c965..7cb8ea1545 100644
--- a/test/map/prefetch.test.cpp
+++ b/test/map/prefetch.test.cpp
@@ -37,11 +37,9 @@ TEST(Map, PrefetchTiles) {
         // The end rendering result should be all green because the map is only
         // considered fully rendered when only ideal tiles are shown.
         if (zoom == int(map.getZoom()) + 1) {
-            response.data = std::make_shared<std::string>(
-                util::read_file("test/fixtures/map/prefetch/tile_green.png"));
+            response.data = util::readFile("test/fixtures/map/prefetch/tile_green.png");
         } else {
-            response.data = std::make_shared<std::string>(
-                util::read_file("test/fixtures/map/prefetch/tile_red.png"));
+            response.data = util::readFile("test/fixtures/map/prefetch/tile_red.png");
         }
 
         return { std::move(response) };
@@ -51,8 +49,8 @@ TEST(Map, PrefetchTiles) {
         tiles.clear();
 
         // Force tile reloading.
-        map.getStyle().loadJSON(util::read_file("test/fixtures/map/prefetch/empty.json"));
-        map.getStyle().loadJSON(util::read_file("test/fixtures/map/prefetch/style.json"));
+        map.getStyle().loadJSON(util::readFile("test/fixtures/map/prefetch/empty.json"));
+        map.getStyle().loadJSON(util::readFile("test/fixtures/map/prefetch/style.json"));
 
         map.setLatLngZoom({ 40.726989, -73.992857 }, zoom); // Manhattan