diff options
Diffstat (limited to 'src')
36 files changed, 188 insertions, 94 deletions
diff --git a/src/mbgl/gl/program.hpp b/src/mbgl/gl/program.hpp index 3b54ec194a..64e3a355d6 100644 --- a/src/mbgl/gl/program.hpp +++ b/src/mbgl/gl/program.hpp @@ -62,7 +62,7 @@ public: try { if (auto cachedBinaryProgram = util::readFile(*cachePath)) { - const BinaryProgram binaryProgram(std::move(*cachedBinaryProgram)); + const BinaryProgram binaryProgram(cachedBinaryProgram); if (binaryProgram.identifier() == identifier) { return Program { context, binaryProgram }; } else { @@ -82,7 +82,7 @@ public: try { if (const auto binaryProgram = result.template get<BinaryProgram>(context, identifier)) { - util::write_file(*cachePath, binaryProgram->serialize()); + util::writeFile(*cachePath, binaryProgram->serialize()); Log::Warning(Event::OpenGL, "Caching program in: %s", (*cachePath).c_str()); } } catch (std::runtime_error& error) { diff --git a/src/mbgl/programs/binary_program.cpp b/src/mbgl/programs/binary_program.cpp index da629194b4..e5943bb5db 100644 --- a/src/mbgl/programs/binary_program.cpp +++ b/src/mbgl/programs/binary_program.cpp @@ -32,9 +32,10 @@ static std::pair<const std::string, Binding> parseBinding(protozero::pbf_reader& namespace mbgl { -BinaryProgram::BinaryProgram(std::string&& data) { +BinaryProgram::BinaryProgram(Blob blob) { + const auto data = blob.uncompressedData(); bool hasFormat = false, hasCode = false; - protozero::pbf_reader pbf(data); + protozero::pbf_reader pbf(*data); while (pbf.next()) { switch (pbf.tag()) { case 1: // format @@ -76,7 +77,7 @@ BinaryProgram::BinaryProgram( uniforms(std::move(uniforms_)) { } -std::string BinaryProgram::serialize() const { +Blob BinaryProgram::serialize() const { std::string data; data.reserve(32 + binaryCode.size() + uniforms.size() * 32 + attributes.size() * 32); protozero::pbf_writer pbf(data); @@ -95,7 +96,7 @@ std::string BinaryProgram::serialize() const { if (!binaryIdentifier.empty()) { pbf.add_string(5 /* identifier */, binaryIdentifier); } - return data; + return { std::move(data), false }; } optional<gl::AttributeLocation> BinaryProgram::attributeLocation(const std::string& name) const { diff --git a/src/mbgl/programs/binary_program.hpp b/src/mbgl/programs/binary_program.hpp index 8690f3fd6f..2d3ea45aa6 100644 --- a/src/mbgl/programs/binary_program.hpp +++ b/src/mbgl/programs/binary_program.hpp @@ -1,6 +1,7 @@ #pragma once #include <mbgl/gl/types.hpp> +#include <mbgl/util/blob.hpp> #include <mbgl/util/optional.hpp> #include <string> @@ -11,7 +12,7 @@ namespace mbgl { class BinaryProgram { public: // Initialize a BinaryProgram object from a serialized represenation. - BinaryProgram(std::string&& data); + BinaryProgram(Blob data); BinaryProgram(gl::BinaryProgramFormat, std::string&& binaryCode, @@ -19,7 +20,7 @@ public: std::vector<std::pair<const std::string, gl::AttributeLocation>>&&, std::vector<std::pair<const std::string, gl::UniformLocation>>&&); - std::string serialize() const; + Blob serialize() const; gl::BinaryProgramFormat format() const { return binaryFormat; diff --git a/src/mbgl/sprite/sprite_loader.cpp b/src/mbgl/sprite/sprite_loader.cpp index 93d6dfd9ae..3bc8add160 100644 --- a/src/mbgl/sprite/sprite_loader.cpp +++ b/src/mbgl/sprite/sprite_loader.cpp @@ -25,8 +25,8 @@ struct SpriteLoader::Loader { worker(scheduler, ActorRef<SpriteLoader>(imageManager, mailbox)) { } - std::shared_ptr<const std::string> image; - std::shared_ptr<const std::string> json; + Blob image; + Blob json; std::unique_ptr<AsyncRequest> jsonRequest; std::unique_ptr<AsyncRequest> spriteRequest; std::shared_ptr<Mailbox> mailbox; @@ -55,7 +55,7 @@ void SpriteLoader::load(const std::string& url, Scheduler& scheduler, FileSource } else if (res.notModified) { return; } else if (res.noContent) { - loader->json = std::make_shared<const std::string>(); + loader->json = {}; emitSpriteLoadedIfComplete(); } else { // Only trigger a sprite loaded event we got new data. @@ -70,7 +70,7 @@ void SpriteLoader::load(const std::string& url, Scheduler& scheduler, FileSource } else if (res.notModified) { return; } else if (res.noContent) { - loader->image = std::make_shared<const std::string>(); + loader->image = {}; emitSpriteLoadedIfComplete(); } else { loader->image = res.data; diff --git a/src/mbgl/sprite/sprite_loader_worker.cpp b/src/mbgl/sprite/sprite_loader_worker.cpp index 4bded33d53..a2e3f331e9 100644 --- a/src/mbgl/sprite/sprite_loader_worker.cpp +++ b/src/mbgl/sprite/sprite_loader_worker.cpp @@ -8,8 +8,7 @@ SpriteLoaderWorker::SpriteLoaderWorker(ActorRef<SpriteLoaderWorker>, ActorRef<Sp : parent(std::move(parent_)) { } -void SpriteLoaderWorker::parse(std::shared_ptr<const std::string> image, - std::shared_ptr<const std::string> json) { +void SpriteLoaderWorker::parse(Blob image, Blob json) { try { if (!image) { // This shouldn't happen, since we always invoke it with a non-empty pointer. @@ -20,7 +19,7 @@ void SpriteLoaderWorker::parse(std::shared_ptr<const std::string> image, throw std::runtime_error("missing sprite metadata"); } - parent.invoke(&SpriteLoader::onParsed, parseSprite(*image, *json)); + parent.invoke(&SpriteLoader::onParsed, parseSprite(std::move(image), std::move(json))); } catch (...) { parent.invoke(&SpriteLoader::onError, std::current_exception()); } diff --git a/src/mbgl/sprite/sprite_loader_worker.hpp b/src/mbgl/sprite/sprite_loader_worker.hpp index d61e07d14f..4f2e38be29 100644 --- a/src/mbgl/sprite/sprite_loader_worker.hpp +++ b/src/mbgl/sprite/sprite_loader_worker.hpp @@ -2,6 +2,7 @@ #include <mbgl/actor/actor_ref.hpp> #include <mbgl/sprite/sprite_parser.hpp> +#include <mbgl/storage/response.hpp> #include <memory> #include <string> @@ -14,7 +15,7 @@ class SpriteLoaderWorker { public: SpriteLoaderWorker(ActorRef<SpriteLoaderWorker>, ActorRef<SpriteLoader>); - void parse(std::shared_ptr<const std::string> image, std::shared_ptr<const std::string> json); + void parse(Blob image, Blob json); private: ActorRef<SpriteLoader> parent; diff --git a/src/mbgl/sprite/sprite_parser.cpp b/src/mbgl/sprite/sprite_parser.cpp index 1a36e3e990..c93d97ba78 100644 --- a/src/mbgl/sprite/sprite_parser.cpp +++ b/src/mbgl/sprite/sprite_parser.cpp @@ -85,11 +85,12 @@ bool getBoolean(const JSValue& value, const char* name, const bool def = false) } // namespace -std::vector<std::unique_ptr<style::Image>> parseSprite(const std::string& encodedImage, const std::string& json) { - const PremultipliedImage raster = decodeImage(encodedImage); +std::vector<std::unique_ptr<style::Image>> parseSprite(Blob imageBlob, Blob jsonBlob) { + const PremultipliedImage raster = decodeImage(imageBlob); + const auto json = jsonBlob.uncompressedData(); JSDocument doc; - doc.Parse<0>(json.c_str()); + doc.Parse<0>(json->c_str()); if (doc.HasParseError()) { std::stringstream message; message << "Failed to parse JSON: " << rapidjson::GetParseError_En(doc.GetParseError()) << " at offset " << doc.GetErrorOffset(); diff --git a/src/mbgl/sprite/sprite_parser.hpp b/src/mbgl/sprite/sprite_parser.hpp index f602818d3b..b17f4cc458 100644 --- a/src/mbgl/sprite/sprite_parser.hpp +++ b/src/mbgl/sprite/sprite_parser.hpp @@ -23,6 +23,6 @@ std::unique_ptr<style::Image> createStyleImage(const std::string& id, bool sdf); // Parses an image and an associated JSON file and returns the sprite objects. -std::vector<std::unique_ptr<style::Image>> parseSprite(const std::string& image, const std::string& json); +std::vector<std::unique_ptr<style::Image>> parseSprite(Blob image, Blob json); } // namespace mbgl diff --git a/src/mbgl/storage/resource.cpp b/src/mbgl/storage/resource.cpp index 207dd2ee69..9bc8495d53 100644 --- a/src/mbgl/storage/resource.cpp +++ b/src/mbgl/storage/resource.cpp @@ -42,14 +42,20 @@ static std::string getTileBBox(int32_t x, int32_t y, int8_t z) { Resource Resource::style(const std::string& url) { return Resource { Resource::Kind::Style, - url + url, + {}, + LoadingMethod::All, + Compression::Uncompressed }; } Resource Resource::source(const std::string& url) { return Resource { Resource::Kind::Source, - url + url, + {}, + LoadingMethod::All, + Compression::Uncompressed }; } @@ -87,7 +93,10 @@ Resource Resource::glyphs(const std::string& urlTemplate, const FontStack& fontS } else { return std::string(); } - }) + }), + {}, + LoadingMethod::All, + Compression::Uncompressed }; } @@ -133,7 +142,8 @@ Resource Resource::tile(const std::string& urlTemplate, y, z }, - loadingMethod + loadingMethod, + Compression::PreferCompressed }; } diff --git a/src/mbgl/style/sources/geojson_source.cpp b/src/mbgl/style/sources/geojson_source.cpp index 4e3478322d..48308ccbc3 100644 --- a/src/mbgl/style/sources/geojson_source.cpp +++ b/src/mbgl/style/sources/geojson_source.cpp @@ -61,7 +61,8 @@ void GeoJSONSource::loadDescription(FileSource& fileSource) { *this, std::make_exception_ptr(std::runtime_error("unexpectedly empty GeoJSON"))); } else { conversion::Error error; - optional<GeoJSON> geoJSON = conversion::convertJSON<GeoJSON>(*res.data, error); + optional<GeoJSON> geoJSON = + conversion::convertJSON<GeoJSON>(*res.data.uncompressedData(), error); if (!geoJSON) { Log::Error(Event::ParseStyle, "Failed to parse GeoJSON data: %s", error.message.c_str()); diff --git a/src/mbgl/style/sources/image_source.cpp b/src/mbgl/style/sources/image_source.cpp index fa268da0ef..5be1b54410 100644 --- a/src/mbgl/style/sources/image_source.cpp +++ b/src/mbgl/style/sources/image_source.cpp @@ -70,7 +70,7 @@ void ImageSource::loadDescription(FileSource& fileSource) { observer->onSourceError(*this, std::make_exception_ptr(std::runtime_error("unexpectedly empty image url"))); } else { try { - baseImpl = makeMutable<Impl>(impl(), decodeImage(*res.data)); + baseImpl = makeMutable<Impl>(impl(), decodeImage(res.data)); } catch (...) { observer->onSourceError(*this, std::current_exception()); } diff --git a/src/mbgl/style/sources/raster_source.cpp b/src/mbgl/style/sources/raster_source.cpp index 53f29d660b..c3c47567aa 100644 --- a/src/mbgl/style/sources/raster_source.cpp +++ b/src/mbgl/style/sources/raster_source.cpp @@ -57,7 +57,7 @@ void RasterSource::loadDescription(FileSource& fileSource) { observer->onSourceError(*this, std::make_exception_ptr(std::runtime_error("unexpectedly empty TileJSON"))); } else { conversion::Error error; - optional<Tileset> tileset = conversion::convertJSON<Tileset>(*res.data, error); + optional<Tileset> tileset = conversion::convertJSON<Tileset>(*res.data.uncompressedData(), error); if (!tileset) { observer->onSourceError(*this, std::make_exception_ptr(std::runtime_error(error.message))); return; diff --git a/src/mbgl/style/sources/vector_source.cpp b/src/mbgl/style/sources/vector_source.cpp index ccdd453c75..d88622f1b8 100644 --- a/src/mbgl/style/sources/vector_source.cpp +++ b/src/mbgl/style/sources/vector_source.cpp @@ -54,7 +54,8 @@ void VectorSource::loadDescription(FileSource& fileSource) { observer->onSourceError(*this, std::make_exception_ptr(std::runtime_error("unexpectedly empty TileJSON"))); } else { conversion::Error error; - optional<Tileset> tileset = conversion::convertJSON<Tileset>(*res.data, error); + optional<Tileset> tileset = + conversion::convertJSON<Tileset>(*res.data.uncompressedData(), error); if (!tileset) { observer->onSourceError(*this, std::make_exception_ptr(std::runtime_error(error.message))); return; diff --git a/src/mbgl/style/style.cpp b/src/mbgl/style/style.cpp index bd8631fc52..82f421ecdf 100644 --- a/src/mbgl/style/style.cpp +++ b/src/mbgl/style/style.cpp @@ -14,7 +14,7 @@ Style::Style(Scheduler& scheduler, FileSource& fileSource, float pixelRatio) Style::~Style() = default; -void Style::loadJSON(const std::string& json) { +void Style::loadJSON(Blob json) { impl->loadJSON(json); } @@ -22,7 +22,7 @@ void Style::loadURL(const std::string& url) { impl->loadURL(url); } -std::string Style::getJSON() const { +Blob Style::getJSON() const { return impl->getJSON(); } diff --git a/src/mbgl/style/style_impl.cpp b/src/mbgl/style/style_impl.cpp index d330b3120a..a952ccb8c2 100644 --- a/src/mbgl/style/style_impl.cpp +++ b/src/mbgl/style/style_impl.cpp @@ -39,7 +39,7 @@ Style::Impl::Impl(Scheduler& scheduler_, FileSource& fileSource_, float pixelRat Style::Impl::~Impl() = default; -void Style::Impl::loadJSON(const std::string& json_) { +void Style::Impl::loadJSON(Blob json_) { lastError = nullptr; observer->onStyleLoading(); @@ -73,15 +73,17 @@ void Style::Impl::loadURL(const std::string& url_) { } else if (res.notModified || res.noContent) { return; } else { - parse(*res.data); + parse(res.data); } }); } -void Style::Impl::parse(const std::string& json_) { +void Style::Impl::parse(Blob json_) { Parser parser; - if (auto error = parser.parse(json_)) { + const auto data = json_.uncompressedData(); + + if (auto error = parser.parse(*data)) { std::string message = "Failed to parse style: " + util::toString(error); Log::Error(Event::ParseStyle, message.c_str()); observer->onStyleError(std::make_exception_ptr(util::StyleParseException(message))); @@ -124,7 +126,7 @@ void Style::Impl::parse(const std::string& json_) { observer->onStyleLoaded(); } -std::string Style::Impl::getJSON() const { +Blob Style::Impl::getJSON() const { return json; } diff --git a/src/mbgl/style/style_impl.hpp b/src/mbgl/style/style_impl.hpp index 3dc222bfad..9dc2704391 100644 --- a/src/mbgl/style/style_impl.hpp +++ b/src/mbgl/style/style_impl.hpp @@ -41,10 +41,10 @@ public: Impl(Scheduler&, FileSource&, float pixelRatio); ~Impl() override; - void loadJSON(const std::string&); + void loadJSON(Blob); void loadURL(const std::string&); - std::string getJSON() const; + Blob getJSON() const; std::string getURL() const; void setObserver(Observer*); @@ -96,13 +96,13 @@ public: bool spriteLoaded = false; private: - void parse(const std::string&); + void parse(Blob); Scheduler& scheduler; FileSource& fileSource; std::string url; - std::string json; + Blob json; std::unique_ptr<AsyncRequest> styleRequest; std::unique_ptr<SpriteLoader> spriteLoader; diff --git a/src/mbgl/text/glyph_manager.cpp b/src/mbgl/text/glyph_manager.cpp index 3130418908..37574eff15 100644 --- a/src/mbgl/text/glyph_manager.cpp +++ b/src/mbgl/text/glyph_manager.cpp @@ -91,7 +91,7 @@ void GlyphManager::processResponse(const Response& res, const FontStack& fontSta std::vector<Glyph> glyphs; try { - glyphs = parseGlyphPBF(range, *res.data); + glyphs = parseGlyphPBF(range, res.data); } catch (...) { observer->onGlyphsError(fontStack, range, std::current_exception()); return; diff --git a/src/mbgl/text/glyph_pbf.cpp b/src/mbgl/text/glyph_pbf.cpp index cfaf803f75..77ea5ec2cb 100644 --- a/src/mbgl/text/glyph_pbf.cpp +++ b/src/mbgl/text/glyph_pbf.cpp @@ -4,11 +4,12 @@ namespace mbgl { -std::vector<Glyph> parseGlyphPBF(const GlyphRange& glyphRange, const std::string& data) { +std::vector<Glyph> parseGlyphPBF(const GlyphRange& glyphRange, Blob blob) { + const auto data = blob.uncompressedData(); std::vector<Glyph> result; result.reserve(256); - protozero::pbf_reader glyphs_pbf(data); + protozero::pbf_reader glyphs_pbf(*data); while (glyphs_pbf.next(1)) { auto fontstack_pbf = glyphs_pbf.get_message(); diff --git a/src/mbgl/text/glyph_pbf.hpp b/src/mbgl/text/glyph_pbf.hpp index 28a28b4114..0c82343ce5 100644 --- a/src/mbgl/text/glyph_pbf.hpp +++ b/src/mbgl/text/glyph_pbf.hpp @@ -8,6 +8,6 @@ namespace mbgl { -std::vector<Glyph> parseGlyphPBF(const GlyphRange&, const std::string& data); +std::vector<Glyph> parseGlyphPBF(const GlyphRange&, Blob blob); } // namespace mbgl diff --git a/src/mbgl/tile/raster_dem_tile.cpp b/src/mbgl/tile/raster_dem_tile.cpp index 5db298cf4c..10f22e378b 100644 --- a/src/mbgl/tile/raster_dem_tile.cpp +++ b/src/mbgl/tile/raster_dem_tile.cpp @@ -45,7 +45,7 @@ void RasterDEMTile::setMetadata(optional<Timestamp> modified_, optional<Timestam expires = expires_; } -void RasterDEMTile::setData(std::shared_ptr<const std::string> data) { +void RasterDEMTile::setData(Blob data) { pending = true; ++correlationID; worker.invoke(&RasterDEMTileWorker::parse, data, correlationID, encoding); diff --git a/src/mbgl/tile/raster_dem_tile.hpp b/src/mbgl/tile/raster_dem_tile.hpp index 0c8dd75961..0c5b3e7bf2 100644 --- a/src/mbgl/tile/raster_dem_tile.hpp +++ b/src/mbgl/tile/raster_dem_tile.hpp @@ -70,7 +70,7 @@ public: void setError(std::exception_ptr); void setMetadata(optional<Timestamp> modified, optional<Timestamp> expires); - void setData(std::shared_ptr<const std::string> data); + void setData(Blob data); void upload(gl::Context&) override; Bucket* getBucket(const style::Layer::Impl&) const override; diff --git a/src/mbgl/tile/raster_dem_tile_worker.cpp b/src/mbgl/tile/raster_dem_tile_worker.cpp index 7338e578c7..ec5aba380a 100644 --- a/src/mbgl/tile/raster_dem_tile_worker.cpp +++ b/src/mbgl/tile/raster_dem_tile_worker.cpp @@ -10,14 +10,14 @@ RasterDEMTileWorker::RasterDEMTileWorker(ActorRef<RasterDEMTileWorker>, ActorRef : parent(std::move(parent_)) { } -void RasterDEMTileWorker::parse(std::shared_ptr<const std::string> data, uint64_t correlationID, Tileset::DEMEncoding encoding) { +void RasterDEMTileWorker::parse(Blob data, uint64_t correlationID, Tileset::DEMEncoding encoding) { if (!data) { parent.invoke(&RasterDEMTile::onParsed, nullptr, correlationID); // No data; empty tile. return; } try { - auto bucket = std::make_unique<HillshadeBucket>(decodeImage(*data), encoding); + auto bucket = std::make_unique<HillshadeBucket>(decodeImage(data), encoding); parent.invoke(&RasterDEMTile::onParsed, std::move(bucket), correlationID); } catch (...) { parent.invoke(&RasterDEMTile::onError, std::current_exception(), correlationID); diff --git a/src/mbgl/tile/raster_dem_tile_worker.hpp b/src/mbgl/tile/raster_dem_tile_worker.hpp index 5a8222bc2d..5b9bce1a19 100644 --- a/src/mbgl/tile/raster_dem_tile_worker.hpp +++ b/src/mbgl/tile/raster_dem_tile_worker.hpp @@ -2,6 +2,7 @@ #include <mbgl/actor/actor_ref.hpp> #include <mbgl/util/tileset.hpp> +#include <mbgl/util/blob.hpp> #include <memory> #include <string> @@ -14,7 +15,7 @@ class RasterDEMTileWorker { public: RasterDEMTileWorker(ActorRef<RasterDEMTileWorker>, ActorRef<RasterDEMTile>); - void parse(std::shared_ptr<const std::string> data, uint64_t correlationID, Tileset::DEMEncoding encoding); + void parse(Blob data, uint64_t correlationID, Tileset::DEMEncoding encoding); private: ActorRef<RasterDEMTile> parent; diff --git a/src/mbgl/tile/raster_tile.cpp b/src/mbgl/tile/raster_tile.cpp index ff23d4493e..4bc9830fda 100644 --- a/src/mbgl/tile/raster_tile.cpp +++ b/src/mbgl/tile/raster_tile.cpp @@ -34,7 +34,7 @@ void RasterTile::setMetadata(optional<Timestamp> modified_, optional<Timestamp> expires = expires_; } -void RasterTile::setData(std::shared_ptr<const std::string> data) { +void RasterTile::setData(Blob data) { pending = true; ++correlationID; worker.invoke(&RasterTileWorker::parse, data, correlationID); diff --git a/src/mbgl/tile/raster_tile.hpp b/src/mbgl/tile/raster_tile.hpp index e25329119a..c1fa6caa99 100644 --- a/src/mbgl/tile/raster_tile.hpp +++ b/src/mbgl/tile/raster_tile.hpp @@ -26,7 +26,7 @@ public: void setError(std::exception_ptr); void setMetadata(optional<Timestamp> modified, optional<Timestamp> expires); - void setData(std::shared_ptr<const std::string> data); + void setData(Blob data); void upload(gl::Context&) override; Bucket* getBucket(const style::Layer::Impl&) const override; diff --git a/src/mbgl/tile/raster_tile_worker.cpp b/src/mbgl/tile/raster_tile_worker.cpp index 4afa876429..d0dd8918a2 100644 --- a/src/mbgl/tile/raster_tile_worker.cpp +++ b/src/mbgl/tile/raster_tile_worker.cpp @@ -10,14 +10,14 @@ RasterTileWorker::RasterTileWorker(ActorRef<RasterTileWorker>, ActorRef<RasterTi : parent(std::move(parent_)) { } -void RasterTileWorker::parse(std::shared_ptr<const std::string> data, uint64_t correlationID) { +void RasterTileWorker::parse(Blob data, uint64_t correlationID) { if (!data) { parent.invoke(&RasterTile::onParsed, nullptr, correlationID); // No data; empty tile. return; } try { - auto bucket = std::make_unique<RasterBucket>(decodeImage(*data)); + auto bucket = std::make_unique<RasterBucket>(decodeImage(data)); parent.invoke(&RasterTile::onParsed, std::move(bucket), correlationID); } catch (...) { parent.invoke(&RasterTile::onError, std::current_exception(), correlationID); diff --git a/src/mbgl/tile/raster_tile_worker.hpp b/src/mbgl/tile/raster_tile_worker.hpp index 520973c3c3..c56da82d0a 100644 --- a/src/mbgl/tile/raster_tile_worker.hpp +++ b/src/mbgl/tile/raster_tile_worker.hpp @@ -1,6 +1,7 @@ #pragma once #include <mbgl/actor/actor_ref.hpp> +#include <mbgl/util/blob.hpp> #include <memory> #include <string> @@ -13,7 +14,7 @@ class RasterTileWorker { public: RasterTileWorker(ActorRef<RasterTileWorker>, ActorRef<RasterTile>); - void parse(std::shared_ptr<const std::string> data, uint64_t correlationID); + void parse(Blob data, uint64_t correlationID); private: ActorRef<RasterTile> parent; diff --git a/src/mbgl/tile/tile_loader_impl.hpp b/src/mbgl/tile/tile_loader_impl.hpp index 1b29638269..48b3b7c87b 100644 --- a/src/mbgl/tile/tile_loader_impl.hpp +++ b/src/mbgl/tile/tile_loader_impl.hpp @@ -106,7 +106,7 @@ void TileLoader<T>::loadedData(const Response& res) { resource.priorExpires = res.expires; resource.priorEtag = res.etag; tile.setMetadata(res.modified, res.expires); - tile.setData(res.noContent ? nullptr : res.data); + tile.setData(res.noContent ? Blob{} : res.data); } } diff --git a/src/mbgl/tile/vector_tile.cpp b/src/mbgl/tile/vector_tile.cpp index 0756d3e526..a815f31e41 100644 --- a/src/mbgl/tile/vector_tile.cpp +++ b/src/mbgl/tile/vector_tile.cpp @@ -21,8 +21,8 @@ void VectorTile::setMetadata(optional<Timestamp> modified_, optional<Timestamp> expires = expires_; } -void VectorTile::setData(std::shared_ptr<const std::string> data_) { - GeometryTile::setData(data_ ? std::make_unique<VectorTileData>(data_) : nullptr); +void VectorTile::setData(Blob data_) { + GeometryTile::setData(data_ ? std::make_unique<VectorTileData>(std::move(data_)) : nullptr); } } // namespace mbgl diff --git a/src/mbgl/tile/vector_tile.hpp b/src/mbgl/tile/vector_tile.hpp index 7dae414fef..87abf2f598 100644 --- a/src/mbgl/tile/vector_tile.hpp +++ b/src/mbgl/tile/vector_tile.hpp @@ -17,7 +17,7 @@ public: void setNecessity(TileNecessity) final; void setMetadata(optional<Timestamp> modified, optional<Timestamp> expires); - void setData(std::shared_ptr<const std::string> data); + void setData(Blob data); private: TileLoader<VectorTile> loader; diff --git a/src/mbgl/tile/vector_tile_data.cpp b/src/mbgl/tile/vector_tile_data.cpp index 2d4a01bda3..eaa91dc9cf 100644 --- a/src/mbgl/tile/vector_tile_data.cpp +++ b/src/mbgl/tile/vector_tile_data.cpp @@ -60,19 +60,24 @@ std::string VectorTileLayer::getName() const { return layer.getName(); } -VectorTileData::VectorTileData(std::shared_ptr<const std::string> data_) : data(std::move(data_)) { +VectorTileData::VectorTileData(Blob blob_) : blob(std::move(blob_)) { } std::unique_ptr<GeometryTileData> VectorTileData::clone() const { - return std::make_unique<VectorTileData>(data); + // Always pass on data that is uncompressed, if we have it. + return std::make_unique<VectorTileData>(data ? Blob{ data, false } : blob); } std::unique_ptr<GeometryTileLayer> VectorTileData::getLayer(const std::string& name) const { - if (!parsed) { + if (!blob) { + return nullptr; + } + + if (!data) { // We're parsing this lazily so that we can construct VectorTileData objects on the main - // thread without incurring the overhead of parsing immediately. + // thread without incurring the overhead of parsing and decompressing immediately. + data = blob.uncompressedData(); layers = mapbox::vector_tile::buffer(*data).getLayers(); - parsed = true; } auto it = layers.find(name); diff --git a/src/mbgl/tile/vector_tile_data.hpp b/src/mbgl/tile/vector_tile_data.hpp index 48beaf9d07..ce120e97c0 100644 --- a/src/mbgl/tile/vector_tile_data.hpp +++ b/src/mbgl/tile/vector_tile_data.hpp @@ -1,5 +1,7 @@ #include <mbgl/tile/geometry_tile_data.hpp> +#include <mbgl/util/blob.hpp> + #include <mapbox/vector_tile.hpp> #include <protozero/pbf_reader.hpp> @@ -38,7 +40,8 @@ private: class VectorTileData : public GeometryTileData { public: - VectorTileData(std::shared_ptr<const std::string> data); + VectorTileData(const VectorTileData&); + VectorTileData(Blob blob); std::unique_ptr<GeometryTileData> clone() const override; std::unique_ptr<GeometryTileLayer> getLayer(const std::string& name) const override; @@ -46,8 +49,8 @@ public: std::vector<std::string> layerNames() const; private: - std::shared_ptr<const std::string> data; - mutable bool parsed = false; + Blob blob; + mutable std::shared_ptr<const std::string> data; mutable std::map<std::string, const protozero::data_view> layers; }; diff --git a/src/mbgl/util/blob.cpp b/src/mbgl/util/blob.cpp new file mode 100644 index 0000000000..53d7644002 --- /dev/null +++ b/src/mbgl/util/blob.cpp @@ -0,0 +1,46 @@ +#include <mbgl/util/blob.hpp> +#include <mbgl/util/compression.hpp> + +namespace mbgl { + +Blob::Blob() = default; + +Blob::Blob(std::shared_ptr<const std::string> bytes_, bool compressed_) + : bytes(std::move(bytes_)), compressed(compressed_) { +} + +Blob::Blob(std::string&& bytes_, bool compressed_) + : bytes(std::make_shared<const std::string>(std::move(bytes_))), compressed(compressed_) { +} + +std::shared_ptr<const std::string> Blob::uncompressedData() const { + if (!bytes) { + throw std::runtime_error("invalid blob"); + } + if (compressed) { + return std::make_shared<const std::string>(util::decompress(*bytes)); + } else { + return bytes; + } +} + +std::shared_ptr<const std::string> Blob::compressedData() const { + if (!bytes) { + throw std::runtime_error("invalid blob"); + } + if (compressed) { + return bytes; + } else { + return std::make_shared<const std::string>(util::compress(*bytes)); + } +} + +void Blob::uncompress() { + if (compressed) { + bytes = uncompressedData(); + compressed = false; + } +} + + +} // namespace mbgl diff --git a/src/mbgl/util/compression.cpp b/src/mbgl/util/compression.cpp index 30e813cbb8..4d4e094f06 100644 --- a/src/mbgl/util/compression.cpp +++ b/src/mbgl/util/compression.cpp @@ -71,7 +71,8 @@ std::string decompress(const std::string &raw) { memset(&inflate_stream, 0, sizeof(inflate_stream)); // TODO: reuse z_streams - if (inflateInit(&inflate_stream) != Z_OK) { + // MAX_WBITS + allows decoding gzip in addition to zlib + if (inflateInit2(&inflate_stream, MAX_WBITS + 32) != Z_OK) { throw std::runtime_error("failed to initialize inflate"); } @@ -100,5 +101,39 @@ std::string decompress(const std::string &raw) { return result; } + +bool isCompressible(const std::string& raw) { + // WebP + if (raw.size() >= 12 && static_cast<uint8_t>(raw[0]) == 'R' && + static_cast<uint8_t>(raw[1]) == 'I' && static_cast<uint8_t>(raw[2]) == 'F' && + static_cast<uint8_t>(raw[3]) == 'F' && static_cast<uint8_t>(raw[8]) == 'W' && + static_cast<uint8_t>(raw[9]) == 'E' && static_cast<uint8_t>(raw[10]) == 'B' && + static_cast<uint8_t>(raw[11]) == 'P') { + // Note: the WebP container format allows uncompressed data as well, but we just assume that + // all WebP files are already compressed. + return false; + } + + // PNG + if (raw.size() >= 8 && static_cast<uint8_t>(raw[0]) == 0x89 && + static_cast<uint8_t>(raw[1]) == 'P' && static_cast<uint8_t>(raw[2]) == 'N' && + static_cast<uint8_t>(raw[3]) == 'G' && static_cast<uint8_t>(raw[4]) == '\r' && + static_cast<uint8_t>(raw[5]) == '\n' && static_cast<uint8_t>(raw[6]) == 0x1a && + static_cast<uint8_t>(raw[7]) == '\n') { + // Note: this assumes the PNG file itself is compressed. However, it is possible to create + // PNG files with uncompressed data in it (zlib compression 0), but they are exceedingly + // rare, so we don't care about them. + return false; + } + + // JPEG + if (raw.size() >= 3 && static_cast<uint8_t>(raw[0]) == 0xff && + static_cast<uint8_t>(raw[1]) == 0xd8 && static_cast<uint8_t>(raw[2]) == 0xff) { + return false; + } + + return true; +} + } // namespace util } // namespace mbgl diff --git a/src/mbgl/util/io.cpp b/src/mbgl/util/io.cpp index 6a6ed7b250..2dc6cabf0f 100644 --- a/src/mbgl/util/io.cpp +++ b/src/mbgl/util/io.cpp @@ -1,43 +1,29 @@ #include <mbgl/util/io.hpp> -#include <cstdio> #include <cerrno> #include <iostream> -#include <sstream> #include <fstream> namespace mbgl { namespace util { -void write_file(const std::string &filename, const std::string &data) { - FILE *fd = fopen(filename.c_str(), "wb"); - if (fd) { - fwrite(data.data(), sizeof(std::string::value_type), data.size(), fd); - fclose(fd); - } else { - throw std::runtime_error(std::string("Failed to open file ") + filename); - } -} - -std::string read_file(const std::string &filename) { - std::ifstream file(filename); +void writeFile(const std::string &filename, Blob blob) { + std::ofstream file(filename, std::ios::binary); if (file.good()) { - std::stringstream data; - data << file.rdbuf(); - return data.str(); + file << *blob.uncompressedData(); } else { - throw std::runtime_error(std::string("Cannot read file ") + filename); + throw IOException(errno, "failed to write file"); } } -optional<std::string> readFile(const std::string &filename) { - std::ifstream file(filename); +Blob readFile(const std::string &filename) { + std::ifstream file(filename, std::ios::binary); if (file.good()) { - std::stringstream data; - data << file.rdbuf(); - return data.str(); + return { { std::istreambuf_iterator<char>(file), std::istreambuf_iterator<char>() }, + false }; + } else { + return {}; } - return {}; } void deleteFile(const std::string& filename) { diff --git a/src/mbgl/util/io.hpp b/src/mbgl/util/io.hpp index 847271acf0..b74f89485b 100644 --- a/src/mbgl/util/io.hpp +++ b/src/mbgl/util/io.hpp @@ -1,6 +1,7 @@ #pragma once #include <mbgl/util/optional.hpp> +#include <mbgl/util/blob.hpp> #include <string> #include <stdexcept> @@ -14,10 +15,8 @@ struct IOException : std::runtime_error { const int code = 0; }; -void write_file(const std::string &filename, const std::string &data); -std::string read_file(const std::string &filename); - -optional<std::string> readFile(const std::string &filename); +void writeFile(const std::string &filename, Blob blob); +Blob readFile(const std::string &filename); void deleteFile(const std::string& filename); } // namespace util |