diff options
author | Dane Springmeyer <dane@mapbox.com> | 2016-05-03 13:57:17 -0700 |
---|---|---|
committer | Dane Springmeyer <dane@mapbox.com> | 2016-05-03 13:57:17 -0700 |
commit | c82b02c508bb4eb1d6ee25320cf83995a807e603 (patch) | |
tree | 6e480d2e91e8fb2a053a99a9d0a6a72a746263b1 | |
parent | 7d1a4c5ec146493bb3e192f7fc20e3ae3b626011 (diff) | |
download | qtlocation-mapboxgl-c82b02c508bb4eb1d6ee25320cf83995a807e603.tar.gz |
[core] port to protozero (#4906)
-rwxr-xr-x | configure | 1 | ||||
-rw-r--r-- | mbgl.gypi | 1 | ||||
-rw-r--r-- | platform/android/scripts/configure.sh | 1 | ||||
-rw-r--r-- | platform/ios/scripts/configure.sh | 1 | ||||
-rw-r--r-- | platform/linux/scripts/configure.sh | 1 | ||||
-rw-r--r-- | platform/osx/scripts/configure.sh | 1 | ||||
-rw-r--r-- | platform/qt/scripts/configure.sh | 1 | ||||
-rw-r--r-- | src/mbgl/text/glyph_pbf.cpp | 70 | ||||
-rw-r--r-- | src/mbgl/tile/vector_tile.cpp | 118 | ||||
-rw-r--r-- | src/mbgl/tile/vector_tile.hpp | 15 | ||||
-rw-r--r-- | src/mbgl/util/pbf.hpp | 184 | ||||
-rw-r--r-- | test/style/glyph_store.cpp | 2 | ||||
-rw-r--r-- | test/style/source.cpp | 2 | ||||
-rw-r--r-- | test/test.gypi | 1 |
14 files changed, 119 insertions, 280 deletions
@@ -97,6 +97,7 @@ CONFIG+=" 'variables': {"$LN print_opengl_flags print_qt_flags +print_flags protozero cflags print_flags boost cflags print_flags boost_libprogram_options static_libs print_flags openssl static_libs cflags ldflags @@ -172,6 +172,7 @@ 'variables': { 'cflags_cc': [ '<@(opengl_cflags)', + '<@(protozero_cflags)', '<@(boost_cflags)', '<@(geojsonvt_cflags)', '<@(rapidjson_cflags)', diff --git a/platform/android/scripts/configure.sh b/platform/android/scripts/configure.sh index c53805a9c6..cf20f4a99e 100644 --- a/platform/android/scripts/configure.sh +++ b/platform/android/scripts/configure.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash +PROTOZERO_VERSION=1.3.0 BOOST_VERSION=1.60.0 LIBPNG_VERSION=1.6.20 LIBJPEG_TURBO_VERSION=1.4.2 diff --git a/platform/ios/scripts/configure.sh b/platform/ios/scripts/configure.sh index 31ef374e8e..769407445a 100644 --- a/platform/ios/scripts/configure.sh +++ b/platform/ios/scripts/configure.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash +PROTOZERO_VERSION=1.3.0 BOOST_VERSION=1.60.0 SQLITE_VERSION=system ZLIB_VERSION=system diff --git a/platform/linux/scripts/configure.sh b/platform/linux/scripts/configure.sh index d58618d4bd..4d1b26fc9b 100644 --- a/platform/linux/scripts/configure.sh +++ b/platform/linux/scripts/configure.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash +PROTOZERO_VERSION=1.3.0 BOOST_VERSION=1.60.0 BOOST_LIBPROGRAM_OPTIONS_VERSION=1.60.0 LIBCURL_VERSION=system diff --git a/platform/osx/scripts/configure.sh b/platform/osx/scripts/configure.sh index 0b4c8eaf3e..6d2eef03c4 100644 --- a/platform/osx/scripts/configure.sh +++ b/platform/osx/scripts/configure.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash +PROTOZERO_VERSION=1.3.0 BOOST_VERSION=1.60.0 BOOST_LIBPROGRAM_OPTIONS_VERSION=1.60.0 GLFW_VERSION=3.1.2 diff --git a/platform/qt/scripts/configure.sh b/platform/qt/scripts/configure.sh index aa5c7615dc..d9a8f15cc2 100644 --- a/platform/qt/scripts/configure.sh +++ b/platform/qt/scripts/configure.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash +PROTOZERO_VERSION=1.3.0 BOOST_VERSION=1.60.0 GEOJSONVT_VERSION=4.1.2 GTEST_VERSION=1.7.0 diff --git a/src/mbgl/text/glyph_pbf.cpp b/src/mbgl/text/glyph_pbf.cpp index 2588afaefa..eab9725075 100644 --- a/src/mbgl/text/glyph_pbf.cpp +++ b/src/mbgl/text/glyph_pbf.cpp @@ -6,52 +6,54 @@ #include <mbgl/text/glyph_store.hpp> #include <mbgl/text/glyph_set.hpp> #include <mbgl/util/exception.hpp> -#include <mbgl/util/pbf.hpp> #include <mbgl/util/string.hpp> #include <mbgl/util/token.hpp> #include <mbgl/util/url.hpp> +#include <protozero/pbf_reader.hpp> + namespace { void parseGlyphPBF(mbgl::GlyphSet& glyphSet, const std::string& data) { - mbgl::pbf glyphs_pbf(reinterpret_cast<const uint8_t *>(data.data()), data.size()); - - while (glyphs_pbf.next()) { - if (glyphs_pbf.tag == 1) { // stacks - mbgl::pbf fontstack_pbf = glyphs_pbf.message(); - while (fontstack_pbf.next()) { - if (fontstack_pbf.tag == 3) { // glyphs - mbgl::pbf glyph_pbf = fontstack_pbf.message(); + protozero::pbf_reader glyphs_pbf(data); - mbgl::SDFGlyph glyph; + while (glyphs_pbf.next(1)) { + auto fontstack_pbf = glyphs_pbf.get_message(); + while (fontstack_pbf.next(3)) { + auto glyph_pbf = fontstack_pbf.get_message(); - while (glyph_pbf.next()) { - if (glyph_pbf.tag == 1) { // id - glyph.id = glyph_pbf.varint(); - } else if (glyph_pbf.tag == 2) { // bitmap - glyph.bitmap = glyph_pbf.string(); - } else if (glyph_pbf.tag == 3) { // width - glyph.metrics.width = glyph_pbf.varint(); - } else if (glyph_pbf.tag == 4) { // height - glyph.metrics.height = glyph_pbf.varint(); - } else if (glyph_pbf.tag == 5) { // left - glyph.metrics.left = glyph_pbf.svarint(); - } else if (glyph_pbf.tag == 6) { // top - glyph.metrics.top = glyph_pbf.svarint(); - } else if (glyph_pbf.tag == 7) { // advance - glyph.metrics.advance = glyph_pbf.varint(); - } else { - glyph_pbf.skip(); - } - } + mbgl::SDFGlyph glyph; - glyphSet.insert(glyph.id, glyph); - } else { - fontstack_pbf.skip(); + while (glyph_pbf.next()) { + switch (glyph_pbf.tag()) { + case 1: // id + glyph.id = glyph_pbf.get_uint32(); + break; + case 2: // bitmap + glyph.bitmap = glyph_pbf.get_string(); + break; + case 3: // width + glyph.metrics.width = glyph_pbf.get_uint32(); + break; + case 4: // height + glyph.metrics.height = glyph_pbf.get_uint32(); + break; + case 5: // left + glyph.metrics.left = glyph_pbf.get_sint32(); + break; + case 6: // top + glyph.metrics.top = glyph_pbf.get_sint32(); + break; + case 7: // advance + glyph.metrics.advance = glyph_pbf.get_uint32(); + break; + default: + glyph_pbf.skip(); + break; } } - } else { - glyphs_pbf.skip(); + + glyphSet.insert(glyph.id, glyph); } } } diff --git a/src/mbgl/tile/vector_tile.cpp b/src/mbgl/tile/vector_tile.cpp index f3a02c5d76..af30fa9986 100644 --- a/src/mbgl/tile/vector_tile.cpp +++ b/src/mbgl/tile/vector_tile.cpp @@ -9,25 +9,24 @@ namespace mbgl { -Value parseValue(pbf data) { +Value parseValue(protozero::pbf_reader data) { while (data.next()) { - switch (data.tag) - { + switch (data.tag()) { case 1: // string_value - return data.string(); + return data.get_string(); case 2: // float_value - return static_cast<double>(data.float32()); + return static_cast<double>(data.get_float()); case 3: // double_value - return data.float64(); + return data.get_double(); case 4: // int_value - return data.varint<int64_t>(); + return data.get_int64(); case 5: // uint_value - return data.varint<uint64_t>(); + return data.get_uint64(); case 6: // sint_value - return data.svarint<int64_t>(); + return data.get_sint64(); case 7: // bool_value - return data.boolean(); + return data.get_bool(); default: data.skip(); break; @@ -36,19 +35,25 @@ Value parseValue(pbf data) { return false; } -VectorTileFeature::VectorTileFeature(pbf feature_pbf, const VectorTileLayer& layer_) +VectorTileFeature::VectorTileFeature(protozero::pbf_reader feature_pbf, const VectorTileLayer& layer_) : layer(layer_) { while (feature_pbf.next()) { - if (feature_pbf.tag == 1) { // id - id = feature_pbf.varint<uint64_t>(); - } else if (feature_pbf.tag == 2) { // tags - tags_pbf = feature_pbf.message(); - } else if (feature_pbf.tag == 3) { // type - type = (FeatureType)feature_pbf.varint(); - } else if (feature_pbf.tag == 4) { // geometry - geometry_pbf = feature_pbf.message(); - } else { + switch (feature_pbf.tag()) { + case 1: // id + id = feature_pbf.get_uint64(); + break; + case 2: // tags + tags_iter = feature_pbf.get_packed_uint32(); + break; + case 3: // type + type = static_cast<FeatureType>(feature_pbf.get_enum()); + break; + case 4: // geometry + geometry_iter = feature_pbf.get_packed_uint32(); + break; + default: feature_pbf.skip(); + break; } } } @@ -59,19 +64,20 @@ optional<Value> VectorTileFeature::getValue(const std::string& key) const { return optional<Value>(); } - pbf tags = tags_pbf; - while (tags) { - uint32_t tag_key = tags.varint(); + auto start_itr = tags_iter.first; + const auto & end_itr = tags_iter.second; + while (start_itr != end_itr) { + uint32_t tag_key = static_cast<uint32_t>(*start_itr++); if (layer.keysMap.size() <= tag_key) { throw std::runtime_error("feature referenced out of range key"); } - if (!tags) { + if (start_itr == end_itr) { throw std::runtime_error("uneven number of feature tag ids"); } - uint32_t tag_val = tags.varint(); + uint32_t tag_val = static_cast<uint32_t>(*start_itr++);; if (layer.values.size() <= tag_val) { throw std::runtime_error("feature referenced out of range value"); } @@ -86,10 +92,11 @@ optional<Value> VectorTileFeature::getValue(const std::string& key) const { std::unordered_map<std::string,Value> VectorTileFeature::getProperties() const { std::unordered_map<std::string,Value> properties; - pbf tags = tags_pbf; - while (tags) { - uint32_t tag_key = tags.varint(); - uint32_t tag_val = tags.varint(); + auto start_itr = tags_iter.first; + const auto & end_itr = tags_iter.second; + while (start_itr != end_itr) { + uint32_t tag_key = static_cast<uint32_t>(*start_itr++); + uint32_t tag_val = static_cast<uint32_t>(*start_itr++); properties[layer.keys.at(tag_key)] = layer.values.at(tag_val); } return properties; @@ -100,7 +107,6 @@ uint64_t VectorTileFeature::getID() const { } GeometryCollection VectorTileFeature::getGeometries() const { - pbf data(geometry_pbf); uint8_t cmd = 1; uint32_t length = 0; int32_t x = 0; @@ -111,9 +117,10 @@ GeometryCollection VectorTileFeature::getGeometries() const { lines.emplace_back(); GeometryCoordinates* line = &lines.back(); - while (data.data < data.end) { + auto g_itr = geometry_iter; + while (g_itr.first != g_itr.second) { if (length == 0) { - uint32_t cmd_length = data.varint(); + uint32_t cmd_length = static_cast<uint32_t>(*g_itr.first++); cmd = cmd_length & 0x7; length = cmd_length >> 3; } @@ -121,8 +128,8 @@ GeometryCollection VectorTileFeature::getGeometries() const { --length; if (cmd == 1 || cmd == 2) { - x += data.svarint(); - y += data.svarint(); + x += protozero::decode_zigzag32(static_cast<uint32_t>(*g_itr.first++)); + y += protozero::decode_zigzag32(static_cast<uint32_t>(*g_itr.first++)); if (cmd == 1 && !line->empty()) { // moveTo lines.emplace_back(); @@ -155,14 +162,10 @@ VectorTile::VectorTile(std::shared_ptr<const std::string> data_) util::ptr<GeometryTileLayer> VectorTile::getLayer(const std::string& name) const { if (!parsed) { parsed = true; - pbf tile_pbf(reinterpret_cast<const unsigned char *>(data->c_str()), data->size()); - while (tile_pbf.next()) { - if (tile_pbf.tag == 3) { // layer - util::ptr<VectorTileLayer> layer = std::make_shared<VectorTileLayer>(tile_pbf.message()); - layers.emplace(layer->name, layer); - } else { - tile_pbf.skip(); - } + protozero::pbf_reader tile_pbf(*data); + while (tile_pbf.next(3)) { + util::ptr<VectorTileLayer> layer = std::make_shared<VectorTileLayer>(tile_pbf.get_message()); + layers.emplace(layer->name, layer); } } @@ -174,20 +177,27 @@ util::ptr<GeometryTileLayer> VectorTile::getLayer(const std::string& name) const return nullptr; } -VectorTileLayer::VectorTileLayer(pbf layer_pbf) { +VectorTileLayer::VectorTileLayer(protozero::pbf_reader layer_pbf) { while (layer_pbf.next()) { - if (layer_pbf.tag == 1) { // name - name = layer_pbf.string(); - } else if (layer_pbf.tag == 2) { // feature - features.push_back(layer_pbf.message()); - } else if (layer_pbf.tag == 3) { // keys - keysMap.emplace(layer_pbf.string(), keysMap.size()); - } else if (layer_pbf.tag == 4) { // values - values.emplace_back(parseValue(layer_pbf.message())); - } else if (layer_pbf.tag == 5) { // extent - extent = layer_pbf.varint(); - } else { + switch (layer_pbf.tag()) { + case 1: // name + name = layer_pbf.get_string(); + break; + case 2: // feature + features.push_back(layer_pbf.get_message()); + break; + case 3: // keys + keysMap.emplace(layer_pbf.get_string(), keysMap.size()); + break; + case 4: // values + values.emplace_back(parseValue(layer_pbf.get_message())); + break; + case 5: // extent + extent = layer_pbf.get_uint32(); + break; + default: layer_pbf.skip(); + break; } } diff --git a/src/mbgl/tile/vector_tile.hpp b/src/mbgl/tile/vector_tile.hpp index 0e583ab33a..441d6827ac 100644 --- a/src/mbgl/tile/vector_tile.hpp +++ b/src/mbgl/tile/vector_tile.hpp @@ -3,7 +3,7 @@ #include <mbgl/tile/geometry_tile.hpp> #include <mbgl/map/tile_id.hpp> -#include <mbgl/util/pbf.hpp> +#include <protozero/pbf_reader.hpp> #include <map> #include <unordered_map> @@ -13,9 +13,12 @@ namespace mbgl { class VectorTileLayer; +using pbf_iter_type = protozero::pbf_reader::const_uint32_iterator; +using packed_iter_type = std::pair<pbf_iter_type,pbf_iter_type>; + class VectorTileFeature : public GeometryTileFeature { public: - VectorTileFeature(pbf, const VectorTileLayer&); + VectorTileFeature(protozero::pbf_reader, const VectorTileLayer&); FeatureType getType() const override { return type; } optional<Value> getValue(const std::string&) const override; @@ -28,13 +31,13 @@ private: const VectorTileLayer& layer; uint64_t id = 0; FeatureType type = FeatureType::Unknown; - pbf tags_pbf; - pbf geometry_pbf; + packed_iter_type tags_iter; + packed_iter_type geometry_iter; }; class VectorTileLayer : public GeometryTileLayer { public: - VectorTileLayer(pbf); + VectorTileLayer(protozero::pbf_reader); std::size_t featureCount() const override { return features.size(); } util::ptr<const GeometryTileFeature> getFeature(std::size_t) const override; @@ -49,7 +52,7 @@ private: std::map<std::string, uint32_t> keysMap; std::vector<std::reference_wrapper<const std::string>> keys; std::vector<Value> values; - std::vector<pbf> features; + std::vector<protozero::pbf_reader> features; }; class VectorTile : public GeometryTile { diff --git a/src/mbgl/util/pbf.hpp b/src/mbgl/util/pbf.hpp deleted file mode 100644 index d017219a52..0000000000 --- a/src/mbgl/util/pbf.hpp +++ /dev/null @@ -1,184 +0,0 @@ -#ifndef MBGL_UTIL_PBF -#define MBGL_UTIL_PBF - -/* - * Some parts are from upb - a minimalist implementation of protocol buffers. - * - * Copyright (c) 2008-2011 Google Inc. See LICENSE for details. - * Author: Josh Haberman <jhaberman@gmail.com> - */ - -#include <string> -#include <cstring> - -namespace mbgl { - -struct pbf { - struct exception : std::exception { const char *what() const noexcept { return "pbf exception"; } }; - struct unterminated_varint_exception : exception { const char *what() const noexcept { return "pbf unterminated varint exception"; } }; - struct varint_too_long_exception : exception { const char *what() const noexcept { return "pbf varint too long exception"; } }; - struct unknown_field_type_exception : exception { const char *what() const noexcept { return "pbf unknown field type exception"; } }; - struct end_of_buffer_exception : exception { const char *what() const noexcept { return "pbf end of buffer exception"; } }; - - inline pbf(const unsigned char *data, size_t length); - inline pbf(); - - inline operator bool() const; - - inline bool next(); - inline bool next(uint32_t tag); - template <typename T = uint32_t> inline T varint(); - template <typename T = uint32_t> inline T svarint(); - - template <typename T = uint32_t, int bytes = 4> inline T fixed(); - inline float float32(); - inline double float64(); - - inline std::string string(); - inline bool boolean(); - - inline pbf message(); - - inline void skip(); - inline void skipValue(uint32_t val); - inline void skipBytes(uint32_t bytes); - - const uint8_t *data = nullptr; - const uint8_t *end = nullptr; - uint32_t value = 0; - uint32_t tag = 0; -}; - -pbf::pbf(const unsigned char *data_, size_t length) - : data(data_), - end(data_ + length), - value(0), - tag(0) { -} - -pbf::pbf() - : data(nullptr), - end(nullptr), - value(0), - tag(0) { -} - - -pbf::operator bool() const { - return data < end; -} - -bool pbf::next() { - if (data < end) { - value = static_cast<uint32_t>(varint()); - tag = value >> 3; - return true; - } - return false; -} - -bool pbf::next(uint32_t requested_tag) { - while (next()) { - if (tag == requested_tag) { - return true; - } else { - skip(); - } - } - return false; -} - -template <typename T> -T pbf::varint() { - uint8_t byte = 0x80; - T result = 0; - int bitpos; - for (bitpos = 0; bitpos < 70 && (byte & 0x80); bitpos += 7) { - if (data >= end) { - throw unterminated_varint_exception(); - } - result |= ((T)(byte = *data) & 0x7F) << bitpos; - - data++; - } - if (bitpos == 70 && (byte & 0x80)) { - throw varint_too_long_exception(); - } - - return result; -} - -template <typename T> -T pbf::svarint() { - T n = varint<T>(); - return (n >> 1) ^ -(T)(n & 1); -} - -template <typename T, int bytes> -T pbf::fixed() { - skipBytes(bytes); - T result; - memcpy(&result, data - bytes, bytes); - return result; -} - -float pbf::float32() { - return fixed<float, 4>(); -} - -double pbf::float64() { - return fixed<double, 8>(); -} - -std::string pbf::string() { - uint32_t bytes = static_cast<uint32_t>(varint()); - const char *string_data = reinterpret_cast<const char*>(data); - skipBytes(bytes); - return std::string(string_data, bytes); -} - -bool pbf::boolean() { - skipBytes(1); - return *(bool *)(data - 1); -} - -pbf pbf::message() { - uint32_t bytes = static_cast<uint32_t>(varint()); - const uint8_t *pos = data; - skipBytes(bytes); - return pbf(pos, bytes); -} - -void pbf::skip() { - skipValue(value); -} - -void pbf::skipValue(uint32_t val) { - switch (val & 0x7) { - case 0: // varint - varint(); - break; - case 1: // 64 bit - skipBytes(8); - break; - case 2: // string/message - skipBytes(static_cast<uint32_t>(varint())); - break; - case 5: // 32 bit - skipBytes(4); - break; - default: - throw unknown_field_type_exception(); - } -} - -void pbf::skipBytes(uint32_t bytes) { - if (data + bytes > end) { - throw end_of_buffer_exception(); - } - data += bytes; -} - -} // end namespace mbgl - -#endif diff --git a/test/style/glyph_store.cpp b/test/style/glyph_store.cpp index 2a874a05a7..08ec6902b9 100644 --- a/test/style/glyph_store.cpp +++ b/test/style/glyph_store.cpp @@ -110,7 +110,7 @@ TEST(GlyphStore, LoadingCorrupted) { EXPECT_EQ(glyphRange, GlyphRange(0, 255)); EXPECT_TRUE(error != nullptr); - EXPECT_EQ(util::toString(error), "pbf unknown field type exception"); + EXPECT_EQ(util::toString(error), "unknown pbf field type exception"); auto glyphSet = test.glyphStore.getGlyphSet({{"Test Stack"}}); ASSERT_TRUE(glyphSet->getSDFs().empty()); diff --git a/test/style/source.cpp b/test/style/source.cpp index 5ec2714b43..7964a05814 100644 --- a/test/style/source.cpp +++ b/test/style/source.cpp @@ -269,7 +269,7 @@ TEST(Source, VectorTileCorrupt) { test.observer.tileError = [&] (Source& source, const TileID& tileID, std::exception_ptr error) { EXPECT_EQ(source.type, SourceType::Vector); EXPECT_EQ(std::string(tileID), "0/0/0"); - EXPECT_EQ(util::toString(error), "pbf unknown field type exception"); + EXPECT_EQ(util::toString(error), "unknown pbf field type exception"); test.end(); }; diff --git a/test/test.gypi b/test/test.gypi index a9e53c88fc..2326fd7cf9 100644 --- a/test/test.gypi +++ b/test/test.gypi @@ -85,6 +85,7 @@ 'cflags_cc': [ '<@(gtest_cflags)', '<@(opengl_cflags)', + '<@(protozero_cflags)', '<@(boost_cflags)', '<@(sqlite_cflags)', '<@(geojsonvt_cflags)', |