[core] port to protozero (#4906)

14 files changed, 119 insertions, 280 deletions

diff --git a/configure b/configure
index 0747197ff1..3290899498 100755
--- a/configure
+++ b/configure
@@ -97,6 +97,7 @@ CONFIG+="  'variables': {"$LN
 print_opengl_flags
 print_qt_flags
 
+print_flags protozero cflags
 print_flags boost cflags
 print_flags boost_libprogram_options static_libs
 print_flags openssl static_libs cflags ldflags
diff --git a/mbgl.gypi b/mbgl.gypi
index 965f2dce93..6619e0011f 100644
--- a/mbgl.gypi
+++ b/mbgl.gypi
@@ -172,6 +172,7 @@
       'variables': {
         'cflags_cc': [
           '<@(opengl_cflags)',
+          '<@(protozero_cflags)',
           '<@(boost_cflags)',
           '<@(geojsonvt_cflags)',
           '<@(rapidjson_cflags)',
diff --git a/platform/android/scripts/configure.sh b/platform/android/scripts/configure.sh
index c53805a9c6..cf20f4a99e 100644
--- a/platform/android/scripts/configure.sh
+++ b/platform/android/scripts/configure.sh
@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+PROTOZERO_VERSION=1.3.0
 BOOST_VERSION=1.60.0
 LIBPNG_VERSION=1.6.20
 LIBJPEG_TURBO_VERSION=1.4.2
diff --git a/platform/ios/scripts/configure.sh b/platform/ios/scripts/configure.sh
index 31ef374e8e..769407445a 100644
--- a/platform/ios/scripts/configure.sh
+++ b/platform/ios/scripts/configure.sh
@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+PROTOZERO_VERSION=1.3.0
 BOOST_VERSION=1.60.0
 SQLITE_VERSION=system
 ZLIB_VERSION=system
diff --git a/platform/linux/scripts/configure.sh b/platform/linux/scripts/configure.sh
index d58618d4bd..4d1b26fc9b 100644
--- a/platform/linux/scripts/configure.sh
+++ b/platform/linux/scripts/configure.sh
@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+PROTOZERO_VERSION=1.3.0
 BOOST_VERSION=1.60.0
 BOOST_LIBPROGRAM_OPTIONS_VERSION=1.60.0
 LIBCURL_VERSION=system
diff --git a/platform/osx/scripts/configure.sh b/platform/osx/scripts/configure.sh
index 0b4c8eaf3e..6d2eef03c4 100644
--- a/platform/osx/scripts/configure.sh
+++ b/platform/osx/scripts/configure.sh
@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+PROTOZERO_VERSION=1.3.0
 BOOST_VERSION=1.60.0
 BOOST_LIBPROGRAM_OPTIONS_VERSION=1.60.0
 GLFW_VERSION=3.1.2
diff --git a/platform/qt/scripts/configure.sh b/platform/qt/scripts/configure.sh
index aa5c7615dc..d9a8f15cc2 100644
--- a/platform/qt/scripts/configure.sh
+++ b/platform/qt/scripts/configure.sh
@@ -1,5 +1,6 @@
 #!/usr/bin/env bash
 
+PROTOZERO_VERSION=1.3.0
 BOOST_VERSION=1.60.0
 GEOJSONVT_VERSION=4.1.2
 GTEST_VERSION=1.7.0
diff --git a/src/mbgl/text/glyph_pbf.cpp b/src/mbgl/text/glyph_pbf.cpp
index 2588afaefa..eab9725075 100644
--- a/src/mbgl/text/glyph_pbf.cpp
+++ b/src/mbgl/text/glyph_pbf.cpp
@@ -6,52 +6,54 @@
 #include <mbgl/text/glyph_store.hpp>
 #include <mbgl/text/glyph_set.hpp>
 #include <mbgl/util/exception.hpp>
-#include <mbgl/util/pbf.hpp>
 #include <mbgl/util/string.hpp>
 #include <mbgl/util/token.hpp>
 #include <mbgl/util/url.hpp>
 
+#include <protozero/pbf_reader.hpp>
+
 namespace {
 
 void parseGlyphPBF(mbgl::GlyphSet& glyphSet, const std::string& data) {
-    mbgl::pbf glyphs_pbf(reinterpret_cast<const uint8_t *>(data.data()), data.size());
-
-    while (glyphs_pbf.next()) {
-        if (glyphs_pbf.tag == 1) { // stacks
-            mbgl::pbf fontstack_pbf = glyphs_pbf.message();
-            while (fontstack_pbf.next()) {
-                if (fontstack_pbf.tag == 3) { // glyphs
-                    mbgl::pbf glyph_pbf = fontstack_pbf.message();
+    protozero::pbf_reader glyphs_pbf(data);
 
-                    mbgl::SDFGlyph glyph;
+    while (glyphs_pbf.next(1)) {
+        auto fontstack_pbf = glyphs_pbf.get_message();
+        while (fontstack_pbf.next(3)) {
+            auto glyph_pbf = fontstack_pbf.get_message();
 
-                    while (glyph_pbf.next()) {
-                        if (glyph_pbf.tag == 1) { // id
-                            glyph.id = glyph_pbf.varint();
-                        } else if (glyph_pbf.tag == 2) { // bitmap
-                            glyph.bitmap = glyph_pbf.string();
-                        } else if (glyph_pbf.tag == 3) { // width
-                            glyph.metrics.width = glyph_pbf.varint();
-                        } else if (glyph_pbf.tag == 4) { // height
-                            glyph.metrics.height = glyph_pbf.varint();
-                        } else if (glyph_pbf.tag == 5) { // left
-                            glyph.metrics.left = glyph_pbf.svarint();
-                        } else if (glyph_pbf.tag == 6) { // top
-                            glyph.metrics.top = glyph_pbf.svarint();
-                        } else if (glyph_pbf.tag == 7) { // advance
-                            glyph.metrics.advance = glyph_pbf.varint();
-                        } else {
-                            glyph_pbf.skip();
-                        }
-                    }
+            mbgl::SDFGlyph glyph;
 
-                    glyphSet.insert(glyph.id, glyph);
-                } else {
-                    fontstack_pbf.skip();
+            while (glyph_pbf.next()) {
+                switch (glyph_pbf.tag()) {
+                case 1: // id
+                    glyph.id = glyph_pbf.get_uint32();
+                    break;
+                case 2: // bitmap
+                    glyph.bitmap = glyph_pbf.get_string();
+                    break;
+                case 3: // width
+                    glyph.metrics.width = glyph_pbf.get_uint32();
+                    break;
+                case 4: // height
+                    glyph.metrics.height = glyph_pbf.get_uint32();
+                    break;
+                case 5: // left
+                    glyph.metrics.left = glyph_pbf.get_sint32();
+                    break;
+                case 6: // top
+                    glyph.metrics.top = glyph_pbf.get_sint32();
+                    break;
+                case 7: // advance
+                    glyph.metrics.advance = glyph_pbf.get_uint32();
+                    break;
+                default:
+                    glyph_pbf.skip();
+                    break;
                 }
             }
-        } else {
-            glyphs_pbf.skip();
+
+            glyphSet.insert(glyph.id, glyph);
         }
     }
 }
diff --git a/src/mbgl/tile/vector_tile.cpp b/src/mbgl/tile/vector_tile.cpp
index f3a02c5d76..af30fa9986 100644
--- a/src/mbgl/tile/vector_tile.cpp
+++ b/src/mbgl/tile/vector_tile.cpp
@@ -9,25 +9,24 @@
 
 namespace mbgl {
 
-Value parseValue(pbf data) {
+Value parseValue(protozero::pbf_reader data) {
     while (data.next())
     {
-        switch (data.tag)
-        {
+        switch (data.tag()) {
         case 1: // string_value
-            return data.string();
+            return data.get_string();
         case 2: // float_value
-            return static_cast<double>(data.float32());
+            return static_cast<double>(data.get_float());
         case 3: // double_value
-            return data.float64();
+            return data.get_double();
         case 4: // int_value
-            return data.varint<int64_t>();
+            return data.get_int64();
         case 5: // uint_value
-            return data.varint<uint64_t>();
+            return data.get_uint64();
         case 6: // sint_value
-            return data.svarint<int64_t>();
+            return data.get_sint64();
         case 7: // bool_value
-            return data.boolean();
+            return data.get_bool();
         default:
             data.skip();
             break;
@@ -36,19 +35,25 @@ Value parseValue(pbf data) {
     return false;
 }
 
-VectorTileFeature::VectorTileFeature(pbf feature_pbf, const VectorTileLayer& layer_)
+VectorTileFeature::VectorTileFeature(protozero::pbf_reader feature_pbf, const VectorTileLayer& layer_)
     : layer(layer_) {
     while (feature_pbf.next()) {
-        if (feature_pbf.tag == 1) { // id
-            id = feature_pbf.varint<uint64_t>();
-        } else if (feature_pbf.tag == 2) { // tags
-            tags_pbf = feature_pbf.message();
-        } else if (feature_pbf.tag == 3) { // type
-            type = (FeatureType)feature_pbf.varint();
-        } else if (feature_pbf.tag == 4) { // geometry
-            geometry_pbf = feature_pbf.message();
-        } else {
+        switch (feature_pbf.tag()) {
+        case 1: // id
+            id = feature_pbf.get_uint64();
+            break;
+        case 2: // tags
+            tags_iter = feature_pbf.get_packed_uint32();
+            break;
+        case 3: // type
+            type = static_cast<FeatureType>(feature_pbf.get_enum());
+            break;
+        case 4: // geometry
+            geometry_iter = feature_pbf.get_packed_uint32();
+            break;
+        default:
             feature_pbf.skip();
+            break;
         }
     }
 }
@@ -59,19 +64,20 @@ optional<Value> VectorTileFeature::getValue(const std::string& key) const {
         return optional<Value>();
     }
 
-    pbf tags = tags_pbf;
-    while (tags) {
-        uint32_t tag_key = tags.varint();
+    auto start_itr = tags_iter.first;
+    const auto & end_itr = tags_iter.second;
+    while (start_itr != end_itr) {
+        uint32_t tag_key = static_cast<uint32_t>(*start_itr++);
 
         if (layer.keysMap.size() <= tag_key) {
             throw std::runtime_error("feature referenced out of range key");
         }
 
-        if (!tags) {
+        if (start_itr == end_itr) {
             throw std::runtime_error("uneven number of feature tag ids");
         }
 
-        uint32_t tag_val = tags.varint();
+        uint32_t tag_val = static_cast<uint32_t>(*start_itr++);;
         if (layer.values.size() <= tag_val) {
             throw std::runtime_error("feature referenced out of range value");
         }
@@ -86,10 +92,11 @@ optional<Value> VectorTileFeature::getValue(const std::string& key) const {
 
 std::unordered_map<std::string,Value> VectorTileFeature::getProperties() const {
     std::unordered_map<std::string,Value> properties;
-    pbf tags = tags_pbf;
-    while (tags) {
-        uint32_t tag_key = tags.varint();
-        uint32_t tag_val = tags.varint();
+    auto start_itr = tags_iter.first;
+    const auto & end_itr = tags_iter.second;
+    while (start_itr != end_itr) {
+        uint32_t tag_key = static_cast<uint32_t>(*start_itr++);
+        uint32_t tag_val = static_cast<uint32_t>(*start_itr++);
         properties[layer.keys.at(tag_key)] = layer.values.at(tag_val);
     }
     return properties;
@@ -100,7 +107,6 @@ uint64_t VectorTileFeature::getID() const {
 }
 
 GeometryCollection VectorTileFeature::getGeometries() const {
-    pbf data(geometry_pbf);
     uint8_t cmd = 1;
     uint32_t length = 0;
     int32_t x = 0;
@@ -111,9 +117,10 @@ GeometryCollection VectorTileFeature::getGeometries() const {
     lines.emplace_back();
     GeometryCoordinates* line = &lines.back();
 
-    while (data.data < data.end) {
+    auto g_itr = geometry_iter;
+    while (g_itr.first != g_itr.second) {
         if (length == 0) {
-            uint32_t cmd_length = data.varint();
+            uint32_t cmd_length = static_cast<uint32_t>(*g_itr.first++);
             cmd = cmd_length & 0x7;
             length = cmd_length >> 3;
         }
@@ -121,8 +128,8 @@ GeometryCollection VectorTileFeature::getGeometries() const {
         --length;
 
         if (cmd == 1 || cmd == 2) {
-            x += data.svarint();
-            y += data.svarint();
+            x += protozero::decode_zigzag32(static_cast<uint32_t>(*g_itr.first++));
+            y += protozero::decode_zigzag32(static_cast<uint32_t>(*g_itr.first++));
 
             if (cmd == 1 && !line->empty()) { // moveTo
                 lines.emplace_back();
@@ -155,14 +162,10 @@ VectorTile::VectorTile(std::shared_ptr<const std::string> data_)
 util::ptr<GeometryTileLayer> VectorTile::getLayer(const std::string& name) const {
     if (!parsed) {
         parsed = true;
-        pbf tile_pbf(reinterpret_cast<const unsigned char *>(data->c_str()), data->size());
-        while (tile_pbf.next()) {
-            if (tile_pbf.tag == 3) { // layer
-                util::ptr<VectorTileLayer> layer = std::make_shared<VectorTileLayer>(tile_pbf.message());
-                layers.emplace(layer->name, layer);
-            } else {
-                tile_pbf.skip();
-            }
+        protozero::pbf_reader tile_pbf(*data);
+        while (tile_pbf.next(3)) {
+            util::ptr<VectorTileLayer> layer = std::make_shared<VectorTileLayer>(tile_pbf.get_message());
+            layers.emplace(layer->name, layer);
         }
     }
 
@@ -174,20 +177,27 @@ util::ptr<GeometryTileLayer> VectorTile::getLayer(const std::string& name) const
     return nullptr;
 }
 
-VectorTileLayer::VectorTileLayer(pbf layer_pbf) {
+VectorTileLayer::VectorTileLayer(protozero::pbf_reader layer_pbf) {
     while (layer_pbf.next()) {
-        if (layer_pbf.tag == 1) { // name
-            name = layer_pbf.string();
-        } else if (layer_pbf.tag == 2) { // feature
-            features.push_back(layer_pbf.message());
-        } else if (layer_pbf.tag == 3) { // keys
-            keysMap.emplace(layer_pbf.string(), keysMap.size());
-        } else if (layer_pbf.tag == 4) { // values
-            values.emplace_back(parseValue(layer_pbf.message()));
-        } else if (layer_pbf.tag == 5) { // extent
-            extent = layer_pbf.varint();
-        } else {
+        switch (layer_pbf.tag()) {
+        case 1: // name
+            name = layer_pbf.get_string();
+            break;
+        case 2: // feature
+            features.push_back(layer_pbf.get_message());
+            break;
+        case 3: // keys
+            keysMap.emplace(layer_pbf.get_string(), keysMap.size());
+            break;
+        case 4: // values
+            values.emplace_back(parseValue(layer_pbf.get_message()));
+            break;
+        case 5: // extent
+            extent = layer_pbf.get_uint32();
+            break;
+        default:
             layer_pbf.skip();
+            break;
         }
     }
 
diff --git a/src/mbgl/tile/vector_tile.hpp b/src/mbgl/tile/vector_tile.hpp
index 0e583ab33a..441d6827ac 100644
--- a/src/mbgl/tile/vector_tile.hpp
+++ b/src/mbgl/tile/vector_tile.hpp
@@ -3,7 +3,7 @@
 
 #include <mbgl/tile/geometry_tile.hpp>
 #include <mbgl/map/tile_id.hpp>
-#include <mbgl/util/pbf.hpp>
+#include <protozero/pbf_reader.hpp>
 
 #include <map>
 #include <unordered_map>
@@ -13,9 +13,12 @@ namespace mbgl {
 
 class VectorTileLayer;
 
+using pbf_iter_type = protozero::pbf_reader::const_uint32_iterator;
+using packed_iter_type = std::pair<pbf_iter_type,pbf_iter_type>;
+
 class VectorTileFeature : public GeometryTileFeature {
 public:
-    VectorTileFeature(pbf, const VectorTileLayer&);
+    VectorTileFeature(protozero::pbf_reader, const VectorTileLayer&);
 
     FeatureType getType() const override { return type; }
     optional<Value> getValue(const std::string&) const override;
@@ -28,13 +31,13 @@ private:
     const VectorTileLayer& layer;
     uint64_t id = 0;
     FeatureType type = FeatureType::Unknown;
-    pbf tags_pbf;
-    pbf geometry_pbf;
+    packed_iter_type tags_iter;
+    packed_iter_type geometry_iter;
 };
 
 class VectorTileLayer : public GeometryTileLayer {
 public:
-    VectorTileLayer(pbf);
+    VectorTileLayer(protozero::pbf_reader);
 
     std::size_t featureCount() const override { return features.size(); }
     util::ptr<const GeometryTileFeature> getFeature(std::size_t) const override;
@@ -49,7 +52,7 @@ private:
     std::map<std::string, uint32_t> keysMap;
     std::vector<std::reference_wrapper<const std::string>> keys;
     std::vector<Value> values;
-    std::vector<pbf> features;
+    std::vector<protozero::pbf_reader> features;
 };
 
 class VectorTile : public GeometryTile {
diff --git a/src/mbgl/util/pbf.hpp b/src/mbgl/util/pbf.hpp
deleted file mode 100644
index d017219a52..0000000000
--- a/src/mbgl/util/pbf.hpp
+++ /dev/null
@@ -1,184 +0,0 @@
-#ifndef MBGL_UTIL_PBF
-#define MBGL_UTIL_PBF
-
-/*
- * Some parts are from upb - a minimalist implementation of protocol buffers.
- *
- * Copyright (c) 2008-2011 Google Inc.  See LICENSE for details.
- * Author: Josh Haberman <jhaberman@gmail.com>
- */
-
-#include <string>
-#include <cstring>
-
-namespace mbgl {
-
-struct pbf {
-    struct exception : std::exception { const char *what() const noexcept { return "pbf exception"; } };
-    struct unterminated_varint_exception : exception { const char *what() const noexcept { return "pbf unterminated varint exception"; } };
-    struct varint_too_long_exception : exception { const char *what() const noexcept { return "pbf varint too long exception"; } };
-    struct unknown_field_type_exception : exception { const char *what() const noexcept { return "pbf unknown field type exception"; } };
-    struct end_of_buffer_exception : exception { const char *what() const noexcept { return "pbf end of buffer exception"; } };
-
-    inline pbf(const unsigned char *data, size_t length);
-    inline pbf();
-
-    inline operator bool() const;
-
-    inline bool next();
-    inline bool next(uint32_t tag);
-    template <typename T = uint32_t> inline T varint();
-    template <typename T = uint32_t> inline T svarint();
-
-    template <typename T = uint32_t, int bytes = 4> inline T fixed();
-    inline float float32();
-    inline double float64();
-
-    inline std::string string();
-    inline bool boolean();
-
-    inline pbf message();
-
-    inline void skip();
-    inline void skipValue(uint32_t val);
-    inline void skipBytes(uint32_t bytes);
-
-    const uint8_t *data = nullptr;
-    const uint8_t *end = nullptr;
-    uint32_t value = 0;
-    uint32_t tag = 0;
-};
-
-pbf::pbf(const unsigned char *data_, size_t length)
-    : data(data_),
-      end(data_ + length),
-      value(0),
-      tag(0) {
-}
-
-pbf::pbf()
-    : data(nullptr),
-      end(nullptr),
-      value(0),
-      tag(0) {
-}
-
-
-pbf::operator bool() const {
-    return data < end;
-}
-
-bool pbf::next() {
-    if (data < end) {
-        value = static_cast<uint32_t>(varint());
-        tag = value >> 3;
-        return true;
-    }
-    return false;
-}
-
-bool pbf::next(uint32_t requested_tag) {
-    while (next()) {
-        if (tag == requested_tag) {
-            return true;
-        } else {
-            skip();
-        }
-    }
-    return false;
-}
-
-template <typename T>
-T pbf::varint() {
-    uint8_t byte = 0x80;
-    T result = 0;
-    int bitpos;
-    for (bitpos = 0; bitpos < 70 && (byte & 0x80); bitpos += 7) {
-        if (data >= end) {
-            throw unterminated_varint_exception();
-        }
-        result |= ((T)(byte = *data) & 0x7F) << bitpos;
-
-        data++;
-    }
-    if (bitpos == 70 && (byte & 0x80)) {
-        throw varint_too_long_exception();
-    }
-
-    return result;
-}
-
-template <typename T>
-T pbf::svarint() {
-    T n = varint<T>();
-    return (n >> 1) ^ -(T)(n & 1);
-}
-
-template <typename T, int bytes>
-T pbf::fixed() {
-    skipBytes(bytes);
-    T result;
-    memcpy(&result, data - bytes, bytes);
-    return result;
-}
-
-float pbf::float32() {
-    return fixed<float, 4>();
-}
-
-double pbf::float64() {
-    return fixed<double, 8>();
-}
-
-std::string pbf::string() {
-    uint32_t bytes = static_cast<uint32_t>(varint());
-    const char *string_data = reinterpret_cast<const char*>(data);
-    skipBytes(bytes);
-    return std::string(string_data, bytes);
-}
-
-bool pbf::boolean() {
-    skipBytes(1);
-    return *(bool *)(data - 1);
-}
-
-pbf pbf::message() {
-    uint32_t bytes = static_cast<uint32_t>(varint());
-    const uint8_t *pos = data;
-    skipBytes(bytes);
-    return pbf(pos, bytes);
-}
-
-void pbf::skip() {
-    skipValue(value);
-}
-
-void pbf::skipValue(uint32_t val) {
-    switch (val & 0x7) {
-        case 0: // varint
-            varint();
-            break;
-        case 1: // 64 bit
-            skipBytes(8);
-            break;
-        case 2: // string/message
-            skipBytes(static_cast<uint32_t>(varint()));
-            break;
-        case 5: // 32 bit
-            skipBytes(4);
-            break;
-        default:
-            throw unknown_field_type_exception();
-    }
-}
-
-void pbf::skipBytes(uint32_t bytes) {
-    if (data + bytes > end) {
-        throw end_of_buffer_exception();
-    }
-    data += bytes;
-}
-
-} // end namespace mbgl
-
-#endif
diff --git a/test/style/glyph_store.cpp b/test/style/glyph_store.cpp
index 2a874a05a7..08ec6902b9 100644
--- a/test/style/glyph_store.cpp
+++ b/test/style/glyph_store.cpp
@@ -110,7 +110,7 @@ TEST(GlyphStore, LoadingCorrupted) {
         EXPECT_EQ(glyphRange, GlyphRange(0, 255));
 
         EXPECT_TRUE(error != nullptr);
-        EXPECT_EQ(util::toString(error), "pbf unknown field type exception");
+        EXPECT_EQ(util::toString(error), "unknown pbf field type exception");
 
         auto glyphSet = test.glyphStore.getGlyphSet({{"Test Stack"}});
         ASSERT_TRUE(glyphSet->getSDFs().empty());
diff --git a/test/style/source.cpp b/test/style/source.cpp
index 5ec2714b43..7964a05814 100644
--- a/test/style/source.cpp
+++ b/test/style/source.cpp
@@ -269,7 +269,7 @@ TEST(Source, VectorTileCorrupt) {
     test.observer.tileError = [&] (Source& source, const TileID& tileID, std::exception_ptr error) {
         EXPECT_EQ(source.type, SourceType::Vector);
         EXPECT_EQ(std::string(tileID), "0/0/0");
-        EXPECT_EQ(util::toString(error), "pbf unknown field type exception");
+        EXPECT_EQ(util::toString(error), "unknown pbf field type exception");
         test.end();
     };
 
diff --git a/test/test.gypi b/test/test.gypi
index a9e53c88fc..2326fd7cf9 100644
--- a/test/test.gypi
+++ b/test/test.gypi
@@ -85,6 +85,7 @@
         'cflags_cc': [
           '<@(gtest_cflags)',
           '<@(opengl_cflags)',
+          '<@(protozero_cflags)',
           '<@(boost_cflags)',
           '<@(sqlite_cflags)',
           '<@(geojsonvt_cflags)',