[core] Use UTF-16 instead of UTF-32 for label features to avoid extra conversions and reduce in-memory size.

Continue to use uint32 as glyph ID to maintain Glyph PBF, even though we're only using 16 bits of that uint32.
Use std::codecvt instead of boost::unicode_iterator for UTF8->UTF16 conversions.

16 files changed, 40 insertions, 39 deletions

diff --git a/platform/default/mbgl/storage/offline_download.cpp b/platform/default/mbgl/storage/offline_download.cpp
index 9e2e11c86f..3edc75845c 100644
--- a/platform/default/mbgl/storage/offline_download.cpp
+++ b/platform/default/mbgl/storage/offline_download.cpp
@@ -184,7 +184,7 @@ void OfflineDownload::activateDownload() {
 
         if (!parser.glyphURL.empty()) {
             for (const auto& fontStack : parser.fontStacks()) {
-                for (uint32_t i = 0; i < GLYPH_RANGES_PER_FONT_STACK; i++) {
+                for (char16_t i = 0; i < GLYPH_RANGES_PER_FONT_STACK; i++) {
                     queueResource(Resource::glyphs(parser.glyphURL, fontStack, getGlyphRange(i * GLYPHS_PER_GLYPH_RANGE)));
                 }
             }
diff --git a/platform/default/string_stdlib.cpp b/platform/default/string_stdlib.cpp
index 90a75c1738..0e97fc54d5 100644
--- a/platform/default/string_stdlib.cpp
+++ b/platform/default/string_stdlib.cpp
@@ -1,10 +1,10 @@
 #include <mbgl/platform/platform.hpp>
-#include <mbgl/util/utf.hpp>
 #define NU_WITH_TOUPPER
 #define NU_WITH_TOLOWER
 #define NU_WITH_UTF8_WRITER
 #include <libnu/libnu.h>
 #include <cstring>
+#include <sstream>
 
 namespace mbgl { namespace platform {
 
diff --git a/src/mbgl/layout/merge_lines.cpp b/src/mbgl/layout/merge_lines.cpp
index f4fdb82617..676cbc092d 100644
--- a/src/mbgl/layout/merge_lines.cpp
+++ b/src/mbgl/layout/merge_lines.cpp
@@ -47,10 +47,10 @@ enum class Side {
 };
 
 size_t
-getKey(const std::u32string& text, const GeometryCollection& geom, Side side) {
+getKey(const std::u16string& text, const GeometryCollection& geom, Side side) {
     const GeometryCoordinate& coord = side == Side::Right ? geom[0].back() : geom[0].front();
 
-    auto hash = std::hash<std::u32string>()(text);
+    auto hash = std::hash<std::u16string>()(text);
     boost::hash_combine(hash, coord.x);
     boost::hash_combine(hash, coord.y);
     return hash;
diff --git a/src/mbgl/layout/symbol_feature.hpp b/src/mbgl/layout/symbol_feature.hpp
index 99db4f9ac5..9e0eacaac5 100644
--- a/src/mbgl/layout/symbol_feature.hpp
+++ b/src/mbgl/layout/symbol_feature.hpp
@@ -10,7 +10,7 @@ namespace mbgl {
 class SymbolFeature {
 public:
     GeometryCollection geometry;
-    optional<std::u32string> text;
+    optional<std::u16string> text;
     optional<std::string> icon;
     std::size_t index;
 };
diff --git a/src/mbgl/layout/symbol_layout.cpp b/src/mbgl/layout/symbol_layout.cpp
index d974e6a357..7f8e8d5f83 100644
--- a/src/mbgl/layout/symbol_layout.cpp
+++ b/src/mbgl/layout/symbol_layout.cpp
@@ -90,10 +90,10 @@ SymbolLayout::SymbolLayout(std::string bucketName_,
                 u8string = platform::lowercase(u8string);
             }
 
-            ft.text = util::utf8_to_utf32::convert(u8string);
+            ft.text = util::utf8_to_utf16::convert( u8string );
 
             // Loop through all characters of this text and collect unique codepoints.
-            for (char32_t chr : *ft.text) {
+            for (char16_t chr : *ft.text) {
                 ranges.insert(getGlyphRange(chr));
             }
         }
@@ -309,7 +309,7 @@ void SymbolLayout::addFeature(const GeometryCollection &lines,
     }
 }
 
-bool SymbolLayout::anchorIsTooClose(const std::u32string &text, const float repeatDistance, Anchor &anchor) {
+bool SymbolLayout::anchorIsTooClose(const std::u16string &text, const float repeatDistance, Anchor &anchor) {
     if (compareText.find(text) == compareText.end()) {
         compareText.emplace(text, Anchors());
     } else {
diff --git a/src/mbgl/layout/symbol_layout.hpp b/src/mbgl/layout/symbol_layout.hpp
index 54acf84aaf..9a580900dd 100644
--- a/src/mbgl/layout/symbol_layout.hpp
+++ b/src/mbgl/layout/symbol_layout.hpp
@@ -64,8 +64,8 @@ private:
                     const GlyphPositions& face,
                     const size_t index);
 
-    bool anchorIsTooClose(const std::u32string& text, const float repeatDistance, Anchor&);
-    std::map<std::u32string, std::vector<Anchor>> compareText;
+    bool anchorIsTooClose(const std::u16string& text, const float repeatDistance, Anchor&);
+    std::map<std::u16string, std::vector<Anchor>> compareText;
 
     void addToDebugBuffers(CollisionTile&, SymbolBucket&);
 
diff --git a/src/mbgl/text/glyph.cpp b/src/mbgl/text/glyph.cpp
index a877d7a799..29929b73e6 100644
--- a/src/mbgl/text/glyph.cpp
+++ b/src/mbgl/text/glyph.cpp
@@ -3,7 +3,7 @@
 namespace mbgl {
 
 // Note: this only works for the BMP
-GlyphRange getGlyphRange(char32_t glyph) {
+GlyphRange getGlyphRange(char16_t glyph) {
     unsigned start = (glyph/256) * 256;
     unsigned end = (start + 255);
     if (start > 65280) start = 65280;
diff --git a/src/mbgl/text/glyph.hpp b/src/mbgl/text/glyph.hpp
index a333f68ff4..d07fbdff21 100644
--- a/src/mbgl/text/glyph.hpp
+++ b/src/mbgl/text/glyph.hpp
@@ -11,7 +11,7 @@
 namespace mbgl {
 
 // Note: this only works for the BMP
-GlyphRange getGlyphRange(char32_t glyph);
+GlyphRange getGlyphRange(char16_t glyph);
 
 struct GlyphMetrics {
     explicit operator bool() const {
@@ -63,10 +63,10 @@ public:
 class Shaping {
     public:
     explicit Shaping() : top(0), bottom(0), left(0), right(0) {}
-    explicit Shaping(float x, float y, std::u32string text_)
+    explicit Shaping(float x, float y, std::u16string text_)
         : text(std::move(text_)), top(y), bottom(y), left(x), right(x) {}
     std::vector<PositionedGlyph> positionedGlyphs;
-    std::u32string text;
+    std::u16string text;
     int32_t top;
     int32_t bottom;
     int32_t left;
diff --git a/src/mbgl/text/glyph_atlas.cpp b/src/mbgl/text/glyph_atlas.cpp
index 031e89d13a..57b8b14017 100644
--- a/src/mbgl/text/glyph_atlas.cpp
+++ b/src/mbgl/text/glyph_atlas.cpp
@@ -81,7 +81,7 @@ void GlyphAtlas::setObserver(GlyphAtlasObserver* observer_) {
 }
 
 void GlyphAtlas::addGlyphs(uintptr_t tileUID,
-                           const std::u32string& text,
+                           const std::u16string& text,
                            const FontStack& fontStack,
                            const GlyphSet& glyphSet,
                            GlyphPositions& face)
@@ -90,7 +90,7 @@ void GlyphAtlas::addGlyphs(uintptr_t tileUID,
 
     const std::map<uint32_t, SDFGlyph>& sdfs = glyphSet.getSDFs();
 
-    for (uint32_t chr : text)
+    for (char16_t chr : text)
     {
         auto sdf_it = sdfs.find(chr);
         if (sdf_it == sdfs.end()) {
diff --git a/src/mbgl/text/glyph_atlas.hpp b/src/mbgl/text/glyph_atlas.hpp
index 550ca4cc17..af14aace5b 100644
--- a/src/mbgl/text/glyph_atlas.hpp
+++ b/src/mbgl/text/glyph_atlas.hpp
@@ -55,7 +55,7 @@ public:
     void setObserver(GlyphAtlasObserver* observer);
 
     void addGlyphs(uintptr_t tileUID,
-                   const std::u32string& text,
+                   const std::u16string& text,
                    const FontStack&,
                    const GlyphSet&,
                    GlyphPositions&);
diff --git a/src/mbgl/text/glyph_set.cpp b/src/mbgl/text/glyph_set.cpp
index c778de207b..f0e3991540 100644
--- a/src/mbgl/text/glyph_set.cpp
+++ b/src/mbgl/text/glyph_set.cpp
@@ -31,7 +31,7 @@ const std::map<uint32_t, SDFGlyph> &GlyphSet::getSDFs() const {
     return sdfs;
 }
 
-const Shaping GlyphSet::getShaping(const std::u32string &string, const float maxWidth,
+const Shaping GlyphSet::getShaping(const std::u16string &string, const float maxWidth,
                                     const float lineHeight, const float horizontalAlign,
                                     const float verticalAlign, const float justify,
                                     const float spacing, const Point<float> &translate) const {
@@ -44,7 +44,7 @@ const Shaping GlyphSet::getShaping(const std::u32string &string, const float max
     const float y = yOffset;
 
     // Loop through all characters of this label and shape.
-    for (uint32_t chr : string) {
+    for (char16_t chr : string) {
         auto it = sdfs.find(chr);
         if (it != sdfs.end()) {
             shaping.positionedGlyphs.emplace_back(chr, x, y);
diff --git a/src/mbgl/text/glyph_set.hpp b/src/mbgl/text/glyph_set.hpp
index fed7960a5f..004cae343d 100644
--- a/src/mbgl/text/glyph_set.hpp
+++ b/src/mbgl/text/glyph_set.hpp
@@ -9,7 +9,7 @@ class GlyphSet {
 public:
     void insert(uint32_t id, SDFGlyph&&);
     const std::map<uint32_t, SDFGlyph> &getSDFs() const;
-    const Shaping getShaping(const std::u32string &string, float maxWidth, float lineHeight,
+    const Shaping getShaping(const std::u16string &string, float maxWidth, float lineHeight,
                              float horizontalAlign, float verticalAlign, float justify,
                              float spacing, const Point<float> &translate) const;
     void lineWrap(Shaping &shaping, float lineHeight, float maxWidth, float horizontalAlign,
diff --git a/src/mbgl/util/i18n.cpp b/src/mbgl/util/i18n.cpp
index dbfa24c5cf..97dda76320 100644
--- a/src/mbgl/util/i18n.cpp
+++ b/src/mbgl/util/i18n.cpp
@@ -8,7 +8,7 @@ namespace {
     @param last The last codepoint in the block, inclusive.
  */
 #define DEFINE_IS_IN_UNICODE_BLOCK(name, first, last)                                              \
-    inline bool isIn##name(uint32_t codepoint) {                                                   \
+    inline bool isIn##name(uint16_t codepoint) {                                                   \
         return codepoint >= first && codepoint <= last;                                            \
     }
 
@@ -294,13 +294,13 @@ namespace mbgl {
 namespace util {
 namespace i18n {
 
-bool isVisible(uint32_t chr) {
+bool isVisible(uint16_t chr) {
     return (chr == 0x0a    /* newline */
             || chr == 0x20 /* space */
             || chr == 0x200b /* zero-width space */);
 }
 
-bool allowsWordBreaking(uint32_t chr) {
+bool allowsWordBreaking(uint16_t chr) {
     return (chr == 0x0a      /* newline */
             || chr == 0x20   /* space */
             || chr == 0x26   /* ampersand */
@@ -314,8 +314,8 @@ bool allowsWordBreaking(uint32_t chr) {
             || chr == 0x2013 /* en dash */);
 }
 
-bool allowsIdeographicBreaking(const std::u32string& string) {
-    for (uint32_t chr : string) {
+bool allowsIdeographicBreaking(const std::u16string& string) {
+    for (uint16_t chr : string) {
         if (!allowsIdeographicBreaking(chr)) {
             return false;
         }
@@ -323,7 +323,7 @@ bool allowsIdeographicBreaking(const std::u32string& string) {
     return true;
 }
 
-bool allowsIdeographicBreaking(uint32_t chr) {
+bool allowsIdeographicBreaking(uint16_t chr) {
     // Return early for characters outside all ideographic ranges.
     if (chr < 0x2E80)
         return false;
diff --git a/src/mbgl/util/i18n.hpp b/src/mbgl/util/i18n.hpp
index fe324f5362..c07dc91ed6 100644
--- a/src/mbgl/util/i18n.hpp
+++ b/src/mbgl/util/i18n.hpp
@@ -7,20 +7,20 @@ namespace util {
 namespace i18n {
 
 /** Returns whether a character is a visible character. */
-bool isVisible(uint32_t chr);
+bool isVisible(uint16_t chr);
 
 /** Returns whether a line break can be inserted after the character indicated
     by the given Unicode codepoint due to word breaking. */
-bool allowsWordBreaking(uint32_t chr);
+bool allowsWordBreaking(uint16_t chr);
 
 /** Returns whether a line break can be inserted after any character in the
     given string. If false, line breaking should occur on word boundaries
     instead. */
-bool allowsIdeographicBreaking(const std::u32string& string);
+bool allowsIdeographicBreaking(const std::u16string& string);
 
 /** Returns whether a line break can be inserted after the character indicated
     by the given Unicode codepoint due to ideographic breaking. */
-bool allowsIdeographicBreaking(uint32_t chr);
+bool allowsIdeographicBreaking(uint16_t chr);
 
 } // namespace i18n
 } // namespace util
diff --git a/src/mbgl/util/utf.hpp b/src/mbgl/util/utf.hpp
index 560ca3ba7f..386e56bef8 100644
--- a/src/mbgl/util/utf.hpp
+++ b/src/mbgl/util/utf.hpp
@@ -2,19 +2,20 @@
 
 #include <memory>
 
-#include <boost/regex/pending/unicode_iterator.hpp>
+#include <locale>
+#include <codecvt>
 
 namespace mbgl {
 namespace util {
 
-class utf8_to_utf32 {
- public:
-    static std::u32string convert(std::string const& utf8)
+class utf8_to_utf16 {
+public:
+    static std::u16string convert(std::string const& utf8)
     {
-        boost::u8_to_u32_iterator<std::string::const_iterator> begin(utf8.begin());
-        boost::u8_to_u32_iterator<std::string::const_iterator> end(utf8.end());
-        return std::u32string(begin,end);
+        std::wstring_convert<std::codecvt_utf8_utf16<char16_t>,char16_t> converter;
+        return converter.from_bytes( utf8 );
     }
+    
 };
 
 } // namespace util
diff --git a/test/util/merge_lines.test.cpp b/test/util/merge_lines.test.cpp
index db81d8b209..30cd1af068 100644
--- a/test/util/merge_lines.test.cpp
+++ b/test/util/merge_lines.test.cpp
@@ -3,8 +3,8 @@
 #include <mbgl/layout/merge_lines.hpp>
 #include <mbgl/layout/symbol_feature.hpp>
 
-const std::u32string aaa = U"a";
-const std::u32string bbb = U"b";
+const std::u16string aaa = u"a";
+const std::u16string bbb = u"b";
 
 TEST(MergeLines, SameText) {
     // merges lines with the same text