[core] Use UTF-16 instead of UTF-32 for label features to avoid extra conversions and reduce in-memory size.

Continue to use uint32 as glyph ID to maintain Glyph PBF, even though we're only using 16 bits of that uint32.
Use std::codecvt instead of boost::unicode_iterator for UTF8->UTF16 conversions.

3 files changed, 18 insertions, 17 deletions

diff --git a/src/mbgl/util/i18n.cpp b/src/mbgl/util/i18n.cpp
index dbfa24c5cf..97dda76320 100644
--- a/src/mbgl/util/i18n.cpp
+++ b/src/mbgl/util/i18n.cpp
@@ -8,7 +8,7 @@ namespace {
     @param last The last codepoint in the block, inclusive.
  */
 #define DEFINE_IS_IN_UNICODE_BLOCK(name, first, last)                                              \
-    inline bool isIn##name(uint32_t codepoint) {                                                   \
+    inline bool isIn##name(uint16_t codepoint) {                                                   \
         return codepoint >= first && codepoint <= last;                                            \
     }
 
@@ -294,13 +294,13 @@ namespace mbgl {
 namespace util {
 namespace i18n {
 
-bool isVisible(uint32_t chr) {
+bool isVisible(uint16_t chr) {
     return (chr == 0x0a    /* newline */
             || chr == 0x20 /* space */
             || chr == 0x200b /* zero-width space */);
 }
 
-bool allowsWordBreaking(uint32_t chr) {
+bool allowsWordBreaking(uint16_t chr) {
     return (chr == 0x0a      /* newline */
             || chr == 0x20   /* space */
             || chr == 0x26   /* ampersand */
@@ -314,8 +314,8 @@ bool allowsWordBreaking(uint32_t chr) {
             || chr == 0x2013 /* en dash */);
 }
 
-bool allowsIdeographicBreaking(const std::u32string& string) {
-    for (uint32_t chr : string) {
+bool allowsIdeographicBreaking(const std::u16string& string) {
+    for (uint16_t chr : string) {
         if (!allowsIdeographicBreaking(chr)) {
             return false;
         }
@@ -323,7 +323,7 @@ bool allowsIdeographicBreaking(const std::u32string& string) {
     return true;
 }
 
-bool allowsIdeographicBreaking(uint32_t chr) {
+bool allowsIdeographicBreaking(uint16_t chr) {
     // Return early for characters outside all ideographic ranges.
     if (chr < 0x2E80)
         return false;
diff --git a/src/mbgl/util/i18n.hpp b/src/mbgl/util/i18n.hpp
index fe324f5362..c07dc91ed6 100644
--- a/src/mbgl/util/i18n.hpp
+++ b/src/mbgl/util/i18n.hpp
@@ -7,20 +7,20 @@ namespace util {
 namespace i18n {
 
 /** Returns whether a character is a visible character. */
-bool isVisible(uint32_t chr);
+bool isVisible(uint16_t chr);
 
 /** Returns whether a line break can be inserted after the character indicated
     by the given Unicode codepoint due to word breaking. */
-bool allowsWordBreaking(uint32_t chr);
+bool allowsWordBreaking(uint16_t chr);
 
 /** Returns whether a line break can be inserted after any character in the
     given string. If false, line breaking should occur on word boundaries
     instead. */
-bool allowsIdeographicBreaking(const std::u32string& string);
+bool allowsIdeographicBreaking(const std::u16string& string);
 
 /** Returns whether a line break can be inserted after the character indicated
     by the given Unicode codepoint due to ideographic breaking. */
-bool allowsIdeographicBreaking(uint32_t chr);
+bool allowsIdeographicBreaking(uint16_t chr);
 
 } // namespace i18n
 } // namespace util
diff --git a/src/mbgl/util/utf.hpp b/src/mbgl/util/utf.hpp
index 560ca3ba7f..386e56bef8 100644
--- a/src/mbgl/util/utf.hpp
+++ b/src/mbgl/util/utf.hpp
@@ -2,19 +2,20 @@
 
 #include <memory>
 
-#include <boost/regex/pending/unicode_iterator.hpp>
+#include <locale>
+#include <codecvt>
 
 namespace mbgl {
 namespace util {
 
-class utf8_to_utf32 {
- public:
-    static std::u32string convert(std::string const& utf8)
+class utf8_to_utf16 {
+public:
+    static std::u16string convert(std::string const& utf8)
     {
-        boost::u8_to_u32_iterator<std::string::const_iterator> begin(utf8.begin());
-        boost::u8_to_u32_iterator<std::string::const_iterator> end(utf8.end());
-        return std::u32string(begin,end);
+        std::wstring_convert<std::codecvt_utf8_utf16<char16_t>,char16_t> converter;
+        return converter.from_bytes( utf8 );
     }
+    
 };
 
 } // namespace util