[core] Introduce LanguageTag for parsing BCP 47 tags

5 files changed, 329 insertions, 0 deletions

diff --git a/cmake/core-files.cmake b/cmake/core-files.cmake
index a06f3a59f5..462ead3b1f 100644
--- a/cmake/core-files.cmake
+++ b/cmake/core-files.cmake
@@ -614,6 +614,8 @@ set(MBGL_CORE_FILES
     src/mbgl/text/glyph_pbf.cpp
     src/mbgl/text/glyph_pbf.hpp
     src/mbgl/text/glyph_range.hpp
+    src/mbgl/text/language_tag.cpp
+    src/mbgl/text/language_tag.hpp
     src/mbgl/text/local_glyph_rasterizer.hpp
     src/mbgl/text/placement.cpp
     src/mbgl/text/placement.hpp
diff --git a/cmake/test-files.cmake b/cmake/test-files.cmake
index ecd7700e97..b2c0172d73 100644
--- a/cmake/test-files.cmake
+++ b/cmake/test-files.cmake
@@ -117,6 +117,7 @@ set(MBGL_TEST_FILES
     test/text/cross_tile_symbol_index.test.cpp
     test/text/glyph_manager.test.cpp
     test/text/glyph_pbf.test.cpp
+    test/text/language_tag.test.cpp
     test/text/local_glyph_rasterizer.test.cpp
     test/text/quads.test.cpp
 
diff --git a/src/mbgl/text/language_tag.cpp b/src/mbgl/text/language_tag.cpp
new file mode 100644
index 0000000000..4c2712f103
--- /dev/null
+++ b/src/mbgl/text/language_tag.cpp
@@ -0,0 +1,237 @@
+#include <mbgl/text/language_tag.hpp>
+
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunknown-pragmas"
+#pragma GCC diagnostic ignored "-Wunused-parameter"
+#pragma GCC diagnostic ignored "-Wshadow"
+#pragma clang diagnostic push
+#pragma clang diagnostic ignored "-Wshorten-64-to-32"
+#pragma clang diagnostic ignored "-Wtautological-constant-compare"
+#include <boost/spirit/include/qi.hpp>
+#include <boost/spirit/include/phoenix_core.hpp>
+#include <boost/spirit/include/phoenix_operator.hpp>
+#pragma clang diagnostic pop
+#pragma GCC diagnostic pop
+
+#include <sstream>
+
+/*
+ ABNF for BCP 47 from: https://tools.ietf.org/html/bcp47
+
+ Language-Tag  = langtag             ; normal language tags
+               / privateuse          ; private use tag
+               / grandfathered       ; grandfathered tags  NOT IMPLEMENTED
+
+ langtag       = language
+                 ["-" script]
+                 ["-" region]
+                 *("-" variant)
+                 *("-" extension)
+                 ["-" privateuse]
+
+ language      = 2*3ALPHA            ; shortest ISO 639 code
+                 ["-" extlang]       ; sometimes followed by
+                                     ; extended language subtags
+               / 4ALPHA              ; or reserved for future use
+               / 5*8ALPHA            ; or registered language subtag
+
+ extlang       = 3ALPHA              ; selected ISO 639 codes
+                 *2("-" 3ALPHA)      ; permanently reserved
+
+ script        = 4ALPHA              ; ISO 15924 code
+
+ region        = 2ALPHA              ; ISO 3166-1 code
+               / 3DIGIT              ; UN M.49 code
+
+ variant       = 5*8alphanum         ; registered variants
+               / (DIGIT 3alphanum)
+
+ extension     = singleton 1*("-" (2*8alphanum))
+
+ ; Single alphanumerics
+ ; "x" reserved for private use
+ singleton     = DIGIT               ; 0 - 9
+               / %x41-57             ; A - W
+               / %x59-5A             ; Y - Z
+               / %x61-77             ; a - w
+               / %x79-7A             ; y - z
+
+ privateuse    = "x" 1*("-" (1*8alphanum))
+
+ grandfathered = irregular           ; non-redundant tags registered
+               / regular             ; during the RFC 3066 era
+
+ irregular     = "en-GB-oed"         ; irregular tags do not match
+                 / "i-ami"             ; the 'langtag' production and
+                 / "i-bnn"             ; would not otherwise be
+                 / "i-default"         ; considered 'well-formed'
+                 / "i-enochian"        ; These tags are all valid,
+                 / "i-hak"             ; but most are deprecated
+                 / "i-klingon"         ; in favor of more modern
+                 / "i-lux"             ; subtags or subtag
+                 / "i-mingo"           ; combination
+                 / "i-navajo"
+                 / "i-pwn"
+                 / "i-tao"
+                 / "i-tay"
+                 / "i-tsu"
+                 / "sgn-BE-FR"
+                 / "sgn-BE-NL"
+                 / "sgn-CH-DE"
+
+ regular       = "art-lojban"        ; these tags match the 'langtag'
+                 / "cel-gaulish"       ; production, but their subtags
+                 / "no-bok"            ; are not extended language
+                 / "no-nyn"            ; or variant subtags: their meaning
+                 / "zh-guoyu"          ; is defined by their registration
+                 / "zh-hakka"          ; and all of these are deprecated
+                 / "zh-min"            ; in favor of a more modern
+                 / "zh-min-nan"        ; subtag or sequence of subtags
+                 / "zh-xiang"
+
+ alphanum      = (ALPHA / DIGIT)     ; letters and numbers
+
+*/
+
+namespace mbgl {
+
+namespace qi = boost::spirit::qi;
+namespace phoenix = boost::phoenix;
+namespace ascii = boost::spirit::ascii;
+
+template <typename Iterator>
+struct bcp47_parser : qi::grammar<Iterator>
+{
+    bcp47_parser() : bcp47_parser::base_type(start)
+    {
+        using qi::lit;
+        using qi::repeat;
+        using qi::inf;
+        using qi::eoi;
+        using ascii::char_;
+        using ascii::no_case;
+        using ascii::digit;
+        using ascii::alnum;
+        using ascii::alpha;
+
+        using boost::spirit::qi::_1;
+
+        start %= no_case[langtag | privateuse | grandfathered];
+
+        langtag %= (language) [phoenix::ref(languageTag.language) = _1]
+            >> -("-" >> (script)[phoenix::ref(languageTag.script) = _1])
+            >> -("-" >> (region)[phoenix::ref(languageTag.region) = _1])
+            >> *("-" >> variant)
+            >> *("-" >> extension)
+            >> -("-" >> privateuse);
+
+        language %= (repeat(2,3)[alpha] >> -("-" >> extlang))     // shortest ISO 639 code
+                                                                  // sometimes followed by extended language subtags
+            | repeat(4)[alpha]                                    // or reserved for future use
+            | repeat(5,8)[alpha];                                 // or registered language subtag
+
+        // We add lookaheads for "-"/eoi so that spurious matches on subtags don't prevent backtracking
+        extlang = repeat(3)[alpha] >> (&lit('-') | eoi) >> repeat(0,2)["-" >> repeat(3)[alpha] >> (&lit('-') | eoi)];
+
+        script = repeat(4)[alpha] >> (&lit('-') | eoi);
+
+        region = (repeat(2)[alpha] | repeat(3)[digit]) >> (&lit('-') | eoi);
+
+        variant = (repeat(5,8)[alnum] | (digit >> repeat(3,inf)[alnum])) >> (&lit('-') | eoi);
+
+        extension = singleton >> +("-" >> repeat(2,8)[alnum]) >> (&lit('-') | eoi);
+
+        singleton = digit | char_('a','w') | char_('y','z'); // "no-case" handles A-W and Y-Z
+
+        privateuse = "x" >> +("-" >> repeat(1,8)[alnum]) >> (&lit('-') | eoi);
+
+        grandfathered = regular | irregular;
+
+        irregular = lit("en-GB-oed")
+            | "i-ami"
+            | "i-bnn"
+            | "i-default"
+            | "i-enochian"
+            | "i-hak"
+            | "i-klingon"
+            | "i-lux"
+            | "i-mingo"
+            | "i-navajo"
+            | "i-pwn"
+            | "i-tao"
+            | "i-tay"
+            | "i-tsu"
+            | "sgn-BE-FR"
+            | "sgn-BE-NL"
+            | "sgn-CH-DE";
+
+        regular = lit("art-lojban")
+            | "cel-gaulish"
+            | "no-bok"
+            | "no-nyn"
+            | "zh-guoyu"
+            | "zh-hakka"
+            | "zh-min"
+            | "zh-min-nan"
+            | "zh-xiang";
+    }
+
+    qi::rule<Iterator> start;
+    qi::rule<Iterator> langtag;
+    qi::rule<Iterator, std::string()> language;
+    qi::rule<Iterator> extlang;
+    qi::rule<Iterator, std::string()> script;
+    qi::rule<Iterator, std::string()> region;
+    qi::rule<Iterator> variant;
+    qi::rule<Iterator> extension;
+    qi::rule<Iterator> singleton;
+    qi::rule<Iterator> privateuse;
+    qi::rule<Iterator> grandfathered;
+    qi::rule<Iterator> irregular;
+    qi::rule<Iterator> regular;
+
+    LanguageTag languageTag;
+};
+
+LanguageTag LanguageTag::fromBCP47(const std::string& bcp47Tag) {
+    typedef std::string::const_iterator iterator_type;
+    typedef bcp47_parser<iterator_type> bcp47_parser;
+
+    bcp47_parser parser;
+    std::string::const_iterator iter = bcp47Tag.begin();
+    std::string::const_iterator end = bcp47Tag.end();
+    bool r = parse(iter, end, parser);
+    if (r && iter == end) {
+        return parser.languageTag;
+    } else {
+        // Invalid tags are treated as empty/"default"
+        return LanguageTag();
+    }
+}
+
+LanguageTag::LanguageTag(optional<std::string> language_, optional<std::string> script_, optional<std::string> region_)
+    : language(std::move(language_))
+    , script(std::move(script_))
+    , region(std::move(region_))
+{}
+
+std::string LanguageTag::toBCP47() const {
+    std::stringstream bcp47;
+    if (!language) {
+        // BCP 47 requires a language, but we're matching implementations that accept ""
+        // to mean something like "default"
+        return bcp47.str();
+    } else {
+        bcp47 << *language;
+    }
+
+    if (script) {
+        bcp47 << "-" << *script;
+    }
+
+    if (region) {
+        bcp47 << "-" << *region;
+    }
+    return bcp47.str();
+}
+} // end namespace mbgl
diff --git a/src/mbgl/text/language_tag.hpp b/src/mbgl/text/language_tag.hpp
new file mode 100644
index 0000000000..7a6a16531f
--- /dev/null
+++ b/src/mbgl/text/language_tag.hpp
@@ -0,0 +1,44 @@
+#pragma once
+
+#include <mbgl/util/optional.hpp>
+
+#include <string>
+
+/*
+    Use LanguageTag to go back and forth between BCP 47 language tags
+    and their component language/script/region.
+
+    This implementation accepts but will not round-trip additional
+    variant/extension/privateuse/grandfathered information in a BCP 47 tag.
+
+    Why implement this?
+        Mapbox Style Spec specifies locales with BCP 47
+        Android and Intl.Collator implementations speak BCP 47 out of the box
+        Darwin implementation requires translation to "Language Identifier"
+        We're OK with not supporting extensions, but we want to succesfully
+         parse any valid BCP 47 tag and get out the base language/script/region.
+
+    Mozilla's version: https://dxr.mozilla.org/mozilla-central/source/intl/locale/MozLocale.cpp
+        Looks like it actually supports a subset of BCP 47.
+        See https://bugzilla.mozilla.org/show_bug.cgi?id=bcp47
+
+    Chromium is based on ICU version: https://ssl.icu-project.org/apiref/icu4c/uloc_8h.html
+        Getting all the locale information is overkill for us, we just want
+         language/script/region.
+ */
+
+namespace mbgl {
+
+struct LanguageTag {
+    LanguageTag() = default;
+    LanguageTag(optional<std::string> language_, optional<std::string> script_, optional<std::string> region_);
+
+    static LanguageTag fromBCP47(const std::string& bcp47Tag);
+    std::string toBCP47() const;
+
+    optional<std::string> language; // ISO 639
+    optional<std::string> script;   // ISO 15924
+    optional<std::string> region;   // ISO 3316-1 || UN M.49
+};
+
+} // end namespace mbgl
diff --git a/test/text/language_tag.test.cpp b/test/text/language_tag.test.cpp
new file mode 100644
index 0000000000..7d70088594
--- /dev/null
+++ b/test/text/language_tag.test.cpp
@@ -0,0 +1,45 @@
+#include <mbgl/test/util.hpp>
+
+#include <mbgl/text/language_tag.hpp>
+
+using namespace mbgl;
+
+TEST(LanguageTag, Parsing) {
+    LanguageTag bareLanguage = LanguageTag::fromBCP47("de");
+    EXPECT_EQ(*(bareLanguage.language), "de");
+    EXPECT_FALSE(bareLanguage.script);
+    EXPECT_FALSE(bareLanguage.region);
+    
+    LanguageTag languageAndScript = LanguageTag::fromBCP47("de-hans");
+    EXPECT_EQ(*(languageAndScript.language), "de");
+    EXPECT_EQ(*(languageAndScript.script), "hans");
+    EXPECT_FALSE(languageAndScript.region);
+    
+    LanguageTag languageAndRegion = LanguageTag::fromBCP47("de-US");
+    EXPECT_EQ(*(languageAndRegion.language), "de");
+    EXPECT_FALSE(languageAndRegion.script);
+    EXPECT_EQ(*(languageAndRegion.region), "US");
+    
+    LanguageTag kitchenSink = LanguageTag::fromBCP47("de-ext-two-Hans-US-variant-x-private");
+    EXPECT_EQ(*(kitchenSink.language), "de");
+    EXPECT_EQ(*(kitchenSink.script), "Hans");
+    EXPECT_EQ(*(kitchenSink.region), "US");
+}
+
+TEST(LanguageTag, Generation) {
+    LanguageTag deHansUS({"de"}, {"hans"}, {"US"});
+    EXPECT_EQ(deHansUS.toBCP47(), "de-hans-US");
+    
+    LanguageTag deUS({"de"}, {}, {"US"});
+    EXPECT_EQ(deUS.toBCP47(), "de-US");
+    
+    LanguageTag noLanguage({}, {"hans"}, {"US"});
+    EXPECT_EQ(noLanguage.toBCP47(), "");
+    
+    LanguageTag deHans({"de"}, {"hans"}, {});
+    EXPECT_EQ(deHans.toBCP47(), "de-hans");
+    
+    LanguageTag de({"de"}, {}, {});
+    EXPECT_EQ(de.toBCP47(), "de");
+}
+