Locale-free default Collator implementation using nunicode.

Remove Qt-specific implementation -- for now just have it use default.

5 files changed, 77 insertions, 66 deletions

diff --git a/platform/android/config.cmake b/platform/android/config.cmake
index 40be54e934..6ddbd67c16 100644
--- a/platform/android/config.cmake
+++ b/platform/android/config.cmake
@@ -21,7 +21,7 @@ set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections -Wl,--ve
 set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} -Wl,--gc-sections -Wl,--version-script=${CMAKE_SOURCE_DIR}/platform/android/version-script")
 
 mason_use(jni.hpp VERSION 3.0.0 HEADER_ONLY)
-mason_use(nunicode VERSION 1.7.1)
+mason_use(nunicode VERSION 1.8)
 mason_use(sqlite VERSION 3.14.2)
 mason_use(gtest VERSION 1.8.0)
 mason_use(icu VERSION 58.1-min-size)
diff --git a/platform/default/collator.cpp b/platform/default/collator.cpp
index 7cfd5bb159..e203aeb417 100644
--- a/platform/default/collator.cpp
+++ b/platform/default/collator.cpp
@@ -1,30 +1,96 @@
 #include <mbgl/style/expression/collator.hpp>
+#include <mbgl/util/platform.hpp>
 
-// TODO: Currently a stub, but should hook up to a mason-bundled ICU for
-// collator functionality. Only used on platforms that don't have natively
-// exposed collators, because the bundling is expensive.
+#define NU_WITH_UTF8
+#define NU_WITH_Z_COLLATION
+#define NU_WITH_UNACCENT
+#include <libnu/libnu.h>
+
+#include <sstream>
+
+/*
+    The default implementation of Collator ignores locale.
+    Case sensitivity and collation order are based on
+    Default Unicode Collation Element Table (DUCET).
+
+    Diacritic-insensitivity is implemented with nunicode's
+    non-standard "unaccent" functionality, which is tailored
+    to European languages.
+
+    It would be possible to implement locale awareness using ICU,
+    but would require bundling locale data.
+*/
 
 namespace mbgl {
 namespace style {
 namespace expression {
 
+std::string unaccent(const std::string& str)
+{
+    std::stringstream output;
+    char const *itr = str.c_str(), *nitr;
+    char const *end = itr + str.length();
+    char lo[5] = { 0 };
+
+    for (; itr < end; itr = nitr)
+    {
+        uint32_t code_point = 0;
+        char const* buf = nullptr;
+
+        nitr = _nu_tounaccent(itr, end, nu_utf8_read, &code_point, &buf, nullptr);
+        if (buf != nullptr)
+        {
+            do
+            {
+                buf = NU_CASEMAP_DECODING_FUNCTION(buf, &code_point);
+                if (code_point == 0) break;
+                output.write(lo, nu_utf8_write(code_point, lo) - lo);
+            }
+            while (code_point != 0);
+        }
+        else
+        {
+            output.write(itr, nitr - itr);
+        }
+    }
+
+    return output.str();
+}
+
 class Collator::Impl {
 public:
-    Impl(bool, bool, optional<std::string>)
+    Impl(bool caseSensitive_, bool diacriticSensitive_, optional<std::string>)
+        : caseSensitive(caseSensitive_)
+        , diacriticSensitive(diacriticSensitive_)
     {}
 
-    bool operator==(const Impl&) const {
-        return false;
+    bool operator==(const Impl& other) const {
+        return caseSensitive == other.caseSensitive &&
+            diacriticSensitive == other.diacriticSensitive;
     }
 
-    int compare(const std::string&, const std::string&) const {
-        return 0;
+    int compare(const std::string& lhs, const std::string& rhs) const {
+        if (caseSensitive && diacriticSensitive) {
+            return nu_strcoll(lhs.c_str(), rhs.c_str(),
+                              nu_utf8_read, nu_utf8_read);
+        } else if (!caseSensitive && diacriticSensitive) {
+            return nu_strcasecoll(lhs.c_str(), rhs.c_str(),
+                                  nu_utf8_read, nu_utf8_read);
+        } else if (caseSensitive && !diacriticSensitive) {
+            return nu_strcoll(unaccent(lhs).c_str(), unaccent(rhs).c_str(),
+                              nu_utf8_read, nu_utf8_read);
+        } else {
+            return nu_strcasecoll(unaccent(lhs).c_str(), unaccent(rhs).c_str(),
+                                  nu_utf8_read, nu_utf8_read);
+        }
     }
 
     std::string resolvedLocale() const {
         return "";
     }
 private:
+    bool caseSensitive;
+    bool diacriticSensitive;
 };
 
 
diff --git a/platform/linux/config.cmake b/platform/linux/config.cmake
index 917a5a2722..4c29b5d4b2 100644
--- a/platform/linux/config.cmake
+++ b/platform/linux/config.cmake
@@ -1,7 +1,7 @@
 mason_use(glfw VERSION 2017-07-13-67c9155)
 mason_use(sqlite VERSION 3.14.2)
 mason_use(libuv VERSION 1.9.1)
-mason_use(nunicode VERSION 1.7.1)
+mason_use(nunicode VERSION 1.8)
 mason_use(libpng VERSION 1.6.25)
 mason_use(libjpeg-turbo VERSION 1.5.0)
 mason_use(webp VERSION 0.5.1)
diff --git a/platform/qt/config.cmake b/platform/qt/config.cmake
index 0d9f504f11..7f35bfa6f3 100644
--- a/platform/qt/config.cmake
+++ b/platform/qt/config.cmake
@@ -44,13 +44,12 @@ macro(mbgl_platform_core)
 
     if(NOT WITH_QT_I18N)
         target_sources(mbgl-core PRIVATE platform/default/bidi.cpp)
-        target_sources(mbgl-core PRIVATE platform/default/collator.cpp)
         target_add_mason_package(mbgl-core PRIVATE icu)
     else()
         target_sources(mbgl-core PRIVATE platform/qt/src/bidi.cpp)
-        target_sources(mbgl-core PRIVATE platform/qt/src/collator.cpp)
     endif()
 
+    target_sources(mbgl-core PRIVATE platform/default/collator.cpp)
     target_sources(mbgl-core PRIVATE platform/default/local_glyph_rasterizer.cpp)
 
     if (CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows")
diff --git a/platform/qt/src/collator.cpp b/platform/qt/src/collator.cpp
deleted file mode 100644
index e57e677805..0000000000
--- a/platform/qt/src/collator.cpp
+++ /dev/null
@@ -1,54 +0,0 @@
-#include <mbgl/style/expression/collator.hpp>
-
-// TODO: This is the Collator implementation for Qt when WITH_QT_I18N flag is set
-// ie we can't link directly to ICU. Since Qt doesn't expose what we need to implement
-// Collator, we have to figure out what to do here (maybe pass through to default string
-// comparison?)
-
-namespace mbgl {
-namespace style {
-namespace expression {
-
-class Collator::Impl {
-public:
-    Impl(bool, bool, optional<std::string>)
-    {}
-
-    bool operator==(const Impl&) const {
-        return false;
-    }
-
-    int compare(const std::string&, const std::string&) const {
-        return 0;
-    }
-
-    std::string resolvedLocale() const {
-        return "";
-    }
-private:
-};
-
-
-Collator::Collator(bool caseSensitive, bool diacriticSensitive, optional<std::string> locale_)
-    : impl(std::make_unique<Impl>(caseSensitive, diacriticSensitive, std::move(locale_)))
-{}
-
-bool Collator::operator==(const Collator& other) const {
-    return *impl == *(other.impl);
-}
-
-int Collator::compare(const std::string& lhs, const std::string& rhs) const {
-    return impl->compare(lhs, rhs);
-}
-
-std::string Collator::resolvedLocale() const {
-    return impl->resolvedLocale();
-}
-
-mbgl::Value Collator::serialize() const {
-    return mbgl::Value(true);
-}
-
-} // namespace expression
-} // namespace style
-} // namespace mbgl