separate out token replacer as a standalone function in preparation for #300

3 files changed, 58 insertions, 31 deletions

diff --git a/include/llmr/style/value.hpp b/include/llmr/style/value.hpp
index c6d6f50b94..8734386253 100644
--- a/include/llmr/style/value.hpp
+++ b/include/llmr/style/value.hpp
@@ -10,6 +10,12 @@ typedef util::variant<bool, int64_t, uint64_t, double, std::string> Value;
 
 std::string toString(const Value &value);
 
+inline std::string& operator+=(std::string& out, const Value& value) {
+    out += toString(value);
+    return out;
+}
+
+
 Value parseValue(pbf data);
 
 namespace util {
diff --git a/include/llmr/util/token.hpp b/include/llmr/util/token.hpp
new file mode 100644
index 0000000000..db5b5fa0c3
--- /dev/null
+++ b/include/llmr/util/token.hpp
@@ -0,0 +1,49 @@
+#ifndef LLMR_UTIL_TOKEN
+#define LLMR_UTIL_TOKEN
+
+#ifdef __linux__
+#include <boost/regex.hpp>
+namespace regex_impl = boost;
+#else
+#include <regex>
+namespace regex_impl = std;
+#endif
+
+namespace llmr {
+namespace util {
+
+namespace detail {
+const regex_impl::regex tokenRegex("\\{\\{(\\w+)\\}\\}");
+const regex_impl::sregex_token_iterator tokensEnd = regex_impl::sregex_token_iterator();
+}
+
+template <typename Tokens>
+std::string replaceTokens(const std::string &source, const Tokens &tokens) {
+    std::string result;
+    result.reserve(source.size());
+
+    bool token = false;
+    for (auto token_it = regex_impl::sregex_token_iterator(source.begin(), source.end(),
+                                                           detail::tokenRegex, {-1, 1});
+         token_it != detail::tokensEnd; ++token_it, token = !token) {
+        if (!token_it->matched) {
+            continue;
+        }
+
+        if (token) {
+            const auto it_prop = tokens.find(token_it->str());
+            if (it_prop != tokens.end()) {
+                result += it_prop->second;
+            }
+        } else {
+            result += token_it->str();
+        }
+    }
+
+    return result;
+}
+
+} // end namespace util
+} // end namespace llmr
+
+#endif
diff --git a/src/map/tile_parser.cpp b/src/map/tile_parser.cpp
index 2c51aa3e50..21d2bf1bfc 100644
--- a/src/map/tile_parser.cpp
+++ b/src/map/tile_parser.cpp
@@ -9,6 +9,7 @@
 #include <llmr/renderer/raster_bucket.hpp>
 #include <llmr/util/raster.hpp>
 #include <llmr/util/constants.hpp>
+#include <llmr/util/token.hpp>
 #include <llmr/geometry/glyph_atlas.hpp>
 #include <llmr/text/glyph_store.hpp>
 #include <llmr/text/glyph.hpp>
@@ -195,9 +196,6 @@ std::unique_ptr<Bucket> TileParser::createTextBucket(const VectorTileLayer &laye
         tile.textVertexBuffer, tile.triangleElementsBuffer, bucket_desc, placement);
 
     util::utf8_to_utf32 ucs4conv;
-    regex_impl::regex token_regex("\\{\\{(\\w+)\\}\\}");
-    const auto tokens_end = regex_impl::sregex_token_iterator();
-
 
     std::vector<std::pair<std::u32string, pbf>> labels;
 
@@ -211,34 +209,8 @@ std::unique_ptr<Bucket> TileParser::createTextBucket(const VectorTileLayer &laye
                 return nullptr;
             VectorTileFeature feature{feature_pbf, layer};
 
-            const std::string &field = bucket_desc.geometry.field;
-            std::string source_string;
-            source_string.reserve(field.size());
-
-            bool token = false;
-            for (auto token_it = regex_impl::sregex_token_iterator(field.begin(), field.end(), token_regex, {-1, 1}); token_it != tokens_end; ++token_it, token = !token) {
-                if (!token_it->matched) {
-                    continue;
-                }
-
-                if (token) {
-                    auto it_prop = feature.properties.find(token_it->str());
-                    if (it_prop == feature.properties.end()) {
-                        // feature does not have the correct property
-                        if (debug::labelTextMissingWarning) {
-                            fprintf(stderr,
-                                    "[WARNING] feature doesn't have property '%s' required for labelling\n",
-                                    token_it->str().c_str());
-                        }
-                        continue;
-                    }
-                    source_string += toString(it_prop->second);
-                } else {
-                    source_string += token_it->str();
-                }
-            }
-
-            const std::u32string string = ucs4conv.convert(source_string);
+            const std::u32string string = ucs4conv.convert(
+                util::replaceTokens(bucket_desc.geometry.field, feature.properties));
 
             // Loop through all characters of this text and collect unique codepoints.
             for (uint32_t chr : string) {