[ios, macos] Convert token strings to expressions on input

Removes mgl_expressionByReplacingTokensWithKeyPaths and associated code. Converting on output is no longer necessary: from the prior commit, core converts token strings to expressions at parse time; all that's necessary is to ensure that the runtime styling API does so as well.

11 files changed, 79 insertions, 174 deletions

diff --git a/platform/darwin/src/MGLBackgroundStyleLayer.mm b/platform/darwin/src/MGLBackgroundStyleLayer.mm
index acea3441fa..cc09ce4e11 100644
--- a/platform/darwin/src/MGLBackgroundStyleLayer.mm
+++ b/platform/darwin/src/MGLBackgroundStyleLayer.mm
@@ -116,8 +116,7 @@
     if (propertyValue.isUndefined()) {
         propertyValue = self.rawLayer->getDefaultBackgroundPattern();
     }
-    NSExpression *expression = MGLStyleValueTransformer<std::string, NSString *>().toExpression(propertyValue);
-    return expression.mgl_expressionByReplacingTokensWithKeyPaths;
+    return MGLStyleValueTransformer<std::string, NSString *>().toExpression(propertyValue);
 }
 
 - (void)setBackgroundPatternTransition:(MGLTransition )transition {
diff --git a/platform/darwin/src/MGLFillExtrusionStyleLayer.mm b/platform/darwin/src/MGLFillExtrusionStyleLayer.mm
index 4f3bfee18e..1cb059fae0 100644
--- a/platform/darwin/src/MGLFillExtrusionStyleLayer.mm
+++ b/platform/darwin/src/MGLFillExtrusionStyleLayer.mm
@@ -231,8 +231,7 @@ namespace mbgl {
     if (propertyValue.isUndefined()) {
         propertyValue = self.rawLayer->getDefaultFillExtrusionPattern();
     }
-    NSExpression *expression = MGLStyleValueTransformer<std::string, NSString *>().toExpression(propertyValue);
-    return expression.mgl_expressionByReplacingTokensWithKeyPaths;
+    return MGLStyleValueTransformer<std::string, NSString *>().toExpression(propertyValue);
 }
 
 - (void)setFillExtrusionPatternTransition:(MGLTransition )transition {
diff --git a/platform/darwin/src/MGLFillStyleLayer.mm b/platform/darwin/src/MGLFillStyleLayer.mm
index 12e3643ce6..2fb9849c42 100644
--- a/platform/darwin/src/MGLFillStyleLayer.mm
+++ b/platform/darwin/src/MGLFillStyleLayer.mm
@@ -220,8 +220,7 @@ namespace mbgl {
     if (propertyValue.isUndefined()) {
         propertyValue = self.rawLayer->getDefaultFillPattern();
     }
-    NSExpression *expression = MGLStyleValueTransformer<std::string, NSString *>().toExpression(propertyValue);
-    return expression.mgl_expressionByReplacingTokensWithKeyPaths;
+    return MGLStyleValueTransformer<std::string, NSString *>().toExpression(propertyValue);
 }
 
 - (void)setFillPatternTransition:(MGLTransition )transition {
diff --git a/platform/darwin/src/MGLLineStyleLayer.mm b/platform/darwin/src/MGLLineStyleLayer.mm
index 84412073cd..e65a474829 100644
--- a/platform/darwin/src/MGLLineStyleLayer.mm
+++ b/platform/darwin/src/MGLLineStyleLayer.mm
@@ -390,8 +390,7 @@ namespace mbgl {
     if (propertyValue.isUndefined()) {
         propertyValue = self.rawLayer->getDefaultLinePattern();
     }
-    NSExpression *expression = MGLStyleValueTransformer<std::string, NSString *>().toExpression(propertyValue);
-    return expression.mgl_expressionByReplacingTokensWithKeyPaths;
+    return MGLStyleValueTransformer<std::string, NSString *>().toExpression(propertyValue);
 }
 
 - (void)setLinePatternTransition:(MGLTransition )transition {
diff --git a/platform/darwin/src/MGLStyleLayer.mm.ejs b/platform/darwin/src/MGLStyleLayer.mm.ejs
index 1e760749e9..b37fbc04bf 100644
--- a/platform/darwin/src/MGLStyleLayer.mm.ejs
+++ b/platform/darwin/src/MGLStyleLayer.mm.ejs
@@ -121,6 +121,16 @@ namespace mbgl {
 - (void)set<%- camelize(property.name) %>:(NSExpression *)<%- objCName(property) %> {
     MGLAssertStyleLayerIsValid();
 
+<% if (property.tokens) { -%>
+    if (<%- objCName(property) %> && <%- objCName(property) %>.expressionType == NSConstantValueExpressionType) {
+        std::string string = ((NSString *)<%- objCName(property) %>.constantValue).UTF8String;
+        if (mbgl::style::conversion::hasTokens(string)) {
+            self.rawLayer->set<%- camelize(originalPropertyName(property)) %>(mbgl::style::DataDrivenPropertyValue<std::string>(
+                mbgl::style::conversion::convertTokenStringToExpression(string)));
+            return;
+        }
+    }
+<% } -%>
 <% if (isDataDriven(property)) { -%>
     auto mbglValue = MGLStyleValueTransformer<<%- valueTransformerArguments(property).join(', ') %>>().toPropertyValue<mbgl::style::DataDrivenPropertyValue<<%- valueTransformerArguments(property)[0] %>>>(<%- objCName(property) %>);
 <% } else { -%>
@@ -136,12 +146,7 @@ namespace mbgl {
     if (propertyValue.isUndefined()) {
         propertyValue = self.rawLayer->getDefault<%- camelize(originalPropertyName(property)) %>();
     }
-<% if (property.type === 'string') { -%>
-    NSExpression *expression = MGLStyleValueTransformer<<%- valueTransformerArguments(property).join(', ') %>>().toExpression(propertyValue);
-    return expression.mgl_expressionByReplacingTokensWithKeyPaths;
-<% } else { -%>
     return MGLStyleValueTransformer<<%- valueTransformerArguments(property).join(', ') %>>().toExpression(propertyValue);
-<% } -%>
 }
 
 <% if (property.original) { -%>
@@ -183,12 +188,7 @@ namespace mbgl {
     if (propertyValue.isUndefined()) {
         propertyValue = self.rawLayer->getDefault<%- camelize(originalPropertyName(property)) %>();
     }
-<% if (property.type === 'string') { -%>
-    NSExpression *expression = MGLStyleValueTransformer<<%- valueTransformerArguments(property).join(', ') %>>().toExpression(propertyValue);
-    return expression.mgl_expressionByReplacingTokensWithKeyPaths;
-<% } else { -%>
     return MGLStyleValueTransformer<<%- valueTransformerArguments(property).join(', ') %>>().toExpression(propertyValue);
-<% } -%>
 }
 <% if (property["transition"]) { -%>
 
diff --git a/platform/darwin/src/MGLSymbolStyleLayer.mm b/platform/darwin/src/MGLSymbolStyleLayer.mm
index 7ec7816c3b..f5522b800d 100644
--- a/platform/darwin/src/MGLSymbolStyleLayer.mm
+++ b/platform/darwin/src/MGLSymbolStyleLayer.mm
@@ -222,6 +222,14 @@ namespace mbgl {
 - (void)setIconImageName:(NSExpression *)iconImageName {
     MGLAssertStyleLayerIsValid();
 
+    if (iconImageName && iconImageName.expressionType == NSConstantValueExpressionType) {
+        std::string string = ((NSString *)iconImageName.constantValue).UTF8String;
+        if (mbgl::style::conversion::hasTokens(string)) {
+            self.rawLayer->setIconImage(mbgl::style::DataDrivenPropertyValue<std::string>(
+                mbgl::style::conversion::convertTokenStringToExpression(string)));
+            return;
+        }
+    }
     auto mbglValue = MGLStyleValueTransformer<std::string, NSString *>().toPropertyValue<mbgl::style::DataDrivenPropertyValue<std::string>>(iconImageName);
     self.rawLayer->setIconImage(mbglValue);
 }
@@ -233,8 +241,7 @@ namespace mbgl {
     if (propertyValue.isUndefined()) {
         propertyValue = self.rawLayer->getDefaultIconImage();
     }
-    NSExpression *expression = MGLStyleValueTransformer<std::string, NSString *>().toExpression(propertyValue);
-    return expression.mgl_expressionByReplacingTokensWithKeyPaths;
+    return MGLStyleValueTransformer<std::string, NSString *>().toExpression(propertyValue);
 }
 
 - (void)setIconImage:(NSExpression *)iconImage {
@@ -568,6 +575,14 @@ namespace mbgl {
 - (void)setText:(NSExpression *)text {
     MGLAssertStyleLayerIsValid();
 
+    if (text && text.expressionType == NSConstantValueExpressionType) {
+        std::string string = ((NSString *)text.constantValue).UTF8String;
+        if (mbgl::style::conversion::hasTokens(string)) {
+            self.rawLayer->setTextField(mbgl::style::DataDrivenPropertyValue<std::string>(
+                mbgl::style::conversion::convertTokenStringToExpression(string)));
+            return;
+        }
+    }
     auto mbglValue = MGLStyleValueTransformer<std::string, NSString *>().toPropertyValue<mbgl::style::DataDrivenPropertyValue<std::string>>(text);
     self.rawLayer->setTextField(mbglValue);
 }
@@ -579,8 +594,7 @@ namespace mbgl {
     if (propertyValue.isUndefined()) {
         propertyValue = self.rawLayer->getDefaultTextField();
     }
-    NSExpression *expression = MGLStyleValueTransformer<std::string, NSString *>().toExpression(propertyValue);
-    return expression.mgl_expressionByReplacingTokensWithKeyPaths;
+    return MGLStyleValueTransformer<std::string, NSString *>().toExpression(propertyValue);
 }
 
 - (void)setTextField:(NSExpression *)textField {
diff --git a/platform/darwin/src/NSExpression+MGLAdditions.mm b/platform/darwin/src/NSExpression+MGLAdditions.mm
index 5cb69d11dc..2a4c80bee6 100644
--- a/platform/darwin/src/NSExpression+MGLAdditions.mm
+++ b/platform/darwin/src/NSExpression+MGLAdditions.mm
@@ -389,112 +389,6 @@ const MGLExpressionInterpolationMode MGLExpressionInterpolationModeCubicBezier =
     return {};
 }
 
-// Selectors of functions that can contain tokens in arguments.
-static NSArray * const MGLTokenizedFunctions = @[
-    @"mgl_interpolateWithCurveType:parameters:stops:",
-    @"mgl_interpolate:withCurveType:parameters:stops:",
-    @"mgl_stepWithMinimum:stops:",
-    @"mgl_step:from:stops:",
-];
-
-/**
- Returns a copy of the given collection with tokens replaced by key path
- expressions.
- 
- If no replacements take place, this method returns the original collection.
- */
-NSArray<NSExpression *> *MGLCollectionByReplacingTokensWithKeyPaths(NSArray<NSExpression *> *collection) {
-    __block NSMutableArray *upgradedCollection;
-    [collection enumerateObjectsUsingBlock:^(NSExpression * _Nonnull item, NSUInteger idx, BOOL * _Nonnull stop) {
-        NSExpression *upgradedItem = item.mgl_expressionByReplacingTokensWithKeyPaths;
-        if (upgradedItem != item) {
-            if (!upgradedCollection) {
-                upgradedCollection = [collection mutableCopy];
-            }
-            upgradedCollection[idx] = upgradedItem;
-        }
-    }];
-    return upgradedCollection ?: collection;
-};
-
-/**
- Returns a copy of the given stop dictionary with tokens replaced by key path
- expressions.
- 
- If no replacements take place, this method returns the original stop
- dictionary.
- */
-NSDictionary<NSNumber *, NSExpression *> *MGLStopDictionaryByReplacingTokensWithKeyPaths(NSDictionary<NSNumber *, NSExpression *> *stops) {
-    __block NSMutableDictionary *upgradedStops;
-    [stops enumerateKeysAndObjectsUsingBlock:^(id _Nonnull zoomLevel, NSExpression * _Nonnull value, BOOL * _Nonnull stop) {
-        if (![value isKindOfClass:[NSExpression class]]) {
-            value = [NSExpression expressionForConstantValue:value];
-        }
-        NSExpression *upgradedValue = value.mgl_expressionByReplacingTokensWithKeyPaths;
-        if (upgradedValue != value) {
-            if (!upgradedStops) {
-                upgradedStops = [stops mutableCopy];
-            }
-            upgradedStops[zoomLevel] = upgradedValue;
-        }
-    }];
-    return upgradedStops ?: stops;
-};
-
-- (NSExpression *)mgl_expressionByReplacingTokensWithKeyPaths {
-    switch (self.expressionType) {
-        case NSConstantValueExpressionType: {
-            NSString *constantValue = self.constantValue;
-            if ([constantValue isKindOfClass:[NSString class]] &&
-                [constantValue containsString:@"{"] && [constantValue containsString:@"}"]) {
-                NSMutableArray *components = [NSMutableArray array];
-                NSScanner *scanner = [NSScanner scannerWithString:constantValue];
-                scanner.charactersToBeSkipped = nil;
-                while (!scanner.isAtEnd) {
-                    NSString *string;
-                    if ([scanner scanUpToString:@"{" intoString:&string]) {
-                        [components addObject:[NSExpression expressionForConstantValue:string]];
-                    }
-                    
-                    NSString *token;
-                    if ([scanner scanString:@"{" intoString:NULL]
-                        && [scanner scanUpToString:@"}" intoString:&token]
-                        && [scanner scanString:@"}" intoString:NULL]) {
-                        [components addObject:[NSExpression expressionForKeyPath:token]];
-                    }
-                }
-                if (components.count == 1) {
-                    return components.firstObject;
-                }
-                return [NSExpression expressionForFunction:@"mgl_join:"
-                                                 arguments:@[[NSExpression expressionForAggregate:components]]];
-            }
-            NSDictionary *stops = self.constantValue;
-            if ([stops isKindOfClass:[NSDictionary class]]) {
-                NSDictionary *localizedStops = MGLStopDictionaryByReplacingTokensWithKeyPaths(stops);
-                if (localizedStops != stops) {
-                    return [NSExpression expressionForConstantValue:localizedStops];
-                }
-            }
-            return self;
-        }
-            
-        case NSFunctionExpressionType: {
-            if ([MGLTokenizedFunctions containsObject:self.function]) {
-                NSArray *arguments = self.arguments;
-                NSArray *localizedArguments = MGLCollectionByReplacingTokensWithKeyPaths(arguments);
-                if (localizedArguments != arguments) {
-                    return [NSExpression expressionForFunction:self.operand selectorName:self.function arguments:localizedArguments];
-                }
-            }
-            return self;
-        }
-            
-        default:
-            return self;
-    }
-}
-
 @end
 
 @implementation NSObject (MGLExpressionAdditions)
diff --git a/platform/darwin/src/NSExpression+MGLPrivateAdditions.h b/platform/darwin/src/NSExpression+MGLPrivateAdditions.h
index a1948f9e45..656d78dd17 100644
--- a/platform/darwin/src/NSExpression+MGLPrivateAdditions.h
+++ b/platform/darwin/src/NSExpression+MGLPrivateAdditions.h
@@ -27,11 +27,6 @@ NS_ASSUME_NONNULL_BEGIN
 @property (nonatomic, readonly) mbgl::FeatureIdentifier mgl_featureIdentifier;
 @property (nonatomic, readonly) std::vector<mbgl::FeatureIdentifier> mgl_aggregateFeatureIdentifier;
 
-/**
- Returns a copy of the receiver with tokens replaced by key path expressions.
- */
-- (NSExpression *)mgl_expressionByReplacingTokensWithKeyPaths;
-
 @end
 
 @interface NSNull (MGLExpressionAdditions)
diff --git a/platform/darwin/test/MGLExpressionTests.mm b/platform/darwin/test/MGLExpressionTests.mm
index 01a279950b..5ae98e5244 100644
--- a/platform/darwin/test/MGLExpressionTests.mm
+++ b/platform/darwin/test/MGLExpressionTests.mm
@@ -995,47 +995,6 @@ using namespace std::string_literals;
 
 #pragma mark - Localization tests
 
-- (void)testTokenReplacement {
-    {
-        NSExpression *original = MGLConstantExpression(@"");
-        NSExpression *expected = original;
-        XCTAssertEqualObjects(original.mgl_expressionByReplacingTokensWithKeyPaths, expected);
-    }
-    {
-        NSExpression *original = MGLConstantExpression(@"{");
-        NSExpression *expected = original;
-        XCTAssertEqualObjects(original.mgl_expressionByReplacingTokensWithKeyPaths, expected);
-    }
-    {
-        NSExpression *original = MGLConstantExpression(@"{token");
-        NSExpression *expected = original;
-        XCTAssertEqualObjects(original.mgl_expressionByReplacingTokensWithKeyPaths, expected);
-    }
-    {
-        NSExpression *original = MGLConstantExpression(@"{token}");
-        NSExpression *expected = [NSExpression expressionForKeyPath:@"token"];
-        XCTAssertEqualObjects(original.mgl_expressionByReplacingTokensWithKeyPaths, expected);
-    }
-    {
-        NSExpression *original = MGLConstantExpression(@"{token} {token}");
-        NSExpression *expected = [NSExpression expressionWithFormat:@"mgl_join({token, ' ', token})"];
-        XCTAssertEqualObjects(original.mgl_expressionByReplacingTokensWithKeyPaths, expected);
-    }
-    {
-        NSExpression *original = [NSExpression expressionWithFormat:@"mgl_step:from:stops:($zoomLevel, '{short}', %@)", @{
-            @1: MGLConstantExpression(@"{short}"),
-            @2: @"…",
-            @3: @"{long}",
-        }];
-        NSExpression *expected = [NSExpression expressionWithFormat:@"mgl_step:from:stops:($zoomLevel, short, %@)", @{
-            @1: [NSExpression expressionForKeyPath:@"short"],
-            @2: @"…",
-            @3: [NSExpression expressionForKeyPath:@"long"],
-        }];
-        XCTAssertEqualObjects(original.mgl_expressionByReplacingTokensWithKeyPaths, expected);
-    }
-}
-
 - (void)testLocalization {
     {
         NSExpression *original = MGLConstantExpression(@"");
diff --git a/platform/darwin/test/MGLStyleLayerTests.mm.ejs b/platform/darwin/test/MGLStyleLayerTests.mm.ejs
index 44996fa125..02c4a04e6f 100644
--- a/platform/darwin/test/MGLStyleLayerTests.mm.ejs
+++ b/platform/darwin/test/MGLStyleLayerTests.mm.ejs
@@ -156,6 +156,23 @@
         XCTAssertEqual(<%- camelizeWithLeadingLowercase(originalPropertyName(property)) %>Transition.delay, transitionTest.delay);
         XCTAssertEqual(<%- camelizeWithLeadingLowercase(originalPropertyName(property)) %>Transition.duration, transitionTest.duration);
 <% } -%>
+<% if (property.tokens) { -%>
+
+        // Tokens test
+        layer.<%- objCName(property) %> = [NSExpression expressionForConstantValue:@"{token}"];
+
+        {
+            using namespace mbgl::style::expression::dsl;
+            propertyValue = mbgl::style::PropertyExpression<<%- mbglType(property) %>>(
+                toString(get(literal("token")))
+            );
+        }
+
+        XCTAssertEqual(rawLayer->get<%- camelize(originalPropertyName(property)) %>(), propertyValue,
+                       @"Setting <%- objCName(property) %> to a constant string with tokens should convert to an expression.");
+        XCTAssertEqualObjects(layer.<%- objCName(property) %>, [NSExpression expressionWithFormat:@"CAST(token, \"NSString\")"],
+                              @"Setting <%- objCName(property) %> to a constant string with tokens should convert to an expression.");
+<% } -%>
     }
 <% } -%>
 }
diff --git a/platform/darwin/test/MGLSymbolStyleLayerTests.mm b/platform/darwin/test/MGLSymbolStyleLayerTests.mm
index 7566617872..8643e8388b 100644
--- a/platform/darwin/test/MGLSymbolStyleLayerTests.mm
+++ b/platform/darwin/test/MGLSymbolStyleLayerTests.mm
@@ -211,6 +211,21 @@
                       @"Unsetting iconImageName should return icon-image to the default value.");
         XCTAssertEqualObjects(layer.iconImageName, defaultExpression,
                               @"iconImageName should return the default value after being unset.");
+
+        // Tokens test
+        layer.iconImageName = [NSExpression expressionForConstantValue:@"{token}"];
+
+        {
+            using namespace mbgl::style::expression::dsl;
+            propertyValue = mbgl::style::PropertyExpression<std::string>(
+                toString(get(literal("token")))
+            );
+        }
+
+        XCTAssertEqual(rawLayer->getIconImage(), propertyValue,
+                       @"Setting iconImageName to a constant string with tokens should convert to an expression.");
+        XCTAssertEqualObjects(layer.iconImageName, [NSExpression expressionWithFormat:@"CAST(token, \"NSString\")"],
+                              @"Setting iconImageName to a constant string with tokens should convert to an expression.");
     }
 
     // icon-offset
@@ -1065,6 +1080,21 @@
                       @"Unsetting text should return text-field to the default value.");
         XCTAssertEqualObjects(layer.text, defaultExpression,
                               @"text should return the default value after being unset.");
+
+        // Tokens test
+        layer.text = [NSExpression expressionForConstantValue:@"{token}"];
+
+        {
+            using namespace mbgl::style::expression::dsl;
+            propertyValue = mbgl::style::PropertyExpression<std::string>(
+                toString(get(literal("token")))
+            );
+        }
+
+        XCTAssertEqual(rawLayer->getTextField(), propertyValue,
+                       @"Setting text to a constant string with tokens should convert to an expression.");
+        XCTAssertEqualObjects(layer.text, [NSExpression expressionWithFormat:@"CAST(token, \"NSString\")"],
+                              @"Setting text to a constant string with tokens should convert to an expression.");
     }
 
     // text-allow-overlap