[ios, macos] Test token replacement, localization

Added unit tests of token replacement and localization of expressions. Only NSExpression is responsible for resolving the preferred language now, since NSLocale tends to tack a region code onto the locale identifier and the NSExpression method can be called independently anyways. Added a private variation of +[MGLVectorTileSource preferredMapboxStreetsLanguage] that takes an array of preferred languages. Fixed localization of non-expressions in stop dictionaries.

7 files changed, 199 insertions, 35 deletions

diff --git a/platform/darwin/src/MGLStyle.mm b/platform/darwin/src/MGLStyle.mm
index cb9e81d3d7..867ac6c451 100644
--- a/platform/darwin/src/MGLStyle.mm
+++ b/platform/darwin/src/MGLStyle.mm
@@ -578,16 +578,6 @@ static_assert(6 == mbgl::util::default_styles::numOrderedStyles,
         }]];
     NSSet<NSString *> *streetsSourceIdentifiers = [streetsSources valueForKey:@"identifier"];
     
-    if (!locale) {
-        NSString *preferredLanguage = [MGLVectorTileSource preferredMapboxStreetsLanguage];
-        if (preferredLanguage) {
-            locale = [NSLocale localeWithLocaleIdentifier:preferredLanguage];
-        }
-    }
-    if (!locale) {
-        locale = [NSLocale localeWithLocaleIdentifier:@"mul"];
-    }
-    
     for (MGLSymbolStyleLayer *layer in self.layers) {
         if (![layer isKindOfClass:[MGLSymbolStyleLayer class]]) {
             continue;
diff --git a/platform/darwin/src/MGLVectorTileSource.mm b/platform/darwin/src/MGLVectorTileSource.mm
index e3666b6c25..c1f7267e4a 100644
--- a/platform/darwin/src/MGLVectorTileSource.mm
+++ b/platform/darwin/src/MGLVectorTileSource.mm
@@ -75,27 +75,54 @@
 
 @implementation MGLVectorTileSource (Private)
 
+/**
+ An array of locale codes with dedicated name fields in the Mapbox Streets
+ source.
+ 
+ https://www.mapbox.com/vector-tiles/mapbox-streets-v7/#overview
+ */
+static NSArray * const MGLMapboxStreetsLanguages = @[
+    @"ar", @"de", @"en", @"es", @"fr", @"pt", @"ru", @"zh", @"zh-Hans",
+];
+
+/**
+ Like `MGLMapboxStreetsLanguages`, but deanglicized for use with
+ `+[NSBundle preferredLocalizationsFromArray:forPreferences:]`.
+ */
+static NSArray * const MGLMapboxStreetsAlternativeLanguages = @[
+    @"mul", @"ar", @"de", @"es", @"fr", @"pt", @"ru", @"zh", @"zh-Hans",
+];
+
 + (NS_SET_OF(NSString *) *)mapboxStreetsLanguages {
-    // https://www.mapbox.com/vector-tiles/mapbox-streets-v7/#overview
     static dispatch_once_t onceToken;
     static NS_SET_OF(NSString *) *mapboxStreetsLanguages;
     dispatch_once(&onceToken, ^{
-        // https://www.mapbox.com/vector-tiles/mapbox-streets-v7/#overview
-        mapboxStreetsLanguages = [NSSet setWithObjects:@"ar", @"de", @"en", @"es", @"fr", @"pt", @"ru", @"zh", @"zh-Hans", nil];
+        mapboxStreetsLanguages = [NSSet setWithArray:MGLMapboxStreetsLanguages];
     });
     return mapboxStreetsLanguages;
 }
 
 + (NSString *)preferredMapboxStreetsLanguage {
-    NSArray<NSString *> *supportedLanguages = [MGLVectorTileSource mapboxStreetsLanguages].allObjects;
-    NSArray<NSString *> *preferredLanguages = [NSBundle preferredLocalizationsFromArray:supportedLanguages
-                                                                         forPreferences:[NSLocale preferredLanguages]];
+    return [self preferredMapboxStreetsLanguageForPreferences:[NSLocale preferredLanguages]];
+}
+
++ (NSString *)preferredMapboxStreetsLanguageForPreferences:(NSArray<NSString *> *)preferencesArray {
+    BOOL acceptsEnglish = [preferencesArray filteredArrayUsingPredicate:
+                           [NSPredicate predicateWithBlock:^BOOL(NSString * _Nullable language, NSDictionary<NSString *,id> * _Nullable bindings) {
+        return [[NSLocale localeWithLocaleIdentifier:language].languageCode isEqualToString:@"en"];
+    }]].count;
+    
+    NSArray<NSString *> *preferredLanguages = [NSBundle preferredLocalizationsFromArray:MGLMapboxStreetsAlternativeLanguages
+                                                                         forPreferences:preferencesArray];
     NSString *mostSpecificLanguage;
     for (NSString *language in preferredLanguages) {
         if (language.length > mostSpecificLanguage.length) {
             mostSpecificLanguage = language;
         }
     }
+    if ([mostSpecificLanguage isEqualToString:@"mul"]) {
+        return acceptsEnglish ? @"en" : nil;
+    }
     return mostSpecificLanguage;
 }
 
diff --git a/platform/darwin/src/MGLVectorTileSource_Private.h b/platform/darwin/src/MGLVectorTileSource_Private.h
index eecd67cfa8..77521869f1 100644
--- a/platform/darwin/src/MGLVectorTileSource_Private.h
+++ b/platform/darwin/src/MGLVectorTileSource_Private.h
@@ -9,6 +9,7 @@ NS_ASSUME_NONNULL_BEGIN
 + (NS_SET_OF(NSString *) *)mapboxStreetsLanguages;
 
 + (nullable NSString *)preferredMapboxStreetsLanguage;
++ (nullable NSString *)preferredMapboxStreetsLanguageForPreferences:(NSArray<NSString *> *)preferencesArray;
 
 - (NS_DICTIONARY_OF(NSString *, NSString *) *)localizedKeysByKeyForPreferredLanguage:(nullable NSString *)preferredLanguage;
 
diff --git a/platform/darwin/src/NSExpression+MGLAdditions.mm b/platform/darwin/src/NSExpression+MGLAdditions.mm
index 4c6e04ffc0..b18391018c 100644
--- a/platform/darwin/src/NSExpression+MGLAdditions.mm
+++ b/platform/darwin/src/NSExpression+MGLAdditions.mm
@@ -10,6 +10,7 @@
 #endif
 #import "NSPredicate+MGLAdditions.h"
 #import "NSValue+MGLStyleAttributeAdditions.h"
+#import "MGLVectorTileSource_Private.h"
 
 #import <objc/runtime.h>
 
@@ -1245,6 +1246,9 @@ NS_ARRAY_OF(NSExpression *) *MGLLocalizedCollection(NS_ARRAY_OF(NSExpression *)
 NS_DICTIONARY_OF(NSNumber *, NSExpression *) *MGLLocalizedStopDictionary(NS_DICTIONARY_OF(NSNumber *, NSExpression *) *stops, NSLocale * _Nullable locale) {
     __block NSMutableDictionary *localizedStops;
     [stops enumerateKeysAndObjectsUsingBlock:^(NSNumber * _Nonnull zoomLevel, NSExpression * _Nonnull value, BOOL * _Nonnull stop) {
+        if (![value isKindOfClass:[NSExpression class]]) {
+            value = [NSExpression expressionForConstantValue:value];
+        }
         NSExpression *localizedValue = [value mgl_expressionLocalizedIntoLocale:locale];
         if (localizedValue != value) {
             if (!localizedStops) {
@@ -1273,8 +1277,12 @@ NS_DICTIONARY_OF(NSNumber *, NSExpression *) *MGLLocalizedStopDictionary(NS_DICT
         case NSKeyPathExpressionType: {
             if ([self.keyPath isEqualToString:@"name"] || [self.keyPath hasPrefix:@"name_"]) {
                 NSString *localizedKeyPath = @"name";
-                if (locale && ![locale.localeIdentifier isEqualToString:@"mul"]) {
-                    localizedKeyPath = [NSString stringWithFormat:@"name_%@", locale.localeIdentifier];
+                if (![locale.localeIdentifier isEqualToString:@"mul"]) {
+                    NSArray *preferences = locale ? @[locale.localeIdentifier] : [NSLocale preferredLanguages];
+                    NSString *preferredLanguage = [MGLVectorTileSource preferredMapboxStreetsLanguageForPreferences:preferences];
+                    if (preferredLanguage) {
+                        localizedKeyPath = [NSString stringWithFormat:@"name_%@", preferredLanguage];
+                    }
                 }
                 return [NSExpression expressionForKeyPath:localizedKeyPath];
             }
diff --git a/platform/darwin/test/MGLExpressionTests.mm b/platform/darwin/test/MGLExpressionTests.mm
index 4d91c41bed..c91cf36d1a 100644
--- a/platform/darwin/test/MGLExpressionTests.mm
+++ b/platform/darwin/test/MGLExpressionTests.mm
@@ -870,32 +870,32 @@ using namespace std::string_literals;
 
 - (void)testTokenReplacement {
     {
-        NSExpression *tokenized = MGLConstantExpression(@"");
-        NSExpression *expected = tokenized;
-        XCTAssertEqualObjects(tokenized.mgl_expressionByReplacingTokensWithKeyPaths, expected);
+        NSExpression *original = MGLConstantExpression(@"");
+        NSExpression *expected = original;
+        XCTAssertEqualObjects(original.mgl_expressionByReplacingTokensWithKeyPaths, expected);
     }
     {
-        NSExpression *tokenized = MGLConstantExpression(@"{");
-        NSExpression *expected = tokenized;
-        XCTAssertEqualObjects(tokenized.mgl_expressionByReplacingTokensWithKeyPaths, expected);
+        NSExpression *original = MGLConstantExpression(@"{");
+        NSExpression *expected = original;
+        XCTAssertEqualObjects(original.mgl_expressionByReplacingTokensWithKeyPaths, expected);
     }
     {
-        NSExpression *tokenized = MGLConstantExpression(@"{token");
-        NSExpression *expected = tokenized;
-        XCTAssertEqualObjects(tokenized.mgl_expressionByReplacingTokensWithKeyPaths, expected);
+        NSExpression *original = MGLConstantExpression(@"{token");
+        NSExpression *expected = original;
+        XCTAssertEqualObjects(original.mgl_expressionByReplacingTokensWithKeyPaths, expected);
     }
     {
-        NSExpression *tokenized = MGLConstantExpression(@"{token}");
+        NSExpression *original = MGLConstantExpression(@"{token}");
         NSExpression *expected = [NSExpression expressionForKeyPath:@"token"];
-        XCTAssertEqualObjects(tokenized.mgl_expressionByReplacingTokensWithKeyPaths, expected);
+        XCTAssertEqualObjects(original.mgl_expressionByReplacingTokensWithKeyPaths, expected);
     }
     {
-        NSExpression *tokenized = MGLConstantExpression(@"{token} {token}");
+        NSExpression *original = MGLConstantExpression(@"{token} {token}");
         NSExpression *expected = [NSExpression expressionWithFormat:@"mgl_join({token, ' ', token})"];
-        XCTAssertEqualObjects(tokenized.mgl_expressionByReplacingTokensWithKeyPaths, expected);
+        XCTAssertEqualObjects(original.mgl_expressionByReplacingTokensWithKeyPaths, expected);
     }
     {
-        NSExpression *tokenized = [NSExpression expressionWithFormat:@"mgl_step:from:stops:($zoomLevel, '{short}', %@)", @{
+        NSExpression *original = [NSExpression expressionWithFormat:@"mgl_step:from:stops:($zoomLevel, '{short}', %@)", @{
             @1: MGLConstantExpression(@"{short}"),
             @2: @"…",
             @3: @"{long}",
@@ -905,7 +905,104 @@ using namespace std::string_literals;
             @2: @"…",
             @3: [NSExpression expressionForKeyPath:@"long"],
         }];
-        XCTAssertEqualObjects(tokenized.mgl_expressionByReplacingTokensWithKeyPaths, expected);
+        XCTAssertEqualObjects(original.mgl_expressionByReplacingTokensWithKeyPaths, expected);
+    }
+}
+
+- (void)testLocalization {
+    {
+        NSExpression *original = MGLConstantExpression(@"");
+        NSExpression *expected = original;
+        XCTAssertEqualObjects([original mgl_expressionLocalizedIntoLocale:nil], expected);
+    }
+    {
+        NSExpression *original = MGLConstantExpression(@"Old MacDonald");
+        NSExpression *expected = original;
+        XCTAssertEqualObjects([original mgl_expressionLocalizedIntoLocale:nil], expected);
+    }
+    {
+        NSExpression *original = MGLConstantExpression(@"{name_en}");
+        NSExpression *expected = original;
+        XCTAssertEqualObjects([original mgl_expressionLocalizedIntoLocale:nil], expected);
+    }
+    {
+        NSExpression *original = [NSExpression expressionForKeyPath:@"name_en"];
+        NSExpression *expected = original;
+        XCTAssertEqualObjects([original mgl_expressionLocalizedIntoLocale:nil], expected);
+    }
+    {
+        NSExpression *original = [NSExpression expressionForKeyPath:@"name_en"];
+        NSExpression *expected = [NSExpression expressionForKeyPath:@"name"];
+        XCTAssertEqualObjects([original mgl_expressionLocalizedIntoLocale:[NSLocale localeWithLocaleIdentifier:@"mul"]], expected);
+    }
+    {
+        NSExpression *original = [NSExpression expressionForKeyPath:@"name_en"];
+        NSExpression *expected = [NSExpression expressionForKeyPath:@"name_fr"];
+        XCTAssertEqualObjects([original mgl_expressionLocalizedIntoLocale:[NSLocale localeWithLocaleIdentifier:@"fr-CA"]], expected);
+    }
+    {
+        NSExpression *original = [NSExpression expressionForKeyPath:@"name_en"];
+        NSExpression *expected = [NSExpression expressionForKeyPath:@"name_zh-Hans"];
+        XCTAssertEqualObjects([original mgl_expressionLocalizedIntoLocale:[NSLocale localeWithLocaleIdentifier:@"zh-Hans"]], expected);
+    }
+    {
+        NSExpression *original = [NSExpression expressionForKeyPath:@"name_en"];
+        NSExpression *expected = [NSExpression expressionForKeyPath:@"name"];
+        XCTAssertEqualObjects([original mgl_expressionLocalizedIntoLocale:[NSLocale localeWithLocaleIdentifier:@"tlh"]], expected);
+    }
+    {
+        NSExpression *original = [NSExpression expressionWithFormat:@"mgl_step:from:stops:($zoomLevel, short, %@)", @{
+            @1: [NSExpression expressionForKeyPath:@"abbr"],
+            @2: @"…",
+            @3: [NSExpression expressionForKeyPath:@"name_fr"],
+        }];
+        NSExpression *expected = [NSExpression expressionWithFormat:@"mgl_step:from:stops:($zoomLevel, short, %@)", @{
+            @1: [NSExpression expressionForKeyPath:@"abbr"],
+            @2: @"…",
+            @3: [NSExpression expressionForKeyPath:@"name_es"],
+        }];
+        XCTAssertEqualObjects([original mgl_expressionLocalizedIntoLocale:[NSLocale localeWithLocaleIdentifier:@"es-PR"]], expected);
+    }
+    {
+        NSArray *jsonExpression = @[
+            @"step",
+            @[@"zoom"],
+            @[
+                @"case",
+                @[
+                    @"<",
+                    @[
+                        @"to-number",
+                        @[@"get", @"area"]
+                    ],
+                    @80000
+                ],
+                @[@"get", @"abbr"],
+                @[@"get", @"name_en"]
+            ],
+            @5, @[@"get", @"name_en"]
+        ];
+        NSArray *localizedJSONExpression = @[
+            @"step",
+            @[@"zoom"],
+            @[
+                @"case",
+                @[
+                    @"<",
+                    @[
+                        @"to-number",
+                        @[@"get", @"area"]
+                    ],
+                    @80000
+                ],
+                @[@"get", @"abbr"],
+                @[@"get", @"name"]
+            ],
+            @5, @[@"get", @"name"]
+        ];
+        NSExpression *expression = [NSExpression expressionWithMGLJSONObject:jsonExpression];
+        NSExpression *localizedExpression = [expression mgl_expressionLocalizedIntoLocale:[NSLocale localeWithLocaleIdentifier:@"mul"]];
+        XCTAssertEqualObjects(localizedExpression.mgl_jsonExpressionObject, localizedJSONExpression);
     }
 }
 
diff --git a/platform/darwin/test/MGLStyleTests.mm b/platform/darwin/test/MGLStyleTests.mm
index 7b1cc56ba4..6048f39ea3 100644
--- a/platform/darwin/test/MGLStyleTests.mm
+++ b/platform/darwin/test/MGLStyleTests.mm
@@ -1,6 +1,7 @@
 #import <Mapbox/Mapbox.h>
 
 #import "NSBundle+MGLAdditions.h"
+#import "MGLVectorTileSource_Private.h"
 
 #import <mbgl/util/default_styles.hpp>
 
@@ -419,4 +420,45 @@
     XCTAssertEqualObjects(layers[startIndex++].identifier, layer4.identifier);
 }
 
+#pragma mark Localization tests
+
+- (void)testLanguageMatching {
+    {
+        NSArray *preferences = @[@"en"];
+        XCTAssertEqualObjects([MGLVectorTileSource preferredMapboxStreetsLanguageForPreferences:preferences], @"en");
+    }
+    {
+        NSArray *preferences = @[@"en-US"];
+        XCTAssertEqualObjects([MGLVectorTileSource preferredMapboxStreetsLanguageForPreferences:preferences], @"en");
+    }
+    {
+        NSArray *preferences = @[@"fr"];
+        XCTAssertEqualObjects([MGLVectorTileSource preferredMapboxStreetsLanguageForPreferences:preferences], @"fr");
+    }
+    {
+        NSArray *preferences = @[@"zh-Hans"];
+        XCTAssertEqualObjects([MGLVectorTileSource preferredMapboxStreetsLanguageForPreferences:preferences], @"zh-Hans");
+    }
+    {
+        NSArray *preferences = @[@"zh-Hans", @"en"];
+        XCTAssertEqualObjects([MGLVectorTileSource preferredMapboxStreetsLanguageForPreferences:preferences], @"zh-Hans");
+    }
+    {
+        NSArray *preferences = @[@"zh-Hant"];
+        XCTAssertNil([MGLVectorTileSource preferredMapboxStreetsLanguageForPreferences:preferences]);
+    }
+    {
+        NSArray *preferences = @[@"tlh"];
+        XCTAssertNil([MGLVectorTileSource preferredMapboxStreetsLanguageForPreferences:preferences]);
+    }
+    {
+        NSArray *preferences = @[@"tlh", @"en"];
+        XCTAssertEqualObjects([MGLVectorTileSource preferredMapboxStreetsLanguageForPreferences:preferences], @"en");
+    }
+    {
+        NSArray *preferences = @[@"mul"];
+        XCTAssertNil([MGLVectorTileSource preferredMapboxStreetsLanguageForPreferences:preferences]);
+    }
+}
+
 @end
diff --git a/platform/macos/macos.xcodeproj/xcshareddata/xcschemes/macosapp.xcscheme b/platform/macos/macos.xcodeproj/xcshareddata/xcschemes/macosapp.xcscheme
index b0ce01fbf0..aab7486fbe 100644
--- a/platform/macos/macos.xcodeproj/xcshareddata/xcschemes/macosapp.xcscheme
+++ b/platform/macos/macos.xcodeproj/xcshareddata/xcschemes/macosapp.xcscheme
@@ -26,7 +26,7 @@
       buildConfiguration = "Debug"
       selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
       selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
-      language = ""
+      language = "en"
       shouldUseLaunchSchemeArgsEnv = "YES">
       <Testables>
          <TestableReference
@@ -56,7 +56,6 @@
       buildConfiguration = "Debug"
       selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
       selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
-      language = ""
       launchStyle = "0"
       useCustomWorkingDirectory = "NO"
       ignoresPersistentStateOnLaunch = "NO"