[ios, macos] Added macro to handle pending iOS tests (rather than disabling via scheme) (#11806)

7 files changed, 51 insertions, 16 deletions

diff --git a/platform/darwin/test/MGLTestUtility.h b/platform/darwin/test/MGLTestUtility.h
new file mode 100644
index 0000000000..2e02e4f9de
--- /dev/null
+++ b/platform/darwin/test/MGLTestUtility.h
@@ -0,0 +1,27 @@
+
+/**
+ MGL_CHECK_IF_PENDING_TEST_SHOULD_RUN()
+
+ Include this macro at the top of a "pending" test - one that would normally fail,
+ but you want to include is as a compile-time test or as a reminder.
+
+ This will output the included warning in the test log, which `xcpretty` will detect.
+ This allows such pending tests to be identified in CI logs.
+
+ If you provide `MAPBOX_RUN_PENDING_TESTS` as an environment variable ALL pending
+ tests will run.
+
+ In addition, it's a good idea to append `PENDING` to the test method name, as this
+ will also be picked up by xcpretty. PENDING tests will be distinguished by a `⧖`
+ and `[PENDING]`
+ */
+#define MGL_CHECK_IF_PENDING_TEST_SHOULD_RUN() \
+    /* By default, skip pending tests. Otherwise check environment for MAPBOX_RUN_PENDING_TESTS */ \
+    { \
+        NSString *runPendingTests = [[NSProcessInfo processInfo] environment][@"MAPBOX_RUN_PENDING_TESTS"]; \
+        if (![runPendingTests boolValue]) { \
+            /* The following warning will be picked up by xcpretty */ \
+            printf("warning: '%s' is a pending test - skipping\n", __PRETTY_FUNCTION__); \
+            return; \
+        } \
+    }
diff --git a/platform/ios/Integration Tests/MGLCameraTransitionTests.mm b/platform/ios/Integration Tests/MGLCameraTransitionTests.mm
index d5c288fbb9..4a9db60982 100644
--- a/platform/ios/Integration Tests/MGLCameraTransitionTests.mm
+++ b/platform/ios/Integration Tests/MGLCameraTransitionTests.mm
@@ -1,4 +1,5 @@
 #import "MGLMapViewIntegrationTest.h"
+#import "MGLTestUtility.h"
 #import "../../darwin/src/MGLGeometry_Private.h"
 
 @interface MBCameraTransitionTests : MGLMapViewIntegrationTest
@@ -336,7 +337,8 @@
 
 #pragma mark - Pending tests
 
-- (void)disabled_testContinuallyResettingNorthInIsChangingPENDING {
+- (void)testContinuallyResettingNorthInIsChangingPENDING {
+    MGL_CHECK_IF_PENDING_TEST_SHOULD_RUN();
 
     // See https://github.com/mapbox/mapbox-gl-native/pull/11614
     // This test currently fails, unsurprisingly, since we're continually
@@ -369,8 +371,9 @@
     XCTAssertEqualWithAccuracy(self.mapView.direction, 0.0, 0.001, @"Camera should have reset to north. %0.3f", self.mapView.direction);
 }
 
-- (void)disabled_testContinuallySettingCoordinateInIsChangingPENDING {
-    // See above comment in `-disabled_testContinuallyResettingNorthInIsChangingPENDING`
+- (void)testContinuallySettingCoordinateInIsChangingPENDING {
+    // See above comment in `-testContinuallyResettingNorthInIsChangingPENDING`
+    MGL_CHECK_IF_PENDING_TEST_SHOULD_RUN();
 
     // Reset to non-zero, prior to testing
     [self.mapView setCenterCoordinate:CLLocationCoordinate2DMake(0.0, 0.0) animated:NO];
diff --git a/platform/ios/Integration Tests/MGLMapViewIntegrationTest.h b/platform/ios/Integration Tests/MGLMapViewIntegrationTest.h
index 6c04ed9f84..3ceec8f04a 100644
--- a/platform/ios/Integration Tests/MGLMapViewIntegrationTest.h
+++ b/platform/ios/Integration Tests/MGLMapViewIntegrationTest.h
@@ -1,5 +1,6 @@
 #import <XCTest/XCTest.h>
 #import <Mapbox/Mapbox.h>
+#import "MGLTestUtility.h"
 
 #define MGLTestFail(myself, ...) \
     _XCTPrimitiveFail(myself, __VA_ARGS__)
diff --git a/platform/ios/ios.xcodeproj/project.pbxproj b/platform/ios/ios.xcodeproj/project.pbxproj
index aef2552b0b..c0909344f3 100644
--- a/platform/ios/ios.xcodeproj/project.pbxproj
+++ b/platform/ios/ios.xcodeproj/project.pbxproj
@@ -1004,6 +1004,7 @@
 		CA4EB8C620863487006AB465 /* MGLStyleLayerIntegrationTests.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = MGLStyleLayerIntegrationTests.m; sourceTree = "<group>"; };
 		CA34C9C2207FD272005C1A06 /* MGLCameraTransitionTests.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = MGLCameraTransitionTests.mm; sourceTree = "<group>"; };
 		CA55CD3E202C16AA00CE7095 /* MGLCameraChangeReason.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = MGLCameraChangeReason.h; sourceTree = "<group>"; };
+		CA5E5042209BDC5F001A8A81 /* MGLTestUtility.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; name = MGLTestUtility.h; path = ../../darwin/test/MGLTestUtility.h; sourceTree = "<group>"; };
 		DA00FC8C1D5EEB0D009AABC8 /* MGLAttributionInfo.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = MGLAttributionInfo.h; sourceTree = "<group>"; };
 		DA00FC8D1D5EEB0D009AABC8 /* MGLAttributionInfo.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = MGLAttributionInfo.mm; sourceTree = "<group>"; };
 		DA0CD58F1CF56F6A00A5F5A5 /* MGLFeatureTests.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; name = MGLFeatureTests.mm; path = ../../darwin/test/MGLFeatureTests.mm; sourceTree = "<group>"; };
@@ -1535,6 +1536,7 @@
 		4031ACFD1E9FD26900A3EA26 /* Test Helpers */ = {
 			isa = PBXGroup;
 			children = (
+				CA5E5042209BDC5F001A8A81 /* MGLTestUtility.h */,
 				4031ACFE1E9FD29F00A3EA26 /* MGLSDKTestHelpers.swift */,
 			);
 			name = "Test Helpers";
diff --git a/platform/ios/ios.xcodeproj/xcshareddata/xcschemes/CI.xcscheme b/platform/ios/ios.xcodeproj/xcshareddata/xcschemes/CI.xcscheme
index 01b565d5d9..6b018e1337 100644
--- a/platform/ios/ios.xcodeproj/xcshareddata/xcschemes/CI.xcscheme
+++ b/platform/ios/ios.xcodeproj/xcshareddata/xcschemes/CI.xcscheme
@@ -54,7 +54,7 @@
       buildConfiguration = "Debug"
       selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
       selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
-      shouldUseLaunchSchemeArgsEnv = "YES">
+      shouldUseLaunchSchemeArgsEnv = "NO">
       <Testables>
          <TestableReference
             skipped = "NO">
@@ -65,14 +65,6 @@
                BlueprintName = "test"
                ReferencedContainer = "container:ios.xcodeproj">
             </BuildableReference>
-            <SkippedTests>
-               <Test
-                  Identifier = "MGLAnnotationViewTests/testAnnotationViewInitWithFrame">
-               </Test>
-               <Test
-                  Identifier = "MGLAnnotationViewTests/testSelectingADisabledAnnotationView">
-               </Test>
-            </SkippedTests>
          </TestableReference>
       </Testables>
       <MacroExpansion>
@@ -84,6 +76,13 @@
             ReferencedContainer = "container:ios.xcodeproj">
          </BuildableReference>
       </MacroExpansion>
+      <EnvironmentVariables>
+         <EnvironmentVariable
+            key = "MAPBOX_RUN_PENDING_TESTS"
+            value = "YES"
+            isEnabled = "NO">
+         </EnvironmentVariable>
+      </EnvironmentVariables>
       <AdditionalOptions>
       </AdditionalOptions>
    </TestAction>
diff --git a/platform/ios/ios.xcodeproj/xcshareddata/xcschemes/Integration Test Harness.xcscheme b/platform/ios/ios.xcodeproj/xcshareddata/xcschemes/Integration Test Harness.xcscheme
index 1638592557..ce264aa19a 100644
--- a/platform/ios/ios.xcodeproj/xcshareddata/xcschemes/Integration Test Harness.xcscheme
+++ b/platform/ios/ios.xcodeproj/xcshareddata/xcschemes/Integration Test Harness.xcscheme
@@ -26,7 +26,7 @@
       buildConfiguration = "Debug"
       selectedDebuggerIdentifier = "Xcode.DebuggerFoundation.Debugger.LLDB"
       selectedLauncherIdentifier = "Xcode.DebuggerFoundation.Launcher.LLDB"
-      shouldUseLaunchSchemeArgsEnv = "YES">
+      shouldUseLaunchSchemeArgsEnv = "NO">
       <Testables>
          <TestableReference
             skipped = "NO">
diff --git a/platform/ios/test/MGLAnnotationViewTests.m b/platform/ios/test/MGLAnnotationViewTests.m
index a2cc4227ed..7bf7c90a33 100644
--- a/platform/ios/test/MGLAnnotationViewTests.m
+++ b/platform/ios/test/MGLAnnotationViewTests.m
@@ -1,5 +1,6 @@
 #import <Mapbox/Mapbox.h>
 #import <XCTest/XCTest.h>
+#import "MGLTestUtility.h"
 
 static NSString * const MGLTestAnnotationReuseIdentifer = @"MGLTestAnnotationReuseIdentifer";
 
@@ -209,7 +210,8 @@ static NSString * const MGLTestAnnotationReuseIdentifer = @"MGLTestAnnotationReu
     XCTAssertEqual(view.dragState, MGLAnnotationViewDragStateNone);
 }
 
-- (void)testAnnotationViewInitWithFrame {
+- (void)testAnnotationViewInitWithFramePENDING {
+    MGL_CHECK_IF_PENDING_TEST_SHOULD_RUN();
     CGRect frame = CGRectMake(10.0, 10.0, 100.0, 100.0);
     MGLAnnotationView *view = [[MGLAnnotationView alloc] initWithFrame:frame];
     [self checkDefaultPropertiesForAnnotationView:view];
@@ -220,11 +222,12 @@ static NSString * const MGLTestAnnotationReuseIdentifer = @"MGLTestAnnotationReu
     [self checkDefaultPropertiesForAnnotationView:view];
 }
 
-- (void)testSelectingADisabledAnnotationView {
+- (void)testSelectingADisabledAnnotationViewPENDING {
+    MGL_CHECK_IF_PENDING_TEST_SHOULD_RUN();
     self.prepareAnnotationView = ^(MGLAnnotationView *view) {
         view.enabled = NO;
     };
-    
+
     // Create annotation
     MGLPointFeature *point = [[MGLPointFeature alloc] init];
     point.title = NSStringFromSelector(_cmd);