[ios, macos] Migrate MGLCustomStyleLayerAdditions to style layer API (#7250)

* [ios, macos] Replaced custom style layer API with MGLOpenGLStyleLayer

Replaced the custom style layer API on MGLMapView with an equally unsupported MGLOpenGLStyleLayer API that nonetheless is consistent with the broader runtime styling API and is compatible with macOS.

Fixed an unrecognized selector crash when wrapping a layer of unrecognized type coming from mbgl.

* [macos] Added lime green layer demo to macosapp

Reprised the demo removed from iosapp in #5091.

* [ios, macos] Rationalized MGLOpenGLStyleLayer API

MGLStyle now strongly references any MGLOpenGLStyleLayer object that’s added to it, in order to prevent pointers from going stale and make it easy for layer drawing code to get more information about the map view. Replaced the MGLOpenGLStyleLayer callback blocks with overridable instance methods. Added internal documentation for each method. Subclassed MGLOpenGLStyleLayer as LimeGreenStyleLayer inside macosapp.

Consolidated -addToMapView: into -addToMapView:belowLayer: to ensure that MGLRedundantLayerException gets raised even if the layer is being inserted rather than added to the bottom of the stack.

* [core] Clarified that rendering happens on the main thread

* [ios, macos] Fixed removing and re-adding MGLOpenGLStyleLayer

Don’t allow index-based layer removal to circumvent -removeFromMapView:, which MGLOpenGLStyleLayer relies on to synchronize the style’s array of MGLOpenGLStyleLayers. When obtaining an MGLOpenGLStyleLayer, get the instance already added to the style instead of creating a new one to wrap the underlying CustomLayer.

8 files changed, 127 insertions, 3 deletions

diff --git a/platform/macos/app/Base.lproj/MainMenu.xib b/platform/macos/app/Base.lproj/MainMenu.xib
index cb9905d4a1..9faf1ba04b 100644
--- a/platform/macos/app/Base.lproj/MainMenu.xib
+++ b/platform/macos/app/Base.lproj/MainMenu.xib
@@ -537,6 +537,12 @@
                                     <action selector="drawAnimatedAnnotation:" target="-1" id="CYM-WB-s97"/>
                                 </connections>
                             </menuItem>
+                            <menuItem title="Add Lime Green Layer" id="UWY-vl-t2m">
+                                <modifierMask key="keyEquivalentModifierMask"/>
+                                <connections>
+                                    <action selector="insertCustomStyleLayer:" target="-1" id="LE5-lz-kx3"/>
+                                </connections>
+                            </menuItem>
                             <menuItem title="Show All Annnotations" keyEquivalent="A" id="yMj-uM-8SN">
                                 <modifierMask key="keyEquivalentModifierMask" shift="YES" command="YES"/>
                                 <connections>
diff --git a/platform/macos/app/LimeGreenStyleLayer.h b/platform/macos/app/LimeGreenStyleLayer.h
new file mode 100644
index 0000000000..35480963a4
--- /dev/null
+++ b/platform/macos/app/LimeGreenStyleLayer.h
@@ -0,0 +1,5 @@
+#import <Mapbox/Mapbox.h>
+
+@interface LimeGreenStyleLayer : MGLOpenGLStyleLayer
+
+@end
diff --git a/platform/macos/app/LimeGreenStyleLayer.m b/platform/macos/app/LimeGreenStyleLayer.m
new file mode 100644
index 0000000000..0d2e642db9
--- /dev/null
+++ b/platform/macos/app/LimeGreenStyleLayer.m
@@ -0,0 +1,60 @@
+#import "LimeGreenStyleLayer.h"
+
+#include <OpenGL/gl.h>
+#include <OpenGL/glext.h>
+
+@implementation LimeGreenStyleLayer {
+    GLuint _program;
+    GLuint _vertexShader;
+    GLuint _fragmentShader;
+    GLuint _buffer;
+    GLuint _aPos;
+}
+
+- (void)didMoveToMapView:(MGLMapView *)mapView {
+    static const GLchar *vertexShaderSource = "attribute vec2 a_pos; void main() { gl_Position = vec4(a_pos, 0, 1); }";
+    static const GLchar *fragmentShaderSource = "void main() { gl_FragColor = vec4(0, 1, 0, 1); }";
+    
+    _program = glCreateProgram();
+    _vertexShader = glCreateShader(GL_VERTEX_SHADER);
+    _fragmentShader = glCreateShader(GL_FRAGMENT_SHADER);
+    
+    glShaderSource(_vertexShader, 1, &vertexShaderSource, NULL);
+    glCompileShader(_vertexShader);
+    glAttachShader(_program, _vertexShader);
+    glShaderSource(_fragmentShader, 1, &fragmentShaderSource, NULL);
+    glCompileShader(_fragmentShader);
+    glAttachShader(_program, _fragmentShader);
+    glLinkProgram(_program);
+    _aPos = glGetAttribLocation(_program, "a_pos");
+    
+    GLfloat background[] = { -1,-1, 1,-1, -1,1, 1,1 };
+    glGenBuffers(1, &_buffer);
+    glBindBuffer(GL_ARRAY_BUFFER, _buffer);
+    glBufferData(GL_ARRAY_BUFFER, 8 * sizeof(GLfloat), background, GL_STATIC_DRAW);
+}
+
+- (void)drawInMapView:(MGLMapView *)mapView withContext:(MGLStyleLayerDrawingContext)context {
+    glUseProgram(_program);
+    glBindBuffer(GL_ARRAY_BUFFER, _buffer);
+    glEnableVertexAttribArray(_aPos);
+    glVertexAttribPointer(_aPos, 2, GL_FLOAT, GL_FALSE, 0, NULL);
+    glDisable(GL_STENCIL_TEST);
+    glDisable(GL_DEPTH_TEST);
+    glDrawArrays(GL_TRIANGLE_STRIP, 0, 4);
+}
+
+- (void)willMoveFromMapView:(MGLMapView *)mapView {
+    if (!_program) {
+        return;
+    }
+    
+    glDeleteBuffers(1, &_buffer);
+    glDetachShader(_program, _vertexShader);
+    glDetachShader(_program, _fragmentShader);
+    glDeleteShader(_vertexShader);
+    glDeleteShader(_fragmentShader);
+    glDeleteProgram(_program);
+}
+
+@end
diff --git a/platform/macos/app/MapDocument.m b/platform/macos/app/MapDocument.m
index 901a5ebb7f..6c36d00d73 100644
--- a/platform/macos/app/MapDocument.m
+++ b/platform/macos/app/MapDocument.m
@@ -1,6 +1,7 @@
 #import "MapDocument.h"
 
 #import "AppDelegate.h"
+#import "LimeGreenStyleLayer.h"
 #import "DroppedPinAnnotation.h"
 
 #import <Mapbox/Mapbox.h>
@@ -604,6 +605,37 @@ NS_ARRAY_OF(id <MGLAnnotation>) *MBXFlattenedShapes(NS_ARRAY_OF(id <MGLAnnotatio
         cos(angle) * 20);
 }
 
+- (IBAction)insertCustomStyleLayer:(id)sender {
+    [self.undoManager registerUndoWithTarget:self handler:^(id  _Nonnull target) {
+        [self removeCustomStyleLayer:sender];
+    }];
+    
+    if (!self.undoManager.isUndoing) {
+        [self.undoManager setActionName:@"Add Lime Green Layer"];
+    }
+    
+    LimeGreenStyleLayer *layer = [[LimeGreenStyleLayer alloc] initWithIdentifier:@"mbx-custom"];
+    MGLStyleLayer *houseNumberLayer = [self.mapView.style layerWithIdentifier:@"housenum-label"];
+    if (houseNumberLayer) {
+        [self.mapView.style insertLayer:layer belowLayer:houseNumberLayer];
+    } else {
+        [self.mapView.style addLayer:layer];
+    }
+}
+
+- (IBAction)removeCustomStyleLayer:(id)sender {
+    [self.undoManager registerUndoWithTarget:self handler:^(id  _Nonnull target) {
+        [self insertCustomStyleLayer:sender];
+    }];
+    
+    if (!self.undoManager.isUndoing) {
+        [self.undoManager setActionName:@"Delete Lime Green Layer"];
+    }
+    
+    MGLStyleLayer *layer = [self.mapView.style layerWithIdentifier:@"mbx-custom"];
+    [self.mapView.style removeLayer:layer];
+}
+
 #pragma mark Offline packs
 
 - (IBAction)addOfflinePack:(id)sender {
@@ -892,6 +924,9 @@ NS_ARRAY_OF(id <MGLAnnotation>) *MBXFlattenedShapes(NS_ARRAY_OF(id <MGLAnnotatio
     if (menuItem.action == @selector(drawAnimatedAnnotation:)) {
         return !_isShowingAnimatedAnnotation;
     }
+    if (menuItem.action == @selector(insertCustomStyleLayer:)) {
+        return ![self.mapView.style layerWithIdentifier:@"mbx-custom"];
+    }
     if (menuItem.action == @selector(showAllAnnotations:) || menuItem.action == @selector(removeAllAnnotations:)) {
         return self.mapView.annotations.count > 0;
     }
diff --git a/platform/macos/macos.xcodeproj/project.pbxproj b/platform/macos/macos.xcodeproj/project.pbxproj
index bb81919292..20fa968a97 100644
--- a/platform/macos/macos.xcodeproj/project.pbxproj
+++ b/platform/macos/macos.xcodeproj/project.pbxproj
@@ -80,6 +80,8 @@
 		DA5589771D320C41006B7F64 /* wms.json in Resources */ = {isa = PBXBuildFile; fileRef = DA5589761D320C41006B7F64 /* wms.json */; };
 		DA6408D71DA4E5DA00908C90 /* MGLVectorStyleLayer.h in Headers */ = {isa = PBXBuildFile; fileRef = DA6408D51DA4E5DA00908C90 /* MGLVectorStyleLayer.h */; settings = {ATTRIBUTES = (Public, ); }; };
 		DA6408D81DA4E5DA00908C90 /* MGLVectorStyleLayer.m in Sources */ = {isa = PBXBuildFile; fileRef = DA6408D61DA4E5DA00908C90 /* MGLVectorStyleLayer.m */; };
+		DA7262071DEEDD460043BB89 /* MGLOpenGLStyleLayer.h in Headers */ = {isa = PBXBuildFile; fileRef = DA7262051DEEDD460043BB89 /* MGLOpenGLStyleLayer.h */; settings = {ATTRIBUTES = (Public, ); }; };
+		DA7262081DEEDD460043BB89 /* MGLOpenGLStyleLayer.mm in Sources */ = {isa = PBXBuildFile; fileRef = DA7262061DEEDD460043BB89 /* MGLOpenGLStyleLayer.mm */; };
 		DA839E971CC2E3400062CAFB /* AppDelegate.m in Sources */ = {isa = PBXBuildFile; fileRef = DA839E961CC2E3400062CAFB /* AppDelegate.m */; };
 		DA839E9A1CC2E3400062CAFB /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = DA839E991CC2E3400062CAFB /* main.m */; };
 		DA839E9D1CC2E3400062CAFB /* MapDocument.m in Sources */ = {isa = PBXBuildFile; fileRef = DA839E9C1CC2E3400062CAFB /* MapDocument.m */; };
@@ -117,6 +119,7 @@
 		DA8F25B21D51CB270010E6B5 /* NSValue+MGLStyleAttributeAdditions.h in Headers */ = {isa = PBXBuildFile; fileRef = DA8F25A61D51CB270010E6B5 /* NSValue+MGLStyleAttributeAdditions.h */; };
 		DA8F25B31D51CB270010E6B5 /* NSValue+MGLStyleAttributeAdditions.mm in Sources */ = {isa = PBXBuildFile; fileRef = DA8F25A71D51CB270010E6B5 /* NSValue+MGLStyleAttributeAdditions.mm */; };
 		DAA48EFD1D6A4731006A7E36 /* StyleLayerIconTransformer.m in Sources */ = {isa = PBXBuildFile; fileRef = DAA48EFC1D6A4731006A7E36 /* StyleLayerIconTransformer.m */; };
+		DAB2CCE51DF632ED001B2FE1 /* LimeGreenStyleLayer.m in Sources */ = {isa = PBXBuildFile; fileRef = DAB2CCE41DF632ED001B2FE1 /* LimeGreenStyleLayer.m */; };
 		DAC2ABC51CC6D343006D18C4 /* MGLAnnotationImage_Private.h in Headers */ = {isa = PBXBuildFile; fileRef = DAC2ABC41CC6D343006D18C4 /* MGLAnnotationImage_Private.h */; };
 		DACC22141CF3D3E200D220D9 /* MGLFeature.h in Headers */ = {isa = PBXBuildFile; fileRef = DACC22121CF3D3E200D220D9 /* MGLFeature.h */; settings = {ATTRIBUTES = (Public, ); }; };
 		DACC22151CF3D3E200D220D9 /* MGLFeature.mm in Sources */ = {isa = PBXBuildFile; fileRef = DACC22131CF3D3E200D220D9 /* MGLFeature.mm */; };
@@ -316,6 +319,8 @@
 		DA5589761D320C41006B7F64 /* wms.json */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.json; path = wms.json; sourceTree = "<group>"; };
 		DA6408D51DA4E5DA00908C90 /* MGLVectorStyleLayer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = MGLVectorStyleLayer.h; sourceTree = "<group>"; };
 		DA6408D61DA4E5DA00908C90 /* MGLVectorStyleLayer.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = MGLVectorStyleLayer.m; sourceTree = "<group>"; };
+		DA7262051DEEDD460043BB89 /* MGLOpenGLStyleLayer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = MGLOpenGLStyleLayer.h; sourceTree = "<group>"; };
+		DA7262061DEEDD460043BB89 /* MGLOpenGLStyleLayer.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = MGLOpenGLStyleLayer.mm; sourceTree = "<group>"; };
 		DA839E921CC2E3400062CAFB /* Mapbox GL.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "Mapbox GL.app"; sourceTree = BUILT_PRODUCTS_DIR; };
 		DA839E951CC2E3400062CAFB /* AppDelegate.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = AppDelegate.h; sourceTree = "<group>"; };
 		DA839E961CC2E3400062CAFB /* AppDelegate.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = AppDelegate.m; sourceTree = "<group>"; };
@@ -361,6 +366,8 @@
 		DA8F25B71D51D2240010E6B5 /* MGLStyleLayer.mm.ejs */ = {isa = PBXFileReference; lastKnownFileType = text; path = MGLStyleLayer.mm.ejs; sourceTree = "<group>"; };
 		DAA48EFB1D6A4731006A7E36 /* StyleLayerIconTransformer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = StyleLayerIconTransformer.h; sourceTree = "<group>"; };
 		DAA48EFC1D6A4731006A7E36 /* StyleLayerIconTransformer.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = StyleLayerIconTransformer.m; sourceTree = "<group>"; };
+		DAB2CCE31DF632ED001B2FE1 /* LimeGreenStyleLayer.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = LimeGreenStyleLayer.h; sourceTree = "<group>"; };
+		DAB2CCE41DF632ED001B2FE1 /* LimeGreenStyleLayer.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = LimeGreenStyleLayer.m; sourceTree = "<group>"; };
 		DAC2ABC41CC6D343006D18C4 /* MGLAnnotationImage_Private.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = MGLAnnotationImage_Private.h; sourceTree = "<group>"; };
 		DACC22121CF3D3E200D220D9 /* MGLFeature.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = MGLFeature.h; sourceTree = "<group>"; };
 		DACC22131CF3D3E200D220D9 /* MGLFeature.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = MGLFeature.mm; sourceTree = "<group>"; };
@@ -500,6 +507,8 @@
 				35602BFE1D3EA9B40050646F /* MGLForegroundStyleLayer.m */,
 				DA8F25891D51CA540010E6B5 /* MGLLineStyleLayer.h */,
 				DA8F258A1D51CA540010E6B5 /* MGLLineStyleLayer.mm */,
+				DA7262051DEEDD460043BB89 /* MGLOpenGLStyleLayer.h */,
+				DA7262061DEEDD460043BB89 /* MGLOpenGLStyleLayer.mm */,
 				DA8F258D1D51CA600010E6B5 /* MGLRasterStyleLayer.h */,
 				DA8F258E1D51CA600010E6B5 /* MGLRasterStyleLayer.mm */,
 				3538AA211D542685008EC33D /* MGLStyleLayer.h */,
@@ -586,6 +595,8 @@
 				DA839E961CC2E3400062CAFB /* AppDelegate.m */,
 				DAE6C2E31CC3050F00DB3429 /* DroppedPinAnnotation.h */,
 				DAE6C2E41CC3050F00DB3429 /* DroppedPinAnnotation.m */,
+				DAB2CCE31DF632ED001B2FE1 /* LimeGreenStyleLayer.h */,
+				DAB2CCE41DF632ED001B2FE1 /* LimeGreenStyleLayer.m */,
 				DAE6C2E51CC3050F00DB3429 /* LocationCoordinate2DTransformer.h */,
 				DAE6C2E61CC3050F00DB3429 /* LocationCoordinate2DTransformer.m */,
 				DA839E9B1CC2E3400062CAFB /* MapDocument.h */,
@@ -952,6 +963,7 @@
 				30E5781B1DAA857E0050F07E /* NSImage+MGLAdditions.h in Headers */,
 				DAE6C3661CC31E0400DB3429 /* MGLShape.h in Headers */,
 				DA551B831DB496AC0009AFAF /* MGLTileSet_Private.h in Headers */,
+				DA7262071DEEDD460043BB89 /* MGLOpenGLStyleLayer.h in Headers */,
 				352742811D4C243B00A1ECE6 /* MGLSource.h in Headers */,
 				DAE6C3C21CC31F4500DB3429 /* Mapbox.h in Headers */,
 				DAE6C3641CC31E0400DB3429 /* MGLPolygon.h in Headers */,
@@ -1169,6 +1181,7 @@
 				DA839E9D1CC2E3400062CAFB /* MapDocument.m in Sources */,
 				DAE6C2ED1CC3050F00DB3429 /* DroppedPinAnnotation.m in Sources */,
 				DAE6C2EE1CC3050F00DB3429 /* LocationCoordinate2DTransformer.m in Sources */,
+				DAB2CCE51DF632ED001B2FE1 /* LimeGreenStyleLayer.m in Sources */,
 				DAE6C2F11CC3050F00DB3429 /* TimeIntervalTransformer.m in Sources */,
 				DA839E9A1CC2E3400062CAFB /* main.m in Sources */,
 				DA839E971CC2E3400062CAFB /* AppDelegate.m in Sources */,
@@ -1191,6 +1204,7 @@
 				DAE6C3B11CC31EF300DB3429 /* MGLAnnotationImage.m in Sources */,
 				3508EC651D749D39009B0EE4 /* NSExpression+MGLAdditions.mm in Sources */,
 				DACC22151CF3D3E200D220D9 /* MGLFeature.mm in Sources */,
+				DA7262081DEEDD460043BB89 /* MGLOpenGLStyleLayer.mm in Sources */,
 				355BA4EE1D41633E00CCC6D5 /* NSColor+MGLAdditions.mm in Sources */,
 				DAE6C3B31CC31EF300DB3429 /* MGLAttributionButton.m in Sources */,
 				35602BFB1D3EA99F0050646F /* MGLFillStyleLayer.mm in Sources */,
diff --git a/platform/macos/src/MGLMapView.mm b/platform/macos/src/MGLMapView.mm
index 204efd4987..a2fe7acbda 100644
--- a/platform/macos/src/MGLMapView.mm
+++ b/platform/macos/src/MGLMapView.mm
@@ -796,7 +796,7 @@ public:
     _mbglMap->setSourceTileCacheSize(cacheSize);
 }
 
-- (void)invalidate {
+- (void)setNeedsGLDisplay {
     MGLAssertIsMainThread();
 
     [self.layer setNeedsDisplay];
@@ -931,7 +931,7 @@ public:
 
 - (void)print:(__unused id)sender {
     _isPrinting = YES;
-    [self invalidate];
+    [self setNeedsGLDisplay];
 }
 
 - (void)printWithImage:(NSImage *)image {
@@ -2623,7 +2623,7 @@ public:
     }
 
     void invalidate() override {
-        [nativeView invalidate];
+        [nativeView setNeedsGLDisplay];
     }
 
     void activate() override {
diff --git a/platform/macos/src/MGLMapView_Private.h b/platform/macos/src/MGLMapView_Private.h
index 2d9fc52a62..f0a61773a9 100644
--- a/platform/macos/src/MGLMapView_Private.h
+++ b/platform/macos/src/MGLMapView_Private.h
@@ -19,6 +19,9 @@
 /// Center longitude set independently of the center latitude in an inspectable.
 @property (nonatomic) CLLocationDegrees pendingLongitude;
 
+/// Asynchronously render a frame of the map.
+- (void)setNeedsGLDisplay;
+
 /// Synchronously render a frame of the map.
 - (void)renderSync;
 
diff --git a/platform/macos/src/Mapbox.h b/platform/macos/src/Mapbox.h
index 73b8624be0..df3f0b27a3 100644
--- a/platform/macos/src/Mapbox.h
+++ b/platform/macos/src/Mapbox.h
@@ -39,6 +39,7 @@ FOUNDATION_EXPORT const unsigned char MapboxVersionString[];
 #import "MGLRasterStyleLayer.h"
 #import "MGLCircleStyleLayer.h"
 #import "MGLBackgroundStyleLayer.h"
+#import "MGLOpenGLStyleLayer.h"
 #import "MGLSource.h"
 #import "MGLVectorSource.h"
 #import "MGLGeoJSONSource.h"