Fix CustomLayer context retain count (#10765)

* [core] RenderCustomLayer should de-initialize old context

* [iOS, macOS] Use toll-free bridging to retain/release MGLOpenGLStyleLayer instead of a separate array.

5 files changed, 16 insertions, 16 deletions

diff --git a/platform/darwin/src/MGLOpenGLStyleLayer.mm b/platform/darwin/src/MGLOpenGLStyleLayer.mm
index 36a3c20c97..8933a77382 100644
--- a/platform/darwin/src/MGLOpenGLStyleLayer.mm
+++ b/platform/darwin/src/MGLOpenGLStyleLayer.mm
@@ -47,7 +47,7 @@ void MGLDrawCustomStyleLayer(void *context, const mbgl::style::CustomLayerRender
     when creating an OpenGL style layer.
  */
 void MGLFinishCustomStyleLayer(void *context) {
-    MGLOpenGLStyleLayer *layer = (__bridge MGLOpenGLStyleLayer *)context;
+    MGLOpenGLStyleLayer *layer = (__bridge_transfer MGLOpenGLStyleLayer *)context;
     [layer willMoveFromMapView:layer.style.mapView];
 }
 
@@ -101,7 +101,7 @@ void MGLFinishCustomStyleLayer(void *context) {
                                                             MGLPrepareCustomStyleLayer,
                                                             MGLDrawCustomStyleLayer,
                                                             MGLFinishCustomStyleLayer,
-                                                            (__bridge void *)self);
+                                                            (__bridge_retained void *)self);
     return self = [super initWithPendingLayer:std::move(layer)];
 }
 
@@ -116,9 +116,7 @@ void MGLFinishCustomStyleLayer(void *context) {
         [NSException raise:@"MGLLayerReuseException"
                     format:@"%@ cannot be added to more than one MGLStyle at a time.", self];
     }
-    _style.openGLLayers[self.identifier] = nil;
     _style = style;
-    _style.openGLLayers[self.identifier] = self;
 }
 
 - (void)addToStyle:(MGLStyle *)style belowLayer:(MGLStyleLayer *)otherLayer {
diff --git a/platform/darwin/src/MGLStyle.mm b/platform/darwin/src/MGLStyle.mm
index 244fb94ef9..987ae5f063 100644
--- a/platform/darwin/src/MGLStyle.mm
+++ b/platform/darwin/src/MGLStyle.mm
@@ -77,7 +77,6 @@
 @property (nonatomic, readonly, weak) MGLMapView *mapView;
 @property (nonatomic, readonly) mbgl::style::Style *rawStyle;
 @property (readonly, copy, nullable) NSURL *URL;
-@property (nonatomic, readwrite, strong) NS_MUTABLE_DICTIONARY_OF(NSString *, MGLOpenGLStyleLayer *) *openGLLayers;
 @property (nonatomic) NS_MUTABLE_DICTIONARY_OF(NSString *, NS_DICTIONARY_OF(NSObject *, MGLTextLanguage *) *) *localizedLayersByIdentifier;
 
 @end
@@ -169,7 +168,6 @@ static NSURL *MGLStyleURL_trafficNight;
     if (self = [super init]) {
         _mapView = mapView;
         _rawStyle = rawStyle;
-        _openGLLayers = [NSMutableDictionary dictionary];
         _localizedLayersByIdentifier = [NSMutableDictionary dictionary];
     }
     return self;
diff --git a/platform/darwin/src/MGLStyle_Private.h b/platform/darwin/src/MGLStyle_Private.h
index e5bd79dc02..4cbe953a44 100644
--- a/platform/darwin/src/MGLStyle_Private.h
+++ b/platform/darwin/src/MGLStyle_Private.h
@@ -26,8 +26,6 @@ namespace mbgl {
 
 - (nullable NS_ARRAY_OF(MGLAttributionInfo *) *)attributionInfosWithFontSize:(CGFloat)fontSize linkColor:(nullable MGLColor *)linkColor;
 
-@property (nonatomic, readonly, strong) NS_MUTABLE_DICTIONARY_OF(NSString *, MGLOpenGLStyleLayer *) *openGLLayers;
-
 - (void)setStyleClasses:(NS_ARRAY_OF(NSString *) *)appliedClasses transitionDuration:(NSTimeInterval)transitionDuration;
 
 @end
diff --git a/src/mbgl/renderer/layers/render_custom_layer.cpp b/src/mbgl/renderer/layers/render_custom_layer.cpp
index 7ece3970da..adafd8583f 100644
--- a/src/mbgl/renderer/layers/render_custom_layer.cpp
+++ b/src/mbgl/renderer/layers/render_custom_layer.cpp
@@ -43,20 +43,25 @@ std::unique_ptr<Bucket> RenderCustomLayer::createBucket(const BucketParameters&,
 }
 
 void RenderCustomLayer::render(PaintParameters& paintParameters, RenderSource*) {
-    if (!initialized) {
+    if (context != impl().context || !initialized) {
+        //If the context changed, deinitialize the previous one before initializing the new one.
+        if (context && !contextDestroyed && impl().deinitializeFn) {
+            impl().deinitializeFn(context);
+        }
+        context = impl().context;
         assert(impl().initializeFn);
         impl().initializeFn(impl().context);
         initialized = true;
     }
 
-    gl::Context& context = paintParameters.context;
+    gl::Context& glContext = paintParameters.context;
     const TransformState& state = paintParameters.state;
 
     // Reset GL state to a known state so the CustomLayer always has a clean slate.
-    context.bindVertexArray = 0;
-    context.setDepthMode(paintParameters.depthModeForSublayer(0, gl::DepthMode::ReadOnly));
-    context.setStencilMode(gl::StencilMode::disabled());
-    context.setColorMode(paintParameters.colorModeForRenderPass());
+    glContext.bindVertexArray = 0;
+    glContext.setDepthMode(paintParameters.depthModeForSublayer(0, gl::DepthMode::ReadOnly));
+    glContext.setStencilMode(gl::StencilMode::disabled());
+    glContext.setColorMode(paintParameters.colorModeForRenderPass());
 
     CustomLayerRenderParameters parameters;
 
@@ -70,12 +75,12 @@ void RenderCustomLayer::render(PaintParameters& paintParameters, RenderSource*)
     parameters.fieldOfView = state.getFieldOfView();
 
     assert(impl().renderFn);
-    impl().renderFn(impl().context, parameters);
+    impl().renderFn(context, parameters);
 
     // Reset the view back to our original one, just in case the CustomLayer changed
     // the viewport or Framebuffer.
     paintParameters.backend.bind();
-    context.setDirtyState();
+    glContext.setDirtyState();
 }
 
 } // namespace mbgl
diff --git a/src/mbgl/renderer/layers/render_custom_layer.hpp b/src/mbgl/renderer/layers/render_custom_layer.hpp
index 32ed9da8da..6d1fea99d3 100644
--- a/src/mbgl/renderer/layers/render_custom_layer.hpp
+++ b/src/mbgl/renderer/layers/render_custom_layer.hpp
@@ -26,6 +26,7 @@ public:
 private:
     bool initialized = false;
     bool contextDestroyed = false;
+    void * context = nullptr;
 };
 
 template <>