wayland/buffer: Only query Wayland EGL buffer if display bound

It's not allowed to call eglQueryWaylandBuffer() if the call to
eglBindWaylandDisplay() failed, and will result in an assert being hit
in mesa if called.

Avoid that by keeping track whether we succeeded to bind, and only
attempt to realize a legacy EGL wl_buffer if binding succeeded.

Part-of: <https://gitlab.gnome.org/GNOME/mutter/-/merge_requests/2415>

(cherry picked from commit 75ec27966d11cdbd83a64128aba032d386633b11)

7 files changed, 63 insertions, 22 deletions

diff --git a/src/wayland/meta-wayland-buffer.c b/src/wayland/meta-wayland-buffer.c
index e6d4df979..ec01546c7 100644
--- a/src/wayland/meta-wayland-buffer.c
+++ b/src/wayland/meta-wayland-buffer.c
@@ -94,7 +94,8 @@ meta_wayland_buffer_destroy_handler (struct wl_listener *listener,
 }
 
 MetaWaylandBuffer *
-meta_wayland_buffer_from_resource (struct wl_resource *resource)
+meta_wayland_buffer_from_resource (MetaWaylandCompositor *compositor,
+                                   struct wl_resource    *resource)
 {
   MetaWaylandBuffer *buffer;
   struct wl_listener *listener;
@@ -112,6 +113,7 @@ meta_wayland_buffer_from_resource (struct wl_resource *resource)
       buffer = g_object_new (META_TYPE_WAYLAND_BUFFER, NULL);
 
       buffer->resource = resource;
+      buffer->compositor = compositor;
       buffer->destroy_listener.notify = meta_wayland_buffer_destroy_handler;
       wl_resource_add_destroy_listener (resource, &buffer->destroy_listener);
     }
@@ -134,12 +136,6 @@ meta_wayland_buffer_is_realized (MetaWaylandBuffer *buffer)
 gboolean
 meta_wayland_buffer_realize (MetaWaylandBuffer *buffer)
 {
-  EGLint format;
-  MetaBackend *backend = meta_get_backend ();
-  MetaEgl *egl = meta_backend_get_egl (backend);
-  ClutterBackend *clutter_backend = meta_backend_get_clutter_backend (backend);
-  CoglContext *cogl_context = clutter_backend_get_cogl_context (clutter_backend);
-  EGLDisplay egl_display = cogl_egl_context_get_egl_display (cogl_context);
 #ifdef HAVE_WAYLAND_EGLSTREAM
   MetaWaylandEglStream *stream;
 #endif
@@ -170,12 +166,24 @@ meta_wayland_buffer_realize (MetaWaylandBuffer *buffer)
     }
 #endif /* HAVE_WAYLAND_EGLSTREAM */
 
-  if (meta_egl_query_wayland_buffer (egl, egl_display, buffer->resource,
-                                     EGL_TEXTURE_FORMAT, &format,
-                                     NULL))
+  if (meta_wayland_compositor_is_egl_display_bound (buffer->compositor))
     {
-      buffer->type = META_WAYLAND_BUFFER_TYPE_EGL_IMAGE;
-      return TRUE;
+      EGLint format;
+      MetaBackend *backend = meta_get_backend ();
+      MetaEgl *egl = meta_backend_get_egl (backend);
+      ClutterBackend *clutter_backend =
+        meta_backend_get_clutter_backend (backend);
+      CoglContext *cogl_context =
+        clutter_backend_get_cogl_context (clutter_backend);
+      EGLDisplay egl_display = cogl_egl_context_get_egl_display (cogl_context);
+
+      if (meta_egl_query_wayland_buffer (egl, egl_display, buffer->resource,
+                                         EGL_TEXTURE_FORMAT, &format,
+                                         NULL))
+        {
+          buffer->type = META_WAYLAND_BUFFER_TYPE_EGL_IMAGE;
+          return TRUE;
+        }
     }
 
   dma_buf = meta_wayland_dma_buf_from_buffer (buffer);
diff --git a/src/wayland/meta-wayland-buffer.h b/src/wayland/meta-wayland-buffer.h
index e00406ba2..c47d2227e 100644
--- a/src/wayland/meta-wayland-buffer.h
+++ b/src/wayland/meta-wayland-buffer.h
@@ -48,6 +48,7 @@ struct _MetaWaylandBuffer
 {
   GObject parent;
 
+  MetaWaylandCompositor *compositor;
   struct wl_resource *resource;
   struct wl_listener destroy_listener;
 
@@ -76,7 +77,8 @@ struct _MetaWaylandBuffer
 G_DECLARE_FINAL_TYPE (MetaWaylandBuffer, meta_wayland_buffer,
                       META, WAYLAND_BUFFER, GObject);
 
-MetaWaylandBuffer *     meta_wayland_buffer_from_resource       (struct wl_resource    *resource);
+MetaWaylandBuffer *     meta_wayland_buffer_from_resource       (MetaWaylandCompositor *compositor,
+                                                                 struct wl_resource    *resource);
 struct wl_resource *    meta_wayland_buffer_get_resource        (MetaWaylandBuffer     *buffer);
 gboolean                meta_wayland_buffer_is_realized         (MetaWaylandBuffer     *buffer);
 gboolean                meta_wayland_buffer_realize             (MetaWaylandBuffer     *buffer);
diff --git a/src/wayland/meta-wayland-dma-buf.c b/src/wayland/meta-wayland-dma-buf.c
index 627a98651..6af20cd01 100644
--- a/src/wayland/meta-wayland-dma-buf.c
+++ b/src/wayland/meta-wayland-dma-buf.c
@@ -133,6 +133,8 @@ struct _MetaWaylandDmaBufBuffer
 {
   GObject parent;
 
+  MetaWaylandDmaBufManager *manager;
+
   int width;
   int height;
   uint32_t drm_format;
@@ -783,7 +785,8 @@ buffer_params_create_common (struct wl_client   *client,
     wl_resource_create (client, &wl_buffer_interface, 1, buffer_id);
   wl_resource_set_implementation (buffer_resource, &dma_buf_buffer_impl,
                                   dma_buf, NULL);
-  buffer = meta_wayland_buffer_from_resource (buffer_resource);
+  buffer = meta_wayland_buffer_from_resource (dma_buf->manager->compositor,
+                                              buffer_resource);
 
   meta_wayland_buffer_realize (buffer);
   if (!meta_wayland_dma_buf_realize_texture (buffer, &error))
@@ -857,10 +860,13 @@ dma_buf_handle_create_buffer_params (struct wl_client   *client,
                                      struct wl_resource *dma_buf_resource,
                                      uint32_t            params_id)
 {
+  MetaWaylandDmaBufManager *dma_buf_manager =
+    wl_resource_get_user_data (dma_buf_resource);
   struct wl_resource *params_resource;
   MetaWaylandDmaBufBuffer *dma_buf;
 
   dma_buf = g_object_new (META_TYPE_WAYLAND_DMA_BUF_BUFFER, NULL);
+  dma_buf->manager = dma_buf_manager;
 
   params_resource =
     wl_resource_create (client,
diff --git a/src/wayland/meta-wayland-egl-stream.c b/src/wayland/meta-wayland-egl-stream.c
index 3a2e1e56a..4f239b012 100644
--- a/src/wayland/meta-wayland-egl-stream.c
+++ b/src/wayland/meta-wayland-egl-stream.c
@@ -46,7 +46,10 @@ attach_eglstream_consumer (struct wl_client   *client,
                            struct wl_resource *wl_surface,
                            struct wl_resource *wl_eglstream)
 {
-  MetaWaylandBuffer *buffer = meta_wayland_buffer_from_resource (wl_eglstream);
+  MetaWaylandCompositor *compositor = wl_resource_get_user_data (wl_eglstream);
+  MetaWaylandBuffer *buffer;
+
+  buffer = meta_wayland_buffer_from_resource (compositor, wl_eglstream);
 
   if (!meta_wayland_buffer_is_realized (buffer))
     meta_wayland_buffer_realize (buffer);
@@ -111,7 +114,7 @@ meta_wayland_eglstream_controller_init (MetaWaylandCompositor *compositor)
 
   if (wl_global_create (compositor->wayland_display,
                         wl_eglstream_controller_interface_ptr, 1,
-                        NULL,
+                        compositor,
                         bind_eglstream_controller) == NULL)
     goto fail;
 
diff --git a/src/wayland/meta-wayland-private.h b/src/wayland/meta-wayland-private.h
index 613dc216e..b0cb38ab4 100644
--- a/src/wayland/meta-wayland-private.h
+++ b/src/wayland/meta-wayland-private.h
@@ -104,4 +104,6 @@ struct _MetaWaylandCompositor
 G_DECLARE_FINAL_TYPE (MetaWaylandCompositor, meta_wayland_compositor,
                       META, WAYLAND_COMPOSITOR, GObject)
 
+gboolean meta_wayland_compositor_is_egl_display_bound (MetaWaylandCompositor *compositor);
+
 #endif /* META_WAYLAND_PRIVATE_H */
diff --git a/src/wayland/meta-wayland-surface.c b/src/wayland/meta-wayland-surface.c
index 8deca4f29..bd6499cf0 100644
--- a/src/wayland/meta-wayland-surface.c
+++ b/src/wayland/meta-wayland-surface.c
@@ -1046,11 +1046,12 @@ wl_surface_attach (struct wl_client   *client,
 {
   MetaWaylandSurface *surface =
     wl_resource_get_user_data (surface_resource);
+  MetaWaylandCompositor *compositor = surface->compositor;
   MetaWaylandSurfaceState *pending = surface->pending_state;
   MetaWaylandBuffer *buffer;
 
   if (buffer_resource)
-    buffer = meta_wayland_buffer_from_resource (buffer_resource);
+    buffer = meta_wayland_buffer_from_resource (compositor, buffer_resource);
   else
     buffer = NULL;
 
diff --git a/src/wayland/meta-wayland.c b/src/wayland/meta-wayland.c
index 6e2b56dd4..856b819cd 100644
--- a/src/wayland/meta-wayland.c
+++ b/src/wayland/meta-wayland.c
@@ -57,7 +57,13 @@
 
 static char *_display_name_override;
 
-G_DEFINE_TYPE (MetaWaylandCompositor, meta_wayland_compositor, G_TYPE_OBJECT)
+typedef struct _MetaWaylandCompositorPrivate
+{
+  gboolean is_wayland_egl_display_bound;
+} MetaWaylandCompositorPrivate;
+
+G_DEFINE_TYPE_WITH_PRIVATE (MetaWaylandCompositor, meta_wayland_compositor,
+                            G_TYPE_OBJECT)
 
 MetaWaylandCompositor *
 meta_wayland_compositor_get_default (void)
@@ -504,6 +510,8 @@ meta_wayland_get_xwayland_auth_file (MetaWaylandCompositor *compositor)
 static void
 meta_wayland_init_egl (MetaWaylandCompositor *compositor)
 {
+  MetaWaylandCompositorPrivate *priv =
+    meta_wayland_compositor_get_instance_private (compositor);
   MetaBackend *backend = meta_get_backend ();
   MetaEgl *egl = meta_backend_get_egl (backend);
   ClutterBackend *clutter_backend = meta_backend_get_clutter_backend (backend);
@@ -524,10 +532,12 @@ meta_wayland_init_egl (MetaWaylandCompositor *compositor)
   meta_topic (META_DEBUG_WAYLAND,
               "Binding Wayland EGL display");
 
-  if (!meta_egl_bind_wayland_display (egl,
-                                      egl_display,
-                                      compositor->wayland_display,
-                                      &error))
+  if (meta_egl_bind_wayland_display (egl,
+                                     egl_display,
+                                     compositor->wayland_display,
+                                     &error))
+    priv->is_wayland_egl_display_bound = TRUE;
+  else
     g_warning ("Failed to bind Wayland display: %s", error->message);
 }
 
@@ -801,3 +811,12 @@ meta_wayland_compositor_notify_surface_id (MetaWaylandCompositor *compositor,
       meta_wayland_compositor_remove_surface_association (compositor, id);
     }
 }
+
+gboolean
+meta_wayland_compositor_is_egl_display_bound (MetaWaylandCompositor *compositor)
+{
+  MetaWaylandCompositorPrivate *priv =
+    meta_wayland_compositor_get_instance_private (compositor);
+
+  return priv->is_wayland_egl_display_bound;
+}