renderer/native: Force EGL config pixel format

We just arbitrarily chose the first EGL config matching the passed
attributes, but we then assumed we always got GBM_FORMAT_XRGB8888. That
was not a correct assumption. Instead, make sure we always pick the
format we expect.

Closes: https://gitlab.gnome.org/GNOME/mutter/issues/2

6 files changed, 196 insertions, 14 deletions

diff --git a/cogl/cogl/winsys/cogl-winsys-egl-private.h b/cogl/cogl/winsys/cogl-winsys-egl-private.h
index 5d21b4f4e..8d3ab512b 100644
--- a/cogl/cogl/winsys/cogl-winsys-egl-private.h
+++ b/cogl/cogl/winsys/cogl-winsys-egl-private.h
@@ -90,6 +90,11 @@ typedef struct _CoglWinsysEGLVtable
   (* add_config_attributes) (CoglDisplay *display,
                              CoglFramebufferConfig *config,
                              EGLint *attributes);
+  CoglBool
+  (* choose_config) (CoglDisplay *display,
+                     EGLint *attributes,
+                     EGLConfig *out_config,
+                     CoglError **error);
 } CoglWinsysEGLVtable;
 
 typedef enum _CoglEGLWinsysFeature
diff --git a/cogl/cogl/winsys/cogl-winsys-egl-x11.c b/cogl/cogl/winsys/cogl-winsys-egl-x11.c
index d72ba02f4..c1d01dccd 100644
--- a/cogl/cogl/winsys/cogl-winsys-egl-x11.c
+++ b/cogl/cogl/winsys/cogl-winsys-egl-x11.c
@@ -337,6 +337,32 @@ _cogl_winsys_egl_add_config_attributes (CoglDisplay *display,
 }
 
 static CoglBool
+_cogl_winsys_egl_choose_config (CoglDisplay *display,
+                                EGLint *attributes,
+                                EGLConfig *out_config,
+                                CoglError **error)
+{
+  CoglRenderer *renderer = display->renderer;
+  CoglRendererEGL *egl_renderer = renderer->winsys;
+  EGLint config_count = 0;
+  EGLBoolean status;
+
+  status = eglChooseConfig (egl_renderer->edpy,
+                            attributes,
+                            out_config, 1,
+                            &config_count);
+  if (status != EGL_TRUE || config_count == 0)
+    {
+      _cogl_set_error (error, COGL_WINSYS_ERROR,
+                       COGL_WINSYS_ERROR_CREATE_CONTEXT,
+                       "No compatible EGL configs found");
+      return FALSE;
+    }
+
+  return TRUE;
+}
+
+static CoglBool
 _cogl_winsys_egl_display_setup (CoglDisplay *display,
                                 CoglError **error)
 {
@@ -841,6 +867,7 @@ static const CoglWinsysEGLVtable
 _cogl_winsys_egl_vtable =
   {
     .add_config_attributes = _cogl_winsys_egl_add_config_attributes,
+    .choose_config = _cogl_winsys_egl_choose_config,
     .display_setup = _cogl_winsys_egl_display_setup,
     .display_destroy = _cogl_winsys_egl_display_destroy,
     .context_created = _cogl_winsys_egl_context_created,
diff --git a/cogl/cogl/winsys/cogl-winsys-egl.c b/cogl/cogl/winsys/cogl-winsys-egl.c
index 8f2bcc7de..fbd6112d4 100644
--- a/cogl/cogl/winsys/cogl-winsys-egl.c
+++ b/cogl/cogl/winsys/cogl-winsys-egl.c
@@ -338,10 +338,9 @@ try_create_context (CoglDisplay *display,
   CoglRendererEGL *egl_renderer = renderer->winsys;
   EGLDisplay edpy;
   EGLConfig config;
-  EGLint config_count = 0;
-  EGLBoolean status;
   EGLint attribs[9];
   EGLint cfg_attribs[MAX_EGL_CONFIG_ATTRIBS];
+  GError *config_error = NULL;
   const char *error_message;
 
   _COGL_RETURN_VAL_IF_FAIL (egl_display->egl_context == NULL, TRUE);
@@ -356,14 +355,16 @@ try_create_context (CoglDisplay *display,
 
   edpy = egl_renderer->edpy;
 
-  status = eglChooseConfig (edpy,
-                            cfg_attribs,
-                            &config, 1,
-                            &config_count);
-  if (status != EGL_TRUE || config_count == 0)
+  if (!egl_renderer->platform_vtable->choose_config (display,
+                                                     cfg_attribs,
+                                                     &config,
+                                                     &config_error))
     {
-      error_message = "Unable to find a usable EGL configuration";
-      goto fail;
+      _cogl_set_error (error, COGL_WINSYS_ERROR,
+                       COGL_WINSYS_ERROR_CREATE_CONTEXT,
+                       "Couldn't choose config: %s", config_error->message);
+      g_error_free (config_error);
+      goto err;
     }
 
   egl_display->egl_config = config;
@@ -419,6 +420,7 @@ fail:
                COGL_WINSYS_ERROR_CREATE_CONTEXT,
                "%s", error_message);
 
+err:
   cleanup_context (display);
 
   return FALSE;
diff --git a/src/backends/meta-egl.c b/src/backends/meta-egl.c
index 775863f88..755ec4908 100644
--- a/src/backends/meta-egl.c
+++ b/src/backends/meta-egl.c
@@ -266,6 +266,72 @@ meta_egl_get_proc_address (MetaEgl    *egl,
 }
 
 gboolean
+meta_egl_get_config_attrib (MetaEgl     *egl,
+                            EGLDisplay   display,
+                            EGLConfig    config,
+                            EGLint       attribute,
+                            EGLint      *value,
+                            GError     **error)
+{
+  if (!eglGetConfigAttrib (display,
+                           config,
+                           attribute,
+                           value))
+    {
+      set_egl_error (error);
+      return FALSE;
+    }
+
+  return TRUE;
+}
+
+EGLConfig *
+meta_egl_choose_all_configs (MetaEgl       *egl,
+                             EGLDisplay     display,
+                             const EGLint  *attrib_list,
+                             EGLint        *out_num_configs,
+                             GError       **error)
+{
+  EGLint num_configs;
+  EGLConfig *configs;
+  EGLint num_matches;
+
+  if (!eglGetConfigs (display, NULL, 0, &num_configs))
+    {
+      set_egl_error (error);
+      return FALSE;
+    }
+
+  if (num_configs < 1)
+    {
+      g_set_error (error, G_IO_ERROR,
+                   G_IO_ERROR_FAILED,
+                   "No EGL configurations available");
+      return FALSE;
+    }
+
+  configs = g_new0 (EGLConfig, num_configs);
+
+  if (!eglChooseConfig (display, attrib_list, configs, num_configs, &num_matches))
+    {
+      g_free (configs);
+      set_egl_error (error);
+      return FALSE;
+    }
+
+  if (num_matches == 0)
+    {
+      g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+                   "No matching EGL configs");
+      g_free (configs);
+      return NULL;
+    }
+
+  *out_num_configs = num_configs;
+  return configs;
+}
+
+gboolean
 meta_egl_choose_first_config (MetaEgl       *egl,
                               EGLDisplay     display,
                               const EGLint  *attrib_list,
diff --git a/src/backends/meta-egl.h b/src/backends/meta-egl.h
index be4e631d2..060c7cd2d 100644
--- a/src/backends/meta-egl.h
+++ b/src/backends/meta-egl.h
@@ -62,6 +62,19 @@ gboolean meta_egl_choose_first_config (MetaEgl       *egl,
                                        EGLConfig     *chosen_config,
                                        GError       **error);
 
+gboolean meta_egl_get_config_attrib (MetaEgl     *egl,
+                                     EGLDisplay   display,
+                                     EGLConfig    config,
+                                     EGLint       attribute,
+                                     EGLint      *value,
+                                     GError     **error);
+
+EGLConfig * meta_egl_choose_all_configs (MetaEgl       *egl,
+                                         EGLDisplay     display,
+                                         const EGLint  *attrib_list,
+                                         EGLint        *out_num_configs,
+                                         GError       **error);
+
 EGLContext meta_egl_create_context (MetaEgl      *egl,
                                     EGLDisplay    display,
                                     EGLConfig     config,
diff --git a/src/backends/native/meta-renderer-native.c b/src/backends/native/meta-renderer-native.c
index 5c29597cc..8175a6b21 100644
--- a/src/backends/native/meta-renderer-native.c
+++ b/src/backends/native/meta-renderer-native.c
@@ -935,6 +935,73 @@ meta_renderer_native_add_egl_config_attributes (CoglDisplay           *cogl_disp
 }
 
 static gboolean
+choose_egl_config_from_gbm_format (MetaEgl       *egl,
+                                   EGLDisplay     egl_display,
+                                   const EGLint  *attributes,
+                                   uint32_t       gbm_format,
+                                   EGLConfig     *out_config,
+                                   GError       **error)
+{
+  EGLConfig *egl_configs;
+  EGLint n_configs;
+  EGLint i;
+
+  egl_configs = meta_egl_choose_all_configs (egl, egl_display,
+                                             attributes,
+                                             &n_configs,
+                                             error);
+  if (!egl_configs)
+    return FALSE;
+
+  for (i = 0; i < n_configs; i++)
+    {
+      EGLint visual_id;
+
+      if (!meta_egl_get_config_attrib (egl, egl_display,
+                                       egl_configs[i],
+                                       EGL_NATIVE_VISUAL_ID,
+                                       &visual_id,
+                                       error))
+        {
+          g_free (egl_configs);
+          return FALSE;
+        }
+
+      if ((uint32_t) visual_id == gbm_format)
+        {
+          *out_config = egl_configs[i];
+          g_free (egl_configs);
+          return TRUE;
+        }
+    }
+
+  g_free (egl_configs);
+  g_set_error (error, G_IO_ERROR, G_IO_ERROR_FAILED,
+               "No EGL config matching supported GBM format found");
+  return FALSE;
+}
+
+static gboolean
+meta_renderer_native_choose_egl_config (CoglDisplay  *cogl_display,
+                                        EGLint       *attributes,
+                                        EGLConfig    *out_config,
+                                        GError      **error)
+{
+  CoglRenderer *cogl_renderer = cogl_display->renderer;
+  CoglRendererEGL *cogl_renderer_egl = cogl_renderer->winsys;
+  MetaBackend *backend = meta_get_backend ();
+  MetaEgl *egl = meta_backend_get_egl (backend);
+  EGLDisplay egl_display = cogl_renderer_egl->edpy;
+
+  return choose_egl_config_from_gbm_format (egl,
+                                            egl_display,
+                                            attributes,
+                                            GBM_FORMAT_XRGB8888,
+                                            out_config,
+                                            error);
+}
+
+static gboolean
 meta_renderer_native_setup_egl_display (CoglDisplay *cogl_display,
                                         GError     **error)
 {
@@ -2408,6 +2475,7 @@ meta_renderer_native_release_onscreen (CoglOnscreen *onscreen)
 static const CoglWinsysEGLVtable
 _cogl_winsys_egl_vtable = {
   .add_config_attributes = meta_renderer_native_add_egl_config_attributes,
+  .choose_config = meta_renderer_native_choose_egl_config,
   .display_setup = meta_renderer_native_setup_egl_display,
   .display_destroy = meta_renderer_native_destroy_egl_display,
   .context_created = meta_renderer_native_egl_context_created,
@@ -2846,11 +2914,12 @@ create_secondary_egl_config (MetaEgl   *egl,
     EGL_NONE
   };
 
-  return meta_egl_choose_first_config (egl,
-                                       egl_display,
-                                       attributes,
-                                       egl_config,
-                                       error);
+  return choose_egl_config_from_gbm_format (egl,
+                                            egl_display,
+                                            attributes,
+                                            GBM_FORMAT_XRGB8888,
+                                            egl_config,
+                                            error);
 }
 
 static EGLContext