vulkan/wsi/wayland: don't expose surface formats not fully supported

Depending on whether an application creates a swapchain with
VK_COMPOSITE_ALPHA_PRE_MULTIPLIED_BIT_KHR or not, we might use 2
different formats with the compositor.

This change makes sure that we support all the underlying formats
before exposing the corresponding VkFormat to the application.

v2: Don't forget get_formats2() (Ivan)

v3: Replace formats with availability boolean (Simon)

Signed-off-by: Lionel Landwerlin <lionel.g.landwerlin@intel.com>
Fixes: 151b65b21190 ("vulkan/wsi/wayland: generalize modifier handling")
Closes: https://gitlab.freedesktop.org/mesa/mesa/-/issues/5522
Reviewed-by: Ivan Briano <ivan.briano@intel.com> (v2)
Reviewed-by: Simon Ser <contact@emersion.fr>
Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/13453>
(cherry picked from commit d944136f3635aeacd97e167176c968c5078d92d7)

2 files changed, 119 insertions, 30 deletions

diff --git a/.pick_status.json b/.pick_status.json
index 48296b88307..72a777b8844 100644
--- a/.pick_status.json
+++ b/.pick_status.json
@@ -454,7 +454,7 @@
         "description": "vulkan/wsi/wayland: don't expose surface formats not fully supported",
         "nominated": true,
         "nomination_type": 1,
-        "resolution": 0,
+        "resolution": 1,
         "main_sha": null,
         "because_sha": "151b65b211907d92f3012997aba12b7dbe004e61"
     },
diff --git a/src/vulkan/wsi/wsi_common_wayland.c b/src/vulkan/wsi/wsi_common_wayland.c
index 906ccde5846..09a4f67b226 100644
--- a/src/vulkan/wsi/wsi_common_wayland.c
+++ b/src/vulkan/wsi/wsi_common_wayland.c
@@ -52,6 +52,8 @@ struct wsi_wayland;
 
 struct wsi_wl_format {
    VkFormat vk_format;
+   uint32_t has_alpha_format;
+   uint32_t has_opaque_format;
    struct u_vector modifiers;
 };
 
@@ -99,12 +101,20 @@ find_format(struct u_vector *formats, VkFormat format)
 
 static struct wsi_wl_format *
 wsi_wl_display_add_vk_format(struct wsi_wl_display *display,
-                             struct u_vector *formats, VkFormat format)
+                             struct u_vector *formats,
+                             VkFormat format,
+                             bool has_alpha_format,
+                             bool has_opaque_format)
 {
    /* Don't add a format that's already in the list */
    struct wsi_wl_format *f = find_format(formats, format);
-   if (f)
+   if (f) {
+      if (has_alpha_format)
+         f->has_alpha_format = true;
+      if (has_opaque_format)
+         f->has_opaque_format = true;
       return f;
+   }
 
    /* Don't add formats that aren't renderable. */
    VkFormatProperties props;
@@ -125,6 +135,8 @@ wsi_wl_display_add_vk_format(struct wsi_wl_display *display,
    }
 
    f->vk_format = format;
+   f->has_alpha_format = has_alpha_format;
+   f->has_opaque_format = has_opaque_format;
    f->modifiers = modifiers;
 
    return f;
@@ -159,14 +171,24 @@ wsi_wl_display_add_drm_format_modifier(struct wsi_wl_display *display,
    /* TODO: These are only available when VK_EXT_4444_formats is enabled, so
     * we probably need to make their use conditional on this extension. */
    case DRM_FORMAT_ARGB4444:
+      format = wsi_wl_display_add_vk_format(display, formats,
+                                            VK_FORMAT_A4R4G4B4_UNORM_PACK16_EXT,
+                                            true, false);
+      break;
    case DRM_FORMAT_XRGB4444:
       format = wsi_wl_display_add_vk_format(display, formats,
-                                            VK_FORMAT_A4R4G4B4_UNORM_PACK16_EXT);
+                                            VK_FORMAT_A4R4G4B4_UNORM_PACK16_EXT,
+                                            false, true);
       break;
    case DRM_FORMAT_ABGR4444:
+      format = wsi_wl_display_add_vk_format(display, formats,
+                                            VK_FORMAT_A4B4G4R4_UNORM_PACK16_EXT,
+                                            true, false);
+      break;
    case DRM_FORMAT_XBGR4444:
       format = wsi_wl_display_add_vk_format(display, formats,
-                                            VK_FORMAT_A4B4G4R4_UNORM_PACK16_EXT);
+                                            VK_FORMAT_A4B4G4R4_UNORM_PACK16_EXT,
+                                            false, true);
       break;
 #endif
 
@@ -174,47 +196,84 @@ wsi_wl_display_add_drm_format_modifier(struct wsi_wl_display *display,
     * on little endian systems, on big endian there exists no analog. */
 #if MESA_LITTLE_ENDIAN
    case DRM_FORMAT_RGBA4444:
+      format = wsi_wl_display_add_vk_format(display, formats,
+                                            VK_FORMAT_R4G4B4A4_UNORM_PACK16,
+                                            true, false);
+      break;
    case DRM_FORMAT_RGBX4444:
       format = wsi_wl_display_add_vk_format(display, formats,
-                                            VK_FORMAT_R4G4B4A4_UNORM_PACK16);
+                                            VK_FORMAT_R4G4B4A4_UNORM_PACK16,
+                                            false, true);
       break;
    case DRM_FORMAT_BGRA4444:
+      format = wsi_wl_display_add_vk_format(display, formats,
+                                            VK_FORMAT_B4G4R4A4_UNORM_PACK16,
+                                            true, false);
+      break;
    case DRM_FORMAT_BGRX4444:
       format = wsi_wl_display_add_vk_format(display, formats,
-                                            VK_FORMAT_B4G4R4A4_UNORM_PACK16);
+                                            VK_FORMAT_B4G4R4A4_UNORM_PACK16,
+                                            false, true);
       break;
    case DRM_FORMAT_RGB565:
       format = wsi_wl_display_add_vk_format(display, formats,
-                                            VK_FORMAT_R5G6B5_UNORM_PACK16);
+                                            VK_FORMAT_R5G6B5_UNORM_PACK16,
+                                            true, true);
       break;
    case DRM_FORMAT_BGR565:
       format = wsi_wl_display_add_vk_format(display, formats,
-                                            VK_FORMAT_B5G6R5_UNORM_PACK16);
+                                            VK_FORMAT_B5G6R5_UNORM_PACK16,
+                                            true, true);
       break;
    case DRM_FORMAT_ARGB1555:
+      format = wsi_wl_display_add_vk_format(display, formats,
+                                            VK_FORMAT_A1R5G5B5_UNORM_PACK16,
+                                            true, false);
+      break;
    case DRM_FORMAT_XRGB1555:
       format = wsi_wl_display_add_vk_format(display, formats,
-                                            VK_FORMAT_A1R5G5B5_UNORM_PACK16);
+                                            VK_FORMAT_A1R5G5B5_UNORM_PACK16,
+                                            false, true);
       break;
    case DRM_FORMAT_RGBA5551:
+      format = wsi_wl_display_add_vk_format(display, formats,
+                                            VK_FORMAT_R5G5B5A1_UNORM_PACK16,
+                                            true, false);
+      break;
    case DRM_FORMAT_RGBX5551:
       format = wsi_wl_display_add_vk_format(display, formats,
-                                            VK_FORMAT_R5G5B5A1_UNORM_PACK16);
+                                            VK_FORMAT_R5G5B5A1_UNORM_PACK16,
+                                            false, true);
       break;
    case DRM_FORMAT_BGRA5551:
+      format = wsi_wl_display_add_vk_format(display, formats,
+                                            VK_FORMAT_B5G5R5A1_UNORM_PACK16,
+                                            true, false);
+      break;
    case DRM_FORMAT_BGRX5551:
       format = wsi_wl_display_add_vk_format(display, formats,
-                                            VK_FORMAT_B5G5R5A1_UNORM_PACK16);
+                                            VK_FORMAT_B5G5R5A1_UNORM_PACK16,
+                                            false, true);
       break;
    case DRM_FORMAT_ARGB2101010:
+      format = wsi_wl_display_add_vk_format(display, formats,
+                                            VK_FORMAT_A2R10G10B10_UNORM_PACK32,
+                                            true, false);
+      break;
    case DRM_FORMAT_XRGB2101010:
       format = wsi_wl_display_add_vk_format(display, formats,
-                                            VK_FORMAT_A2R10G10B10_UNORM_PACK32);
+                                            VK_FORMAT_A2R10G10B10_UNORM_PACK32,
+                                            false, true);
       break;
    case DRM_FORMAT_ABGR2101010:
+      format = wsi_wl_display_add_vk_format(display, formats,
+                                            VK_FORMAT_A2B10G10R10_UNORM_PACK32,
+                                            true, false);
+      break;
    case DRM_FORMAT_XBGR2101010:
       format = wsi_wl_display_add_vk_format(display, formats,
-                                            VK_FORMAT_A2B10G10R10_UNORM_PACK32);
+                                            VK_FORMAT_A2B10G10R10_UNORM_PACK32,
+                                            false, true);
       break;
 #endif
 
@@ -230,27 +289,35 @@ wsi_wl_display_add_drm_format_modifier(struct wsi_wl_display *display,
     * Vulkan interprets the pixel data. */
    case DRM_FORMAT_XBGR8888:
       srgb_format = wsi_wl_display_add_vk_format(display, formats,
-                                                 VK_FORMAT_R8G8B8_SRGB);
+                                                 VK_FORMAT_R8G8B8_SRGB,
+                                                 true, true);
       format = wsi_wl_display_add_vk_format(display, formats,
-                                            VK_FORMAT_R8G8B8_UNORM);
+                                            VK_FORMAT_R8G8B8_UNORM,
+                                            true, true);
       FALLTHROUGH;
    case DRM_FORMAT_ABGR8888:
       srgb_format = wsi_wl_display_add_vk_format(display, formats,
-                                                 VK_FORMAT_R8G8B8A8_SRGB);
+                                                 VK_FORMAT_R8G8B8A8_SRGB,
+                                                 true, true);
       format = wsi_wl_display_add_vk_format(display, formats,
-                                            VK_FORMAT_R8G8B8A8_UNORM);
+                                            VK_FORMAT_R8G8B8A8_UNORM,
+                                            true, true);
       break;
    case DRM_FORMAT_XRGB8888:
       srgb_format = wsi_wl_display_add_vk_format(display, formats,
-                                                 VK_FORMAT_B8G8R8_SRGB);
+                                                 VK_FORMAT_B8G8R8_SRGB,
+                                                 true, true);
       format = wsi_wl_display_add_vk_format(display, formats,
-                                            VK_FORMAT_B8G8R8_UNORM);
+                                            VK_FORMAT_B8G8R8_UNORM,
+                                            true, true);
       FALLTHROUGH;
    case DRM_FORMAT_ARGB8888:
       srgb_format = wsi_wl_display_add_vk_format(display, formats,
-                                                 VK_FORMAT_B8G8R8A8_SRGB);
+                                                 VK_FORMAT_B8G8R8A8_SRGB,
+                                                 true, true);
       format = wsi_wl_display_add_vk_format(display, formats,
-                                            VK_FORMAT_B8G8R8A8_UNORM);
+                                            VK_FORMAT_B8G8R8A8_UNORM,
+                                            true, true);
       break;
    }
 
@@ -268,27 +335,35 @@ wsi_wl_display_add_wl_shm_format(struct wsi_wl_display *display,
    switch (wl_shm_format) {
    case WL_SHM_FORMAT_XBGR8888:
       wsi_wl_display_add_vk_format(display, formats,
-                                   VK_FORMAT_R8G8B8_SRGB);
+                                   VK_FORMAT_R8G8B8_SRGB,
+                                   false, true);
       wsi_wl_display_add_vk_format(display, formats,
-                                   VK_FORMAT_R8G8B8_UNORM);
+                                   VK_FORMAT_R8G8B8_UNORM,
+                                   false, true);
       FALLTHROUGH;
    case WL_SHM_FORMAT_ABGR8888:
       wsi_wl_display_add_vk_format(display, formats,
-                                   VK_FORMAT_R8G8B8A8_SRGB);
+                                   VK_FORMAT_R8G8B8A8_SRGB,
+                                   true, false);
       wsi_wl_display_add_vk_format(display, formats,
-                                   VK_FORMAT_R8G8B8A8_UNORM);
+                                   VK_FORMAT_R8G8B8A8_UNORM,
+                                   true, false);
       break;
    case WL_SHM_FORMAT_XRGB8888:
       wsi_wl_display_add_vk_format(display, formats,
-                                   VK_FORMAT_B8G8R8_SRGB);
+                                   VK_FORMAT_B8G8R8_SRGB,
+                                   false, true);
       wsi_wl_display_add_vk_format(display, formats,
-                                   VK_FORMAT_B8G8R8_UNORM);
+                                   VK_FORMAT_B8G8R8_UNORM,
+                                   false, true);
       FALLTHROUGH;
    case WL_SHM_FORMAT_ARGB8888:
       wsi_wl_display_add_vk_format(display, formats,
-                                   VK_FORMAT_B8G8R8A8_SRGB);
+                                   VK_FORMAT_B8G8R8A8_SRGB,
+                                   true, false);
       wsi_wl_display_add_vk_format(display, formats,
-                                   VK_FORMAT_B8G8R8A8_UNORM);
+                                   VK_FORMAT_B8G8R8A8_UNORM,
+                                   true, false);
       break;
    }
 }
@@ -698,6 +773,13 @@ wsi_wl_surface_get_formats(VkIcdSurfaceBase *icd_surface,
 
    struct wsi_wl_format *disp_fmt;
    u_vector_foreach(disp_fmt, &display.formats) {
+      /* Skip formats for which we can't support both alpha & opaque
+       * formats.
+       */
+      if (!disp_fmt->has_opaque_format ||
+          !disp_fmt->has_alpha_format)
+         continue;
+
       vk_outarray_append(&out, out_fmt) {
          out_fmt->format = disp_fmt->vk_format;
          out_fmt->colorSpace = VK_COLORSPACE_SRGB_NONLINEAR_KHR;
@@ -729,6 +811,13 @@ wsi_wl_surface_get_formats2(VkIcdSurfaceBase *icd_surface,
 
    struct wsi_wl_format *disp_fmt;
    u_vector_foreach(disp_fmt, &display.formats) {
+      /* Skip formats for which we can't support both alpha & opaque
+       * formats.
+       */
+      if (!disp_fmt->has_opaque_format ||
+          !disp_fmt->has_alpha_format)
+         continue;
+
       vk_outarray_append(&out, out_fmt) {
          out_fmt->surfaceFormat.format = disp_fmt->vk_format;
          out_fmt->surfaceFormat.colorSpace = VK_COLORSPACE_SRGB_NONLINEAR_KHR;