gl-renderer: Add support for a few YUV dmabuf formats

Namely the single-planar YUYV, the two-planar NV12, and the
three-planar YUV420, using the shaders already present in Weston.

Signed-off-by: Emmanuel Gil Peyrot <emmanuel.peyrot@collabora.com>
Reviewed-by: Derek Foreman <derekf@osg.samsung.com>

Maniphest Tasks: T13

Differential Revision: https://phabricator.freedesktop.org/D334

1 files changed, 252 insertions, 43 deletions

diff --git a/src/gl-renderer.c b/src/gl-renderer.c
index d4d92879..0ee20756 100644
--- a/src/gl-renderer.c
+++ b/src/gl-renderer.c
@@ -101,11 +101,36 @@ struct egl_image {
 	int refcount;
 };
 
+enum import_type {
+	IMPORT_TYPE_INVALID,
+	IMPORT_TYPE_DIRECT,
+	IMPORT_TYPE_GL_CONVERSION
+};
+
 struct dmabuf_image {
 	struct linux_dmabuf_buffer *dmabuf;
 	int num_images;
 	struct egl_image *images[3];
 	struct wl_list link;
+
+	enum import_type import_type;
+	GLenum target;
+	struct gl_shader *shader;
+};
+
+struct yuv_plane_descriptor {
+	int width_divisor;
+	int height_divisor;
+	uint32_t format;
+	int plane_index;
+};
+
+struct yuv_format_descriptor {
+	uint32_t format;
+	int input_planes;
+	int output_planes;
+	int texture_type;
+	struct yuv_plane_descriptor plane[4];
 };
 
 struct gl_surface_state {
@@ -1517,25 +1542,219 @@ import_simple_dmabuf(struct gl_renderer *gr,
 	return image;
 }
 
+/* The kernel header drm_fourcc.h defines the DRM formats below.  We duplicate
+ * some of the definitions here so that building Weston won't require
+ * bleeding-edge kernel headers.
+ */
+#ifndef DRM_FORMAT_R8
+#define DRM_FORMAT_R8            fourcc_code('R', '8', ' ', ' ') /* [7:0] R */
+#endif
+
+#ifndef DRM_FORMAT_GR88
+#define DRM_FORMAT_GR88          fourcc_code('G', 'R', '8', '8') /* [15:0] G:R 8:8 little endian */
+#endif
+
+struct yuv_format_descriptor yuv_formats[] = {
+	{
+		.format = DRM_FORMAT_YUYV,
+		.input_planes = 1,
+		.output_planes = 2,
+		.texture_type = EGL_TEXTURE_Y_XUXV_WL,
+		{{
+			.width_divisor = 1,
+			.height_divisor = 1,
+			.format = DRM_FORMAT_GR88,
+			.plane_index = 0
+		}, {
+			.width_divisor = 2,
+			.height_divisor = 1,
+			.format = DRM_FORMAT_ARGB8888,
+			.plane_index = 0
+		}}
+	}, {
+		.format = DRM_FORMAT_NV12,
+		.input_planes = 2,
+		.output_planes = 2,
+		.texture_type = EGL_TEXTURE_Y_UV_WL,
+		{{
+			.width_divisor = 1,
+			.height_divisor = 1,
+			.format = DRM_FORMAT_R8,
+			.plane_index = 0
+		}, {
+			.width_divisor = 2,
+			.height_divisor = 2,
+			.format = DRM_FORMAT_GR88,
+			.plane_index = 1
+		}}
+	}, {
+		.format = DRM_FORMAT_YUV420,
+		.input_planes = 3,
+		.output_planes = 3,
+		.texture_type = EGL_TEXTURE_Y_U_V_WL,
+		{{
+			.width_divisor = 1,
+			.height_divisor = 1,
+			.format = DRM_FORMAT_R8,
+			.plane_index = 0
+		}, {
+			.width_divisor = 2,
+			.height_divisor = 2,
+			.format = DRM_FORMAT_R8,
+			.plane_index = 1
+		}, {
+			.width_divisor = 2,
+			.height_divisor = 2,
+			.format = DRM_FORMAT_R8,
+			.plane_index = 2
+		}}
+	}
+};
+
+static struct egl_image *
+import_dmabuf_single_plane(struct gl_renderer *gr,
+                           const struct dmabuf_attributes *attributes,
+                           struct yuv_plane_descriptor *descriptor)
+{
+	struct dmabuf_attributes plane;
+	struct egl_image *image;
+	char fmt[4];
+
+	plane.width = attributes->width / descriptor->width_divisor;
+	plane.height = attributes->height / descriptor->height_divisor;
+	plane.format = descriptor->format;
+	plane.n_planes = 1;
+	plane.fd[0] = attributes->fd[descriptor->plane_index];
+	plane.offset[0] = attributes->offset[descriptor->plane_index];
+	plane.stride[0] = attributes->stride[descriptor->plane_index];
+	plane.modifier[0] = attributes->modifier[descriptor->plane_index];
+
+	image = import_simple_dmabuf(gr, &plane);
+	if (!image) {
+		weston_log("Failed to import plane %d as %.4s\n",
+		           descriptor->plane_index,
+		           dump_format(descriptor->format, fmt));
+		return NULL;
+	}
+
+	return image;
+}
+
+static bool
+import_yuv_dmabuf(struct gl_renderer *gr,
+                  struct dmabuf_image *image)
+{
+	unsigned i;
+	int j;
+	int ret;
+	struct yuv_format_descriptor *format = NULL;
+	struct dmabuf_attributes *attributes = &image->dmabuf->attributes;
+	char fmt[4];
+
+	for (i = 0; i < ARRAY_LENGTH(yuv_formats); ++i) {
+		if (yuv_formats[i].format == attributes->format) {
+			format = &yuv_formats[i];
+			break;
+		}
+	}
+
+	if (!format) {
+		weston_log("Error during import, and no known conversion for format "
+		           "%.4s in the renderer",
+		           dump_format(attributes->format, fmt));
+		return false;
+	}
+
+	if (attributes->n_planes != format->input_planes) {
+		weston_log("%.4s dmabuf must contain %d plane%s (%d provided)",
+		           dump_format(format->format, fmt),
+		           format->input_planes,
+		           (format->input_planes > 1) ? "s" : "",
+		           attributes->n_planes);
+		return false;
+	}
+
+	for (j = 0; j < format->output_planes; ++j) {
+		image->images[j] = import_dmabuf_single_plane(gr, attributes,
+		                                              &format->plane[j]);
+		if (!image->images[j]) {
+			while (j) {
+				ret = egl_image_unref(image->images[--j]);
+				assert(ret == 0);
+			}
+			return false;
+		}
+	}
+
+	image->num_images = format->output_planes;
+
+	switch (format->texture_type) {
+	case EGL_TEXTURE_Y_XUXV_WL:
+		image->shader = &gr->texture_shader_y_xuxv;
+		break;
+	case EGL_TEXTURE_Y_UV_WL:
+		image->shader = &gr->texture_shader_y_uv;
+		break;
+	case EGL_TEXTURE_Y_U_V_WL:
+		image->shader = &gr->texture_shader_y_u_v;
+		break;
+	default:
+		assert(false);
+	}
+
+	return true;
+}
+
+static GLenum
+choose_texture_target(struct dmabuf_attributes *attributes)
+{
+	if (attributes->n_planes > 1)
+		return GL_TEXTURE_EXTERNAL_OES;
+
+	switch (attributes->format & ~DRM_FORMAT_BIG_ENDIAN) {
+	case DRM_FORMAT_YUYV:
+	case DRM_FORMAT_YVYU:
+	case DRM_FORMAT_UYVY:
+	case DRM_FORMAT_VYUY:
+	case DRM_FORMAT_AYUV:
+		return GL_TEXTURE_EXTERNAL_OES;
+	default:
+		return GL_TEXTURE_2D;
+	}
+}
+
 static struct dmabuf_image *
 import_dmabuf(struct gl_renderer *gr,
 	      struct linux_dmabuf_buffer *dmabuf)
 {
 	struct egl_image *egl_image;
 	struct dmabuf_image *image;
-	char fmt[4];
-
-	egl_image = import_simple_dmabuf(gr, &dmabuf->attributes);
-	if (!egl_image) {
-		weston_log("Format %.4s unsupported by EGL, aborting\n",
-		           dump_format(dmabuf->attributes.format, fmt));
-		return NULL;
-	}
 
 	image = dmabuf_image_create();
 	image->dmabuf = dmabuf;
-	image->num_images = 1;
-	image->images[0] = egl_image;
+
+	egl_image = import_simple_dmabuf(gr, &dmabuf->attributes);
+	if (egl_image) {
+		image->num_images = 1;
+		image->images[0] = egl_image;
+		image->import_type = IMPORT_TYPE_DIRECT;
+		image->target = choose_texture_target(&dmabuf->attributes);
+
+		switch (image->target) {
+		case GL_TEXTURE_2D:
+			image->shader = &gr->texture_shader_rgba;
+			break;
+		default:
+			image->shader = &gr->texture_shader_egl_external;
+		}
+	} else {
+		if (!import_yuv_dmabuf(gr, image)) {
+			dmabuf_image_destroy(image);
+			return NULL;
+		}
+		image->import_type = IMPORT_TYPE_GL_CONVERSION;
+		image->target = GL_TEXTURE_2D;
+	}
 
 	return image;
 }
@@ -1571,22 +1790,28 @@ gl_renderer_import_dmabuf(struct weston_compositor *ec,
 	return true;
 }
 
-static GLenum
-choose_texture_target(struct dmabuf_attributes *attributes)
+static bool
+import_known_dmabuf(struct gl_renderer *gr,
+                    struct dmabuf_image *image)
 {
-	if (attributes->n_planes > 1)
-		return GL_TEXTURE_EXTERNAL_OES;
+	switch (image->import_type) {
+	case IMPORT_TYPE_DIRECT:
+		image->images[0] = import_simple_dmabuf(gr, &image->dmabuf->attributes);
+		if (!image->images[0])
+			return false;
+		break;
+
+	case IMPORT_TYPE_GL_CONVERSION:
+		if (!import_yuv_dmabuf(gr, image))
+			return false;
+		break;
 
-	switch (attributes->format & ~DRM_FORMAT_BIG_ENDIAN) {
-	case DRM_FORMAT_YUYV:
-	case DRM_FORMAT_YVYU:
-	case DRM_FORMAT_UYVY:
-	case DRM_FORMAT_VYUY:
-	case DRM_FORMAT_AYUV:
-		return GL_TEXTURE_EXTERNAL_OES;
 	default:
-		return GL_TEXTURE_2D;
+		weston_log("Invalid import type for dmabuf\n");
+		return false;
 	}
+
+	return true;
 }
 
 static void
@@ -1615,15 +1840,6 @@ gl_renderer_attach_dmabuf(struct weston_surface *surface,
 		egl_image_unref(gs->images[i]);
 	gs->num_images = 0;
 
-	gs->target = choose_texture_target(&dmabuf->attributes);
-	switch (gs->target) {
-	case GL_TEXTURE_2D:
-		gs->shader = &gr->texture_shader_rgba;
-		break;
-	default:
-		gs->shader = &gr->texture_shader_egl_external;
-	}
-
 	/*
 	 * We try to always hold an imported EGLImage from the dmabuf
 	 * to prevent the client from preventing re-imports. But, we also
@@ -1642,24 +1858,16 @@ gl_renderer_attach_dmabuf(struct weston_surface *surface,
 		assert(ret == 0);
 	}
 
-	for (i = 0; i < image->num_images; ++i) {
-		image->images[i] = import_simple_dmabuf(gr, &image->dmabuf->attributes);
-		if (!image->images[i]) {
-			image->num_images = 0;
-			while (i) {
-				ret = egl_image_unref(image->images[--i]);
-				assert(ret == 0);
-			}
-			linux_dmabuf_buffer_send_server_error(dmabuf,
-							      "EGL dmabuf import failed");
-			return;
-		}
+	if (!import_known_dmabuf(gr, image)) {
+		linux_dmabuf_buffer_send_server_error(dmabuf, "EGL dmabuf import failed");
+		return;
 	}
 
 	gs->num_images = image->num_images;
 	for (i = 0; i < gs->num_images; ++i)
 		gs->images[i] = egl_image_ref(image->images[i]);
 
+	gs->target = image->target;
 	ensure_textures(gs, gs->num_images);
 	for (i = 0; i < gs->num_images; ++i) {
 		glActiveTexture(GL_TEXTURE0 + i);
@@ -1667,6 +1875,7 @@ gl_renderer_attach_dmabuf(struct weston_surface *surface,
 		gr->image_target_texture_2d(gs->target, gs->images[i]->image);
 	}
 
+	gs->shader = image->shader;
 	gs->pitch = buffer->width;
 	gs->height = buffer->height;
 	gs->buffer_type = BUFFER_TYPE_EGL;