v4l2: fix colorimetry for NV12

Replicate V4L2_MAP_QUANTIZATION_DEFAULT macro behavior.
At #v4l it was described that documentation might be wrong and that
we should trust this macro instead.

https://bugzilla.gnome.org/show_bug.cgi?id=762529

1 files changed, 13 insertions, 4 deletions

diff --git a/sys/v4l2/gstv4l2object.c b/sys/v4l2/gstv4l2object.c
index 3fc52b2cd..c326849dc 100644
--- a/sys/v4l2/gstv4l2object.c
+++ b/sys/v4l2/gstv4l2object.c
@@ -1878,7 +1878,7 @@ gst_v4l2_object_get_interlace_mode (enum v4l2_field field,
 static gboolean
 gst_v4l2_object_get_colorspace (enum v4l2_colorspace colorspace,
     enum v4l2_quantization range, enum v4l2_ycbcr_encoding matrix,
-    enum v4l2_xfer_func transfer, GstVideoColorimetry * cinfo)
+    enum v4l2_xfer_func transfer, gboolean is_rgb, GstVideoColorimetry * cinfo)
 {
   gboolean ret = TRUE;
 
@@ -1958,7 +1958,15 @@ gst_v4l2_object_get_colorspace (enum v4l2_colorspace colorspace,
       cinfo->range = GST_VIDEO_COLOR_RANGE_16_235;
       break;
     case V4L2_QUANTIZATION_DEFAULT:
-      /* nothing, just use defaults for colorspace */
+      /* replicated V4L2_MAP_QUANTIZATION_DEFAULT macro behavior */
+      if (is_rgb && colorspace == V4L2_COLORSPACE_BT2020)
+        cinfo->range = GST_VIDEO_COLOR_RANGE_16_235;
+      else if (is_rgb || matrix == V4L2_YCBCR_ENC_XV601
+          || matrix == V4L2_YCBCR_ENC_XV709
+          || colorspace == V4L2_COLORSPACE_JPEG)
+        cinfo->range = GST_VIDEO_COLOR_RANGE_0_255;
+      else
+        cinfo->range = GST_VIDEO_COLOR_RANGE_16_235;
       break;
     default:
       GST_WARNING ("Unknown enum v4l2_quantization value %d", range);
@@ -2138,6 +2146,7 @@ gst_v4l2_object_add_colorspace (GstV4l2Object * v4l2object, GstStructure * s,
     enum v4l2_quantization range;
     enum v4l2_ycbcr_encoding matrix;
     enum v4l2_xfer_func transfer;
+    gboolean is_rgb = gst_v4l2_object_v4l2fourcc_is_rgb (pixelformat);
 
     if (V4L2_TYPE_IS_MULTIPLANAR (v4l2object->type)) {
       colorspace = fmt.fmt.pix_mp.colorspace;
@@ -2152,11 +2161,11 @@ gst_v4l2_object_add_colorspace (GstV4l2Object * v4l2object, GstStructure * s,
     }
 
     if (gst_v4l2_object_get_colorspace (colorspace, range, matrix, transfer,
-            &cinfo)) {
+            is_rgb, &cinfo)) {
       /* Set identity matrix for R'G'B' formats to avoid creating
        * confusion. This though is cosmetic as it's now properly ignored by
        * the video info API and videoconvert. */
-      if (gst_v4l2_object_v4l2fourcc_is_rgb (pixelformat))
+      if (is_rgb)
         cinfo.matrix = GST_VIDEO_COLOR_MATRIX_RGB;
 
       g_value_init (&colorimetry, G_TYPE_STRING);