vaapidecode: Rework the re-negotiation code to handle multi resoultion videos

Delaying the pool re-negotiation untill we push all decoded (and queued)
frames downstream. Otherwise for the multi-resolution videos, the
GstVideoVideoMemory will be having wrong resolution and which leads
to nasty behaviours, especially when using software renderers.

sample media file: RAP_B_Bossen_1.bin
case explained:
The first SPS Nal will report resoultion of 448x256 and having crop rectangles to
get the final resoultion 416x240.
Starting from 25 th frame, the resolution will change to 416x240. But parser
elements won't report this since the effective croped resolution is same in
both cases. Here the core libgstvaapi will detect this through it's internal
parsing and do all context/pool destory/reset stuffs. Also it will notify this
change to plugins in advance. But if the plugin try to do re-negotiaion of pool
immediately, this will not sync with the resolution of already decoded and queued
frames and which will lead to failure in gst_video_frame_map() in downstream(if we use the
software renderer). So we have to delay the pool renegotiation in vaapidecode,
untill we push all decoded frames downstream.

https://bugzilla.gnome.org/show_bug.cgi?id=753914

2 files changed, 34 insertions, 6 deletions

diff --git a/gst/vaapi/gstvaapidecode.c b/gst/vaapi/gstvaapidecode.c
index e40ee94b..e0378b52 100644
--- a/gst/vaapi/gstvaapidecode.c
+++ b/gst/vaapi/gstvaapidecode.c
@@ -107,6 +107,8 @@ G_DEFINE_TYPE_WITH_CODE(
     GST_VAAPI_PLUGIN_BASE_INIT_INTERFACES)
 /* *INDENT-ON* */
 
+static gboolean
+gst_vaapidecode_update_sink_caps (GstVaapiDecode * decode, GstCaps * caps);
 static gboolean gst_vaapidecode_update_src_caps (GstVaapiDecode * decode);
 
 static gboolean
@@ -130,12 +132,10 @@ gst_vaapi_decoder_state_changed (GstVaapiDecoder * decoder,
 
   if (!gst_vaapi_decode_input_state_replace (decode, codec_state))
     return;
-  if (!gst_vaapidecode_update_src_caps (decode))
-    return;
-  if (!gst_video_decoder_negotiate (vdec))
-    return;
-  if (!gst_vaapi_plugin_base_set_caps (plugin, NULL, decode->srcpad_caps))
-    return;
+  if (!gst_vaapidecode_update_sink_caps (decode, decode->input_state->caps))
+    return FALSE;
+
+  decode->do_renego = TRUE;
 }
 
 static GstVideoCodecState *
@@ -379,6 +379,31 @@ error_commit_buffer:
   }
 }
 
+static gboolean
+gst_vaapidecode_negotiate (GstVaapiDecode * decode)
+{
+  GstVideoDecoder *const vdec = GST_VIDEO_DECODER (decode);
+  GstVaapiPluginBase *const plugin = GST_VAAPI_PLUGIN_BASE (vdec);
+
+  if (!decode->do_renego)
+    return TRUE;
+
+  GST_DEBUG_OBJECT (decode, "Input codec state changed, doing renegotiation");
+
+  if (!gst_vaapi_plugin_base_set_caps (plugin, decode->sinkpad_caps, NULL))
+    return FALSE;
+  if (!gst_vaapidecode_update_src_caps (decode))
+    return FALSE;
+  if (!gst_video_decoder_negotiate (vdec))
+    return FALSE;
+  if (!gst_vaapi_plugin_base_set_caps (plugin, NULL, decode->srcpad_caps))
+    return FALSE;
+
+  decode->do_renego = FALSE;
+
+  return TRUE;
+}
+
 static GstFlowReturn
 gst_vaapidecode_push_all_decoded_frames (GstVaapiDecode * decode)
 {
@@ -397,6 +422,8 @@ gst_vaapidecode_push_all_decoded_frames (GstVaapiDecode * decode)
           return ret;
         break;
       case GST_VAAPI_DECODER_STATUS_ERROR_NO_DATA:
+        if (!gst_vaapidecode_negotiate (decode))
+          return GST_FLOW_ERROR;
         return GST_FLOW_OK;
       default:
         GST_VIDEO_DECODER_ERROR (vdec, 1, STREAM, DECODE, ("Decoding failed"),
diff --git a/gst/vaapi/gstvaapidecode.h b/gst/vaapi/gstvaapidecode.h
index 48807dda..13c57bc0 100644
--- a/gst/vaapi/gstvaapidecode.h
+++ b/gst/vaapi/gstvaapidecode.h
@@ -73,6 +73,7 @@ struct _GstVaapiDecode {
 
     GstVideoCodecState *input_state;
     volatile gboolean   active;
+    volatile gboolean   do_renego;
 };
 
 struct _GstVaapiDecodeClass {