multiview: initial attempt at stereo/multiview support

Add support for marking caps and buffers for multiview or
stereoscopic output.

https://bugzilla.gnome.org/show_bug.cgi?id=750835

6 files changed, 93 insertions, 2 deletions

diff --git a/gst-libs/gst/vaapi/gstvaapidecoder.c b/gst-libs/gst/vaapi/gstvaapidecoder.c
index d1bf5a19..72367d9f 100644
--- a/gst-libs/gst/vaapi/gstvaapidecoder.c
+++ b/gst-libs/gst/vaapi/gstvaapidecoder.c
@@ -924,6 +924,35 @@ gst_vaapi_decoder_set_interlaced (GstVaapiDecoder * decoder,
           GST_VIDEO_INTERLACE_MODE_PROGRESSIVE));
 }
 
+#if GST_CHECK_VERSION(1,5,0)
+void
+gst_vaapi_decoder_set_multiview_mode (GstVaapiDecoder * decoder,
+    gint views, GstVideoMultiviewMode mv_mode, GstVideoMultiviewFlags mv_flags)
+{
+  GstVideoCodecState *const codec_state = decoder->codec_state;
+  GstVideoInfo *info = &codec_state->info;
+
+  if (GST_VIDEO_INFO_VIEWS (info) != views ||
+      GST_VIDEO_INFO_MULTIVIEW_MODE (info) != mv_mode ||
+      GST_VIDEO_INFO_MULTIVIEW_FLAGS (info) != mv_flags) {
+    const gchar *mv_mode_str = gst_video_multiview_mode_to_caps_string (mv_mode);
+
+    GST_DEBUG ("Multiview mode changed to %s flags 0x%x views %d",
+        mv_mode_str, mv_flags, views);
+    GST_VIDEO_INFO_MULTIVIEW_MODE (info) = mv_mode;
+    GST_VIDEO_INFO_MULTIVIEW_FLAGS (info) = mv_flags;
+    GST_VIDEO_INFO_VIEWS (info) = views;
+
+    gst_caps_set_simple (codec_state->caps, "multiview-mode",
+        G_TYPE_STRING, mv_mode_str,
+        "multiview-flags", GST_TYPE_VIDEO_MULTIVIEW_FLAGSET, mv_flags,
+        GST_FLAG_SET_MASK_EXACT, "views", G_TYPE_INT, views, NULL);
+
+    notify_codec_state_changed (decoder);
+  }
+}
+#endif
+
 gboolean
 gst_vaapi_decoder_ensure_context (GstVaapiDecoder * decoder,
     GstVaapiContextInfo * cip)
diff --git a/gst-libs/gst/vaapi/gstvaapidecoder_h264.c b/gst-libs/gst/vaapi/gstvaapidecoder_h264.c
index 7a90f4b2..3947a445 100644
--- a/gst-libs/gst/vaapi/gstvaapidecoder_h264.c
+++ b/gst-libs/gst/vaapi/gstvaapidecoder_h264.c
@@ -1443,11 +1443,41 @@ ensure_context(GstVaapiDecoderH264 *decoder, GstH264SPS *sps)
     }
 
     if (!priv->profile || (priv->profile != profile && priv->max_views == 1)) {
-        GST_DEBUG("profile changed");
+        GST_DEBUG("profile changed to %x", profile);
         reset_context = TRUE;
         priv->profile = profile;
     }
 
+#if GST_CHECK_VERSION(1,5,0)
+    /* Multiview flags only available in >= 1.5 */
+    if (reset_context) {
+        switch (num_views) {
+          case 1:
+            /* Frame-packed mode details should be copied from the parser
+             * if we set NONE */
+            gst_vaapi_decoder_set_multiview_mode (base_decoder,
+                num_views, GST_VIDEO_MULTIVIEW_MODE_NONE,
+                GST_VIDEO_MULTIVIEW_FLAGS_NONE);
+            break;
+          case 2: /* Assume stereo */
+            if (profile == GST_VAAPI_PROFILE_H264_STEREO_HIGH) {
+              GST_DEBUG ("Stereo profile - frame-by-frame output, %d views", num_views);
+              gst_vaapi_decoder_set_multiview_mode (base_decoder,
+                  num_views, GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME,
+                  GST_VIDEO_MULTIVIEW_FLAGS_NONE);
+              break;
+            }
+            /* non-stereo 2 views. Fall through */
+          default:
+            GST_DEBUG ("Multiview profile - frame-by-frame output, %d views", num_views);
+            gst_vaapi_decoder_set_multiview_mode (base_decoder,
+                num_views, GST_VIDEO_MULTIVIEW_MODE_MULTIVIEW_FRAME_BY_FRAME,
+                GST_VIDEO_MULTIVIEW_FLAGS_NONE);
+            break;
+        }
+    }
+#endif
+
     chroma_type = gst_vaapi_utils_h264_get_chroma_type(sps->chroma_format_idc);
     if (!chroma_type) {
         GST_ERROR("unsupported chroma_format_idc %u", sps->chroma_format_idc);
diff --git a/gst-libs/gst/vaapi/gstvaapidecoder_priv.h b/gst-libs/gst/vaapi/gstvaapidecoder_priv.h
index 2ac56e8b..3b7aa0e8 100644
--- a/gst-libs/gst/vaapi/gstvaapidecoder_priv.h
+++ b/gst-libs/gst/vaapi/gstvaapidecoder_priv.h
@@ -263,6 +263,13 @@ void
 gst_vaapi_decoder_set_interlaced (GstVaapiDecoder * decoder,
     gboolean interlaced);
 
+#if GST_CHECK_VERSION(1,5,0)
+G_GNUC_INTERNAL
+void
+gst_vaapi_decoder_set_multiview_mode (GstVaapiDecoder * decoder,
+    gint views, GstVideoMultiviewMode mv_mode, GstVideoMultiviewFlags mv_flags);
+#endif
+
 G_GNUC_INTERNAL
 gboolean
 gst_vaapi_decoder_ensure_context (GstVaapiDecoder * decoder,
diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_h264.c b/gst-libs/gst/vaapi/gstvaapiencoder_h264.c
index 36edd9dd..c3a7c5fe 100644
--- a/gst-libs/gst/vaapi/gstvaapiencoder_h264.c
+++ b/gst-libs/gst/vaapi/gstvaapiencoder_h264.c
@@ -2450,7 +2450,6 @@ reset_properties (GstVaapiEncoderH264 * encoder)
   encoder->max_pic_order_cnt = (1 << encoder->log2_max_pic_order_cnt);
   encoder->idr_num = 0;
 
-  encoder->is_mvc = encoder->num_views > 1;
   for (i = 0; i < encoder->num_views; i++) {
     GstVaapiH264ViewRefPool *const ref_pool = &encoder->ref_pools[i];
     ref_pool->max_reflist0_count = 1;
@@ -2630,6 +2629,7 @@ gst_vaapi_encoder_h264_reordering (GstVaapiEncoder * base_encoder,
 
   /* encoding views alternatively for MVC */
   if (encoder->is_mvc) {
+    /* FIXME: Use first-in-bundle flag on buffers to reset view idx? */
     if (frame)
       encoder->view_idx = frame->system_frame_number % encoder->num_views;
     else
@@ -2779,6 +2779,7 @@ gst_vaapi_encoder_h264_reconfigure (GstVaapiEncoder * base_encoder)
 {
   GstVaapiEncoderH264 *const encoder =
       GST_VAAPI_ENCODER_H264_CAST (base_encoder);
+  GstVideoInfo *const vip = GST_VAAPI_ENCODER_VIDEO_INFO (encoder);
   GstVaapiEncoderStatus status;
   guint mb_width, mb_height;
 
@@ -2792,6 +2793,15 @@ gst_vaapi_encoder_h264_reconfigure (GstVaapiEncoder * base_encoder)
     encoder->config_changed = TRUE;
   }
 
+#if GST_CHECK_VERSION(1,5,0)
+  /* Take number of MVC views from input caps if provided */
+  if (GST_VIDEO_INFO_MULTIVIEW_MODE (vip) == GST_VIDEO_MULTIVIEW_MODE_FRAME_BY_FRAME ||
+      GST_VIDEO_INFO_MULTIVIEW_MODE (vip) == GST_VIDEO_MULTIVIEW_MODE_MULTIVIEW_FRAME_BY_FRAME)
+    encoder->num_views = GST_VIDEO_INFO_VIEWS (vip);
+#endif
+
+  encoder->is_mvc = encoder->num_views > 1;
+
   status = ensure_profile_and_level (encoder);
   if (status != GST_VAAPI_ENCODER_STATUS_SUCCESS)
     return status;
diff --git a/gst/vaapi/gstvaapidecode.c b/gst/vaapi/gstvaapidecode.c
index 43509d8f..313e4049 100644
--- a/gst/vaapi/gstvaapidecode.c
+++ b/gst/vaapi/gstvaapidecode.c
@@ -326,6 +326,14 @@ gst_vaapidecode_push_decoded_frame (GstVideoDecoder * vdec,
     }
     GST_BUFFER_FLAG_SET (out_frame->output_buffer, out_flags);
 
+#if GST_CHECK_VERSION(1,5,0)
+    /* First-in-bundle flag only appeared in 1.5 dev */
+    if (flags & GST_VAAPI_SURFACE_PROXY_FLAG_FFB) {
+      GST_BUFFER_FLAG_SET (out_frame->output_buffer,
+          GST_VIDEO_BUFFER_FLAG_FIRST_IN_BUNDLE);
+    }
+#endif
+
     crop_rect = gst_vaapi_surface_proxy_get_crop_rect (proxy);
     if (crop_rect) {
       GstVideoCropMeta *const crop_meta =
diff --git a/gst/vaapi/gstvaapipluginutil.c b/gst/vaapi/gstvaapipluginutil.c
index ef3c6cff..997e8e0c 100644
--- a/gst/vaapi/gstvaapipluginutil.c
+++ b/gst/vaapi/gstvaapipluginutil.c
@@ -633,6 +633,13 @@ gst_video_info_change_format (GstVideoInfo * vip, GstVideoFormat format,
   vip->par_d = vi.par_d;
   vip->fps_n = vi.fps_n;
   vip->fps_d = vi.fps_d;
+
+#if GST_CHECK_VERSION(1,5,0)
+  GST_VIDEO_INFO_MULTIVIEW_MODE (vip) =
+      GST_VIDEO_INFO_MULTIVIEW_MODE (&vi);
+  GST_VIDEO_INFO_MULTIVIEW_FLAGS (vip) =
+      GST_VIDEO_INFO_MULTIVIEW_FLAGS (&vi);
+#endif
 }
 
 /**