libs: encoder: h265: modify set_max_profile to set_allowed_profiles.

In h265, bigger profile idc may not be compatible with the small profile
idc. And more important, there are multi profiles with the same profile
idc. Such as main-422-10, main-444 and main-444-10, they all have profile
idc 4.
So recording the max profile idc is not enough, the encoder needs to know
all allowed profiles when deciding the real profile.

Part-of: <https://gitlab.freedesktop.org/gstreamer/gstreamer-vaapi/-/merge_requests/349>

3 files changed, 24 insertions, 27 deletions

diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_h265.c b/gst-libs/gst/vaapi/gstvaapiencoder_h265.c
index 17e7a4d1..4cc274cd 100644
--- a/gst-libs/gst/vaapi/gstvaapiencoder_h265.c
+++ b/gst-libs/gst/vaapi/gstvaapiencoder_h265.c
@@ -97,7 +97,7 @@ struct _GstVaapiEncoderH265
   GstVaapiLevelH265 level;
   GstVaapiEntrypoint entrypoint;
   guint8 profile_idc;
-  guint8 max_profile_idc;
+  GArray *allowed_profiles;
   guint8 level_idc;
   guint32 idr_period;
   guint32 init_qp;
@@ -1104,11 +1104,17 @@ error_unsupported_profile:
 static gboolean
 ensure_profile_limits (GstVaapiEncoderH265 * encoder)
 {
+  gint i;
 
-  if (!encoder->max_profile_idc
-      || encoder->profile_idc <= encoder->max_profile_idc)
+  if (!encoder->allowed_profiles)
     return TRUE;
 
+  for (i = 0; i < encoder->allowed_profiles->len; i++) {
+    if (encoder->profile ==
+        g_array_index (encoder->allowed_profiles, GstVaapiProfile, i))
+      return TRUE;
+  }
+
   GST_WARNING
       ("Needs to lower coding tools to meet target decoder constraints");
   GST_WARNING ("Only supporting Main profile, reset profile to Main");
@@ -3195,6 +3201,8 @@ gst_vaapi_encoder_h265_init (GstVaapiEncoderH265 * encoder)
   ref_pool->max_ref_frames = 0;
   ref_pool->max_reflist0_count = 1;
   ref_pool->max_reflist1_count = 1;
+
+  encoder->allowed_profiles = NULL;
 }
 
 struct _GstVaapiEncoderH265Class
@@ -3238,6 +3246,9 @@ gst_vaapi_encoder_h265_finalize (GObject * object)
 
   reset_tile (encoder);
 
+  if (encoder->allowed_profiles)
+    g_array_unref (encoder->allowed_profiles);
+
   G_OBJECT_CLASS (gst_vaapi_encoder_h265_parent_class)->finalize (object);
 }
 
@@ -3678,36 +3689,22 @@ gst_vaapi_encoder_h265_new (GstVaapiDisplay * display)
 }
 
 /**
- * gst_vaapi_encoder_h265_set_max_profile:
+ * gst_vaapi_encoder_h265_set_allowed_profiles:
  * @encoder: a #GstVaapiEncoderH265
- * @profile: an H.265 #GstVaapiProfile
- *
- * Notifies the @encoder to use coding tools from the supplied
- * @profile at most.
+ * @profiles: a #GArray of all allowed #GstVaapiProfile.
  *
- * This means that if the minimal profile derived to
- * support the specified coding tools is greater than this @profile,
- * then an error is returned when the @encoder is configured.
+ * Set the all allowed profiles for the encoder.
  *
  * Return value: %TRUE on success
  */
 gboolean
-gst_vaapi_encoder_h265_set_max_profile (GstVaapiEncoderH265 * encoder,
-    GstVaapiProfile profile)
+gst_vaapi_encoder_h265_set_allowed_profiles (GstVaapiEncoderH265 * encoder,
+    GArray * profiles)
 {
-  guint8 profile_idc;
-
   g_return_val_if_fail (encoder != NULL, FALSE);
-  g_return_val_if_fail (profile != GST_VAAPI_PROFILE_UNKNOWN, FALSE);
-
-  if (gst_vaapi_profile_get_codec (profile) != GST_VAAPI_CODEC_H265)
-    return FALSE;
-
-  profile_idc = gst_vaapi_utils_h265_get_profile_idc (profile);
-  if (!profile_idc)
-    return FALSE;
+  g_return_val_if_fail (profiles, FALSE);
 
-  encoder->max_profile_idc = profile_idc;
+  encoder->allowed_profiles = g_array_ref (profiles);
   return TRUE;
 }
 
diff --git a/gst-libs/gst/vaapi/gstvaapiencoder_h265.h b/gst-libs/gst/vaapi/gstvaapiencoder_h265.h
index 09449058..60da074a 100644
--- a/gst-libs/gst/vaapi/gstvaapiencoder_h265.h
+++ b/gst-libs/gst/vaapi/gstvaapiencoder_h265.h
@@ -45,8 +45,8 @@ GstVaapiEncoder *
 gst_vaapi_encoder_h265_new (GstVaapiDisplay * display);
 
 gboolean
-gst_vaapi_encoder_h265_set_max_profile (GstVaapiEncoderH265 * encoder,
-    GstVaapiProfile profile);
+gst_vaapi_encoder_h265_set_allowed_profiles (GstVaapiEncoderH265 * encoder,
+    GArray * profiles);
 
 gboolean
 gst_vaapi_encoder_h265_get_profile_tier_level (GstVaapiEncoderH265 * encoder,
diff --git a/gst/vaapi/gstvaapiencode_h265.c b/gst/vaapi/gstvaapiencode_h265.c
index c94d7460..9ee8855e 100644
--- a/gst/vaapi/gstvaapiencode_h265.c
+++ b/gst/vaapi/gstvaapiencode_h265.c
@@ -153,7 +153,7 @@ gst_vaapiencode_h265_set_config (GstVaapiEncode * base_encode)
   if (profile) {
     GST_INFO ("using %s profile as target decoder constraints",
         gst_vaapi_utils_h265_get_profile_string (profile));
-    if (!gst_vaapi_encoder_h265_set_max_profile (encoder, profile))
+    if (!gst_vaapi_encoder_h265_set_allowed_profiles (encoder, NULL))
       return FALSE;
   }
   return TRUE;