avfilter/vf_libplacebo: support more output formats

Rather than hard-coding AV_PIX_FMT_VULKAN, expand this to the full list
of formats supported by <libplacebo/utils/libav.h>. We re-use the
existing `format` option to allow selecting specific software formats in
addition to specific vulkan hwframe formats.

Some minor changes are necessary to account for the fact that
`ff_vk_filter_config_output` is now only called optionally, the fact
that the output format must now be parsed before `query_format` gets
called, and the fact that we need to call a different function to
retrieve data from the `pl_frame` in the non-hwaccel case.

Signed-off-by: Niklas Haas <git@haasn.dev>

1 files changed, 59 insertions, 28 deletions

diff --git a/libavfilter/vf_libplacebo.c b/libavfilter/vf_libplacebo.c
index ce57e7539e..3678b60b7d 100644
--- a/libavfilter/vf_libplacebo.c
+++ b/libavfilter/vf_libplacebo.c
@@ -62,10 +62,11 @@ typedef struct LibplaceboContext {
     pl_vulkan vulkan;
     pl_gpu gpu;
     pl_renderer renderer;
-    pl_tex tex[4];
+    pl_tex tex[8];
 
     /* settings */
     char *out_format_string;
+    enum AVPixelFormat out_format;
     char *w_expr;
     char *h_expr;
     AVRational target_sar;
@@ -215,6 +216,8 @@ static int find_scaler(AVFilterContext *avctx,
     return AVERROR(EINVAL);
 }
 
+static void libplacebo_uninit(AVFilterContext *avctx);
+
 static int libplacebo_init(AVFilterContext *avctx)
 {
     LibplaceboContext *s = avctx->priv;
@@ -229,6 +232,18 @@ static int libplacebo_init(AVFilterContext *avctx)
     if (!s->log)
         return AVERROR(ENOMEM);
 
+    if (s->out_format_string) {
+        s->out_format = av_get_pix_fmt(s->out_format_string);
+        if (s->out_format == AV_PIX_FMT_NONE) {
+            av_log(avctx, AV_LOG_ERROR, "Invalid output format: %s\n",
+                   s->out_format_string);
+            libplacebo_uninit(avctx);
+            return AVERROR(EINVAL);
+        }
+    } else {
+        s->out_format = AV_PIX_FMT_NONE;
+    }
+
     /* Note: s->vulkan etc. are initialized later, when hwctx is available */
     return 0;
 }
@@ -320,6 +335,7 @@ static int process_frames(AVFilterContext *avctx, AVFrame *out, AVFrame *in)
     LibplaceboContext *s = avctx->priv;
     struct pl_render_params params;
     enum pl_tone_map_mode tonemapping_mode = s->tonemapping_mode;
+    const AVPixFmtDescriptor *outdesc = av_pix_fmt_desc_get(out->format);
     enum pl_gamut_mode gamut_mode = s->gamut_mode;
     struct pl_frame image, target;
     ok = pl_map_avframe_ex(s->gpu, &image, pl_avframe_params(
@@ -328,10 +344,14 @@ static int process_frames(AVFilterContext *avctx, AVFrame *out, AVFrame *in)
         .map_dovi = s->apply_dovi,
     ));
 
-    ok &= pl_map_avframe_ex(s->gpu, &target, pl_avframe_params(
-        .frame    = out,
-        .map_dovi = false,
-    ));
+    if (outdesc->flags & AV_PIX_FMT_FLAG_HWACCEL) {
+        ok &= pl_map_avframe_ex(s->gpu, &target, pl_avframe_params(
+            .frame    = out,
+            .map_dovi = false,
+        ));
+    } else {
+        ok &= pl_frame_recreate_from_avframe(s->gpu, &target, s->tex + 4, out);
+    }
 
     if (!ok) {
         err = AVERROR_EXTERNAL;
@@ -434,7 +454,13 @@ static int process_frames(AVFilterContext *avctx, AVFrame *out, AVFrame *in)
 
     pl_render_image(s->renderer, &image, &target, &params);
     pl_unmap_avframe(s->gpu, &image);
-    pl_unmap_avframe(s->gpu, &target);
+
+    if (outdesc->flags & AV_PIX_FMT_FLAG_HWACCEL) {
+        pl_unmap_avframe(s->gpu, &target);
+    } else if (!pl_download_avframe(s->gpu, &target, out)) {
+        err = AVERROR_EXTERNAL;
+        goto fail;
+    }
 
     /* Flush the command queues for performance */
     pl_gpu_flush(s->gpu);
@@ -516,10 +542,7 @@ static int libplacebo_query_format(AVFilterContext *ctx)
     int err = 0;
     LibplaceboContext *s = ctx->priv;
     const AVPixFmtDescriptor *desc = NULL;
-    AVFilterFormats *in_fmts = NULL;
-    static const enum AVPixelFormat out_fmts[] = {
-        AV_PIX_FMT_VULKAN, AV_PIX_FMT_NONE,
-    };
+    AVFilterFormats *formats = NULL;
 
     RET(init_vulkan(ctx));
 
@@ -534,14 +557,24 @@ static int libplacebo_query_format(AVFilterContext *ctx)
 #endif
 
         if (pl_test_pixfmt(s->gpu, pixfmt)) {
-            if ((err = ff_add_format(&in_fmts, pixfmt)) < 0)
+            if ((err = ff_add_format(&formats, pixfmt)) < 0)
                 return err;
         }
     }
 
-    RET(ff_formats_ref(in_fmts, &ctx->inputs[0]->outcfg.formats));
-    RET(ff_formats_ref(ff_make_format_list(out_fmts),
-                       &ctx->outputs[0]->incfg.formats));
+    RET(ff_formats_ref(formats, &ctx->inputs[0]->outcfg.formats));
+
+    if (s->out_format != AV_PIX_FMT_NONE) {
+        /* Support only requested format, and hwaccel (vulkan) */
+        const enum AVPixelFormat out_fmts[] = {
+            s->out_format, AV_PIX_FMT_VULKAN, AV_PIX_FMT_NONE,
+        };
+        RET(ff_formats_ref(ff_make_format_list(out_fmts),
+                           &ctx->outputs[0]->incfg.formats));
+    } else {
+        /* Support all formats */
+        RET(ff_formats_ref(formats, &ctx->outputs[0]->incfg.formats));
+    }
 
     return 0;
 
@@ -572,17 +605,15 @@ static int libplacebo_config_output(AVFilterLink *outlink)
     AVHWFramesContext *hwfc;
     AVVulkanFramesContext *vkfc;
     AVRational scale_sar;
-    int *out_w = &s->vkctx.output_width;
-    int *out_h = &s->vkctx.output_height;
 
     RET(ff_scale_eval_dimensions(s, s->w_expr, s->h_expr, inlink, outlink,
-                                 out_w, out_h));
+                                 &outlink->w, &outlink->h));
 
-    ff_scale_adjust_dimensions(inlink, out_w, out_h,
+    ff_scale_adjust_dimensions(inlink, &outlink->w, &outlink->h,
                                s->force_original_aspect_ratio,
                                s->force_divisible_by);
 
-    scale_sar = (AVRational){*out_h * inlink->w, *out_w * inlink->h};
+    scale_sar = (AVRational){outlink->h * inlink->w, outlink->w * inlink->h};
     if (inlink->sample_aspect_ratio.num)
         scale_sar = av_mul_q(scale_sar, inlink->sample_aspect_ratio);
 
@@ -598,17 +629,17 @@ static int libplacebo_config_output(AVFilterLink *outlink)
             outlink->sample_aspect_ratio = scale_sar;
     }
 
-    if (s->out_format_string) {
-        s->vkctx.output_format = av_get_pix_fmt(s->out_format_string);
-        if (s->vkctx.output_format == AV_PIX_FMT_NONE) {
-            av_log(avctx, AV_LOG_ERROR, "Invalid output format.\n");
-            return AVERROR(EINVAL);
-        }
-    } else {
-        /* Default to re-using the input format */
+    if (outlink->format != AV_PIX_FMT_VULKAN)
+        return 0;
+
+    s->vkctx.output_width = outlink->w;
+    s->vkctx.output_height = outlink->h;
+    /* Default to re-using the input format */
+    if (s->out_format == AV_PIX_FMT_NONE || s->out_format == AV_PIX_FMT_VULKAN) {
         s->vkctx.output_format = s->vkctx.input_format;
+    } else {
+        s->vkctx.output_format = s->out_format;
     }
-
     RET(ff_vk_filter_config_output(outlink));
     hwfc = (AVHWFramesContext *) outlink->hw_frames_ctx->data;
     vkfc = hwfc->hwctx;