avfilter/formats: fix format negotiation when multiple channel_layouts are provided

For example
ffmpeg -f lavfi -i sine -af "aformat=cl=stereo|5.1|7.1,lowpass,aformat=cl=7.1|5.1|stereo" -f null -

Reviewed-by: Paul B Mahol <onemda@gmail.com>
Signed-off-by: James Almer <jamrial@gmail.com>

2 files changed, 31 insertions, 10 deletions

diff --git a/libavfilter/avfiltergraph.c b/libavfilter/avfiltergraph.c
index 744f480e1d..a200ab856c 100644
--- a/libavfilter/avfiltergraph.c
+++ b/libavfilter/avfiltergraph.c
@@ -441,7 +441,7 @@ static int query_formats(AVFilterGraph *graph, void *log_ctx)
 
             neg = ff_filter_get_negotiation(link);
             av_assert0(neg);
-            for (neg_step = 1; neg_step < neg->nb_mergers; neg_step++) {
+            for (neg_step = 0; neg_step < neg->nb_mergers; neg_step++) {
                 const AVFilterFormatsMerger *m = &neg->mergers[neg_step];
                 void *a = FF_FIELD_AT(void *, m->offset, link->incfg);
                 void *b = FF_FIELD_AT(void *, m->offset, link->outcfg);
diff --git a/libavfilter/formats.c b/libavfilter/formats.c
index e8c2888c0c..d1c97daf64 100644
--- a/libavfilter/formats.c
+++ b/libavfilter/formats.c
@@ -209,11 +209,10 @@ static int merge_samplerates(void *a, void *b)
 /**
  * See merge_pix_fmts().
  */
-static int merge_channel_layouts(void *va, void *vb)
+static int merge_channel_layouts_internal(AVFilterChannelLayouts *a,
+                                          AVFilterChannelLayouts *b, int check)
 {
-    AVFilterChannelLayouts *a = va;
-    AVFilterChannelLayouts *b = vb;
-    AVChannelLayout *channel_layouts;
+    AVChannelLayout *channel_layouts = NULL;
     unsigned a_all = a->all_layouts + a->all_counts;
     unsigned b_all = b->all_layouts + b->all_counts;
     int ret_max, ret_nb = 0, i, j, round;
@@ -231,8 +230,11 @@ static int merge_channel_layouts(void *va, void *vb)
         if (a_all == 1 && !b_all) {
             /* keep only known layouts in b; works also for b_all = 1 */
             for (i = j = 0; i < b->nb_channel_layouts; i++)
-                if (KNOWN(&b->channel_layouts[i]) && i != j++)
+                if (KNOWN(&b->channel_layouts[i]) && i != j++) {
+                    if (check)
+                        return 1;
                     av_channel_layout_copy(&b->channel_layouts[j], &b->channel_layouts[i]);
+                }
             /* Not optimal: the unknown layouts of b may become known after
                another merge. */
             if (!j)
@@ -244,7 +246,7 @@ static int merge_channel_layouts(void *va, void *vb)
     }
 
     ret_max = a->nb_channel_layouts + b->nb_channel_layouts;
-    if (!(channel_layouts = av_calloc(ret_max, sizeof(*channel_layouts))))
+    if (!check && !(channel_layouts = av_calloc(ret_max, sizeof(*channel_layouts))))
         return AVERROR(ENOMEM);
 
     /* a[known] intersect b[known] */
@@ -253,6 +255,8 @@ static int merge_channel_layouts(void *va, void *vb)
             continue;
         for (j = 0; j < b->nb_channel_layouts; j++) {
             if (!av_channel_layout_compare(&a->channel_layouts[i], &b->channel_layouts[j])) {
+                if (check)
+                    return 1;
                 av_channel_layout_copy(&channel_layouts[ret_nb++], &a->channel_layouts[i]);
                 av_channel_layout_uninit(&a->channel_layouts[i]);
                 av_channel_layout_uninit(&b->channel_layouts[j]);
@@ -269,8 +273,11 @@ static int merge_channel_layouts(void *va, void *vb)
                 continue;
             bfmt = FF_COUNT2LAYOUT(fmt->nb_channels);
             for (j = 0; j < b->nb_channel_layouts; j++)
-                if (!av_channel_layout_compare(&b->channel_layouts[j], &bfmt))
+                if (!av_channel_layout_compare(&b->channel_layouts[j], &bfmt)) {
+                    if (check)
+                        return 1;
                     av_channel_layout_copy(&channel_layouts[ret_nb++], fmt);
+                }
         }
         /* 1st round: swap to prepare 2nd round; 2nd round: put it back */
         FFSWAP(AVFilterChannelLayouts *, a, b);
@@ -280,8 +287,11 @@ static int merge_channel_layouts(void *va, void *vb)
         if (KNOWN(&a->channel_layouts[i]))
             continue;
         for (j = 0; j < b->nb_channel_layouts; j++)
-            if (!av_channel_layout_compare(&a->channel_layouts[i], &b->channel_layouts[j]))
+            if (!av_channel_layout_compare(&a->channel_layouts[i], &b->channel_layouts[j])) {
+                if (check)
+                    return 1;
                 av_channel_layout_copy(&channel_layouts[ret_nb++], &a->channel_layouts[i]);
+            }
     }
 
     if (!ret_nb) {
@@ -300,6 +310,17 @@ static int merge_channel_layouts(void *va, void *vb)
     return 1;
 }
 
+static int can_merge_channel_layouts(const void *a, const void *b)
+{
+    return merge_channel_layouts_internal((AVFilterChannelLayouts *)a,
+                                          (AVFilterChannelLayouts *)b, 1);
+}
+
+static int merge_channel_layouts(void *a, void *b)
+{
+    return merge_channel_layouts_internal(a, b, 0);
+}
+
 static const AVFilterFormatsMerger mergers_video[] = {
     {
         .offset     = offsetof(AVFilterFormatsConfig, formats),
@@ -312,7 +333,7 @@ static const AVFilterFormatsMerger mergers_audio[] = {
     {
         .offset     = offsetof(AVFilterFormatsConfig, channel_layouts),
         .merge      = merge_channel_layouts,
-        .can_merge  = NULL,
+        .can_merge  = can_merge_channel_layouts,
     },
     {
         .offset     = offsetof(AVFilterFormatsConfig, samplerates),