video: support priority order for VO format selection

Author: sfan5

filters/f_autoconvert.c

@@ -24,6 +24,7 @@ struct priv {
 
     int *imgfmts;
     int *subfmts;
+    uint8_t *priorities;
     int num_imgfmts;
     struct mp_image_params imgparams;
     bool imgparams_set;
@@ -69,15 +70,20 @@ void mp_autoconvert_clear(struct mp_autoconvert *c)
     p->force_update = true;
 }
 
-void mp_autoconvert_add_imgfmt(struct mp_autoconvert *c, int imgfmt, int subfmt)
+void mp_autoconvert_add_imgfmt(struct mp_autoconvert *c, int imgfmt,
+                               int subfmt, uint8_t priority)
 {
     struct priv *p = c->f->priv;
 
-    MP_TARRAY_GROW(p, p->imgfmts, p->num_imgfmts);
-    MP_TARRAY_GROW(p, p->subfmts, p->num_imgfmts);
+    const int i = p->num_imgfmts;
 
-    p->imgfmts[p->num_imgfmts] = imgfmt;
-    p->subfmts[p->num_imgfmts] = subfmt;
+    MP_TARRAY_GROW(p, p->imgfmts, i);
+    MP_TARRAY_GROW(p, p->subfmts, i);
+    MP_TARRAY_GROW(p, p->priorities, i);
+
+    p->imgfmts[i] = imgfmt;
+    p->subfmts[i] = subfmt;
+    p->priorities[i] = priority;
 
     p->num_imgfmts += 1;
     p->force_update = true;
@@ -97,14 +103,14 @@ void mp_autoconvert_set_target_image_params(struct mp_autoconvert *c,
     p->imgparams_set = true;
 
     p->num_imgfmts = 0;
-    mp_autoconvert_add_imgfmt(c, par->imgfmt, par->hw_subfmt);
+    mp_autoconvert_add_imgfmt(c, par->imgfmt, par->hw_subfmt, 1);
 }
 
 void mp_autoconvert_add_all_sw_imgfmts(struct mp_autoconvert *c)
 {
     for (int n = IMGFMT_START; n < IMGFMT_END; n++) {
         if (!IMGFMT_IS_HWACCEL(n))
-            mp_autoconvert_add_imgfmt(c, n, 0);
+            mp_autoconvert_add_imgfmt(c, n, 0, 1);
     }
 }
 
@@ -307,7 +313,8 @@ static bool build_image_converter(struct mp_autoconvert *c, struct mp_log *log,
 
         sws->force_scaler = c->force_scaler;
 
-        int out = mp_sws_find_best_out_format(sws, src_fmt, fmts, num_fmts);
+        int out = mp_sws_find_best_out_format(sws, src_fmt, fmts,
+                                              p->priorities, num_fmts);
         if (!out) {
             mp_err(log, "can't find video conversion for %s\n",
                    mp_imgfmt_to_name(src_fmt));

filters/f_autoconvert.h

@@ -39,7 +39,9 @@ void mp_autoconvert_set_target_image_params(struct mp_autoconvert *c,
 // formats are allowed (even non-video).
 // subfmt can be used to specify underlying surface formats for hardware formats,
 // otherwise must be 0. (Mismatches lead to conversion errors.)
-void mp_autoconvert_add_imgfmt(struct mp_autoconvert *c, int imgfmt, int subfmt);
+// priority: higher is better
+void mp_autoconvert_add_imgfmt(struct mp_autoconvert *c, int imgfmt,
+                               int subfmt, uint8_t priority);
 
 // Add all sw image formats. The effect is that hardware video image formats are
 // disallowed. The semantics are the same as calling mp_autoconvert_add_imgfmt()

filters/f_output_chain.c

@@ -84,9 +84,14 @@ static void update_output_caps(struct chain *p)
         uint8_t allowed_output_formats[IMGFMT_END - IMGFMT_START] = {0};
         vo_query_formats(p->vo, allowed_output_formats);
 
+        MP_TRACE(p, "VO reports supported formats:\n");
         for (int n = 0; n < MP_ARRAY_SIZE(allowed_output_formats); n++) {
-            if (allowed_output_formats[n])
-                mp_autoconvert_add_imgfmt(p->convert, IMGFMT_START + n, 0);
+            const uint8_t prio = allowed_output_formats[n];
+            if (prio) {
+                const int imgfmt = IMGFMT_START + n;
+                mp_autoconvert_add_imgfmt(p->convert, imgfmt, 0, prio);
+                MP_TRACE(p, "  %-14s (%d)\n", mp_imgfmt_to_name(imgfmt), (int) prio);
+            }
         }
     }
 }

filters/f_swscale.c

@@ -41,23 +41,32 @@
 #include "filter_internal.h"
 
 int mp_sws_find_best_out_format(struct mp_sws_filter *sws, int in_format,
-                                int *out_formats, int num_out_formats)
+                                int *out_formats, uint8_t *priorities,
+                                int num_out_formats)
 {
     sws->sws->force_scaler = sws->force_scaler;
 
     int best = 0;
+    uint8_t prio = 0;
     for (int n = 0; n < num_out_formats; n++) {
         int out_format = out_formats[n];
 
+        if (priorities && priorities[n] < prio)
+            continue;
+
         if (!mp_sws_supports_formats(sws->sws, out_format, in_format))
             continue;
 
         if (best) {
             int candidate = mp_imgfmt_select_best(best, out_format, in_format);
-            if (candidate)
+            if (candidate) {
                 best = candidate;
+                if (best == out_format && priorities)
+                    prio = priorities[n];
+            }
         } else {
             best = out_format;
+            prio = priorities ? priorities[n] : 0;
         }
     }
     return best;

filters/f_swscale.h

@@ -24,11 +24,13 @@ struct mp_sws_filter *mp_sws_filter_create(struct mp_filter *parent);
 
 // Return the best format based on the input format and a list of allowed output
 // formats. This tries to set the output format to the one that will result in
-// the least loss. Returns a format from out_formats[], or 0 if no format could
+// the least loss. If not NULL, priorities[] must be an array with the same size
+// as out_formats and will be taken into account when choosing (higher is better).
+// Returns a format from out_formats[], or 0 if no format could
 // be chosen (or it's not supported by libswscale).
 int mp_sws_find_best_out_format(struct mp_sws_filter *sws,
                                 int in_format, int *out_formats,
-                                int num_out_formats);
+                                uint8_t *priorities, int num_out_formats);
 
 // Whether the given format is supported as input format.
 bool mp_sws_supports_input(int imgfmt);

player/playloop.c

@@ -1024,16 +1024,19 @@ int handle_force_window(struct MPContext *mpctx, bool force)
 
     if (!mpctx->video_out->config_ok || force) {
         struct vo *vo = mpctx->video_out;
-        // Pick whatever works
+        // Pick most-preferred format
         int config_format = 0;
+        uint8_t prio = 0;
         uint8_t fmts[IMGFMT_END - IMGFMT_START] = {0};
         vo_query_formats(vo, fmts);
         for (int fmt = IMGFMT_START; fmt < IMGFMT_END; fmt++) {
-            if (fmts[fmt - IMGFMT_START]) {
+            if (fmts[fmt - IMGFMT_START] > prio) {
                 config_format = fmt;
-                break;
+                prio = fmts[fmt - IMGFMT_START];
             }
         }
+        if (!config_format)
+            goto err;
 
         // Use a 16:9 aspect ratio so that fullscreen on a 16:9 screen will not
         // have vertical margins, which can lead to a different size or position

video/filter/refqueue.c

@@ -81,7 +81,7 @@ struct mp_refqueue *mp_refqueue_alloc(struct mp_filter *f)
 
 void mp_refqueue_add_in_format(struct mp_refqueue *q, int fmt, int subfmt)
 {
-    mp_autoconvert_add_imgfmt(q->conv, fmt, subfmt);
+    mp_autoconvert_add_imgfmt(q->conv, fmt, subfmt, 1);
 }
 
 // The minimum number of frames required before and after the current frame.

video/filter/vf_format.c

@@ -223,7 +223,7 @@ static struct mp_filter *vf_format_create(struct mp_filter *parent, void *option
     priv->conv->force_scaler = priv->opts->force_scaler;
 
     if (priv->opts->fmt)
-        mp_autoconvert_add_imgfmt(priv->conv, priv->opts->fmt, 0);
+        mp_autoconvert_add_imgfmt(priv->conv, priv->opts->fmt, 0, 1);
 
     return f;
 }

video/filter/vf_vapoursynth.c

@@ -788,7 +788,7 @@ static struct mp_filter *vf_vapoursynth_create(struct mp_filter *parent,
     for (int n = 0; mpvs_fmt_table[n].bits; n++) {
         int imgfmt = mp_from_vs(mpvs_fmt_table[n].vs);
         if (imgfmt)
-            mp_autoconvert_add_imgfmt(conv, imgfmt, 0);
+            mp_autoconvert_add_imgfmt(conv, imgfmt, 0, 1);
     }
 
     struct mp_filter *dur = mp_compute_frame_duration_create(f);

video/out/vo.h

@@ -323,8 +323,10 @@ struct vo_driver {
 
     /*
      * Whether the given image format is supported and config() will succeed.
+     * VO can choose fallback order of formats. Priority N-1 will only be considered if
+     * no format of priority N is usable.
      * format: one of IMGFMT_*
-     * returns: 0 on not supported, otherwise 1
+     * returns: 0 if not supported, priority otherwise
      */
     int (*query_format)(struct vo *vo, int format);