draw_bmp: add RGB rendering to fix image quality issues

As pointed out in commit ed01df, the quality loss due to frequent
conversion between RGB and YUV is too much when drawing OSD and
subtitles.

Fix this by staying in the same colorspace when drawing subtitles.
Render directly to RGB, without converting to YUV first.

The bad thing about packed RGB is that there are many pixel formats,
which would all require special code for blending. It's also completely
incompatible to planar YUV. Use planar RGB instead, which allows us to
reuse all code originally written for planar YUV. The only thing that
needs to be changed is the color conversion in the libass case. (In
exchange for simpler code, the image has to be copied, but this is
still much better than converting to YUV.)

Unfortunately, libswscale doesn't support planar RGB output. Add a hack
to sws_utils.c to handle conversion to planar RGB. In the common case,
when converting 32 bit per pixel RGB, calling swscale can be avoided
entirely.

The change in mp_image.c is needed to allocate GBRP images correctly.

(The issue with vo_x11 could be easily solved by always backing up the
same bounding box as the bitmap drawing RGB<->YUV conversion does, but
this commit is probably the better fix.)

sub/draw_bmp.c

@@ -307,7 +307,14 @@ static void draw_ass(struct mp_draw_sub_cache **cache, struct mp_rect bb,
         int b = (sb->libass.color >> 8) & 0xFF;
         int a = 255 - (sb->libass.color & 0xFF);
         int color_yuv[3] = {r, g, b};
-        mp_map_int_color(rgb2yuv, bits, color_yuv);
+        if (dst.flags & MP_IMGFLAG_YUV) {
+            mp_map_int_color(rgb2yuv, bits, color_yuv);
+        } else {
+            assert(dst.imgfmt == IMGFMT_GBRP);
+            color_yuv[0] = g;
+            color_yuv[1] = b;
+            color_yuv[2] = r;
+        }
 
         int bytes = (bits + 7) / 8;
         uint8_t *alpha_p = (uint8_t *)sb->bitmap + src_y * sb->stride + src_x;
@@ -411,6 +418,10 @@ static void get_closest_y444_format(int imgfmt, int *out_format, int *out_bits)
                     return;
             }
         }
+    } else {
+        *out_format = IMGFMT_GBRP;
+        *out_bits = 8;
+        return;
     }
     *out_format = IMGFMT_444P16;
     *out_bits = 16;

video/mp_image.c

@@ -149,6 +149,10 @@ void mp_image_setfmt(mp_image_t* mpi,unsigned int out_fmt){
     if (out_fmt == IMGFMT_GBRP) {
         mpi->bpp=24;
         mpi->flags|=MP_IMGFLAG_PLANAR;
+        mpi->chroma_x_shift = 0;
+        mpi->chroma_y_shift = 0;
+        mpi->chroma_width=mpi->width;
+        mpi->chroma_height=mpi->height;
         return;
     }
     mpi->flags|=MP_IMGFLAG_YUV;

video/sws_utils.c

@@ -151,9 +151,54 @@ static int mp_csp_to_sws_colorspace(enum mp_csp csp)
     }
 }
 
+// component_offset[]: byte index of each r (0), g (1), b (2), a (3) component
+static void planarize32(struct mp_image *dst, struct mp_image *src,
+                        int component_offset[4])
+{
+    for (int y = 0; y < dst->h; y++) {
+        for (int p = 0; p < 3; p++) {
+            uint8_t *d_line = dst->planes[p] + y * dst->stride[p];
+            uint8_t *s_line = src->planes[0] + y * src->stride[0];
+            s_line += component_offset[(p + 1) % 3]; // GBR => RGB
+            for (int x = 0; x < dst->w; x++) {
+                d_line[x] = s_line[x * 4];
+            }
+        }
+    }
+}
+
+#define SET_COMPS(comp, r, g, b, a) \
+    { (comp)[0] = (r); (comp)[1] = (g); (comp)[2] = (b); (comp)[3] = (a); }
+
+static void to_gbrp(struct mp_image *dst, struct mp_image *src,
+                    int my_sws_flags)
+{
+    struct mp_image *temp = NULL;
+    int comp[4];
+
+    switch (src->imgfmt) {
+    case IMGFMT_ABGR: SET_COMPS(comp, 3, 2, 1, 0); break;
+    case IMGFMT_BGRA: SET_COMPS(comp, 2, 1, 0, 3); break;
+    case IMGFMT_ARGB: SET_COMPS(comp, 1, 2, 3, 0); break;
+    case IMGFMT_RGBA: SET_COMPS(comp, 0, 1, 2, 3); break;
+    default:
+        temp = alloc_mpi(dst->w, dst->h, IMGFMT_RGBA);
+        mp_image_swscale(temp, src, my_sws_flags);
+        src = temp;
+        SET_COMPS(comp, 0, 1, 2, 3);
+    }
+
+    planarize32(dst, src, comp);
+
+    talloc_free(temp);
+}
+
 void mp_image_swscale(struct mp_image *dst, struct mp_image *src,
                       int my_sws_flags)
 {
+    if (dst->imgfmt == IMGFMT_GBRP)
+        return to_gbrp(dst, src, my_sws_flags);
+
     enum PixelFormat s_fmt = imgfmt2pixfmt(src->imgfmt);
     if (src->imgfmt == IMGFMT_RGB8 || src->imgfmt == IMGFMT_BGR8)
         s_fmt = PIX_FMT_PAL8;