video: work around libswscale for PNG pixel formats

The intention is that we can test vo_opengl with high bit depth PNGs
better. This throws libswscale completely out of the loop, which before
was needed in order to convert from big endian to little endian.

Also apply a minimal cleanup to fmt-conversion.c (unrelated).

video/decode/vd_lavc.c

@@ -45,6 +45,7 @@
 #include "demux/stheader.h"
 #include "demux/packet.h"
 #include "video/csputils.h"
+#include "video/sws_utils.h"
 
 #include "lavc.h"
 
@@ -648,7 +649,7 @@ static int decode(struct dec_video *vd, struct demux_packet *packet,
     if (ctx->hwdec && ctx->hwdec->process_image)
         mpi = ctx->hwdec->process_image(ctx, mpi);
 
-    *out_image = mpi;
+    *out_image = mp_img_swap_to_native(mpi);
     return 1;
 }
 

video/fmt-conversion.c

@@ -88,29 +88,30 @@ static const struct {
 
     {IMGFMT_XYZ12,   AV_PIX_FMT_XYZ12},
 
-    // ffmpeg only
-#if LIBAVUTIL_VERSION_MICRO >= 100
+#ifdef AV_PIX_FMT_YUV420P12
     {IMGFMT_420P12,  AV_PIX_FMT_YUV420P12},
     {IMGFMT_420P14,  AV_PIX_FMT_YUV420P14},
     {IMGFMT_422P12,  AV_PIX_FMT_YUV422P12},
     {IMGFMT_422P14,  AV_PIX_FMT_YUV422P14},
     {IMGFMT_444P12,  AV_PIX_FMT_YUV444P12},
     {IMGFMT_444P14,  AV_PIX_FMT_YUV444P14},
+#endif
 
+#ifdef AV_PIX_FMT_RGBA64
     {IMGFMT_RGBA64,  AV_PIX_FMT_RGBA64},
     {IMGFMT_BGRA64,  AV_PIX_FMT_BGRA64},
+#endif
 
+#if LIBAVUTIL_VERSION_MICRO >= 100
     {IMGFMT_BGR0,  AV_PIX_FMT_BGR0},
     {IMGFMT_0RGB,  AV_PIX_FMT_0RGB},
     {IMGFMT_RGB0,  AV_PIX_FMT_RGB0},
     {IMGFMT_0BGR,  AV_PIX_FMT_0BGR},
-    {IMGFMT_BGR0,  AV_PIX_FMT_BGR0},
 #else
     {IMGFMT_BGR0,  AV_PIX_FMT_BGRA},
     {IMGFMT_0RGB,  AV_PIX_FMT_ARGB},
     {IMGFMT_RGB0,  AV_PIX_FMT_RGBA},
     {IMGFMT_0BGR,  AV_PIX_FMT_ABGR},
-    {IMGFMT_BGR0,  AV_PIX_FMT_BGRA},
 #endif
 
 #ifdef AV_PIX_FMT_YA16

video/sws_utils.c

@@ -20,6 +20,7 @@
 
 #include <libswscale/swscale.h>
 #include <libavcodec/avcodec.h>
+#include <libavutil/bswap.h>
 #include <libavutil/opt.h>
 
 #include "config.h"
@@ -34,6 +35,7 @@
 #include "csputils.h"
 #include "common/msg.h"
 #include "video/filter/vf.h"
+#include "osdep/endian.h"
 
 //global sws_flags from the command line
 struct sws_opts {
@@ -317,4 +319,38 @@ int mp_sws_set_vf_equalizer(struct mp_sws_context *sws, struct vf_seteq *eq)
     return mp_sws_reinit(sws) >= 0 ? 1 : -1;
 }
 
+static const int endian_swaps[][2] = {
+#if BYTE_ORDER == LITTLE_ENDIAN
+#if defined(AV_PIX_FMT_YA16) && defined(AV_PIX_FMT_RGBA64)
+    {AV_PIX_FMT_YA16BE,     AV_PIX_FMT_YA16LE},
+    {AV_PIX_FMT_RGBA64BE,   AV_PIX_FMT_RGBA64LE},
+    {AV_PIX_FMT_GRAY16BE,   AV_PIX_FMT_GRAY16LE},
+    {AV_PIX_FMT_RGB48BE,    AV_PIX_FMT_RGB48LE},
+#endif
+#endif
+    {AV_PIX_FMT_NONE,       AV_PIX_FMT_NONE}
+};
+
+// Swap _some_ non-native endian formats to native. We do this specifically
+// for pixel formats used by PNG, to avoid going through libswscale, which
+// might reduce the effective bit depth in some cases.
+struct mp_image *mp_img_swap_to_native(struct mp_image *img)
+{
+    int to = AV_PIX_FMT_NONE;
+    for (int n = 0; endian_swaps[n][0] != AV_PIX_FMT_NONE; n++) {
+        if (endian_swaps[n][0] == img->fmt.avformat)
+            to = endian_swaps[n][1];
+    }
+    if (to == AV_PIX_FMT_NONE || !mp_image_make_writeable(img))
+        return img;
+    int elems = img->fmt.bytes[0] / 2 * img->w;
+    for (int y = 0; y < img->h; y++) {
+        uint16_t *p = (uint16_t *)(img->planes[0] + y * img->stride[0]);
+        for (int i = 0; i < elems; i++)
+            p[i] = av_be2ne16(p[i]);
+    }
+    mp_image_setfmt(img, pixfmt2imgfmt(to));
+    return img;
+}
+
 // vim: ts=4 sw=4 et tw=80

video/sws_utils.h

@@ -59,6 +59,8 @@ struct vf_seteq;
 int mp_sws_set_vf_equalizer(struct mp_sws_context *sws, struct vf_seteq *eq);
 int mp_sws_get_vf_equalizer(struct mp_sws_context *sws, struct vf_seteq *eq);
 
+struct mp_image *mp_img_swap_to_native(struct mp_image *img);
+
 #endif /* MP_SWS_UTILS_H */
 
 // vim: ts=4 sw=4 et tw=80