mp_image: change how palette is handled

According to DOCS/OUTDATED-tech/colorspaces.txt, the following formats
are supposed to be palettized:

    IMGFMT_BGR8
    IMGFMT_RGB8,
    IMGFMT_BGR4_CHAR
    IMGFMT_RGB4_CHAR
    IMGFMT_BGR4
    IMGFMT_RGB4

Of these, only BGR8 and RGB8 are actually treated as palettized in some
way. ffmpeg has only one palettized format (AV_PIX_FMT_PAL8), and
IMGFMT_BGR8 was inconsistently mapped to packed non-palettized RGB
formats too (AV_PIX_FMT_BGR8). Moreover, vf_scale.c contained messy
hacks to generate a palette when AV_PIX_FMT_BGR8 is output. (libswscale
does not support AV_PIX_FMT_PAL8 output in the first place.)

Get rid of all of this, and introduce IMGFMT_PAL8, which directly maps
to AV_PIX_FMT_PAL8. Remove the palette creation code from vf_scale.c.
IMGFMT_BGR8 maps to AV_PIX_FMT_RGB8 (don't ask me why it's swapped),
without any palette use. Enabling it in vo_x11 or using it as vf_scale
input seems to give correct results.

video/decode/vd_lavc.c

@@ -639,14 +639,6 @@ static void release_buffer_hwdec(AVCodecContext *avctx, AVFrame *pic)
         pic->data[i] = NULL;
 }
 
-static av_unused void swap_palette(void *pal)
-{
-    int i;
-    uint32_t *p = pal;
-    for (i = 0; i < AVPALETTE_COUNT; i++)
-        p[i] = le2me_32(p[i]);
-}
-
 static void fb_ref(void *b)
 {
     mp_buffer_ref(b);
@@ -740,12 +732,6 @@ static int decode(struct sh_video *sh, struct demux_packet *packet, void *data,
 
     assert(mpi->imgfmt == pixfmt2imgfmt(avctx->pix_fmt));
 
-#if BYTE_ORDER == BIG_ENDIAN
-    // FIXME: this might cause problems for buffers with FF_BUFFER_HINTS_PRESERVE
-    if (mpi->bpp == 8)
-        swap_palette(mpi->planes[1]);
-#endif
-
     mpi->colorspace = sh->colorspace;
     mpi->levels = sh->color_range;
     mpi->qscale = pic->qscale_table;

video/filter/vf.c

@@ -380,6 +380,9 @@ void vf_clone_mpi_attributes(mp_image_t *dst, mp_image_t *src)
         dst->colorspace = src->colorspace;
         dst->levels = src->levels;
     }
+    if (dst->imgfmt == IMGFMT_PAL8 && src->imgfmt == IMGFMT_PAL8) {
+        memcpy(dst->planes[1], src->planes[1], MP_PALETTE_SIZE);
+    }
 }
 
 // Used by filters to add a filtered frame to the output queue.

video/filter/vf_dlopen.c

@@ -240,7 +240,7 @@ static int query_format(struct vf_instance *vf, unsigned int fmt)
 {
     if (IMGFMT_IS_HWACCEL(fmt))
         return 0;  // these can't really be filtered
-    if (fmt == IMGFMT_RGB8 || fmt == IMGFMT_BGR8)
+    if (fmt == IMGFMT_PAL8)
         return 0;  // we don't have palette support, sorry
     const char *fmtname = mp_imgfmt_to_name(fmt);
     if (!fmtname)

video/filter/vf_mirror.c

@@ -102,7 +102,6 @@ static struct mp_image *filter(struct vf_instance *vf, struct mp_image *mpi)
 	mirror(dmpi->planes[0],mpi->planes[0],
 	       dmpi->stride[0],mpi->stride[0],
 	       dmpi->w,dmpi->h,dmpi->bpp>>3,mpi->imgfmt);
-	dmpi->planes[1]=mpi->planes[1]; // passthrough rgb8 palette
     }
 
     talloc_free(mpi);

video/filter/vf_rotate.c

@@ -99,7 +99,6 @@ static struct mp_image *filter(struct vf_instance *vf, struct mp_image *mpi)
 	rotate(dmpi->planes[0],mpi->planes[0],
 	       dmpi->stride[0],mpi->stride[0],
 	       dmpi->w,dmpi->h,dmpi->bpp>>3,vf->priv->direction);
-	dmpi->planes[1] = mpi->planes[1]; // passthrough rgb8 palette
     }
 
     talloc_free(mpi);

video/filter/vf_scale.c

@@ -47,7 +47,6 @@ static struct vf_priv_s {
     double param[2];
     unsigned int fmt;
     struct SwsContext *ctx;
-    unsigned char* palette;
     int interlaced;
     int noup;
     int accurate_rnd;
@@ -57,9 +56,6 @@ static struct vf_priv_s {
   -1,-1,
   0,
   {SWS_PARAM_DEFAULT, SWS_PARAM_DEFAULT},
-  0,
-  NULL,
-  NULL
 };
 
 static int mp_sws_set_colorspace(struct SwsContext *sws,
@@ -162,6 +158,7 @@ static int preferred_conversions[][2] = {
     {IMGFMT_GBRP, IMGFMT_RGB24},
     {IMGFMT_GBRP, IMGFMT_BGR32},
     {IMGFMT_GBRP, IMGFMT_RGB32},
+    {IMGFMT_PAL8, IMGFMT_BGR32},
     {0, 0}
 };
 
@@ -210,7 +207,6 @@ static int config(struct vf_instance *vf,
     unsigned int best=find_best_out(vf, outfmt);
     int int_sws_flags=0;
     int round_w=0, round_h=0;
-    int i;
     SwsFilter *srcFilter, *dstFilter;
     enum PixelFormat dfmt, sfmt;
 
@@ -221,7 +217,6 @@ static int config(struct vf_instance *vf,
 	return 0;
     }
     sfmt = imgfmt2pixfmt(outfmt);
-    if (outfmt == IMGFMT_RGB8 || outfmt == IMGFMT_BGR8) sfmt = PIX_FMT_PAL8;
     dfmt = imgfmt2pixfmt(best);
 
     vf->next->query_format(vf->next,best);
@@ -316,51 +311,6 @@ static int config(struct vf_instance *vf,
     }
     vf->priv->fmt=best;
 
-    free(vf->priv->palette);
-    vf->priv->palette=NULL;
-    switch(best){
-    case IMGFMT_RGB8: {
-      /* set 332 palette for 8 bpp */
-	vf->priv->palette=malloc(4*256);
-	for(i=0; i<256; i++){
-	    vf->priv->palette[4*i+0]=4*(i>>6)*21;
-	    vf->priv->palette[4*i+1]=4*((i>>3)&7)*9;
-	    vf->priv->palette[4*i+2]=4*((i&7)&7)*9;
-            vf->priv->palette[4*i+3]=0;
-	}
-	break; }
-    case IMGFMT_BGR8: {
-      /* set 332 palette for 8 bpp */
-	vf->priv->palette=malloc(4*256);
-	for(i=0; i<256; i++){
-	    vf->priv->palette[4*i+0]=4*(i&3)*21;
-	    vf->priv->palette[4*i+1]=4*((i>>2)&7)*9;
-	    vf->priv->palette[4*i+2]=4*((i>>5)&7)*9;
-            vf->priv->palette[4*i+3]=0;
-	}
-	break; }
-    case IMGFMT_BGR4:
-    case IMGFMT_BG4B: {
-	vf->priv->palette=malloc(4*16);
-	for(i=0; i<16; i++){
-	    vf->priv->palette[4*i+0]=4*(i&1)*63;
-	    vf->priv->palette[4*i+1]=4*((i>>1)&3)*21;
-	    vf->priv->palette[4*i+2]=4*((i>>3)&1)*63;
-            vf->priv->palette[4*i+3]=0;
-	}
-	break; }
-    case IMGFMT_RGB4:
-    case IMGFMT_RG4B: {
-	vf->priv->palette=malloc(4*16);
-	for(i=0; i<16; i++){
-	    vf->priv->palette[4*i+0]=4*(i>>3)*63;
-	    vf->priv->palette[4*i+1]=4*((i>>1)&3)*21;
-	    vf->priv->palette[4*i+2]=4*((i&1)&1)*63;
-            vf->priv->palette[4*i+3]=0;
-	}
-	break; }
-    }
-
     if (!opts->screen_size_x && !opts->screen_size_y
         && !(opts->screen_size_xy >= 0.001)) {
 	// Compute new d_width and d_height, preserving aspect
@@ -566,7 +516,6 @@ static int query_format(struct vf_instance *vf, unsigned int fmt){
 
 static void uninit(struct vf_instance *vf){
     if(vf->priv->ctx) sws_freeContext(vf->priv->ctx);
-    free(vf->priv->palette);
     free(vf->priv);
 }
 

video/filter/vf_sub.c

@@ -144,7 +144,6 @@ static void prepare_image(struct vf_instance *vf, struct mp_image *dmpi,
 		   mpi->h,
                    dmpi->stride[0],
 		   mpi->stride[0]);
-        dmpi->planes[1] = mpi->planes[1];   // passthrough rgb8 palette
     }
     if (tmargin)
         blank(dmpi, 0, tmargin);

video/fmt-conversion.c

@@ -54,7 +54,7 @@ static const struct {
     {IMGFMT_RGB12LE, PIX_FMT_BGR444LE},
     {IMGFMT_RGB8,  PIX_FMT_BGR8},
     {IMGFMT_RGB4,  PIX_FMT_BGR4},
-    {IMGFMT_BGR8,  PIX_FMT_PAL8},
+    {IMGFMT_PAL8,  PIX_FMT_PAL8},
     {IMGFMT_GBRP,  PIX_FMT_GBRP},
     {IMGFMT_YUY2,  PIX_FMT_YUYV422},
     {IMGFMT_UYVY,  PIX_FMT_UYVY422},

video/img_format.c

@@ -113,6 +113,7 @@ struct mp_imgfmt_entry mp_imgfmt_list[] = {
     {"rgb15", IMGFMT_RGB15},
     {"rgb12", IMGFMT_RGB12},
     {"rgb8", IMGFMT_RGB8},
+    {"pal8", IMGFMT_PAL8},
     {"rgb4", IMGFMT_RGB4},
     {"rg4b", IMGFMT_RG4B},
     {"rgb1", IMGFMT_RGB1},

video/img_format.h

@@ -33,8 +33,6 @@
 #define MP_IMGFLAG_YUV 0x200
 // set if it's swapped (BGR or YVU) plane/byteorder
 #define MP_IMGFLAG_SWAPPED 0x400
-// set if you want memory for palette allocated and managed by vf_get_image etc.
-#define MP_IMGFLAG_RGB_PALETTE 0x800
 // set if the format is standard YUV format:
 // - planar and yuv colorspace
 // - chroma shift 0-2
@@ -135,7 +133,6 @@ struct mp_imgfmt_desc mp_imgfmt_get_desc(unsigned int out_fmt);
 #define IMGFMT_RG4B  IMGFMT_RGB4_CHAR
 #define IMGFMT_BG4B  IMGFMT_BGR4_CHAR
 
-
 // AV_PIX_FMT_BGR0
 #define IMGFMT_BGR0  0x1DC70000
 
@@ -159,6 +156,8 @@ static inline bool IMGFMT_IS_BGR(unsigned int fmt)
 #define IMGFMT_Y16LE 0x1DC70001
 // AV_PIX_FMT_GRAY16BE
 #define IMGFMT_Y16BE 0x1DC70002
+// AV_PIX_FMT_PAL8
+#define IMGFMT_PAL8  0x1DC70003
 
 #if BYTE_ORDER == BIG_ENDIAN
 #define IMGFMT_Y16 IMGFMT_Y16BE

video/mp_image.c

@@ -105,7 +105,7 @@ static void mp_image_alloc_planes(struct mp_image *mpi)
         mpi->stride[n] = FFALIGN(line_bytes, SWS_MIN_BYTE_ALIGN);
         plane_size[n] = mpi->stride[n] * mpi->plane_h[n];
     }
-    if (mpi->flags & MP_IMGFLAG_RGB_PALETTE)
+    if (mpi->imgfmt == IMGFMT_PAL8)
         plane_size[1] = MP_PALETTE_SIZE;
 
     size_t sum = 0;
@@ -184,8 +184,6 @@ struct mp_image *mp_image_alloc(unsigned int imgfmt, int w, int h)
     mpi->refcount = m_refcount_new();
     mpi->refcount->free = av_free;
     mpi->refcount->arg = mpi->planes[0];
-    // NOTE: palette isn't free'd. Palette handling should be fixed instead.
-
     return mpi;
 }
 
@@ -318,6 +316,8 @@ void mp_image_copy(struct mp_image *dst, struct mp_image *src)
         memcpy_pic(dst->planes[n], src->planes[n], line_bytes, dst->plane_h[n],
                    dst->stride[n], src->stride[n]);
     }
+    if (dst->imgfmt == IMGFMT_PAL8)
+        memcpy(dst->planes[1], src->planes[1], MP_PALETTE_SIZE);
 }
 
 enum mp_csp mp_image_csp(struct mp_image *img)

video/out/vo_lavc.c

@@ -87,8 +87,7 @@ static void uninit(struct vo *vo)
 
     if (vc->lastimg) {
         // palette hack
-        if (vc->lastimg->imgfmt == IMGFMT_RGB8
-                || vc->lastimg->imgfmt == IMGFMT_BGR8)
+        if (vc->lastimg->imgfmt == IMGFMT_PAL8)
             vc->lastimg->planes[1] = NULL;
         free_mp_image(vc->lastimg);
         vc->lastimg = NULL;
@@ -175,9 +174,8 @@ static int config(struct vo *vo, uint32_t width, uint32_t height,
     vc->lastimg = alloc_mpi(width, height, format);
 
     // palette hack
-    if (vc->lastimg->imgfmt == IMGFMT_RGB8 ||
-            vc->lastimg->imgfmt == IMGFMT_BGR8)
-        vc->lastimg->planes[1] = talloc_zero_size(vc, 1024);
+    if (vc->lastimg->imgfmt == IMGFMT_PAL8)
+        vc->lastimg->planes[1] = talloc_zero_size(vc, MP_PALETTE_SIZE);
 
     return 0;
 
@@ -461,9 +459,8 @@ static void draw_image(struct vo *vo, mp_image_t *mpi)
             vc->lastimg_wants_osd = true;
 
             // palette hack
-            if (vc->lastimg->imgfmt == IMGFMT_RGB8 ||
-                    vc->lastimg->imgfmt == IMGFMT_BGR8)
-                memcpy(vc->lastimg->planes[1], mpi->planes[1], 1024);
+            if (vc->lastimg->imgfmt == IMGFMT_PAL8)
+                memcpy(vc->lastimg->planes[1], mpi->planes[1], MP_PALETTE_SIZE);
 
             vc->lastframeipts = vc->lastipts = frameipts;
             if (ectx->options->rawts && vc->lastipts < 0) {

video/out/vo_x11.c

@@ -526,8 +526,8 @@ static int query_format(struct vo *vo, uint32_t format)
            "vo_x11: query_format was called: %x (%s)\n", format,
            vo_format_name(format));
     if (IMGFMT_IS_BGR(format)) {
-        if (IMGFMT_BGR_DEPTH(format) <= 8)
-            return 0;           // TODO 8bpp not yet fully implemented
+        if (IMGFMT_BGR_DEPTH(format) < 8)
+            return 0;
         if (IMGFMT_BGR_DEPTH(format) == vo->x11->depthonscreen)
             return VFCAP_CSP_SUPPORTED | VFCAP_CSP_SUPPORTED_BY_HW |
                    VFCAP_OSD | VFCAP_FLIP;

video/sws_utils.c

@@ -103,8 +103,6 @@ struct SwsContext *sws_getContextFromCmdLine(int srcW, int srcH,
 
     dfmt = imgfmt2pixfmt(dstFormat);
     sfmt = imgfmt2pixfmt(srcFormat);
-    if (srcFormat == IMGFMT_RGB8 || srcFormat == IMGFMT_BGR8)
-        sfmt = PIX_FMT_PAL8;
     sws_getFlagsAndFilterFromCmdLine(&flags, &srcFilterParam, &dstFilterParam);
 
     return sws_getContext(srcW, srcH, sfmt, dstW, dstH, dfmt, flags,
@@ -176,8 +174,6 @@ static void mp_sws_set_conv(struct SwsContext *sws, struct mp_image *dst,
                             struct mp_image *src, int my_sws_flags)
 {
     enum PixelFormat s_fmt = imgfmt2pixfmt(src->imgfmt);
-    if (src->imgfmt == IMGFMT_RGB8 || src->imgfmt == IMGFMT_BGR8)
-        s_fmt = PIX_FMT_PAL8;
     int s_csp = mp_csp_to_sws_colorspace(mp_image_csp(src));
     int s_range = mp_image_levels(src) == MP_CSP_LEVELS_PC;