From de3f6718124372f64443a0b86527c1abf86d6f8c Mon Sep 17 00:00:00 2001
From: reimar <reimar@b3059339-0415-0410-9bf9-f77b7e298cf2>
Date: Sat, 5 Nov 2011 18:20:44 +0000
Subject: [PATCH] vd_ffmpeg, vf: fix crashes with some game formats

Fixes for palette allocation handling.

git-svn-id: svn://svn.mplayerhq.hu/mplayer/trunk@34304 b3059339-0415-0410-9bf9-f77b7e298cf2

This caused a crash with http://samples.ffmpeg.org/cdxl/fruit.cdxl if
direct rendering was used. (Which is rarely these days.)

With small changes: avoid av_freep() use, as this function is not sane.
---
 libmpcodecs/vd_ffmpeg.c |  3 ---
 libmpcodecs/vf.c        | 10 ++++++++--
 2 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/libmpcodecs/vd_ffmpeg.c b/libmpcodecs/vd_ffmpeg.c
index d36e4122a0..343a26d8d3 100644
--- a/libmpcodecs/vd_ffmpeg.c
+++ b/libmpcodecs/vd_ffmpeg.c
@@ -583,9 +583,6 @@ static void release_buffer(struct AVCodecContext *avctx, AVFrame *pic)
     }
 
     if (mpi) {
-        // Palette support: free palette buffer allocated in get_buffer
-        if (mpi->bpp == 8)
-            av_freep(&mpi->planes[1]);
         // release mpi (in case MPI_IMGTYPE_NUMBERED is used, e.g. for VDPAU)
         mpi->usage_count--;
     }
diff --git a/libmpcodecs/vf.c b/libmpcodecs/vf.c
index 5ec7d19570..8cf6002246 100644
--- a/libmpcodecs/vf.c
+++ b/libmpcodecs/vf.c
@@ -252,6 +252,8 @@ mp_image_t *vf_get_image(vf_instance_t *vf, unsigned int outfmt,
         break;
     }
     if (mpi) {
+        int missing_palette = !(mpi->flags & MP_IMGFLAG_RGB_PALETTE) &&
+                              (mp_imgflag & MP_IMGFLAG_RGB_PALETTE);
         mpi->type = mp_imgtype;
         mpi->w = vf->w;
         mpi->h = vf->h;
@@ -263,11 +265,15 @@ mp_image_t *vf_get_image(vf_instance_t *vf, unsigned int outfmt,
                           MP_IMGFLAG_DRAW_CALLBACK | MP_IMGFLAG_RGB_PALETTE);
         if (!vf->draw_slice)
             mpi->flags &= ~MP_IMGFLAG_DRAW_CALLBACK;
-        if (mpi->width != w2 || mpi->height != h) {
+        if (mpi->width != w2 || mpi->height != h || missing_palette) {
             if (mpi->flags & MP_IMGFLAG_ALLOCATED) {
-                if (mpi->width < w2 || mpi->height < h) {
+                if (mpi->width < w2 || mpi->height < h || missing_palette) {
                     // need to re-allocate buffer memory:
                     av_free(mpi->planes[0]);
+                    if (mpi->flags & MP_IMGFLAG_RGB_PALETTE)
+                        av_free(mpi->planes[1]);
+                    for (int n = 0; n < MP_MAX_PLANES; n++)
+                        mpi->planes[n] = NULL;
                     mpi->flags &= ~MP_IMGFLAG_ALLOCATED;
                     mp_msg(MSGT_VFILTER, MSGL_V,
                            "vf.c: have to REALLOCATE buffer memory :(\n");