avcodec/zlib_wrapper: Add wrappers for zlib inflateInit, inflateEnd

It is not documented to be safe to call inflateEnd() on a z_stream
that has never been successfully been initialized by inflateInit(),
but just zeroed. It just happens to work and several codecs rely
on this (they have FF_CODEC_CAP_INIT_CLEANUP set and even call
inflateEnd() when inflateInit() failed or has never been called).
To avoid this, other codecs recorded whether their zstream has been
initialized successfully or not.

This commit adds wrappers for inflateInit() and inflateEnd() that
do what these other codecs do; furthermore, they also take care of
properly setting up the zstream before inflateInit() and emit
an error message in case of error.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>

configure

@@ -2461,6 +2461,7 @@ CONFIG_EXTRA="
     idctdsp
     iirfilter
     mdct15
+    inflate_wrapper
     intrax8
     iso_media
     ividsp
@@ -2722,6 +2723,7 @@ faanidct_select="idctdsp"
 h264dsp_select="startcode"
 hevcparse_select="atsc_a53 golomb"
 frame_thread_encoder_deps="encoders threads"
+inflate_wrapper_deps="zlib"
 intrax8_select="blockdsp idctdsp"
 iso_media_select="mpeg4audio"
 mdct_select="fft"

libavcodec/Makefile

@@ -105,6 +105,7 @@ OBJS-$(CONFIG_HUFFYUVENCDSP)           += huffyuvencdsp.o
 OBJS-$(CONFIG_IDCTDSP)                 += idctdsp.o simple_idct.o jrevdct.o
 OBJS-$(CONFIG_IIRFILTER)               += iirfilter.o
 OBJS-$(CONFIG_MDCT15)                  += mdct15.o
+OBJS-$(CONFIG_INFLATE_WRAPPER)         += zlib_wrapper.o
 OBJS-$(CONFIG_INTRAX8)                 += intrax8.o intrax8dsp.o msmpeg4data.o
 OBJS-$(CONFIG_IVIDSP)                  += ivi_dsp.o
 OBJS-$(CONFIG_JNI)                     += ffjni.o jni.o

libavcodec/zlib_wrapper.c

@@ -0,0 +1,57 @@
+/*
+ * Wrappers for zlib
+ * Copyright (C) 2022 Andreas Rheinhardt
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#include <zlib.h>
+
+#include "libavutil/error.h"
+#include "libavutil/log.h"
+#include "zlib_wrapper.h"
+
+int ff_inflate_init(FFZStream *z, void *logctx)
+{
+    z_stream *const zstream = &z->zstream;
+    int zret;
+
+    z->inited = 0;
+    zstream->next_in  = Z_NULL;
+    zstream->avail_in = 0;
+    zstream->zalloc   = Z_NULL;
+    zstream->zfree    = Z_NULL;
+    zstream->opaque   = Z_NULL;
+
+    zret = inflateInit(zstream);
+    if (zret == Z_OK) {
+        z->inited = 1;
+    } else {
+        av_log(logctx, AV_LOG_ERROR, "inflateInit error %d, message: %s\n",
+               zret, zstream->msg ? zstream->msg : "");
+        return AVERROR_EXTERNAL;
+    }
+    return 0;
+}
+
+void ff_inflate_end(FFZStream *z)
+{
+    if (z->inited) {
+        z->inited = 0;
+        inflateEnd(&z->zstream);
+    }
+}

libavcodec/zlib_wrapper.h

@@ -0,0 +1,51 @@
+/*
+ * Wrappers for zlib
+ * Copyright (C) 2022 Andreas Rheinhardt
+ *
+ * This file is part of FFmpeg.
+ *
+ * FFmpeg is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * FFmpeg is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with FFmpeg; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
+ */
+
+#ifndef AVCODEC_ZLIB_WRAPPER_H
+#define AVCODEC_ZLIB_WRAPPER_H
+
+#include <zlib.h>
+
+typedef struct FFZStream {
+    z_stream zstream;
+    int inited;
+} FFZStream;
+
+/**
+ * Wrapper around inflateInit(). It initializes the fields that zlib
+ * requires to be initialized before inflateInit().
+ * In case of error it also returns an error message to the provided logctx;
+ * in any case, it sets zstream->inited to indicate whether inflateInit()
+ * succeeded.
+ * @return Returns 0 on success or a negative error code on failure
+ */
+int ff_inflate_init(FFZStream *zstream, void *logctx);
+
+/**
+ * Wrapper around inflateEnd(). It calls inflateEnd() iff
+ * zstream->inited is set and resets zstream->inited.
+ * It is therefore safe to be called even if
+ * ff_inflate_init() has never been called on it (or errored out)
+ * provided that the FFZStream (or just FFZStream.inited) has been zeroed.
+ */
+void ff_inflate_end(FFZStream *zstream);
+
+#endif /* AVCODEC_ZLIB_WRAPPER_H */