lavu/fifo: disallow overly large fifo sizes

The API currently allows creating FIFOs up to
- UINT_MAX: av_fifo_alloc(), av_fifo_realloc(), av_fifo_grow()
- SIZE_MAX: av_fifo_alloc_array()
However the usable limit is determined by
- rndx/wndx being uint32_t
- av_fifo_[size,space] returning int
so no FIFO should be larger than the smallest of
- INT_MAX
- UINT32_MAX
- SIZE_MAX
(which should be INT_MAX an all commonly used platforms).
Return an error on trying to allocate FIFOs larger than this limit.
This commit is contained in:
Anton Khirnov 2022-01-03 13:09:33 +01:00 committed by Andreas Rheinhardt
parent 2d71f93c7c
commit 5939c8d361
1 changed files with 13 additions and 1 deletions

View File

@ -20,14 +20,23 @@
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
*/ */
#include <stdint.h>
#include "avassert.h" #include "avassert.h"
#include "common.h" #include "common.h"
#include "fifo.h" #include "fifo.h"
#define OLD_FIFO_SIZE_MAX (size_t)FFMIN3(INT_MAX, UINT32_MAX, SIZE_MAX)
AVFifoBuffer *av_fifo_alloc_array(size_t nmemb, size_t size) AVFifoBuffer *av_fifo_alloc_array(size_t nmemb, size_t size)
{ {
AVFifoBuffer *f; AVFifoBuffer *f;
void *buffer = av_realloc_array(NULL, nmemb, size); void *buffer;
if (nmemb > OLD_FIFO_SIZE_MAX / size)
return NULL;
buffer = av_realloc_array(NULL, nmemb, size);
if (!buffer) if (!buffer)
return NULL; return NULL;
f = av_mallocz(sizeof(AVFifoBuffer)); f = av_mallocz(sizeof(AVFifoBuffer));
@ -82,6 +91,9 @@ int av_fifo_realloc2(AVFifoBuffer *f, unsigned int new_size)
{ {
unsigned int old_size = f->end - f->buffer; unsigned int old_size = f->end - f->buffer;
if (new_size > OLD_FIFO_SIZE_MAX)
return AVERROR(EINVAL);
if (old_size < new_size) { if (old_size < new_size) {
size_t offset_r = f->rptr - f->buffer; size_t offset_r = f->rptr - f->buffer;
size_t offset_w = f->wptr - f->buffer; size_t offset_w = f->wptr - f->buffer;