From f1e17eb446577180ee9976730aacb46563766518 Mon Sep 17 00:00:00 2001 From: Paul B Mahol Date: Wed, 25 Sep 2019 17:18:32 +0200 Subject: [PATCH] avcodec/qtrleenc: fix undefined behaviour Fixes #7991. --- libavcodec/qtrleenc.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libavcodec/qtrleenc.c b/libavcodec/qtrleenc.c index cdd864bf82..6669c1302f 100644 --- a/libavcodec/qtrleenc.c +++ b/libavcodec/qtrleenc.c @@ -259,9 +259,10 @@ static void qtrle_encode_line(QtrleEncContext *s, const AVFrame *p, int line, ui /* These bulk costs increase every iteration */ lowest_bulk_cost += s->pixel_size; sec_lowest_bulk_cost += s->pixel_size; - - this_line -= s->pixel_size; - prev_line -= s->pixel_size; + if (this_line >= p->data[0] + s->pixel_size) + this_line -= s->pixel_size; + if (prev_line >= s->previous_frame->data[0] + s->pixel_size) + prev_line -= s->pixel_size; } /* Good! Now we have the best sequence for this line, let's output it. */