diff --git a/libavcodec/lagarith.c b/libavcodec/lagarith.c
index 10cc71e0d8..9200773a65 100644
--- a/libavcodec/lagarith.c
+++ b/libavcodec/lagarith.c
@@ -245,21 +245,21 @@ static void lag_pred_line(LagarithContext *l, uint8_t *buf,
 {
     int L, TL;
 
+    /* Left pixel is actually prev_row[width] */
+    L = buf[width - stride - 1];
     if (!line) {
         /* Left prediction only for first line */
         L = l->dsp.add_hfyu_left_prediction(buf + 1, buf + 1,
                                             width - 1, buf[0]);
         return;
     } else if (line == 1) {
-        /* Second line, left predict first pixel, the rest of the line is median predicted */
-        /* FIXME: In the case of RGB this pixel is top predicted */
-        TL = buf[-stride];
+        /* Second line, left predict first pixel, the rest of the line is median predicted
+         * NOTE: In the case of RGB this pixel is top predicted */
+        TL = l->avctx->pix_fmt == PIX_FMT_YUV420P ? buf[-stride] : L;
     } else {
         /* Top left is 2 rows back, last pixel */
         TL = buf[width - (2 * stride) - 1];
     }
-    /* Left pixel is actually prev_row[width] */
-    L = buf[width - stride - 1];
 
     add_lag_median_prediction(buf, buf - stride, buf,
                               width, &L, &TL);