diff --git a/libavcodec/libx264.c b/libavcodec/libx264.c
index 0685b300a0..aba02821aa 100644
--- a/libavcodec/libx264.c
+++ b/libavcodec/libx264.c
@@ -268,11 +268,11 @@ static int X264_frame(AVCodecContext *ctx, AVPacket *pkt, const AVFrame *frame,
     }
     do {
         if (x264_encoder_encode(x4->enc, &nal, &nnal, frame? &x4->pic: NULL, &pic_out) < 0)
-            return -1;
+            return AVERROR_UNKNOWN;
 
         ret = encode_nals(ctx, pkt, nal, nnal);
         if (ret < 0)
-            return -1;
+            return ret;
     } while (!ret && !frame && x264_encoder_delayed_frames(x4->enc));
 
     pkt->pts = pic_out.i_pts;
@@ -687,7 +687,7 @@ static av_cold int X264_init(AVCodecContext *avctx)
 
     x4->enc = x264_encoder_open(&x4->params);
     if (!x4->enc)
-        return -1;
+        return AVERROR_UNKNOWN;
 
     avctx->coded_frame = av_frame_alloc();
     if (!avctx->coded_frame)