diff --git a/libavcodec/hevc/hevcdec.c b/libavcodec/hevc/hevcdec.c
index cda52e05ef..0bf68ea45c 100644
--- a/libavcodec/hevc/hevcdec.c
+++ b/libavcodec/hevc/hevcdec.c
@@ -3006,7 +3006,11 @@ static int hevc_frame_start(HEVCContext *s)
     if (ret < 0)
         goto fail;
 
-    if (!s->avctx->hwaccel)
+    if (s->avctx->hwaccel) {
+        ret = FF_HW_CALL(s->avctx, start_frame, NULL, 0);
+        if (ret < 0)
+            goto fail;
+    } else
         ff_thread_finish_setup(s->avctx);
 
     return 0;
@@ -3148,12 +3152,6 @@ static int decode_nal_unit(HEVCContext *s, const H2645NAL *nal)
             }
         }
 
-        if (s->sh.first_slice_in_pic_flag && s->avctx->hwaccel) {
-            ret = FF_HW_CALL(s->avctx, start_frame, NULL, 0);
-            if (ret < 0)
-                goto fail;
-        }
-
         if (s->avctx->hwaccel) {
             ret = FF_HW_CALL(s->avctx, decode_slice, nal->raw_data, nal->raw_size);
             if (ret < 0)