diff --git a/libavcodec/hevc.c b/libavcodec/hevc.c
index 63a2c2f66d..fa5a505189 100644
--- a/libavcodec/hevc.c
+++ b/libavcodec/hevc.c
@@ -2455,7 +2455,7 @@ static int hls_slice_data_wpp(HEVCContext *s, const HEVCNAL *nal)
     offset = (lc->gb.index >> 3);
 
     for (j = 0, cmpt = 0, startheader = offset + s->sh.entry_point_offset[0]; j < nal->skipped_bytes; j++) {
-        if (s->skipped_bytes_pos[j] >= offset && s->skipped_bytes_pos[j] < startheader) {
+        if (nal->skipped_bytes_pos[j] >= offset && nal->skipped_bytes_pos[j] < startheader) {
             startheader--;
             cmpt++;
         }
@@ -2465,7 +2465,7 @@ static int hls_slice_data_wpp(HEVCContext *s, const HEVCNAL *nal)
         offset += (s->sh.entry_point_offset[i - 1] - cmpt);
         for (j = 0, cmpt = 0, startheader = offset
              + s->sh.entry_point_offset[i]; j < nal->skipped_bytes; j++) {
-            if (s->skipped_bytes_pos[j] >= offset && s->skipped_bytes_pos[j] < startheader) {
+            if (nal->skipped_bytes_pos[j] >= offset && nal->skipped_bytes_pos[j] < startheader) {
                 startheader--;
                 cmpt++;
             }
@@ -2781,8 +2781,6 @@ static int decode_nal_units(HEVCContext *s, const uint8_t *buf, int length)
 
     /* decode the NAL units */
     for (i = 0; i < s->pkt.nb_nals; i++) {
-        s->skipped_bytes_pos = s->pkt.nals[i].skipped_bytes_pos_nal;
-
         ret = decode_nal_unit(s, &s->pkt.nals[i]);
         if (ret < 0) {
             av_log(s->avctx, AV_LOG_WARNING,
@@ -3011,7 +3009,7 @@ static av_cold int hevc_decode_free(AVCodecContext *avctx)
 
     for (i = 0; i < s->pkt.nals_allocated; i++) {
         av_freep(&s->pkt.nals[i].rbsp_buffer);
-        av_freep(&s->pkt.nals[i].skipped_bytes_pos_nal);
+        av_freep(&s->pkt.nals[i].skipped_bytes_pos);
     }
     av_freep(&s->pkt.nals);
     s->pkt.nals_allocated = 0;
diff --git a/libavcodec/hevc.h b/libavcodec/hevc.h
index 8ae1ace3ac..2c1aff77b3 100644
--- a/libavcodec/hevc.h
+++ b/libavcodec/hevc.h
@@ -761,8 +761,8 @@ typedef struct HEVCNAL {
     int temporal_id;
 
     int skipped_bytes;
-    int skipped_bytes_pos_size_nal;
-    int *skipped_bytes_pos_nal;
+    int skipped_bytes_pos_size;
+    int *skipped_bytes_pos;
 } HEVCNAL;
 
 /* an input packet split into unescaped NAL units */
@@ -901,8 +901,6 @@ typedef struct HEVCContext {
 
     int enable_parallel_tiles;
     int wpp_err;
-    int *skipped_bytes_pos;
-    int skipped_bytes_pos_size;
 
     const uint8_t *data;
 
diff --git a/libavcodec/hevc_parse.c b/libavcodec/hevc_parse.c
index 453a656cd3..5ba5150fed 100644
--- a/libavcodec/hevc_parse.c
+++ b/libavcodec/hevc_parse.c
@@ -110,18 +110,21 @@ int ff_hevc_extract_rbsp(HEVCContext *s, const uint8_t *src, int length,
                 dst[di++] = 0;
                 si       += 3;
 
-                if (s) {
-                        nal->skipped_bytes++;
-                        if (s->skipped_bytes_pos_size < nal->skipped_bytes) {
-                        s->skipped_bytes_pos_size *= 2;
-                        av_reallocp_array(&s->skipped_bytes_pos,
-                                s->skipped_bytes_pos_size,
-                                sizeof(*s->skipped_bytes_pos));
-                        if (!s->skipped_bytes_pos)
-                                return AVERROR(ENOMEM);
+                if (s && nal->skipped_bytes_pos) {
+                    nal->skipped_bytes++;
+                    if (nal->skipped_bytes_pos_size < nal->skipped_bytes) {
+                        nal->skipped_bytes_pos_size *= 2;
+                        av_assert0(nal->skipped_bytes_pos_size >= nal->skipped_bytes);
+                        av_reallocp_array(&nal->skipped_bytes_pos,
+                                nal->skipped_bytes_pos_size,
+                                sizeof(*nal->skipped_bytes_pos));
+                        if (!nal->skipped_bytes_pos) {
+                            nal->skipped_bytes_pos_size = 0;
+                            return AVERROR(ENOMEM);
                         }
-                        if (s->skipped_bytes_pos)
-                        s->skipped_bytes_pos[nal->skipped_bytes-1] = di - 1;
+                    }
+                    if (nal->skipped_bytes_pos)
+                        nal->skipped_bytes_pos[nal->skipped_bytes-1] = di - 1;
                 }
                 continue;
             } else // next start code
@@ -219,23 +222,19 @@ int ff_hevc_split_packet(HEVCContext *s, HEVCPacket *pkt, const uint8_t *buf, in
                    (new_size - pkt->nals_allocated) * sizeof(*pkt->nals));
 
             nal = &pkt->nals[pkt->nb_nals];
-            nal->skipped_bytes_pos_size_nal = 1024; // initial buffer size
-            nal->skipped_bytes_pos_nal = av_malloc_array(nal->skipped_bytes_pos_size_nal, sizeof(*s->skipped_bytes_pos));
-            if (!nal->skipped_bytes_pos_nal)
+            nal->skipped_bytes_pos_size = 1024; // initial buffer size
+            nal->skipped_bytes_pos = av_malloc_array(nal->skipped_bytes_pos_size, sizeof(*nal->skipped_bytes_pos));
+            if (!nal->skipped_bytes_pos)
                 return AVERROR(ENOMEM);
 
             pkt->nals_allocated = new_size;
         }
         nal = &pkt->nals[pkt->nb_nals];
-        s->skipped_bytes_pos_size = nal->skipped_bytes_pos_size_nal;
-        s->skipped_bytes_pos = nal->skipped_bytes_pos_nal;
 
         consumed = ff_hevc_extract_rbsp(s, buf, extract_length, nal);
         if (consumed < 0)
             return consumed;
 
-        nal->skipped_bytes_pos_size_nal = s->skipped_bytes_pos_size;
-        nal->skipped_bytes_pos_nal      = s->skipped_bytes_pos;
         pkt->nb_nals++;
 
         ret = init_get_bits8(&nal->gb, nal->data, nal->size);
diff --git a/libavcodec/hevc_parser.c b/libavcodec/hevc_parser.c
index d6171b0472..5c1ff3038d 100644
--- a/libavcodec/hevc_parser.c
+++ b/libavcodec/hevc_parser.c
@@ -313,7 +313,6 @@ static int hevc_init(AVCodecParserContext *s)
     h->HEVClc = av_mallocz(sizeof(HEVCLocalContext));
     if (!h->HEVClc)
         return AVERROR(ENOMEM);
-    h->skipped_bytes_pos_size = INT_MAX;
 
     return 0;
 }
@@ -326,7 +325,6 @@ static void hevc_close(AVCodecParserContext *s)
     HEVCParamSets *ps = &h->ps;
     HEVCPacket   *pkt = &h->pkt;
 
-    av_freep(&h->skipped_bytes_pos);
     av_freep(&h->HEVClc);
     av_freep(&pc->buffer);
 
@@ -339,8 +337,10 @@ static void hevc_close(AVCodecParserContext *s)
 
     ps->sps = NULL;
 
-    for (i = 0; i < pkt->nals_allocated; i++)
+    for (i = 0; i < pkt->nals_allocated; i++) {
         av_freep(&pkt->nals[i].rbsp_buffer);
+        av_freep(&pkt->nals[i].skipped_bytes_pos);
+    }
     av_freep(&pkt->nals);
     pkt->nals_allocated = 0;
 }