diff --git a/libavcodec/hevc/hevcdec.c b/libavcodec/hevc/hevcdec.c
index 46db7923fe..ae4a5888e5 100644
--- a/libavcodec/hevc/hevcdec.c
+++ b/libavcodec/hevc/hevcdec.c
@@ -625,8 +625,8 @@ static int hls_slice_header(HEVCContext *s, GetBitContext *gb)
     if (s->nal_unit_type == HEVC_NAL_CRA_NUT && s->last_eos == 1)
         sh->no_output_of_prior_pics_flag = 1;
 
-    if (s->ps.sps != s->ps.sps_list[s->ps.pps->sps_id]) {
-        const HEVCSPS *sps = s->ps.sps_list[s->ps.pps->sps_id];
+    if (s->ps.sps != s->ps.pps->sps) {
+        const HEVCSPS *sps = s->ps.pps->sps;
         enum AVPixelFormat pix_fmt;
 
         ff_hevc_clear_refs(s);
diff --git a/libavcodec/hevc/parser.c b/libavcodec/hevc/parser.c
index 056e1b4aa4..d0d5e7fbc2 100644
--- a/libavcodec/hevc/parser.c
+++ b/libavcodec/hevc/parser.c
@@ -80,12 +80,8 @@ static int hevc_parse_slice_header(AVCodecParserContext *s, H2645NAL *nal,
     }
     ps->pps = ps->pps_list[pps_id];
 
-    if (ps->pps->sps_id >= HEVC_MAX_SPS_COUNT || !ps->sps_list[ps->pps->sps_id]) {
-        av_log(avctx, AV_LOG_ERROR, "SPS id out of range: %d\n", ps->pps->sps_id);
-        return AVERROR_INVALIDDATA;
-    }
-    if (ps->sps != ps->sps_list[ps->pps->sps_id]) {
-        ps->sps  = ps->sps_list[ps->pps->sps_id];
+    if (ps->sps != ps->pps->sps) {
+        ps->sps  = ps->pps->sps;
         ps->vps  = ps->vps_list[ps->sps->vps_id];
     }
     ow  = &ps->sps->output_window;
diff --git a/libavcodec/hevc/ps.c b/libavcodec/hevc/ps.c
index 2dd4f834a4..98217e337b 100644
--- a/libavcodec/hevc/ps.c
+++ b/libavcodec/hevc/ps.c
@@ -1363,6 +1363,8 @@ static void hevc_pps_free(FFRefStructOpaque unused, void *obj)
 {
     HEVCPPS *pps = obj;
 
+    ff_refstruct_unref(&pps->sps);
+
     av_freep(&pps->column_width);
     av_freep(&pps->row_height);
     av_freep(&pps->col_bd);
@@ -1828,6 +1830,8 @@ int ff_hevc_decode_nal_pps(GetBitContext *gb, AVCodecContext *avctx,
     sps = ps->sps_list[pps->sps_id];
     vps = ps->vps_list[sps->vps_id];
 
+    pps->sps = ff_refstruct_ref_c(sps);
+
     pps->dependent_slice_segments_enabled_flag = get_bits1(gb);
     pps->output_flag_present_flag              = get_bits1(gb);
     pps->num_extra_slice_header_bits           = get_bits(gb, 3);
diff --git a/libavcodec/hevc/ps.h b/libavcodec/hevc/ps.h
index 99d70cefd2..7c9aacf057 100644
--- a/libavcodec/hevc/ps.h
+++ b/libavcodec/hevc/ps.h
@@ -437,6 +437,8 @@ typedef struct HEVCPPS {
 
     uint8_t *data;
     int data_size;
+
+    const HEVCSPS *sps;     ///< RefStruct reference
 } HEVCPPS;
 
 typedef struct HEVCParamSets {