diff --git a/libswscale/swscale_internal.h b/libswscale/swscale_internal.h
index 9676d463d1..721a9cafaa 100644
--- a/libswscale/swscale_internal.h
+++ b/libswscale/swscale_internal.h
@@ -322,6 +322,7 @@ typedef struct SwsContext {
     uint8_t *cascaded_tmp[4];
     int cascaded1_tmpStride[4];
     uint8_t *cascaded1_tmp[4];
+    int cascaded_mainindex;
 
     double gamma_value;
     int gamma_flag;
diff --git a/libswscale/utils.c b/libswscale/utils.c
index 5265157d3e..eb1c940371 100644
--- a/libswscale/utils.c
+++ b/libswscale/utils.c
@@ -858,8 +858,8 @@ int sws_setColorspaceDetails(struct SwsContext *c, const int inv_table[4],
     c->dstFormatBpp = av_get_bits_per_pixel(desc_dst);
     c->srcFormatBpp = av_get_bits_per_pixel(desc_src);
 
-    if (c->cascaded_context[0])
-        return sws_setColorspaceDetails(c->cascaded_context[0],inv_table, srcRange,table, dstRange, brightness,  contrast, saturation);
+    if (c->cascaded_context[c->cascaded_mainindex])
+        return sws_setColorspaceDetails(c->cascaded_context[c->cascaded_mainindex],inv_table, srcRange,table, dstRange, brightness,  contrast, saturation);
 
     if ((isYUV(c->dstFormat) || isGray(c->dstFormat)) && (isYUV(c->srcFormat) || isGray(c->srcFormat))) {
         if (!c->cascaded_context[0] &&
@@ -1485,6 +1485,7 @@ av_cold int sws_init_context(SwsContext *c, SwsFilter *srcFilter,
             usesHFilter || usesVFilter ||
             c->srcRange != c->dstRange
         ) {
+            c->cascaded_mainindex = 1;
             ret = av_image_alloc(c->cascaded_tmp, c->cascaded_tmpStride,
                                 srcW, srcH, tmpFormat, 64);
             if (ret < 0)