diff --git a/libavcodec/nvenc.c b/libavcodec/nvenc.c
index 2fdfdcf17b..4be0405113 100644
--- a/libavcodec/nvenc.c
+++ b/libavcodec/nvenc.c
@@ -828,8 +828,6 @@ static av_cold int nvenc_setup_encoder(AVCodecContext *avctx)
     int res = 0;
     int dw, dh;
 
-    ctx->last_dts = AV_NOPTS_VALUE;
-
     ctx->encode_config.version = NV_ENC_CONFIG_VER;
     ctx->init_encode_params.version = NV_ENC_INITIALIZE_PARAMS_VER;
 
@@ -897,9 +895,8 @@ static av_cold int nvenc_setup_encoder(AVCodecContext *avctx)
         ctx->encode_config.gopLength = 1;
     }
 
-    /* when there're b frames, set dts offset */
-    if (ctx->encode_config.frameIntervalP >= 2)
-        ctx->last_dts = -2;
+    ctx->initial_pts[0] = AV_NOPTS_VALUE;
+    ctx->initial_pts[1] = AV_NOPTS_VALUE;
 
     nvenc_setup_rate_control(avctx);
 
@@ -1419,19 +1416,30 @@ static int nvenc_set_timestamp(AVCodecContext *avctx,
 
     pkt->pts = params->outputTimeStamp;
     pkt->duration = params->outputDuration;
+
+    /* generate the first dts by linearly extrapolating the
+     * first two pts values to the past */
+    if (avctx->max_b_frames > 0 && !ctx->first_packet_output &&
+        ctx->initial_pts[1] != AV_NOPTS_VALUE) {
+        int64_t ts0 = ctx->initial_pts[0], ts1 = ctx->initial_pts[1];
+        int64_t delta;
+
+        if ((ts0 < 0 && ts1 > INT64_MAX + ts0) ||
+            (ts0 > 0 && ts1 < INT64_MIN + ts0))
+            return AVERROR(ERANGE);
+        delta = ts1 - ts0;
+
+        if ((delta < 0 && ts0 > INT64_MAX + delta) ||
+            (delta > 0 && ts0 < INT64_MIN + delta))
+            return AVERROR(ERANGE);
+        pkt->dts = ts0 - delta;
+
+        ctx->first_packet_output = 1;
+        return 0;
+    }
+
     pkt->dts = timestamp_queue_dequeue(ctx->timestamp_list);
 
-    /* when there're b frame(s), set dts offset */
-    if (ctx->encode_config.frameIntervalP >= 2)
-        pkt->dts -= 1;
-
-    if (pkt->dts > pkt->pts)
-        pkt->dts = pkt->pts;
-
-    if (ctx->last_dts != AV_NOPTS_VALUE && pkt->dts <= ctx->last_dts)
-        pkt->dts = ctx->last_dts + 1;
-
-    ctx->last_dts = pkt->dts;
     return 0;
 }
 
@@ -1551,6 +1559,12 @@ static int output_ready(AVCodecContext *avctx, int flush)
     NvencContext *ctx = avctx->priv_data;
     int nb_ready, nb_pending;
 
+    /* when B-frames are enabled, we wait for two initial timestamps to
+     * calculate the first dts */
+    if (!flush && avctx->max_b_frames > 0 &&
+        (ctx->initial_pts[0] == AV_NOPTS_VALUE || ctx->initial_pts[1] == AV_NOPTS_VALUE))
+        return 0;
+
     nb_ready   = av_fifo_size(ctx->output_surface_ready_queue)   / sizeof(NvencSurface*);
     nb_pending = av_fifo_size(ctx->output_surface_queue)         / sizeof(NvencSurface*);
     if (flush)
@@ -1617,6 +1631,11 @@ int ff_nvenc_encode_frame(AVCodecContext *avctx, AVPacket *pkt,
     if (frame) {
         av_fifo_generic_write(ctx->output_surface_queue, &inSurf, sizeof(inSurf), NULL);
         timestamp_queue_enqueue(ctx->timestamp_list, frame->pts);
+
+        if (ctx->initial_pts[0] == AV_NOPTS_VALUE)
+            ctx->initial_pts[0] = frame->pts;
+        else if (ctx->initial_pts[1] == AV_NOPTS_VALUE)
+            ctx->initial_pts[1] = frame->pts;
     }
 
     /* all the pending buffers are now ready for output */
diff --git a/libavcodec/nvenc.h b/libavcodec/nvenc.h
index 7abaf33756..961cbc7e64 100644
--- a/libavcodec/nvenc.h
+++ b/libavcodec/nvenc.h
@@ -157,7 +157,10 @@ typedef struct NvencContext
      * AVCodecContext.pix_fmt when using hwaccel frames on input */
     enum AVPixelFormat data_pix_fmt;
 
-    int64_t last_dts;
+    /* timestamps of the first two frames, for computing the first dts
+     * when B-frames are present */
+    int64_t initial_pts[2];
+    int first_packet_output;
 
     void *nvencoder;