diff --git a/go.mod b/go.mod
index 01d1c45b..0009f113 100644
--- a/go.mod
+++ b/go.mod
@@ -5,7 +5,7 @@ go 1.15
 require (
 	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
 	github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d // indirect
-	github.com/aler9/gortsplib v0.0.0-20210513074841-4ef801b9e015
+	github.com/aler9/gortsplib v0.0.0-20210514205337-76fab391dc85
 	github.com/asticode/go-astits v0.0.0-00010101000000-000000000000
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/fsnotify/fsnotify v1.4.9
diff --git a/go.sum b/go.sum
index 920b3b24..987a678b 100644
--- a/go.sum
+++ b/go.sum
@@ -4,8 +4,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2c
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
 github.com/aler9/go-astits v0.0.0-20210423195926-582b09ed7c04 h1:CXgQLsU4uxWAmsXNOjGLbj0A+0IlRcpZpMgI13fmVwo=
 github.com/aler9/go-astits v0.0.0-20210423195926-582b09ed7c04/go.mod h1:DkOWmBNQpnr9mv24KfZjq4JawCFX1FCqjLVGvO0DygQ=
-github.com/aler9/gortsplib v0.0.0-20210513074841-4ef801b9e015 h1:IivP6Fx5QipZ2fB7we4y78qJRSuHu+JLMvZyVMOviqE=
-github.com/aler9/gortsplib v0.0.0-20210513074841-4ef801b9e015/go.mod h1:zVCg+TQX445hh1pC5QgAuuBvvXZMWLY1XYz626dGFqY=
+github.com/aler9/gortsplib v0.0.0-20210514205337-76fab391dc85 h1:OFulQt+GaeWy+mjPspkh4OBiGpGPVK4JPWVU12wqsQ8=
+github.com/aler9/gortsplib v0.0.0-20210514205337-76fab391dc85/go.mod h1:zVCg+TQX445hh1pC5QgAuuBvvXZMWLY1XYz626dGFqY=
 github.com/aler9/rtmp v0.0.0-20210403095203-3be4a5535927 h1:95mXJ5fUCYpBRdSOnLAQAdJHHKxxxJrVCiaqDi965YQ=
 github.com/aler9/rtmp v0.0.0-20210403095203-3be4a5535927/go.mod h1:vzuE21rowz+lT1NGsWbreIvYulgBpCGnQyeTyFblUHc=
 github.com/asticode/go-astikit v0.20.0 h1:+7N+J4E4lWx2QOkRdOf6DafWJMv6O4RRfgClwQokrH8=
diff --git a/internal/hlsconverter/converter.go b/internal/hlsconverter/converter.go
index 05c85ef7..7e2f8eeb 100644
--- a/internal/hlsconverter/converter.go
+++ b/internal/hlsconverter/converter.go
@@ -17,6 +17,7 @@ import (
 	"github.com/aler9/gortsplib/pkg/ringbuffer"
 	"github.com/aler9/gortsplib/pkg/rtpaac"
 	"github.com/aler9/gortsplib/pkg/rtph264"
+	"github.com/pion/rtp"
 
 	"github.com/aler9/rtsp-simple-server/internal/h264"
 	"github.com/aler9/rtsp-simple-server/internal/logger"
@@ -337,6 +338,7 @@ func (c *Converter) runInner(innerCtx context.Context) error {
 			startPCR := time.Now()
 			var videoBuf [][]byte
 			videoDTSEst := h264.NewDTSEstimator()
+			videoInitialized := false
 			audioAUCount := 0
 
 			for {
@@ -347,7 +349,26 @@ func (c *Converter) runInner(innerCtx context.Context) error {
 				pair := data.(trackIDPayloadPair)
 
 				if videoTrack != nil && pair.trackID == videoTrack.ID {
-					nalus, pts, err := h264Decoder.Decode(pair.buf)
+					var pkt rtp.Packet
+					err := pkt.Unmarshal(pair.buf)
+					if err != nil {
+						c.log(logger.Warn, "unable to decode RTP packet: %v", err)
+						continue
+					}
+
+					// skip packets that are part of frames sent before
+					// the initialization of the converter
+					if !videoInitialized {
+						typ := pkt.Payload[0] & 0x1F
+						start := pkt.Payload[1] >> 7
+						if typ == 28 && start != 1 { // FU-A
+							continue
+						}
+
+						videoInitialized = true
+					}
+
+					nalus, pts, err := h264Decoder.DecodeRTP(&pkt)
 					if err != nil {
 						if err != rtph264.ErrMorePacketsNeeded {
 							c.log(logger.Warn, "unable to decode video track: %v", err)