diff --git a/libavfilter/af_silencedetect.c b/libavfilter/af_silencedetect.c
index c53b9d9218..b4184c2486 100644
--- a/libavfilter/af_silencedetect.c
+++ b/libavfilter/af_silencedetect.c
@@ -39,7 +39,7 @@ typedef struct SilenceDetectContext {
     int mono;                   ///< mono mode : check each channel separately (default = check when ALL channels are silent)
     int independant_channels;   ///< number of entries in following arrays (always 1 in mono mode)
     int64_t *nb_null_samples;   ///< (array) current number of continuous zero samples
-    int64_t *start;             ///< (array) if silence is detected, this value contains the time of the first zero sample
+    int64_t *start;             ///< (array) if silence is detected, this value contains the time of the first zero sample (default/unset = INT64_MIN)
     int last_sample_rate;       ///< last sample rate to check for sample rate changes
 
     void (*silencedetect)(struct SilenceDetectContext *s, AVFrame *insamples,
@@ -76,7 +76,7 @@ static av_always_inline void update(SilenceDetectContext *s, AVFrame *insamples,
 {
     int channel = current_sample % s->independant_channels;
     if (is_silence) {
-        if (!s->start[channel]) {
+        if (s->start[channel] == INT64_MIN) {
             s->nb_null_samples[channel]++;
             if (s->nb_null_samples[channel] >= nb_samples_notify) {
                 s->start[channel] = insamples->pts - (int64_t)(s->duration / av_q2d(time_base) + .5);
@@ -89,7 +89,7 @@ static av_always_inline void update(SilenceDetectContext *s, AVFrame *insamples,
             }
         }
     } else {
-        if (s->start[channel]) {
+        if (s->start[channel] > INT64_MIN) {
             int64_t end_pts = insamples->pts;
             int64_t duration_ts = end_pts - s->start[channel];
             set_meta(insamples, s->mono ? channel + 1 : 0, "silence_end",
@@ -102,7 +102,8 @@ static av_always_inline void update(SilenceDetectContext *s, AVFrame *insamples,
                     av_ts2timestr(end_pts, &time_base),
                     av_ts2timestr(duration_ts, &time_base));
         }
-        s->nb_null_samples[channel] = s->start[channel] = 0;
+        s->nb_null_samples[channel] = 0;
+        s->start[channel] = INT64_MIN;
     }
 }
 
@@ -129,14 +130,17 @@ static int config_input(AVFilterLink *inlink)
 {
     AVFilterContext *ctx = inlink->dst;
     SilenceDetectContext *s = ctx->priv;
+    int c;
 
     s->independant_channels = s->mono ? inlink->channels : 1;
     s->nb_null_samples = av_mallocz_array(sizeof(*s->nb_null_samples), s->independant_channels);
     if (!s->nb_null_samples)
         return AVERROR(ENOMEM);
-    s->start = av_mallocz_array(sizeof(*s->start), s->independant_channels);
+    s->start = av_malloc_array(sizeof(*s->start), s->independant_channels);
     if (!s->start)
         return AVERROR(ENOMEM);
+    for (c = 0; c < s->independant_channels; c++)
+        s->start[c] = INT64_MIN;
 
     switch (inlink->format) {
     case AV_SAMPLE_FMT_DBL: s->silencedetect = silencedetect_dbl; break;