diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go
index d2a69634aa..06f46f8d72 100644
--- a/cmd/prometheus/main.go
+++ b/cmd/prometheus/main.go
@@ -989,18 +989,12 @@ func main() {
 	listeners, err := webHandler.Listeners()
 	if err != nil {
 		logger.Error("Unable to start web listener", "err", err)
-		if err := queryEngine.Close(); err != nil {
-			logger.Warn("Closing query engine failed", "err", err)
-		}
 		os.Exit(1)
 	}
 
 	err = toolkit_web.Validate(*webConfig)
 	if err != nil {
 		logger.Error("Unable to validate web configuration file", "err", err)
-		if err := queryEngine.Close(); err != nil {
-			logger.Warn("Closing query engine failed", "err", err)
-		}
 		os.Exit(1)
 	}
 
@@ -1022,9 +1016,6 @@ func main() {
 				case <-cancel:
 					reloadReady.Close()
 				}
-				if err := queryEngine.Close(); err != nil {
-					logger.Warn("Closing query engine failed", "err", err)
-				}
 				return nil
 			},
 			func(err error) {
diff --git a/promql/engine.go b/promql/engine.go
index a23e899d04..bbd8410268 100644
--- a/promql/engine.go
+++ b/promql/engine.go
@@ -436,6 +436,8 @@ func NewEngine(opts EngineOpts) *Engine {
 }
 
 // Close closes ng.
+// Callers must ensure the engine is really no longer in use before calling this to avoid
+// issues failures like in https://github.com/prometheus/prometheus/issues/15232
 func (ng *Engine) Close() error {
 	if ng == nil {
 		return nil