Log runtime errors during query evaluation instead of panicking.

promql/engine.go

@@ -22,6 +22,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/prometheus/log"
 	"golang.org/x/net/context"
 
 	clientmodel "github.com/prometheus/client_golang/model"
@@ -260,7 +261,7 @@ func contextDone(ctx context.Context, env string) error {
 	}
 }
 
-// Engine handles the liftetime of queries from beginning to end.
+// Engine handles the lifetime of queries from beginning to end.
 // It is connected to a storage.
 type Engine struct {
 	// The storage on which the engine operates.
@@ -554,11 +555,16 @@ func (ev *evaluator) error(err error) {
 func (ev *evaluator) recover(errp *error) {
 	e := recover()
 	if e != nil {
-		// Do not recover from runtime errors.
 		if _, ok := e.(runtime.Error); ok {
-			panic(e)
+			// Print the stack trace but do not inhibit the running application.
+			buf := make([]byte, 64<<10)
+			buf = buf[:runtime.Stack(buf, false)]
+
+			log.Errorf("parser panic: %v\n%s", e, buf)
+			*errp = fmt.Errorf("unexpected error")
+		} else {
+			*errp = e.(error)
 		}
-		*errp = e.(error)
 	}
 }
 

promql/engine_test.go

@@ -1,6 +1,7 @@
 package promql
 
 import (
+	"fmt"
 	"testing"
 	"time"
 
@@ -179,3 +180,30 @@ func TestEngineShutdown(t *testing.T) {
 		t.Fatalf("expected cancelation error, got %q", res2.Err)
 	}
 }
+
+func TestRecoverEvaluatorRuntime(t *testing.T) {
+	var ev *evaluator
+	var err error
+	defer ev.recover(&err)
+
+	// Cause a runtime panic.
+	var a []int
+	a[123] = 1
+
+	if err.Error() != "unexpected error" {
+		t.Fatalf("wrong error message: %q, expected %q", err, "unexpected error")
+	}
+}
+
+func TestRecoverEvaluatorError(t *testing.T) {
+	var ev *evaluator
+	var err error
+	defer ev.recover(&err)
+
+	e := fmt.Errorf("custom error")
+	panic(e)
+
+	if err.Error() != e.Error() {
+		t.Fatalf("wrong error message: %q, expected %q", err, e)
+	}
+}

promql/parse_test.go

@@ -1428,7 +1428,7 @@ func TestParseSeries(t *testing.T) {
 	}
 }
 
-func TestRecoverRuntime(t *testing.T) {
+func TestRecoverParserRuntime(t *testing.T) {
 	var p *parser
 	var err error
 	defer p.recover(&err)
@@ -1442,7 +1442,7 @@ func TestRecoverRuntime(t *testing.T) {
 	}
 }
 
-func TestRecoverError(t *testing.T) {
+func TestRecoverParserError(t *testing.T) {
 	var p *parser
 	var err error
 	defer p.recover(&err)