Adding new metric type to track in-flight queries via the remote read API endpoint. (#4699)

* Adding new metric type to track in-flight queries via the remote read API endpoint.

Signed-off-by: tariqibrahim <tariq.ibrahim@microsoft.com>

* fix review comments

Signed-off-by: tariqibrahim <tariq.ibrahim@microsoft.com>

* fix comments

Signed-off-by: tariqibrahim <tariq.ibrahim@microsoft.com>

web/api/v1/api.go

@@ -31,6 +31,7 @@ import (
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
 	jsoniter "github.com/json-iterator/go"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/common/route"
 	"github.com/prometheus/tsdb"
@@ -51,6 +52,11 @@ import (
 	tsdbLabels "github.com/prometheus/tsdb/labels"
 )
 
+const (
+	namespace = "prometheus"
+	subsystem = "api"
+)
+
 type status string
 
 const (
@@ -78,6 +84,13 @@ var corsHeaders = map[string]string{
 	"Access-Control-Expose-Headers": "Date",
 }
 
+var remoteReadQueries = prometheus.NewGauge(prometheus.GaugeOpts{
+	Namespace: namespace,
+	Subsystem: subsystem,
+	Name:      "remote_read_queries",
+	Help:      "The current number of remote read queries being executed or waiting.",
+})
+
 type apiError struct {
 	typ errorType
 	err error
@@ -139,6 +152,11 @@ type API struct {
 	remoteReadGate        *gate.Gate
 }
 
+func init() {
+	jsoniter.RegisterTypeEncoderFunc("promql.Point", marshalPointJSON, marshalPointJSONIsEmpty)
+	prometheus.MustRegister(remoteReadQueries)
+}
+
 // NewAPI returns an initialized API type.
 func NewAPI(
 	qe *promql.Engine,
@@ -762,7 +780,10 @@ func (api *API) serveFlags(r *http.Request) (interface{}, *apiError, func()) {
 
 func (api *API) remoteRead(w http.ResponseWriter, r *http.Request) {
 	api.remoteReadGate.Start(r.Context())
+	remoteReadQueries.Inc()
+
 	defer api.remoteReadGate.Done()
+	defer remoteReadQueries.Dec()
 
 	req, err := remote.DecodeReadRequest(r)
 	if err != nil {
@@ -1083,10 +1104,6 @@ func parseDuration(s string) (time.Duration, error) {
 	return 0, fmt.Errorf("cannot parse %q to a valid duration", s)
 }
 
-func init() {
-	jsoniter.RegisterTypeEncoderFunc("promql.Point", marshalPointJSON, marshalPointJSONIsEmpty)
-}
-
 func marshalPointJSON(ptr unsafe.Pointer, stream *jsoniter.Stream) {
 	p := *((*promql.Point)(ptr))
 	stream.WriteArrayStart()