Add HTTP config support

Signed-off-by: rustyclock <rustyclock@protonmail.com>

cmd/main.go

@@ -108,7 +108,7 @@ func probeHandler(w http.ResponseWriter, r *http.Request, logger log.Logger, con
 	registry.MustRegister(probeSuccessGauge)
 	registry.MustRegister(probeDurationGauge)
 
-	data, err := internal.FetchJson(ctx, logger, target, config.Headers)
+	data, err := internal.FetchJson(ctx, logger, target, config)
 	if err != nil {
 		level.Error(logger).Log("msg", "Failed to fetch JSON response", "err", err) //nolint:errcheck
 		duration := time.Since(start).Seconds()

config/config.go

@@ -16,6 +16,7 @@ package config
 import (
 	"io/ioutil"
 
+	pconfig "github.com/prometheus/common/config"
 	"gopkg.in/yaml.v2"
 )
 
@@ -38,13 +39,14 @@ const (
 
 // Config contains metrics and headers defining a configuration
 type Config struct {
-	Headers map[string]string
+	Headers          map[string]string        `yaml:"headers,omitempty"`
-	Metrics []Metric
+	Metrics          []Metric                 `yaml:"metrics"`
-	Global  GlobalConfig
+	Global           GlobalConfig             `yaml:"global_config,omitempty"`
+	HTTPClientConfig pconfig.HTTPClientConfig `yaml:"http_client_config,omitempty"`
 }
 
 type GlobalConfig struct {
-	TimeoutSeconds float64
+	TimeoutSeconds float64 `yaml:"timeout_seconds,omitempty"`
 }
 
 func (metric *Metric) LabelNames() []string {

examples/config.yml

@@ -21,3 +21,16 @@ metrics:
 
 headers:
   X-Dummy: my-test-header
+
+# For full http client config parameters, ref: https://pkg.go.dev/github.com/prometheus/common/config?tab=doc#HTTPClientConfig
+#
+# http_client_config:
+#   tls_config:
+#     insecure_skip_verify: false
+#   basic_auth:
+#     username: myuser
+#     #password: veryverysecret
+#     password_file: /tmp/mysecret.txt
+#
+# global_config:
+#   timeout_seconds: 30 // defaults to 10

go.sum

@@ -60,6 +60,7 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223 h1:F9x/1yl3T2AeKLr2AMdilSD8+f9bvMnNN8VS5iDtovc=
 github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
@@ -86,6 +87,7 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -98,6 +100,7 @@ github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980 h1:dfGZHvZk057jK2MCeWus/TowKpJ8y4AmooUzdBSR9GU=
 golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -109,6 +112,7 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80=
 golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=

internal/util.go

@@ -27,6 +27,7 @@ import (
 	"github.com/kawamuray/jsonpath"
 	"github.com/prometheus-community/json_exporter/config"
 	"github.com/prometheus/client_golang/prometheus"
+	pconfig "github.com/prometheus/common/config"
 )
 
 func MakeMetricName(parts ...string) string {
@@ -106,8 +107,13 @@ func CreateMetricsList(r *prometheus.Registry, c config.Config) ([]JsonGaugeColl
 	return metrics, nil
 }
 
-func FetchJson(ctx context.Context, logger log.Logger, endpoint string, headers map[string]string) ([]byte, error) {
+func FetchJson(ctx context.Context, logger log.Logger, endpoint string, config config.Config) ([]byte, error) {
-	client := &http.Client{}
+	httpClientConfig := config.HTTPClientConfig
+	client, err := pconfig.NewClientFromConfig(httpClientConfig, "fetch_json", true)
+	if err != nil {
+		level.Error(logger).Log("msg", "Error generating HTTP client", "err", err)
+		return nil, err
+	}
 	req, err := http.NewRequest("GET", endpoint, nil)
 	req = req.WithContext(ctx)
 	if err != nil {
@@ -115,7 +121,7 @@ func FetchJson(ctx context.Context, logger log.Logger, endpoint string, headers
 		return nil, err
 	}
 
-	for key, value := range headers {
+	for key, value := range config.Headers {
 		req.Header.Add(key, value)
 	}
 	if req.Header.Get("Accept") == "" {