Support modules configuration (#146)

* support modules configuration

Signed-off-by: Ben Ye <ben.ye@bytedance.com>

* fallback default module if the param is missing

Signed-off-by: Ben Ye <ben.ye@bytedance.com>

* update readme and example config file

Signed-off-by: Ben Ye <ben.ye@bytedance.com>

* fix lint

Signed-off-by: Ben Ye <ben.ye@bytedance.com>

README.md

@@ -39,35 +39,37 @@ $ cat examples/data.json
 
 $ cat examples/config.yml
 ---
-metrics:
+modules:
-- name: example_global_value
+  default:
-  path: "{ .counter }"
+    metrics:
-  help: Example of a top-level global value scrape in the json
+    - name: example_global_value
-  labels:
+      path: "{ .counter }"
-    environment: beta # static label
+      help: Example of a top-level global value scrape in the json
-    location: "planet-{.location}"          # dynamic label
+      labels:
+        environment: beta # static label
+        location: "planet-{.location}"          # dynamic label
 
-- name: example_value
+    - name: example_value
-  type: object
+      type: object
-  help: Example of sub-level value scrapes from a json
+      help: Example of sub-level value scrapes from a json
-  path: '{.values[?(@.state == "ACTIVE")]}'
+      path: '{.values[?(@.state == "ACTIVE")]}'
-  labels:
+      labels:
-    environment: beta # static label
+        environment: beta # static label
-    id: '{.id}'          # dynamic label
+        id: '{.id}'          # dynamic label
-  values:
+      values:
-    active: 1      # static value
+        active: 1      # static value
-    count: '{.count}' # dynamic value
+        count: '{.count}' # dynamic value
-    boolean: '{.some_boolean}'
+        boolean: '{.some_boolean}'
 
-headers:
+    headers:
-  X-Dummy: my-test-header
+      X-Dummy: my-test-header
 
 $ python -m SimpleHTTPServer 8000 &
 Serving HTTP on 0.0.0.0 port 8000 ...
 
 $ ./json_exporter --config.file examples/config.yml &
 
-$ curl "http://localhost:7979/probe?target=http://localhost:8000/examples/data.json" | grep ^example
+$ curl "http://localhost:7979/probe?module=default&target=http://localhost:8000/examples/data.json" | grep ^example
 example_global_value{environment="beta",location="planet-mars"} 1234
 example_value_active{environment="beta",id="id-A"} 1
 example_value_active{environment="beta",id="id-C"} 1

cmd/main.go

@@ -16,6 +16,7 @@ package cmd
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"os"
 
@@ -86,9 +87,19 @@ func probeHandler(w http.ResponseWriter, r *http.Request, logger log.Logger, con
 	defer cancel()
 	r = r.WithContext(ctx)
 
+	module := r.URL.Query().Get("module")
+	if module == "" {
+		module = "default"
+	}
+	if _, ok := config.Modules[module]; !ok {
+		http.Error(w, fmt.Sprintf("Unknown module %q", module), http.StatusBadRequest)
+		level.Debug(logger).Log("msg", "Unknown module", "module", module)
+		return
+	}
+
 	registry := prometheus.NewPedanticRegistry()
 
-	metrics, err := exporter.CreateMetricsList(config)
+	metrics, err := exporter.CreateMetricsList(config.Modules[module])
 	if err != nil {
 		level.Error(logger).Log("msg", "Failed to create metrics list from config", "err", err)
 	}
@@ -102,7 +113,7 @@ func probeHandler(w http.ResponseWriter, r *http.Request, logger log.Logger, con
 		return
 	}
 
-	fetcher := exporter.NewJSONFetcher(ctx, logger, config, r.URL.Query())
+	fetcher := exporter.NewJSONFetcher(ctx, logger, config.Modules[module], r.URL.Query())
 	data, err := fetcher.FetchJSON(target)
 	if err != nil {
 		http.Error(w, "Failed to fetch JSON response. TARGET: "+target+", ERROR: "+err.Error(), http.StatusServiceUnavailable)

cmd/main_test.go

@@ -32,9 +32,9 @@ func TestFailIfSelfSignedCA(t *testing.T) {
 	}))
 	defer target.Close()
 
-	req := httptest.NewRequest("GET", "http://example.com/foo"+"?target="+target.URL, nil)
+	req := httptest.NewRequest("GET", "http://example.com/foo"+"?module=default&target="+target.URL, nil)
 	recorder := httptest.NewRecorder()
-	probeHandler(recorder, req, log.NewNopLogger(), config.Config{})
+	probeHandler(recorder, req, log.NewNopLogger(), config.Config{Modules: map[string]config.Module{"default": {}}})
 
 	resp := recorder.Result()
 	body, _ := ioutil.ReadAll(resp.Body)
@@ -45,13 +45,21 @@ func TestFailIfSelfSignedCA(t *testing.T) {
 }
 
 func TestSucceedIfSelfSignedCA(t *testing.T) {
-	c := config.Config{}
+	c := config.Config{
-	c.HTTPClientConfig.TLSConfig.InsecureSkipVerify = true
+		Modules: map[string]config.Module{
+			"default": {
+				HTTPClientConfig: pconfig.HTTPClientConfig{
+					TLSConfig: pconfig.TLSConfig{
+						InsecureSkipVerify: true,
+					},
+				},
+			}},
+	}
 	target := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 	}))
 	defer target.Close()
 
-	req := httptest.NewRequest("GET", "http://example.com/foo"+"?target="+target.URL, nil)
+	req := httptest.NewRequest("GET", "http://example.com/foo"+"?module=default&target="+target.URL, nil)
 	recorder := httptest.NewRecorder()
 	probeHandler(recorder, req, log.NewNopLogger(), c)
 
@@ -63,6 +71,29 @@ func TestSucceedIfSelfSignedCA(t *testing.T) {
 	}
 }
 
+func TestDefaultModule(t *testing.T) {
+	target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	}))
+	defer target.Close()
+
+	req := httptest.NewRequest("GET", "http://example.com/foo"+"?target="+target.URL, nil)
+	recorder := httptest.NewRecorder()
+	probeHandler(recorder, req, log.NewNopLogger(), config.Config{Modules: map[string]config.Module{"default": {}}})
+
+	resp := recorder.Result()
+	if resp.StatusCode != http.StatusOK {
+		t.Fatalf("Default module test fails unexpectedly, expected 200, got %d", resp.StatusCode)
+	}
+
+	// Module doesn't exist.
+	recorder = httptest.NewRecorder()
+	probeHandler(recorder, req, log.NewNopLogger(), config.Config{Modules: map[string]config.Module{"foo": {}}})
+	resp = recorder.Result()
+	if resp.StatusCode != http.StatusBadRequest {
+		t.Fatalf("Default module test fails unexpectedly, expected 400, got %d", resp.StatusCode)
+	}
+}
+
 func TestFailIfTargetMissing(t *testing.T) {
 	req := httptest.NewRequest("GET", "http://example.com/foo", nil)
 	recorder := httptest.NewRecorder()
@@ -86,9 +117,9 @@ func TestDefaultAcceptHeader(t *testing.T) {
 	}))
 	defer target.Close()
 
-	req := httptest.NewRequest("GET", "http://example.com/foo"+"?target="+target.URL, nil)
+	req := httptest.NewRequest("GET", "http://example.com/foo"+"?module=default&target="+target.URL, nil)
 	recorder := httptest.NewRecorder()
-	probeHandler(recorder, req, log.NewNopLogger(), config.Config{})
+	probeHandler(recorder, req, log.NewNopLogger(), config.Config{Modules: map[string]config.Module{"default": {}}})
 
 	resp := recorder.Result()
 	body, _ := ioutil.ReadAll(resp.Body)
@@ -118,7 +149,7 @@ func TestCorrectResponse(t *testing.T) {
 			t.Fatalf("Failed to load config file %s", test.ConfigFile)
 		}
 
-		req := httptest.NewRequest("GET", "http://example.com/foo"+"?target="+target.URL+test.ServeFile, nil)
+		req := httptest.NewRequest("GET", "http://example.com/foo"+"?module=default&target="+target.URL+test.ServeFile, nil)
 		recorder := httptest.NewRecorder()
 		probeHandler(recorder, req, log.NewNopLogger(), c)
 
@@ -145,15 +176,21 @@ func TestBasicAuth(t *testing.T) {
 	}))
 	defer target.Close()
 
-	req := httptest.NewRequest("GET", "http://example.com/foo"+"?target="+target.URL, nil)
+	req := httptest.NewRequest("GET", "http://example.com/foo"+"?module=default&target="+target.URL, nil)
 	recorder := httptest.NewRecorder()
-	c := config.Config{}
+	c := config.Config{
-	auth := &pconfig.BasicAuth{
+		Modules: map[string]config.Module{
-		Username: username,
+			"default": {
-		Password: pconfig.Secret(password),
+				HTTPClientConfig: pconfig.HTTPClientConfig{
+					BasicAuth: &pconfig.BasicAuth{
+						Username: username,
+						Password: pconfig.Secret(password),
+					},
+				},
+			},
+		},
 	}
 
-	c.HTTPClientConfig.BasicAuth = auth
 	probeHandler(recorder, req, log.NewNopLogger(), c)
 
 	resp := recorder.Result()
@@ -175,11 +212,16 @@ func TestBearerToken(t *testing.T) {
 	}))
 	defer target.Close()
 
-	req := httptest.NewRequest("GET", "http://example.com/foo"+"?target="+target.URL, nil)
+	req := httptest.NewRequest("GET", "http://example.com/foo"+"?module=default&target="+target.URL, nil)
 	recorder := httptest.NewRecorder()
-	c := config.Config{}
+	c := config.Config{
+		Modules: map[string]config.Module{"default": {
+			HTTPClientConfig: pconfig.HTTPClientConfig{
+				BearerToken: pconfig.Secret(token),
+			},
+		}},
+	}
 
-	c.HTTPClientConfig.BearerToken = pconfig.Secret(token)
 	probeHandler(recorder, req, log.NewNopLogger(), c)
 
 	resp := recorder.Result()
@@ -206,10 +248,15 @@ func TestHTTPHeaders(t *testing.T) {
 	}))
 	defer target.Close()
 
-	req := httptest.NewRequest("GET", "http://example.com/foo"+"?target="+target.URL, nil)
+	req := httptest.NewRequest("GET", "http://example.com/foo"+"?module=default&target="+target.URL, nil)
 	recorder := httptest.NewRecorder()
-	c := config.Config{}
+	c := config.Config{
-	c.Headers = headers
+		Modules: map[string]config.Module{
+			"default": {
+				Headers: headers,
+			},
+		},
+	}
 
 	probeHandler(recorder, req, log.NewNopLogger(), c)
 
@@ -264,9 +311,15 @@ func TestBodyPostTemplate(t *testing.T) {
 			w.WriteHeader(http.StatusOK)
 		}))
 
-		req := httptest.NewRequest("POST", "http://example.com/foo"+"?target="+target.URL, strings.NewReader(test.Body.Content))
+		req := httptest.NewRequest("POST", "http://example.com/foo"+"?module=default&target="+target.URL, strings.NewReader(test.Body.Content))
 		recorder := httptest.NewRecorder()
-		c := config.Config{Body: test.Body}
+		c := config.Config{
+			Modules: map[string]config.Module{
+				"default": {
+					Body: test.Body,
+				},
+			},
+		}
 
 		probeHandler(recorder, req, log.NewNopLogger(), c)
 
@@ -351,7 +404,7 @@ func TestBodyPostQuery(t *testing.T) {
 			w.WriteHeader(http.StatusOK)
 		}))
 
-		req := httptest.NewRequest("POST", "http://example.com/foo"+"?target="+target.URL, strings.NewReader(test.Body.Content))
+		req := httptest.NewRequest("POST", "http://example.com/foo"+"?module=default&target="+target.URL, strings.NewReader(test.Body.Content))
 		q := req.URL.Query()
 		for k, v := range test.QueryParams {
 			q.Add(k, v)
@@ -359,7 +412,13 @@ func TestBodyPostQuery(t *testing.T) {
 		req.URL.RawQuery = q.Encode()
 
 		recorder := httptest.NewRecorder()
-		c := config.Config{Body: test.Body}
+		c := config.Config{
+			Modules: map[string]config.Module{
+				"default": {
+					Body: test.Body,
+				},
+			},
+		}
 
 		probeHandler(recorder, req, log.NewNopLogger(), c)
 

config/config.go

@@ -46,8 +46,13 @@ const (
 	ValueTypeUntyped ValueType = "untyped"
 )
 
-// Config contains metrics and headers defining a configuration
+// Config contains multiple modules.
 type Config struct {
+	Modules map[string]Module `yaml:"modules"`
+}
+
+// Module contains metrics and headers defining a configuration
+type Module struct {
 	Headers          map[string]string        `yaml:"headers,omitempty"`
 	Metrics          []Metric                 `yaml:"metrics"`
 	HTTPClientConfig pconfig.HTTPClientConfig `yaml:"http_client_config,omitempty"`
@@ -71,15 +76,17 @@ func LoadConfig(configPath string) (Config, error) {
 	}
 
 	// Complete Defaults
-	for i := 0; i < len(config.Metrics); i++ {
+	for _, module := range config.Modules {
-		if config.Metrics[i].Type == "" {
+		for i := 0; i < len(module.Metrics); i++ {
-			config.Metrics[i].Type = ValueScrape
+			if module.Metrics[i].Type == "" {
-		}
+				module.Metrics[i].Type = ValueScrape
-		if config.Metrics[i].Help == "" {
+			}
-			config.Metrics[i].Help = config.Metrics[i].Name
+			if module.Metrics[i].Help == "" {
-		}
+				module.Metrics[i].Help = module.Metrics[i].Name
-		if config.Metrics[i].ValueType == "" {
+			}
-			config.Metrics[i].ValueType = ValueTypeUntyped
+			if module.Metrics[i].ValueType == "" {
+				module.Metrics[i].ValueType = ValueTypeUntyped
+			}
 		}
 	}
 

examples/config.yml

@@ -1,44 +1,45 @@
 ---
-metrics:
+modules:
-- name: example_global_value
+  default:
-  path: "{ .counter }"
+    metrics:
-  help: Example of a top-level global value scrape in the json
+    - name: example_global_value
-  labels:
+      path: "{ .counter }"
-    environment: beta # static label
+      help: Example of a top-level global value scrape in the json
-    location: "planet-{.location}"          # dynamic label
+      labels:
+        environment: beta # static label
+        location: "planet-{.location}"          # dynamic label
 
-- name: example_value
+    - name: example_value
-  type: object
+      type: object
-  help: Example of sub-level value scrapes from a json
+      help: Example of sub-level value scrapes from a json
-  path: '{.values[?(@.state == "ACTIVE")]}'
+      path: '{.values[?(@.state == "ACTIVE")]}'
-  labels:
+      labels:
-    environment: beta # static label
+        environment: beta # static label
-    id: '{.id}'          # dynamic label
+        id: '{.id}'          # dynamic label
-  values:
+      values:
-    active: 1      # static value
+        active: 1      # static value
-    count: '{.count}' # dynamic value
+        count: '{.count}' # dynamic value
-    boolean: '{.some_boolean}'
+        boolean: '{.some_boolean}'
+    headers:
+      X-Dummy: my-test-header
 
-headers:
+    # If 'body' is set, it will be sent by the exporter as the body content in the scrape request. The HTTP method will also be set as 'POST' in this case.
-  X-Dummy: my-test-header
+    # body:
+    #   content: |
+    #     {"time_diff": "1m25s", "anotherVar": "some value"}
 
-# If 'body' is set, it will be sent by the exporter as the body content in the scrape request. The HTTP method will also be set as 'POST' in this case.
+    # The body content can also be a Go Template (https://golang.org/pkg/text/template), with all the functions from the Sprig library (https://masterminds.github.io/sprig/) available. All the query parameters sent by prometheus in the scrape query to the exporter, are available in the template.
-# body:
+    # body:
-#   content: |
+    #   content: |
-#     {"time_diff": "1m25s", "anotherVar": "some value"}
+    #     {"time_diff": "{{ duration `95` }}","anotherVar": "{{ .myVal | first }}"}
+    #   templatize: true
 
-# The body content can also be a Go Template (https://golang.org/pkg/text/template), with all the functions from the Sprig library (https://masterminds.github.io/sprig/) available. All the query parameters sent by prometheus in the scrape query to the exporter, are available in the template.
+    # For full http client config parameters, ref: https://pkg.go.dev/github.com/prometheus/common/config?tab=doc#HTTPClientConfig
-# body:
+    #
-#   content: |
+    # http_client_config:
-#     {"time_diff": "{{ duration `95` }}","anotherVar": "{{ .myVal | first }}"}
+    #   tls_config:
-#   templatize: true
+    #     insecure_skip_verify: true
-
+    #   basic_auth:
-# For full http client config parameters, ref: https://pkg.go.dev/github.com/prometheus/common/config?tab=doc#HTTPClientConfig
+    #     username: myuser
-#
+    #     #password: veryverysecret
-# http_client_config:
+    #     password_file: /tmp/mysecret.txt
-#   tls_config:
-#     insecure_skip_verify: true
-#   basic_auth:
-#     username: myuser
-#     #password: veryverysecret
-#     password_file: /tmp/mysecret.txt

examples/prometheus.yml

@@ -15,6 +15,8 @@ scrape_configs:
   ## gather the metrics from third party json sources, via the json exporter
 - job_name: json
   metrics_path: /probe
+  params:
+    module: [default]
   static_configs:
     - targets:
       - http://host-1.foobar.com/dummy/data.json

exporter/util.go

@@ -62,7 +62,7 @@ func SanitizeValue(s string) (float64, error) {
 	return value, fmt.Errorf(resultErr)
 }
 
-func CreateMetricsList(c config.Config) ([]JSONMetric, error) {
+func CreateMetricsList(c config.Module) ([]JSONMetric, error) {
 	var (
 		metrics   []JSONMetric
 		valueType prometheus.ValueType
@@ -127,17 +127,17 @@ func CreateMetricsList(c config.Config) ([]JSONMetric, error) {
 }
 
 type JSONFetcher struct {
-	config config.Config
+	module config.Module
 	ctx    context.Context
 	logger log.Logger
 	method string
 	body   io.Reader
 }
 
-func NewJSONFetcher(ctx context.Context, logger log.Logger, c config.Config, tplValues url.Values) *JSONFetcher {
+func NewJSONFetcher(ctx context.Context, logger log.Logger, m config.Module, tplValues url.Values) *JSONFetcher {
-	method, body := renderBody(logger, c.Body, tplValues)
+	method, body := renderBody(logger, m.Body, tplValues)
 	return &JSONFetcher{
-		config: c,
+		module: m,
 		ctx:    ctx,
 		logger: logger,
 		method: method,
@@ -146,7 +146,7 @@ func NewJSONFetcher(ctx context.Context, logger log.Logger, c config.Config, tpl
 }
 
 func (f *JSONFetcher) FetchJSON(endpoint string) ([]byte, error) {
-	httpClientConfig := f.config.HTTPClientConfig
+	httpClientConfig := f.module.HTTPClientConfig
 	client, err := pconfig.NewClientFromConfig(httpClientConfig, "fetch_json", pconfig.WithKeepAlivesDisabled(), pconfig.WithHTTP2Disabled())
 	if err != nil {
 		level.Error(f.logger).Log("msg", "Error generating HTTP client", "err", err)
@@ -161,7 +161,7 @@ func (f *JSONFetcher) FetchJSON(endpoint string) ([]byte, error) {
 		return nil, err
 	}
 
-	for key, value := range f.config.Headers {
+	for key, value := range f.module.Headers {
 		req.Header.Add(key, value)
 	}
 	if req.Header.Get("Accept") == "" {

test/config/good.yml

@@ -1,21 +1,22 @@
 ---
-metrics:
+modules:
-- name: example_global_value
+  default:
-  path: "{ .counter }"
+    metrics:
-  help: Example of a top-level global value scrape in the json
+    - name: example_global_value
-  labels:
+      path: "{ .counter }"
-    environment: beta # static label
+      help: Example of a top-level global value scrape in the json
-    location: "planet-{.location}"          # dynamic label
+      labels:
-
+        environment: beta # static label
-- name: example_value
+        location: "planet-{.location}"          # dynamic label
-  type: object
-  help: Example of sub-level value scrapes from a json
-  path: '{.values[?(@.state == "ACTIVE")]}'
-  labels:
-    environment: beta # static label
-    id: '{.id}'          # dynamic label
-  values:
-    active: 1      # static value
-    count: '{.count}' # dynamic value
-    boolean: '{.some_boolean}'
 
+    - name: example_value
+      type: object
+      help: Example of sub-level value scrapes from a json
+      path: '{.values[?(@.state == "ACTIVE")]}'
+      labels:
+        environment: beta # static label
+        id: '{.id}'          # dynamic label
+      values:
+        active: 1      # static value
+        count: '{.count}' # dynamic value
+        boolean: '{.some_boolean}'