From 239211151e8cf8d3e2a9701e9b54724ee3990d11 Mon Sep 17 00:00:00 2001 From: Yuto Kawamura Date: Mon, 8 Feb 2016 22:48:30 +0900 Subject: [PATCH] Initial import --- .gitignore | 3 + README.md | 35 ++++++- example/config.yml | 14 +++ example/data.json | 20 ++++ gow | 20 ++++ json_exporter.go | 13 +++ jsonexporter/collector.go | 82 ++++++++++++++++ jsonexporter/config.go | 47 +++++++++ jsonexporter/init.go | 80 +++++++++++++++ jsonexporter/scraper.go | 202 ++++++++++++++++++++++++++++++++++++++ jsonexporter/version.go | 3 + 11 files changed, 517 insertions(+), 2 deletions(-) create mode 100644 example/config.yml create mode 100644 example/data.json create mode 100755 gow create mode 100644 json_exporter.go create mode 100644 jsonexporter/collector.go create mode 100644 jsonexporter/config.go create mode 100644 jsonexporter/init.go create mode 100644 jsonexporter/scraper.go create mode 100644 jsonexporter/version.go diff --git a/.gitignore b/.gitignore index daf913b..073cfdf 100644 --- a/.gitignore +++ b/.gitignore @@ -22,3 +22,6 @@ _testmain.go *.exe *.test *.prof + +/build +/json_exporter diff --git a/README.md b/README.md index f5b2f5c..7595374 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,33 @@ -# prometheus-json-exporter -A prometheus exporter which scrapes remote JSON +prometheus-json-exporter +======================== + +A [prometheus](https://prometheus.io/) exporter which scrapes remote JSON by JSONPath. + +Build +===== +```sh +./gow get . +./gow build -o json_exporter . +``` + +Example Usage +============= +```sh +$ python -m SimpleHTTPServer 8000 & +Serving HTTP on 0.0.0.0 port 8000 ... +$ ./json_exporter http://localhost:8000/example/data.json example/config.yml & +INFO[2016-02-08T22:44:38+09:00] metric registered;name: +INFO[2016-02-08T22:44:38+09:00] metric registered;name: +INFO[2016-02-08T22:44:38+09:00] metric registered;name: +127.0.0.1 - - [08/Feb/2016 22:44:38] "GET /example/data.json HTTP/1.1" 200 - +$ curl http://localhost:7979/metrics | grep ^example +example_global_value{environment="beta"} 1234 +example_value_active{environment="beta",id="id-A"} 1 +example_value_active{environment="beta",id="id-C"} 1 +example_value_count{environment="beta",id="id-A"} 1 +example_value_count{environment="beta",id="id-C"} 3 +``` + +See Also +======== +- [nicksardo/jsonpath](https://github.com/nicksardo/jsonpath) : For syntax reference of JSONPath diff --git a/example/config.yml b/example/config.yml new file mode 100644 index 0000000..d4c62db --- /dev/null +++ b/example/config.yml @@ -0,0 +1,14 @@ +- name: example_global_value + path: $.counter + labels: + environment: beta # static label + +- name: example_value + type: object + path: $.values[*]?(@.state == "ACTIVE") + labels: + environment: beta # static label + id: $.id # dynamic label + values: + active: 1 # static value + count: $.count # dynamic value diff --git a/example/data.json b/example/data.json new file mode 100644 index 0000000..45e2b79 --- /dev/null +++ b/example/data.json @@ -0,0 +1,20 @@ +{ + "counter": 1234, + "values": [ + { + "id": "id-A", + "count": 1, + "state": "ACTIVE" + }, + { + "id": "id-B", + "count": 2, + "state": "INACTIVE" + }, + { + "id": "id-C", + "count": 3, + "state": "ACTIVE" + }, + ] +} diff --git a/gow b/gow new file mode 100755 index 0000000..872b7ea --- /dev/null +++ b/gow @@ -0,0 +1,20 @@ +#!/bin/bash +set -e + +GO=go +PACKAGE_LOCATION_GIT_REMOTE=origin +PACKAGE_LOCATION=$(git config --get "remote.$PACKAGE_LOCATION_GIT_REMOTE.url" | sed 's/^[^@]*@//' | sed 's/^.*:\/\///' | sed 's/\.git$//') + +PROJECT_ROOT_DIR=$(cd $(dirname $0); pwd) +GOPATH=$PROJECT_ROOT_DIR/build + +LOCAL_PACKAGE_PATH=$GOPATH/src/$(echo "$PACKAGE_LOCATION" | tr ':' '/') +LOCAL_PACKAGE_DIR=$(dirname $LOCAL_PACKAGE_PATH) + +mkdir -p $LOCAL_PACKAGE_DIR +ln -sfh $PROJECT_ROOT_DIR $LOCAL_PACKAGE_PATH + +export GOBIN="$GOPATH/bin" +export GOPATH + +exec $GO "$@" diff --git a/json_exporter.go b/json_exporter.go new file mode 100644 index 0000000..d85067e --- /dev/null +++ b/json_exporter.go @@ -0,0 +1,13 @@ +package main + +import ( + "github.com/kawamuray/prometheus-exporter-harness/harness" + "github.com/kawamuray/prometheus-json-exporter/jsonexporter" +) + +func main() { + opts := harness.NewExporterOpts("json_exporter", jsonexporter.Version) + opts.Usage = "[OPTIONS] HTTP_ENDPOINT CONFIG_PATH" + opts.Init = jsonexporter.Init + harness.Main(opts) +} diff --git a/jsonexporter/collector.go b/jsonexporter/collector.go new file mode 100644 index 0000000..316f481 --- /dev/null +++ b/jsonexporter/collector.go @@ -0,0 +1,82 @@ +package jsonexporter + +import ( + "fmt" + "github.com/NickSardo/jsonpath" + log "github.com/Sirupsen/logrus" + "github.com/kawamuray/prometheus-exporter-harness/harness" + "io/ioutil" + "net/http" +) + +type Collector struct { + Endpoint string + scrapers []JsonScraper +} + +func compilePath(path string) (*jsonpath.Path, error) { + // All paths in this package is for extracting a value. + // Complete trailing '+' sign if necessary. + if path[len(path)-1] != '+' { + path += "+" + } + + paths, err := jsonpath.ParsePaths(path) + if err != nil { + return nil, err + } + return paths[0], nil +} + +func compilePaths(paths map[string]string) (map[string]*jsonpath.Path, error) { + compiledPaths := make(map[string]*jsonpath.Path) + for name, value := range paths { + if len(value) < 1 || value[0] != '$' { + // Static value + continue + } + compiledPath, err := compilePath(value) + if err != nil { + return nil, fmt.Errorf("failed to parse path;path:<%s>,err:<%s>", value, err) + } + compiledPaths[name] = compiledPath + } + return compiledPaths, nil +} + +func NewCollector(endpoint string, scrapers []JsonScraper) *Collector { + return &Collector{ + Endpoint: endpoint, + scrapers: scrapers, + } +} + +func (col *Collector) fetchJson() ([]byte, error) { + resp, err := http.Get(col.Endpoint) + if err != nil { + return nil, fmt.Errorf("failed to fetch json from endpoint;endpoint:<%s>,err:<%s>", col.Endpoint, err) + } + defer resp.Body.Close() + + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("failed to read response body;err:<%s>", err) + } + + return data, nil +} + +func (col *Collector) Collect(reg *harness.MetricRegistry) { + json, err := col.fetchJson() + if err != nil { + log.Error(err) + return + } + + for _, scraper := range col.scrapers { + if err := scraper.Scrape(json, reg); err != nil { + log.Errorf("error while scraping json;err:<%s>", err) + continue + } + } +} diff --git a/jsonexporter/config.go b/jsonexporter/config.go new file mode 100644 index 0000000..c0a421f --- /dev/null +++ b/jsonexporter/config.go @@ -0,0 +1,47 @@ +package jsonexporter + +import ( + "fmt" + "gopkg.in/yaml.v2" + "io/ioutil" +) + +type Config struct { + Name string `yaml:name` + Path string `yaml:path` + Labels map[string]string `yaml:labels` + Type string `yaml:type` + Help string `yaml:help` + Values map[string]string `yaml:values` +} + +func (config *Config) labelNames() []string { + labelNames := make([]string, 0, len(config.Labels)) + for name := range config.Labels { + labelNames = append(labelNames, name) + } + return labelNames +} + +func loadConfig(configPath string) ([]*Config, error) { + data, err := ioutil.ReadFile(configPath) + if err != nil { + return nil, fmt.Errorf("failed to load config;path:<%s>,err:<%s>", configPath, err) + } + + var configs []*Config + if err := yaml.Unmarshal(data, &configs); err != nil { + return nil, fmt.Errorf("failed to parse yaml;err:<%s>", err) + } + // Complete defaults + for _, config := range configs { + if config.Type == "" { + config.Type = DefaultScrapeType + } + if config.Help == "" { + config.Help = config.Name + } + } + + return configs, nil +} diff --git a/jsonexporter/init.go b/jsonexporter/init.go new file mode 100644 index 0000000..c6590c2 --- /dev/null +++ b/jsonexporter/init.go @@ -0,0 +1,80 @@ +package jsonexporter + +import ( + "fmt" + "github.com/codegangsta/cli" + "github.com/kawamuray/prometheus-exporter-harness/harness" + "github.com/prometheus/client_golang/prometheus" +) + +type ScrapeType struct { + Configure func(*Config, *harness.MetricRegistry) + NewScraper func(*Config) (JsonScraper, error) +} + +var ScrapeTypes = map[string]*ScrapeType{ + "object": { + Configure: func(config *Config, reg *harness.MetricRegistry) { + for subName := range config.Values { + name := harness.MakeMetricName(config.Name, subName) + reg.Register( + name, + prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: name, + Help: config.Help + " - " + subName, + }, config.labelNames()), + ) + } + }, + NewScraper: NewObjectScraper, + }, + "value": { + Configure: func(config *Config, reg *harness.MetricRegistry) { + reg.Register( + config.Name, + prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: config.Name, + Help: config.Help, + }, config.labelNames()), + ) + }, + NewScraper: NewValueScraper, + }, +} + +var DefaultScrapeType = "value" + +func Init(c *cli.Context, reg *harness.MetricRegistry) (harness.Collector, error) { + args := c.Args() + + if len(args) < 2 { + cli.ShowAppHelp(c) + return nil, fmt.Errorf("not enough arguments") + } + + var ( + endpoint = args[0] + configPath = args[1] + ) + + configs, err := loadConfig(configPath) + if err != nil { + return nil, err + } + + scrapers := make([]JsonScraper, len(configs)) + for i, config := range configs { + tpe := ScrapeTypes[config.Type] + if tpe == nil { + return nil, fmt.Errorf("unknown scrape type;type:<%s>", config.Type) + } + tpe.Configure(config, reg) + scraper, err := tpe.NewScraper(config) + if err != nil { + return nil, fmt.Errorf("failed to create scraper;name:<%s>,err:<%s>", config.Name, err) + } + scrapers[i] = scraper + } + + return NewCollector(endpoint, scrapers), nil +} diff --git a/jsonexporter/scraper.go b/jsonexporter/scraper.go new file mode 100644 index 0000000..9b335b1 --- /dev/null +++ b/jsonexporter/scraper.go @@ -0,0 +1,202 @@ +package jsonexporter + +import ( + "fmt" + "github.com/NickSardo/jsonpath" + log "github.com/Sirupsen/logrus" + "github.com/kawamuray/prometheus-exporter-harness/harness" + "github.com/prometheus/client_golang/prometheus" + "strconv" +) + +type JsonScraper interface { + Scrape(data []byte, reg *harness.MetricRegistry) error +} + +type ValueScraper struct { + *Config + valueJsonPath *jsonpath.Path +} + +func NewValueScraper(config *Config) (JsonScraper, error) { + valuepath, err := compilePath(config.Path) + if err != nil { + return nil, fmt.Errorf("failed to parse path;path:<%s>,err:<%s>", config.Path, err) + } + + scraper := &ValueScraper{ + Config: config, + valueJsonPath: valuepath, + } + return scraper, nil +} + +func (vs *ValueScraper) parseValue(bytes []byte) (float64, error) { + value, err := strconv.ParseFloat(string(bytes), 64) + if err != nil { + return -1.0, fmt.Errorf("failed to parse value as float;value:<%s>", bytes) + } + return value, nil +} + +func (vs *ValueScraper) forTargetValue(data []byte, handle func(*jsonpath.Result)) error { + eval, err := jsonpath.EvalPathsInBytes(data, []*jsonpath.Path{vs.valueJsonPath}) + if err != nil { + return fmt.Errorf("failed to eval jsonpath;path:<%s>,json:<%s>", vs.valueJsonPath, data) + } + + for { + result, ok := eval.Next() + if !ok { + break + } + handle(result) + } + return nil +} + +func (vs *ValueScraper) Scrape(data []byte, reg *harness.MetricRegistry) error { + isFirst := true + return vs.forTargetValue(data, func(result *jsonpath.Result) { + if !isFirst { + log.Infof("ignoring non-first value;path:<%s>", vs.valueJsonPath) + return + } + isFirst = false + + if result.Type != jsonpath.JsonNumber { + log.Warnf("skipping not numerical result;path:<%s>,value:<%s>", + vs.valueJsonPath, result.Value) + return + } + + value, err := vs.parseValue(result.Value) + if err != nil { + // Should never happen. + log.Errorf("could not parse numerical value as float;path:<%s>,value:<%s>", + vs.valueJsonPath, result.Value) + return + } + + log.Debugf("metric updated;name:<%s>,labels:<%s>,value:<%.2f>", vs.Name, vs.Labels, value) + reg.Get(vs.Name).(*prometheus.GaugeVec).With(vs.Labels).Set(value) + }) +} + +type ObjectScraper struct { + *ValueScraper + labelJsonPaths map[string]*jsonpath.Path + valueJsonPaths map[string]*jsonpath.Path +} + +func NewObjectScraper(config *Config) (JsonScraper, error) { + valueScraper, err := NewValueScraper(config) + if err != nil { + return nil, err + } + + labelPaths, err := compilePaths(config.Labels) + if err != nil { + return nil, err + } + valuePaths, err := compilePaths(config.Values) + if err != nil { + return nil, err + } + scraper := &ObjectScraper{ + ValueScraper: valueScraper.(*ValueScraper), + labelJsonPaths: labelPaths, + valueJsonPaths: valuePaths, + } + return scraper, nil +} + +func (obsc *ObjectScraper) newLabels() map[string]string { + labels := make(map[string]string) + for name, value := range obsc.Labels { + if _, ok := obsc.labelJsonPaths[name]; !ok { + // Static label value. + labels[name] = value + } + } + return labels +} + +func (obsc *ObjectScraper) extractFirstValue(data []byte, path *jsonpath.Path) (*jsonpath.Result, error) { + eval, err := jsonpath.EvalPathsInBytes(data, []*jsonpath.Path{path}) + if err != nil { + return nil, fmt.Errorf("failed to eval jsonpath;err:<%s>", err) + } + + result, ok := eval.Next() + if !ok { + return nil, fmt.Errorf("no value found for path") + } + return result, nil +} + +func (obsc *ObjectScraper) Scrape(data []byte, reg *harness.MetricRegistry) error { + return obsc.forTargetValue(data, func(result *jsonpath.Result) { + if result.Type != jsonpath.JsonObject && result.Type != jsonpath.JsonArray { + log.Warnf("skipping not structual result;path:<%s>,value:<%s>", + obsc.valueJsonPath, result.Value) + return + } + + labels := obsc.newLabels() + for name, path := range obsc.labelJsonPaths { + firstResult, err := obsc.extractFirstValue(result.Value, path) + if err != nil { + log.Warnf("could not find value for label path;path:<%s>,json:<%s>,err:<%s>", path, result.Value, err) + continue + } + value := firstResult.Value + if firstResult.Type == jsonpath.JsonString { + // Strip quotes + value = value[1 : len(value)-1] + } + labels[name] = string(value) + } + + for name, configValue := range obsc.Values { + var metricValue float64 + path := obsc.valueJsonPaths[name] + + if path == nil { + // Static value + value, err := obsc.parseValue([]byte(configValue)) + if err != nil { + log.Errorf("could not use configured value as float number;name:<%s>,err:<%s>", err) + continue + } + metricValue = value + } else { + // Dynamic value + firstResult, err := obsc.extractFirstValue(result.Value, path) + if err != nil { + log.Warnf("could not find value for value path;path:<%s>,json:<%s>,err:<%s>", path, result.Value, err) + continue + } + + if firstResult.Type != jsonpath.JsonNumber { + log.Warnf("skipping not numerical result;path:<%s>,value:<%s>", + obsc.valueJsonPath, result.Value) + continue + } + + value, err := obsc.parseValue(firstResult.Value) + if err != nil { + // Should never happen. + log.Errorf("could not parse numerical value as float;path:<%s>,value:<%s>", + obsc.valueJsonPath, firstResult.Value) + continue + } + metricValue = value + } + + fqn := harness.MakeMetricName(obsc.Name, name) + log.Debugf("metric updated;name:<%s>,labels:<%s>,value:<%.2f>", fqn, labels, metricValue) + reg.Get(fqn).(*prometheus.GaugeVec).With(labels).Set(metricValue) + } + }) +} diff --git a/jsonexporter/version.go b/jsonexporter/version.go new file mode 100644 index 0000000..e23a780 --- /dev/null +++ b/jsonexporter/version.go @@ -0,0 +1,3 @@ +package jsonexporter + +const Version = "0.0.1"