From 682f949ab1e15f63daed02182bd9f09cd4914cd8 Mon Sep 17 00:00:00 2001 From: Brian Brazil Date: Wed, 24 Jun 2015 08:07:17 +0100 Subject: [PATCH] Add 'hashmod' relabel action. This takes the modulus of a hash of some labels. Combined with a keep relabel action, this allows for sharding of targets across multiple prometheus servers. --- config/config.go | 24 +++++++++++++++++------- config/config_test.go | 16 ++++++++++++++++ config/testdata/conf.good.yml | 7 +++++++ config/testdata/modulus_missing.bad.yml | 5 +++++ retrieval/relabel.go | 6 ++++++ retrieval/relabel_test.go | 22 ++++++++++++++++++++++ 6 files changed, 73 insertions(+), 7 deletions(-) create mode 100644 config/testdata/modulus_missing.bad.yml diff --git a/config/config.go b/config/config.go index c3494c765..10652e76f 100644 --- a/config/config.go +++ b/config/config.go @@ -455,9 +455,11 @@ const ( // Performs a regex replacement. RelabelReplace RelabelAction = "replace" // Drops targets for which the input does not match the regex. - RelabelKeep = "keep" + RelabelKeep RelabelAction = "keep" // Drops targets for which the input does match the regex. - RelabelDrop = "drop" + RelabelDrop RelabelAction = "drop" + // Sets a label to the modulus of a hash of labels. + RelabelHashMod RelabelAction = "hashmod" ) // UnmarshalYAML implements the yaml.Unmarshaler interface. @@ -467,7 +469,7 @@ func (a *RelabelAction) UnmarshalYAML(unmarshal func(interface{}) error) error { return err } switch act := RelabelAction(strings.ToLower(s)); act { - case RelabelReplace, RelabelKeep, RelabelDrop: + case RelabelReplace, RelabelKeep, RelabelDrop, RelabelHashMod: *a = act return nil } @@ -482,7 +484,9 @@ type RelabelConfig struct { // Separator is the string between concatenated values from the source labels. Separator string `yaml:"separator,omitempty"` // Regex against which the concatenation is matched. - Regex *Regexp `yaml:"regex"` + Regex *Regexp `yaml:"regex",omitempty` + // Modulus to take of the hash of concatenated values from the source labels. + Modulus uint64 `yaml:"modulus,omitempty"` // The label to which the resulting string is written in a replacement. TargetLabel clientmodel.LabelName `yaml:"target_label,omitempty"` // Replacement is the regex replacement pattern to be used. @@ -501,9 +505,12 @@ func (c *RelabelConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { if err := unmarshal((*plain)(c)); err != nil { return err } - if c.Regex == nil { + if c.Regex == nil && c.Action != RelabelHashMod { return fmt.Errorf("relabel configuration requires a regular expression") } + if c.Modulus == 0 && c.Action == RelabelHashMod { + return fmt.Errorf("relabel configuration for hashmod requires non-zero modulus") + } return checkOverflow(c.XXX, "relabel_config") } @@ -527,8 +534,11 @@ func (re *Regexp) UnmarshalYAML(unmarshal func(interface{}) error) error { } // MarshalYAML implements the yaml.Marshaler interface. -func (re Regexp) MarshalYAML() (interface{}, error) { - return re.String(), nil +func (re *Regexp) MarshalYAML() (interface{}, error) { + if re != nil { + return re.String(), nil + } + return nil, nil } // Duration encapsulates a time.Duration and makes it YAML marshallable. diff --git a/config/config_test.go b/config/config_test.go index 8d902c049..e41228d29 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -114,6 +114,19 @@ var expectedConf = &Config{ Separator: ";", Action: RelabelDrop, }, + { + SourceLabels: clientmodel.LabelNames{"__address__"}, + TargetLabel: "__hash", + Modulus: 8, + Separator: ";", + Action: RelabelHashMod, + }, + { + SourceLabels: clientmodel.LabelNames{"__hash"}, + Regex: &Regexp{*regexp.MustCompile("^1$")}, + Separator: ";", + Action: RelabelKeep, + }, }, MetricRelabelConfigs: []*RelabelConfig{ { @@ -194,6 +207,9 @@ var expectedErrors = []struct { }, { filename: "regex_missing.bad.yml", errMsg: "relabel configuration requires a regular expression", + }, { + filename: "modulus_missing.bad.yml", + errMsg: "relabel configuration for hashmod requires non-zero modulus", }, { filename: "rules.bad.yml", errMsg: "invalid rule file path", diff --git a/config/testdata/conf.good.yml b/config/testdata/conf.good.yml index c0fefc1a7..9bbf24f19 100644 --- a/config/testdata/conf.good.yml +++ b/config/testdata/conf.good.yml @@ -71,6 +71,13 @@ scrape_configs: - source_labels: [job] regex: (.*)some-[regex]$ action: drop + - source_labels: [__address__] + modulus: 8 + target_label: __hash + action: hashmod + - source_labels: [__hash] + regex: ^1$ + action: keep metric_relabel_configs: - source_labels: [__name__] diff --git a/config/testdata/modulus_missing.bad.yml b/config/testdata/modulus_missing.bad.yml new file mode 100644 index 000000000..864a57c68 --- /dev/null +++ b/config/testdata/modulus_missing.bad.yml @@ -0,0 +1,5 @@ +scrape_configs: + - job_name: prometheus + relabel_configs: + - regex: abcdef + action: hashmod diff --git a/retrieval/relabel.go b/retrieval/relabel.go index 337a1b67a..0b2785f58 100644 --- a/retrieval/relabel.go +++ b/retrieval/relabel.go @@ -2,6 +2,7 @@ package retrieval import ( "fmt" + "hash/fnv" "strings" clientmodel "github.com/prometheus/client_golang/model" @@ -56,6 +57,11 @@ func relabel(labels clientmodel.LabelSet, cfg *config.RelabelConfig) (clientmode } else { labels[cfg.TargetLabel] = clientmodel.LabelValue(res) } + case config.RelabelHashMod: + hasher := fnv.New64a() + hasher.Write([]byte(val)) + mod := hasher.Sum64() % cfg.Modulus + labels[cfg.TargetLabel] = clientmodel.LabelValue(fmt.Sprintf("%d", mod)) default: panic(fmt.Errorf("retrieval.relabel: unknown relabel action type %q", cfg.Action)) } diff --git a/retrieval/relabel_test.go b/retrieval/relabel_test.go index 071f8f1f9..455067c83 100644 --- a/retrieval/relabel_test.go +++ b/retrieval/relabel_test.go @@ -151,6 +151,28 @@ func TestRelabel(t *testing.T) { "a": "boo", }, }, + { + input: clientmodel.LabelSet{ + "a": "foo", + "b": "bar", + "c": "baz", + }, + relabel: []*config.RelabelConfig{ + { + SourceLabels: clientmodel.LabelNames{"c"}, + TargetLabel: clientmodel.LabelName("d"), + Separator: ";", + Action: config.RelabelHashMod, + Modulus: 1000, + }, + }, + output: clientmodel.LabelSet{ + "a": "foo", + "b": "bar", + "c": "baz", + "d": "58", + }, + }, } for i, test := range tests {