Add relabelling to target management.

This commit adds a relabelling stage on the set of base
labels from which a target is created. It allows to drop
targets and rewrite any regular or internal label.

config/config.go

@@ -14,6 +14,7 @@
 package config
 
 import (
+	"errors"
 	"fmt"
 	"regexp"
 	"strings"
@@ -190,6 +191,11 @@ func (c *ScrapeConfig) Validate() error {
 			return fmt.Errorf("invalid DNS config: %s", err)
 		}
 	}
+	for _, rlcfg := range c.RelabelConfigs() {
+		if err := rlcfg.Validate(); err != nil {
+			return fmt.Errorf("invalid relabelling config: %s", err)
+		}
+	}
 	return nil
 }
 
@@ -203,6 +209,15 @@ func (c *ScrapeConfig) DNSConfigs() []*DNSConfig {
 	return dnscfgs
 }
 
+// RelabelConfigs returns the relabel configs of the scrape config.
+func (c *ScrapeConfig) RelabelConfigs() []*RelabelConfig {
+	var rlcfgs []*RelabelConfig
+	for _, rc := range c.GetRelabelConfig() {
+		rlcfgs = append(rlcfgs, &RelabelConfig{*rc})
+	}
+	return rlcfgs
+}
+
 // DNSConfig encapsulates the protobuf configuration object for DNS based
 // service discovery.
 type DNSConfig struct {
@@ -222,6 +237,17 @@ func (c *DNSConfig) RefreshInterval() time.Duration {
 	return stringToDuration(c.GetRefreshInterval())
 }
 
+type RelabelConfig struct {
+	pb.RelabelConfig
+}
+
+func (c *RelabelConfig) Validate() error {
+	if len(c.GetSourceLabel()) == 0 {
+		return errors.New("at least one source label is required")
+	}
+	return nil
+}
+
 // TargetGroup is derived from a protobuf TargetGroup and attaches a source to it
 // that identifies the origin of the group.
 type TargetGroup struct {

config/config.proto

@@ -58,9 +58,32 @@ message DNSConfig {
 	optional string refresh_interval = 2 [default = "30s"];
 }
 
+// The configuration for relabeling of target label sets.
+message RelabelConfig {
+	// A list of labels from which values are taken and concatenated
+	// with the configured separator in order.
+	repeated string source_label = 1;
+	// Regex against which the concatenation is matched.
+	required string regex = 2;
+	// The label to which the resulting string is written in a replacement.
+	optional string target_label = 3;
+	// Replacement is the regex replacement pattern to be used.
+	optional string replacement = 4;
+	// Separator is the string between concatenated values from the source labels.
+	optional string separator = 5 [default = ";"];
+
+	// Action is the action to be performed for the relabeling.
+	enum Action {
+		REPLACE = 0; // Performs a regex replacement.
+		KEEP = 1;    // Drops targets for which the input does not match the regex.
+		DROP = 2;    // Drops targets for which the input does match the regex.
+	}
+	optional Action action = 6 [default = REPLACE];
+}
+
 // The configuration for a Prometheus job to scrape.
 //
-// The next field no. is 10.
+// The next field no. is 11.
 message ScrapeConfig {
 	// The job name. Must adhere to the regex "[a-zA-Z_][a-zA-Z0-9_-]*".
 	required string job_name = 1;
@@ -75,6 +98,8 @@ message ScrapeConfig {
 	repeated DNSConfig dns_config = 9;
 	// List of labeled target groups for this job.
 	repeated TargetGroup target_group = 5;
+	// List of relabel configurations.
+	repeated RelabelConfig relabel_config = 10;
 	// The HTTP resource path on which to fetch metrics from targets.
 	optional string metrics_path = 6 [default = "/metrics"];
 	// The URL scheme with which to fetch metrics from targets.

config/generated/config.pb.go

@@ -14,6 +14,7 @@ It has these top-level messages:
 	GlobalConfig
 	TargetGroup
 	DNSConfig
+	RelabelConfig
 	ScrapeConfig
 	PrometheusConfig
 */
@@ -26,6 +27,43 @@ import math "math"
 var _ = proto.Marshal
 var _ = math.Inf
 
+// Action is the action to be performed for the relabeling.
+type RelabelConfig_Action int32
+
+const (
+	RelabelConfig_REPLACE RelabelConfig_Action = 0
+	RelabelConfig_KEEP    RelabelConfig_Action = 1
+	RelabelConfig_DROP    RelabelConfig_Action = 2
+)
+
+var RelabelConfig_Action_name = map[int32]string{
+	0: "REPLACE",
+	1: "KEEP",
+	2: "DROP",
+}
+var RelabelConfig_Action_value = map[string]int32{
+	"REPLACE": 0,
+	"KEEP":    1,
+	"DROP":    2,
+}
+
+func (x RelabelConfig_Action) Enum() *RelabelConfig_Action {
+	p := new(RelabelConfig_Action)
+	*p = x
+	return p
+}
+func (x RelabelConfig_Action) String() string {
+	return proto.EnumName(RelabelConfig_Action_name, int32(x))
+}
+func (x *RelabelConfig_Action) UnmarshalJSON(data []byte) error {
+	value, err := proto.UnmarshalJSONEnum(RelabelConfig_Action_value, data, "RelabelConfig_Action")
+	if err != nil {
+		return err
+	}
+	*x = RelabelConfig_Action(value)
+	return nil
+}
+
 // A label/value pair suitable for attaching to timeseries.
 type LabelPair struct {
 	// The name of the label. Must adhere to the regex "[a-zA-Z_][a-zA-Z0-9_]*".
@@ -178,9 +216,75 @@ func (m *DNSConfig) GetRefreshInterval() string {
 	return Default_DNSConfig_RefreshInterval
 }
 
+// The configuration for relabeling of target label sets.
+type RelabelConfig struct {
+	// A list of labels from which values are taken and concatenated
+	// with the configured separator in order.
+	SourceLabel []string `protobuf:"bytes,1,rep,name=source_label" json:"source_label,omitempty"`
+	// Regex against which the concatenation is matched.
+	Regex *string `protobuf:"bytes,2,req,name=regex" json:"regex,omitempty"`
+	// The label to which the resulting string is written in a replacement.
+	TargetLabel *string `protobuf:"bytes,3,opt,name=target_label" json:"target_label,omitempty"`
+	// Replacement is the regex replacement pattern to be used.
+	Replacement *string `protobuf:"bytes,4,opt,name=replacement" json:"replacement,omitempty"`
+	// Separator is the string between concatenated values from the source labels.
+	Separator        *string               `protobuf:"bytes,5,opt,name=separator,def=;" json:"separator,omitempty"`
+	Action           *RelabelConfig_Action `protobuf:"varint,6,opt,name=action,enum=io.prometheus.RelabelConfig_Action,def=0" json:"action,omitempty"`
+	XXX_unrecognized []byte                `json:"-"`
+}
+
+func (m *RelabelConfig) Reset()         { *m = RelabelConfig{} }
+func (m *RelabelConfig) String() string { return proto.CompactTextString(m) }
+func (*RelabelConfig) ProtoMessage()    {}
+
+const Default_RelabelConfig_Separator string = ";"
+const Default_RelabelConfig_Action RelabelConfig_Action = RelabelConfig_REPLACE
+
+func (m *RelabelConfig) GetSourceLabel() []string {
+	if m != nil {
+		return m.SourceLabel
+	}
+	return nil
+}
+
+func (m *RelabelConfig) GetRegex() string {
+	if m != nil && m.Regex != nil {
+		return *m.Regex
+	}
+	return ""
+}
+
+func (m *RelabelConfig) GetTargetLabel() string {
+	if m != nil && m.TargetLabel != nil {
+		return *m.TargetLabel
+	}
+	return ""
+}
+
+func (m *RelabelConfig) GetReplacement() string {
+	if m != nil && m.Replacement != nil {
+		return *m.Replacement
+	}
+	return ""
+}
+
+func (m *RelabelConfig) GetSeparator() string {
+	if m != nil && m.Separator != nil {
+		return *m.Separator
+	}
+	return Default_RelabelConfig_Separator
+}
+
+func (m *RelabelConfig) GetAction() RelabelConfig_Action {
+	if m != nil && m.Action != nil {
+		return *m.Action
+	}
+	return Default_RelabelConfig_Action
+}
+
 // The configuration for a Prometheus job to scrape.
 //
-// The next field no. is 10.
+// The next field no. is 11.
 type ScrapeConfig struct {
 	// The job name. Must adhere to the regex "[a-zA-Z_][a-zA-Z0-9_-]*".
 	JobName *string `protobuf:"bytes,1,req,name=job_name" json:"job_name,omitempty"`
@@ -195,6 +299,8 @@ type ScrapeConfig struct {
 	DnsConfig []*DNSConfig `protobuf:"bytes,9,rep,name=dns_config" json:"dns_config,omitempty"`
 	// List of labeled target groups for this job.
 	TargetGroup []*TargetGroup `protobuf:"bytes,5,rep,name=target_group" json:"target_group,omitempty"`
+	// List of relabel configurations.
+	RelabelConfig []*RelabelConfig `protobuf:"bytes,10,rep,name=relabel_config" json:"relabel_config,omitempty"`
 	// The HTTP resource path on which to fetch metrics from targets.
 	MetricsPath *string `protobuf:"bytes,6,opt,name=metrics_path,def=/metrics" json:"metrics_path,omitempty"`
 	// The URL scheme with which to fetch metrics from targets.
@@ -245,6 +351,13 @@ func (m *ScrapeConfig) GetTargetGroup() []*TargetGroup {
 	return nil
 }
 
+func (m *ScrapeConfig) GetRelabelConfig() []*RelabelConfig {
+	if m != nil {
+		return m.RelabelConfig
+	}
+	return nil
+}
+
 func (m *ScrapeConfig) GetMetricsPath() string {
 	if m != nil && m.MetricsPath != nil {
 		return *m.MetricsPath
@@ -289,4 +402,5 @@ func (m *PrometheusConfig) GetScrapeConfig() []*ScrapeConfig {
 }
 
 func init() {
+	proto.RegisterEnum("io.prometheus.RelabelConfig_Action", RelabelConfig_Action_name, RelabelConfig_Action_value)
 }

retrieval/discovery/dns.go

@@ -32,6 +32,7 @@ const (
 	resolvConf = "/etc/resolv.conf"
 
 	dnsSourcePrefix = "dns"
+	DNSNameLabel    = clientmodel.MetaLabelPrefix + "dns_srv_name"
 
 	// Constants for instrumentation.
 	namespace = "prometheus"
@@ -148,6 +149,7 @@ func (dd *DNSDiscovery) refresh(name string, ch chan<- *config.TargetGroup) erro
 		target := clientmodel.LabelValue(fmt.Sprintf("%s:%d", addr.Target, addr.Port))
 		tg.Targets = append(tg.Targets, clientmodel.LabelSet{
 			clientmodel.AddressLabel: target,
+			DNSNameLabel:             clientmodel.LabelValue(name),
 		})
 	}
 

retrieval/relabel.go

@@ -0,0 +1,70 @@
+package retrieval
+
+import (
+	"regexp"
+	"strings"
+
+	clientmodel "github.com/prometheus/client_golang/model"
+
+	"github.com/prometheus/prometheus/config"
+	pb "github.com/prometheus/prometheus/config/generated"
+)
+
+// Relabel returns a relabeled copy of the given label set. The relabel configurations
+// are applied in order of input.
+// If a label set is dropped, nil is returned.
+func Relabel(labels clientmodel.LabelSet, cfgs ...*config.RelabelConfig) (clientmodel.LabelSet, error) {
+	out := clientmodel.LabelSet{}
+	for ln, lv := range labels {
+		out[ln] = lv
+	}
+	var err error
+	for _, cfg := range cfgs {
+		if out, err = relabel(out, cfg); err != nil {
+			return nil, err
+		}
+		if out == nil {
+			return nil, nil
+		}
+	}
+	return out, nil
+}
+
+func relabel(labels clientmodel.LabelSet, cfg *config.RelabelConfig) (clientmodel.LabelSet, error) {
+	pat, err := regexp.Compile(cfg.GetRegex())
+	if err != nil {
+		return nil, err
+	}
+
+	values := make([]string, 0, len(cfg.GetSourceLabel()))
+	for _, name := range cfg.GetSourceLabel() {
+		values = append(values, string(labels[clientmodel.LabelName(name)]))
+	}
+	val := strings.Join(values, cfg.GetSeparator())
+
+	switch cfg.GetAction() {
+	case pb.RelabelConfig_DROP:
+		if pat.MatchString(val) {
+			return nil, nil
+		}
+	case pb.RelabelConfig_KEEP:
+		if !pat.MatchString(val) {
+			return nil, nil
+		}
+	case pb.RelabelConfig_REPLACE:
+		// If there is no match no replacement must take place.
+		if !pat.MatchString(val) {
+			break
+		}
+		res := pat.ReplaceAllString(val, cfg.GetReplacement())
+		ln := clientmodel.LabelName(cfg.GetTargetLabel())
+		if res == "" {
+			delete(labels, ln)
+		} else {
+			labels[ln] = clientmodel.LabelValue(res)
+		}
+	default:
+		panic("retrieval.relabel: unknown relabel action type")
+	}
+	return labels, nil
+}

retrieval/relabel_test.go

@@ -0,0 +1,172 @@
+package retrieval
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/golang/protobuf/proto"
+
+	clientmodel "github.com/prometheus/client_golang/model"
+
+	"github.com/prometheus/prometheus/config"
+	pb "github.com/prometheus/prometheus/config/generated"
+)
+
+func TestRelabel(t *testing.T) {
+	tests := []struct {
+		input   clientmodel.LabelSet
+		relabel []pb.RelabelConfig
+		output  clientmodel.LabelSet
+	}{
+		{
+			input: clientmodel.LabelSet{
+				"a": "foo",
+				"b": "bar",
+				"c": "baz",
+			},
+			relabel: []pb.RelabelConfig{
+				{
+					SourceLabel: []string{"a"},
+					Regex:       proto.String("f(.*)"),
+					TargetLabel: proto.String("d"),
+					Separator:   proto.String(";"),
+					Replacement: proto.String("ch${1}-ch${1}"),
+					Action:      pb.RelabelConfig_REPLACE.Enum(),
+				},
+			},
+			output: clientmodel.LabelSet{
+				"a": "foo",
+				"b": "bar",
+				"c": "baz",
+				"d": "choo-choo",
+			},
+		},
+		{
+			input: clientmodel.LabelSet{
+				"a": "foo",
+				"b": "bar",
+				"c": "baz",
+			},
+			relabel: []pb.RelabelConfig{
+				{
+					SourceLabel: []string{"a", "b"},
+					Regex:       proto.String("^f(.*);(.*)r$"),
+					TargetLabel: proto.String("a"),
+					Separator:   proto.String(";"),
+					Replacement: proto.String("b${1}${2}m"), // boobam
+				},
+				{
+					SourceLabel: []string{"c", "a"},
+					Regex:       proto.String("(b).*b(.*)ba(.*)"),
+					TargetLabel: proto.String("d"),
+					Separator:   proto.String(";"),
+					Replacement: proto.String("$1$2$2$3"),
+					Action:      pb.RelabelConfig_REPLACE.Enum(),
+				},
+			},
+			output: clientmodel.LabelSet{
+				"a": "boobam",
+				"b": "bar",
+				"c": "baz",
+				"d": "boooom",
+			},
+		},
+		{
+			input: clientmodel.LabelSet{
+				"a": "foo",
+			},
+			relabel: []pb.RelabelConfig{
+				{
+					SourceLabel: []string{"a"},
+					Regex:       proto.String("o$"),
+					Action:      pb.RelabelConfig_DROP.Enum(),
+				}, {
+					SourceLabel: []string{"a"},
+					Regex:       proto.String("f(.*)"),
+					TargetLabel: proto.String("d"),
+					Separator:   proto.String(";"),
+					Replacement: proto.String("ch$1-ch$1"),
+					Action:      pb.RelabelConfig_REPLACE.Enum(),
+				},
+			},
+			output: nil,
+		},
+		{
+			input: clientmodel.LabelSet{
+				"a": "foo",
+			},
+			relabel: []pb.RelabelConfig{
+				{
+					SourceLabel: []string{"a"},
+					Regex:       proto.String("no-match"),
+					Action:      pb.RelabelConfig_DROP.Enum(),
+				},
+			},
+			output: clientmodel.LabelSet{
+				"a": "foo",
+			},
+		},
+		{
+			input: clientmodel.LabelSet{
+				"a": "foo",
+			},
+			relabel: []pb.RelabelConfig{
+				{
+					SourceLabel: []string{"a"},
+					Regex:       proto.String("no-match"),
+					Action:      pb.RelabelConfig_KEEP.Enum(),
+				},
+			},
+			output: nil,
+		},
+		{
+			input: clientmodel.LabelSet{
+				"a": "foo",
+			},
+			relabel: []pb.RelabelConfig{
+				{
+					SourceLabel: []string{"a"},
+					Regex:       proto.String("^f"),
+					Action:      pb.RelabelConfig_KEEP.Enum(),
+				},
+			},
+			output: clientmodel.LabelSet{
+				"a": "foo",
+			},
+		},
+		{
+			// No replacement must be applied if there is no match.
+			input: clientmodel.LabelSet{
+				"a": "boo",
+			},
+			relabel: []pb.RelabelConfig{
+				{
+					SourceLabel: []string{"a"},
+					Regex:       proto.String("^f"),
+					Action:      pb.RelabelConfig_REPLACE.Enum(),
+					TargetLabel: proto.String("b"),
+					Replacement: proto.String("bar"),
+				},
+			},
+			output: clientmodel.LabelSet{
+				"a": "boo",
+			},
+		},
+	}
+
+	for i, test := range tests {
+		var relabel []*config.RelabelConfig
+		for _, rl := range test.relabel {
+			proto.SetDefaults(&rl)
+			relabel = append(relabel, &config.RelabelConfig{rl})
+		}
+		res, err := Relabel(test.input, relabel...)
+		if err != nil {
+			t.Errorf("Test %d: error relabeling: %s", i+1, err)
+		}
+
+		if !reflect.DeepEqual(res, test.output) {
+			t.Errorf("Test %d: relabel output mismatch: expected %#v, got %#v", i+1, test.output, res)
+		}
+	}
+}

retrieval/target.go

@@ -172,10 +172,10 @@ type target struct {
 }
 
 // NewTarget creates a reasonably configured target for querying.
-func NewTarget(address string, cfg *config.ScrapeConfig, baseLabels clientmodel.LabelSet) Target {
+func NewTarget(cfg *config.ScrapeConfig, baseLabels clientmodel.LabelSet) Target {
 	t := &target{
 		url: &url.URL{
-			Host: address,
+			Host: string(baseLabels[clientmodel.AddressLabel]),
 		},
 		scraperStopping: make(chan struct{}),
 		scraperStopped:  make(chan struct{}),
@@ -197,16 +197,16 @@ func (t *target) Update(cfg *config.ScrapeConfig, baseLabels clientmodel.LabelSe
 	t.deadline = cfg.ScrapeTimeout()
 	t.httpClient = utility.NewDeadlineClient(cfg.ScrapeTimeout())
 
-	t.baseLabels = clientmodel.LabelSet{
-		clientmodel.InstanceLabel: clientmodel.LabelValue(t.InstanceIdentifier()),
-	}
-
+	t.baseLabels = clientmodel.LabelSet{}
 	// All remaining internal labels will not be part of the label set.
 	for name, val := range baseLabels {
 		if !strings.HasPrefix(string(name), clientmodel.ReservedLabelPrefix) {
 			t.baseLabels[name] = val
 		}
 	}
+	if _, ok := t.baseLabels[clientmodel.InstanceLabel]; !ok {
+		t.baseLabels[clientmodel.InstanceLabel] = clientmodel.LabelValue(t.InstanceIdentifier())
+	}
 }
 
 func (t *target) String() string {

retrieval/targetmanager.go

@@ -81,19 +81,19 @@ func (tm *TargetManager) Run() {
 	sources := map[string]struct{}{}
 
 	for scfg, provs := range tm.providers {
-		for _, p := range provs {
+		for _, prov := range provs {
 			ch := make(chan *config.TargetGroup)
 			go tm.handleTargetUpdates(scfg, ch)
 
-			for _, src := range p.Sources() {
+			for _, src := range prov.Sources() {
 				src = fullSource(scfg, src)
 				sources[src] = struct{}{}
 			}
 
 			// Run the target provider after cleanup of the stale targets is done.
-			defer func(c chan *config.TargetGroup) {
+			defer func(p TargetProvider, c chan *config.TargetGroup) {
 				go p.Run(c)
-			}(ch)
+			}(prov, ch)
 		}
 	}
 
@@ -326,9 +326,17 @@ func (tm *TargetManager) targetsFromGroup(tg *config.TargetGroup, cfg *config.Sc
 			}
 		}
 
-		address, ok := labels[clientmodel.AddressLabel]
-		if !ok {
-			return nil, fmt.Errorf("Instance %d in target group %s has no address", i, tg)
+		if _, ok := labels[clientmodel.AddressLabel]; !ok {
+			return nil, fmt.Errorf("instance %d in target group %s has no address", i, tg)
+		}
+
+		labels, err := Relabel(labels, cfg.RelabelConfigs()...)
+		if err != nil {
+			return nil, fmt.Errorf("error while relabelling instance %d in target group %s: %s", i, tg, err)
+		}
+		// Check if the target was dropped.
+		if labels == nil {
+			continue
 		}
 
 		for ln := range labels {
@@ -338,8 +346,8 @@ func (tm *TargetManager) targetsFromGroup(tg *config.TargetGroup, cfg *config.Sc
 				delete(labels, ln)
 			}
 		}
-		targets = append(targets, NewTarget(string(address), cfg, labels))
-
+		tr := NewTarget(cfg, labels)
+		targets = append(targets, tr)
 	}
 
 	return targets, nil

retrieval/targetmanager_test.go

@@ -164,10 +164,45 @@ func TestTargetManagerConfigUpdate(t *testing.T) {
 		JobName:        proto.String("test_job2"),
 		ScrapeInterval: proto.String("1m"),
 		TargetGroup: []*pb.TargetGroup{
-			{Target: []string{"example.org:8080", "example.com:8081"}},
+			{
+				Target: []string{"example.org:8080", "example.com:8081"},
+				Labels: &pb.LabelPairs{Label: []*pb.LabelPair{
+					{Name: proto.String("foo"), Value: proto.String("bar")},
+					{Name: proto.String("boom"), Value: proto.String("box")},
+				}},
+			},
 			{Target: []string{"test.com:1234"}},
+			{
+				Target: []string{"test.com:1235"},
+				Labels: &pb.LabelPairs{Label: []*pb.LabelPair{
+					{Name: proto.String("instance"), Value: proto.String("fixed")},
+				}},
+			},
+		},
+		RelabelConfig: []*pb.RelabelConfig{
+			{
+				SourceLabel: []string{string(clientmodel.AddressLabel)},
+				Regex:       proto.String(`^test\.(.*?):(.*)`),
+				Replacement: proto.String("foo.${1}:${2}"),
+				TargetLabel: proto.String(string(clientmodel.AddressLabel)),
+			}, {
+				// Add a new label for example.* targets.
+				SourceLabel: []string{string(clientmodel.AddressLabel), "boom", "foo"},
+				Regex:       proto.String("^example.*?-b([a-z-]+)r$"),
+				TargetLabel: proto.String("new"),
+				Replacement: proto.String("$1"),
+				Separator:   proto.String("-"),
+			}, {
+				// Drop an existing label.
+				SourceLabel: []string{"boom"},
+				Regex:       proto.String(".*"),
+				TargetLabel: proto.String("boom"),
+				Replacement: proto.String(""),
+			},
 		},
 	}
+	proto.SetDefaults(testJob1)
+	proto.SetDefaults(testJob2)
 
 	sequence := []struct {
 		scrapeConfigs []*pb.ScrapeConfig
@@ -197,11 +232,14 @@ func TestTargetManagerConfigUpdate(t *testing.T) {
 					{clientmodel.JobLabel: "test_job1", clientmodel.InstanceLabel: "example.com:80"},
 				},
 				"test_job2:static:0": {
-					{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "example.org:8080"},
-					{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "example.com:8081"},
+					{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "example.org:8080", "foo": "bar", "new": "ox-ba"},
+					{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "example.com:8081", "foo": "bar", "new": "ox-ba"},
 				},
 				"test_job2:static:1": {
-					{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "test.com:1234"},
+					{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "foo.com:1234"},
+				},
+				"test_job2:static:2": {
+					{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "fixed"},
 				},
 			},
 		}, {
@@ -211,11 +249,14 @@ func TestTargetManagerConfigUpdate(t *testing.T) {
 			scrapeConfigs: []*pb.ScrapeConfig{testJob2},
 			expected: map[string][]clientmodel.LabelSet{
 				"test_job2:static:0": {
-					{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "example.org:8080"},
-					{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "example.com:8081"},
+					{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "example.org:8080", "foo": "bar", "new": "ox-ba"},
+					{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "example.com:8081", "foo": "bar", "new": "ox-ba"},
 				},
 				"test_job2:static:1": {
-					{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "test.com:1234"},
+					{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "foo.com:1234"},
+				},
+				"test_job2:static:2": {
+					{clientmodel.JobLabel: "test_job2", clientmodel.InstanceLabel: "fixed"},
 				},
 			},
 		},