diff --git a/config/config.go b/config/config.go index e00e24535..dd482ad2f 100644 --- a/config/config.go +++ b/config/config.go @@ -33,7 +33,7 @@ var ( patJobName = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z0-9_-]*$`) patFileSDName = regexp.MustCompile(`^[^*]*(\*[^/]*)?\.(json|yml|yaml|JSON|YML|YAML)$`) patRulePath = regexp.MustCompile(`^[^*]*(\*[^/]*)?$`) - patAuthLine = regexp.MustCompile(`((?:username|password|bearer_token):\s+)(".+"|'.+'|[^\s]+)`) + patAuthLine = regexp.MustCompile(`((?:username|password|bearer_token|secret_key):\s+)(".+"|'.+'|[^\s]+)`) ) // Load parses the YAML input s into a Config. @@ -128,6 +128,12 @@ var ( RequestTimeout: Duration(10 * time.Second), RetryInterval: Duration(1 * time.Second), } + + // DefaultEC2SDConfig is the default EC2 SD configuration. + DefaultEC2SDConfig = EC2SDConfig{ + Port: 80, + RefreshInterval: Duration(60 * time.Second), + } ) // URL is a custom URL type that allows validation at configuration load time. @@ -351,6 +357,8 @@ type ScrapeConfig struct { MarathonSDConfigs []*MarathonSDConfig `yaml:"marathon_sd_configs,omitempty"` // List of Kubernetes service discovery configurations. KubernetesSDConfigs []*KubernetesSDConfig `yaml:"kubernetes_sd_configs,omitempty"` + // List of EC2 service discovery configurations. + EC2SDConfigs []*EC2SDConfig `yaml:"ec2_sd_configs,omitempty"` // List of target relabel configurations. RelabelConfigs []*RelabelConfig `yaml:"relabel_configs,omitempty"` @@ -664,6 +672,31 @@ func (c *KubernetesSDConfig) UnmarshalYAML(unmarshal func(interface{}) error) er return checkOverflow(c.XXX, "kubernetes_sd_config") } +// EC2SDConfig is the configuration for EC2 based service discovery. +type EC2SDConfig struct { + Region string `yaml:"region"` + AccessKey string `yaml:"access_key,omitempty"` + SecretKey string `yaml:"secret_key,omitempty"` + RefreshInterval Duration `yaml:"refresh_interval,omitempty"` + Port int `yaml:"port"` + // Catches all undefined fields and must be empty after parsing. + XXX map[string]interface{} `yaml:",inline"` +} + +// UnmarshalYAML implements the yaml.Unmarshaler interface. +func (c *EC2SDConfig) UnmarshalYAML(unmarshal func(interface{}) error) error { + *c = DefaultEC2SDConfig + type plain EC2SDConfig + err := unmarshal((*plain)(c)) + if err != nil { + return err + } + if c.Region == "" { + return fmt.Errorf("EC2 SD configuration requires a region") + } + return checkOverflow(c.XXX, "ec2_sd_config") +} + // RelabelAction is the action to be performed on relabeling. type RelabelAction string diff --git a/config/config_test.go b/config/config_test.go index 3e38cda33..053e94c21 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -230,6 +230,25 @@ var expectedConf = &Config{ }, }, }, + { + JobName: "service-ec2", + + ScrapeInterval: Duration(15 * time.Second), + ScrapeTimeout: DefaultGlobalConfig.ScrapeTimeout, + + MetricsPath: DefaultScrapeConfig.MetricsPath, + Scheme: DefaultScrapeConfig.Scheme, + + EC2SDConfigs: []*EC2SDConfig{ + { + Region: "us-east-1", + AccessKey: "access", + SecretKey: "secret", + RefreshInterval: Duration(60 * time.Second), + Port: 80, + }, + }, + }, }, original: "", } diff --git a/config/testdata/conf.good.yml b/config/testdata/conf.good.yml index 3e04c0a9a..783c75393 100644 --- a/config/testdata/conf.good.yml +++ b/config/testdata/conf.good.yml @@ -114,3 +114,9 @@ scrape_configs: marathon_sd_configs: - servers: - 'http://marathon.example.com:8080' + +- job_name: service-ec2 + ec2_sd_configs: + - region: us-east-1 + access_key: access + secret_key: secret diff --git a/retrieval/discovery/ec2.go b/retrieval/discovery/ec2.go new file mode 100644 index 000000000..795355509 --- /dev/null +++ b/retrieval/discovery/ec2.go @@ -0,0 +1,131 @@ +// Copyright 2015 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package discovery + +import ( + "fmt" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/prometheus/common/log" + "github.com/prometheus/common/model" + + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/util/strutil" +) + +const ( + ec2Label = model.MetaLabelPrefix + "ec2_" + ec2LabelInstanceID = ec2Label + "instance_id" + ec2LabelPublicIP = ec2Label + "public_ip" + ec2LabelPrivateIP = ec2Label + "private_ip" + ec2LabelTag = ec2Label + "tag_" +) + +// EC2Discovery periodically performs EC2-SD requests. It implements +// the TargetProvider interface. +type EC2Discovery struct { + aws *aws.Config + done chan struct{} + interval time.Duration + port int +} + +// NewEC2Discovery returns a new EC2Discovery which periodically refreshes its targets. +func NewEC2Discovery(conf *config.EC2SDConfig) *EC2Discovery { + creds := credentials.NewStaticCredentials(conf.AccessKey, conf.SecretKey, "") + if conf.AccessKey == "" && conf.SecretKey == "" { + creds = credentials.NewEnvCredentials() + } + return &EC2Discovery{ + aws: &aws.Config{ + Region: &conf.Region, + Credentials: creds, + }, + done: make(chan struct{}), + interval: time.Duration(conf.RefreshInterval), + port: conf.Port, + } +} + +// Run implements the TargetProvider interface. +func (ed *EC2Discovery) Run(ch chan<- *config.TargetGroup, done <-chan struct{}) { + defer close(ch) + + ticker := time.NewTicker(ed.interval) + defer ticker.Stop() + + // Get an initial set right away. + tg, err := ed.refresh() + if err != nil { + log.Error(err) + } else { + ch <- tg + } + + for { + select { + case <-ticker.C: + tg, err := ed.refresh() + if err != nil { + log.Error(err) + } else { + ch <- tg + } + case <-done: + return + } + } +} + +// Sources implements the TargetProvider interface. +func (ed *EC2Discovery) Sources() []string { + return []string{*ed.aws.Region} +} + +func (ed *EC2Discovery) refresh() (*config.TargetGroup, error) { + ec2s := ec2.New(ed.aws) + tg := &config.TargetGroup{ + Source: *ed.aws.Region, + } + if err := ec2s.DescribeInstancesPages(nil, func(p *ec2.DescribeInstancesOutput, lastPage bool) bool { + for _, r := range p.Reservations { + for _, inst := range r.Instances { + if inst.PrivateIpAddress == nil { + continue + } + labels := model.LabelSet{ + ec2LabelInstanceID: model.LabelValue(*inst.InstanceId), + } + if inst.PublicIpAddress != nil { + labels[ec2LabelPublicIP] = model.LabelValue(*inst.PublicIpAddress) + } + labels[ec2LabelPrivateIP] = model.LabelValue(*inst.PrivateIpAddress) + addr := fmt.Sprintf("%s:%d", *inst.PrivateIpAddress, ed.port) + labels[model.AddressLabel] = model.LabelValue(addr) + for _, t := range inst.Tags { + name := strutil.SanitizeLabelName(*t.Key) + labels[ec2LabelTag+model.LabelName(name)] = model.LabelValue(*t.Value) + } + tg.Targets = append(tg.Targets, labels) + } + } + return true + }); err != nil { + return nil, fmt.Errorf("could not describe instances: %s", err) + } + return tg, nil +} diff --git a/retrieval/targetmanager.go b/retrieval/targetmanager.go index 1c101425c..3f3a4377b 100644 --- a/retrieval/targetmanager.go +++ b/retrieval/targetmanager.go @@ -438,6 +438,9 @@ func providersFromConfig(cfg *config.ScrapeConfig) []TargetProvider { for i, c := range cfg.ServersetSDConfigs { app("serverset", i, discovery.NewServersetDiscovery(c)) } + for i, c := range cfg.EC2SDConfigs { + app("ec2", i, discovery.NewEC2Discovery(c)) + } if len(cfg.TargetGroups) > 0 { app("static", 0, NewStaticProvider(cfg.TargetGroups)) }