From 5b78aa0649689fd1e437e3f4ea0649b4f6e316ab Mon Sep 17 00:00:00 2001 From: Robert Fratto Date: Mon, 8 Mar 2021 14:20:09 -0500 Subject: [PATCH] Contribute grafana/agent sigv4 code (#8509) * Contribute grafana/agent sigv4 code * address review feedback - move validation logic for RemoteWrite into unmarshal - copy configuration fields from ec2 SD config - remove enabled field, use pointer for enabling sigv4 * Update config/config.go * Don't provide credentials if secret key / access key left blank * Add SigV4 headers to the list of unchangeable headers. * sigv4: don't include all headers in signature * only test for equality in the authorization header, not the signed date * address review feedback 1. s/httpClientConfigEnabled/httpClientConfigAuthEnabled 2. bearer_token tuples to "authorization" 3. Un-export NewSigV4RoundTripper * add x-amz-content-sha256 to list of unchangeable headers * Document sigv4 configuration * add suggestion for using default AWS SDK credentials Signed-off-by: Robert Fratto Co-authored-by: Julien Pivotto --- config/config.go | 30 +++++- docs/configuration/configuration.md | 29 +++++- storage/remote/client.go | 12 ++- storage/remote/sigv4.go | 138 ++++++++++++++++++++++++++++ storage/remote/sigv4_test.go | 92 +++++++++++++++++++ storage/remote/write.go | 1 + 6 files changed, 295 insertions(+), 7 deletions(-) create mode 100644 storage/remote/sigv4.go create mode 100644 storage/remote/sigv4_test.go diff --git a/config/config.go b/config/config.go index 3ba890ae1e..792ef5c1b0 100644 --- a/config/config.go +++ b/config/config.go @@ -51,6 +51,11 @@ var ( "accept-encoding": {}, "x-prometheus-remote-write-version": {}, "x-prometheus-remote-read-version": {}, + + // Added by SigV4. + "x-amz-date": {}, + "x-amz-security-token": {}, + "x-amz-content-sha256": {}, } ) @@ -601,6 +606,7 @@ type RemoteWriteConfig struct { HTTPClientConfig config.HTTPClientConfig `yaml:",inline"` QueueConfig QueueConfig `yaml:"queue_config,omitempty"` MetadataConfig MetadataConfig `yaml:"metadata_config,omitempty"` + SigV4Config *SigV4Config `yaml:"sigv4,omitempty"` } // SetDirectory joins any relative file paths with dir. @@ -630,7 +636,18 @@ func (c *RemoteWriteConfig) UnmarshalYAML(unmarshal func(interface{}) error) err // The UnmarshalYAML method of HTTPClientConfig is not being called because it's not a pointer. // We cannot make it a pointer as the parser panics for inlined pointer structs. // Thus we just do its validation here. - return c.HTTPClientConfig.Validate() + if err := c.HTTPClientConfig.Validate(); err != nil { + return err + } + + httpClientConfigAuthEnabled := c.HTTPClientConfig.BasicAuth != nil || + c.HTTPClientConfig.Authorization != nil + + if httpClientConfigAuthEnabled && c.SigV4Config != nil { + return fmt.Errorf("at most one of basic_auth, authorization, & sigv4 must be configured") + } + + return nil } func validateHeaders(headers map[string]string) error { @@ -679,6 +696,17 @@ type MetadataConfig struct { SendInterval model.Duration `yaml:"send_interval"` } +// SigV4Config is the configuration for signing remote write requests with +// AWS's SigV4 verification process. Empty values will be retrieved using the +// AWS default credentials chain. +type SigV4Config struct { + Region string `yaml:"region,omitempty"` + AccessKey string `yaml:"access_key,omitempty"` + SecretKey config.Secret `yaml:"secret_key,omitempty"` + Profile string `yaml:"profile,omitempty"` + RoleARN string `yaml:"role_arn,omitempty"` +} + // RemoteReadConfig is the configuration for reading from remote storage. type RemoteReadConfig struct { URL *config.URL `yaml:"url"` diff --git a/docs/configuration/configuration.md b/docs/configuration/configuration.md index 678e13da8c..8a7f02a42b 100644 --- a/docs/configuration/configuration.md +++ b/docs/configuration/configuration.md @@ -1222,13 +1222,13 @@ namespaces: names: [ - ] -# Optional label and field selectors to limit the discovery process to a subset of available resources. +# Optional label and field selectors to limit the discovery process to a subset of available resources. # See https://kubernetes.io/docs/concepts/overview/working-with-objects/field-selectors/ -# and https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ to learn more about the possible +# and https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/ to learn more about the possible # filters that can be used. Endpoints role supports pod, service and endpoints selectors, other roles # only support selectors matching the role itself (e.g. node role can only contain node selectors). -# Note: When making decision about using field/label selector make sure that this +# Note: When making decision about using field/label selector make sure that this # is the best approach - it will prevent Prometheus from reusing single list/watch # for all scrape configs. This might result in a bigger load on the Kubernetes API, # because per each selector combination there will be additional LIST/WATCH. On the other hand, @@ -1788,7 +1788,7 @@ headers: write_relabel_configs: [ - ... ] -# Name of the remote write config, which if specified must be unique among remote write configs. +# Name of the remote write config, which if specified must be unique among remote write configs. # The name will be used in metrics and logging in place of a generated value to help users distinguish between # remote write configs. [ name: ] @@ -1812,6 +1812,25 @@ authorization: # It is mutually exclusive with `credentials`. [ credentials_file: ] +# Optionally configures AWS's Signature Verification 4 signing process to +# sign requests. Cannot be set at the same time as basic_auth or authorization. +# To use the default credentials from the AWS SDK, use `sigv4: {}`. +sigv4: + # The AWS region. If blank, the region from the default credentials chain + # is used. + [ region: ] + + # The AWS API keys. If blank, the environment variables `AWS_ACCESS_KEY_ID` + # and `AWS_SECRET_ACCESS_KEY` are used. + [ access_key: ] + [ secret_key: ] + + # Named AWS profile used to authenticate. + [ profile: ] + + # AWS Role ARN, an alternative to using AWS API keys. + [ role_arn: ] + # Configures the remote write request's TLS settings. tls_config: [ ] @@ -1865,7 +1884,7 @@ with this feature. # The URL of the endpoint to query from. url: -# Name of the remote read config, which if specified must be unique among remote read configs. +# Name of the remote read config, which if specified must be unique among remote read configs. # The name will be used in metrics and logging in place of a generated value to help users distinguish between # remote read configs. [ name: ] diff --git a/storage/remote/client.go b/storage/remote/client.go index 40b22a1b78..346f66c2f7 100644 --- a/storage/remote/client.go +++ b/storage/remote/client.go @@ -35,6 +35,7 @@ import ( "github.com/prometheus/common/model" "github.com/prometheus/common/version" + "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/prompb" ) @@ -96,6 +97,7 @@ type ClientConfig struct { URL *config_util.URL Timeout model.Duration HTTPClientConfig config_util.HTTPClientConfig + SigV4Config *config.SigV4Config Headers map[string]string RetryOnRateLimit bool } @@ -138,11 +140,19 @@ func NewWriteClient(name string, conf *ClientConfig) (WriteClient, error) { if err != nil { return nil, err } - t := httpClient.Transport + + if conf.SigV4Config != nil { + t, err = newSigV4RoundTripper(conf.SigV4Config, httpClient.Transport) + if err != nil { + return nil, err + } + } + if len(conf.Headers) > 0 { t = newInjectHeadersRoundTripper(conf.Headers, t) } + httpClient.Transport = &nethttp.Transport{ RoundTripper: t, } diff --git a/storage/remote/sigv4.go b/storage/remote/sigv4.go new file mode 100644 index 0000000000..4a8974f711 --- /dev/null +++ b/storage/remote/sigv4.go @@ -0,0 +1,138 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "bytes" + "fmt" + "io" + "io/ioutil" + "net/http" + "net/textproto" + "sync" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/credentials/stscreds" + "github.com/aws/aws-sdk-go/aws/session" + signer "github.com/aws/aws-sdk-go/aws/signer/v4" + "github.com/prometheus/prometheus/config" +) + +var sigv4HeaderDenylist = []string{ + "uber-trace-id", +} + +type sigV4RoundTripper struct { + region string + next http.RoundTripper + pool sync.Pool + + signer *signer.Signer +} + +// newSigV4RoundTripper returns a new http.RoundTripper that will sign requests +// using Amazon's Signature Verification V4 signing procedure. The request will +// then be handed off to the next RoundTripper provided by next. If next is nil, +// http.DefaultTransport will be used. +// +// Credentials for signing are retrieved using the the default AWS credential +// chain. If credentials cannot be found, an error will be returned. +func newSigV4RoundTripper(cfg *config.SigV4Config, next http.RoundTripper) (http.RoundTripper, error) { + if next == nil { + next = http.DefaultTransport + } + + creds := credentials.NewStaticCredentials(cfg.AccessKey, string(cfg.SecretKey), "") + if cfg.AccessKey == "" && cfg.SecretKey == "" { + creds = nil + } + + sess, err := session.NewSessionWithOptions(session.Options{ + Config: aws.Config{ + Region: aws.String(cfg.Region), + Credentials: creds, + }, + Profile: cfg.Profile, + }) + if err != nil { + return nil, fmt.Errorf("could not create new AWS session: %w", err) + } + if _, err := sess.Config.Credentials.Get(); err != nil { + return nil, fmt.Errorf("could not get SigV4 credentials: %w", err) + } + if aws.StringValue(sess.Config.Region) == "" { + return nil, fmt.Errorf("region not configured in sigv4 or in default credentials chain") + } + + signerCreds := sess.Config.Credentials + if cfg.RoleARN != "" { + signerCreds = stscreds.NewCredentials(sess, cfg.RoleARN) + } + + rt := &sigV4RoundTripper{ + region: cfg.Region, + next: next, + signer: signer.NewSigner(signerCreds), + } + rt.pool.New = rt.newBuf + return rt, nil +} + +func (rt *sigV4RoundTripper) newBuf() interface{} { + return bytes.NewBuffer(make([]byte, 0, 1024)) +} + +func (rt *sigV4RoundTripper) RoundTrip(req *http.Request) (*http.Response, error) { + // rt.signer.Sign needs a seekable body, so we replace the body with a + // buffered reader filled with the contents of original body. + buf := rt.pool.Get().(*bytes.Buffer) + defer func() { + buf.Reset() + rt.pool.Put(buf) + }() + if _, err := io.Copy(buf, req.Body); err != nil { + return nil, err + } + // Close the original body since we don't need it anymore. + _ = req.Body.Close() + + // Ensure our seeker is back at the start of the buffer once we return. + var seeker io.ReadSeeker = bytes.NewReader(buf.Bytes()) + defer func() { + _, _ = seeker.Seek(0, io.SeekStart) + }() + req.Body = ioutil.NopCloser(seeker) + + // Clone the request and trim out headers that we don't want to sign. + signReq := req.Clone(req.Context()) + for _, header := range sigv4HeaderDenylist { + signReq.Header.Del(header) + } + + headers, err := rt.signer.Sign(signReq, seeker, "aps", rt.region, time.Now().UTC()) + if err != nil { + return nil, fmt.Errorf("failed to sign request: %w", err) + } + + // Copy over signed headers. Authorization header is not returned by + // rt.signer.Sign and needs to be copied separately. + for k, v := range headers { + req.Header[textproto.CanonicalMIMEHeaderKey(k)] = v + } + req.Header.Set("Authorization", signReq.Header.Get("Authorization")) + + return rt.next.RoundTrip(req) +} diff --git a/storage/remote/sigv4_test.go b/storage/remote/sigv4_test.go new file mode 100644 index 0000000000..97d0878533 --- /dev/null +++ b/storage/remote/sigv4_test.go @@ -0,0 +1,92 @@ +// Copyright 2021 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package remote + +import ( + "net/http" + "os" + "strings" + "testing" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + signer "github.com/aws/aws-sdk-go/aws/signer/v4" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/stretchr/testify/require" +) + +func TestSigV4_Inferred_Region(t *testing.T) { + os.Setenv("AWS_ACCESS_KEY_ID", "secret") + os.Setenv("AWS_SECRET_ACCESS_KEY", "token") + os.Setenv("AWS_REGION", "us-west-2") + + sess, err := session.NewSession(&aws.Config{ + // Setting to an empty string to demostrate the default value from the yaml + // won't override the environment's region. + Region: aws.String(""), + }) + require.NoError(t, err) + _, err = sess.Config.Credentials.Get() + require.NoError(t, err) + + require.NotNil(t, sess.Config.Region) + require.Equal(t, "us-west-2", *sess.Config.Region) +} + +func TestSigV4RoundTripper(t *testing.T) { + var gotReq *http.Request + + rt := &sigV4RoundTripper{ + region: "us-east-2", + next: promhttp.RoundTripperFunc(func(req *http.Request) (*http.Response, error) { + gotReq = req + return &http.Response{StatusCode: http.StatusOK}, nil + }), + signer: signer.NewSigner(credentials.NewStaticCredentials( + "test-id", + "secret", + "token", + )), + } + rt.pool.New = rt.newBuf + + cli := &http.Client{Transport: rt} + + req, err := http.NewRequest(http.MethodPost, "google.com", strings.NewReader("Hello, world!")) + require.NoError(t, err) + + _, err = cli.Do(req) + require.NoError(t, err) + require.NotNil(t, gotReq) + + origReq := gotReq + require.NotEmpty(t, origReq.Header.Get("Authorization")) + require.NotEmpty(t, origReq.Header.Get("X-Amz-Date")) + + // Perform the same request but with a header that shouldn't included in the + // signature; validate that the Authorization signature matches. + t.Run("Ignored Headers", func(t *testing.T) { + req, err := http.NewRequest(http.MethodPost, "google.com", strings.NewReader("Hello, world!")) + require.NoError(t, err) + + req.Header.Add("Uber-Trace-Id", "some-trace-id") + + _, err = cli.Do(req) + require.NoError(t, err) + require.NotNil(t, gotReq) + + require.Equal(t, origReq.Header.Get("Authorization"), gotReq.Header.Get("Authorization")) + }) +} diff --git a/storage/remote/write.go b/storage/remote/write.go index c62b143a94..4b75eca339 100644 --- a/storage/remote/write.go +++ b/storage/remote/write.go @@ -134,6 +134,7 @@ func (rws *WriteStorage) ApplyConfig(conf *config.Config) error { URL: rwConf.URL, Timeout: rwConf.RemoteTimeout, HTTPClientConfig: rwConf.HTTPClientConfig, + SigV4Config: rwConf.SigV4Config, Headers: rwConf.Headers, RetryOnRateLimit: rwConf.QueueConfig.RetryOnRateLimit, })