collect cpu and trace profiles with the promtool debug command (#4897)
Signed-off-by: Krasi Georgiev <kgeorgie@redhat.com>
This commit is contained in:
Krasi Georgiev
GitHub
parent
f25a6baedb
commit
080e6ed31a
|
|
@ -14,112 +14,56 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"github.com/google/pprof/profile"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
type debugWriterConfig struct {
|
||||
serverURL string
|
||||
tarballName string
|
||||
pathToFileName map[string]string
|
||||
postProcess func(b []byte) ([]byte, error)
|
||||
endPointGroups []endpointsGroup
|
||||
}
|
||||
|
||||
type debugWriter struct {
|
||||
archiver
|
||||
httpClient
|
||||
requestToFile map[*http.Request]string
|
||||
postProcess func(b []byte) ([]byte, error)
|
||||
}
|
||||
|
||||
func newDebugWriter(cfg debugWriterConfig) (*debugWriter, error) {
|
||||
client, err := newPrometheusHTTPClient(cfg.serverURL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
func debugWrite(cfg debugWriterConfig) error {
|
||||
archiver, err := newTarGzFileWriter(cfg.tarballName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return errors.Wrap(err, "error creating a new archiver")
|
||||
}
|
||||
reqs := make(map[*http.Request]string)
|
||||
for path, filename := range cfg.pathToFileName {
|
||||
req, err := http.NewRequest(http.MethodGet, client.urlJoin(path), nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
reqs[req] = filename
|
||||
}
|
||||
return &debugWriter{
|
||||
archiver,
|
||||
client,
|
||||
reqs,
|
||||
cfg.postProcess,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (w *debugWriter) Write() int {
|
||||
for req, filename := range w.requestToFile {
|
||||
_, body, err := w.do(req)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error executing HTTP request:", err)
|
||||
return 1
|
||||
for _, endPointGroup := range cfg.endPointGroups {
|
||||
for url, filename := range endPointGroup.urlToFilename {
|
||||
url := cfg.serverURL + url
|
||||
fmt.Println("collecting:", url)
|
||||
res, err := http.Get(url)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error executing HTTP request")
|
||||
}
|
||||
body, err := ioutil.ReadAll(res.Body)
|
||||
res.Body.Close()
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error reading the response body")
|
||||
}
|
||||
|
||||
if endPointGroup.postProcess != nil {
|
||||
body, err = endPointGroup.postProcess(body)
|
||||
if err != nil {
|
||||
return errors.Wrap(err, "error post-processing HTTP response body")
|
||||
}
|
||||
}
|
||||
if err := archiver.write(filename, body); err != nil {
|
||||
return errors.Wrap(err, "error writing into the archive")
|
||||
}
|
||||
}
|
||||
|
||||
buf, err := w.postProcess(body)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error post-processing HTTP response body:", err)
|
||||
return 1
|
||||
}
|
||||
|
||||
if err := w.archiver.write(filename, buf); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error writing into archive:", err)
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
if err := w.close(); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error closing archiver:", err)
|
||||
return 1
|
||||
if err := archiver.close(); err != nil {
|
||||
return errors.Wrap(err, "error closing archive writer")
|
||||
}
|
||||
|
||||
fmt.Printf("Compiling debug information complete, all files written in %q.\n", w.filename())
|
||||
return 0
|
||||
}
|
||||
|
||||
func validate(b []byte) (*profile.Profile, error) {
|
||||
p, err := profile.Parse(bytes.NewReader(b))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
var pprofPostProcess = func(b []byte) ([]byte, error) {
|
||||
p, err := validate(b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := p.WriteUncompressed(&buf); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
fmt.Println(p.String())
|
||||
return buf.Bytes(), nil
|
||||
}
|
||||
|
||||
var metricsPostProcess = func(b []byte) ([]byte, error) {
|
||||
fmt.Println(string(b))
|
||||
return b, nil
|
||||
}
|
||||
|
||||
var allPostProcess = func(b []byte) ([]byte, error) {
|
||||
_, err := validate(b)
|
||||
if err != nil {
|
||||
return metricsPostProcess(b)
|
||||
}
|
||||
return pprofPostProcess(b)
|
||||
fmt.Printf("Compiling debug information complete, all files written in %q.\n", cfg.tarballName)
|
||||
return nil
|
||||
}
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
|
@ -27,6 +28,8 @@ import (
|
|||
|
||||
"gopkg.in/alecthomas/kingpin.v2"
|
||||
|
||||
"github.com/google/pprof/profile"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/prometheus/client_golang/api"
|
||||
"github.com/prometheus/client_golang/api/prometheus/v1"
|
||||
config_util "github.com/prometheus/common/config"
|
||||
|
|
@ -510,61 +513,85 @@ func parseTime(s string) (time.Time, error) {
|
|||
return time.Time{}, fmt.Errorf("cannot parse %q to a valid timestamp", s)
|
||||
}
|
||||
|
||||
func debugPprof(url string) int {
|
||||
w, err := newDebugWriter(debugWriterConfig{
|
||||
serverURL: url,
|
||||
tarballName: "debug.tar.gz",
|
||||
pathToFileName: map[string]string{
|
||||
"/debug/pprof/block": "block.pb",
|
||||
"/debug/pprof/goroutine": "goroutine.pb",
|
||||
"/debug/pprof/heap": "heap.pb",
|
||||
"/debug/pprof/mutex": "mutex.pb",
|
||||
"/debug/pprof/threadcreate": "threadcreate.pb",
|
||||
type endpointsGroup struct {
|
||||
urlToFilename map[string]string
|
||||
postProcess func(b []byte) ([]byte, error)
|
||||
}
|
||||
|
||||
var (
|
||||
pprofEndpoints = []endpointsGroup{
|
||||
{
|
||||
urlToFilename: map[string]string{
|
||||
"/debug/pprof/profile?seconds=30": "cpu.pb",
|
||||
"/debug/pprof/block": "block.pb",
|
||||
"/debug/pprof/goroutine": "goroutine.pb",
|
||||
"/debug/pprof/heap": "heap.pb",
|
||||
"/debug/pprof/mutex": "mutex.pb",
|
||||
"/debug/pprof/threadcreate": "threadcreate.pb",
|
||||
},
|
||||
postProcess: func(b []byte) ([]byte, error) {
|
||||
p, err := profile.Parse(bytes.NewReader(b))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var buf bytes.Buffer
|
||||
if err := p.WriteUncompressed(&buf); err != nil {
|
||||
return nil, errors.Wrap(err, "writing the profile to the buffer")
|
||||
}
|
||||
|
||||
return buf.Bytes(), nil
|
||||
},
|
||||
},
|
||||
postProcess: pprofPostProcess,
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error creating debug writer:", err)
|
||||
{
|
||||
urlToFilename: map[string]string{
|
||||
"/debug/pprof/trace?seconds=30": "trace.pb",
|
||||
},
|
||||
},
|
||||
}
|
||||
metricsEndpoints = []endpointsGroup{
|
||||
{
|
||||
urlToFilename: map[string]string{
|
||||
"/metrics": "metrics.txt",
|
||||
},
|
||||
},
|
||||
}
|
||||
allEndpoints = append(pprofEndpoints, metricsEndpoints...)
|
||||
)
|
||||
|
||||
func debugPprof(url string) int {
|
||||
if err := debugWrite(debugWriterConfig{
|
||||
serverURL: url,
|
||||
tarballName: "debug.tar.gz",
|
||||
endPointGroups: pprofEndpoints,
|
||||
}); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error completing debug command:", err)
|
||||
return 1
|
||||
}
|
||||
return w.Write()
|
||||
return 0
|
||||
}
|
||||
|
||||
func debugMetrics(url string) int {
|
||||
w, err := newDebugWriter(debugWriterConfig{
|
||||
serverURL: url,
|
||||
tarballName: "debug.tar.gz",
|
||||
pathToFileName: map[string]string{
|
||||
"/metrics": "metrics.txt",
|
||||
},
|
||||
postProcess: metricsPostProcess,
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error creating debug writer:", err)
|
||||
if err := debugWrite(debugWriterConfig{
|
||||
serverURL: url,
|
||||
tarballName: "debug.tar.gz",
|
||||
endPointGroups: metricsEndpoints,
|
||||
}); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error completing debug command:", err)
|
||||
return 1
|
||||
}
|
||||
return w.Write()
|
||||
return 0
|
||||
}
|
||||
|
||||
func debugAll(url string) int {
|
||||
w, err := newDebugWriter(debugWriterConfig{
|
||||
serverURL: url,
|
||||
tarballName: "debug.tar.gz",
|
||||
pathToFileName: map[string]string{
|
||||
"/debug/pprof/block": "block.pb",
|
||||
"/debug/pprof/goroutine": "goroutine.pb",
|
||||
"/debug/pprof/heap": "heap.pb",
|
||||
"/debug/pprof/mutex": "mutex.pb",
|
||||
"/debug/pprof/threadcreate": "threadcreate.pb",
|
||||
"/metrics": "metrics.txt",
|
||||
},
|
||||
postProcess: allPostProcess,
|
||||
})
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error creating debug writer:", err)
|
||||
if err := debugWrite(debugWriterConfig{
|
||||
serverURL: url,
|
||||
tarballName: "debug.tar.gz",
|
||||
endPointGroups: allEndpoints,
|
||||
}); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error completing debug command:", err)
|
||||
return 1
|
||||
}
|
||||
return w.Write()
|
||||
return 0
|
||||
}
|
||||
|
||||
type printer interface {
|
||||
|
|
@ -583,7 +610,7 @@ func (p *promqlPrinter) printSeries(val []model.LabelSet) {
|
|||
fmt.Println(v)
|
||||
}
|
||||
}
|
||||
func (j *promqlPrinter) printLabelValues(val model.LabelValues) {
|
||||
func (p *promqlPrinter) printLabelValues(val model.LabelValues) {
|
||||
for _, v := range val {
|
||||
fmt.Println(v)
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue