add output dir parameter

Signed-off-by: jessicagreben <jessicagrebens@gmail.com>
This commit is contained in:
jessicagreben 2020-09-13 08:38:32 -07:00
parent dfa510086b
commit 2e526cf2a7
2 changed files with 11 additions and 16 deletions

View File

@ -135,6 +135,7 @@ func main() {
Required().String()
backfillRuleEnd := backfillRuleCmd.Flag("end", "If an end time is provided, all recording rules in the rule files provided will be backfilled to the end time. Default will backfill up to 3 hrs ago. End time should be RFC3339 or Unix timestamp.").
Default("-3h").String()
backfillOutputDir := backfillRuleCmd.Flag("output dir", "The filepath on the local filesystem to write the output to. Output will be blocks containing the data of the backfilled recording rules.").Default("backfilldata/").String()
backfillRuleURL := backfillRuleCmd.Flag("url", "Prometheus API url with the data where the rule will be backfilled from.").Default("localhost:9090").String()
backfillRuleEvalInterval := backfillRuleCmd.Flag("evaluation_interval", "How frequently to evaluate rules when backfilling.").
Default("15s").Duration()
@ -200,7 +201,7 @@ func main() {
os.Exit(checkErr(dumpSamples(*dumpPath, *dumpMinTime, *dumpMaxTime)))
case backfillRuleCmd.FullCommand():
os.Exit(BackfillRule(*backfillRuleURL, *backfillRuleStart, *backfillRuleEnd, *backfillRuleEvalInterval, *backfillRuleFiles...))
os.Exit(BackfillRule(*backfillRuleURL, *backfillRuleStart, *backfillRuleEnd, *backfillOutputDir, *backfillRuleEvalInterval, *backfillRuleFiles...))
}
}
@ -766,8 +767,9 @@ func (j *jsonPrinter) printLabelValues(v model.LabelValues) {
json.NewEncoder(os.Stdout).Encode(v)
}
// BackfillRule backfills rules from the files provided.
func BackfillRule(url, start, end string, evalInterval time.Duration, files ...string) int {
// BackfillRule backfills recording rules from the files provided. The output are blocks of data
// at the outputDir location.
func BackfillRule(url, start, end, outputDir string, evalInterval time.Duration, files ...string) int {
ctx := context.Background()
stime, etime, err := parseStartTimeAndEndTime(start, end)
if err != nil {
@ -777,6 +779,7 @@ func BackfillRule(url, start, end string, evalInterval time.Duration, files ...s
cfg := RuleImporterConfig{
Start: stime,
End: etime,
OutputDir: outputDir,
EvalInterval: evalInterval,
URL: url,
}

View File

@ -16,7 +16,6 @@ package main
import (
"context"
"fmt"
"io/ioutil"
"os"
"time"
@ -49,6 +48,7 @@ type RuleImporter struct {
type RuleImporterConfig struct {
Start time.Time
End time.Time
OutputDir string
EvalInterval time.Duration
URL string
}
@ -64,12 +64,10 @@ func NewRuleImporter(logger log.Logger, config RuleImporterConfig) *RuleImporter
// Init initializes the rule importer which creates a new block writer
// and creates an Prometheus API client.
func (importer *RuleImporter) Init() error {
// todo: clean up dir
newBlockDir, err := ioutil.TempDir("", "rule_blocks")
if err != nil {
return err
}
importer.writer = blocks.NewMultiWriter(importer.logger, newBlockDir, importer.config.EvalInterval.Nanoseconds())
importer.writer = blocks.NewMultiWriter(importer.logger,
importer.config.OutputDir,
importer.config.EvalInterval.Nanoseconds(),
)
config := api.Config{
Address: importer.config.URL,
@ -84,7 +82,6 @@ func (importer *RuleImporter) Init() error {
// Close cleans up any open resources.
func (importer *RuleImporter) Close() error {
// todo: clean up any dirs that were created
return importer.writer.Close()
}
@ -99,21 +96,17 @@ func (importer *RuleImporter) LoadGroups(ctx context.Context, filenames []string
}
for _, ruleGroup := range rgs.Groups {
itv := importer.config.EvalInterval
if ruleGroup.Interval != 0 {
itv = time.Duration(ruleGroup.Interval)
}
rgRules := make([]rules.Rule, 0, len(ruleGroup.Rules))
for _, r := range ruleGroup.Rules {
expr, err := importer.groupLoader.Parse(r.Expr.Value)
if err != nil {
return []error{errors.Wrap(err, filename)}
}
rgRules = append(rgRules, rules.NewRecordingRule(
r.Record.Value,
expr,
@ -158,7 +151,6 @@ func (importer *RuleImporter) ImportAll(ctx context.Context) []error {
// ImportRule imports the historical data for a single rule.
func (importer *RuleImporter) ImportRule(ctx context.Context, ruleExpr string, stimeWithAlignment time.Time, internval time.Duration) error {
ts := stimeWithAlignment
appender := importer.writer.Appender()
for ts.Before(importer.config.End) {