prometheus/retrieval/scheduler.go

119 lines
4.3 KiB
Go

// Copyright 2013 Prometheus Team
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package retrieval
import (
"github.com/prometheus/prometheus/utility"
"math"
"time"
)
const (
// The default increment for exponential backoff when querying a target.
DEFAULT_BACKOFF_VALUE = 2
// The base units for the exponential backoff.
DEFAULT_BACKOFF_VALUE_UNIT = time.Second
// The maximum allowed backoff time.
MAXIMUM_BACKOFF_VALUE = 30 * time.Minute
)
// scheduler is an interface that various scheduling strategies must fulfill
// in order to set the scheduling order for a target.
//
// Target takes advantage of this type by embedding an instance of scheduler
// in each Target instance itself. The emitted scheduler.ScheduledFor() is
// the basis for sorting the order of pending queries.
//
// This type is described as an interface to maximize testability.
type scheduler interface {
// ScheduledFor emits the earliest time at which the given object is allowed
// to be run. This time may or not be a reflection of the earliest parameter
// provided in Reschedule; that is up to the underlying strategy
// implementations.
ScheduledFor() time.Time
// Instruct the scheduled item to re-schedule itself given new state data and
// the earliest time at which the outside system thinks the operation should
// be scheduled for.
Reschedule(earliest time.Time, future TargetState)
}
// healthScheduler is an implementation of scheduler that uses health data
// provided by the target field as well as unreachability counts to determine
// when to next schedule an operation.
//
// The type is almost capable of being used with default initialization, except
// that a target field must be provided for which the system compares current
// health against future proposed values.
type healthScheduler struct {
scheduledFor time.Time
target healthReporter
time utility.Time
unreachableCount int
}
func (s healthScheduler) ScheduledFor() time.Time {
return s.scheduledFor
}
// Reschedule, like the protocol described in scheduler, uses the current and
// proposed future health state to determine how and when a given subject is to
// be scheduled.
//
// If a subject has been at given moment marked as unhealthy, an exponential
// backoff scheme is applied to it. The reason for this backoff is to ensure
// that known-healthy targets can consume valuable request queuing resources
// first. Depending on the retrieval interval and number of consecutive
// unhealthy markings, the query of these unhealthy individuals may come before
// the healthy ones for a short time to help ensure expeditious retrieval.
// The inflection point that drops these to the back of the queue is beneficial
// to save resources in the long-run.
//
// If a subject is healthy, its next scheduling opportunity is set to
// earliest, for this ensures fair querying of all remaining healthy targets and
// removes bias in the ordering. In order for the anti-bias value to have any
// value, the earliest opportunity should be set to a value that is constant
// for a given batch of subjects who are to be scraped on a given interval.
func (s *healthScheduler) Reschedule(e time.Time, f TargetState) {
currentState := s.target.State()
// XXX: Handle metrics surrounding health.
switch currentState {
case UNKNOWN, UNREACHABLE:
switch f {
case ALIVE:
s.unreachableCount = 0
break
case UNREACHABLE:
s.unreachableCount++
break
}
case ALIVE:
switch f {
case UNREACHABLE:
s.unreachableCount++
}
}
if s.unreachableCount == 0 {
s.scheduledFor = e
} else {
backoff := MAXIMUM_BACKOFF_VALUE
exponential := time.Duration(math.Pow(DEFAULT_BACKOFF_VALUE, float64(s.unreachableCount))) * DEFAULT_BACKOFF_VALUE_UNIT
if backoff > exponential {
backoff = exponential
}
s.scheduledFor = s.time.Now().Add(backoff)
}
}