150 lines
4.3 KiB
Go
150 lines
4.3 KiB
Go
// Copyright 2013 Prometheus Team
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package remote
|
|
|
|
import (
|
|
"time"
|
|
|
|
"github.com/golang/glog"
|
|
|
|
clientmodel "github.com/prometheus/client_golang/model"
|
|
)
|
|
|
|
const (
|
|
// The maximum number of concurrent send requests to the TSDB.
|
|
maxConcurrentSends = 10
|
|
// The maximum number of samples to fit into a single request to the TSDB.
|
|
maxSamplesPerSend = 100
|
|
// The deadline after which to send queued samples even if the maximum batch
|
|
// size has not been reached.
|
|
batchSendDeadline = 5 * time.Second
|
|
)
|
|
|
|
// TSDBClient defines an interface for sending a batch of samples to an
|
|
// external timeseries database (TSDB).
|
|
type TSDBClient interface {
|
|
Store(clientmodel.Samples) error
|
|
}
|
|
|
|
// TSDBQueueManager manages a queue of samples to be sent to the TSDB indicated
|
|
// by the provided TSDBClient.
|
|
type TSDBQueueManager struct {
|
|
tsdb TSDBClient
|
|
queue chan clientmodel.Samples
|
|
pendingSamples clientmodel.Samples
|
|
sendSemaphore chan bool
|
|
drained chan bool
|
|
}
|
|
|
|
// NewTSDBQueueManager builds a new TSDBQueueManager.
|
|
func NewTSDBQueueManager(tsdb TSDBClient, queueCapacity int) *TSDBQueueManager {
|
|
return &TSDBQueueManager{
|
|
tsdb: tsdb,
|
|
queue: make(chan clientmodel.Samples, queueCapacity),
|
|
sendSemaphore: make(chan bool, maxConcurrentSends),
|
|
drained: make(chan bool),
|
|
}
|
|
}
|
|
|
|
// Queue queues a sample batch to be sent to the TSDB. It drops the most
|
|
// recently queued samples on the floor if the queue is full.
|
|
func (t *TSDBQueueManager) Queue(s clientmodel.Samples) {
|
|
select {
|
|
case t.queue <- s:
|
|
default:
|
|
samplesCount.IncrementBy(map[string]string{result: dropped}, float64(len(s)))
|
|
glog.Warningf("TSDB queue full, discarding %d samples", len(s))
|
|
}
|
|
}
|
|
|
|
func (t *TSDBQueueManager) sendSamples(s clientmodel.Samples) {
|
|
t.sendSemaphore <- true
|
|
defer func() {
|
|
<-t.sendSemaphore
|
|
}()
|
|
|
|
// Samples are sent to the TSDB on a best-effort basis. If a sample isn't
|
|
// sent correctly the first time, it's simply dropped on the floor.
|
|
begin := time.Now()
|
|
err := t.tsdb.Store(s)
|
|
recordOutcome(time.Since(begin), len(s), err)
|
|
|
|
if err != nil {
|
|
glog.Warningf("error sending %d samples to TSDB: %s", len(s), err)
|
|
}
|
|
}
|
|
|
|
// reportQueues reports notification queue occupancy and capacity.
|
|
func (t *TSDBQueueManager) reportQueues() {
|
|
queueSize.Set(map[string]string{facet: occupancy}, float64(len(t.queue)))
|
|
queueSize.Set(map[string]string{facet: capacity}, float64(cap(t.queue)))
|
|
}
|
|
|
|
// Run continuously sends samples to the TSDB.
|
|
func (t *TSDBQueueManager) Run() {
|
|
defer func() {
|
|
close(t.drained)
|
|
}()
|
|
|
|
queueReportTicker := time.NewTicker(time.Second)
|
|
go func() {
|
|
for _ = range queueReportTicker.C {
|
|
t.reportQueues()
|
|
}
|
|
}()
|
|
defer queueReportTicker.Stop()
|
|
|
|
// Send batches of at most maxSamplesPerSend samples to the TSDB. If we
|
|
// have fewer samples than that, flush them out after a deadline anyways.
|
|
for {
|
|
select {
|
|
case s, ok := <-t.queue:
|
|
if !ok {
|
|
glog.Infof("Flushing %d samples to OpenTSDB...", len(t.pendingSamples))
|
|
t.flush()
|
|
glog.Infof("Done flushing.")
|
|
return
|
|
}
|
|
|
|
t.pendingSamples = append(t.pendingSamples, s...)
|
|
|
|
for len(t.pendingSamples) >= maxSamplesPerSend {
|
|
go t.sendSamples(t.pendingSamples[:maxSamplesPerSend])
|
|
t.pendingSamples = t.pendingSamples[maxSamplesPerSend:]
|
|
}
|
|
case <-time.After(batchSendDeadline):
|
|
t.flush()
|
|
}
|
|
}
|
|
}
|
|
|
|
// Flush flushes remaining queued samples.
|
|
func (t *TSDBQueueManager) flush() {
|
|
if len(t.pendingSamples) > 0 {
|
|
go t.sendSamples(t.pendingSamples)
|
|
}
|
|
t.pendingSamples = t.pendingSamples[:0]
|
|
}
|
|
|
|
// Close stops sending samples to the TSDB and waits for pending sends to
|
|
// complete.
|
|
func (t *TSDBQueueManager) Close() {
|
|
glog.Infof("TSDB queue manager shutting down...")
|
|
close(t.queue)
|
|
<-t.drained
|
|
for i := 0; i < maxConcurrentSends; i++ {
|
|
t.sendSemaphore <- true
|
|
}
|
|
}
|