2015-01-21 19:07:45 +00:00
|
|
|
// Copyright 2014 The Prometheus Authors
|
2014-10-23 13:18:32 +00:00
|
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
// you may not use this file except in compliance with the License.
|
|
|
|
// You may obtain a copy of the License at
|
|
|
|
//
|
|
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
//
|
|
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
// See the License for the specific language governing permissions and
|
|
|
|
// limitations under the License.
|
|
|
|
|
|
|
|
package local
|
|
|
|
|
|
|
|
import "github.com/prometheus/client_golang/prometheus"
|
|
|
|
|
|
|
|
// Usually, a separate file for instrumentation is frowned upon. Metrics should
|
2014-11-20 20:03:51 +00:00
|
|
|
// be close to where they are used. However, the metrics below are set all over
|
2014-10-23 13:18:32 +00:00
|
|
|
// the place, so we go for a separate instrumentation file in this case.
|
|
|
|
var (
|
|
|
|
chunkOps = prometheus.NewCounterVec(
|
|
|
|
prometheus.CounterOpts{
|
|
|
|
Namespace: namespace,
|
|
|
|
Subsystem: subsystem,
|
|
|
|
Name: "chunk_ops_total",
|
|
|
|
Help: "The total number of chunk operations by their type.",
|
|
|
|
},
|
|
|
|
[]string{opTypeLabel},
|
|
|
|
)
|
|
|
|
chunkDescOps = prometheus.NewCounterVec(
|
|
|
|
prometheus.CounterOpts{
|
|
|
|
Namespace: namespace,
|
|
|
|
Subsystem: subsystem,
|
|
|
|
Name: "chunkdesc_ops_total",
|
|
|
|
Help: "The total number of chunk descriptor operations by their type.",
|
|
|
|
},
|
|
|
|
[]string{opTypeLabel},
|
|
|
|
)
|
2014-11-27 17:25:03 +00:00
|
|
|
numMemChunkDescs = prometheus.NewGauge(prometheus.GaugeOpts{
|
|
|
|
Namespace: namespace,
|
|
|
|
Subsystem: subsystem,
|
|
|
|
Name: "memory_chunkdescs",
|
|
|
|
Help: "The current number of chunk descriptors in memory.",
|
|
|
|
})
|
2014-10-23 13:18:32 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
const (
|
|
|
|
namespace = "prometheus"
|
|
|
|
subsystem = "local_storage"
|
|
|
|
|
|
|
|
opTypeLabel = "type"
|
|
|
|
|
|
|
|
// Op-types for seriesOps.
|
2014-11-12 16:12:57 +00:00
|
|
|
create = "create"
|
|
|
|
archive = "archive"
|
|
|
|
unarchive = "unarchive"
|
|
|
|
memoryPurge = "purge_from_memory"
|
|
|
|
archivePurge = "purge_from_archive"
|
2016-01-11 16:22:16 +00:00
|
|
|
requestedPurge = "purge_on_request"
|
2014-11-12 16:12:57 +00:00
|
|
|
memoryMaintenance = "maintenance_in_memory"
|
|
|
|
archiveMaintenance = "maintenance_in_archive"
|
Handle errors caused by data corruption more gracefully
This requires all the panic calls upon unexpected data to be converted
into errors returned. This pollute the function signatures quite
lot. Well, this is Go...
The ideas behind this are the following:
- panic only if it's a programming error. Data corruptions happen, and
they are not programming errors.
- If we detect a data corruption, we "quarantine" the series,
essentially removing it from the database and putting its data into
a separate directory for forensics.
- Failure during writing to a series file is not considered corruption
automatically. It will call setDirty, though, so that a
crashrecovery upon the next restart will commence and check for
that.
- Series quarantining and setDirty calls are logged and counted in
metrics, but are hidden from the user of the interfaces in
interface.go, whith the notable exception of Append(). The reasoning
is that we treat corruption by removing the corrupted series, i.e. a
query for it will return no results on its next call anyway, so
return no results right now. In the case of Append(), we want to
tell the user that no data has been appended, though.
Minor side effects:
- Now consistently using filepath.* instead of path.*.
- Introduced structured logging where I touched it. This makes things
less consistent, but a complete change to structured logging would
be out of scope for this PR.
2016-02-25 11:23:42 +00:00
|
|
|
completedQurantine = "quarantine_completed"
|
|
|
|
droppedQuarantine = "quarantine_dropped"
|
|
|
|
failedQuarantine = "quarantine_failed"
|
2014-10-23 13:18:32 +00:00
|
|
|
|
|
|
|
// Op-types for chunkOps.
|
|
|
|
createAndPin = "create" // A chunkDesc creation with refCount=1.
|
|
|
|
persistAndUnpin = "persist"
|
|
|
|
pin = "pin" // Excluding the pin on creation.
|
|
|
|
unpin = "unpin" // Excluding the unpin on persisting.
|
|
|
|
clone = "clone"
|
|
|
|
transcode = "transcode"
|
2015-02-26 14:19:44 +00:00
|
|
|
drop = "drop"
|
2014-10-23 13:18:32 +00:00
|
|
|
|
|
|
|
// Op-types for chunkOps and chunkDescOps.
|
|
|
|
evict = "evict"
|
|
|
|
load = "load"
|
2015-03-19 16:06:16 +00:00
|
|
|
|
|
|
|
seriesLocationLabel = "location"
|
|
|
|
|
|
|
|
// Maintenance types for maintainSeriesDuration.
|
|
|
|
maintainInMemory = "memory"
|
|
|
|
maintainArchived = "archived"
|
2016-04-25 14:43:52 +00:00
|
|
|
|
|
|
|
discardReasonLabel = "reason"
|
|
|
|
|
|
|
|
// Reasons to discard samples.
|
|
|
|
outOfOrderTimestamp = "timestamp_out_of_order"
|
|
|
|
duplicateSample = "multiple_values_for_timestamp"
|
2014-10-23 13:18:32 +00:00
|
|
|
)
|
|
|
|
|
|
|
|
func init() {
|
|
|
|
prometheus.MustRegister(chunkOps)
|
|
|
|
prometheus.MustRegister(chunkDescOps)
|
2014-11-27 17:25:03 +00:00
|
|
|
prometheus.MustRegister(numMemChunkDescs)
|
2014-10-23 13:18:32 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
var (
|
2014-11-27 17:25:03 +00:00
|
|
|
// Global counter, also used internally, so not implemented as
|
2016-06-29 06:14:23 +00:00
|
|
|
// metrics. Collected in MemorySeriesStorage.Collect.
|
2015-07-15 17:53:15 +00:00
|
|
|
// TODO(beorn7): As it is used internally, it is actually very bad style
|
|
|
|
// to have it as a global variable.
|
2014-11-27 17:25:03 +00:00
|
|
|
numMemChunks int64
|
2014-10-23 13:18:32 +00:00
|
|
|
|
|
|
|
// Metric descriptors for the above.
|
|
|
|
numMemChunksDesc = prometheus.NewDesc(
|
|
|
|
prometheus.BuildFQName(namespace, subsystem, "memory_chunks"),
|
|
|
|
"The current number of chunks in memory, excluding cloned chunks (i.e. chunks without a descriptor).",
|
|
|
|
nil, nil,
|
|
|
|
)
|
|
|
|
)
|