Files
Martin DisibioandGitHub 2679389f06 [Rhythm] Improve metrics generator + Kafka performance and stability (#4721)
* Add new concurrency config options and validate

* Reduce allocations of IDmap when replaying wal blocks since we know the number of traces

* Generator read from kafka concurrency, add shared ingest lag metric, option to uniqify strings

* Memoize spanmetrics sanitizelabelname, move to better location

* Moved to shared queue for localblocks wal completion, allow concurrency within a tenant

* Honor max live traces in non-flushing local blocks processor

* Add metric for enqueue time

* Add missing mutex lock for enqueuing on replay

* Fix mutex lock while reloading blocks

* Increase default concurrency

* Simplify local blocks complete queue

* Remove uniqify, fix test

* Switch completequeue to reference counting and shut it down when last local blocks processor is stopped

* cache rename/cleanup

* Cleanup/denoising PR

* Lint/cleanup

* lint

* Update config manifest

* fix race condition

* cleanup

* review feedback

* Update config manifest

* changelog
2025-02-21 11:18:42 -05:00

99 lines
3.3 KiB
Go

package localblocks
import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
)
const (
namespace = "tempo"
subsystem = "metrics_generator_processor_local_blocks"
reasonLiveTracesExceeded = "live_traces_exceeded"
reasonTraceSizeExceeded = "trace_too_large"
reasonWaitingForLiveTraces = "waiting_for_live_traces"
reasonWaitingForWAL = "waiting_for_wal"
)
var (
metricTotalTraces = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "traces_total",
Help: "Total number of traces created",
}, []string{"tenant"})
metricTotalSpans = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "spans_total",
Help: "Total number of spans after filtering",
}, []string{"tenant"})
metricDroppedSpans = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "spans_dropped_total",
Help: "Number of spans dropped",
}, []string{"tenant", "reason"})
metricLiveTraces = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "live_traces",
Help: "Number of live traces",
}, []string{"tenant"})
metricLiveTraceBytes = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "live_trace_bytes",
Help: "Total number of traces created",
}, []string{"tenant"})
metricDroppedTraces = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "traces_dropped_total",
Help: "Number of traces dropped",
}, []string{"tenant", "reason"})
metricBlockSize = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "bytes",
Help: "Total size of local blocks",
}, []string{"tenant"})
metricCompletedBlocks = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "completed_blocks",
Help: "Number of blocks completed by the local blocks processor",
}, []string{"tenant"})
metricCutBlocks = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "cut_blocks",
Help: "Number of blocks cut by the local blocks processor",
}, []string{"tenant"})
metricFlushedBlocks = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "flushed_blocks",
Help: "Number of blocks flushed by the local blocks processor",
}, []string{"tenant"})
metricFlushQueueSize = promauto.NewGaugeVec(prometheus.GaugeOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "flush_queue_size",
Help: "Size of the flush queue",
}, []string{"tenant"})
metricFailedFlushes = promauto.NewCounter(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "failed_flushes_total",
Help: "The total number of failed flushes",
})
metricBackPressure = promauto.NewCounterVec(prometheus.CounterOpts{
Namespace: namespace,
Subsystem: subsystem,
Name: "back_pressure_seconds_total",
Help: "The total amount of time spent waiting to process data from queue",
}, []string{"reason"})
)