mirror of
https://github.com/hanzoai/traces.git
synced 2026-10-30 18:20:41 +00:00
Suraj Nathandzeekay
5e4cd1913d
* metrics-generator: Add per-label cardinality sanitizer Adds a per-label cardinality cap that replaces only the high-cardinality label value with "__overflow__" while preserving all other dimensions. Implementation: - CardinalitySanitizer: per-label HLL sketch tracking with cached overLimit flag updated every 15s. Always inserts original value hash to prevent estimate oscillation. - ChainSanitizer: composes sanitizers in order (DRAIN first, then cardinality sanitizer as hard backstop). - New config field: max_cardinality_per_label (0 = disabled). * metrics-generator: Add docs for per-label cardinality sanitizer Rename overflow value to __cardinality_overflow__ for clarity and add troubleshooting docs explaining the per-label cardinality limit, its PromQL query, configuration, and how it interacts with the series and entity limiters. * metrics-generator: Move max_cardinality_per_label to per-tenant override Move max_cardinality_per_label from static registry config to a per-tenant runtime override, consistent with max_active_series and max_active_entities. The CardinalitySanitizer now receives the Overrides interface and refreshes the cached value every maintenance tick (15s), allowing operators to tune it per tenant without restart. * move from piggybacking on sanitizer and have a standalone per label limiter sanitizer is there to sanitize the labels, so we shouldn't piggyback and rely on the chained sanitizer hack to ensure per label limiter runs after all sanitizer. we create a new LabelLimiter interface and run it after Sanitizer and decouple from Sanitizer * extract maxSeriesFunc and maxEntityFunc to type * test the overflow -> recovery -> overflow liftcycle * make the test more strict * rework docs * track label_names and add a demand estimate metric as well * Add tests for MetricsGeneratorMaxCardinalityPerLabel override * optimize hot path * Update TestPerLabelLimiter_ConcurrentAccess test * Add BenchmarkPerLabelLimiter to check perf and allocs * Add CHANGELOG.md * Update config index page * drop noopLabelLimiter from tests * always run s.doPeriodicMaintenance first to refresh the config value
122 lines
4.5 KiB
Go
122 lines
4.5 KiB
Go
package localserieslimiter
|
|
|
|
import (
|
|
"go.uber.org/atomic"
|
|
|
|
"github.com/grafana/tempo/modules/generator/registry"
|
|
tempo_log "github.com/grafana/tempo/pkg/util/log"
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
"github.com/prometheus/client_golang/prometheus/promauto"
|
|
"github.com/prometheus/prometheus/model/labels"
|
|
)
|
|
|
|
type localSeriesLimiterMetrics struct {
|
|
totalSeriesLimited *prometheus.CounterVec
|
|
activeSeries *prometheus.GaugeVec
|
|
maxActiveSeries *prometheus.GaugeVec
|
|
totalSeriesAdded *prometheus.CounterVec
|
|
totalSeriesRemoved *prometheus.CounterVec
|
|
}
|
|
|
|
func newMetrics(reg prometheus.Registerer) localSeriesLimiterMetrics {
|
|
return localSeriesLimiterMetrics{
|
|
totalSeriesLimited: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
|
|
Namespace: "tempo",
|
|
Name: "metrics_generator_registry_series_limited_total",
|
|
Help: "The total amount of series not created because of limits per tenant",
|
|
}, []string{"tenant"}),
|
|
activeSeries: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{
|
|
Namespace: "tempo",
|
|
Name: "metrics_generator_registry_active_series",
|
|
Help: "The active series per tenant",
|
|
}, []string{"tenant"}),
|
|
maxActiveSeries: promauto.With(reg).NewGaugeVec(prometheus.GaugeOpts{
|
|
Namespace: "tempo",
|
|
Name: "metrics_generator_registry_max_active_series",
|
|
Help: "The maximum active series per tenant",
|
|
}, []string{"tenant"}),
|
|
totalSeriesAdded: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
|
|
Namespace: "tempo",
|
|
Name: "metrics_generator_registry_series_added_total",
|
|
Help: "The total amount of series created per tenant",
|
|
}, []string{"tenant"}),
|
|
totalSeriesRemoved: promauto.With(reg).NewCounterVec(prometheus.CounterOpts{
|
|
Namespace: "tempo",
|
|
Name: "metrics_generator_registry_series_removed_total",
|
|
Help: "The total amount of series removed after they have become stale per tenant",
|
|
}, []string{"tenant"}),
|
|
}
|
|
}
|
|
|
|
var metrics = newMetrics(prometheus.DefaultRegisterer)
|
|
|
|
type maxSeriesFunc func(tenant string) uint32
|
|
|
|
type LocalSeriesLimiter struct {
|
|
tenant string
|
|
activeSeries atomic.Uint32
|
|
maxSeriesFunc maxSeriesFunc
|
|
limitLogger *tempo_log.RateLimitedLogger
|
|
metricTotalSeriesLimited prometheus.Counter
|
|
metricActiveSeries prometheus.Gauge
|
|
metricMaxActiveSeries prometheus.Gauge
|
|
metricTotalSeriesAdded prometheus.Counter
|
|
metricTotalSeriesRemoved prometheus.Counter
|
|
|
|
overflowEntity labels.Labels
|
|
overflowEntityHash uint64
|
|
}
|
|
|
|
var _ registry.Limiter = (*LocalSeriesLimiter)(nil)
|
|
|
|
func New(maxSeriesF maxSeriesFunc, tenant string, limitLogger *tempo_log.RateLimitedLogger) *LocalSeriesLimiter {
|
|
l := &LocalSeriesLimiter{
|
|
tenant: tenant,
|
|
maxSeriesFunc: maxSeriesF,
|
|
limitLogger: limitLogger,
|
|
metricTotalSeriesLimited: metrics.totalSeriesLimited.WithLabelValues(tenant),
|
|
metricActiveSeries: metrics.activeSeries.WithLabelValues(tenant),
|
|
metricMaxActiveSeries: metrics.maxActiveSeries.WithLabelValues(tenant),
|
|
metricTotalSeriesAdded: metrics.totalSeriesAdded.WithLabelValues(tenant),
|
|
metricTotalSeriesRemoved: metrics.totalSeriesRemoved.WithLabelValues(tenant),
|
|
|
|
overflowEntity: registry.OverflowEntity,
|
|
overflowEntityHash: registry.OverflowEntity.Hash(),
|
|
}
|
|
return l
|
|
}
|
|
|
|
func (l *LocalSeriesLimiter) OnAdd(hash uint64, seriesCount uint32, lbls labels.Labels) (labels.Labels, uint64) {
|
|
maxSeries := l.maxSeriesFunc(l.tenant)
|
|
|
|
if hash == l.overflowEntityHash {
|
|
// The overflow entity is always accepted.
|
|
return l.overflowEntity, l.overflowEntityHash
|
|
}
|
|
|
|
overLimit := maxSeries != 0 && l.activeSeries.Load()+seriesCount > maxSeries
|
|
if overLimit {
|
|
// We're at the limit, return overflow labels
|
|
l.metricTotalSeriesLimited.Add(float64(seriesCount))
|
|
l.limitLogger.Log("msg", "reached max active series", "active_series", l.activeSeries.Load(), "max_active_series", maxSeries)
|
|
return l.overflowEntity, l.overflowEntityHash
|
|
}
|
|
|
|
// We're under the limit, add the new series
|
|
l.activeSeries.Add(seriesCount)
|
|
l.metricActiveSeries.Set(float64(l.activeSeries.Load()))
|
|
l.metricMaxActiveSeries.Set(float64(maxSeries))
|
|
l.metricTotalSeriesAdded.Add(float64(seriesCount))
|
|
return lbls, hash
|
|
}
|
|
|
|
func (l *LocalSeriesLimiter) OnUpdate(uint64, uint32) {
|
|
// No-op, we rely on OnDelete to clean up the series
|
|
}
|
|
|
|
func (l *LocalSeriesLimiter) OnDelete(_ uint64, seriesCount uint32) {
|
|
l.activeSeries.Sub(seriesCount)
|
|
l.metricActiveSeries.Set(float64(l.activeSeries.Load()))
|
|
l.metricTotalSeriesRemoved.Add(float64(seriesCount))
|
|
}
|