mirror of
https://github.com/hanzoai/traces.git
synced 2026-10-30 18:20:41 +00:00
Suraj Nathandzeekay
5e4cd1913d
* metrics-generator: Add per-label cardinality sanitizer Adds a per-label cardinality cap that replaces only the high-cardinality label value with "__overflow__" while preserving all other dimensions. Implementation: - CardinalitySanitizer: per-label HLL sketch tracking with cached overLimit flag updated every 15s. Always inserts original value hash to prevent estimate oscillation. - ChainSanitizer: composes sanitizers in order (DRAIN first, then cardinality sanitizer as hard backstop). - New config field: max_cardinality_per_label (0 = disabled). * metrics-generator: Add docs for per-label cardinality sanitizer Rename overflow value to __cardinality_overflow__ for clarity and add troubleshooting docs explaining the per-label cardinality limit, its PromQL query, configuration, and how it interacts with the series and entity limiters. * metrics-generator: Move max_cardinality_per_label to per-tenant override Move max_cardinality_per_label from static registry config to a per-tenant runtime override, consistent with max_active_series and max_active_entities. The CardinalitySanitizer now receives the Overrides interface and refreshes the cached value every maintenance tick (15s), allowing operators to tune it per tenant without restart. * move from piggybacking on sanitizer and have a standalone per label limiter sanitizer is there to sanitize the labels, so we shouldn't piggyback and rely on the chained sanitizer hack to ensure per label limiter runs after all sanitizer. we create a new LabelLimiter interface and run it after Sanitizer and decouple from Sanitizer * extract maxSeriesFunc and maxEntityFunc to type * test the overflow -> recovery -> overflow liftcycle * make the test more strict * rework docs * track label_names and add a demand estimate metric as well * Add tests for MetricsGeneratorMaxCardinalityPerLabel override * optimize hot path * Update TestPerLabelLimiter_ConcurrentAccess test * Add BenchmarkPerLabelLimiter to check perf and allocs * Add CHANGELOG.md * Update config index page * drop noopLabelLimiter from tests * always run s.doPeriodicMaintenance first to refresh the config value
85 lines
2.2 KiB
Go
85 lines
2.2 KiB
Go
package registry
|
|
|
|
import (
|
|
"sync"
|
|
|
|
"github.com/prometheus/common/model"
|
|
"github.com/prometheus/prometheus/model/labels"
|
|
)
|
|
|
|
type safeBuilderPool struct {
|
|
pool sync.Pool
|
|
}
|
|
|
|
func newSafeBuilderPool() *safeBuilderPool {
|
|
return &safeBuilderPool{
|
|
pool: sync.Pool{
|
|
New: func() interface{} {
|
|
return labels.NewBuilder(labels.New())
|
|
},
|
|
},
|
|
}
|
|
}
|
|
|
|
func (p *safeBuilderPool) Get() *labels.Builder {
|
|
return p.pool.Get().(*labels.Builder)
|
|
}
|
|
|
|
func (p *safeBuilderPool) Put(builder *labels.Builder) {
|
|
builder.Reset(labels.New())
|
|
p.pool.Put(builder)
|
|
}
|
|
|
|
var builderPool = newSafeBuilderPool()
|
|
|
|
type labelBuilder struct {
|
|
builder *labels.Builder
|
|
sanitizer Sanitizer
|
|
perLabelLimiter LabelLimiter
|
|
|
|
maxLabelNameLength int
|
|
maxLabelValueLength int
|
|
}
|
|
|
|
var _ LabelBuilder = (*labelBuilder)(nil)
|
|
|
|
func NewLabelBuilder(maxLabelNameLength int, maxLabelValueLength int, sanitizer Sanitizer, perLabelLimiter LabelLimiter) LabelBuilder {
|
|
builder := builderPool.Get()
|
|
return &labelBuilder{
|
|
builder: builder,
|
|
sanitizer: sanitizer,
|
|
perLabelLimiter: perLabelLimiter,
|
|
maxLabelNameLength: maxLabelNameLength,
|
|
maxLabelValueLength: maxLabelValueLength,
|
|
}
|
|
}
|
|
|
|
func (b *labelBuilder) Add(name, value string) {
|
|
if b.maxLabelNameLength > 0 && len(name) > b.maxLabelNameLength {
|
|
name = name[:b.maxLabelNameLength]
|
|
}
|
|
if b.maxLabelValueLength > 0 && len(value) > b.maxLabelValueLength {
|
|
value = value[:b.maxLabelValueLength]
|
|
}
|
|
b.builder.Set(name, value)
|
|
}
|
|
|
|
func (b *labelBuilder) CloseAndBuildLabels() (labels.Labels, bool) {
|
|
// We always run sanitizer first and then run per label limiter to ensure that
|
|
// per label limits are always applied after sanitizer.
|
|
// Pipeline: sanitize labels --> per-label cardinality limit --> entity/series limit
|
|
lbls := b.sanitizer.Sanitize(b.builder.Labels())
|
|
lbls = b.perLabelLimiter.Limit(lbls)
|
|
// it's no longer safe to use the builder after this point, so we drop our
|
|
// reference to it. this may cause a nil panic if the builder is used after
|
|
// this point, but it's better than memory corruption.
|
|
builderPool.Put(b.builder)
|
|
b.builder = nil
|
|
|
|
if !lbls.IsValid(model.UTF8Validation) {
|
|
return lbls, false
|
|
}
|
|
|
|
return lbls, true
|
|
}
|