mirror of
https://github.com/hanzoai/traces.git
synced 2026-10-30 18:20:41 +00:00
* [Generator] Produce overflow series when limiting This PR implements an "overflow series" mechanism when the entity limit is hit. To avoid bike shedding, I adopted the spec from otel's metrics sdk, since it has a similar mechanism. https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/sdk.md#cardinality-limits * Simplify how overflow entities are tracked * Move label sanitization into OnAdd * Remove demand tracker from builder * Remove unused type * Add overflow entity * Fix linter * Fix integration tests * Add changelog entry for overflow series
165 lines
3.5 KiB
Go
165 lines
3.5 KiB
Go
package registry
|
|
|
|
import (
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/prometheus/prometheus/model/labels"
|
|
"github.com/prometheus/prometheus/storage"
|
|
"go.uber.org/atomic"
|
|
)
|
|
|
|
var _ metric = (*gauge)(nil)
|
|
|
|
// this is mostly copied from counter
|
|
type gauge struct {
|
|
//nolint unused
|
|
metric
|
|
|
|
metricName string
|
|
|
|
// seriesMtx is used to sync modifications to the map, not to the data in series
|
|
seriesMtx sync.RWMutex
|
|
series map[uint64]*gaugeSeries
|
|
seriesDemand *Cardinality
|
|
|
|
lifecycler Limiter
|
|
|
|
externalLabels map[string]string
|
|
}
|
|
|
|
type gaugeSeries struct {
|
|
labels labels.Labels
|
|
value *atomic.Float64
|
|
lastUpdated *atomic.Int64
|
|
}
|
|
|
|
var (
|
|
_ Gauge = (*gauge)(nil)
|
|
_ metric = (*gauge)(nil)
|
|
)
|
|
|
|
const (
|
|
add = "add"
|
|
set = "set"
|
|
)
|
|
|
|
func newGauge(name string, lifecycler Limiter, externalLabels map[string]string, staleDuration time.Duration) *gauge {
|
|
return &gauge{
|
|
metricName: name,
|
|
series: make(map[uint64]*gaugeSeries),
|
|
seriesDemand: NewCardinality(staleDuration, removeStaleSeriesInterval),
|
|
lifecycler: lifecycler,
|
|
externalLabels: externalLabels,
|
|
}
|
|
}
|
|
|
|
func (g *gauge) Set(lbls labels.Labels, value float64) {
|
|
g.updateSeries(lbls, value, set, true)
|
|
}
|
|
|
|
func (g *gauge) Inc(lbls labels.Labels, value float64) {
|
|
g.updateSeries(lbls, value, add, true)
|
|
}
|
|
|
|
func (g *gauge) SetForTargetInfo(lbls labels.Labels, value float64) {
|
|
g.updateSeries(lbls, value, set, false)
|
|
}
|
|
|
|
func (g *gauge) updateSeries(lbls labels.Labels, value float64, operation string, updateIfAlreadyExist bool) {
|
|
hash := lbls.Hash()
|
|
|
|
g.seriesMtx.RLock()
|
|
s, ok := g.series[hash]
|
|
g.seriesMtx.RUnlock()
|
|
|
|
g.seriesDemand.Insert(hash)
|
|
|
|
if ok {
|
|
// target_info will always be 1 so if the series exists, we don't need to go through this loop
|
|
if !updateIfAlreadyExist {
|
|
return
|
|
}
|
|
g.updateSeriesValue(hash, s, value, operation)
|
|
return
|
|
}
|
|
|
|
g.seriesMtx.Lock()
|
|
defer g.seriesMtx.Unlock()
|
|
|
|
s, lbls, hash = resolveSeries(g.series, hash, lbls, g.lifecycler, 1)
|
|
if s != nil {
|
|
if !updateIfAlreadyExist {
|
|
return
|
|
}
|
|
g.updateSeriesValue(hash, s, value, operation)
|
|
return
|
|
}
|
|
|
|
g.series[hash] = g.newSeries(lbls, value)
|
|
}
|
|
|
|
func (g *gauge) newSeries(lbls labels.Labels, value float64) *gaugeSeries {
|
|
return &gaugeSeries{
|
|
labels: getSeriesLabels(g.metricName, lbls, g.externalLabels),
|
|
value: atomic.NewFloat64(value),
|
|
lastUpdated: atomic.NewInt64(time.Now().UnixMilli()),
|
|
}
|
|
}
|
|
|
|
func (g *gauge) updateSeriesValue(hash uint64, s *gaugeSeries, value float64, operation string) {
|
|
if operation == add {
|
|
s.value.Add(value)
|
|
} else {
|
|
s.value.Store(value)
|
|
}
|
|
s.lastUpdated.Store(time.Now().UnixMilli())
|
|
g.lifecycler.OnUpdate(hash, 1)
|
|
}
|
|
|
|
func (g *gauge) name() string {
|
|
return g.metricName
|
|
}
|
|
|
|
func (g *gauge) collectMetrics(appender storage.Appender, timeMs int64) error {
|
|
g.seriesMtx.RLock()
|
|
defer g.seriesMtx.RUnlock()
|
|
|
|
for _, s := range g.series {
|
|
_, err := appender.Append(0, s.labels, timeMs, s.value.Load())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
// TODO: support exemplars
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (g *gauge) countActiveSeries() int {
|
|
g.seriesMtx.RLock()
|
|
defer g.seriesMtx.RUnlock()
|
|
|
|
return len(g.series)
|
|
}
|
|
|
|
func (g *gauge) countSeriesDemand() int {
|
|
g.seriesMtx.RLock()
|
|
defer g.seriesMtx.RUnlock()
|
|
|
|
return int(g.seriesDemand.Estimate())
|
|
}
|
|
|
|
func (g *gauge) removeStaleSeries(staleTimeMs int64) {
|
|
g.seriesMtx.Lock()
|
|
defer g.seriesMtx.Unlock()
|
|
|
|
for hash, s := range g.series {
|
|
if s.lastUpdated.Load() < staleTimeMs {
|
|
delete(g.series, hash)
|
|
g.lifecycler.OnDelete(hash, 1)
|
|
}
|
|
}
|
|
g.seriesDemand.Advance()
|
|
}
|