hanzoai/traces
2
0
Fork 0
mirror of https://github.com/hanzoai/traces.git synced 2026-10-30 18:20:41 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
preserve/dbc-local
traces/modules/generator/registry/histogram_test.go
T
Patrick OyarzunandGitHub 1970f26b28 [Generator] Produce overflow series instead of dropping (#5954) 
* [Generator] Produce overflow series when limiting

This PR implements an "overflow series" mechanism when the entity limit is hit.

To avoid bike shedding, I adopted the spec from otel's metrics sdk, since it has a similar mechanism.

https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/metrics/sdk.md#cardinality-limits

* Simplify how overflow entities are tracked

* Move label sanitization into OnAdd

* Remove demand tracker from builder

* Remove unused type

* Add overflow entity

* Fix linter

* Fix integration tests

* Add changelog entry for overflow series
2025-12-10 06:15:10 -05:00

521 lines
30 KiB
Go
Raw Permalink Blame History

package registry
import (
"fmt"
"math/rand"
"sync"
"testing"
"time"
"github.com/prometheus/prometheus/model/exemplar"
"github.com/prometheus/prometheus/model/labels"
"github.com/stretchr/testify/assert"
"go.uber.org/atomic"
)
func Test_histogram(t *testing.T) {
var seriesAdded int
lifecycler := &mockLimiter{
onAddFunc: func(hash uint64, _ uint32, lbls labels.Labels) (labels.Labels, uint64) {
seriesAdded++
return lbls, hash
},
}
h := newHistogram("my_histogram", []float64{1.0, 2.0}, lifecycler, "trace_id", nil, 15*time.Minute)
h.ObserveWithExemplar(buildTestLabels([]string{"label"}, []string{"value-1"}), 1.0, "trace-1", 1.0)
h.ObserveWithExemplar(buildTestLabels([]string{"label"}, []string{"value-2"}), 1.5, "trace-2", 1.0)
assert.Equal(t, 2, seriesAdded)
collectionTimeMs := time.Now().UnixMilli()
endOfLastMinuteMs := getEndOfLastMinuteMs(collectionTimeMs)
expectedSamples := []sample{
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1"}, endOfLastMinuteMs, 0), // Zero entry for value-1 series
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-1"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "1"}, endOfLastMinuteMs, 0), // Zero entry for bucket
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "1"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "2"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "2"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "+Inf"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "+Inf"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-2"}, endOfLastMinuteMs, 0), // Zero entry for value-2 series
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-2"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-2"}, collectionTimeMs, 1.5),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "1"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "1"}, collectionTimeMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "2"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "2"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "+Inf"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "+Inf"}, collectionTimeMs, 1),
}
expectedExemplars := []exemplarSample{
newExemplar(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "1"}, exemplar.Exemplar{
Labels: labels.FromMap(map[string]string{"trace_id": "trace-1"}),
Value: 1.0,
Ts: collectionTimeMs,
}),
newExemplar(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "2"}, exemplar.Exemplar{
Labels: labels.FromMap(map[string]string{"trace_id": "trace-2"}),
Value: 1.5,
Ts: collectionTimeMs,
}),
}
collectMetricAndAssert(t, h, collectionTimeMs, 10, expectedSamples, expectedExemplars)
h.ObserveWithExemplar(buildTestLabels([]string{"label"}, []string{"value-2"}), 2.5, "trace-2.2", 1.0)
h.ObserveWithExemplar(buildTestLabels([]string{"label"}, []string{"value-3"}), 3.0, "trace-3", 1.0)
assert.Equal(t, 3, seriesAdded)
collectionTimeMs = time.Now().UnixMilli()
endOfLastMinuteMs = getEndOfLastMinuteMs(collectionTimeMs)
expectedSamples = []sample{
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-1"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "1"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "2"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "+Inf"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-2"}, collectionTimeMs, 2),
newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-2"}, collectionTimeMs, 4.0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "1"}, collectionTimeMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "2"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "+Inf"}, collectionTimeMs, 2),
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-3"}, endOfLastMinuteMs, 0), // Zero entry for value-3 series
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-3"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-3"}, collectionTimeMs, 3),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-3", "le": "1"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-3", "le": "1"}, collectionTimeMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-3", "le": "2"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-3", "le": "2"}, collectionTimeMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-3", "le": "+Inf"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-3", "le": "+Inf"}, collectionTimeMs, 1),
}
expectedExemplars = []exemplarSample{
newExemplar(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "+Inf"}, exemplar.Exemplar{
Labels: labels.FromMap(map[string]string{"trace_id": "trace-2.2"}),
Value: 2.5,
Ts: collectionTimeMs,
}),
newExemplar(map[string]string{"__name__": "my_histogram_bucket", "label": "value-3", "le": "+Inf"}, exemplar.Exemplar{
Labels: labels.FromMap(map[string]string{"trace_id": "trace-3"}),
Value: 3.0,
Ts: collectionTimeMs,
}),
}
collectMetricAndAssert(t, h, collectionTimeMs, 15, expectedSamples, expectedExemplars)
h.ObserveWithExemplar(buildTestLabels([]string{"label"}, []string{"value-2"}), 2.5, "trace-2.2", 20.0)
h.ObserveWithExemplar(buildTestLabels([]string{"label"}, []string{"value-3"}), 3.0, "trace-3", 13.5)
h.ObserveWithExemplar(buildTestLabels([]string{"label"}, []string{"value-3"}), 1.0, "trace-3", 7.5)
assert.Equal(t, 3, seriesAdded)
collectionTimeMs = time.Now().UnixMilli()
expectedSamples = []sample{
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-1"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "1"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "2"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "+Inf"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-2"}, collectionTimeMs, 22),
newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-2"}, collectionTimeMs, 54.0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "1"}, collectionTimeMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "2"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "+Inf"}, collectionTimeMs, 22),
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-3"}, collectionTimeMs, 22.0),
newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-3"}, collectionTimeMs, 51.0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-3", "le": "1"}, collectionTimeMs, 7.5),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-3", "le": "2"}, collectionTimeMs, 7.5),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-3", "le": "+Inf"}, collectionTimeMs, 22.0),
}
expectedExemplars = []exemplarSample{
newExemplar(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "+Inf"}, exemplar.Exemplar{
Labels: labels.FromMap(map[string]string{"trace_id": "trace-2.2"}),
Value: 2.5,
Ts: collectionTimeMs,
}),
newExemplar(map[string]string{"__name__": "my_histogram_bucket", "label": "value-3", "le": "1"}, exemplar.Exemplar{
Labels: labels.FromMap(map[string]string{"trace_id": "trace-3"}),
Value: 1.0,
Ts: collectionTimeMs,
}),
newExemplar(map[string]string{"__name__": "my_histogram_bucket", "label": "value-3", "le": "+Inf"}, exemplar.Exemplar{
Labels: labels.FromMap(map[string]string{"trace_id": "trace-3"}),
Value: 3.0,
Ts: collectionTimeMs,
}),
}
collectMetricAndAssert(t, h, collectionTimeMs, 15, expectedSamples, expectedExemplars)
}
func Test_histogram_cantAdd(t *testing.T) {
canAdd := false
overflowLabels := labels.FromStrings("metric_overflow", "true")
overflowHash := overflowLabels.Hash()
lifecycler := &mockLimiter{
onAddFunc: func(_ uint64, count uint32, lbls labels.Labels) (labels.Labels, uint64) {
assert.Equal(t, uint32(5), count)
if canAdd {
return lbls, lbls.Hash()
}
return overflowLabels, overflowHash
},
}
h := newHistogram("my_histogram", []float64{1.0, 2.0}, lifecycler, "", nil, 15*time.Minute)
// allow adding new series
canAdd = true
h.ObserveWithExemplar(buildTestLabels([]string{"label"}, []string{"value-1"}), 1.0, "", 1.0)
h.ObserveWithExemplar(buildTestLabels([]string{"label"}, []string{"value-2"}), 1.5, "", 1.0)
collectionTimeMs := time.Now().UnixMilli()
endOfLastMinuteMs := getEndOfLastMinuteMs(collectionTimeMs)
expectedSamples := []sample{
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-1"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "1"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "1"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "2"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "2"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "+Inf"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "+Inf"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-2"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-2"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-2"}, collectionTimeMs, 1.5),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "1"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "1"}, collectionTimeMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "2"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "2"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "+Inf"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "+Inf"}, collectionTimeMs, 1),
}
collectMetricAndAssert(t, h, collectionTimeMs, 10, expectedSamples, nil)
// block new series - existing series can still be updated
canAdd = false
h.ObserveWithExemplar(buildTestLabels([]string{"label"}, []string{"value-2"}), 2.5, "", 1.0)
h.ObserveWithExemplar(buildTestLabels([]string{"label"}, []string{"value-3"}), 3.0, "", 1.0)
collectionTimeMs = time.Now().UnixMilli()
expectedSamples = []sample{
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-1"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "1"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "2"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "+Inf"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-2"}, collectionTimeMs, 2),
newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-2"}, collectionTimeMs, 4.0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "1"}, collectionTimeMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "2"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "+Inf"}, collectionTimeMs, 2),
newSample(map[string]string{"__name__": "my_histogram_count", "metric_overflow": "true"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_count", "metric_overflow": "true"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_sum", "metric_overflow": "true"}, collectionTimeMs, 3),
newSample(map[string]string{"__name__": "my_histogram_bucket", "metric_overflow": "true", "le": "1"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "metric_overflow": "true", "le": "1"}, collectionTimeMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "metric_overflow": "true", "le": "2"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "metric_overflow": "true", "le": "2"}, collectionTimeMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "metric_overflow": "true", "le": "+Inf"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "metric_overflow": "true", "le": "+Inf"}, collectionTimeMs, 1),
}
collectMetricAndAssert(t, h, collectionTimeMs, 15, expectedSamples, nil)
}
func Test_histogram_removeStaleSeries(t *testing.T) {
var removedSeries int
lifecycler := &mockLimiter{
onDeleteFunc: func(_ uint64, count uint32) {
assert.Equal(t, uint32(5), count)
removedSeries++
},
}
h := newHistogram("my_histogram", []float64{1.0, 2.0}, lifecycler, "", nil, 15*time.Minute)
timeMs := time.Now().UnixMilli()
h.ObserveWithExemplar(buildTestLabels([]string{"label"}, []string{"value-1"}), 1.0, "", 1.0)
h.ObserveWithExemplar(buildTestLabels([]string{"label"}, []string{"value-2"}), 1.5, "", 1.0)
h.removeStaleSeries(timeMs)
assert.Equal(t, 0, removedSeries)
collectionTimeMs := time.Now().UnixMilli()
endOfLastMinuteMs := getEndOfLastMinuteMs(collectionTimeMs)
expectedSamples := []sample{
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-1"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "1"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "1"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "2"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "2"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "+Inf"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "+Inf"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-2"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-2"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-2"}, collectionTimeMs, 1.5),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "1"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "1"}, collectionTimeMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "2"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "2"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "+Inf"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "+Inf"}, collectionTimeMs, 1),
}
collectMetricAndAssert(t, h, collectionTimeMs, 10, expectedSamples, nil)
time.Sleep(10 * time.Millisecond)
timeMs = time.Now().UnixMilli()
// update value-2 series
h.ObserveWithExemplar(buildTestLabels([]string{"label"}, []string{"value-2"}), 2.5, "", 1.0)
h.removeStaleSeries(timeMs)
assert.Equal(t, 1, removedSeries)
collectionTimeMs = time.Now().UnixMilli()
expectedSamples = []sample{
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-2"}, collectionTimeMs, 2),
newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-2"}, collectionTimeMs, 4.0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "1"}, collectionTimeMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "2"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "+Inf"}, collectionTimeMs, 2),
}
collectMetricAndAssert(t, h, collectionTimeMs, 5, expectedSamples, nil)
}
func Test_histogram_externalLabels(t *testing.T) {
extLabels := map[string]string{"external_label": "external_value"}
h := newHistogram("my_histogram", []float64{1.0, 2.0}, noopLimiter, "", extLabels, 15*time.Minute)
h.ObserveWithExemplar(buildTestLabels([]string{"label"}, []string{"value-1"}), 1.0, "", 1.0)
h.ObserveWithExemplar(buildTestLabels([]string{"label"}, []string{"value-2"}), 1.5, "", 1.0)
collectionTimeMs := time.Now().UnixMilli()
endOfLastMinuteMs := getEndOfLastMinuteMs(collectionTimeMs)
expectedSamples := []sample{
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1", "external_label": "external_value"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1", "external_label": "external_value"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-1", "external_label": "external_value"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "1", "external_label": "external_value"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "1", "external_label": "external_value"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "2", "external_label": "external_value"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "2", "external_label": "external_value"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "+Inf", "external_label": "external_value"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "+Inf", "external_label": "external_value"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-2", "external_label": "external_value"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-2", "external_label": "external_value"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-2", "external_label": "external_value"}, collectionTimeMs, 1.5),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "1", "external_label": "external_value"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "1", "external_label": "external_value"}, collectionTimeMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "2", "external_label": "external_value"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "2", "external_label": "external_value"}, collectionTimeMs, 1),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "+Inf", "external_label": "external_value"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-2", "le": "+Inf", "external_label": "external_value"}, collectionTimeMs, 1),
}
collectMetricAndAssert(t, h, collectionTimeMs, 10, expectedSamples, nil)
}
func Test_histogram_concurrencyDataRace(t *testing.T) {
h := newHistogram("my_histogram", []float64{1.0, 2.0}, noopLimiter, "", nil, 15*time.Minute)
end := make(chan struct{})
accessor := func(f func()) {
for {
select {
case <-end:
return
default:
f()
}
}
}
for i := 0; i < 4; i++ {
go accessor(func() {
h.ObserveWithExemplar(buildTestLabels([]string{"label"}, []string{"value-1"}), 1.0, "", 1.0)
h.ObserveWithExemplar(buildTestLabels([]string{"label"}, []string{"value-2"}), 1.5, "", 1.0)
})
}
// this goroutine constantly creates new series
letters := []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")
go accessor(func() {
s := make([]rune, 6)
for i := range s {
s[i] = letters[rand.Intn(len(letters))]
}
h.ObserveWithExemplar(buildTestLabels([]string{"label"}, []string{string(s)}), 1.0, "", 1.0)
})
go accessor(func() {
err := h.collectMetrics(&noopAppender{}, 0)
assert.NoError(t, err)
})
go accessor(func() {
h.removeStaleSeries(time.Now().UnixMilli())
})
time.Sleep(200 * time.Millisecond)
close(end)
}
func Test_histogram_concurrencyCorrectness(t *testing.T) {
h := newHistogram("my_histogram", []float64{1.0, 2.0}, noopLimiter, "", nil, 15*time.Minute)
var wg sync.WaitGroup
end := make(chan struct{})
totalCount := atomic.NewUint64(0)
for i := 0; i < 4; i++ {
wg.Add(1)
go func() {
defer wg.Done()
for {
select {
case <-end:
return
default:
h.ObserveWithExemplar(buildTestLabels([]string{"label"}, []string{"value-1"}), 2.0, "", 1.0)
totalCount.Inc()
}
}
}()
}
time.Sleep(200 * time.Millisecond)
close(end)
wg.Wait()
collectionTimeMs := time.Now().UnixMilli()
endOfLastMinuteMs := getEndOfLastMinuteMs(collectionTimeMs)
expectedSamples := []sample{
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1"}, collectionTimeMs, float64(totalCount.Load())),
newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-1"}, collectionTimeMs, 2*float64(totalCount.Load())),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "1"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "1"}, collectionTimeMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "2"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "2"}, collectionTimeMs, float64(totalCount.Load())),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "+Inf"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "+Inf"}, collectionTimeMs, float64(totalCount.Load())),
}
collectMetricAndAssert(t, h, collectionTimeMs, 5, expectedSamples, nil)
}
func Test_histogram_span_multiplier(t *testing.T) {
h := newHistogram("my_histogram", []float64{1.0, 2.0}, noopLimiter, "", nil, 15*time.Minute)
h.ObserveWithExemplar(buildTestLabels([]string{"label"}, []string{"value-1"}), 1.0, "", 1.5)
h.ObserveWithExemplar(buildTestLabels([]string{"label"}, []string{"value-1"}), 2.0, "", 5)
collectionTimeMs := time.Now().UnixMilli()
endOfLastMinuteMs := getEndOfLastMinuteMs(collectionTimeMs)
expectedSamples := []sample{
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_count", "label": "value-1"}, collectionTimeMs, 6.5),
newSample(map[string]string{"__name__": "my_histogram_sum", "label": "value-1"}, collectionTimeMs, 11.5),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "1"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "1"}, collectionTimeMs, 1.5),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "2"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "2"}, collectionTimeMs, 6.5),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "+Inf"}, endOfLastMinuteMs, 0),
newSample(map[string]string{"__name__": "my_histogram_bucket", "label": "value-1", "le": "+Inf"}, collectionTimeMs, 6.5),
}
collectMetricAndAssert(t, h, collectionTimeMs, 5, expectedSamples, nil)
}
func Test_histogram_demandTracking(t *testing.T) {
h := newHistogram("my_histogram", []float64{1.0, 2.0}, noopLimiter, "", nil, 15*time.Minute)
// Initially, demand should be 0
assert.Equal(t, 0, h.countSeriesDemand())
// Add some histogram series (each histogram creates multiple series)
for i := 0; i < 20; i++ {
lbls := buildTestLabels([]string{"label"}, []string{fmt.Sprintf("value-%d", i)})
h.ObserveWithExemplar(lbls, 1.5, "", 1.0)
}
// Demand should be approximately 20 histogram series * 5 (sum, count, 3 buckets) = 100 total series
// Within HLL error
demand := h.countSeriesDemand()
assert.Greater(t, demand, 80, "demand should be close to 100")
assert.Less(t, demand, 120, "demand should be close to 100")
// Active series should exactly match 20 * 5
expectedActive := 20 * int(h.activeSeriesPerHistogramSerie())
assert.Equal(t, expectedActive, h.countActiveSeries())
}
func Test_histogram_activeSeriesPerHistogramSerie(t *testing.T) {
// Test with 2 buckets (creates: sum, count, bucket1, bucket2, +Inf)
h := newHistogram("my_histogram", []float64{1.0, 2.0}, noopLimiter, "", nil, 15*time.Minute)
assert.Equal(t, uint32(5), h.activeSeriesPerHistogramSerie(), "should be sum + count + 3 buckets")
// Test with 3 buckets
h2 := newHistogram("my_histogram", []float64{1.0, 2.0, 3.0}, noopLimiter, "", nil, 15*time.Minute)
assert.Equal(t, uint32(6), h2.activeSeriesPerHistogramSerie(), "should be sum + count + 4 buckets")
// Test with no buckets (still has +Inf)
h3 := newHistogram("my_histogram", []float64{}, noopLimiter, "", nil, 15*time.Minute)
assert.Equal(t, uint32(3), h3.activeSeriesPerHistogramSerie(), "should be sum + count + +Inf bucket")
}
func Test_histogram_demandVsActiveSeries(t *testing.T) {
limitReached := false
overflowLabels := labels.FromStrings("metric_overflow", "true")
overflowHash := overflowLabels.Hash()
lifecycler := &mockLimiter{
onAddFunc: func(hash uint64, _ uint32, lbls labels.Labels) (labels.Labels, uint64) {
if !limitReached {
return lbls, hash
}
return overflowLabels, overflowHash
},
}
h := newHistogram("my_histogram", []float64{1.0, 2.0}, lifecycler, "", nil, 15*time.Minute)
// Add some histogram series
for i := 0; i < 10; i++ {
lbls := buildTestLabels([]string{"label"}, []string{fmt.Sprintf("value-%d", i)})
h.ObserveWithExemplar(lbls, 1.5, "", 1.0)
}
expectedActive := 10 * int(h.activeSeriesPerHistogramSerie())
assert.Equal(t, expectedActive, h.countActiveSeries())
// Hit the limit
limitReached = true
// Try to add more series (they should be mapped to overflow)
for i := 10; i < 20; i++ {
lbls := buildTestLabels([]string{"label"}, []string{fmt.Sprintf("value-%d", i)})
h.ObserveWithExemplar(lbls, 1.5, "", 1.0)
}
// Active series should have increased by 1 overflow series (5 series for classic histogram)
expectedOverflowSeries := int(h.activeSeriesPerHistogramSerie())
assert.Equal(t, expectedActive+expectedOverflowSeries, h.countActiveSeries())
// But demand should show all attempted series
demand := h.countSeriesDemand()
expectedDemand := 20 * int(h.activeSeriesPerHistogramSerie())
assert.Greater(t, demand, expectedDemand-20, "demand should track all attempted series")
assert.Greater(t, demand, h.countActiveSeries(), "demand should exceed active series")
}
Reference in New Issue View Git Blame Copy Permalink