mirror of
https://github.com/hanzoai/otel-collector.git
synced 2026-11-03 02:20:04 +00:00
f621ffd9d2
## Pull Request ### Summary Briefly describe the change and its purpose. This PR aims to introduce Value Suggestions for JSON logs body feature, along with deprecating jsontypeexporter (Will be raised separately for removal) by merging into metadataexporter - [x] String suggestions - [x] Number suggestions **JSON body walker** (`json_writer.go`, `types.go`) - On each log record, walks the JSON body once to collect: - Path → ClickHouse type mappings → written to `distributed_json_path_types` - Path → value suggestions → written to `distributed_tag_attributes_v2` with `tag_type=body` - Respects cardinality limits; a background ticker periodically fetches high-cardinality keys to skip. - Enabled via a new `json.enabled` config flag. ### Refactor (no behavior change) - `PushLogs` now fans out to a slice of `LogsMetadataWriter` implementations run in parallel via `errgroup`. The existing attribute writing logic is moved as-is into `attributeMetadataWriter` — no logic changed there. ### Related Issues <!-- Reference any related issues (e.g. Fixes #123, Closes #456) --> Closes SigNoz/engineering-pod#4157 ### Resource Impact **Will this change affect the application's resource usage?** - [ ] Yes - [x] No If **Yes**, describe: - **Resources impacted:** (CPU, Memory, Disk, Network, etc.) - **Expected change:** (e.g., +10% CPU, -150MB RAM) - **Benchmarks:** Provide before/after results and how they were measured. <!-- Ideally the test should be done where you keep ingestion rate and configuration same and observe the changes in CPU, Memory, Ingestion Rate and the Lag on Kafka> --------- Co-authored-by: Srikanth Chekuri <[email protected]>
62 lines
1.3 KiB
Go
62 lines
1.3 KiB
Go
package metadataexporter
|
|
|
|
import (
|
|
"time"
|
|
|
|
lru "github.com/hashicorp/golang-lru/v2"
|
|
"github.com/jellydator/ttlcache/v3"
|
|
)
|
|
|
|
type ValueTracker struct {
|
|
ttl *ttlcache.Cache[string, *lru.Cache[any, struct{}]]
|
|
maxValuesPerKey int
|
|
}
|
|
|
|
func NewValueTracker(
|
|
maxKeys int,
|
|
maxValuesPerKey int,
|
|
ttl time.Duration,
|
|
) *ValueTracker {
|
|
cache := ttlcache.New(
|
|
ttlcache.WithTTL[string, *lru.Cache[any, struct{}]](ttl),
|
|
ttlcache.WithCapacity[string, *lru.Cache[any, struct{}]](uint64(maxKeys)),
|
|
)
|
|
|
|
go cache.Start()
|
|
|
|
return &ValueTracker{
|
|
ttl: cache,
|
|
maxValuesPerKey: maxValuesPerKey,
|
|
}
|
|
}
|
|
|
|
func (vt *ValueTracker) AddValue(key string, value any) {
|
|
// Get or create LRU cache for this key
|
|
var valueCache *lru.Cache[any, struct{}]
|
|
if item := vt.ttl.Get(key); item != nil {
|
|
valueCache = item.Value()
|
|
} else {
|
|
// Create new LRU cache for this key
|
|
newCache, err := lru.New[any, struct{}](vt.maxValuesPerKey)
|
|
if err != nil {
|
|
return
|
|
}
|
|
valueCache = newCache
|
|
vt.ttl.Set(key, valueCache, ttlcache.DefaultTTL)
|
|
}
|
|
|
|
// Add value to the LRU cache
|
|
valueCache.Add(value, struct{}{})
|
|
}
|
|
|
|
func (vt *ValueTracker) GetUniqueValueCount(key string) int {
|
|
if item := vt.ttl.Get(key); item != nil {
|
|
return item.Value().Len()
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func (vt *ValueTracker) Close() {
|
|
vt.ttl.Stop()
|
|
}
|