mirror of
https://github.com/hanzoai/otel-collector.git
synced 2026-11-03 02:20:04 +00:00
194862803e
## Pull Request ### Summary Briefly describe the change and its purpose. This PR aims to introduce Value Suggestions for JSON logs body feature, along with deprecating jsontypeexporter (Will be raised separately for removal) by merging into metadataexporter - [x] String suggestions - [x] Number suggestions **JSON body walker** (`json_writer.go`, `types.go`) - On each log record, walks the JSON body once to collect: - Path → ClickHouse type mappings → written to `distributed_json_path_types` - Path → value suggestions → written to `distributed_tag_attributes_v2` with `tag_type=body` - Respects cardinality limits; a background ticker periodically fetches high-cardinality keys to skip. - Enabled via a new `json.enabled` config flag. ### Refactor (no behavior change) - `PushLogs` now fans out to a slice of `LogsMetadataWriter` implementations run in parallel via `errgroup`. The existing attribute writing logic is moved as-is into `attributeMetadataWriter` — no logic changed there. ### Related Issues <!-- Reference any related issues (e.g. Fixes #123, Closes #456) --> Closes SigNoz/engineering-pod#4157 ### Resource Impact **Will this change affect the application's resource usage?** - [ ] Yes - [x] No If **Yes**, describe: - **Resources impacted:** (CPU, Memory, Disk, Network, etc.) - **Expected change:** (e.g., +10% CPU, -150MB RAM) - **Benchmarks:** Provide before/after results and how they were measured. <!-- Ideally the test should be done where you keep ingestion rate and configuration same and observe the changes in CPU, Memory, Ingestion Rate and the Lag on Kafka> --------- Co-authored-by: Srikanth Chekuri <[email protected]>
105 lines
3.7 KiB
Go
105 lines
3.7 KiB
Go
package metadataexporter
|
|
|
|
import (
|
|
"time"
|
|
|
|
"go.opentelemetry.io/collector/config/configoptional"
|
|
"go.opentelemetry.io/collector/config/configretry"
|
|
"go.opentelemetry.io/collector/exporter/exporterhelper"
|
|
)
|
|
|
|
type CacheProvider string
|
|
|
|
const (
|
|
CacheProviderInMemory CacheProvider = "in_memory"
|
|
CacheProviderRedis CacheProvider = "redis"
|
|
)
|
|
|
|
type LimitsConfig struct {
|
|
MaxKeys uint64 `mapstructure:"max_keys"`
|
|
MaxStringDistinctValues uint64 `mapstructure:"max_string_distinct_values"`
|
|
MaxStringLength uint64 `mapstructure:"max_string_length"`
|
|
FetchInterval time.Duration `mapstructure:"fetch_interval"`
|
|
}
|
|
|
|
type MaxDistinctValuesConfig struct {
|
|
Traces LimitsConfig `mapstructure:"traces"`
|
|
Logs LimitsConfig `mapstructure:"logs"`
|
|
Metrics LimitsConfig `mapstructure:"metrics"`
|
|
}
|
|
|
|
type AlwaysIncludeAttributesConfig struct {
|
|
Traces []string `mapstructure:"traces"`
|
|
Logs []string `mapstructure:"logs"`
|
|
Metrics []string `mapstructure:"metrics"`
|
|
}
|
|
|
|
type InMemoryCacheConfig struct {
|
|
}
|
|
|
|
type RedisCacheConfig struct {
|
|
Addr string `mapstructure:"addr"`
|
|
Username string `mapstructure:"username"`
|
|
Password string `mapstructure:"password"`
|
|
DB int `mapstructure:"db"`
|
|
}
|
|
|
|
type CacheLimits struct {
|
|
MaxResources uint64 `mapstructure:"max_resources"`
|
|
MaxCardinalityPerResource uint64 `mapstructure:"max_cardinality_per_resource"`
|
|
MaxTotalCardinality uint64 `mapstructure:"max_total_cardinality"`
|
|
}
|
|
|
|
type CacheConfig struct {
|
|
Provider CacheProvider `mapstructure:"provider"`
|
|
InMemory InMemoryCacheConfig `mapstructure:"in_memory"`
|
|
Redis RedisCacheConfig `mapstructure:"redis"`
|
|
Traces CacheLimits `mapstructure:"traces_limits"`
|
|
Metrics CacheLimits `mapstructure:"metrics_limits"`
|
|
Logs CacheLimits `mapstructure:"logs_limits"`
|
|
// Iterate over all the keys in the cache and print the cardinality
|
|
// Since this is expensive, it is disabled by default
|
|
Debug bool `mapstructure:"debug"`
|
|
}
|
|
|
|
const (
|
|
defaultJSONMaxDepthTraverse = 22
|
|
defaultJSONMaxArrayElementsAllowed = 100
|
|
defaultJSONMaxKeysAtLevel = 1024
|
|
defaultJSONKeyCacheSize = 10_000
|
|
)
|
|
|
|
// JSONConfig holds configuration for JSON field processing (body).
|
|
type JSONConfig struct {
|
|
// Enabled gates all JSON field processing (type collection + value suggestions).
|
|
Enabled bool `mapstructure:"enabled"`
|
|
// MaxDepthTraverse is the maximum nesting depth to traverse.
|
|
MaxDepthTraverse *int `mapstructure:"max_depth_traverse"`
|
|
// MaxArrayElementsAllowed is the maximum number of array elements to inspect.
|
|
MaxArrayElementsAllowed *int `mapstructure:"max_array_elements_allowed"`
|
|
// MaxKeysAtLevel is the maximum number of keys allowed at any single map level.
|
|
MaxKeysAtLevel *int `mapstructure:"max_keys_at_level"`
|
|
}
|
|
|
|
// Config defines configuration for Metadata exporter.
|
|
type Config struct {
|
|
exporterhelper.TimeoutConfig `mapstructure:",squash"` // squash ensures fields are correctly decoded in embedded struct.
|
|
QueueBatchConfig configoptional.Optional[exporterhelper.QueueBatchConfig] `mapstructure:"sending_queue"`
|
|
BackOffConfig configretry.BackOffConfig `mapstructure:"retry_on_failure"`
|
|
|
|
DSN string `mapstructure:"dsn"`
|
|
|
|
MaxDistinctValues MaxDistinctValuesConfig `mapstructure:"max_distinct_values"`
|
|
|
|
AlwaysIncludeAttributes AlwaysIncludeAttributesConfig `mapstructure:"always_include_attributes"`
|
|
|
|
Cache CacheConfig `mapstructure:"cache"`
|
|
|
|
TenantID string `mapstructure:"tenant_id"`
|
|
|
|
Enabled bool `mapstructure:"enabled"`
|
|
|
|
// JSON configures JSON field processing for body (and attributes in future).
|
|
JSON JSONConfig `mapstructure:"json"`
|
|
}
|