mirror of
https://github.com/hanzoai/otel-collector.git
synced 2026-10-30 18:26:25 +00:00
194862803e
## Pull Request ### Summary Briefly describe the change and its purpose. This PR aims to introduce Value Suggestions for JSON logs body feature, along with deprecating jsontypeexporter (Will be raised separately for removal) by merging into metadataexporter - [x] String suggestions - [x] Number suggestions **JSON body walker** (`json_writer.go`, `types.go`) - On each log record, walks the JSON body once to collect: - Path → ClickHouse type mappings → written to `distributed_json_path_types` - Path → value suggestions → written to `distributed_tag_attributes_v2` with `tag_type=body` - Respects cardinality limits; a background ticker periodically fetches high-cardinality keys to skip. - Enabled via a new `json.enabled` config flag. ### Refactor (no behavior change) - `PushLogs` now fans out to a slice of `LogsMetadataWriter` implementations run in parallel via `errgroup`. The existing attribute writing logic is moved as-is into `attributeMetadataWriter` — no logic changed there. ### Related Issues <!-- Reference any related issues (e.g. Fixes #123, Closes #456) --> Closes SigNoz/engineering-pod#4157 ### Resource Impact **Will this change affect the application's resource usage?** - [ ] Yes - [x] No If **Yes**, describe: - **Resources impacted:** (CPU, Memory, Disk, Network, etc.) - **Expected change:** (e.g., +10% CPU, -150MB RAM) - **Benchmarks:** Provide before/after results and how they were measured. <!-- Ideally the test should be done where you keep ingestion rate and configuration same and observe the changes in CPU, Memory, Ingestion Rate and the Lag on Kafka> --------- Co-authored-by: Srikanth Chekuri <[email protected]>
185 lines
3.5 KiB
Go
185 lines
3.5 KiB
Go
package keycheck
|
|
|
|
import "testing"
|
|
|
|
func Test_IsRandomKey(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
key string
|
|
want bool
|
|
}{
|
|
{
|
|
name: "Valid UUID",
|
|
key: "123e4567-e89b-12d3-a456-426614174000",
|
|
want: true,
|
|
},
|
|
{
|
|
name: "Valid Hex String",
|
|
key: "abcdef1234567890abcdef1234567890",
|
|
want: true,
|
|
},
|
|
{
|
|
name: "Valid Base64 String",
|
|
key: "QWxhZGRpbjpvcGVuIHNlc2FtZQ==",
|
|
want: true,
|
|
},
|
|
{
|
|
name: "Long String Without Vowels",
|
|
key: "bcdfghjklmnpqrstvwxyz",
|
|
want: false,
|
|
},
|
|
{
|
|
name: "Mixed Case with Digits",
|
|
key: "Abcdef123456",
|
|
want: false,
|
|
},
|
|
{
|
|
name: "Short String",
|
|
key: "short",
|
|
want: false,
|
|
},
|
|
{
|
|
name: "String with Vowels",
|
|
key: "aeiouAEIOU",
|
|
want: false,
|
|
},
|
|
{
|
|
name: "Mixed Case without Digits",
|
|
key: "Abcdefghijklmnop",
|
|
want: false,
|
|
},
|
|
{
|
|
name: "Empty String",
|
|
key: "",
|
|
want: false,
|
|
},
|
|
{
|
|
name: "String with Special Characters",
|
|
key: "abc!@#123",
|
|
want: false,
|
|
},
|
|
{
|
|
name: "String with Only Digits",
|
|
key: "1234567890123456",
|
|
want: true,
|
|
},
|
|
{
|
|
name: "Flattened JSON Key with UUID",
|
|
key: "x.y.123e4567-e89b-12d3-a456-426614174000",
|
|
want: true,
|
|
},
|
|
{
|
|
name: "Flattened JSON Key with Hex",
|
|
key: "x.y.abcdef1234567890abcdef1234567890",
|
|
want: true,
|
|
},
|
|
{
|
|
name: "Flattened JSON Key with Base64",
|
|
key: "x.y.QWxhZGRpbjpvcGVuIHNlc2FtZQ==",
|
|
want: true,
|
|
},
|
|
{
|
|
name: "Flattened JSON Key with Regular String",
|
|
key: "x.y.orderId",
|
|
want: false,
|
|
},
|
|
{
|
|
name: "Deeply Nested JSON Path with ID",
|
|
key: "user.profile.settings.preferences.id.123456789",
|
|
want: false,
|
|
},
|
|
{
|
|
name: "Multiple UUIDs in Path",
|
|
key: "parent.123e4567-e89b-12d3-a456-426614174000.child.987fcdeb-51a2-43d7-9876-543210987654",
|
|
want: true,
|
|
},
|
|
{
|
|
name: "Mixed Random and Meaningful Keys",
|
|
key: "user.123e4567-e89b-12d3-a456-426614174000.name",
|
|
want: true,
|
|
},
|
|
{
|
|
name: "Long Path with Multiple Random Segments",
|
|
key: "a.b.c.d.e.f.g.h.i.j.k.l.m.n.o.p.q.r.s.t.u.v.w.x.y.z",
|
|
want: false,
|
|
},
|
|
{
|
|
name: "Path with Base64 in Middle",
|
|
key: "user.data.QWxhZGRpbjpvcGVuIHNlc2FtZQ==.profile",
|
|
want: true,
|
|
},
|
|
{
|
|
name: "Path with Hex in Middle",
|
|
key: "config.abcdef1234567890abcdef1234567890.settings",
|
|
want: true,
|
|
},
|
|
{
|
|
name: "Path with Multiple Special Characters",
|
|
key: "[email protected]",
|
|
want: false,
|
|
},
|
|
{
|
|
name: "Path with Timestamp-like Numbers",
|
|
key: "events.1647123456789.data",
|
|
want: true,
|
|
},
|
|
{
|
|
name: "Path with Multiple Dots and Numbers",
|
|
key: "v1.2.3.4.5.6.7.8.9.0",
|
|
want: false,
|
|
},
|
|
{
|
|
name: "Path With orderID",
|
|
key: "methodArgs.txn_map.KWIK3J3DCFYS3584069MX",
|
|
want: true,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
if got := IsRandomKey(tt.key); got != tt.want {
|
|
t.Errorf("isRandomKey() = %v, want %v", got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func Test_IsCardinal(t *testing.T) {
|
|
tests := []struct {
|
|
key string
|
|
isCardinal bool
|
|
}{
|
|
{
|
|
key: "value-test",
|
|
isCardinal: false,
|
|
},
|
|
{
|
|
key: "value@test",
|
|
isCardinal: false,
|
|
},
|
|
{
|
|
key: "value:test",
|
|
isCardinal: false,
|
|
},
|
|
{
|
|
key: "value_test",
|
|
isCardinal: false,
|
|
},
|
|
{
|
|
key: "value1",
|
|
isCardinal: false,
|
|
},
|
|
{
|
|
key: "value`",
|
|
isCardinal: true,
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.key, func(t *testing.T) {
|
|
if got := IsCardinal(tt.key); got != tt.isCardinal {
|
|
t.Errorf("IsCardinal() = %v, want %v", got, tt.isCardinal)
|
|
}
|
|
})
|
|
}
|
|
}
|