mirror of
https://github.com/hanzoai/o11y.git
synced 2026-11-02 14:03:38 +00:00
zeekayandGitHub
babbdec4b8
* debrand: signoz/SigNoz/SIGNOZ -> o11y/O11y/O11Y across the tree Rename all SigNoz *branding* to O11y in file contents, file/dir names, package names, env vars, config keys, comments, docs, SDK code, and the ClickHouse schema identifiers the querier reads. Collector dependency: - Repoint github.com/hanzoai/signoz-otel-collector -> github.com/hanzoai/otel-collector and bump v0.144.6 -> v0.144.7 (go.mod + go.sum reconciled via go mod tidy). Internal package paths also debranded upstream: signozschemamigrator -> o11yschemamigrator, signozlogspipelineprocessor -> o11ylogspipelineprocessor. - Fix: v0.144.7 dropped otelconst.DistributedFieldKeysTable from the collector's public constants; pin FieldKeysTable = "distributed_field_keys" locally in pkg/telemetrymetadata/tables.go. Package/dir renames (package decl + all importers): pkg/apiserver/signozapiserver -> o11yapiserver, plus signozalertmanager, signozauthzapi, signozglobal, signozquerier, signozruler, and pkg/signoz -> pkg/o11y. Schema (read plane): signoz_traces/metrics/logs/metadata/analytics/meter and signoz_index_v3/index_v2/error_index_v2/spans (+ distributed_*) -> o11y_*. Data-preserving cutover migration added: deploy/clickhouse/migrations/0001_rename_signoz_to_o11y.sql (metadata-only RENAME DATABASE/TABLE, no drop/recreate). Preserved (attribution / external / wire contracts): - LICENSE + NOTICE verbatim (incl. "software from SigNoz", "Copyright ... SigNoz Inc."). - Upstream github.com/SigNoz/* repo URLs + @SigNoz/* CODEOWNERS teams. - Billing resource-attribute regex "signoz.workspace.*" (external wire contract). go build ./pkg/... ./cmd/... = 0; gofmt clean; renamed+schema pkg tests pass. * o11y: bump collector to v0.144.8 (o11y_* physical schema writer) + lockstep deploy plan Collector v0.144.8's schema-migrator + exporters now CREATE/WRITE the same o11y_* physical databases/tables the querier reads and the RENAME migration (0001_rename_signoz_to_o11y.sql) targets. Cross-checked byte-identical: writer DB set == migration target DB set (6); writer prefixed-table set == migration target table set (8); o11y querier reads and cloud reads are subsets of that set. Zero table-name mismatches between writer, readers, and migration. Collector go.mod unchanged between v0.144.7 and v0.144.8 (identical go.mod hash) — pure source rename, no module-graph change. Adds deploy/clickhouse/DEPLOY_signoz_to_o11y_cutover.md: the lockstep order (collector v0.144.8 -> RENAME migration -> o11y+cloud readers) with a fresh/scratch-ClickHouse pre-flight (create via migrator, emit trace+metric, query back) and rollback. Ships together or telemetry blackholes. go build ./pkg/... ./cmd/community = 0.
592 lines
14 KiB
Go
592 lines
14 KiB
Go
package logparsingpipeline
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"strconv"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/hanzoai/o11y/pkg/query-service/model"
|
|
v3 "github.com/hanzoai/o11y/pkg/query-service/model/v3"
|
|
"github.com/hanzoai/o11y/pkg/query-service/utils"
|
|
"github.com/hanzoai/o11y/pkg/types/pipelinetypes"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
// Tests for pipeline processors other than the ones
|
|
// covered in dedicated files.
|
|
|
|
func TestRegexProcessor(t *testing.T) {
|
|
require := require.New(t)
|
|
|
|
testPipelines := []pipelinetypes.GettablePipeline{
|
|
{
|
|
StoreablePipeline: pipelinetypes.StoreablePipeline{
|
|
OrderID: 1,
|
|
Name: "pipeline1",
|
|
Alias: "pipeline1",
|
|
Enabled: true,
|
|
},
|
|
Filter: &v3.FilterSet{
|
|
Operator: "AND",
|
|
Items: []v3.FilterItem{
|
|
{
|
|
Key: v3.AttributeKey{
|
|
Key: "method",
|
|
DataType: v3.AttributeKeyDataTypeString,
|
|
Type: v3.AttributeKeyTypeTag,
|
|
},
|
|
Operator: "=",
|
|
Value: "GET",
|
|
},
|
|
},
|
|
},
|
|
Config: []pipelinetypes.PipelineOperator{},
|
|
},
|
|
}
|
|
|
|
var parserOp pipelinetypes.PipelineOperator
|
|
err := json.Unmarshal([]byte(`
|
|
{
|
|
"orderId": 1,
|
|
"enabled": true,
|
|
"type": "regex_parser",
|
|
"name": "Test regex parser",
|
|
"id": "test-regex-parser",
|
|
"parse_from": "body",
|
|
"parse_to": "attributes",
|
|
"regex": "PAN: (?P<pan>[a-zA-Z]{5}[0-9]{4}[a-zA-Z]{1}) "
|
|
}
|
|
`), &parserOp)
|
|
require.Nil(err)
|
|
testPipelines[0].Config = append(testPipelines[0].Config, parserOp)
|
|
|
|
testPan := "GDTSJ4756E"
|
|
testLog := makeTestO11yLog(
|
|
fmt.Sprintf("test string with PAN: %s and some more text", testPan),
|
|
map[string]interface{}{
|
|
"method": "GET",
|
|
},
|
|
)
|
|
|
|
result, collectorWarnAndErrorLogs, err := SimulatePipelinesProcessing(
|
|
context.Background(),
|
|
testPipelines,
|
|
[]model.O11yLog{
|
|
testLog,
|
|
},
|
|
)
|
|
require.Nil(err)
|
|
require.Equal(1, len(result))
|
|
require.Equal(0, len(collectorWarnAndErrorLogs), strings.Join(collectorWarnAndErrorLogs, "\n"))
|
|
processed := result[0]
|
|
require.Equal(testPan, processed.Attributes_string["pan"])
|
|
}
|
|
|
|
func TestGrokProcessor(t *testing.T) {
|
|
require := require.New(t)
|
|
|
|
testPipelines := []pipelinetypes.GettablePipeline{
|
|
{
|
|
StoreablePipeline: pipelinetypes.StoreablePipeline{
|
|
OrderID: 1,
|
|
Name: "pipeline1",
|
|
Alias: "pipeline1",
|
|
Enabled: true,
|
|
},
|
|
Filter: &v3.FilterSet{
|
|
Operator: "AND",
|
|
Items: []v3.FilterItem{
|
|
{
|
|
Key: v3.AttributeKey{
|
|
Key: "method",
|
|
DataType: v3.AttributeKeyDataTypeString,
|
|
Type: v3.AttributeKeyTypeTag,
|
|
},
|
|
Operator: "=",
|
|
Value: "GET",
|
|
},
|
|
},
|
|
},
|
|
Config: []pipelinetypes.PipelineOperator{},
|
|
},
|
|
}
|
|
|
|
var parserOp pipelinetypes.PipelineOperator
|
|
err := json.Unmarshal([]byte(`
|
|
{
|
|
"orderId": 1,
|
|
"enabled": true,
|
|
"type": "grok_parser",
|
|
"name": "Test grok parser",
|
|
"id": "test-grok-parser",
|
|
"parse_from": "body",
|
|
"parse_to": "attributes",
|
|
"pattern": "status: %{INT:http_status_code:int}"
|
|
}
|
|
`), &parserOp)
|
|
require.Nil(err)
|
|
testPipelines[0].Config = append(testPipelines[0].Config, parserOp)
|
|
|
|
testStatusCode := int64(404)
|
|
testLog := makeTestO11yLog(
|
|
fmt.Sprintf("some http log with status: %d and some more text", testStatusCode),
|
|
map[string]interface{}{
|
|
"method": "GET",
|
|
},
|
|
)
|
|
|
|
result, collectorWarnAndErrorLogs, err := SimulatePipelinesProcessing(
|
|
context.Background(),
|
|
testPipelines,
|
|
[]model.O11yLog{
|
|
testLog,
|
|
},
|
|
)
|
|
require.Nil(err)
|
|
require.Equal(1, len(result))
|
|
require.Equal(0, len(collectorWarnAndErrorLogs), strings.Join(collectorWarnAndErrorLogs, "\n"))
|
|
processed := result[0]
|
|
require.Equal(testStatusCode, processed.Attributes_int64["http_status_code"])
|
|
}
|
|
|
|
func TestJSONProcessor(t *testing.T) {
|
|
require := require.New(t)
|
|
|
|
testPipelines := []pipelinetypes.GettablePipeline{
|
|
{
|
|
StoreablePipeline: pipelinetypes.StoreablePipeline{
|
|
OrderID: 1,
|
|
Name: "pipeline1",
|
|
Alias: "pipeline1",
|
|
Enabled: true,
|
|
},
|
|
Filter: &v3.FilterSet{
|
|
Operator: "AND",
|
|
Items: []v3.FilterItem{
|
|
{
|
|
Key: v3.AttributeKey{
|
|
Key: "method",
|
|
DataType: v3.AttributeKeyDataTypeString,
|
|
Type: v3.AttributeKeyTypeTag,
|
|
},
|
|
Operator: "=",
|
|
Value: "GET",
|
|
},
|
|
},
|
|
},
|
|
Config: []pipelinetypes.PipelineOperator{},
|
|
},
|
|
}
|
|
|
|
var parserOp pipelinetypes.PipelineOperator
|
|
err := json.Unmarshal([]byte(`
|
|
{
|
|
"orderId": 1,
|
|
"enabled": true,
|
|
"type": "json_parser",
|
|
"name": "Test json parser",
|
|
"id": "test-json-parser",
|
|
"parse_from": "body",
|
|
"parse_to": "attributes"
|
|
}
|
|
`), &parserOp)
|
|
require.Nil(err)
|
|
testPipelines[0].Config = append(testPipelines[0].Config, parserOp)
|
|
|
|
testLog := makeTestO11yLog(
|
|
`{"test_str": "value", "test_float": 1.1}`,
|
|
map[string]interface{}{
|
|
"method": "GET",
|
|
},
|
|
)
|
|
|
|
result, collectorWarnAndErrorLogs, err := SimulatePipelinesProcessing(
|
|
context.Background(),
|
|
testPipelines,
|
|
[]model.O11yLog{
|
|
testLog,
|
|
},
|
|
)
|
|
require.Nil(err)
|
|
require.Equal(1, len(result))
|
|
require.Equal(0, len(collectorWarnAndErrorLogs), strings.Join(collectorWarnAndErrorLogs, "\n"))
|
|
processed := result[0]
|
|
require.Equal("value", processed.Attributes_string["test_str"])
|
|
require.Equal(1.1, processed.Attributes_float64["test_float"])
|
|
}
|
|
|
|
func TestTraceParsingProcessor(t *testing.T) {
|
|
require := require.New(t)
|
|
|
|
testPipelines := []pipelinetypes.GettablePipeline{
|
|
{
|
|
StoreablePipeline: pipelinetypes.StoreablePipeline{
|
|
OrderID: 1,
|
|
Name: "pipeline1",
|
|
Alias: "pipeline1",
|
|
Enabled: true,
|
|
},
|
|
Filter: &v3.FilterSet{
|
|
Operator: "AND",
|
|
Items: []v3.FilterItem{
|
|
{
|
|
Key: v3.AttributeKey{
|
|
Key: "method",
|
|
DataType: v3.AttributeKeyDataTypeString,
|
|
Type: v3.AttributeKeyTypeTag,
|
|
},
|
|
Operator: "=",
|
|
Value: "GET",
|
|
},
|
|
},
|
|
},
|
|
Config: []pipelinetypes.PipelineOperator{},
|
|
},
|
|
}
|
|
|
|
// Start with JSON serialized trace parser to validate deserialization too
|
|
var traceParserOp pipelinetypes.PipelineOperator
|
|
err := json.Unmarshal([]byte(`
|
|
{
|
|
"orderId": 1,
|
|
"enabled": true,
|
|
"type": "trace_parser",
|
|
"name": "Test trace parser",
|
|
"id": "test-trace-parser",
|
|
"trace_id": {
|
|
"parse_from": "attributes.test_trace_id"
|
|
},
|
|
"span_id": {
|
|
"parse_from": "attributes.test_span_id"
|
|
},
|
|
"trace_flags": {
|
|
"parse_from": "attributes.test_trace_flags"
|
|
}
|
|
}
|
|
`), &traceParserOp)
|
|
require.Nil(err)
|
|
testPipelines[0].Config = append(testPipelines[0].Config, traceParserOp)
|
|
|
|
testTraceId, err := utils.RandomHex(16)
|
|
require.Nil(err)
|
|
|
|
testSpanId, err := utils.RandomHex(8)
|
|
require.Nil(err)
|
|
|
|
testTraceFlags, err := utils.RandomHex(1)
|
|
require.Nil(err)
|
|
|
|
testLog := model.O11yLog{
|
|
Timestamp: uint64(time.Now().UnixNano()),
|
|
Body: "test log",
|
|
Attributes_string: map[string]string{
|
|
"method": "GET",
|
|
"test_trace_id": testTraceId,
|
|
"test_span_id": testSpanId,
|
|
"test_trace_flags": testTraceFlags,
|
|
},
|
|
SpanID: "",
|
|
TraceID: "",
|
|
TraceFlags: 0,
|
|
}
|
|
|
|
result, collectorWarnAndErrorLogs, err := SimulatePipelinesProcessing(
|
|
context.Background(),
|
|
testPipelines,
|
|
[]model.O11yLog{
|
|
testLog,
|
|
},
|
|
)
|
|
require.Nil(err)
|
|
require.Equal(1, len(result))
|
|
require.Equal(0, len(collectorWarnAndErrorLogs))
|
|
processed := result[0]
|
|
|
|
require.Equal(testTraceId, processed.TraceID)
|
|
require.Equal(testSpanId, processed.SpanID)
|
|
|
|
expectedTraceFlags, err := strconv.ParseUint(testTraceFlags, 16, 16)
|
|
require.Nil(err)
|
|
require.Equal(uint32(expectedTraceFlags), processed.TraceFlags)
|
|
|
|
// trace parser should work even if parse_from value is empty
|
|
testPipelines[0].Config[0].SpanId.ParseFrom = ""
|
|
result, collectorWarnAndErrorLogs, err = SimulatePipelinesProcessing(
|
|
context.Background(),
|
|
testPipelines,
|
|
[]model.O11yLog{
|
|
testLog,
|
|
},
|
|
)
|
|
require.Nil(err)
|
|
require.Equal(1, len(result))
|
|
require.Equal(0, len(collectorWarnAndErrorLogs))
|
|
require.Equal("", result[0].SpanID)
|
|
}
|
|
|
|
func TestAddProcessor(t *testing.T) {
|
|
require := require.New(t)
|
|
|
|
testPipelines := []pipelinetypes.GettablePipeline{
|
|
{
|
|
StoreablePipeline: pipelinetypes.StoreablePipeline{
|
|
OrderID: 1,
|
|
Name: "pipeline1",
|
|
Alias: "pipeline1",
|
|
Enabled: true,
|
|
},
|
|
Filter: &v3.FilterSet{
|
|
Operator: "AND",
|
|
Items: []v3.FilterItem{
|
|
{
|
|
Key: v3.AttributeKey{
|
|
Key: "method",
|
|
DataType: v3.AttributeKeyDataTypeString,
|
|
Type: v3.AttributeKeyTypeTag,
|
|
},
|
|
Operator: "=",
|
|
Value: "GET",
|
|
},
|
|
},
|
|
},
|
|
Config: []pipelinetypes.PipelineOperator{},
|
|
},
|
|
}
|
|
|
|
var parserOp pipelinetypes.PipelineOperator
|
|
err := json.Unmarshal([]byte(`
|
|
{
|
|
"orderId": 1,
|
|
"enabled": true,
|
|
"type": "add",
|
|
"name": "Test add parser",
|
|
"id": "test-add-parser",
|
|
"field": "attributes.test",
|
|
"value": "test"
|
|
}
|
|
`), &parserOp)
|
|
require.Nil(err)
|
|
testPipelines[0].Config = append(testPipelines[0].Config, parserOp)
|
|
|
|
testLog := makeTestO11yLog(
|
|
"test log",
|
|
map[string]interface{}{
|
|
"method": "GET",
|
|
},
|
|
)
|
|
|
|
result, collectorWarnAndErrorLogs, err := SimulatePipelinesProcessing(
|
|
context.Background(),
|
|
testPipelines,
|
|
[]model.O11yLog{
|
|
testLog,
|
|
},
|
|
)
|
|
require.Nil(err)
|
|
require.Equal(1, len(result))
|
|
require.Equal(0, len(collectorWarnAndErrorLogs), strings.Join(collectorWarnAndErrorLogs, "\n"))
|
|
processed := result[0]
|
|
require.Equal("test", processed.Attributes_string["test"])
|
|
}
|
|
|
|
func TestRemoveProcessor(t *testing.T) {
|
|
require := require.New(t)
|
|
|
|
testPipelines := []pipelinetypes.GettablePipeline{
|
|
{
|
|
StoreablePipeline: pipelinetypes.StoreablePipeline{
|
|
OrderID: 1,
|
|
Name: "pipeline1",
|
|
Alias: "pipeline1",
|
|
Enabled: true,
|
|
},
|
|
Filter: &v3.FilterSet{
|
|
Operator: "AND",
|
|
Items: []v3.FilterItem{
|
|
{
|
|
Key: v3.AttributeKey{
|
|
Key: "method",
|
|
DataType: v3.AttributeKeyDataTypeString,
|
|
Type: v3.AttributeKeyTypeTag,
|
|
},
|
|
Operator: "=",
|
|
Value: "GET",
|
|
},
|
|
},
|
|
},
|
|
Config: []pipelinetypes.PipelineOperator{},
|
|
},
|
|
}
|
|
|
|
var parserOp pipelinetypes.PipelineOperator
|
|
err := json.Unmarshal([]byte(`
|
|
{
|
|
"orderId": 1,
|
|
"enabled": true,
|
|
"type": "remove",
|
|
"name": "Test remove parser",
|
|
"id": "test-remove-parser",
|
|
"field": "attributes.method"
|
|
}
|
|
`), &parserOp)
|
|
require.Nil(err)
|
|
testPipelines[0].Config = append(testPipelines[0].Config, parserOp)
|
|
|
|
testLog := makeTestO11yLog(
|
|
"test log",
|
|
map[string]interface{}{
|
|
"method": "GET",
|
|
},
|
|
)
|
|
|
|
result, collectorWarnAndErrorLogs, err := SimulatePipelinesProcessing(
|
|
context.Background(),
|
|
testPipelines,
|
|
[]model.O11yLog{
|
|
testLog,
|
|
},
|
|
)
|
|
require.Nil(err)
|
|
require.Equal(1, len(result))
|
|
require.Equal(0, len(collectorWarnAndErrorLogs), strings.Join(collectorWarnAndErrorLogs, "\n"))
|
|
processed := result[0]
|
|
_, methodExists := processed.Attributes_string["method"]
|
|
require.False(methodExists)
|
|
}
|
|
|
|
func TestCopyProcessor(t *testing.T) {
|
|
require := require.New(t)
|
|
|
|
testPipelines := []pipelinetypes.GettablePipeline{
|
|
{
|
|
StoreablePipeline: pipelinetypes.StoreablePipeline{
|
|
OrderID: 1,
|
|
Name: "pipeline1",
|
|
Alias: "pipeline1",
|
|
Enabled: true,
|
|
},
|
|
Filter: &v3.FilterSet{
|
|
Operator: "AND",
|
|
Items: []v3.FilterItem{
|
|
{
|
|
Key: v3.AttributeKey{
|
|
Key: "method",
|
|
DataType: v3.AttributeKeyDataTypeString,
|
|
Type: v3.AttributeKeyTypeTag,
|
|
},
|
|
Operator: "=",
|
|
Value: "GET",
|
|
},
|
|
},
|
|
},
|
|
Config: []pipelinetypes.PipelineOperator{},
|
|
},
|
|
}
|
|
|
|
var parserOp pipelinetypes.PipelineOperator
|
|
err := json.Unmarshal([]byte(`
|
|
{
|
|
"orderId": 1,
|
|
"enabled": true,
|
|
"type": "copy",
|
|
"name": "Test copy parser",
|
|
"id": "test-add-parser",
|
|
"from": "attributes.method",
|
|
"to": "attributes.copied_method"
|
|
}
|
|
`), &parserOp)
|
|
require.Nil(err)
|
|
testPipelines[0].Config = append(testPipelines[0].Config, parserOp)
|
|
|
|
testLog := makeTestO11yLog(
|
|
"test log",
|
|
map[string]interface{}{
|
|
"method": "GET",
|
|
},
|
|
)
|
|
|
|
result, collectorWarnAndErrorLogs, err := SimulatePipelinesProcessing(
|
|
context.Background(),
|
|
testPipelines,
|
|
[]model.O11yLog{
|
|
testLog,
|
|
},
|
|
)
|
|
require.Nil(err)
|
|
require.Equal(1, len(result))
|
|
require.Equal(0, len(collectorWarnAndErrorLogs), strings.Join(collectorWarnAndErrorLogs, "\n"))
|
|
processed := result[0]
|
|
require.Equal("GET", processed.Attributes_string["method"])
|
|
require.Equal("GET", processed.Attributes_string["copied_method"])
|
|
}
|
|
|
|
func TestMoveProcessor(t *testing.T) {
|
|
require := require.New(t)
|
|
|
|
testPipelines := []pipelinetypes.GettablePipeline{
|
|
{
|
|
StoreablePipeline: pipelinetypes.StoreablePipeline{
|
|
OrderID: 1,
|
|
Name: "pipeline1",
|
|
Alias: "pipeline1",
|
|
Enabled: true,
|
|
},
|
|
Filter: &v3.FilterSet{
|
|
Operator: "AND",
|
|
Items: []v3.FilterItem{
|
|
{
|
|
Key: v3.AttributeKey{
|
|
Key: "method",
|
|
DataType: v3.AttributeKeyDataTypeString,
|
|
Type: v3.AttributeKeyTypeTag,
|
|
},
|
|
Operator: "=",
|
|
Value: "GET",
|
|
},
|
|
},
|
|
},
|
|
Config: []pipelinetypes.PipelineOperator{},
|
|
},
|
|
}
|
|
|
|
var parserOp pipelinetypes.PipelineOperator
|
|
err := json.Unmarshal([]byte(`
|
|
{
|
|
"orderId": 1,
|
|
"enabled": true,
|
|
"type": "move",
|
|
"name": "Test move parser",
|
|
"id": "test-move-parser",
|
|
"from": "attributes.method",
|
|
"to": "attributes.moved_method"
|
|
}
|
|
`), &parserOp)
|
|
require.Nil(err)
|
|
testPipelines[0].Config = append(testPipelines[0].Config, parserOp)
|
|
|
|
testLog := makeTestO11yLog(
|
|
"test log",
|
|
map[string]interface{}{
|
|
"method": "GET",
|
|
},
|
|
)
|
|
|
|
result, collectorWarnAndErrorLogs, err := SimulatePipelinesProcessing(
|
|
context.Background(),
|
|
testPipelines,
|
|
[]model.O11yLog{
|
|
testLog,
|
|
},
|
|
)
|
|
require.Nil(err)
|
|
require.Equal(1, len(result))
|
|
require.Equal(0, len(collectorWarnAndErrorLogs), strings.Join(collectorWarnAndErrorLogs, "\n"))
|
|
processed := result[0]
|
|
require.Equal("", processed.Attributes_string["method"])
|
|
require.Equal("GET", processed.Attributes_string["moved_method"])
|
|
}
|