hanzoai/otel-collector
2
0
Fork 0
mirror of https://github.com/hanzoai/otel-collector.git synced 2026-11-03 02:20:04 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
main
otel-collector/cmd/o11yschemamigrator/schema_migrator/metadata_migrations.go
T
zeekayandGitHub 940df58087 debrand: rename physical ClickHouse signoz_* schema -> o11y_* (writer side) (#7) 
Rename ONLY the physical ClickHouse DATABASE/TABLE identifiers the
schema-migrator CREATEs and the exporters WRITE, matching the o11y read
plane + the data-preserving RENAME migration (hanzoai/o11y#28):

  databases: signoz_{traces,metrics,logs,metadata,analytics,meter}
             -> o11y_{traces,metrics,logs,metadata,analytics,meter}
  tables:    (distributed_)signoz_{index_v3,index_v2,error_index_v2,spans}
             -> (distributed_)o11y_{index_v3,index_v2,error_index_v2,spans}

Word-boundary anchored (\bsignoz_<db>\b) so wire contracts are untouched:
spanmetrics/usage METRIC names (signoz_calls_total, signoz_latency,
signoz_*_count/_bytes, signoz_metadata_exporter_json_logs_processed,
signoz_normalize_operator_logs_processed), signoz.* resource ATTRIBUTES
(signoz.gen_ai.*), OTel component TYPE strings (signozclickhousemetrics,
signozclickhousemeter, signozlogspipeline, signozllmpricing), and all
github.com/SigNoz/* deps + LICENSE/NOTICE attribution all preserved.

Coordinated cutover: deploy this collector, run o11y#28 RENAME migration,
then deploy o11y + cloud readers — together, or telemetry blackholes.

go build ./... = 0. schema-migrator tests green. Two pre-existing test
failures (flaky TestExporterPushTracesData usage-collector timing;
o11ydatastoremeter go.signoz.io scope-name debrand) reproduce identically
on unmodified v0.144.7.
2026-07-09 13:08:10 -07:00

155 lines
6.4 KiB
Go
Raw Permalink Blame History

package schemamigrator
import (
"fmt"
"time"
)
var MetadataMigrations = []SchemaMigrationRecord{
{
MigrationID: 1000,
UpItems: []Operation{
CreateTableOperation{
Database: "o11y_metadata",
Table: "attributes_metadata",
Columns: []Column{
{Name: "unix_milli", Type: ColumnTypeUInt64},
{Name: "data_source", Type: ColumnTypeString},
{Name: "resource_fingerprint", Type: ColumnTypeUInt64},
{Name: "attrs_fingerprint", Type: ColumnTypeUInt64},
{Name: "resource_attributes", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}},
{Name: "attributes", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}},
},
Indexes: []Index{
{Name: "idx_resource_attributes_map_keys", Expression: "mapKeys(resource_attributes)", Type: "tokenbf_v1(1024, 2, 0)", Granularity: 1},
{Name: "idx_attributes_map_keys", Expression: "mapKeys(attributes)", Type: "tokenbf_v1(1024, 2, 0)", Granularity: 1},
{Name: "idx_resource_attributes_map_values", Expression: "mapValues(resource_attributes)", Type: "ngrambf_v1(4, 5000, 2, 0)", Granularity: 1},
{Name: "idx_attributes_map_values", Expression: "mapValues(attributes)", Type: "ngrambf_v1(4, 5000, 2, 0)", Granularity: 1},
},
Engine: ReplacingMergeTree{
MergeTree: MergeTree{
PartitionBy: "toDate(unix_milli / 1000)",
OrderBy: "(data_source, unix_milli, resource_fingerprint, attrs_fingerprint)",
TTL: "toDateTime(unix_milli / 1000) + INTERVAL 2592000 SECOND DELETE",
Settings: TableSettings{
{Name: "ttl_only_drop_parts", Value: "1"},
{Name: "index_granularity", Value: "8192"},
},
},
},
},
CreateTableOperation{
Database: "o11y_metadata",
Table: "distributed_attributes_metadata",
Columns: []Column{
{Name: "unix_milli", Type: ColumnTypeUInt64},
{Name: "data_source", Type: ColumnTypeString},
{Name: "resource_fingerprint", Type: ColumnTypeUInt64},
{Name: "attrs_fingerprint", Type: ColumnTypeUInt64},
{Name: "resource_attributes", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}},
{Name: "attributes", Type: MapColumnType{KeyType: LowCardinalityColumnType{ColumnTypeString}, ValueType: ColumnTypeString}},
},
Engine: Distributed{
Database: "o11y_metadata",
Table: "attributes_metadata",
ShardingKey: "cityHash64(data_source, resource_fingerprint, attrs_fingerprint)",
},
},
},
DownItems: []Operation{
DropTableOperation{
Database: "o11y_metadata",
Table: "distributed_attributes_metadata",
},
DropTableOperation{
Database: "o11y_metadata",
Table: "attributes_metadata",
},
},
},
{
MigrationID: 1001,
UpItems: []Operation{
CreateTableOperation{
Database: "o11y_metadata",
Table: "column_evolution_metadata",
Columns: []Column{
{Name: "signal", Type: ColumnTypeString, Codec: "ZSTD(1)"},
{Name: "column_name", Type: ColumnTypeString, Codec: "ZSTD(1)"},
{Name: "column_type", Type: ColumnTypeString, Codec: "ZSTD(1)"},
{Name: "field_context", Type: ColumnTypeString, Codec: "ZSTD(1)"},
{Name: "field_name", Type: ColumnTypeString, Codec: "ZSTD(1)"},
{Name: "version", Type: ColumnTypeUInt32, Codec: "DoubleDelta, ZSTD(1)"},
{Name: "release_time", Type: SimpleAggregateFunction{FunctionName: "min", Arguments: []ColumnType{ColumnTypeFloat64}}, Codec: "ZSTD(1)"},
},
Engine: AggregatingMergeTree{
MergeTree: MergeTree{
OrderBy: "(signal, column_name, column_type, field_context, field_name, version)",
PartitionBy: "toDate(release_time / 1000000000)",
},
},
},
CreateTableOperation{
Database: "o11y_metadata",
Table: "distributed_column_evolution_metadata",
Columns: []Column{
{Name: "signal", Type: ColumnTypeString, Codec: "ZSTD(1)"},
{Name: "column_name", Type: ColumnTypeString, Codec: "ZSTD(1)"},
{Name: "column_type", Type: ColumnTypeString, Codec: "ZSTD(1)"},
{Name: "field_context", Type: ColumnTypeString, Codec: "ZSTD(1)"},
{Name: "field_name", Type: ColumnTypeString, Codec: "ZSTD(1)"},
{Name: "version", Type: ColumnTypeUInt32, Codec: "DoubleDelta, ZSTD(1)"},
{Name: "release_time", Type: SimpleAggregateFunction{FunctionName: "min", Arguments: []ColumnType{ColumnTypeFloat64}}, Codec: "ZSTD(1)"},
},
Engine: Distributed{
Database: "o11y_metadata",
Table: "column_evolution_metadata",
ShardingKey: "cityHash64(signal,column_name)",
},
},
InsertIntoTable{
Database: "o11y_metadata",
Table: "distributed_column_evolution_metadata",
LightWeight: true,
Synchronous: true,
Columns: []string{"signal", "column_name", "column_type", "field_context", "field_name", "version", "release_time"},
Values: "('logs', 'resources_string', 'Map(LowCardinality(String), Float64)', 'resource', '__all__', 0, 0)",
},
InsertIntoTable{
Database: "o11y_metadata",
Table: "distributed_column_evolution_metadata",
LightWeight: true,
Synchronous: true,
Columns: []string{"signal", "column_name", "column_type", "field_context", "field_name", "version", "release_time"},
Values: fmt.Sprintf("('logs', 'resource', 'JSON()', 'resource', '__all__', 1, %d)", time.Now().UnixNano()),
},
InsertIntoTable{
Database: "o11y_metadata",
Table: "distributed_column_evolution_metadata",
LightWeight: true,
Synchronous: true,
Columns: []string{"signal", "column_name", "column_type", "field_context", "field_name", "version", "release_time"},
Values: "('traces', 'resources_string', 'Map(LowCardinality(String), Float64)', 'resource', '__all__', 0, 0)",
},
InsertIntoTable{
Database: "o11y_metadata",
Table: "distributed_column_evolution_metadata",
LightWeight: true,
Synchronous: true,
Columns: []string{"signal", "column_name", "column_type", "field_context", "field_name", "version", "release_time"},
Values: fmt.Sprintf("('traces', 'resource', 'JSON()', 'resource', '__all__', 1, %d)", time.Now().UnixNano()),
},
},
DownItems: []Operation{
DropTableOperation{
Database: "o11y_metadata",
Table: "distributed_column_evolution_metadata",
},
DropTableOperation{
Database: "o11y_metadata",
Table: "column_evolution_metadata",
},
},
},
}
Reference in New Issue View Git Blame Copy Permalink