mirror of
https://github.com/zap-proto/telemetry.git
synced 2026-10-30 15:03:28 +00:00
The three signals already shared everything that matters — the same envelope, the same port, the same node handshake, the same convention of carrying the message type in the header flags high byte. What they did not share was a way to be turned on. Each service dialed three times, configured three exporters and installed three providers: three chances to get two right and one wrong, silently, because a provider that was never installed discards every measurement while the code still reads as instrumented. That failure already happened twice in this fleet. Start does all three against one endpoint and returns one Shutdown. The zero value is a noop. An empty Endpoint installs nothing and dials nothing, so a service can call Start unconditionally at the top of main and pick up telemetry when its deployment supplies an endpoint. There is no separate disabled path to keep working — the branch not taken is the one that rots. An unreachable collector does NOT fail Start. Exporters connect lazily and retry; a service that refuses to boot because telemetry is down has made observation a dependency of serving. Signals are attempted independently, so a collector rejecting metrics does not cost the service its traces. Partial telemetry beats none, and the failure is returned rather than swallowed. Endpoints are normalized to host:port. They are widely configured as URLs from the OTLP era, and passing one through dials a host named http — a baffling failure for a config that looks correct. No OTLP, no gRPC, no protobuf in the graph.
263 lines
7.9 KiB
Go
263 lines
7.9 KiB
Go
// Package telemetry starts a service's traces, metrics and logs on the ZAP
|
|
// wire in one call.
|
|
//
|
|
// The three signals already share everything that matters: the same envelope,
|
|
// the same port, the same node handshake, and the same convention of carrying
|
|
// the message type in the header flags' high byte. What they did not share was
|
|
// a way to be turned on. Each service dialed three times, configured three
|
|
// exporters, and installed three providers — three chances to get two right and
|
|
// one wrong, silently, because a provider that was never installed discards
|
|
// every measurement while the code still reads as instrumented.
|
|
//
|
|
// tel, err := telemetry.Start(telemetry.Config{
|
|
// AppName: "iam",
|
|
// Endpoint: "o11y.hanzo.svc:4317",
|
|
// })
|
|
// if err != nil { return err }
|
|
// defer tel.Shutdown(ctx)
|
|
//
|
|
// # Noop is the zero value
|
|
//
|
|
// An empty Endpoint means the whole thing is inert: nothing dials, no provider
|
|
// is installed, and the OTel globals stay as they were. That is what makes it
|
|
// safe to call unconditionally at the top of main — a service picks up
|
|
// telemetry when its deployment supplies an endpoint, and costs nothing when it
|
|
// does not. There is no separate "disabled" path to keep working.
|
|
//
|
|
// # Not the event bus
|
|
//
|
|
// Telemetry is observation: fire-and-forget, lossy-tolerable, and it never
|
|
// affects business state. Events are facts — a payment settled, an org was
|
|
// created — which are durable, ordered and acted upon. Routing spans through
|
|
// the event bus makes telemetry volume compete with money-path delivery and
|
|
// both get worse. Facts go to the event bus; this goes to o11y.
|
|
package telemetry
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"strings"
|
|
|
|
"go.opentelemetry.io/otel"
|
|
"go.opentelemetry.io/otel/attribute"
|
|
otellog "go.opentelemetry.io/otel/log/global"
|
|
"go.opentelemetry.io/otel/propagation"
|
|
sdklog "go.opentelemetry.io/otel/sdk/log"
|
|
sdkmetric "go.opentelemetry.io/otel/sdk/metric"
|
|
"go.opentelemetry.io/otel/sdk/resource"
|
|
|
|
luxlog "github.com/luxfi/log"
|
|
luxmetric "github.com/luxfi/metric"
|
|
luxtrace "github.com/luxfi/trace"
|
|
)
|
|
|
|
// Signal names a telemetry stream.
|
|
type Signal string
|
|
|
|
const (
|
|
Traces Signal = "traces"
|
|
Metrics Signal = "metrics"
|
|
Logs Signal = "logs"
|
|
)
|
|
|
|
// Config describes what to start.
|
|
type Config struct {
|
|
// AppName is the service name on every emitted batch. Required unless the
|
|
// Endpoint is empty.
|
|
AppName string
|
|
|
|
// Version is reported alongside AppName.
|
|
Version string
|
|
|
|
// Endpoint is the collector address as host:port. Empty ⇒ everything is a
|
|
// noop. A scheme or path is trimmed: they have no meaning on this wire, and
|
|
// silently dialing "https" as a hostname is a worse failure than ignoring it.
|
|
Endpoint string
|
|
|
|
// Resource attributes attached to every batch (k8s namespace, pod, region…).
|
|
Resource map[string]string
|
|
|
|
// Signals to start. Empty ⇒ all three. Naming a subset is for a service that
|
|
// genuinely has nothing to say on one of them, not for turning telemetry
|
|
// down: an unwanted signal costs a provider and no traffic.
|
|
Signals []Signal
|
|
|
|
// SampleRate for traces, 0..1. Zero ⇒ 1 (sample everything). A service that
|
|
// wants no traces should leave Traces out of Signals rather than set 0 here,
|
|
// so the intent is visible.
|
|
SampleRate float64
|
|
}
|
|
|
|
// Telemetry owns what Start installed.
|
|
type Telemetry struct {
|
|
tracer luxtrace.Tracer
|
|
mp *sdkmetric.MeterProvider
|
|
lp *sdklog.LoggerProvider
|
|
|
|
enabled bool
|
|
}
|
|
|
|
// Enabled reports whether anything was actually started. A caller that logs its
|
|
// telemetry state should read this rather than assume Start succeeded — the
|
|
// no-endpoint case returns a valid, inert Telemetry and a nil error.
|
|
func (t *Telemetry) Enabled() bool { return t != nil && t.enabled }
|
|
|
|
// Start installs the providers for the configured signals.
|
|
//
|
|
// It returns a usable Telemetry even when disabled, so callers never need a nil
|
|
// check before Shutdown.
|
|
func Start(cfg Config) (*Telemetry, error) {
|
|
endpoint := normalizeEndpoint(cfg.Endpoint)
|
|
if endpoint == "" {
|
|
return &Telemetry{}, nil
|
|
}
|
|
if cfg.AppName == "" {
|
|
return &Telemetry{}, errors.New("telemetry: AppName is required when Endpoint is set")
|
|
}
|
|
if cfg.SampleRate <= 0 {
|
|
cfg.SampleRate = 1
|
|
}
|
|
|
|
want := map[Signal]bool{}
|
|
if len(cfg.Signals) == 0 {
|
|
want[Traces], want[Metrics], want[Logs] = true, true, true
|
|
} else {
|
|
for _, s := range cfg.Signals {
|
|
want[s] = true
|
|
}
|
|
}
|
|
|
|
res := resource.NewSchemaless(resourceAttrs(cfg)...)
|
|
t := &Telemetry{}
|
|
|
|
// Each signal is attempted independently. One collector rejecting metrics
|
|
// must not cost the service its traces — partial telemetry beats none, and
|
|
// the failure is reported rather than hidden.
|
|
var errs []error
|
|
|
|
if want[Traces] {
|
|
tracer, err := luxtrace.New(luxtrace.Config{
|
|
ExporterConfig: luxtrace.ExporterConfig{Type: luxtrace.ZAP, Endpoint: endpoint},
|
|
TraceSampleRate: cfg.SampleRate,
|
|
AppName: cfg.AppName,
|
|
Version: cfg.Version,
|
|
})
|
|
if err != nil {
|
|
errs = append(errs, fmt.Errorf("traces: %w", err))
|
|
} else {
|
|
t.tracer = tracer
|
|
t.enabled = true
|
|
}
|
|
}
|
|
|
|
if want[Metrics] {
|
|
exp, err := luxmetric.NewOTelZAPExporter(luxmetric.ZAPExporterConfig{
|
|
Endpoint: endpoint,
|
|
AppName: cfg.AppName,
|
|
Version: cfg.Version,
|
|
Resource: cfg.Resource,
|
|
})
|
|
if err != nil {
|
|
errs = append(errs, fmt.Errorf("metrics: %w", err))
|
|
} else {
|
|
mp := sdkmetric.NewMeterProvider(
|
|
sdkmetric.WithReader(sdkmetric.NewPeriodicReader(exp)),
|
|
sdkmetric.WithResource(res),
|
|
)
|
|
otel.SetMeterProvider(mp)
|
|
t.mp = mp
|
|
t.enabled = true
|
|
}
|
|
}
|
|
|
|
if want[Logs] {
|
|
exp, err := luxlog.NewZAPExporter(luxlog.ZAPExporterConfig{
|
|
Endpoint: endpoint,
|
|
AppName: cfg.AppName,
|
|
Version: cfg.Version,
|
|
Resource: cfg.Resource,
|
|
})
|
|
if err != nil {
|
|
errs = append(errs, fmt.Errorf("logs: %w", err))
|
|
} else {
|
|
lp := sdklog.NewLoggerProvider(
|
|
sdklog.WithProcessor(sdklog.NewBatchProcessor(exp)),
|
|
sdklog.WithResource(res),
|
|
)
|
|
otellog.SetLoggerProvider(lp)
|
|
t.lp = lp
|
|
t.enabled = true
|
|
}
|
|
}
|
|
|
|
// W3C trace context so a span links to the caller's trace across the edge,
|
|
// giving one end-to-end trace rather than a per-service fragment.
|
|
otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(
|
|
propagation.TraceContext{}, propagation.Baggage{},
|
|
))
|
|
|
|
if len(errs) > 0 {
|
|
return t, errors.Join(errs...)
|
|
}
|
|
return t, nil
|
|
}
|
|
|
|
// Shutdown flushes and stops everything Start installed. Safe on a disabled
|
|
// Telemetry and safe to call twice.
|
|
func (t *Telemetry) Shutdown(ctx context.Context) error {
|
|
if t == nil {
|
|
return nil
|
|
}
|
|
var errs []error
|
|
if t.lp != nil {
|
|
if err := t.lp.Shutdown(ctx); err != nil {
|
|
errs = append(errs, fmt.Errorf("logs: %w", err))
|
|
}
|
|
t.lp = nil
|
|
}
|
|
if t.mp != nil {
|
|
if err := t.mp.Shutdown(ctx); err != nil {
|
|
errs = append(errs, fmt.Errorf("metrics: %w", err))
|
|
}
|
|
t.mp = nil
|
|
}
|
|
if t.tracer != nil {
|
|
if err := t.tracer.Close(); err != nil {
|
|
errs = append(errs, fmt.Errorf("traces: %w", err))
|
|
}
|
|
t.tracer = nil
|
|
}
|
|
t.enabled = false
|
|
return errors.Join(errs...)
|
|
}
|
|
|
|
func resourceAttrs(cfg Config) []attribute.KeyValue {
|
|
attrs := make([]attribute.KeyValue, 0, len(cfg.Resource)+2)
|
|
attrs = append(attrs, attribute.String("service.name", cfg.AppName))
|
|
if cfg.Version != "" {
|
|
attrs = append(attrs, attribute.String("service.version", cfg.Version))
|
|
}
|
|
for k, v := range cfg.Resource {
|
|
attrs = append(attrs, attribute.String(k, v))
|
|
}
|
|
return attrs
|
|
}
|
|
|
|
// normalizeEndpoint reduces a configured value to host:port.
|
|
//
|
|
// Endpoints are widely configured as URLs from the OTLP era. A scheme and path
|
|
// mean nothing on this wire, and passing them through produces a dial against a
|
|
// hostname like "http" — a confusing failure for a config that looks right.
|
|
func normalizeEndpoint(ep string) string {
|
|
ep = strings.TrimSpace(ep)
|
|
if ep == "" {
|
|
return ""
|
|
}
|
|
ep = strings.TrimPrefix(strings.TrimPrefix(ep, "https://"), "http://")
|
|
if i := strings.IndexByte(ep, '/'); i >= 0 {
|
|
ep = ep[:i]
|
|
}
|
|
return ep
|
|
}
|